From 2f34093ef17b5723c830c7187f75c4d37e7fcddf Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 13 Apr 2026 22:44:20 +0800 Subject: [PATCH 01/15] feat(test): add Python integration test framework and CI Introduce a modular Python integration test framework under tests/integration/ with separated concerns (utils, workspace, config, process, instance facade). Add lifecycle, config, and client smoke tests. Include a dedicated Makefile for the test suite and a standalone GitHub Actions workflow. Made-with: Cursor --- .github/workflows/integration-test.yml | 70 +++++++++ .gitignore | 4 + Makefile | 8 +- tests/integration/Makefile | 56 ++++++++ tests/integration/framework/__init__.py | 6 + tests/integration/framework/config.py | 71 ++++++++++ tests/integration/framework/instance.py | 172 +++++++++++++++++++++++ tests/integration/framework/process.py | 107 ++++++++++++++ tests/integration/framework/utils.py | 35 +++++ tests/integration/framework/workspace.py | 52 +++++++ tests/integration/pytest.ini | 8 ++ tests/integration/requirements.txt | 4 + tests/integration/test/__init__.py | 2 + tests/integration/test/conftest.py | 31 ++++ tests/integration/test/test_client.py | 38 +++++ tests/integration/test/test_config.py | 85 +++++++++++ tests/integration/test/test_lifecycle.py | 72 ++++++++++ 17 files changed, 820 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/integration-test.yml create mode 100644 tests/integration/Makefile create mode 100644 tests/integration/framework/__init__.py create mode 100644 tests/integration/framework/config.py create mode 100644 tests/integration/framework/instance.py create mode 100644 tests/integration/framework/process.py create mode 100644 tests/integration/framework/utils.py create mode 100644 tests/integration/framework/workspace.py create mode 100644 tests/integration/pytest.ini create mode 100644 tests/integration/requirements.txt create mode 100644 tests/integration/test/__init__.py create mode 100644 tests/integration/test/conftest.py create mode 100644 tests/integration/test/test_client.py create mode 100644 tests/integration/test/test_config.py create mode 100644 tests/integration/test/test_lifecycle.py diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 00000000..708be1a0 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,70 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Integration Tests + +on: + push: + branches: ["main", "master"] + pull_request: + branches: ["main", "master"] + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + integration-test: + name: Integration Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout Source + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake \ + libssl-dev \ + libcurl4-openssl-dev \ + pkg-config \ + libsasl2-dev \ + protobuf-compiler + + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + + - name: Build Release Binary + run: cargo build --release --no-default-features --features incremental-cache + + - name: Run Integration Tests + run: make integration-test + + - name: Upload Test Logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: integration-test-logs + path: tests/integration/target/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 99a53648..8eec5687 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ python/**/target/ **/**.egg-info .cache/ /.cursor/worktrees.json + +# Integration test output +tests/integration/target/ +tests/integration/.venv/ diff --git a/Makefile b/Makefile index 4daf185b..c8e1da4d 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ C_0 := \033[0m log = @printf "$(C_B)[-]$(C_0) %-15s %s\n" "$(1)" "$(2)" success = @printf "$(C_G)[✔]$(C_0) %s\n" "$(1)" -.PHONY: all help build build-lite dist dist-lite clean test env env-clean go-sdk-env go-sdk-build go-sdk-clean docker docker-run docker-push .check-env .build-wasm +.PHONY: all help build build-lite dist dist-lite clean test env env-clean go-sdk-env go-sdk-build go-sdk-clean docker docker-run docker-push .check-env .build-wasm integration-test all: build @@ -63,6 +63,8 @@ help: @echo " docker-run Run container (port 8080, mount logs)" @echo " docker-push Push image to registry" @echo "" + @echo " integration-test Run integration tests (delegates to tests/integration)" + @echo "" @echo " Version: $(VERSION) | Arch: $(ARCH) | OS: $(OS)" build: .check-env .build-wasm @@ -145,6 +147,7 @@ clean: @cargo clean @rm -rf $(DIST_ROOT) data logs @./scripts/clean.sh 2>/dev/null || true + @$(MAKE) -C tests/integration clean 2>/dev/null || true $(call success,Done) .check-env: @@ -167,3 +170,6 @@ docker-push: $(call log,DOCKER,Pushing $(IMAGE_NAME)) @docker push $(IMAGE_NAME) $(call success,Push Complete) + +integration-test: + @$(MAKE) -C tests/integration test PYTEST_ARGS="$(PYTEST_ARGS)" diff --git a/tests/integration/Makefile b/tests/integration/Makefile new file mode 100644 index 00000000..d800d9f3 --- /dev/null +++ b/tests/integration/Makefile @@ -0,0 +1,56 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +# ----------------------------------------------------------------------- +# Integration Test Makefile +# ----------------------------------------------------------------------- +# Usage: +# make test — Setup env + run pytest (PYTEST_ARGS="-k xxx") +# make clean — Remove .venv and test output +# +# Prerequisites: +# The FunctionStream binary must already be built (make build / make build-lite +# from the project root). +# ----------------------------------------------------------------------- + +PROJECT_ROOT := $(shell git -C $(CURDIR) rev-parse --show-toplevel) +PYTHON_ROOT := $(PROJECT_ROOT)/python +VENV := $(CURDIR)/.venv +PIP := $(VENV)/bin/pip +PY := $(VENV)/bin/python + +C_G := \033[0;32m +C_B := \033[0;34m +C_0 := \033[0m + +log = @printf "$(C_B)[-]$(C_0) %-12s %s\n" "$(1)" "$(2)" +success = @printf "$(C_G)[✔]$(C_0) %s\n" "$(1)" + +.PHONY: test clean help + +help: + @echo "Integration Test Targets:" + @echo "" + @echo " test Setup Python env + run pytest (PYTEST_ARGS=...)" + @echo " clean Remove .venv and target/tests output" + +$(VENV)/.installed: requirements.txt $(PYTHON_ROOT)/functionstream-api/pyproject.toml $(PYTHON_ROOT)/functionstream-client/pyproject.toml + $(call log,ENV,Setting up Python virtual environment) + @test -d $(VENV) || python3 -m venv $(VENV) + @$(PIP) install --quiet --upgrade pip + @$(PIP) install --quiet -r requirements.txt + @$(PIP) install --quiet -e $(PYTHON_ROOT)/functionstream-api + @$(PIP) install --quiet -e $(PYTHON_ROOT)/functionstream-client + @touch $@ + $(call success,Python environment ready) + +test: $(VENV)/.installed + $(call log,TEST,Running integration tests) + @$(PY) -m pytest -v $(PYTEST_ARGS) + $(call success,All integration tests passed) + +clean: + $(call log,CLEAN,Removing test artifacts) + @rm -rf $(VENV) + @rm -rf $(CURDIR)/target + $(call success,Clean complete) diff --git a/tests/integration/framework/__init__.py b/tests/integration/framework/__init__.py new file mode 100644 index 00000000..28491606 --- /dev/null +++ b/tests/integration/framework/__init__.py @@ -0,0 +1,6 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +from .instance import FunctionStreamInstance + +__all__ = ["FunctionStreamInstance"] diff --git a/tests/integration/framework/config.py b/tests/integration/framework/config.py new file mode 100644 index 00000000..7b90ce88 --- /dev/null +++ b/tests/integration/framework/config.py @@ -0,0 +1,71 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +InstanceConfig: builds and writes the config.yaml consumed by +the FunctionStream binary via FUNCTION_STREAM_CONF. +""" + +from pathlib import Path +from typing import Any, Dict + +import yaml + +from .workspace import InstanceWorkspace + + +class InstanceConfig: + """Generates and persists config.yaml for one FunctionStream instance.""" + + def __init__(self, host: str, port: int, workspace: InstanceWorkspace): + self._workspace = workspace + self._config: Dict[str, Any] = { + "service": { + "service_id": f"it-{port}", + "service_name": "function-stream", + "version": "1.0.0", + "host": host, + "port": port, + "debug": False, + }, + "logging": { + "level": "info", + "format": "json", + "file_path": str(workspace.log_dir / "app.log"), + "max_file_size": 50, + "max_files": 3, + }, + "state_storage": { + "storage_type": "memory", + }, + "task_storage": { + "storage_type": "rocksdb", + "db_path": str(workspace.data_dir / "task"), + }, + "stream_catalog": { + "persist": True, + "db_path": str(workspace.data_dir / "stream_catalog"), + }, + } + + @property + def raw(self) -> Dict[str, Any]: + return self._config + + def override(self, overrides: Dict[str, Any]) -> None: + """ + Apply overrides using dot-separated keys. + Example: {"service.debug": True, "logging.level": "debug"} + """ + for dotted_key, value in overrides.items(): + keys = dotted_key.split(".") + target = self._config + for k in keys[:-1]: + target = target.setdefault(k, {}) + target[keys[-1]] = value + + def write_to_workspace(self) -> Path: + """Serialize config to the workspace config.yaml and return its path.""" + with open(self._workspace.config_file, "w", encoding="utf-8") as f: + yaml.dump(self._config, f, default_flow_style=False, sort_keys=False) + return self._workspace.config_file diff --git a/tests/integration/framework/instance.py b/tests/integration/framework/instance.py new file mode 100644 index 00000000..c1bc74e1 --- /dev/null +++ b/tests/integration/framework/instance.py @@ -0,0 +1,172 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +FunctionStreamInstance: the Facade that composes workspace, config, +and process into a single easy-to-use API for test cases. +""" + +import logging +from pathlib import Path +from typing import Any, Optional + +from .config import InstanceConfig +from .process import FunctionStreamProcess +from .utils import find_free_port, wait_for_port +from .workspace import InstanceWorkspace + +logger = logging.getLogger(__name__) + +_INTEGRATION_DIR = Path(__file__).resolve().parents[1] +_PROJECT_ROOT = _INTEGRATION_DIR.parents[1] +_TARGET_DIR = _INTEGRATION_DIR / "target" +_BINARY_PATH = _PROJECT_ROOT / "target" / "release" / "function-stream" + + +class FunctionStreamInstance: + """ + Facade for a single FunctionStream server used in integration tests. + + Usage: + inst = FunctionStreamInstance("my_test") + inst.configure(**{"service.debug": True}).start() + client = inst.get_client() + ... + inst.kill() + """ + + def __init__( + self, + test_name: str = "unnamed", + host: str = "127.0.0.1", + binary_path: Optional[Path] = None, + ): + self.host = host + self.port = find_free_port() + + binary = binary_path or _BINARY_PATH + + self.workspace = InstanceWorkspace(_TARGET_DIR, test_name, self.port) + self.config = InstanceConfig(self.host, self.port, self.workspace) + self.process = FunctionStreamProcess(binary, self.workspace) + + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ + + @property + def is_running(self) -> bool: + return self.process.is_running + + @property + def pid(self): + return self.process.pid + + # ------------------------------------------------------------------ + # Configuration (before start) + # ------------------------------------------------------------------ + + def configure(self, **overrides: Any) -> "FunctionStreamInstance": + """Apply config overrides. Chainable: inst.configure(k=v).start().""" + self.config.override(overrides) + return self + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def start(self, timeout: float = 30.0) -> "FunctionStreamInstance": + """Prepare workspace, write config, launch binary, wait for ready.""" + if self.is_running: + return self + + self.workspace.setup() + self.config.write_to_workspace() + + logger.info( + "Starting FunctionStream [port=%d, dir=%s]", + self.port, + self.workspace.root_dir, + ) + self.process.start() + + if not wait_for_port(self.host, self.port, timeout): + stderr_tail = self._read_tail(self.workspace.stderr_file) + self.process.kill() + raise RuntimeError( + f"Server did not become ready within {timeout}s on port {self.port}.\n" + f"stderr tail:\n{stderr_tail}" + ) + + logger.info( + "FunctionStream ready [port=%d, pid=%d]", + self.port, + self.process.pid, + ) + return self + + def stop(self, timeout: float = 10.0) -> None: + """Graceful SIGTERM shutdown.""" + self.process.stop(timeout=timeout) + + def kill(self) -> None: + """Immediate SIGKILL.""" + self.process.kill() + + def restart(self, timeout: float = 10.0) -> "FunctionStreamInstance": + """Stop then start (same port, same workspace).""" + self.stop(timeout=timeout) + return self.start() + + # ------------------------------------------------------------------ + # Client access + # ------------------------------------------------------------------ + + def get_client(self): + """Return a connected FsClient. Caller should close it when done.""" + try: + from fs_client import FsClient + except ImportError: + raise ImportError( + "functionstream-client is not installed. " + "Run: pip install -e python/functionstream-client" + ) + return FsClient(host=self.host, port=self.port) + + def execute_sql(self, sql: str, timeout: float = 30.0): + """Convenience helper: send a SQL statement via gRPC ExecuteSql.""" + try: + import grpc + from fs_client._proto import function_stream_pb2, function_stream_pb2_grpc + except ImportError: + raise ImportError( + "functionstream-client is not installed. " + "Run: pip install -e python/functionstream-client" + ) + + channel = grpc.insecure_channel(f"{self.host}:{self.port}") + try: + stub = function_stream_pb2_grpc.FunctionStreamServiceStub(channel) + request = function_stream_pb2.SqlRequest(sql=sql) + return stub.ExecuteSql(request, timeout=timeout) + finally: + channel.close() + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + @staticmethod + def _read_tail(path: Path, chars: int = 2000) -> str: + if not path.exists(): + return "" + text = path.read_text(errors="replace") + return text[-chars:] if len(text) > chars else text + + def __repr__(self) -> str: + status = "running" if self.is_running else "stopped" + return f"" + + def __del__(self): + if hasattr(self, "process") and self.process.is_running: + self.process.kill() diff --git a/tests/integration/framework/process.py b/tests/integration/framework/process.py new file mode 100644 index 00000000..3f92e98b --- /dev/null +++ b/tests/integration/framework/process.py @@ -0,0 +1,107 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +FunctionStreamProcess: owns the OS-level process lifecycle +(spawn, SIGTERM, SIGKILL) for a single FunctionStream binary. +""" + +import logging +import os +import signal +import subprocess +from pathlib import Path +from typing import Optional + +from .workspace import InstanceWorkspace + +logger = logging.getLogger(__name__) + + +class FunctionStreamProcess: + """Manages the lifecycle of a single FunctionStream OS process.""" + + def __init__(self, binary_path: Path, workspace: InstanceWorkspace): + self._binary = binary_path + self._workspace = workspace + self._process: Optional[subprocess.Popen] = None + self._stdout_fh = None + self._stderr_fh = None + + @property + def pid(self) -> Optional[int]: + return self._process.pid if self._process else None + + @property + def is_running(self) -> bool: + return self._process is not None and self._process.poll() is None + + def start(self) -> None: + """Launch the binary, redirecting stdout/stderr to log files.""" + if not self._binary.exists(): + raise FileNotFoundError( + f"Binary not found: {self._binary}. " + "Run 'make integration-build' first." + ) + + env = os.environ.copy() + env["FUNCTION_STREAM_CONF"] = str(self._workspace.config_file) + env["FUNCTION_STREAM_HOME"] = str(self._workspace.root_dir) + + self._stdout_fh = open(self._workspace.stdout_file, "w") + self._stderr_fh = open(self._workspace.stderr_file, "w") + + self._process = subprocess.Popen( + [str(self._binary)], + env=env, + cwd=str(self._workspace.root_dir), + stdout=self._stdout_fh, + stderr=self._stderr_fh, + preexec_fn=os.setsid if os.name != "nt" else None, + ) + logger.info("Process started [pid=%d]", self._process.pid) + + def stop(self, timeout: float = 10.0) -> None: + """Graceful shutdown via SIGTERM, falls back to SIGKILL on timeout.""" + if not self.is_running: + self._close_handles() + return + + logger.info("Sending SIGTERM [pid=%d]", self._process.pid) + try: + os.killpg(os.getpgid(self._process.pid), signal.SIGTERM) + self._process.wait(timeout=timeout) + except (ProcessLookupError, PermissionError): + pass + except subprocess.TimeoutExpired: + logger.warning("SIGTERM timed out, escalating to SIGKILL [pid=%d]", self._process.pid) + self.kill() + return + + self._close_handles() + + def kill(self) -> None: + """Immediately SIGKILL the process group.""" + if not self.is_running: + self._close_handles() + return + + logger.info("Sending SIGKILL [pid=%d]", self._process.pid) + try: + os.killpg(os.getpgid(self._process.pid), signal.SIGKILL) + self._process.wait(timeout=5) + except (ProcessLookupError, PermissionError): + pass + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait(timeout=5) + + self._close_handles() + + def _close_handles(self) -> None: + for fh in (self._stdout_fh, self._stderr_fh): + if fh and not fh.closed: + fh.close() + self._stdout_fh = None + self._stderr_fh = None + self._process = None diff --git a/tests/integration/framework/utils.py b/tests/integration/framework/utils.py new file mode 100644 index 00000000..39793887 --- /dev/null +++ b/tests/integration/framework/utils.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Stateless utility functions: port allocation, health checks. +""" + +import random +import socket +import time + + +def find_free_port(start: int = 18000, end: int = 28000) -> int: + """Find a random available TCP port in the given range.""" + for _ in range(200): + port = random.randint(start, end) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("127.0.0.1", port)) + return port + except OSError: + continue + raise RuntimeError(f"Could not find a free port in [{start}, {end}]") + + +def wait_for_port(host: str, port: int, timeout: float = 30.0) -> bool: + """Block until the given TCP port is accepting connections.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(0.5) + if s.connect_ex((host, port)) == 0: + return True + time.sleep(0.3) + return False diff --git a/tests/integration/framework/workspace.py b/tests/integration/framework/workspace.py new file mode 100644 index 00000000..d34d3472 --- /dev/null +++ b/tests/integration/framework/workspace.py @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +InstanceWorkspace: manages the directory tree for a single +FunctionStream test instance. + +Layout: + tests/integration/target///FunctionStream-/ + conf/config.yaml + data/ + logs/stdout.log, stderr.log, app.log +""" + +import shutil +from datetime import datetime +from pathlib import Path + + +class InstanceWorkspace: + """Owns the on-disk directory environment for one FunctionStream instance.""" + + def __init__(self, target_dir: Path, test_name: str, port: int): + self.target_dir = target_dir + self.test_name = test_name + self.port = port + self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + self.root_dir = ( + self.target_dir + / self.test_name + / self.timestamp + / f"FunctionStream-{self.port}" + ) + self.conf_dir = self.root_dir / "conf" + self.data_dir = self.root_dir / "data" + self.log_dir = self.root_dir / "logs" + + self.config_file = self.conf_dir / "config.yaml" + self.stdout_file = self.log_dir / "stdout.log" + self.stderr_file = self.log_dir / "stderr.log" + + def setup(self) -> None: + """Create the full directory tree.""" + for d in (self.conf_dir, self.data_dir, self.log_dir): + d.mkdir(parents=True, exist_ok=True) + + def cleanup_data(self) -> None: + """Remove the data directory but preserve logs for debugging.""" + if self.data_dir.exists(): + shutil.rmtree(self.data_dir) + self.data_dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini new file mode 100644 index 00000000..4c39a85a --- /dev/null +++ b/tests/integration/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +testpaths = test +python_files = test_*.py +python_classes = Test* +python_functions = test_* +log_cli = true +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)s] %(name)s: %(message)s diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 00000000..67e33688 --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1,4 @@ +pytest>=7.0 +pyyaml>=6.0 +grpcio>=1.60.0 +protobuf>=4.25.0 diff --git a/tests/integration/test/__init__.py b/tests/integration/test/__init__.py new file mode 100644 index 00000000..a47de27e --- /dev/null +++ b/tests/integration/test/__init__.py @@ -0,0 +1,2 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. diff --git a/tests/integration/test/conftest.py b/tests/integration/test/conftest.py new file mode 100644 index 00000000..3539ff6e --- /dev/null +++ b/tests/integration/test/conftest.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Pytest conftest: provides a per-test FunctionStreamInstance fixture. +The fixture yields an *un-started* instance so tests can call +.configure() before .start(). +""" + +import sys +from pathlib import Path + +import pytest + +# Ensure the framework package is importable from tests/integration/ +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from framework import FunctionStreamInstance + + +@pytest.fixture +def fs_instance(request): + """ + Provides a fresh FunctionStreamInstance per test function. + The instance is NOT started automatically — call .start() in the test. + Teardown always kills the process to avoid leaks. + """ + test_name = request.node.name + instance = FunctionStreamInstance(test_name=test_name) + yield instance + instance.kill() diff --git a/tests/integration/test/test_client.py b/tests/integration/test/test_client.py new file mode 100644 index 00000000..9ecba350 --- /dev/null +++ b/tests/integration/test/test_client.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Client access tests: verify gRPC connectivity, SQL execution, +and basic CRUD via the Python FsClient. +""" + + +class TestClientConnect: + + def test_client_target_matches(self, fs_instance): + """FsClient.target points to the correct host:port.""" + fs_instance.start() + with fs_instance.get_client() as client: + assert client.target == f"{fs_instance.host}:{fs_instance.port}" + + def test_show_functions_empty_on_fresh_server(self, fs_instance): + """A fresh instance has no functions registered.""" + fs_instance.start() + with fs_instance.get_client() as client: + result = client.show_functions() + assert result.functions == [] + + +class TestSqlExecution: + + def test_execute_show_tables(self, fs_instance): + """SHOW TABLES on a fresh instance returns a successful response.""" + fs_instance.start() + response = fs_instance.execute_sql("SHOW TABLES") + assert response.status_code < 400 + + def test_execute_invalid_sql(self, fs_instance): + """Invalid SQL returns an error status code (>= 400).""" + fs_instance.start() + response = fs_instance.execute_sql("THIS IS NOT SQL") + assert response.status_code >= 400 diff --git a/tests/integration/test/test_config.py b/tests/integration/test/test_config.py new file mode 100644 index 00000000..05f55e9c --- /dev/null +++ b/tests/integration/test/test_config.py @@ -0,0 +1,85 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Configuration tests: verify config overrides take effect, +workspace directories and config files are created correctly. +""" + +import yaml + + +class TestConfigOverride: + + def test_debug_mode(self, fs_instance): + """service.debug override is written to config.yaml and server starts.""" + fs_instance.configure(**{ + "service.debug": True, + "logging.level": "debug", + }).start() + + assert fs_instance.is_running + + written = yaml.safe_load(fs_instance.workspace.config_file.read_text()) + assert written["service"]["debug"] is True + assert written["logging"]["level"] == "debug" + + def test_config_port_matches_instance(self, fs_instance): + """The port in config.yaml must match the instance's allocated port.""" + fs_instance.workspace.setup() + fs_instance.config.write_to_workspace() + + written = yaml.safe_load(fs_instance.workspace.config_file.read_text()) + assert written["service"]["port"] == fs_instance.port + + def test_nested_override(self, fs_instance): + """Deep dot-separated keys create nested YAML structures.""" + fs_instance.configure(**{ + "state_storage.storage_type": "rocksdb", + "state_storage.base_dir": "/tmp/custom", + }) + fs_instance.workspace.setup() + fs_instance.config.write_to_workspace() + + written = yaml.safe_load(fs_instance.workspace.config_file.read_text()) + assert written["state_storage"]["storage_type"] == "rocksdb" + assert written["state_storage"]["base_dir"] == "/tmp/custom" + + +class TestWorkspaceLayout: + + def test_directories_created(self, fs_instance): + """setup() creates conf/, data/, logs/ directories.""" + fs_instance.workspace.setup() + + assert fs_instance.workspace.conf_dir.exists() + assert fs_instance.workspace.data_dir.exists() + assert fs_instance.workspace.log_dir.exists() + + def test_instance_dir_name_contains_port(self, fs_instance): + """Instance root directory is named FunctionStream-.""" + expected_suffix = f"FunctionStream-{fs_instance.port}" + assert fs_instance.workspace.root_dir.name == expected_suffix + + def test_log_files_preserved_after_stop(self, fs_instance): + """After shutdown, log files still exist on disk.""" + fs_instance.start() + log_dir = fs_instance.workspace.log_dir + fs_instance.stop() + + assert log_dir.exists() + assert fs_instance.workspace.stdout_file.exists() + assert fs_instance.workspace.stderr_file.exists() + + def test_cleanup_data_preserves_logs(self, fs_instance): + """cleanup_data() removes data/ but keeps logs/.""" + fs_instance.start() + + data_marker = fs_instance.workspace.data_dir / "marker.txt" + data_marker.write_text("should be deleted") + + fs_instance.workspace.cleanup_data() + + assert not data_marker.exists() + assert fs_instance.workspace.data_dir.exists() + assert fs_instance.workspace.log_dir.exists() diff --git a/tests/integration/test/test_lifecycle.py b/tests/integration/test/test_lifecycle.py new file mode 100644 index 00000000..3a7fcdf2 --- /dev/null +++ b/tests/integration/test/test_lifecycle.py @@ -0,0 +1,72 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Lifecycle tests: start, stop, kill, restart, parallel instances. +""" + +from framework import FunctionStreamInstance + + +class TestStartAndStop: + + def test_start_and_graceful_stop(self, fs_instance): + """Server starts on a random port and stops gracefully via SIGTERM.""" + fs_instance.start() + + assert fs_instance.is_running + assert fs_instance.port > 0 + assert fs_instance.pid is not None + + fs_instance.stop() + assert not fs_instance.is_running + + def test_force_kill(self, fs_instance): + """Server can be forcefully killed via SIGKILL.""" + fs_instance.start() + assert fs_instance.is_running + + fs_instance.kill() + assert not fs_instance.is_running + + +class TestRestart: + + def test_restart_preserves_port(self, fs_instance): + """After restart, port stays the same but PID changes.""" + fs_instance.start() + original_pid = fs_instance.pid + original_port = fs_instance.port + + fs_instance.restart() + + assert fs_instance.is_running + assert fs_instance.port == original_port + assert fs_instance.pid != original_pid + + def test_restart_server_responds(self, fs_instance): + """After restart, the server still accepts gRPC requests.""" + fs_instance.start() + fs_instance.restart() + + with fs_instance.get_client() as client: + result = client.show_functions() + assert result.functions == [] + + +class TestParallel: + + def test_multiple_instances_run_simultaneously(self): + """Two independent instances can run on different ports at the same time.""" + inst1 = FunctionStreamInstance(test_name="test_parallel_a") + inst2 = FunctionStreamInstance(test_name="test_parallel_b") + try: + inst1.start() + inst2.start() + + assert inst1.is_running + assert inst2.is_running + assert inst1.port != inst2.port + finally: + inst1.kill() + inst2.kill() From cfef9e969e54ef74082d9970ac67f3e732605881 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Apr 2026 00:08:07 +0800 Subject: [PATCH 02/15] update --- .github/workflows/integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 708be1a0..817b0f8f 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -66,5 +66,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: integration-test-logs - path: tests/integration/target/ - retention-days: 7 + path: tests/integration/target/**/logs/ + retention-days: 30 From 366a5dacb1ab7ee4e5c9119d83b6b64af32184e2 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Apr 2026 00:12:15 +0800 Subject: [PATCH 03/15] feat(test): add streaming SQL and WASM function integration tests Add two test subdirectories under tests/integration/test/: - streaming/: SQL DDL tests (CREATE/DROP/SHOW TABLE, catalog persistence across restarts) - wasm/: WASM function lifecycle tests (list, drop, start, stop) and Python function creation error handling Made-with: Cursor --- tests/integration/test/streaming/__init__.py | 2 + .../streaming/test_catalog_persistence.py | 53 +++++++ .../test/streaming/test_sql_ddl.py | 144 ++++++++++++++++++ tests/integration/test/wasm/__init__.py | 2 + .../test/wasm/test_function_lifecycle.py | 61 ++++++++ .../test/wasm/test_python_function.py | 37 +++++ 6 files changed, 299 insertions(+) create mode 100644 tests/integration/test/streaming/__init__.py create mode 100644 tests/integration/test/streaming/test_catalog_persistence.py create mode 100644 tests/integration/test/streaming/test_sql_ddl.py create mode 100644 tests/integration/test/wasm/__init__.py create mode 100644 tests/integration/test/wasm/test_function_lifecycle.py create mode 100644 tests/integration/test/wasm/test_python_function.py diff --git a/tests/integration/test/streaming/__init__.py b/tests/integration/test/streaming/__init__.py new file mode 100644 index 00000000..a47de27e --- /dev/null +++ b/tests/integration/test/streaming/__init__.py @@ -0,0 +1,2 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. diff --git a/tests/integration/test/streaming/test_catalog_persistence.py b/tests/integration/test/streaming/test_catalog_persistence.py new file mode 100644 index 00000000..5aa63cfe --- /dev/null +++ b/tests/integration/test/streaming/test_catalog_persistence.py @@ -0,0 +1,53 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Catalog persistence tests: verify that table metadata survives +a server restart when stream_catalog.persist is enabled. +""" + + +class TestCatalogPersistence: + + def test_table_survives_restart(self, fs_instance): + """A table created before restart is still visible after restart.""" + fs_instance.configure(**{"stream_catalog.persist": True}).start() + + fs_instance.execute_sql(""" + CREATE TABLE persistent_tbl ( + id BIGINT, + ts TIMESTAMP, + WATERMARK FOR ts AS ts - INTERVAL '3' SECOND + ) WITH ( + 'connector' = 'kafka', + 'topic' = 'persist-topic', + 'bootstrap.servers' = 'localhost:9092', + 'format' = 'json' + ) + """) + + fs_instance.restart() + + resp = fs_instance.execute_sql("SHOW TABLES") + assert resp.status_code < 400 + + def test_dropped_table_gone_after_restart(self, fs_instance): + """A table that was dropped should not reappear after restart.""" + fs_instance.configure(**{"stream_catalog.persist": True}).start() + + fs_instance.execute_sql(""" + CREATE TABLE temp_tbl ( + id BIGINT + ) WITH ( + 'connector' = 'kafka', + 'topic' = 'temp-topic', + 'bootstrap.servers' = 'localhost:9092', + 'format' = 'json' + ) + """) + fs_instance.execute_sql("DROP TABLE temp_tbl") + + fs_instance.restart() + + resp = fs_instance.execute_sql("SHOW CREATE TABLE temp_tbl") + assert resp.status_code >= 400 diff --git a/tests/integration/test/streaming/test_sql_ddl.py b/tests/integration/test/streaming/test_sql_ddl.py new file mode 100644 index 00000000..76f5ac86 --- /dev/null +++ b/tests/integration/test/streaming/test_sql_ddl.py @@ -0,0 +1,144 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Streaming SQL DDL tests: CREATE TABLE, DROP TABLE, SHOW TABLES, +SHOW CREATE TABLE, CREATE STREAMING TABLE. +""" + + +class TestShowTables: + + def test_show_tables_empty(self, fs_instance): + """SHOW TABLES returns success on a fresh instance.""" + fs_instance.start() + resp = fs_instance.execute_sql("SHOW TABLES") + assert resp.status_code < 400 + + def test_show_streaming_tables_empty(self, fs_instance): + """SHOW STREAMING TABLES returns success on a fresh instance.""" + fs_instance.start() + resp = fs_instance.execute_sql("SHOW STREAMING TABLES") + assert resp.status_code < 400 + + +class TestCreateTable: + + def test_create_source_table(self, fs_instance): + """CREATE TABLE with connector options registers a source table.""" + fs_instance.start() + + create_sql = """ + CREATE TABLE test_source ( + id BIGINT, + name VARCHAR, + event_time TIMESTAMP, + WATERMARK FOR event_time AS event_time - INTERVAL '5' SECOND + ) WITH ( + 'connector' = 'kafka', + 'topic' = 'test-topic', + 'bootstrap.servers' = 'localhost:9092', + 'format' = 'json' + ) + """ + resp = fs_instance.execute_sql(create_sql) + assert resp.status_code < 400 + + show_resp = fs_instance.execute_sql("SHOW TABLES") + assert show_resp.status_code < 400 + + def test_create_duplicate_table_fails(self, fs_instance): + """Creating the same table twice should return an error.""" + fs_instance.start() + + create_sql = """ + CREATE TABLE dup_table ( + id BIGINT + ) WITH ( + 'connector' = 'kafka', + 'topic' = 'dup-topic', + 'bootstrap.servers' = 'localhost:9092', + 'format' = 'json' + ) + """ + resp1 = fs_instance.execute_sql(create_sql) + assert resp1.status_code < 400 + + resp2 = fs_instance.execute_sql(create_sql) + assert resp2.status_code >= 400 + + +class TestDropTable: + + def test_drop_existing_table(self, fs_instance): + """DROP TABLE removes a previously created table.""" + fs_instance.start() + + fs_instance.execute_sql(""" + CREATE TABLE to_drop ( + id BIGINT + ) WITH ( + 'connector' = 'kafka', + 'topic' = 'drop-topic', + 'bootstrap.servers' = 'localhost:9092', + 'format' = 'json' + ) + """) + + resp = fs_instance.execute_sql("DROP TABLE to_drop") + assert resp.status_code < 400 + + def test_drop_nonexistent_table_fails(self, fs_instance): + """DROP TABLE on a non-existent table returns an error.""" + fs_instance.start() + resp = fs_instance.execute_sql("DROP TABLE no_such_table") + assert resp.status_code >= 400 + + def test_drop_if_exists_nonexistent_succeeds(self, fs_instance): + """DROP TABLE IF EXISTS on a non-existent table should succeed.""" + fs_instance.start() + resp = fs_instance.execute_sql("DROP TABLE IF EXISTS no_such_table") + assert resp.status_code < 400 + + +class TestShowCreateTable: + + def test_show_create_table(self, fs_instance): + """SHOW CREATE TABLE returns DDL for an existing table.""" + fs_instance.start() + + fs_instance.execute_sql(""" + CREATE TABLE show_me ( + id BIGINT, + value VARCHAR + ) WITH ( + 'connector' = 'kafka', + 'topic' = 'show-topic', + 'bootstrap.servers' = 'localhost:9092', + 'format' = 'json' + ) + """) + + resp = fs_instance.execute_sql("SHOW CREATE TABLE show_me") + assert resp.status_code < 400 + + def test_show_create_nonexistent_fails(self, fs_instance): + """SHOW CREATE TABLE on a missing table returns an error.""" + fs_instance.start() + resp = fs_instance.execute_sql("SHOW CREATE TABLE ghost_table") + assert resp.status_code >= 400 + + +class TestSqlErrorHandling: + + def test_invalid_sql_syntax(self, fs_instance): + """Malformed SQL returns an error status.""" + fs_instance.start() + resp = fs_instance.execute_sql("NOT VALID SQL AT ALL") + assert resp.status_code >= 400 + + def test_empty_sql(self, fs_instance): + """Empty SQL string returns an error status.""" + fs_instance.start() + resp = fs_instance.execute_sql("") + assert resp.status_code >= 400 diff --git a/tests/integration/test/wasm/__init__.py b/tests/integration/test/wasm/__init__.py new file mode 100644 index 00000000..a47de27e --- /dev/null +++ b/tests/integration/test/wasm/__init__.py @@ -0,0 +1,2 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. diff --git a/tests/integration/test/wasm/test_function_lifecycle.py b/tests/integration/test/wasm/test_function_lifecycle.py new file mode 100644 index 00000000..9a046226 --- /dev/null +++ b/tests/integration/test/wasm/test_function_lifecycle.py @@ -0,0 +1,61 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +WASM function lifecycle tests: create, list, start, stop, drop functions +via the gRPC FsClient API. +""" + + +class TestFunctionList: + + def test_show_functions_empty(self, fs_instance): + """A fresh server has no functions registered.""" + fs_instance.start() + with fs_instance.get_client() as client: + result = client.show_functions() + assert result.functions == [] + + def test_show_functions_after_drop(self, fs_instance): + """After dropping a function, show_functions reflects the removal.""" + fs_instance.start() + with fs_instance.get_client() as client: + result = client.show_functions() + initial_count = len(result.functions) + assert initial_count == 0 + + +class TestFunctionDrop: + + def test_drop_nonexistent_function_fails(self, fs_instance): + """Dropping a function that does not exist should raise an error.""" + fs_instance.start() + with fs_instance.get_client() as client: + try: + client.drop_function("no_such_function") + assert False, "Expected an error when dropping non-existent function" + except Exception: + pass + + +class TestFunctionStartStop: + + def test_start_nonexistent_function_fails(self, fs_instance): + """Starting a function that does not exist should raise an error.""" + fs_instance.start() + with fs_instance.get_client() as client: + try: + client.start_function("ghost_function") + assert False, "Expected an error when starting non-existent function" + except Exception: + pass + + def test_stop_nonexistent_function_fails(self, fs_instance): + """Stopping a function that does not exist should raise an error.""" + fs_instance.start() + with fs_instance.get_client() as client: + try: + client.stop_function("phantom_function") + assert False, "Expected an error when stopping non-existent function" + except Exception: + pass diff --git a/tests/integration/test/wasm/test_python_function.py b/tests/integration/test/wasm/test_python_function.py new file mode 100644 index 00000000..63e1a936 --- /dev/null +++ b/tests/integration/test/wasm/test_python_function.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +""" +Python WASM function tests: create and manage Python functions +via the CreatePythonFunction gRPC API. +""" + + +class TestCreatePythonFunction: + + def test_create_with_empty_modules_fails(self, fs_instance): + """Creating a Python function with no modules should fail.""" + fs_instance.start() + with fs_instance.get_client() as client: + try: + client.create_python_function( + class_name="EmptyDriver", + modules=[], + config_content="task_name: empty", + ) + assert False, "Expected ValueError for empty modules" + except (ValueError, Exception): + pass + + def test_create_with_invalid_class_name(self, fs_instance): + """Creating a Python function with a non-existent class should fail at server side.""" + fs_instance.start() + with fs_instance.get_client() as client: + try: + client.create_python_function( + class_name="NoSuchClass", + modules=[("fake_module", b"x = 1\n")], + config_content="task_name: bad_class_test", + ) + except Exception: + pass From ca2dc8e60aadf19f976071b41719437aabe03262 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Apr 2026 00:24:18 +0800 Subject: [PATCH 04/15] update --- .gitignore | 1 + tests/integration/Makefile | 4 +- tests/integration/install | 0 tests/integration/test/conftest.py | 31 ------- tests/integration/test/streaming/conftest.py | 19 +++++ tests/integration/test/test_client.py | 38 --------- tests/integration/test/test_config.py | 85 ------------------- tests/integration/test/test_lifecycle.py | 72 ---------------- tests/integration/test/wasm/conftest.py | 19 +++++ .../test/wasm/test_function_lifecycle.py | 61 ------------- .../test/wasm/test_python_function.py | 37 -------- 11 files changed, 41 insertions(+), 326 deletions(-) create mode 100644 tests/integration/install delete mode 100644 tests/integration/test/conftest.py create mode 100644 tests/integration/test/streaming/conftest.py delete mode 100644 tests/integration/test/test_client.py delete mode 100644 tests/integration/test/test_config.py delete mode 100644 tests/integration/test/test_lifecycle.py create mode 100644 tests/integration/test/wasm/conftest.py delete mode 100644 tests/integration/test/wasm/test_function_lifecycle.py delete mode 100644 tests/integration/test/wasm/test_python_function.py diff --git a/.gitignore b/.gitignore index 8eec5687..5dffeb0f 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ python/**/target/ # Integration test output tests/integration/target/ tests/integration/.venv/ +tests/integration/install diff --git a/tests/integration/Makefile b/tests/integration/Makefile index d800d9f3..47b0d99e 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -34,7 +34,7 @@ help: @echo " test Setup Python env + run pytest (PYTEST_ARGS=...)" @echo " clean Remove .venv and target/tests output" -$(VENV)/.installed: requirements.txt $(PYTHON_ROOT)/functionstream-api/pyproject.toml $(PYTHON_ROOT)/functionstream-client/pyproject.toml +install $(VENV)/.installed: requirements.txt $(PYTHON_ROOT)/functionstream-api/pyproject.toml $(PYTHON_ROOT)/functionstream-client/pyproject.toml $(call log,ENV,Setting up Python virtual environment) @test -d $(VENV) || python3 -m venv $(VENV) @$(PIP) install --quiet --upgrade pip @@ -44,7 +44,7 @@ $(VENV)/.installed: requirements.txt $(PYTHON_ROOT)/functionstream-api/pyproject @touch $@ $(call success,Python environment ready) -test: $(VENV)/.installed +test: install $(call log,TEST,Running integration tests) @$(PY) -m pytest -v $(PYTEST_ARGS) $(call success,All integration tests passed) diff --git a/tests/integration/install b/tests/integration/install new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test/conftest.py b/tests/integration/test/conftest.py deleted file mode 100644 index 3539ff6e..00000000 --- a/tests/integration/test/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -Pytest conftest: provides a per-test FunctionStreamInstance fixture. -The fixture yields an *un-started* instance so tests can call -.configure() before .start(). -""" - -import sys -from pathlib import Path - -import pytest - -# Ensure the framework package is importable from tests/integration/ -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from framework import FunctionStreamInstance - - -@pytest.fixture -def fs_instance(request): - """ - Provides a fresh FunctionStreamInstance per test function. - The instance is NOT started automatically — call .start() in the test. - Teardown always kills the process to avoid leaks. - """ - test_name = request.node.name - instance = FunctionStreamInstance(test_name=test_name) - yield instance - instance.kill() diff --git a/tests/integration/test/streaming/conftest.py b/tests/integration/test/streaming/conftest.py new file mode 100644 index 00000000..26539187 --- /dev/null +++ b/tests/integration/test/streaming/conftest.py @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from framework import FunctionStreamInstance + + +@pytest.fixture +def fs_instance(request): + """Provides a fresh FunctionStreamInstance per test. Not started automatically.""" + instance = FunctionStreamInstance(test_name=request.node.name) + yield instance + instance.kill() diff --git a/tests/integration/test/test_client.py b/tests/integration/test/test_client.py deleted file mode 100644 index 9ecba350..00000000 --- a/tests/integration/test/test_client.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -Client access tests: verify gRPC connectivity, SQL execution, -and basic CRUD via the Python FsClient. -""" - - -class TestClientConnect: - - def test_client_target_matches(self, fs_instance): - """FsClient.target points to the correct host:port.""" - fs_instance.start() - with fs_instance.get_client() as client: - assert client.target == f"{fs_instance.host}:{fs_instance.port}" - - def test_show_functions_empty_on_fresh_server(self, fs_instance): - """A fresh instance has no functions registered.""" - fs_instance.start() - with fs_instance.get_client() as client: - result = client.show_functions() - assert result.functions == [] - - -class TestSqlExecution: - - def test_execute_show_tables(self, fs_instance): - """SHOW TABLES on a fresh instance returns a successful response.""" - fs_instance.start() - response = fs_instance.execute_sql("SHOW TABLES") - assert response.status_code < 400 - - def test_execute_invalid_sql(self, fs_instance): - """Invalid SQL returns an error status code (>= 400).""" - fs_instance.start() - response = fs_instance.execute_sql("THIS IS NOT SQL") - assert response.status_code >= 400 diff --git a/tests/integration/test/test_config.py b/tests/integration/test/test_config.py deleted file mode 100644 index 05f55e9c..00000000 --- a/tests/integration/test/test_config.py +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -Configuration tests: verify config overrides take effect, -workspace directories and config files are created correctly. -""" - -import yaml - - -class TestConfigOverride: - - def test_debug_mode(self, fs_instance): - """service.debug override is written to config.yaml and server starts.""" - fs_instance.configure(**{ - "service.debug": True, - "logging.level": "debug", - }).start() - - assert fs_instance.is_running - - written = yaml.safe_load(fs_instance.workspace.config_file.read_text()) - assert written["service"]["debug"] is True - assert written["logging"]["level"] == "debug" - - def test_config_port_matches_instance(self, fs_instance): - """The port in config.yaml must match the instance's allocated port.""" - fs_instance.workspace.setup() - fs_instance.config.write_to_workspace() - - written = yaml.safe_load(fs_instance.workspace.config_file.read_text()) - assert written["service"]["port"] == fs_instance.port - - def test_nested_override(self, fs_instance): - """Deep dot-separated keys create nested YAML structures.""" - fs_instance.configure(**{ - "state_storage.storage_type": "rocksdb", - "state_storage.base_dir": "/tmp/custom", - }) - fs_instance.workspace.setup() - fs_instance.config.write_to_workspace() - - written = yaml.safe_load(fs_instance.workspace.config_file.read_text()) - assert written["state_storage"]["storage_type"] == "rocksdb" - assert written["state_storage"]["base_dir"] == "/tmp/custom" - - -class TestWorkspaceLayout: - - def test_directories_created(self, fs_instance): - """setup() creates conf/, data/, logs/ directories.""" - fs_instance.workspace.setup() - - assert fs_instance.workspace.conf_dir.exists() - assert fs_instance.workspace.data_dir.exists() - assert fs_instance.workspace.log_dir.exists() - - def test_instance_dir_name_contains_port(self, fs_instance): - """Instance root directory is named FunctionStream-.""" - expected_suffix = f"FunctionStream-{fs_instance.port}" - assert fs_instance.workspace.root_dir.name == expected_suffix - - def test_log_files_preserved_after_stop(self, fs_instance): - """After shutdown, log files still exist on disk.""" - fs_instance.start() - log_dir = fs_instance.workspace.log_dir - fs_instance.stop() - - assert log_dir.exists() - assert fs_instance.workspace.stdout_file.exists() - assert fs_instance.workspace.stderr_file.exists() - - def test_cleanup_data_preserves_logs(self, fs_instance): - """cleanup_data() removes data/ but keeps logs/.""" - fs_instance.start() - - data_marker = fs_instance.workspace.data_dir / "marker.txt" - data_marker.write_text("should be deleted") - - fs_instance.workspace.cleanup_data() - - assert not data_marker.exists() - assert fs_instance.workspace.data_dir.exists() - assert fs_instance.workspace.log_dir.exists() diff --git a/tests/integration/test/test_lifecycle.py b/tests/integration/test/test_lifecycle.py deleted file mode 100644 index 3a7fcdf2..00000000 --- a/tests/integration/test/test_lifecycle.py +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -Lifecycle tests: start, stop, kill, restart, parallel instances. -""" - -from framework import FunctionStreamInstance - - -class TestStartAndStop: - - def test_start_and_graceful_stop(self, fs_instance): - """Server starts on a random port and stops gracefully via SIGTERM.""" - fs_instance.start() - - assert fs_instance.is_running - assert fs_instance.port > 0 - assert fs_instance.pid is not None - - fs_instance.stop() - assert not fs_instance.is_running - - def test_force_kill(self, fs_instance): - """Server can be forcefully killed via SIGKILL.""" - fs_instance.start() - assert fs_instance.is_running - - fs_instance.kill() - assert not fs_instance.is_running - - -class TestRestart: - - def test_restart_preserves_port(self, fs_instance): - """After restart, port stays the same but PID changes.""" - fs_instance.start() - original_pid = fs_instance.pid - original_port = fs_instance.port - - fs_instance.restart() - - assert fs_instance.is_running - assert fs_instance.port == original_port - assert fs_instance.pid != original_pid - - def test_restart_server_responds(self, fs_instance): - """After restart, the server still accepts gRPC requests.""" - fs_instance.start() - fs_instance.restart() - - with fs_instance.get_client() as client: - result = client.show_functions() - assert result.functions == [] - - -class TestParallel: - - def test_multiple_instances_run_simultaneously(self): - """Two independent instances can run on different ports at the same time.""" - inst1 = FunctionStreamInstance(test_name="test_parallel_a") - inst2 = FunctionStreamInstance(test_name="test_parallel_b") - try: - inst1.start() - inst2.start() - - assert inst1.is_running - assert inst2.is_running - assert inst1.port != inst2.port - finally: - inst1.kill() - inst2.kill() diff --git a/tests/integration/test/wasm/conftest.py b/tests/integration/test/wasm/conftest.py new file mode 100644 index 00000000..26539187 --- /dev/null +++ b/tests/integration/test/wasm/conftest.py @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from framework import FunctionStreamInstance + + +@pytest.fixture +def fs_instance(request): + """Provides a fresh FunctionStreamInstance per test. Not started automatically.""" + instance = FunctionStreamInstance(test_name=request.node.name) + yield instance + instance.kill() diff --git a/tests/integration/test/wasm/test_function_lifecycle.py b/tests/integration/test/wasm/test_function_lifecycle.py deleted file mode 100644 index 9a046226..00000000 --- a/tests/integration/test/wasm/test_function_lifecycle.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -WASM function lifecycle tests: create, list, start, stop, drop functions -via the gRPC FsClient API. -""" - - -class TestFunctionList: - - def test_show_functions_empty(self, fs_instance): - """A fresh server has no functions registered.""" - fs_instance.start() - with fs_instance.get_client() as client: - result = client.show_functions() - assert result.functions == [] - - def test_show_functions_after_drop(self, fs_instance): - """After dropping a function, show_functions reflects the removal.""" - fs_instance.start() - with fs_instance.get_client() as client: - result = client.show_functions() - initial_count = len(result.functions) - assert initial_count == 0 - - -class TestFunctionDrop: - - def test_drop_nonexistent_function_fails(self, fs_instance): - """Dropping a function that does not exist should raise an error.""" - fs_instance.start() - with fs_instance.get_client() as client: - try: - client.drop_function("no_such_function") - assert False, "Expected an error when dropping non-existent function" - except Exception: - pass - - -class TestFunctionStartStop: - - def test_start_nonexistent_function_fails(self, fs_instance): - """Starting a function that does not exist should raise an error.""" - fs_instance.start() - with fs_instance.get_client() as client: - try: - client.start_function("ghost_function") - assert False, "Expected an error when starting non-existent function" - except Exception: - pass - - def test_stop_nonexistent_function_fails(self, fs_instance): - """Stopping a function that does not exist should raise an error.""" - fs_instance.start() - with fs_instance.get_client() as client: - try: - client.stop_function("phantom_function") - assert False, "Expected an error when stopping non-existent function" - except Exception: - pass diff --git a/tests/integration/test/wasm/test_python_function.py b/tests/integration/test/wasm/test_python_function.py deleted file mode 100644 index 63e1a936..00000000 --- a/tests/integration/test/wasm/test_python_function.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -Python WASM function tests: create and manage Python functions -via the CreatePythonFunction gRPC API. -""" - - -class TestCreatePythonFunction: - - def test_create_with_empty_modules_fails(self, fs_instance): - """Creating a Python function with no modules should fail.""" - fs_instance.start() - with fs_instance.get_client() as client: - try: - client.create_python_function( - class_name="EmptyDriver", - modules=[], - config_content="task_name: empty", - ) - assert False, "Expected ValueError for empty modules" - except (ValueError, Exception): - pass - - def test_create_with_invalid_class_name(self, fs_instance): - """Creating a Python function with a non-existent class should fail at server side.""" - fs_instance.start() - with fs_instance.get_client() as client: - try: - client.create_python_function( - class_name="NoSuchClass", - modules=[("fake_module", b"x = 1\n")], - config_content="task_name: bad_class_test", - ) - except Exception: - pass From 99e4c8f82b78170c36c8719ac34c736eba8e3384 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Apr 2026 00:25:14 +0800 Subject: [PATCH 05/15] update --- tests/integration/test/streaming/conftest.py | 19 --- .../streaming/test_catalog_persistence.py | 53 ------- .../test/streaming/test_sql_ddl.py | 144 ------------------ tests/integration/test/wasm/conftest.py | 19 --- 4 files changed, 235 deletions(-) delete mode 100644 tests/integration/test/streaming/conftest.py delete mode 100644 tests/integration/test/streaming/test_catalog_persistence.py delete mode 100644 tests/integration/test/streaming/test_sql_ddl.py delete mode 100644 tests/integration/test/wasm/conftest.py diff --git a/tests/integration/test/streaming/conftest.py b/tests/integration/test/streaming/conftest.py deleted file mode 100644 index 26539187..00000000 --- a/tests/integration/test/streaming/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from framework import FunctionStreamInstance - - -@pytest.fixture -def fs_instance(request): - """Provides a fresh FunctionStreamInstance per test. Not started automatically.""" - instance = FunctionStreamInstance(test_name=request.node.name) - yield instance - instance.kill() diff --git a/tests/integration/test/streaming/test_catalog_persistence.py b/tests/integration/test/streaming/test_catalog_persistence.py deleted file mode 100644 index 5aa63cfe..00000000 --- a/tests/integration/test/streaming/test_catalog_persistence.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -Catalog persistence tests: verify that table metadata survives -a server restart when stream_catalog.persist is enabled. -""" - - -class TestCatalogPersistence: - - def test_table_survives_restart(self, fs_instance): - """A table created before restart is still visible after restart.""" - fs_instance.configure(**{"stream_catalog.persist": True}).start() - - fs_instance.execute_sql(""" - CREATE TABLE persistent_tbl ( - id BIGINT, - ts TIMESTAMP, - WATERMARK FOR ts AS ts - INTERVAL '3' SECOND - ) WITH ( - 'connector' = 'kafka', - 'topic' = 'persist-topic', - 'bootstrap.servers' = 'localhost:9092', - 'format' = 'json' - ) - """) - - fs_instance.restart() - - resp = fs_instance.execute_sql("SHOW TABLES") - assert resp.status_code < 400 - - def test_dropped_table_gone_after_restart(self, fs_instance): - """A table that was dropped should not reappear after restart.""" - fs_instance.configure(**{"stream_catalog.persist": True}).start() - - fs_instance.execute_sql(""" - CREATE TABLE temp_tbl ( - id BIGINT - ) WITH ( - 'connector' = 'kafka', - 'topic' = 'temp-topic', - 'bootstrap.servers' = 'localhost:9092', - 'format' = 'json' - ) - """) - fs_instance.execute_sql("DROP TABLE temp_tbl") - - fs_instance.restart() - - resp = fs_instance.execute_sql("SHOW CREATE TABLE temp_tbl") - assert resp.status_code >= 400 diff --git a/tests/integration/test/streaming/test_sql_ddl.py b/tests/integration/test/streaming/test_sql_ddl.py deleted file mode 100644 index 76f5ac86..00000000 --- a/tests/integration/test/streaming/test_sql_ddl.py +++ /dev/null @@ -1,144 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -""" -Streaming SQL DDL tests: CREATE TABLE, DROP TABLE, SHOW TABLES, -SHOW CREATE TABLE, CREATE STREAMING TABLE. -""" - - -class TestShowTables: - - def test_show_tables_empty(self, fs_instance): - """SHOW TABLES returns success on a fresh instance.""" - fs_instance.start() - resp = fs_instance.execute_sql("SHOW TABLES") - assert resp.status_code < 400 - - def test_show_streaming_tables_empty(self, fs_instance): - """SHOW STREAMING TABLES returns success on a fresh instance.""" - fs_instance.start() - resp = fs_instance.execute_sql("SHOW STREAMING TABLES") - assert resp.status_code < 400 - - -class TestCreateTable: - - def test_create_source_table(self, fs_instance): - """CREATE TABLE with connector options registers a source table.""" - fs_instance.start() - - create_sql = """ - CREATE TABLE test_source ( - id BIGINT, - name VARCHAR, - event_time TIMESTAMP, - WATERMARK FOR event_time AS event_time - INTERVAL '5' SECOND - ) WITH ( - 'connector' = 'kafka', - 'topic' = 'test-topic', - 'bootstrap.servers' = 'localhost:9092', - 'format' = 'json' - ) - """ - resp = fs_instance.execute_sql(create_sql) - assert resp.status_code < 400 - - show_resp = fs_instance.execute_sql("SHOW TABLES") - assert show_resp.status_code < 400 - - def test_create_duplicate_table_fails(self, fs_instance): - """Creating the same table twice should return an error.""" - fs_instance.start() - - create_sql = """ - CREATE TABLE dup_table ( - id BIGINT - ) WITH ( - 'connector' = 'kafka', - 'topic' = 'dup-topic', - 'bootstrap.servers' = 'localhost:9092', - 'format' = 'json' - ) - """ - resp1 = fs_instance.execute_sql(create_sql) - assert resp1.status_code < 400 - - resp2 = fs_instance.execute_sql(create_sql) - assert resp2.status_code >= 400 - - -class TestDropTable: - - def test_drop_existing_table(self, fs_instance): - """DROP TABLE removes a previously created table.""" - fs_instance.start() - - fs_instance.execute_sql(""" - CREATE TABLE to_drop ( - id BIGINT - ) WITH ( - 'connector' = 'kafka', - 'topic' = 'drop-topic', - 'bootstrap.servers' = 'localhost:9092', - 'format' = 'json' - ) - """) - - resp = fs_instance.execute_sql("DROP TABLE to_drop") - assert resp.status_code < 400 - - def test_drop_nonexistent_table_fails(self, fs_instance): - """DROP TABLE on a non-existent table returns an error.""" - fs_instance.start() - resp = fs_instance.execute_sql("DROP TABLE no_such_table") - assert resp.status_code >= 400 - - def test_drop_if_exists_nonexistent_succeeds(self, fs_instance): - """DROP TABLE IF EXISTS on a non-existent table should succeed.""" - fs_instance.start() - resp = fs_instance.execute_sql("DROP TABLE IF EXISTS no_such_table") - assert resp.status_code < 400 - - -class TestShowCreateTable: - - def test_show_create_table(self, fs_instance): - """SHOW CREATE TABLE returns DDL for an existing table.""" - fs_instance.start() - - fs_instance.execute_sql(""" - CREATE TABLE show_me ( - id BIGINT, - value VARCHAR - ) WITH ( - 'connector' = 'kafka', - 'topic' = 'show-topic', - 'bootstrap.servers' = 'localhost:9092', - 'format' = 'json' - ) - """) - - resp = fs_instance.execute_sql("SHOW CREATE TABLE show_me") - assert resp.status_code < 400 - - def test_show_create_nonexistent_fails(self, fs_instance): - """SHOW CREATE TABLE on a missing table returns an error.""" - fs_instance.start() - resp = fs_instance.execute_sql("SHOW CREATE TABLE ghost_table") - assert resp.status_code >= 400 - - -class TestSqlErrorHandling: - - def test_invalid_sql_syntax(self, fs_instance): - """Malformed SQL returns an error status.""" - fs_instance.start() - resp = fs_instance.execute_sql("NOT VALID SQL AT ALL") - assert resp.status_code >= 400 - - def test_empty_sql(self, fs_instance): - """Empty SQL string returns an error status.""" - fs_instance.start() - resp = fs_instance.execute_sql("") - assert resp.status_code >= 400 diff --git a/tests/integration/test/wasm/conftest.py b/tests/integration/test/wasm/conftest.py deleted file mode 100644 index 26539187..00000000 --- a/tests/integration/test/wasm/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from framework import FunctionStreamInstance - - -@pytest.fixture -def fs_instance(request): - """Provides a fresh FunctionStreamInstance per test. Not started automatically.""" - instance = FunctionStreamInstance(test_name=request.node.name) - yield instance - instance.kill() From edbf1f72d31d7b681805e372e950e8e0db023036 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Apr 2026 00:32:05 +0800 Subject: [PATCH 06/15] update --- tests/integration/Makefile | 9 +++++++++ tests/integration/framework/__init__.py | 9 +++++++++ tests/integration/framework/config.py | 9 +++++++++ tests/integration/framework/instance.py | 9 +++++++++ tests/integration/framework/process.py | 9 +++++++++ tests/integration/framework/utils.py | 9 +++++++++ tests/integration/framework/workspace.py | 9 +++++++++ tests/integration/install | 0 tests/integration/pytest.ini | 12 ++++++++++++ tests/integration/requirements.txt | 12 ++++++++++++ tests/integration/test/__init__.py | 9 +++++++++ tests/integration/test/streaming/__init__.py | 9 +++++++++ tests/integration/test/wasm/__init__.py | 9 +++++++++ 13 files changed, 114 insertions(+) delete mode 100644 tests/integration/install diff --git a/tests/integration/Makefile b/tests/integration/Makefile index 47b0d99e..35a2fcf6 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -1,5 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # ----------------------------------------------------------------------- # Integration Test Makefile diff --git a/tests/integration/framework/__init__.py b/tests/integration/framework/__init__.py index 28491606..5cba3938 100644 --- a/tests/integration/framework/__init__.py +++ b/tests/integration/framework/__init__.py @@ -1,5 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from .instance import FunctionStreamInstance diff --git a/tests/integration/framework/config.py b/tests/integration/framework/config.py index 7b90ce88..e810c8f1 100644 --- a/tests/integration/framework/config.py +++ b/tests/integration/framework/config.py @@ -1,5 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ InstanceConfig: builds and writes the config.yaml consumed by diff --git a/tests/integration/framework/instance.py b/tests/integration/framework/instance.py index c1bc74e1..dbf9ba0e 100644 --- a/tests/integration/framework/instance.py +++ b/tests/integration/framework/instance.py @@ -1,5 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ FunctionStreamInstance: the Facade that composes workspace, config, diff --git a/tests/integration/framework/process.py b/tests/integration/framework/process.py index 3f92e98b..6b04de67 100644 --- a/tests/integration/framework/process.py +++ b/tests/integration/framework/process.py @@ -1,5 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ FunctionStreamProcess: owns the OS-level process lifecycle diff --git a/tests/integration/framework/utils.py b/tests/integration/framework/utils.py index 39793887..d4783a81 100644 --- a/tests/integration/framework/utils.py +++ b/tests/integration/framework/utils.py @@ -1,5 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Stateless utility functions: port allocation, health checks. diff --git a/tests/integration/framework/workspace.py b/tests/integration/framework/workspace.py index d34d3472..d41133cf 100644 --- a/tests/integration/framework/workspace.py +++ b/tests/integration/framework/workspace.py @@ -1,5 +1,14 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ InstanceWorkspace: manages the directory tree for a single diff --git a/tests/integration/install b/tests/integration/install deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 4c39a85a..6e745e7f 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -1,3 +1,15 @@ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. + [pytest] testpaths = test python_files = test_*.py diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 67e33688..4c25aaff 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,3 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + pytest>=7.0 pyyaml>=6.0 grpcio>=1.60.0 diff --git a/tests/integration/test/__init__.py b/tests/integration/test/__init__.py index a47de27e..4d9a9249 100644 --- a/tests/integration/test/__init__.py +++ b/tests/integration/test/__init__.py @@ -1,2 +1,11 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/test/streaming/__init__.py b/tests/integration/test/streaming/__init__.py index a47de27e..4d9a9249 100644 --- a/tests/integration/test/streaming/__init__.py +++ b/tests/integration/test/streaming/__init__.py @@ -1,2 +1,11 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/test/wasm/__init__.py b/tests/integration/test/wasm/__init__.py index a47de27e..4d9a9249 100644 --- a/tests/integration/test/wasm/__init__.py +++ b/tests/integration/test/wasm/__init__.py @@ -1,2 +1,11 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 64c47a4c8e4ecc3ad5cfc37b1f193d2db5bee3ec Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Apr 2026 21:43:04 +0800 Subject: [PATCH 07/15] update --- tests/integration/test/wasm/go_sdk/__init__.py | 11 +++++++++++ tests/integration/test/wasm/python_sdk/__init__.py | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/integration/test/wasm/go_sdk/__init__.py create mode 100644 tests/integration/test/wasm/python_sdk/__init__.py diff --git a/tests/integration/test/wasm/go_sdk/__init__.py b/tests/integration/test/wasm/go_sdk/__init__.py new file mode 100644 index 00000000..4d9a9249 --- /dev/null +++ b/tests/integration/test/wasm/go_sdk/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/integration/test/wasm/python_sdk/__init__.py b/tests/integration/test/wasm/python_sdk/__init__.py new file mode 100644 index 00000000..4d9a9249 --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From d1cc9f8471125ab47ce4af208b07b5e1ab1f73d4 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Apr 2026 00:00:35 +0800 Subject: [PATCH 08/15] update --- src/server/handler.rs | 28 +- tests/integration/Makefile | 5 +- tests/integration/framework/config.py | 20 + tests/integration/requirements.txt | 5 + .../integration/test/wasm/go_sdk/__init__.py | 11 - .../test/wasm/python_sdk/conftest.py | 43 ++ .../python_sdk/test_client_error_handling.py | 304 ++++++++ .../test/wasm/python_sdk/test_config_unit.py | 293 ++++++++ .../test/wasm/python_sdk/test_python_api.py | 694 ++++++++++++++++++ 9 files changed, 1386 insertions(+), 17 deletions(-) delete mode 100644 tests/integration/test/wasm/go_sdk/__init__.py create mode 100644 tests/integration/test/wasm/python_sdk/conftest.py create mode 100644 tests/integration/test/wasm/python_sdk/test_client_error_handling.py create mode 100644 tests/integration/test/wasm/python_sdk/test_config_unit.py create mode 100644 tests/integration/test/wasm/python_sdk/test_python_api.py diff --git a/src/server/handler.rs b/src/server/handler.rs index 82ccb803..0319e352 100644 --- a/src/server/handler.rs +++ b/src/server/handler.rs @@ -91,6 +91,25 @@ impl FunctionStreamServiceImpl { } } + fn classify_error(message: &str) -> StatusCode { + let lower = message.to_lowercase(); + if lower.contains("not found") || lower.contains("not exist") { + StatusCode::NotFound + } else if lower.contains("uniqueness violation") + || lower.contains("already exists") + || lower.contains("duplicate") + { + StatusCode::Conflict + } else if lower.contains("invalid") + || lower.contains("unsupported") + || lower.contains("missing") + { + StatusCode::BadRequest + } else { + StatusCode::InternalServerError + } + } + async fn execute_statement( &self, stmt: &dyn Statement, @@ -101,7 +120,8 @@ impl FunctionStreamServiceImpl { if result.success { Self::build_success_response(success_status, result.message, result.data) } else { - Self::build_error_response(StatusCode::InternalServerError, result.message) + let status = Self::classify_error(&result.message); + Self::build_error_response(status, result.message) } } } @@ -139,8 +159,9 @@ impl FunctionStreamService for FunctionStreamServiceImpl { if !result.success { error!("SQL execution aborted: {}", result.message); + let status = Self::classify_error(&result.message); return Ok(TonicResponse::new(Self::build_error_response( - StatusCode::InternalServerError, + status, result.message, ))); } @@ -235,8 +256,9 @@ impl FunctionStreamService for FunctionStreamServiceImpl { if !result.success { error!("show_functions execution failed: {}", result.message); + let status = Self::classify_error(&result.message); return Ok(TonicResponse::new(ShowFunctionsResponse { - status_code: StatusCode::InternalServerError as i32, + status_code: status as i32, message: result.message, functions: vec![], })); diff --git a/tests/integration/Makefile b/tests/integration/Makefile index 35a2fcf6..f16d640b 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -43,13 +43,11 @@ help: @echo " test Setup Python env + run pytest (PYTEST_ARGS=...)" @echo " clean Remove .venv and target/tests output" -install $(VENV)/.installed: requirements.txt $(PYTHON_ROOT)/functionstream-api/pyproject.toml $(PYTHON_ROOT)/functionstream-client/pyproject.toml +install: requirements.txt $(PYTHON_ROOT)/functionstream-api/pyproject.toml $(PYTHON_ROOT)/functionstream-client/pyproject.toml $(call log,ENV,Setting up Python virtual environment) @test -d $(VENV) || python3 -m venv $(VENV) @$(PIP) install --quiet --upgrade pip @$(PIP) install --quiet -r requirements.txt - @$(PIP) install --quiet -e $(PYTHON_ROOT)/functionstream-api - @$(PIP) install --quiet -e $(PYTHON_ROOT)/functionstream-client @touch $@ $(call success,Python environment ready) @@ -62,4 +60,5 @@ clean: $(call log,CLEAN,Removing test artifacts) @rm -rf $(VENV) @rm -rf $(CURDIR)/target + @rm -rf $(CURDIR)/install $(call success,Clean complete) diff --git a/tests/integration/framework/config.py b/tests/integration/framework/config.py index e810c8f1..1ddb97ce 100644 --- a/tests/integration/framework/config.py +++ b/tests/integration/framework/config.py @@ -22,6 +22,21 @@ from .workspace import InstanceWorkspace +_INTEGRATION_DIR = Path(__file__).resolve().parents[1] +_PROJECT_ROOT = _INTEGRATION_DIR.parents[1] + + +def _find_python_wasm() -> str: + """Locate the Python WASM runtime for server initialisation.""" + candidates = [ + _PROJECT_ROOT / "python" / "functionstream-runtime" / "target" / "functionstream-python-runtime.wasm", + _PROJECT_ROOT / "dist" / "function-stream" / "data" / "cache" / "python-runner" / "functionstream-python-runtime.wasm", + ] + for c in candidates: + if c.exists(): + return str(c.resolve()) + return str(candidates[0]) + class InstanceConfig: """Generates and persists config.yaml for one FunctionStream instance.""" @@ -44,6 +59,11 @@ def __init__(self, host: str, port: int, workspace: InstanceWorkspace): "max_file_size": 50, "max_files": 3, }, + "python": { + "wasm_path": _find_python_wasm(), + "cache_dir": str(workspace.data_dir / "cache" / "python-runner"), + "enable_cache": True, + }, "state_storage": { "storage_type": "memory", }, diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 4c25aaff..45a16ecc 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -10,7 +10,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Third-party dependencies pytest>=7.0 pyyaml>=6.0 grpcio>=1.60.0 protobuf>=4.25.0 + +# FunctionStream Python packages (local editable installs) +-e ../../python/functionstream-api +-e ../../python/functionstream-client diff --git a/tests/integration/test/wasm/go_sdk/__init__.py b/tests/integration/test/wasm/go_sdk/__init__.py deleted file mode 100644 index 4d9a9249..00000000 --- a/tests/integration/test/wasm/go_sdk/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/integration/test/wasm/python_sdk/conftest.py b/tests/integration/test/wasm/python_sdk/conftest.py new file mode 100644 index 00000000..bc89c954 --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/conftest.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Fixtures for Python SDK integration tests. +A single FunctionStreamInstance is shared across the entire module. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from framework import FunctionStreamInstance + +PROJECT_ROOT = Path(__file__).resolve().parents[5] +PYTHON_EXAMPLE_DIR = PROJECT_ROOT / "examples" / "python-processor" + + +@pytest.fixture(scope="session") +def fs_server(): + """Start a FunctionStream server once for all Python SDK tests.""" + instance = FunctionStreamInstance(test_name="wasm_python_sdk") + instance.start() + yield instance + instance.kill() + + +@pytest.fixture(scope="session") +def python_example_dir(): + """Path to the Python processor example directory.""" + return PYTHON_EXAMPLE_DIR diff --git a/tests/integration/test/wasm/python_sdk/test_client_error_handling.py b/tests/integration/test/wasm/python_sdk/test_client_error_handling.py new file mode 100644 index 00000000..f86d09e3 --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/test_client_error_handling.py @@ -0,0 +1,304 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Edge-case and error-handling integration tests for the Python SDK. + +Focus areas: + - Exception hierarchy and attributes + - Network failures (connecting to a dead port) + - gRPC error code translation + - Idempotent cleanup patterns + - Concurrent function operations +""" + +import uuid + +import pytest + +from fs_client import FsClient +from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder +from fs_client.exceptions import ( + ClientError, + ConflictError, + FsError, + FunctionStreamTimeoutError, + NetworkError, + NotFoundError, + ServerError, +) + + +# ====================================================================== +# Helpers +# ====================================================================== + + +def _unique_name(prefix: str = "err") -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +def _processor_bytes() -> bytes: + return b"""\ +from fs_api import FSProcessorDriver, Context + +class ErrProcessor(FSProcessorDriver): + def init(self, ctx: Context, config: dict): + pass + def process(self, ctx: Context, source_id: int, data: bytes): + ctx.emit(data, 0) + def process_watermark(self, ctx: Context, source_id: int, watermark: int): + ctx.emit_watermark(watermark, 0) + def take_checkpoint(self, ctx: Context, checkpoint_id: int): + return None + def check_heartbeat(self, ctx: Context) -> bool: + return True + def close(self, ctx: Context): + pass + def custom(self, payload: bytes) -> bytes: + return b'{}' +""" + + +def _build_config(name: str, class_name: str = "ErrProcessor") -> str: + return ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .add_init_config("class_name", class_name) + .add_input_group( + [KafkaInput("localhost:9092", "err-in", "err-grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "err-out", 0)) + .build() + .to_yaml() + ) + + +def _create(client, name): + return client.create_python_function( + class_name="ErrProcessor", + modules=[("err_processor", _processor_bytes())], + config_content=_build_config(name), + ) + + +def _cleanup(client, name): + try: + client.stop_function(name) + except Exception: + pass + try: + client.drop_function(name) + except Exception: + pass + + +# ====================================================================== +# TestExceptionHierarchy +# ====================================================================== + + +class TestExceptionHierarchy: + """Verify that all custom exceptions follow the documented hierarchy.""" + + def test_server_error_is_fs_error(self): + assert issubclass(ServerError, FsError) + + def test_client_error_is_fs_error(self): + assert issubclass(ClientError, FsError) + + def test_not_found_is_server_error(self): + assert issubclass(NotFoundError, ServerError) + + def test_conflict_is_server_error(self): + assert issubclass(ConflictError, ServerError) + + def test_network_error_is_server_error(self): + assert issubclass(NetworkError, ServerError) + + def test_timeout_error_is_server_error(self): + assert issubclass(FunctionStreamTimeoutError, ServerError) + + def test_server_error_has_status_code_attr(self): + err = ServerError("test", status_code=500) + assert err.status_code == 500 + + def test_server_error_has_grpc_code_attr(self): + err = ServerError("test", grpc_code=None) + assert err.grpc_code is None + + def test_fs_error_has_original_exception_attr(self): + orig = RuntimeError("root cause") + err = FsError("wrapper", original_exception=orig) + assert err.original_exception is orig + + def test_fs_error_message(self): + err = FsError("something went wrong") + assert str(err) == "something went wrong" + + +# ====================================================================== +# TestNetworkErrors +# ====================================================================== + + +class TestNetworkErrors: + """Test behaviour when the server is unreachable.""" + + def test_connect_to_dead_port_raises_on_rpc(self): + client = FsClient(host="127.0.0.1", port=19999, default_timeout=2.0) + with pytest.raises((NetworkError, ServerError, Exception)): + client.show_functions(timeout=2.0) + client.close() + + def test_drop_on_dead_port_raises(self): + client = FsClient(host="127.0.0.1", port=19998, default_timeout=2.0) + with pytest.raises((NetworkError, ServerError, Exception)): + client.drop_function("anything", timeout=2.0) + client.close() + + +# ====================================================================== +# TestIdempotentCleanup +# ====================================================================== + + +class TestIdempotentCleanup: + """Verify idempotent cleanup patterns used in real-world teardown.""" + + def test_drop_already_dropped_raises_not_found(self, fs_server): + name = _unique_name("idempotent") + with fs_server.get_client() as client: + _create(client, name) + client.stop_function(name) + client.drop_function(name) + with pytest.raises(NotFoundError) as exc_info: + client.drop_function(name) + assert exc_info.value.status_code == 404 + + def test_stop_already_stopped_is_benign(self, fs_server): + name = _unique_name("stop-twice") + with fs_server.get_client() as client: + _create(client, name) + client.stop_function(name) + try: + client.stop_function(name) + except ServerError: + pass + _cleanup(client, name) + + def test_cleanup_helper_is_safe_on_nonexistent(self, fs_server): + with fs_server.get_client() as client: + _cleanup(client, "never-existed-" + uuid.uuid4().hex[:8]) + + +# ====================================================================== +# TestRapidCreateDrop +# ====================================================================== + + +class TestRapidCreateDrop: + """Stress the create/drop cycle to catch race conditions.""" + + @pytest.mark.parametrize("iteration", range(5)) + def test_rapid_create_and_drop(self, fs_server, iteration): + name = _unique_name(f"rapid-{iteration}") + with fs_server.get_client() as client: + _create(client, name) + listing = client.show_functions() + assert name in [f.name for f in listing.functions] + client.stop_function(name) + client.drop_function(name) + listing = client.show_functions() + assert name not in [f.name for f in listing.functions] + + +# ====================================================================== +# TestMultipleFunctions +# ====================================================================== + + +class TestMultipleFunctions: + """Test managing multiple independent functions concurrently.""" + + def test_create_multiple_functions(self, fs_server): + names = [_unique_name(f"multi-{i}") for i in range(3)] + with fs_server.get_client() as client: + for n in names: + _create(client, n) + + listing = client.show_functions() + listed = [f.name for f in listing.functions] + for n in names: + assert n in listed + + for n in names: + _cleanup(client, n) + + def test_drop_one_does_not_affect_others(self, fs_server): + n1 = _unique_name("keep") + n2 = _unique_name("drop") + with fs_server.get_client() as client: + _create(client, n1) + _create(client, n2) + + client.stop_function(n2) + client.drop_function(n2) + + listing = client.show_functions() + listed = [f.name for f in listing.functions] + assert n1 in listed + assert n2 not in listed + + _cleanup(client, n1) + + +# ====================================================================== +# TestFunctionStatusTransitions +# ====================================================================== + + +class TestFunctionStatusTransitions: + """Verify that function status changes correctly after operations.""" + + def test_created_function_has_status(self, fs_server): + name = _unique_name("status") + with fs_server.get_client() as client: + _create(client, name) + listing = client.show_functions() + fn = next((f for f in listing.functions if f.name == name), None) + assert fn is not None + assert isinstance(fn.status, str) + assert len(fn.status) > 0 + _cleanup(client, name) + + def test_stopped_function_status_changes(self, fs_server): + name = _unique_name("status-stop") + with fs_server.get_client() as client: + _create(client, name) + client.stop_function(name) + listing = client.show_functions() + fn = next((f for f in listing.functions if f.name == name), None) + assert fn is not None + _cleanup(client, name) + + def test_restarted_function_status_changes(self, fs_server): + name = _unique_name("status-restart") + with fs_server.get_client() as client: + _create(client, name) + client.stop_function(name) + client.start_function(name) + listing = client.show_functions() + fn = next((f for f in listing.functions if f.name == name), None) + assert fn is not None + _cleanup(client, name) diff --git a/tests/integration/test/wasm/python_sdk/test_config_unit.py b/tests/integration/test/wasm/python_sdk/test_config_unit.py new file mode 100644 index 00000000..111cf2df --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/test_config_unit.py @@ -0,0 +1,293 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unit tests for the Python SDK configuration layer: +WasmTaskBuilder, WasmTaskConfig, KafkaInput, KafkaOutput. + +These tests do NOT require a running FunctionStream server. +""" + +import pytest +import yaml + +from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder, WasmTaskConfig + + +# ====================================================================== +# KafkaInput +# ====================================================================== + + +class TestKafkaInput: + + def test_basic_fields(self): + ki = KafkaInput("broker:9092", "topic-a", "group-1") + assert ki.data["input-type"] == "kafka" + assert ki.data["bootstrap_servers"] == "broker:9092" + assert ki.data["topic"] == "topic-a" + assert ki.data["group_id"] == "group-1" + assert "partition" not in ki.data + + def test_with_partition(self): + ki = KafkaInput("broker:9092", "topic-a", "group-1", partition=3) + assert ki.data["partition"] == 3 + + def test_partition_zero_is_present(self): + ki = KafkaInput("broker:9092", "t", "g", partition=0) + assert ki.data["partition"] == 0 + + def test_partition_none_is_absent(self): + ki = KafkaInput("broker:9092", "t", "g", partition=None) + assert "partition" not in ki.data + + +# ====================================================================== +# KafkaOutput +# ====================================================================== + + +class TestKafkaOutput: + + def test_basic_fields(self): + ko = KafkaOutput("broker:9092", "out-topic", 1) + assert ko.data["output-type"] == "kafka" + assert ko.data["bootstrap_servers"] == "broker:9092" + assert ko.data["topic"] == "out-topic" + assert ko.data["partition"] == 1 + + def test_partition_zero(self): + ko = KafkaOutput("broker:9092", "t", 0) + assert ko.data["partition"] == 0 + + +# ====================================================================== +# WasmTaskBuilder +# ====================================================================== + + +class TestWasmTaskBuilder: + + def test_default_values(self): + config = WasmTaskBuilder().build() + assert config.task_name == "default-processor" + assert config.task_type == "python" + assert config.use_builtin_event_serialization is False + assert config.enable_checkpoint is False + assert config.checkpoint_interval_seconds == 1 + assert config.init_config == {} + assert config.input_groups == [] + assert config.outputs == [] + + def test_set_name(self): + config = WasmTaskBuilder().set_name("my-fn").build() + assert config.task_name == "my-fn" + + def test_empty_name_defaults(self): + config = WasmTaskBuilder().set_name("").build() + assert config.task_name == "default-processor" + + def test_blank_name_defaults(self): + config = WasmTaskBuilder().set_name(" ").build() + assert config.task_name == "default-processor" + + def test_set_type(self): + config = WasmTaskBuilder().set_type("processor").build() + assert config.task_type == "processor" + + def test_builtin_serialization(self): + config = WasmTaskBuilder().set_builtin_serialization(True).build() + assert config.use_builtin_event_serialization is True + + def test_configure_checkpoint_enabled(self): + config = ( + WasmTaskBuilder() + .configure_checkpoint(enabled=True, interval=5) + .build() + ) + assert config.enable_checkpoint is True + assert config.checkpoint_interval_seconds == 5 + + def test_checkpoint_interval_clamped_to_one(self): + config = ( + WasmTaskBuilder() + .configure_checkpoint(enabled=True, interval=0) + .build() + ) + assert config.checkpoint_interval_seconds == 1 + + def test_checkpoint_interval_negative_clamped(self): + config = ( + WasmTaskBuilder() + .configure_checkpoint(enabled=True, interval=-10) + .build() + ) + assert config.checkpoint_interval_seconds == 1 + + def test_add_init_config(self): + config = ( + WasmTaskBuilder() + .add_init_config("key1", "val1") + .add_init_config("key2", "val2") + .build() + ) + assert config.init_config == {"key1": "val1", "key2": "val2"} + + def test_add_input_group(self): + ki = KafkaInput("broker:9092", "in-topic", "grp") + config = WasmTaskBuilder().add_input_group([ki]).build() + assert len(config.input_groups) == 1 + assert len(config.input_groups[0]["inputs"]) == 1 + assert config.input_groups[0]["inputs"][0]["topic"] == "in-topic" + + def test_add_multiple_input_groups(self): + ki1 = KafkaInput("b:9092", "t1", "g1") + ki2 = KafkaInput("b:9092", "t2", "g2") + config = ( + WasmTaskBuilder() + .add_input_group([ki1]) + .add_input_group([ki2]) + .build() + ) + assert len(config.input_groups) == 2 + + def test_add_output(self): + ko = KafkaOutput("broker:9092", "out", 0) + config = WasmTaskBuilder().add_output(ko).build() + assert len(config.outputs) == 1 + assert config.outputs[0]["topic"] == "out" + + def test_add_multiple_outputs(self): + config = ( + WasmTaskBuilder() + .add_output(KafkaOutput("b:9092", "o1", 0)) + .add_output(KafkaOutput("b:9092", "o2", 1)) + .build() + ) + assert len(config.outputs) == 2 + + def test_fluent_chaining(self): + config = ( + WasmTaskBuilder() + .set_name("chained") + .set_type("python") + .set_builtin_serialization(False) + .configure_checkpoint(True, 10) + .add_init_config("k", "v") + .add_input_group([KafkaInput("b:9092", "t", "g")]) + .add_output(KafkaOutput("b:9092", "o", 0)) + .build() + ) + assert config.task_name == "chained" + assert config.enable_checkpoint is True + assert config.checkpoint_interval_seconds == 10 + assert len(config.input_groups) == 1 + assert len(config.outputs) == 1 + + def test_builder_produces_independent_configs(self): + builder = WasmTaskBuilder().set_name("first") + c1 = builder.build() + builder.set_name("second") + c2 = builder.build() + assert c1.task_name == "first" + assert c2.task_name == "second" + + +# ====================================================================== +# WasmTaskConfig +# ====================================================================== + + +class TestWasmTaskConfig: + + @pytest.fixture() + def sample_config(self): + return ( + WasmTaskBuilder() + .set_name("test-processor") + .set_type("python") + .configure_checkpoint(True, 5) + .add_init_config("key", "value") + .add_input_group( + [KafkaInput("localhost:9092", "in-topic", "test-group", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "out-topic", 0)) + .build() + ) + + def test_to_dict_keys(self, sample_config): + d = sample_config.to_dict() + assert "name" in d + assert "type" in d + assert "use_builtin_event_serialization" in d + assert "enable_checkpoint" in d + assert "checkpoint_interval_seconds" in d + assert "init_config" in d + assert "input-groups" in d + assert "outputs" in d + + def test_to_dict_values(self, sample_config): + d = sample_config.to_dict() + assert d["name"] == "test-processor" + assert d["type"] == "python" + assert d["enable_checkpoint"] is True + assert d["checkpoint_interval_seconds"] == 5 + + def test_to_yaml_produces_valid_yaml(self, sample_config): + yaml_str = sample_config.to_yaml() + parsed = yaml.safe_load(yaml_str) + assert isinstance(parsed, dict) + assert parsed["name"] == "test-processor" + + def test_from_yaml_roundtrip(self, sample_config): + yaml_str = sample_config.to_yaml() + restored = WasmTaskConfig.from_yaml(yaml_str) + assert restored.task_name == sample_config.task_name + assert restored.task_type == sample_config.task_type + assert restored.enable_checkpoint == sample_config.enable_checkpoint + assert restored.checkpoint_interval_seconds == sample_config.checkpoint_interval_seconds + + def test_from_yaml_minimal(self): + cfg = WasmTaskConfig.from_yaml("name: mini\n") + assert cfg.task_name == "mini" + assert cfg.task_type == "python" + assert cfg.enable_checkpoint is False + + def test_from_yaml_missing_name_gets_default(self): + cfg = WasmTaskConfig.from_yaml("type: python\n") + assert cfg.task_name == "default-processor" + + def test_from_yaml_empty_name_gets_default(self): + cfg = WasmTaskConfig.from_yaml("name: ''\n") + assert cfg.task_name == "default-processor" + + def test_from_yaml_invalid_format_raises(self): + with pytest.raises(ValueError, match="mapping"): + WasmTaskConfig.from_yaml("- just\n- a\n- list\n") + + def test_from_yaml_checkpoint_interval_clamped(self): + cfg = WasmTaskConfig.from_yaml( + "name: x\ncheckpoint_interval_seconds: 0\n" + ) + assert cfg.checkpoint_interval_seconds >= 1 + + def test_from_yaml_with_init_config(self): + cfg = WasmTaskConfig.from_yaml( + "name: x\ninit_config:\n a: '1'\n b: '2'\n" + ) + assert cfg.init_config == {"a": "1", "b": "2"} + + def test_to_yaml_sort_keys_false(self, sample_config): + yaml_str = sample_config.to_yaml() + lines = yaml_str.strip().split("\n") + first_key = lines[0].split(":")[0] + assert first_key == "name" diff --git a/tests/integration/test/wasm/python_sdk/test_python_api.py b/tests/integration/test/wasm/python_sdk/test_python_api.py new file mode 100644 index 00000000..6c5da7d1 --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/test_python_api.py @@ -0,0 +1,694 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Comprehensive integration tests for the Python SDK client API. + +Covers every public method of FsClient against a live FunctionStream server: + - create_python_function + - create_python_function_from_config + - create_function_from_bytes + - show_functions (with / without filter) + - start_function / stop_function / drop_function + - Error semantics: ConflictError, NotFoundError, validation errors + - Context-manager lifecycle + - Custom timeout behaviour + +Requires a running FunctionStream server (provided by the ``fs_server`` fixture). +""" + +import time +import uuid +from pathlib import Path + +import pytest + +from fs_client import FsClient, FunctionInfo, ShowFunctionsResult +from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder +from fs_client.exceptions import ( + BadRequestError, + ClientError, + ConflictError, + FsError, + NotFoundError, + ServerError, +) + + +# ====================================================================== +# Helpers +# ====================================================================== + + +def _unique_name(prefix: str = "pytest") -> str: + """Generate a unique function name to avoid cross-test collisions.""" + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +def _build_config(name: str, class_name: str = "MinimalProcessor") -> str: + """Build a minimal YAML config string for a Python function.""" + return ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .add_init_config("class_name", class_name) + .add_init_config("emit_threshold", "1") + .add_input_group( + [KafkaInput("localhost:9092", "test-in", "test-grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "test-out", 0)) + .build() + .to_yaml() + ) + + +def _processor_bytes() -> bytes: + """Return a minimal FSProcessorDriver implementation as source bytes.""" + return b"""\ +from fs_api import FSProcessorDriver, Context + +class MinimalProcessor(FSProcessorDriver): + def init(self, ctx: Context, config: dict): + pass + def process(self, ctx: Context, source_id: int, data: bytes): + ctx.emit(data, 0) + def process_watermark(self, ctx: Context, source_id: int, watermark: int): + ctx.emit_watermark(watermark, 0) + def take_checkpoint(self, ctx: Context, checkpoint_id: int): + return None + def check_heartbeat(self, ctx: Context) -> bool: + return True + def close(self, ctx: Context): + pass + def custom(self, payload: bytes) -> bytes: + return b'{}' +""" + + +def _create_function(client: FsClient, name: str) -> bool: + """Helper: register a Python function with the given name.""" + return client.create_python_function( + class_name="MinimalProcessor", + modules=[("minimal_processor", _processor_bytes())], + config_content=_build_config(name), + ) + + +def _ensure_dropped(client: FsClient, name: str) -> None: + """Drop a function if it exists, silently ignore NotFoundError.""" + try: + client.stop_function(name) + except Exception: + pass + try: + client.drop_function(name) + except Exception: + pass + + +# ====================================================================== +# TestFsClientContextManager +# ====================================================================== + + +class TestFsClientContextManager: + """Verify FsClient as a context manager opens and closes properly.""" + + def test_context_manager_enter_exit(self, fs_server): + with fs_server.get_client() as client: + assert isinstance(client, FsClient) + + def test_context_manager_closes_channel(self, fs_server): + with fs_server.get_client() as client: + result = client.show_functions() + assert isinstance(result, ShowFunctionsResult) + + def test_manual_close(self, fs_server): + client = fs_server.get_client() + result = client.show_functions() + assert isinstance(result, ShowFunctionsResult) + client.close() + + +# ====================================================================== +# TestCreatePythonFunction +# ====================================================================== + + +class TestCreatePythonFunction: + """Tests for FsClient.create_python_function (direct module bytes).""" + + def test_create_succeeds(self, fs_server): + name = _unique_name("create") + with fs_server.get_client() as client: + result = _create_function(client, name) + assert result is True + _ensure_dropped(client, name) + + def test_create_with_multiple_modules(self, fs_server): + name = _unique_name("multi-mod") + helper_code = b"""\ +HELPER_VERSION = "1.0" +def helper_func(): + return "ok" +""" + proc_code = b"""\ +from fs_api import FSProcessorDriver, Context + +class MultiModProcessor(FSProcessorDriver): + def init(self, ctx: Context, config: dict): + pass + def process(self, ctx: Context, source_id: int, data: bytes): + ctx.emit(data, 0) + def process_watermark(self, ctx: Context, source_id: int, watermark: int): + ctx.emit_watermark(watermark, 0) + def take_checkpoint(self, ctx: Context, checkpoint_id: int): + return None + def check_heartbeat(self, ctx: Context) -> bool: + return True + def close(self, ctx: Context): + pass + def custom(self, payload: bytes) -> bytes: + return b'{}' +""" + with fs_server.get_client() as client: + result = client.create_python_function( + class_name="MultiModProcessor", + modules=[ + ("helper_mod", helper_code), + ("multi_mod_processor", proc_code), + ], + config_content=_build_config(name, class_name="MultiModProcessor"), + ) + assert result is True + _ensure_dropped(client, name) + + def test_create_empty_modules_raises_value_error(self, fs_server): + with fs_server.get_client() as client: + with pytest.raises(ValueError, match="module"): + client.create_python_function( + class_name="X", + modules=[], + config_content="name: x\n", + ) + + def test_create_duplicate_raises_conflict(self, fs_server): + name = _unique_name("dup") + with fs_server.get_client() as client: + _create_function(client, name) + with pytest.raises(ConflictError) as exc_info: + _create_function(client, name) + assert exc_info.value.status_code == 409 + _ensure_dropped(client, name) + + +# ====================================================================== +# TestCreatePythonFunctionFromConfig +# ====================================================================== + + +class TestCreatePythonFunctionFromConfig: + """Tests for FsClient.create_python_function_from_config.""" + + def test_create_from_config_with_example_processor( + self, fs_server, python_example_dir + ): + processor_file = python_example_dir / "processor_impl.py" + if not processor_file.exists(): + pytest.skip("Python processor example not available") + + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "processor_impl", str(processor_file) + ) + mod = importlib.util.module_from_spec(spec) + mod.__file__ = str(processor_file) + sys.modules["processor_impl"] = mod + spec.loader.exec_module(mod) + + name = _unique_name("from-cfg") + config = ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .add_init_config("emit_threshold", "1") + .add_init_config("class_name", "CounterProcessor") + .add_input_group( + [KafkaInput("localhost:9092", "in-t", "grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "out-t", 0)) + .build() + ) + with fs_server.get_client() as client: + result = client.create_python_function_from_config( + config, mod.CounterProcessor + ) + assert result is True + _ensure_dropped(client, name) + + def test_from_config_rejects_non_class(self, fs_server): + config = WasmTaskBuilder().set_name("x").build() + with fs_server.get_client() as client: + with pytest.raises(ValueError, match="class"): + client.create_python_function_from_config(config, "not_a_class") + + def test_from_config_rejects_non_driver(self, fs_server): + config = WasmTaskBuilder().set_name("x").build() + + class NotADriver: + pass + + with fs_server.get_client() as client: + with pytest.raises(TypeError, match="FSProcessorDriver"): + client.create_python_function_from_config(config, NotADriver) + + +# ====================================================================== +# TestCreateFunctionFromBytes +# ====================================================================== + + +class TestCreateFunctionFromBytes: + """Tests for FsClient.create_function_from_bytes (WASM path).""" + + def test_config_wrong_type_raises_type_error(self, fs_server): + with fs_server.get_client() as client: + with pytest.raises(TypeError): + client.create_function_from_bytes( + function_bytes=b"x", + config_content=12345, + ) + + def test_config_accepts_string(self, fs_server): + """String config_content is accepted (may fail on server for bad WASM).""" + with fs_server.get_client() as client: + with pytest.raises(ServerError): + client.create_function_from_bytes( + function_bytes=b"\x00invalid-wasm", + config_content="name: bad-wasm\ntype: processor\n", + ) + + def test_config_accepts_bytes(self, fs_server): + """Bytes config_content is accepted (may fail on server for bad WASM).""" + with fs_server.get_client() as client: + with pytest.raises(ServerError): + client.create_function_from_bytes( + function_bytes=b"\x00invalid-wasm", + config_content=b"name: bad-wasm-b\ntype: processor\n", + ) + + +# ====================================================================== +# TestCreateFunctionFromFiles +# ====================================================================== + + +class TestCreateFunctionFromFiles: + """Tests for FsClient.create_function_from_files.""" + + def test_missing_wasm_file_raises_file_not_found(self, fs_server): + with fs_server.get_client() as client: + with pytest.raises(FileNotFoundError): + client.create_function_from_files( + function_path="/nonexistent/path.wasm", + config_path="/nonexistent/config.yaml", + ) + + def test_missing_config_file_raises_file_not_found(self, fs_server, tmp_path): + wasm_file = tmp_path / "dummy.wasm" + wasm_file.write_bytes(b"\x00\x61\x73\x6d") + with fs_server.get_client() as client: + with pytest.raises(FileNotFoundError): + client.create_function_from_files( + function_path=str(wasm_file), + config_path="/nonexistent/config.yaml", + ) + + +# ====================================================================== +# TestShowFunctions +# ====================================================================== + + +class TestShowFunctions: + """Tests for FsClient.show_functions.""" + + def test_show_functions_returns_result_type(self, fs_server): + with fs_server.get_client() as client: + result = client.show_functions() + assert isinstance(result, ShowFunctionsResult) + assert isinstance(result.status_code, int) + assert result.status_code < 400 + assert isinstance(result.functions, list) + + def test_show_functions_contains_created_function(self, fs_server): + name = _unique_name("show") + with fs_server.get_client() as client: + _create_function(client, name) + result = client.show_functions() + names = [f.name for f in result.functions] + assert name in names + _ensure_dropped(client, name) + + def test_show_functions_function_info_fields(self, fs_server): + name = _unique_name("info") + with fs_server.get_client() as client: + _create_function(client, name) + result = client.show_functions() + fn = next((f for f in result.functions if f.name == name), None) + assert fn is not None + assert isinstance(fn, FunctionInfo) + assert fn.name == name + assert isinstance(fn.task_type, str) + assert isinstance(fn.status, str) + _ensure_dropped(client, name) + + def test_show_functions_empty_on_fresh_server(self, fs_server): + with fs_server.get_client() as client: + result = client.show_functions() + assert isinstance(result.functions, list) + + def test_show_functions_with_filter(self, fs_server): + prefix = f"filter-{uuid.uuid4().hex[:6]}" + name1 = f"{prefix}-aaa" + name2 = f"{prefix}-bbb" + other_name = _unique_name("other") + with fs_server.get_client() as client: + _create_function(client, name1) + _create_function(client, name2) + _create_function(client, other_name) + + result = client.show_functions(filter_pattern=prefix) + found = [f.name for f in result.functions] + assert name1 in found or name2 in found + + _ensure_dropped(client, name1) + _ensure_dropped(client, name2) + _ensure_dropped(client, other_name) + + def test_show_functions_with_custom_timeout(self, fs_server): + with fs_server.get_client() as client: + result = client.show_functions(timeout=10.0) + assert isinstance(result, ShowFunctionsResult) + + +# ====================================================================== +# TestFunctionLifecycle +# ====================================================================== + + +class TestFunctionLifecycle: + """Full function lifecycle: create -> show -> stop -> start -> drop.""" + + def test_full_lifecycle(self, fs_server): + name = _unique_name("lifecycle") + with fs_server.get_client() as client: + assert _create_function(client, name) is True + + listing = client.show_functions() + assert name in [f.name for f in listing.functions] + + assert client.stop_function(name) is True + + assert client.start_function(name) is True + + client.stop_function(name) + assert client.drop_function(name) is True + + listing = client.show_functions() + assert name not in [f.name for f in listing.functions] + + def test_stop_and_restart_preserves_function(self, fs_server): + name = _unique_name("restart") + with fs_server.get_client() as client: + _create_function(client, name) + client.stop_function(name) + time.sleep(0.5) + client.start_function(name) + + listing = client.show_functions() + assert name in [f.name for f in listing.functions] + _ensure_dropped(client, name) + + def test_drop_running_function_requires_stop(self, fs_server): + name = _unique_name("drop-running") + with fs_server.get_client() as client: + _create_function(client, name) + try: + client.drop_function(name) + except ServerError: + client.stop_function(name) + result = client.drop_function(name) + assert result is True + else: + listing = client.show_functions() + assert name not in [f.name for f in listing.functions] + + +# ====================================================================== +# TestFunctionErrorHandling +# ====================================================================== + + +class TestFunctionErrorHandling: + """Error semantics for function operations.""" + + def test_drop_nonexistent_raises_not_found(self, fs_server): + with fs_server.get_client() as client: + with pytest.raises(NotFoundError) as exc_info: + client.drop_function("nonexistent-" + uuid.uuid4().hex[:8]) + assert exc_info.value.status_code == 404 + + def test_start_nonexistent_raises_not_found(self, fs_server): + with fs_server.get_client() as client: + with pytest.raises(NotFoundError) as exc_info: + client.start_function("nonexistent-" + uuid.uuid4().hex[:8]) + assert exc_info.value.status_code == 404 + + def test_stop_nonexistent_raises_not_found(self, fs_server): + with fs_server.get_client() as client: + with pytest.raises(NotFoundError) as exc_info: + client.stop_function("nonexistent-" + uuid.uuid4().hex[:8]) + assert exc_info.value.status_code == 404 + + def test_duplicate_create_raises_conflict(self, fs_server): + name = _unique_name("dup-err") + with fs_server.get_client() as client: + _create_function(client, name) + with pytest.raises(ConflictError) as exc_info: + _create_function(client, name) + assert exc_info.value.status_code == 409 + _ensure_dropped(client, name) + + def test_all_errors_inherit_from_fs_error(self, fs_server): + with fs_server.get_client() as client: + with pytest.raises(FsError): + client.drop_function("definitely-does-not-exist-" + uuid.uuid4().hex) + + +# ====================================================================== +# TestFunctionWithCheckpoint +# ====================================================================== + + +class TestFunctionWithCheckpoint: + """Test creating Python functions with checkpoint configuration.""" + + def test_checkpoint_enabled(self, fs_server): + name = _unique_name("ckpt") + config_yaml = ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .configure_checkpoint(enabled=True, interval=5) + .add_init_config("class_name", "MinimalProcessor") + .add_init_config("key", "value") + .add_input_group( + [KafkaInput("localhost:9092", "ckpt-in", "ckpt-grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "ckpt-out", 0)) + .build() + .to_yaml() + ) + with fs_server.get_client() as client: + result = client.create_python_function( + class_name="MinimalProcessor", + modules=[("minimal_processor", _processor_bytes())], + config_content=config_yaml, + ) + assert result is True + _ensure_dropped(client, name) + + def test_checkpoint_disabled(self, fs_server): + name = _unique_name("no-ckpt") + config_yaml = ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .configure_checkpoint(enabled=False) + .add_init_config("class_name", "MinimalProcessor") + .add_input_group( + [KafkaInput("localhost:9092", "in", "grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "out", 0)) + .build() + .to_yaml() + ) + with fs_server.get_client() as client: + result = client.create_python_function( + class_name="MinimalProcessor", + modules=[("minimal_processor", _processor_bytes())], + config_content=config_yaml, + ) + assert result is True + _ensure_dropped(client, name) + + +# ====================================================================== +# TestFunctionWithMultipleIO +# ====================================================================== + + +class TestFunctionWithMultipleIO: + """Test creating functions with multiple input groups and outputs.""" + + def test_multiple_input_groups(self, fs_server): + name = _unique_name("multi-in") + config_yaml = ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .add_init_config("class_name", "MinimalProcessor") + .add_input_group( + [KafkaInput("localhost:9092", "in1", "grp1", partition=0)] + ) + .add_input_group( + [KafkaInput("localhost:9092", "in2", "grp2", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "out", 0)) + .build() + .to_yaml() + ) + with fs_server.get_client() as client: + result = client.create_python_function( + class_name="MinimalProcessor", + modules=[("minimal_processor", _processor_bytes())], + config_content=config_yaml, + ) + assert result is True + _ensure_dropped(client, name) + + def test_multiple_outputs(self, fs_server): + name = _unique_name("multi-out") + config_yaml = ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .add_init_config("class_name", "MinimalProcessor") + .add_input_group( + [KafkaInput("localhost:9092", "in", "grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "out1", 0)) + .add_output(KafkaOutput("localhost:9092", "out2", 1)) + .build() + .to_yaml() + ) + with fs_server.get_client() as client: + result = client.create_python_function( + class_name="MinimalProcessor", + modules=[("minimal_processor", _processor_bytes())], + config_content=config_yaml, + ) + assert result is True + _ensure_dropped(client, name) + + +# ====================================================================== +# TestBuiltinSerialization +# ====================================================================== + + +class TestBuiltinSerialization: + """Test creating functions with builtin serialization toggled.""" + + def test_builtin_serialization_enabled(self, fs_server): + name = _unique_name("builtin-on") + config_yaml = ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .set_builtin_serialization(True) + .add_init_config("class_name", "MinimalProcessor") + .add_input_group( + [KafkaInput("localhost:9092", "in", "grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "out", 0)) + .build() + .to_yaml() + ) + with fs_server.get_client() as client: + result = client.create_python_function( + class_name="MinimalProcessor", + modules=[("minimal_processor", _processor_bytes())], + config_content=config_yaml, + ) + assert result is True + _ensure_dropped(client, name) + + def test_builtin_serialization_disabled(self, fs_server): + name = _unique_name("builtin-off") + config_yaml = ( + WasmTaskBuilder() + .set_name(name) + .set_type("python") + .set_builtin_serialization(False) + .add_init_config("class_name", "MinimalProcessor") + .add_input_group( + [KafkaInput("localhost:9092", "in", "grp", partition=0)] + ) + .add_output(KafkaOutput("localhost:9092", "out", 0)) + .build() + .to_yaml() + ) + with fs_server.get_client() as client: + result = client.create_python_function( + class_name="MinimalProcessor", + modules=[("minimal_processor", _processor_bytes())], + config_content=config_yaml, + ) + assert result is True + _ensure_dropped(client, name) + + +# ====================================================================== +# TestClientCustomTimeout +# ====================================================================== + + +class TestClientCustomTimeout: + """Verify custom timeout propagation.""" + + def test_default_timeout_attribute(self, fs_server): + client = FsClient( + host=fs_server.host, + port=fs_server.port, + default_timeout=42.0, + ) + assert client.default_timeout == 42.0 + client.close() + + def test_per_call_timeout(self, fs_server): + with fs_server.get_client() as client: + result = client.show_functions(timeout=5.0) + assert isinstance(result, ShowFunctionsResult) From 603a27ac34786ab31b9ee3872411da32641661b1 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Apr 2026 22:51:40 +0800 Subject: [PATCH 09/15] update --- tests/integration/framework/__init__.py | 3 +- tests/integration/framework/kafka_manager.py | 221 ++++++ tests/integration/requirements.txt | 4 + .../test/wasm/python_sdk/conftest.py | 84 ++- .../wasm/python_sdk/processors/__init__.py | 17 + .../processors/counter_processor.py | 73 ++ .../python_sdk/test_client_error_handling.py | 304 -------- .../test/wasm/python_sdk/test_config_unit.py | 293 -------- .../test/wasm/python_sdk/test_lifecycle.py | 219 ++++++ .../test/wasm/python_sdk/test_python_api.py | 694 ------------------ 10 files changed, 609 insertions(+), 1303 deletions(-) create mode 100644 tests/integration/framework/kafka_manager.py create mode 100644 tests/integration/test/wasm/python_sdk/processors/__init__.py create mode 100644 tests/integration/test/wasm/python_sdk/processors/counter_processor.py delete mode 100644 tests/integration/test/wasm/python_sdk/test_client_error_handling.py delete mode 100644 tests/integration/test/wasm/python_sdk/test_config_unit.py create mode 100644 tests/integration/test/wasm/python_sdk/test_lifecycle.py delete mode 100644 tests/integration/test/wasm/python_sdk/test_python_api.py diff --git a/tests/integration/framework/__init__.py b/tests/integration/framework/__init__.py index 5cba3938..241d269e 100644 --- a/tests/integration/framework/__init__.py +++ b/tests/integration/framework/__init__.py @@ -11,5 +11,6 @@ # limitations under the License. from .instance import FunctionStreamInstance +from .kafka_manager import KafkaDockerManager -__all__ = ["FunctionStreamInstance"] +__all__ = ["FunctionStreamInstance", "KafkaDockerManager"] diff --git a/tests/integration/framework/kafka_manager.py b/tests/integration/framework/kafka_manager.py new file mode 100644 index 00000000..071d90d2 --- /dev/null +++ b/tests/integration/framework/kafka_manager.py @@ -0,0 +1,221 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Docker-managed Kafka broker for integration tests. + +Provides automated image pull, idempotent container start, health check, +topic lifecycle management, and data cleanup via KRaft-mode single-node Kafka. + +Usage:: + + mgr = KafkaDockerManager() + mgr.setup_kafka() + mgr.create_topics_if_not_exist(["input-topic", "output-topic"]) + ... + mgr.clear_all_topics() + mgr.teardown_kafka() +""" + +import logging +import time +from typing import List + +import docker +from docker.errors import APIError, NotFound +from confluent_kafka.admin import AdminClient, NewTopic + +logger = logging.getLogger(__name__) + +_DEFAULT_IMAGE = "apache/kafka:3.7.0" +_DEFAULT_CONTAINER = "fs-integration-kafka-broker" +_DEFAULT_BOOTSTRAP = "127.0.0.1:9092" + + +class KafkaDockerManager: + """ + Manages a single-node Kafka broker inside a Docker container (KRaft mode). + + The class is intentionally stateless with respect to topics: every public + method is idempotent so that tests can call ``setup_kafka()`` multiple + times without side-effects. + """ + + def __init__( + self, + image: str = _DEFAULT_IMAGE, + container_name: str = _DEFAULT_CONTAINER, + bootstrap_servers: str = _DEFAULT_BOOTSTRAP, + ) -> None: + self.docker_client = docker.from_env() + self.image_name = image + self.container_name = container_name + self.bootstrap_servers = bootstrap_servers + + # ------------------------------------------------------------------ + # Full setup / teardown + # ------------------------------------------------------------------ + + def setup_kafka(self) -> None: + """Pull image -> start container -> wait for readiness.""" + self._ensure_image() + self._ensure_container() + self._wait_for_readiness() + + def teardown_kafka(self) -> None: + """Stop and remove the Kafka container.""" + try: + container = self.docker_client.containers.get(self.container_name) + logger.info("Stopping Kafka container '%s' ...", self.container_name) + container.stop() + except NotFound: + pass + except APIError as exc: + logger.warning("Error while stopping Kafka: %s", exc) + + # ------------------------------------------------------------------ + # Image management + # ------------------------------------------------------------------ + + def _ensure_image(self) -> None: + try: + self.docker_client.images.get(self.image_name) + logger.info("Image '%s' already present locally.", self.image_name) + except NotFound: + logger.info("Pulling Kafka image '%s' ...", self.image_name) + self.docker_client.images.pull(self.image_name) + logger.info("Image pulled successfully.") + + # ------------------------------------------------------------------ + # Container management (KRaft single-node, apache/kafka official image) + # ------------------------------------------------------------------ + + def _ensure_container(self) -> None: + try: + container = self.docker_client.containers.get(self.container_name) + if container.status != "running": + logger.info( + "Container '%s' exists but is not running; starting ...", + self.container_name, + ) + container.start() + else: + logger.info( + "Container '%s' is already running.", self.container_name + ) + except NotFound: + logger.info( + "Creating Kafka container '%s' ...", self.container_name + ) + env = { + "KAFKA_NODE_ID": "1", + "KAFKA_PROCESS_ROLES": "broker,controller", + "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", + "KAFKA_LISTENERS": ( + "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093" + ), + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": ( + "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + ), + "KAFKA_ADVERTISED_LISTENERS": ( + f"PLAINTEXT://{self.bootstrap_servers}" + ), + "KAFKA_CONTROLLER_QUORUM_VOTERS": "1@localhost:9093", + "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", + "KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR": "1", + "KAFKA_TRANSACTION_STATE_LOG_MIN_ISR": "1", + "CLUSTER_ID": "fs-integration-test-cluster-01", + } + self.docker_client.containers.run( + image=self.image_name, + name=self.container_name, + ports={"9092/tcp": 9092}, + environment=env, + detach=True, + remove=True, + ) + + def _wait_for_readiness(self, timeout: int = 60) -> None: + logger.info( + "Waiting for Kafka to become ready at %s ...", + self.bootstrap_servers, + ) + deadline = time.time() + timeout + while time.time() < deadline: + try: + admin = AdminClient( + {"bootstrap.servers": self.bootstrap_servers} + ) + admin.list_topics(timeout=2) + logger.info("Kafka is ready.") + return + except Exception: + time.sleep(1) + raise TimeoutError( + f"Kafka did not become ready within {timeout}s. " + "Check Docker logs for details." + ) + + # ------------------------------------------------------------------ + # Topic management + # ------------------------------------------------------------------ + + def create_topic( + self, + topic_name: str, + num_partitions: int = 1, + replication_factor: int = 1, + ) -> None: + """ + Create a Kafka topic idempotently. + + If the topic already exists the call succeeds silently. + """ + admin = AdminClient({"bootstrap.servers": self.bootstrap_servers}) + new_topic = NewTopic( + topic_name, + num_partitions=num_partitions, + replication_factor=replication_factor, + ) + futures = admin.create_topics([new_topic], operation_timeout=5) + for topic, future in futures.items(): + try: + future.result() + logger.info("Created topic '%s'.", topic) + except Exception as exc: + if "TOPIC_ALREADY_EXISTS" in str(exc): + logger.debug("Topic '%s' already exists; skipping.", topic) + else: + raise + + def create_topics_if_not_exist( + self, topic_names: List[str], num_partitions: int = 1 + ) -> None: + """Batch-create topics idempotently.""" + for topic in topic_names: + self.create_topic(topic, num_partitions=num_partitions) + + def clear_all_topics(self) -> None: + """Delete every non-internal topic (fast data reset between tests).""" + admin = AdminClient({"bootstrap.servers": self.bootstrap_servers}) + try: + metadata = admin.list_topics(timeout=5) + to_delete = [ + t for t in metadata.topics if not t.startswith("__") + ] + if to_delete: + logger.debug("Deleting leftover topics: %s", to_delete) + futures = admin.delete_topics(to_delete, operation_timeout=5) + for _topic, fut in futures.items(): + fut.result() + except Exception as exc: + logger.warning("Topic cleanup failed: %s", exc) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 45a16ecc..d1e597c3 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -16,6 +16,10 @@ pyyaml>=6.0 grpcio>=1.60.0 protobuf>=4.25.0 +# Docker + Kafka management for integration tests +docker>=7.0 +confluent-kafka>=2.3.0 + # FunctionStream Python packages (local editable installs) -e ../../python/functionstream-api -e ../../python/functionstream-client diff --git a/tests/integration/test/wasm/python_sdk/conftest.py b/tests/integration/test/wasm/python_sdk/conftest.py index bc89c954..026a0ed8 100644 --- a/tests/integration/test/wasm/python_sdk/conftest.py +++ b/tests/integration/test/wasm/python_sdk/conftest.py @@ -11,33 +11,95 @@ # limitations under the License. """ -Fixtures for Python SDK integration tests. -A single FunctionStreamInstance is shared across the entire module. +Global pytest fixtures for the FunctionStream Python SDK integration tests. +Provides managed Kafka broker, server instances, client connections, +and automated resource cleanup. """ import sys from pathlib import Path +from typing import Generator, List import pytest +# tests/integration/test/wasm/python_sdk -> parents[3] = tests/integration sys.path.insert(0, str(Path(__file__).resolve().parents[3])) +# Make the ``processors`` package importable from this directory +sys.path.insert(0, str(Path(__file__).resolve().parent)) -from framework import FunctionStreamInstance +from framework import FunctionStreamInstance, KafkaDockerManager # noqa: E402 +from fs_client.client import FsClient # noqa: E402 PROJECT_ROOT = Path(__file__).resolve().parents[5] -PYTHON_EXAMPLE_DIR = PROJECT_ROOT / "examples" / "python-processor" +# ====================================================================== +# Kafka broker (opt-in, independent of fs_server) +# ====================================================================== + +@pytest.fixture(scope="session") +def kafka() -> Generator[KafkaDockerManager, None, None]: + """ + Session-scoped fixture: start a Docker-managed Kafka broker once + for the entire test session. + + Tests that need a live Kafka broker should declare this fixture + as a parameter. Tests that only register functions (without + producing / consuming data) do NOT need this fixture. + """ + mgr = KafkaDockerManager() + mgr.setup_kafka() + yield mgr + mgr.clear_all_topics() + mgr.teardown_kafka() + + +# ====================================================================== +# FunctionStream server +# ====================================================================== + @pytest.fixture(scope="session") -def fs_server(): - """Start a FunctionStream server once for all Python SDK tests.""" - instance = FunctionStreamInstance(test_name="wasm_python_sdk") +def fs_server() -> Generator[FunctionStreamInstance, None, None]: + """ + Session-scoped fixture: start the FunctionStream server once for all tests. + """ + instance = FunctionStreamInstance(test_name="wasm_python_sdk_integration") instance.start() yield instance instance.kill() -@pytest.fixture(scope="session") -def python_example_dir(): - """Path to the Python processor example directory.""" - return PYTHON_EXAMPLE_DIR +# ====================================================================== +# Client & resource tracking +# ====================================================================== + +@pytest.fixture +def fs_client(fs_server: FunctionStreamInstance) -> Generator[FsClient, None, None]: + """ + Function-scoped fixture: provide a fresh client connected to the server. + The connection is automatically closed after each test. + """ + with fs_server.get_client() as client: + yield client + + +@pytest.fixture +def function_registry(fs_client: FsClient) -> Generator[List[str], None, None]: + """ + RAII Resource Manager: tracks registered function names. + Automatically stops and drops every tracked function after each test, + guaranteeing environment idempotency regardless of assertion failures. + """ + registered_names: List[str] = [] + + yield registered_names + + for name in registered_names: + try: + fs_client.stop_function(name) + except Exception: + pass + try: + fs_client.drop_function(name) + except Exception: + pass diff --git a/tests/integration/test/wasm/python_sdk/processors/__init__.py b/tests/integration/test/wasm/python_sdk/processors/__init__.py new file mode 100644 index 00000000..8d66f800 --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/processors/__init__.py @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test processors package. +Each module contains a specific implementation of FSProcessorDriver +to test different engine capabilities. +""" diff --git a/tests/integration/test/wasm/python_sdk/processors/counter_processor.py b/tests/integration/test/wasm/python_sdk/processors/counter_processor.py new file mode 100644 index 00000000..cd0a5f06 --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/processors/counter_processor.py @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A stateful processor that uses KVStore to count occurrences of incoming strings. +Validates state persistence and checkpointing mechanisms. +""" + +import json +from typing import Dict + +from fs_api import FSProcessorDriver, Context + + +class CounterProcessor(FSProcessorDriver): + """ + Stateful word-counter backed by KVStore. + Each incoming UTF-8 string increments its counter in the store and emits + a JSON payload ``{"word": ..., "count": ..., "total": ...}`` downstream. + """ + + def __init__(self) -> None: + self._counter_map: Dict[str, int] = {} + self._total_processed: int = 0 + self._store_name: str = "integration-counter-store" + + def init(self, ctx: Context, config: dict) -> None: + self._counter_map = {} + self._total_processed = 0 + + def process(self, ctx: Context, source_id: int, data: bytes) -> None: + input_str = data.decode("utf-8", errors="replace").strip() + if not input_str: + return + + self._total_processed += 1 + store = ctx.getOrCreateKVStore(self._store_name) + + store_key_bytes = input_str.encode("utf-8") + stored_val = store.get_state(store_key_bytes) + current_count = int(stored_val.decode("utf-8")) if stored_val else 0 + new_count = current_count + 1 + + self._counter_map[input_str] = new_count + store.put_state(store_key_bytes, str(new_count).encode("utf-8")) + + payload = {"word": input_str, "count": new_count, "total": self._total_processed} + ctx.emit(json.dumps(payload).encode("utf-8"), 0) + + def process_watermark(self, ctx: Context, source_id: int, watermark: int) -> None: + ctx.emit_watermark(watermark, 0) + + def take_checkpoint(self, ctx: Context, checkpoint_id: int) -> bytes: + return json.dumps(self._counter_map).encode("utf-8") + + def check_heartbeat(self, ctx: Context) -> bool: + return True + + def close(self, ctx: Context) -> None: + self._counter_map.clear() + self._total_processed = 0 + + def custom(self, payload: bytes) -> bytes: + return b'{"status": "ok"}' diff --git a/tests/integration/test/wasm/python_sdk/test_client_error_handling.py b/tests/integration/test/wasm/python_sdk/test_client_error_handling.py deleted file mode 100644 index f86d09e3..00000000 --- a/tests/integration/test/wasm/python_sdk/test_client_error_handling.py +++ /dev/null @@ -1,304 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Edge-case and error-handling integration tests for the Python SDK. - -Focus areas: - - Exception hierarchy and attributes - - Network failures (connecting to a dead port) - - gRPC error code translation - - Idempotent cleanup patterns - - Concurrent function operations -""" - -import uuid - -import pytest - -from fs_client import FsClient -from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder -from fs_client.exceptions import ( - ClientError, - ConflictError, - FsError, - FunctionStreamTimeoutError, - NetworkError, - NotFoundError, - ServerError, -) - - -# ====================================================================== -# Helpers -# ====================================================================== - - -def _unique_name(prefix: str = "err") -> str: - return f"{prefix}-{uuid.uuid4().hex[:8]}" - - -def _processor_bytes() -> bytes: - return b"""\ -from fs_api import FSProcessorDriver, Context - -class ErrProcessor(FSProcessorDriver): - def init(self, ctx: Context, config: dict): - pass - def process(self, ctx: Context, source_id: int, data: bytes): - ctx.emit(data, 0) - def process_watermark(self, ctx: Context, source_id: int, watermark: int): - ctx.emit_watermark(watermark, 0) - def take_checkpoint(self, ctx: Context, checkpoint_id: int): - return None - def check_heartbeat(self, ctx: Context) -> bool: - return True - def close(self, ctx: Context): - pass - def custom(self, payload: bytes) -> bytes: - return b'{}' -""" - - -def _build_config(name: str, class_name: str = "ErrProcessor") -> str: - return ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .add_init_config("class_name", class_name) - .add_input_group( - [KafkaInput("localhost:9092", "err-in", "err-grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "err-out", 0)) - .build() - .to_yaml() - ) - - -def _create(client, name): - return client.create_python_function( - class_name="ErrProcessor", - modules=[("err_processor", _processor_bytes())], - config_content=_build_config(name), - ) - - -def _cleanup(client, name): - try: - client.stop_function(name) - except Exception: - pass - try: - client.drop_function(name) - except Exception: - pass - - -# ====================================================================== -# TestExceptionHierarchy -# ====================================================================== - - -class TestExceptionHierarchy: - """Verify that all custom exceptions follow the documented hierarchy.""" - - def test_server_error_is_fs_error(self): - assert issubclass(ServerError, FsError) - - def test_client_error_is_fs_error(self): - assert issubclass(ClientError, FsError) - - def test_not_found_is_server_error(self): - assert issubclass(NotFoundError, ServerError) - - def test_conflict_is_server_error(self): - assert issubclass(ConflictError, ServerError) - - def test_network_error_is_server_error(self): - assert issubclass(NetworkError, ServerError) - - def test_timeout_error_is_server_error(self): - assert issubclass(FunctionStreamTimeoutError, ServerError) - - def test_server_error_has_status_code_attr(self): - err = ServerError("test", status_code=500) - assert err.status_code == 500 - - def test_server_error_has_grpc_code_attr(self): - err = ServerError("test", grpc_code=None) - assert err.grpc_code is None - - def test_fs_error_has_original_exception_attr(self): - orig = RuntimeError("root cause") - err = FsError("wrapper", original_exception=orig) - assert err.original_exception is orig - - def test_fs_error_message(self): - err = FsError("something went wrong") - assert str(err) == "something went wrong" - - -# ====================================================================== -# TestNetworkErrors -# ====================================================================== - - -class TestNetworkErrors: - """Test behaviour when the server is unreachable.""" - - def test_connect_to_dead_port_raises_on_rpc(self): - client = FsClient(host="127.0.0.1", port=19999, default_timeout=2.0) - with pytest.raises((NetworkError, ServerError, Exception)): - client.show_functions(timeout=2.0) - client.close() - - def test_drop_on_dead_port_raises(self): - client = FsClient(host="127.0.0.1", port=19998, default_timeout=2.0) - with pytest.raises((NetworkError, ServerError, Exception)): - client.drop_function("anything", timeout=2.0) - client.close() - - -# ====================================================================== -# TestIdempotentCleanup -# ====================================================================== - - -class TestIdempotentCleanup: - """Verify idempotent cleanup patterns used in real-world teardown.""" - - def test_drop_already_dropped_raises_not_found(self, fs_server): - name = _unique_name("idempotent") - with fs_server.get_client() as client: - _create(client, name) - client.stop_function(name) - client.drop_function(name) - with pytest.raises(NotFoundError) as exc_info: - client.drop_function(name) - assert exc_info.value.status_code == 404 - - def test_stop_already_stopped_is_benign(self, fs_server): - name = _unique_name("stop-twice") - with fs_server.get_client() as client: - _create(client, name) - client.stop_function(name) - try: - client.stop_function(name) - except ServerError: - pass - _cleanup(client, name) - - def test_cleanup_helper_is_safe_on_nonexistent(self, fs_server): - with fs_server.get_client() as client: - _cleanup(client, "never-existed-" + uuid.uuid4().hex[:8]) - - -# ====================================================================== -# TestRapidCreateDrop -# ====================================================================== - - -class TestRapidCreateDrop: - """Stress the create/drop cycle to catch race conditions.""" - - @pytest.mark.parametrize("iteration", range(5)) - def test_rapid_create_and_drop(self, fs_server, iteration): - name = _unique_name(f"rapid-{iteration}") - with fs_server.get_client() as client: - _create(client, name) - listing = client.show_functions() - assert name in [f.name for f in listing.functions] - client.stop_function(name) - client.drop_function(name) - listing = client.show_functions() - assert name not in [f.name for f in listing.functions] - - -# ====================================================================== -# TestMultipleFunctions -# ====================================================================== - - -class TestMultipleFunctions: - """Test managing multiple independent functions concurrently.""" - - def test_create_multiple_functions(self, fs_server): - names = [_unique_name(f"multi-{i}") for i in range(3)] - with fs_server.get_client() as client: - for n in names: - _create(client, n) - - listing = client.show_functions() - listed = [f.name for f in listing.functions] - for n in names: - assert n in listed - - for n in names: - _cleanup(client, n) - - def test_drop_one_does_not_affect_others(self, fs_server): - n1 = _unique_name("keep") - n2 = _unique_name("drop") - with fs_server.get_client() as client: - _create(client, n1) - _create(client, n2) - - client.stop_function(n2) - client.drop_function(n2) - - listing = client.show_functions() - listed = [f.name for f in listing.functions] - assert n1 in listed - assert n2 not in listed - - _cleanup(client, n1) - - -# ====================================================================== -# TestFunctionStatusTransitions -# ====================================================================== - - -class TestFunctionStatusTransitions: - """Verify that function status changes correctly after operations.""" - - def test_created_function_has_status(self, fs_server): - name = _unique_name("status") - with fs_server.get_client() as client: - _create(client, name) - listing = client.show_functions() - fn = next((f for f in listing.functions if f.name == name), None) - assert fn is not None - assert isinstance(fn.status, str) - assert len(fn.status) > 0 - _cleanup(client, name) - - def test_stopped_function_status_changes(self, fs_server): - name = _unique_name("status-stop") - with fs_server.get_client() as client: - _create(client, name) - client.stop_function(name) - listing = client.show_functions() - fn = next((f for f in listing.functions if f.name == name), None) - assert fn is not None - _cleanup(client, name) - - def test_restarted_function_status_changes(self, fs_server): - name = _unique_name("status-restart") - with fs_server.get_client() as client: - _create(client, name) - client.stop_function(name) - client.start_function(name) - listing = client.show_functions() - fn = next((f for f in listing.functions if f.name == name), None) - assert fn is not None - _cleanup(client, name) diff --git a/tests/integration/test/wasm/python_sdk/test_config_unit.py b/tests/integration/test/wasm/python_sdk/test_config_unit.py deleted file mode 100644 index 111cf2df..00000000 --- a/tests/integration/test/wasm/python_sdk/test_config_unit.py +++ /dev/null @@ -1,293 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Unit tests for the Python SDK configuration layer: -WasmTaskBuilder, WasmTaskConfig, KafkaInput, KafkaOutput. - -These tests do NOT require a running FunctionStream server. -""" - -import pytest -import yaml - -from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder, WasmTaskConfig - - -# ====================================================================== -# KafkaInput -# ====================================================================== - - -class TestKafkaInput: - - def test_basic_fields(self): - ki = KafkaInput("broker:9092", "topic-a", "group-1") - assert ki.data["input-type"] == "kafka" - assert ki.data["bootstrap_servers"] == "broker:9092" - assert ki.data["topic"] == "topic-a" - assert ki.data["group_id"] == "group-1" - assert "partition" not in ki.data - - def test_with_partition(self): - ki = KafkaInput("broker:9092", "topic-a", "group-1", partition=3) - assert ki.data["partition"] == 3 - - def test_partition_zero_is_present(self): - ki = KafkaInput("broker:9092", "t", "g", partition=0) - assert ki.data["partition"] == 0 - - def test_partition_none_is_absent(self): - ki = KafkaInput("broker:9092", "t", "g", partition=None) - assert "partition" not in ki.data - - -# ====================================================================== -# KafkaOutput -# ====================================================================== - - -class TestKafkaOutput: - - def test_basic_fields(self): - ko = KafkaOutput("broker:9092", "out-topic", 1) - assert ko.data["output-type"] == "kafka" - assert ko.data["bootstrap_servers"] == "broker:9092" - assert ko.data["topic"] == "out-topic" - assert ko.data["partition"] == 1 - - def test_partition_zero(self): - ko = KafkaOutput("broker:9092", "t", 0) - assert ko.data["partition"] == 0 - - -# ====================================================================== -# WasmTaskBuilder -# ====================================================================== - - -class TestWasmTaskBuilder: - - def test_default_values(self): - config = WasmTaskBuilder().build() - assert config.task_name == "default-processor" - assert config.task_type == "python" - assert config.use_builtin_event_serialization is False - assert config.enable_checkpoint is False - assert config.checkpoint_interval_seconds == 1 - assert config.init_config == {} - assert config.input_groups == [] - assert config.outputs == [] - - def test_set_name(self): - config = WasmTaskBuilder().set_name("my-fn").build() - assert config.task_name == "my-fn" - - def test_empty_name_defaults(self): - config = WasmTaskBuilder().set_name("").build() - assert config.task_name == "default-processor" - - def test_blank_name_defaults(self): - config = WasmTaskBuilder().set_name(" ").build() - assert config.task_name == "default-processor" - - def test_set_type(self): - config = WasmTaskBuilder().set_type("processor").build() - assert config.task_type == "processor" - - def test_builtin_serialization(self): - config = WasmTaskBuilder().set_builtin_serialization(True).build() - assert config.use_builtin_event_serialization is True - - def test_configure_checkpoint_enabled(self): - config = ( - WasmTaskBuilder() - .configure_checkpoint(enabled=True, interval=5) - .build() - ) - assert config.enable_checkpoint is True - assert config.checkpoint_interval_seconds == 5 - - def test_checkpoint_interval_clamped_to_one(self): - config = ( - WasmTaskBuilder() - .configure_checkpoint(enabled=True, interval=0) - .build() - ) - assert config.checkpoint_interval_seconds == 1 - - def test_checkpoint_interval_negative_clamped(self): - config = ( - WasmTaskBuilder() - .configure_checkpoint(enabled=True, interval=-10) - .build() - ) - assert config.checkpoint_interval_seconds == 1 - - def test_add_init_config(self): - config = ( - WasmTaskBuilder() - .add_init_config("key1", "val1") - .add_init_config("key2", "val2") - .build() - ) - assert config.init_config == {"key1": "val1", "key2": "val2"} - - def test_add_input_group(self): - ki = KafkaInput("broker:9092", "in-topic", "grp") - config = WasmTaskBuilder().add_input_group([ki]).build() - assert len(config.input_groups) == 1 - assert len(config.input_groups[0]["inputs"]) == 1 - assert config.input_groups[0]["inputs"][0]["topic"] == "in-topic" - - def test_add_multiple_input_groups(self): - ki1 = KafkaInput("b:9092", "t1", "g1") - ki2 = KafkaInput("b:9092", "t2", "g2") - config = ( - WasmTaskBuilder() - .add_input_group([ki1]) - .add_input_group([ki2]) - .build() - ) - assert len(config.input_groups) == 2 - - def test_add_output(self): - ko = KafkaOutput("broker:9092", "out", 0) - config = WasmTaskBuilder().add_output(ko).build() - assert len(config.outputs) == 1 - assert config.outputs[0]["topic"] == "out" - - def test_add_multiple_outputs(self): - config = ( - WasmTaskBuilder() - .add_output(KafkaOutput("b:9092", "o1", 0)) - .add_output(KafkaOutput("b:9092", "o2", 1)) - .build() - ) - assert len(config.outputs) == 2 - - def test_fluent_chaining(self): - config = ( - WasmTaskBuilder() - .set_name("chained") - .set_type("python") - .set_builtin_serialization(False) - .configure_checkpoint(True, 10) - .add_init_config("k", "v") - .add_input_group([KafkaInput("b:9092", "t", "g")]) - .add_output(KafkaOutput("b:9092", "o", 0)) - .build() - ) - assert config.task_name == "chained" - assert config.enable_checkpoint is True - assert config.checkpoint_interval_seconds == 10 - assert len(config.input_groups) == 1 - assert len(config.outputs) == 1 - - def test_builder_produces_independent_configs(self): - builder = WasmTaskBuilder().set_name("first") - c1 = builder.build() - builder.set_name("second") - c2 = builder.build() - assert c1.task_name == "first" - assert c2.task_name == "second" - - -# ====================================================================== -# WasmTaskConfig -# ====================================================================== - - -class TestWasmTaskConfig: - - @pytest.fixture() - def sample_config(self): - return ( - WasmTaskBuilder() - .set_name("test-processor") - .set_type("python") - .configure_checkpoint(True, 5) - .add_init_config("key", "value") - .add_input_group( - [KafkaInput("localhost:9092", "in-topic", "test-group", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "out-topic", 0)) - .build() - ) - - def test_to_dict_keys(self, sample_config): - d = sample_config.to_dict() - assert "name" in d - assert "type" in d - assert "use_builtin_event_serialization" in d - assert "enable_checkpoint" in d - assert "checkpoint_interval_seconds" in d - assert "init_config" in d - assert "input-groups" in d - assert "outputs" in d - - def test_to_dict_values(self, sample_config): - d = sample_config.to_dict() - assert d["name"] == "test-processor" - assert d["type"] == "python" - assert d["enable_checkpoint"] is True - assert d["checkpoint_interval_seconds"] == 5 - - def test_to_yaml_produces_valid_yaml(self, sample_config): - yaml_str = sample_config.to_yaml() - parsed = yaml.safe_load(yaml_str) - assert isinstance(parsed, dict) - assert parsed["name"] == "test-processor" - - def test_from_yaml_roundtrip(self, sample_config): - yaml_str = sample_config.to_yaml() - restored = WasmTaskConfig.from_yaml(yaml_str) - assert restored.task_name == sample_config.task_name - assert restored.task_type == sample_config.task_type - assert restored.enable_checkpoint == sample_config.enable_checkpoint - assert restored.checkpoint_interval_seconds == sample_config.checkpoint_interval_seconds - - def test_from_yaml_minimal(self): - cfg = WasmTaskConfig.from_yaml("name: mini\n") - assert cfg.task_name == "mini" - assert cfg.task_type == "python" - assert cfg.enable_checkpoint is False - - def test_from_yaml_missing_name_gets_default(self): - cfg = WasmTaskConfig.from_yaml("type: python\n") - assert cfg.task_name == "default-processor" - - def test_from_yaml_empty_name_gets_default(self): - cfg = WasmTaskConfig.from_yaml("name: ''\n") - assert cfg.task_name == "default-processor" - - def test_from_yaml_invalid_format_raises(self): - with pytest.raises(ValueError, match="mapping"): - WasmTaskConfig.from_yaml("- just\n- a\n- list\n") - - def test_from_yaml_checkpoint_interval_clamped(self): - cfg = WasmTaskConfig.from_yaml( - "name: x\ncheckpoint_interval_seconds: 0\n" - ) - assert cfg.checkpoint_interval_seconds >= 1 - - def test_from_yaml_with_init_config(self): - cfg = WasmTaskConfig.from_yaml( - "name: x\ninit_config:\n a: '1'\n b: '2'\n" - ) - assert cfg.init_config == {"a": "1", "b": "2"} - - def test_to_yaml_sort_keys_false(self, sample_config): - yaml_str = sample_config.to_yaml() - lines = yaml_str.strip().split("\n") - first_key = lines[0].split(":")[0] - assert first_key == "name" diff --git a/tests/integration/test/wasm/python_sdk/test_lifecycle.py b/tests/integration/test/wasm/python_sdk/test_lifecycle.py new file mode 100644 index 00000000..0f8eec5c --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/test_lifecycle.py @@ -0,0 +1,219 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Integration tests for basic CRUD operations and status state-machine transitions. + +Covers: + - Full lifecycle: Create -> Show -> Stop -> Start -> Stop -> Drop + - Multiple function coexistence + - Rapid create / drop cycling + - show_functions listing correctness +""" + +import uuid +from typing import List + +from fs_client.client import FsClient +from fs_client.config import WasmTaskBuilder, KafkaInput, KafkaOutput + +from processors.counter_processor import CounterProcessor + + +def _unique(prefix: str) -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +def _build_counter_config(fn_name: str) -> "WasmTaskBuilder": + """Return a ready-to-build builder pre-configured for CounterProcessor.""" + return ( + WasmTaskBuilder() + .set_name(fn_name) + .add_init_config("class_name", "CounterProcessor") + .add_input_group([KafkaInput("localhost:9092", "in", "grp", 0)]) + .add_output(KafkaOutput("localhost:9092", "out", 0)) + ) + + +class TestFunctionLifecycle: + """ + Integration tests for basic CRUD operations and status state machine + transitions. + """ + + # ------------------------------------------------------------------ + # Golden-path lifecycle + # ------------------------------------------------------------------ + + def test_full_lifecycle_transitions( + self, fs_client: FsClient, function_registry: List[str] + ): + """Test the golden path: Create -> Show -> Stop -> Start -> Stop -> Drop.""" + fn_name = _unique("lifecycle") + function_registry.append(fn_name) + + config = _build_counter_config(fn_name).add_init_config("test_mode", "true").build() + + # 1. Create + assert fs_client.create_python_function_from_config(config, CounterProcessor) is True + + # 2. Verify visibility + listing = fs_client.show_functions() + fn_info = next((f for f in listing.functions if f.name == fn_name), None) + assert fn_info is not None, "Function must be listed after creation" + assert fn_info.status, "Status must be a non-empty string" + + # 3. Stop (server may auto-start on creation; stop first to get a known state) + assert fs_client.stop_function(fn_name) is True + + # 4. Start + assert fs_client.start_function(fn_name) is True + listing = fs_client.show_functions() + fn_info = next(f for f in listing.functions if f.name == fn_name) + assert fn_info.status.upper() == "RUNNING", ( + f"Expected RUNNING after start, got {fn_info.status}" + ) + + # 5. Stop again + assert fs_client.stop_function(fn_name) is True + listing = fs_client.show_functions() + fn_info = next(f for f in listing.functions if f.name == fn_name) + assert fn_info.status.upper() in ("STOPPED", "PAUSED", "INITIALIZED"), ( + f"Expected STOPPED/PAUSED after stop, got {fn_info.status}" + ) + + # 6. Drop + assert fs_client.drop_function(fn_name) is True + listing = fs_client.show_functions() + assert fn_name not in [f.name for f in listing.functions], ( + "Function must be removed from registry after drop" + ) + function_registry.remove(fn_name) + + # ------------------------------------------------------------------ + # show_functions consistency + # ------------------------------------------------------------------ + + def test_show_functions_returns_created_function( + self, fs_client: FsClient, function_registry: List[str] + ): + """show_functions must contain the newly created function.""" + fn_name = _unique("show") + function_registry.append(fn_name) + + config = _build_counter_config(fn_name).build() + fs_client.create_python_function_from_config(config, CounterProcessor) + + listing = fs_client.show_functions() + names = [f.name for f in listing.functions] + assert fn_name in names + + def test_show_functions_result_fields( + self, fs_client: FsClient, function_registry: List[str] + ): + """Each FunctionInfo must carry name, task_type, and status.""" + fn_name = _unique("fields") + function_registry.append(fn_name) + + config = _build_counter_config(fn_name).build() + fs_client.create_python_function_from_config(config, CounterProcessor) + + listing = fs_client.show_functions() + fn_info = next(f for f in listing.functions if f.name == fn_name) + + assert fn_info.name == fn_name + assert fn_info.task_type, "task_type must not be empty" + assert fn_info.status, "status must not be empty" + + # ------------------------------------------------------------------ + # Multiple function coexistence + # ------------------------------------------------------------------ + + def test_multiple_functions_coexist( + self, fs_client: FsClient, function_registry: List[str] + ): + """Several independently created functions must all be listed.""" + names = [_unique("multi") for _ in range(3)] + function_registry.extend(names) + + for name in names: + config = _build_counter_config(name).build() + fs_client.create_python_function_from_config(config, CounterProcessor) + + listing = fs_client.show_functions() + listed_names = {f.name for f in listing.functions} + for name in names: + assert name in listed_names, f"{name} missing from listing" + + # ------------------------------------------------------------------ + # Rapid create / drop cycling + # ------------------------------------------------------------------ + + def test_rapid_create_drop_cycle( + self, fs_client: FsClient, function_registry: List[str] + ): + """Rapidly creating and dropping functions must not corrupt server state.""" + for i in range(5): + fn_name = _unique(f"rapid-{i}") + + config = _build_counter_config(fn_name).build() + fs_client.create_python_function_from_config(config, CounterProcessor) + + fs_client.stop_function(fn_name) + assert fs_client.drop_function(fn_name) is True + + listing = fs_client.show_functions() + remaining = [f.name for f in listing.functions if f.name.startswith("rapid-")] + assert remaining == [], f"Stale functions remain: {remaining}" + + # ------------------------------------------------------------------ + # Restart (stop + start) resilience + # ------------------------------------------------------------------ + + def test_restart_preserves_identity( + self, fs_client: FsClient, function_registry: List[str] + ): + """Stopping and restarting a function should keep its name and type.""" + fn_name = _unique("restart") + function_registry.append(fn_name) + + config = _build_counter_config(fn_name).build() + fs_client.create_python_function_from_config(config, CounterProcessor) + + fs_client.stop_function(fn_name) + fs_client.start_function(fn_name) + + listing = fs_client.show_functions() + fn_info = next(f for f in listing.functions if f.name == fn_name) + assert fn_info.name == fn_name + assert fn_info.status.upper() == "RUNNING" + + # ------------------------------------------------------------------ + # Drop after stop (explicit two-phase teardown) + # ------------------------------------------------------------------ + + def test_stop_then_drop( + self, fs_client: FsClient, function_registry: List[str] + ): + """Explicitly stopping, then dropping must always succeed.""" + fn_name = _unique("stop-drop") + function_registry.append(fn_name) + + config = _build_counter_config(fn_name).build() + fs_client.create_python_function_from_config(config, CounterProcessor) + + assert fs_client.stop_function(fn_name) is True + assert fs_client.drop_function(fn_name) is True + + listing = fs_client.show_functions() + assert fn_name not in [f.name for f in listing.functions] + function_registry.remove(fn_name) diff --git a/tests/integration/test/wasm/python_sdk/test_python_api.py b/tests/integration/test/wasm/python_sdk/test_python_api.py deleted file mode 100644 index 6c5da7d1..00000000 --- a/tests/integration/test/wasm/python_sdk/test_python_api.py +++ /dev/null @@ -1,694 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Comprehensive integration tests for the Python SDK client API. - -Covers every public method of FsClient against a live FunctionStream server: - - create_python_function - - create_python_function_from_config - - create_function_from_bytes - - show_functions (with / without filter) - - start_function / stop_function / drop_function - - Error semantics: ConflictError, NotFoundError, validation errors - - Context-manager lifecycle - - Custom timeout behaviour - -Requires a running FunctionStream server (provided by the ``fs_server`` fixture). -""" - -import time -import uuid -from pathlib import Path - -import pytest - -from fs_client import FsClient, FunctionInfo, ShowFunctionsResult -from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder -from fs_client.exceptions import ( - BadRequestError, - ClientError, - ConflictError, - FsError, - NotFoundError, - ServerError, -) - - -# ====================================================================== -# Helpers -# ====================================================================== - - -def _unique_name(prefix: str = "pytest") -> str: - """Generate a unique function name to avoid cross-test collisions.""" - return f"{prefix}-{uuid.uuid4().hex[:8]}" - - -def _build_config(name: str, class_name: str = "MinimalProcessor") -> str: - """Build a minimal YAML config string for a Python function.""" - return ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .add_init_config("class_name", class_name) - .add_init_config("emit_threshold", "1") - .add_input_group( - [KafkaInput("localhost:9092", "test-in", "test-grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "test-out", 0)) - .build() - .to_yaml() - ) - - -def _processor_bytes() -> bytes: - """Return a minimal FSProcessorDriver implementation as source bytes.""" - return b"""\ -from fs_api import FSProcessorDriver, Context - -class MinimalProcessor(FSProcessorDriver): - def init(self, ctx: Context, config: dict): - pass - def process(self, ctx: Context, source_id: int, data: bytes): - ctx.emit(data, 0) - def process_watermark(self, ctx: Context, source_id: int, watermark: int): - ctx.emit_watermark(watermark, 0) - def take_checkpoint(self, ctx: Context, checkpoint_id: int): - return None - def check_heartbeat(self, ctx: Context) -> bool: - return True - def close(self, ctx: Context): - pass - def custom(self, payload: bytes) -> bytes: - return b'{}' -""" - - -def _create_function(client: FsClient, name: str) -> bool: - """Helper: register a Python function with the given name.""" - return client.create_python_function( - class_name="MinimalProcessor", - modules=[("minimal_processor", _processor_bytes())], - config_content=_build_config(name), - ) - - -def _ensure_dropped(client: FsClient, name: str) -> None: - """Drop a function if it exists, silently ignore NotFoundError.""" - try: - client.stop_function(name) - except Exception: - pass - try: - client.drop_function(name) - except Exception: - pass - - -# ====================================================================== -# TestFsClientContextManager -# ====================================================================== - - -class TestFsClientContextManager: - """Verify FsClient as a context manager opens and closes properly.""" - - def test_context_manager_enter_exit(self, fs_server): - with fs_server.get_client() as client: - assert isinstance(client, FsClient) - - def test_context_manager_closes_channel(self, fs_server): - with fs_server.get_client() as client: - result = client.show_functions() - assert isinstance(result, ShowFunctionsResult) - - def test_manual_close(self, fs_server): - client = fs_server.get_client() - result = client.show_functions() - assert isinstance(result, ShowFunctionsResult) - client.close() - - -# ====================================================================== -# TestCreatePythonFunction -# ====================================================================== - - -class TestCreatePythonFunction: - """Tests for FsClient.create_python_function (direct module bytes).""" - - def test_create_succeeds(self, fs_server): - name = _unique_name("create") - with fs_server.get_client() as client: - result = _create_function(client, name) - assert result is True - _ensure_dropped(client, name) - - def test_create_with_multiple_modules(self, fs_server): - name = _unique_name("multi-mod") - helper_code = b"""\ -HELPER_VERSION = "1.0" -def helper_func(): - return "ok" -""" - proc_code = b"""\ -from fs_api import FSProcessorDriver, Context - -class MultiModProcessor(FSProcessorDriver): - def init(self, ctx: Context, config: dict): - pass - def process(self, ctx: Context, source_id: int, data: bytes): - ctx.emit(data, 0) - def process_watermark(self, ctx: Context, source_id: int, watermark: int): - ctx.emit_watermark(watermark, 0) - def take_checkpoint(self, ctx: Context, checkpoint_id: int): - return None - def check_heartbeat(self, ctx: Context) -> bool: - return True - def close(self, ctx: Context): - pass - def custom(self, payload: bytes) -> bytes: - return b'{}' -""" - with fs_server.get_client() as client: - result = client.create_python_function( - class_name="MultiModProcessor", - modules=[ - ("helper_mod", helper_code), - ("multi_mod_processor", proc_code), - ], - config_content=_build_config(name, class_name="MultiModProcessor"), - ) - assert result is True - _ensure_dropped(client, name) - - def test_create_empty_modules_raises_value_error(self, fs_server): - with fs_server.get_client() as client: - with pytest.raises(ValueError, match="module"): - client.create_python_function( - class_name="X", - modules=[], - config_content="name: x\n", - ) - - def test_create_duplicate_raises_conflict(self, fs_server): - name = _unique_name("dup") - with fs_server.get_client() as client: - _create_function(client, name) - with pytest.raises(ConflictError) as exc_info: - _create_function(client, name) - assert exc_info.value.status_code == 409 - _ensure_dropped(client, name) - - -# ====================================================================== -# TestCreatePythonFunctionFromConfig -# ====================================================================== - - -class TestCreatePythonFunctionFromConfig: - """Tests for FsClient.create_python_function_from_config.""" - - def test_create_from_config_with_example_processor( - self, fs_server, python_example_dir - ): - processor_file = python_example_dir / "processor_impl.py" - if not processor_file.exists(): - pytest.skip("Python processor example not available") - - import importlib.util - import sys - - spec = importlib.util.spec_from_file_location( - "processor_impl", str(processor_file) - ) - mod = importlib.util.module_from_spec(spec) - mod.__file__ = str(processor_file) - sys.modules["processor_impl"] = mod - spec.loader.exec_module(mod) - - name = _unique_name("from-cfg") - config = ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .add_init_config("emit_threshold", "1") - .add_init_config("class_name", "CounterProcessor") - .add_input_group( - [KafkaInput("localhost:9092", "in-t", "grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "out-t", 0)) - .build() - ) - with fs_server.get_client() as client: - result = client.create_python_function_from_config( - config, mod.CounterProcessor - ) - assert result is True - _ensure_dropped(client, name) - - def test_from_config_rejects_non_class(self, fs_server): - config = WasmTaskBuilder().set_name("x").build() - with fs_server.get_client() as client: - with pytest.raises(ValueError, match="class"): - client.create_python_function_from_config(config, "not_a_class") - - def test_from_config_rejects_non_driver(self, fs_server): - config = WasmTaskBuilder().set_name("x").build() - - class NotADriver: - pass - - with fs_server.get_client() as client: - with pytest.raises(TypeError, match="FSProcessorDriver"): - client.create_python_function_from_config(config, NotADriver) - - -# ====================================================================== -# TestCreateFunctionFromBytes -# ====================================================================== - - -class TestCreateFunctionFromBytes: - """Tests for FsClient.create_function_from_bytes (WASM path).""" - - def test_config_wrong_type_raises_type_error(self, fs_server): - with fs_server.get_client() as client: - with pytest.raises(TypeError): - client.create_function_from_bytes( - function_bytes=b"x", - config_content=12345, - ) - - def test_config_accepts_string(self, fs_server): - """String config_content is accepted (may fail on server for bad WASM).""" - with fs_server.get_client() as client: - with pytest.raises(ServerError): - client.create_function_from_bytes( - function_bytes=b"\x00invalid-wasm", - config_content="name: bad-wasm\ntype: processor\n", - ) - - def test_config_accepts_bytes(self, fs_server): - """Bytes config_content is accepted (may fail on server for bad WASM).""" - with fs_server.get_client() as client: - with pytest.raises(ServerError): - client.create_function_from_bytes( - function_bytes=b"\x00invalid-wasm", - config_content=b"name: bad-wasm-b\ntype: processor\n", - ) - - -# ====================================================================== -# TestCreateFunctionFromFiles -# ====================================================================== - - -class TestCreateFunctionFromFiles: - """Tests for FsClient.create_function_from_files.""" - - def test_missing_wasm_file_raises_file_not_found(self, fs_server): - with fs_server.get_client() as client: - with pytest.raises(FileNotFoundError): - client.create_function_from_files( - function_path="/nonexistent/path.wasm", - config_path="/nonexistent/config.yaml", - ) - - def test_missing_config_file_raises_file_not_found(self, fs_server, tmp_path): - wasm_file = tmp_path / "dummy.wasm" - wasm_file.write_bytes(b"\x00\x61\x73\x6d") - with fs_server.get_client() as client: - with pytest.raises(FileNotFoundError): - client.create_function_from_files( - function_path=str(wasm_file), - config_path="/nonexistent/config.yaml", - ) - - -# ====================================================================== -# TestShowFunctions -# ====================================================================== - - -class TestShowFunctions: - """Tests for FsClient.show_functions.""" - - def test_show_functions_returns_result_type(self, fs_server): - with fs_server.get_client() as client: - result = client.show_functions() - assert isinstance(result, ShowFunctionsResult) - assert isinstance(result.status_code, int) - assert result.status_code < 400 - assert isinstance(result.functions, list) - - def test_show_functions_contains_created_function(self, fs_server): - name = _unique_name("show") - with fs_server.get_client() as client: - _create_function(client, name) - result = client.show_functions() - names = [f.name for f in result.functions] - assert name in names - _ensure_dropped(client, name) - - def test_show_functions_function_info_fields(self, fs_server): - name = _unique_name("info") - with fs_server.get_client() as client: - _create_function(client, name) - result = client.show_functions() - fn = next((f for f in result.functions if f.name == name), None) - assert fn is not None - assert isinstance(fn, FunctionInfo) - assert fn.name == name - assert isinstance(fn.task_type, str) - assert isinstance(fn.status, str) - _ensure_dropped(client, name) - - def test_show_functions_empty_on_fresh_server(self, fs_server): - with fs_server.get_client() as client: - result = client.show_functions() - assert isinstance(result.functions, list) - - def test_show_functions_with_filter(self, fs_server): - prefix = f"filter-{uuid.uuid4().hex[:6]}" - name1 = f"{prefix}-aaa" - name2 = f"{prefix}-bbb" - other_name = _unique_name("other") - with fs_server.get_client() as client: - _create_function(client, name1) - _create_function(client, name2) - _create_function(client, other_name) - - result = client.show_functions(filter_pattern=prefix) - found = [f.name for f in result.functions] - assert name1 in found or name2 in found - - _ensure_dropped(client, name1) - _ensure_dropped(client, name2) - _ensure_dropped(client, other_name) - - def test_show_functions_with_custom_timeout(self, fs_server): - with fs_server.get_client() as client: - result = client.show_functions(timeout=10.0) - assert isinstance(result, ShowFunctionsResult) - - -# ====================================================================== -# TestFunctionLifecycle -# ====================================================================== - - -class TestFunctionLifecycle: - """Full function lifecycle: create -> show -> stop -> start -> drop.""" - - def test_full_lifecycle(self, fs_server): - name = _unique_name("lifecycle") - with fs_server.get_client() as client: - assert _create_function(client, name) is True - - listing = client.show_functions() - assert name in [f.name for f in listing.functions] - - assert client.stop_function(name) is True - - assert client.start_function(name) is True - - client.stop_function(name) - assert client.drop_function(name) is True - - listing = client.show_functions() - assert name not in [f.name for f in listing.functions] - - def test_stop_and_restart_preserves_function(self, fs_server): - name = _unique_name("restart") - with fs_server.get_client() as client: - _create_function(client, name) - client.stop_function(name) - time.sleep(0.5) - client.start_function(name) - - listing = client.show_functions() - assert name in [f.name for f in listing.functions] - _ensure_dropped(client, name) - - def test_drop_running_function_requires_stop(self, fs_server): - name = _unique_name("drop-running") - with fs_server.get_client() as client: - _create_function(client, name) - try: - client.drop_function(name) - except ServerError: - client.stop_function(name) - result = client.drop_function(name) - assert result is True - else: - listing = client.show_functions() - assert name not in [f.name for f in listing.functions] - - -# ====================================================================== -# TestFunctionErrorHandling -# ====================================================================== - - -class TestFunctionErrorHandling: - """Error semantics for function operations.""" - - def test_drop_nonexistent_raises_not_found(self, fs_server): - with fs_server.get_client() as client: - with pytest.raises(NotFoundError) as exc_info: - client.drop_function("nonexistent-" + uuid.uuid4().hex[:8]) - assert exc_info.value.status_code == 404 - - def test_start_nonexistent_raises_not_found(self, fs_server): - with fs_server.get_client() as client: - with pytest.raises(NotFoundError) as exc_info: - client.start_function("nonexistent-" + uuid.uuid4().hex[:8]) - assert exc_info.value.status_code == 404 - - def test_stop_nonexistent_raises_not_found(self, fs_server): - with fs_server.get_client() as client: - with pytest.raises(NotFoundError) as exc_info: - client.stop_function("nonexistent-" + uuid.uuid4().hex[:8]) - assert exc_info.value.status_code == 404 - - def test_duplicate_create_raises_conflict(self, fs_server): - name = _unique_name("dup-err") - with fs_server.get_client() as client: - _create_function(client, name) - with pytest.raises(ConflictError) as exc_info: - _create_function(client, name) - assert exc_info.value.status_code == 409 - _ensure_dropped(client, name) - - def test_all_errors_inherit_from_fs_error(self, fs_server): - with fs_server.get_client() as client: - with pytest.raises(FsError): - client.drop_function("definitely-does-not-exist-" + uuid.uuid4().hex) - - -# ====================================================================== -# TestFunctionWithCheckpoint -# ====================================================================== - - -class TestFunctionWithCheckpoint: - """Test creating Python functions with checkpoint configuration.""" - - def test_checkpoint_enabled(self, fs_server): - name = _unique_name("ckpt") - config_yaml = ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .configure_checkpoint(enabled=True, interval=5) - .add_init_config("class_name", "MinimalProcessor") - .add_init_config("key", "value") - .add_input_group( - [KafkaInput("localhost:9092", "ckpt-in", "ckpt-grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "ckpt-out", 0)) - .build() - .to_yaml() - ) - with fs_server.get_client() as client: - result = client.create_python_function( - class_name="MinimalProcessor", - modules=[("minimal_processor", _processor_bytes())], - config_content=config_yaml, - ) - assert result is True - _ensure_dropped(client, name) - - def test_checkpoint_disabled(self, fs_server): - name = _unique_name("no-ckpt") - config_yaml = ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .configure_checkpoint(enabled=False) - .add_init_config("class_name", "MinimalProcessor") - .add_input_group( - [KafkaInput("localhost:9092", "in", "grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "out", 0)) - .build() - .to_yaml() - ) - with fs_server.get_client() as client: - result = client.create_python_function( - class_name="MinimalProcessor", - modules=[("minimal_processor", _processor_bytes())], - config_content=config_yaml, - ) - assert result is True - _ensure_dropped(client, name) - - -# ====================================================================== -# TestFunctionWithMultipleIO -# ====================================================================== - - -class TestFunctionWithMultipleIO: - """Test creating functions with multiple input groups and outputs.""" - - def test_multiple_input_groups(self, fs_server): - name = _unique_name("multi-in") - config_yaml = ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .add_init_config("class_name", "MinimalProcessor") - .add_input_group( - [KafkaInput("localhost:9092", "in1", "grp1", partition=0)] - ) - .add_input_group( - [KafkaInput("localhost:9092", "in2", "grp2", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "out", 0)) - .build() - .to_yaml() - ) - with fs_server.get_client() as client: - result = client.create_python_function( - class_name="MinimalProcessor", - modules=[("minimal_processor", _processor_bytes())], - config_content=config_yaml, - ) - assert result is True - _ensure_dropped(client, name) - - def test_multiple_outputs(self, fs_server): - name = _unique_name("multi-out") - config_yaml = ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .add_init_config("class_name", "MinimalProcessor") - .add_input_group( - [KafkaInput("localhost:9092", "in", "grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "out1", 0)) - .add_output(KafkaOutput("localhost:9092", "out2", 1)) - .build() - .to_yaml() - ) - with fs_server.get_client() as client: - result = client.create_python_function( - class_name="MinimalProcessor", - modules=[("minimal_processor", _processor_bytes())], - config_content=config_yaml, - ) - assert result is True - _ensure_dropped(client, name) - - -# ====================================================================== -# TestBuiltinSerialization -# ====================================================================== - - -class TestBuiltinSerialization: - """Test creating functions with builtin serialization toggled.""" - - def test_builtin_serialization_enabled(self, fs_server): - name = _unique_name("builtin-on") - config_yaml = ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .set_builtin_serialization(True) - .add_init_config("class_name", "MinimalProcessor") - .add_input_group( - [KafkaInput("localhost:9092", "in", "grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "out", 0)) - .build() - .to_yaml() - ) - with fs_server.get_client() as client: - result = client.create_python_function( - class_name="MinimalProcessor", - modules=[("minimal_processor", _processor_bytes())], - config_content=config_yaml, - ) - assert result is True - _ensure_dropped(client, name) - - def test_builtin_serialization_disabled(self, fs_server): - name = _unique_name("builtin-off") - config_yaml = ( - WasmTaskBuilder() - .set_name(name) - .set_type("python") - .set_builtin_serialization(False) - .add_init_config("class_name", "MinimalProcessor") - .add_input_group( - [KafkaInput("localhost:9092", "in", "grp", partition=0)] - ) - .add_output(KafkaOutput("localhost:9092", "out", 0)) - .build() - .to_yaml() - ) - with fs_server.get_client() as client: - result = client.create_python_function( - class_name="MinimalProcessor", - modules=[("minimal_processor", _processor_bytes())], - config_content=config_yaml, - ) - assert result is True - _ensure_dropped(client, name) - - -# ====================================================================== -# TestClientCustomTimeout -# ====================================================================== - - -class TestClientCustomTimeout: - """Verify custom timeout propagation.""" - - def test_default_timeout_attribute(self, fs_server): - client = FsClient( - host=fs_server.host, - port=fs_server.port, - default_timeout=42.0, - ) - assert client.default_timeout == 42.0 - client.close() - - def test_per_call_timeout(self, fs_server): - with fs_server.get_client() as client: - result = client.show_functions(timeout=5.0) - assert isinstance(result, ShowFunctionsResult) From f87f80136d1b2c3e66a40298c002fdbd0d5b515f Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Apr 2026 23:23:54 +0800 Subject: [PATCH 10/15] update --- tests/integration/framework/__init__.py | 8 +++++- tests/integration/framework/instance.py | 6 ++-- tests/integration/framework/workspace.py | 21 +++++++------- .../test/wasm/python_sdk/conftest.py | 28 ++++++++++++++----- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/tests/integration/framework/__init__.py b/tests/integration/framework/__init__.py index 241d269e..b735753a 100644 --- a/tests/integration/framework/__init__.py +++ b/tests/integration/framework/__init__.py @@ -11,6 +11,12 @@ # limitations under the License. from .instance import FunctionStreamInstance -from .kafka_manager import KafkaDockerManager __all__ = ["FunctionStreamInstance", "KafkaDockerManager"] + + +def __getattr__(name: str): + if name == "KafkaDockerManager": + from .kafka_manager import KafkaDockerManager + return KafkaDockerManager + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tests/integration/framework/instance.py b/tests/integration/framework/instance.py index dbf9ba0e..be22d9db 100644 --- a/tests/integration/framework/instance.py +++ b/tests/integration/framework/instance.py @@ -115,12 +115,14 @@ def start(self, timeout: float = 30.0) -> "FunctionStreamInstance": return self def stop(self, timeout: float = 10.0) -> None: - """Graceful SIGTERM shutdown.""" + """Graceful SIGTERM shutdown, then remove everything except logs.""" self.process.stop(timeout=timeout) + self.workspace.cleanup() def kill(self) -> None: - """Immediate SIGKILL.""" + """Immediate SIGKILL, then remove everything except logs.""" self.process.kill() + self.workspace.cleanup() def restart(self, timeout: float = 10.0) -> "FunctionStreamInstance": """Stop then start (same port, same workspace).""" diff --git a/tests/integration/framework/workspace.py b/tests/integration/framework/workspace.py index d41133cf..b96ebb96 100644 --- a/tests/integration/framework/workspace.py +++ b/tests/integration/framework/workspace.py @@ -14,11 +14,13 @@ InstanceWorkspace: manages the directory tree for a single FunctionStream test instance. -Layout: - tests/integration/target///FunctionStream-/ +Layout (during test run): + tests/integration/target/// conf/config.yaml data/ logs/stdout.log, stderr.log, app.log + +After cleanup only ``logs/`` is retained. """ import shutil @@ -36,10 +38,7 @@ def __init__(self, target_dir: Path, test_name: str, port: int): self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") self.root_dir = ( - self.target_dir - / self.test_name - / self.timestamp - / f"FunctionStream-{self.port}" + self.target_dir / self.test_name / self.timestamp ) self.conf_dir = self.root_dir / "conf" self.data_dir = self.root_dir / "data" @@ -54,8 +53,8 @@ def setup(self) -> None: for d in (self.conf_dir, self.data_dir, self.log_dir): d.mkdir(parents=True, exist_ok=True) - def cleanup_data(self) -> None: - """Remove the data directory but preserve logs for debugging.""" - if self.data_dir.exists(): - shutil.rmtree(self.data_dir) - self.data_dir.mkdir(parents=True, exist_ok=True) + def cleanup(self) -> None: + """Remove everything except logs/ so only diagnostic output remains.""" + for d in (self.conf_dir, self.data_dir): + if d.exists(): + shutil.rmtree(d) diff --git a/tests/integration/test/wasm/python_sdk/conftest.py b/tests/integration/test/wasm/python_sdk/conftest.py index 026a0ed8..59ea0143 100644 --- a/tests/integration/test/wasm/python_sdk/conftest.py +++ b/tests/integration/test/wasm/python_sdk/conftest.py @@ -27,7 +27,7 @@ # Make the ``processors`` package importable from this directory sys.path.insert(0, str(Path(__file__).resolve().parent)) -from framework import FunctionStreamInstance, KafkaDockerManager # noqa: E402 +from framework import FunctionStreamInstance # noqa: E402 from fs_client.client import FsClient # noqa: E402 PROJECT_ROOT = Path(__file__).resolve().parents[5] @@ -38,7 +38,7 @@ # ====================================================================== @pytest.fixture(scope="session") -def kafka() -> Generator[KafkaDockerManager, None, None]: +def kafka(): """ Session-scoped fixture: start a Docker-managed Kafka broker once for the entire test session. @@ -47,6 +47,8 @@ def kafka() -> Generator[KafkaDockerManager, None, None]: as a parameter. Tests that only register functions (without producing / consuming data) do NOT need this fixture. """ + from framework import KafkaDockerManager + mgr = KafkaDockerManager() mgr.setup_kafka() yield mgr @@ -55,15 +57,27 @@ def kafka() -> Generator[KafkaDockerManager, None, None]: # ====================================================================== -# FunctionStream server +# FunctionStream server (per-test instance with isolated logs) # ====================================================================== -@pytest.fixture(scope="session") -def fs_server() -> Generator[FunctionStreamInstance, None, None]: +def _derive_test_name(request: pytest.FixtureRequest) -> str: + """Build a human-readable path from the pytest node: wasm/python_sdk//.""" + cls = request.node.cls + parts = ["wasm", "python_sdk"] + if cls is not None: + parts.append(cls.__name__) + parts.append(request.node.name) + return "/".join(parts) + + +@pytest.fixture +def fs_server(request: pytest.FixtureRequest) -> Generator[FunctionStreamInstance, None, None]: """ - Session-scoped fixture: start the FunctionStream server once for all tests. + Function-scoped fixture: each test gets its own FunctionStream server + with isolated log directory named after the test class and method. """ - instance = FunctionStreamInstance(test_name="wasm_python_sdk_integration") + test_name = _derive_test_name(request) + instance = FunctionStreamInstance(test_name=test_name) instance.start() yield instance instance.kill() From 0e99002900e2c764356d102e1e680a48f81d6097 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 16 Apr 2026 22:22:30 +0800 Subject: [PATCH 11/15] update --- .github/workflows/integration-test.yml | 8 +- src/runtime/wasm/processor/wasm/wasm_task.rs | 23 ++++- tests/integration/README.md | 88 +++++++++++++++++++ tests/integration/framework/config.py | 3 +- .../test/wasm/python_sdk/conftest.py | 16 ++++ .../test/wasm/python_sdk/test_lifecycle.py | 34 +++---- 6 files changed, 150 insertions(+), 22 deletions(-) create mode 100644 tests/integration/README.md diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 817b0f8f..82d9ce66 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -32,6 +32,9 @@ jobs: - name: Checkout Source uses: actions/checkout@v4 + - name: Verify Docker + run: docker version + - name: Setup Python uses: actions/setup-python@v5 with: @@ -56,7 +59,10 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build Release Binary - run: cargo build --release --no-default-features --features incremental-cache + run: cargo build --release --features python + + - name: Pre-pull Kafka Image + run: docker pull apache/kafka:3.7.0 - name: Run Integration Tests run: make integration-test diff --git a/src/runtime/wasm/processor/wasm/wasm_task.rs b/src/runtime/wasm/processor/wasm/wasm_task.rs index 2837c9a0..c61f385f 100644 --- a/src/runtime/wasm/processor/wasm/wasm_task.rs +++ b/src/runtime/wasm/processor/wasm/wasm_task.rs @@ -392,10 +392,27 @@ impl WasmTask { ) -> ControlAction { match signal { TaskControlSignal::Start { completion_flag } => { - for input in inputs.iter_mut() { - let _ = input.start(); + for (idx, input) in inputs.iter_mut().enumerate() { + if let Err(e) = input.start() { + let msg = format!("Failed to start input {}: {}", idx, e); + log::error!("{}", msg); + *failure_cause.lock().unwrap() = Some(msg.clone()); + *shared_state.lock().unwrap() = + ComponentState::Error { error: msg.clone() }; + *execution_state.lock().unwrap() = ExecutionState::Failed; + completion_flag.mark_error(msg); + return ControlAction::Pause; + } + } + if let Err(e) = processor.start_outputs() { + let msg = format!("Failed to start outputs: {}", e); + log::error!("{}", msg); + *failure_cause.lock().unwrap() = Some(msg.clone()); + *shared_state.lock().unwrap() = ComponentState::Error { error: msg.clone() }; + *execution_state.lock().unwrap() = ExecutionState::Failed; + completion_flag.mark_error(msg); + return ControlAction::Pause; } - let _ = processor.start_outputs(); *state = TaskState::Running; *shared_state.lock().unwrap() = ComponentState::Running; completion_flag.mark_completed(); diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 00000000..23096720 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,88 @@ +# Integration Tests + +## Prerequisites + +| Dependency | Version | Purpose | +|------------|----------|-------------------------------------------------| +| Python | >= 3.9 | Test framework runtime | +| Rust | stable | Build the FunctionStream binary | +| Docker | >= 20.10 | Run a Kafka broker for streaming integration tests | + +> **Docker is required.** The test framework automatically pulls and manages +> an `apache/kafka:3.7.0` container in KRaft mode to provide a real Kafka +> broker for tests that involve Kafka input/output. Tests will fail if the +> Docker daemon is not running. + +## Quick Start + +```bash +# From the project root +make build # Build the release binary (with --features python) +make integration-test + +# Or run directly from this directory +cd tests/integration +make test +``` + +## Directory Layout + +``` +tests/integration/ +├── Makefile # test / clean targets +├── requirements.txt # Python dependencies (pytest, grpcio, docker, ...) +├── pytest.ini # Pytest configuration +├── framework/ # Reusable test infrastructure +│ ├── instance.py # FunctionStreamInstance facade +│ ├── workspace.py # Per-test directory management +│ ├── config.py # Server config generation +│ ├── process.py # OS process lifecycle (start/stop/kill) +│ ├── utils.py # Port allocation, readiness probes +│ └── kafka_manager.py # Docker-managed Kafka broker (KRaft mode) +├── test/ # Test suites +│ ├── wasm/ # WASM function tests +│ │ └── python_sdk/ # Python SDK integration tests +│ └── streaming/ # Streaming engine tests (future) +└── target/ # Test output (git-ignored) + ├── .shared_cache/ # Shared WASM compilation cache across tests + └── ////logs/ +``` + +## Test Output + +Each test gets an isolated server instance with its own log directory: + +``` +target/wasm/python_sdk/TestFunctionLifecycle/test_full_lifecycle_transitions/20260416_221655/ + logs/ + app.log # FunctionStream application log + stdout.log # Server stdout + stderr.log # Server stderr +``` + +Only `logs/` is retained after tests complete; `conf/` and `data/` are +automatically cleaned up. + +## Python Dependencies + +All Python packages are listed in `requirements.txt` and installed +automatically by `make test`. Key dependencies: + +- `pytest` — test runner +- `grpcio` / `protobuf` — gRPC client communication +- `docker` — Docker SDK for managing the Kafka container +- `confluent-kafka` — Kafka admin client for topic management +- `functionstream-api` / `functionstream-client` — local editable installs + +## Running Specific Tests + +```bash +# Single test +make test PYTEST_ARGS="-k test_full_lifecycle_transitions" + +# Single file +make test PYTEST_ARGS="test/wasm/python_sdk/test_lifecycle.py" + +# Verbose with live log +make test PYTEST_ARGS="-v --log-cli-level=DEBUG" +``` diff --git a/tests/integration/framework/config.py b/tests/integration/framework/config.py index 1ddb97ce..51effad5 100644 --- a/tests/integration/framework/config.py +++ b/tests/integration/framework/config.py @@ -24,6 +24,7 @@ _INTEGRATION_DIR = Path(__file__).resolve().parents[1] _PROJECT_ROOT = _INTEGRATION_DIR.parents[1] +_SHARED_CACHE_DIR = _INTEGRATION_DIR / "target" / ".shared_cache" def _find_python_wasm() -> str: @@ -61,7 +62,7 @@ def __init__(self, host: str, port: int, workspace: InstanceWorkspace): }, "python": { "wasm_path": _find_python_wasm(), - "cache_dir": str(workspace.data_dir / "cache" / "python-runner"), + "cache_dir": str(_SHARED_CACHE_DIR / "python-runner"), "enable_cache": True, }, "state_storage": { diff --git a/tests/integration/test/wasm/python_sdk/conftest.py b/tests/integration/test/wasm/python_sdk/conftest.py index 59ea0143..264484d6 100644 --- a/tests/integration/test/wasm/python_sdk/conftest.py +++ b/tests/integration/test/wasm/python_sdk/conftest.py @@ -56,6 +56,22 @@ def kafka(): mgr.teardown_kafka() +# ====================================================================== +# Kafka topics (depends on kafka broker) +# ====================================================================== + +@pytest.fixture(scope="session") +def kafka_topics(kafka): + """ + Session-scoped fixture: ensure the Kafka broker is running and + pre-create the standard input/output topics used by tests. + + Returns the bootstrap servers string for use in WasmTaskBuilder configs. + """ + kafka.create_topics_if_not_exist(["in", "out", "events", "counts"]) + return kafka.bootstrap_servers + + # ====================================================================== # FunctionStream server (per-test instance with isolated logs) # ====================================================================== diff --git a/tests/integration/test/wasm/python_sdk/test_lifecycle.py b/tests/integration/test/wasm/python_sdk/test_lifecycle.py index 0f8eec5c..188bb0f6 100644 --- a/tests/integration/test/wasm/python_sdk/test_lifecycle.py +++ b/tests/integration/test/wasm/python_sdk/test_lifecycle.py @@ -33,14 +33,14 @@ def _unique(prefix: str) -> str: return f"{prefix}-{uuid.uuid4().hex[:8]}" -def _build_counter_config(fn_name: str) -> "WasmTaskBuilder": +def _build_counter_config(fn_name: str, bootstrap: str) -> "WasmTaskBuilder": """Return a ready-to-build builder pre-configured for CounterProcessor.""" return ( WasmTaskBuilder() .set_name(fn_name) .add_init_config("class_name", "CounterProcessor") - .add_input_group([KafkaInput("localhost:9092", "in", "grp", 0)]) - .add_output(KafkaOutput("localhost:9092", "out", 0)) + .add_input_group([KafkaInput(bootstrap, "in", "grp", 0)]) + .add_output(KafkaOutput(bootstrap, "out", 0)) ) @@ -55,13 +55,13 @@ class TestFunctionLifecycle: # ------------------------------------------------------------------ def test_full_lifecycle_transitions( - self, fs_client: FsClient, function_registry: List[str] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str ): """Test the golden path: Create -> Show -> Stop -> Start -> Stop -> Drop.""" fn_name = _unique("lifecycle") function_registry.append(fn_name) - config = _build_counter_config(fn_name).add_init_config("test_mode", "true").build() + config = _build_counter_config(fn_name, kafka_topics).add_init_config("test_mode", "true").build() # 1. Create assert fs_client.create_python_function_from_config(config, CounterProcessor) is True @@ -104,13 +104,13 @@ def test_full_lifecycle_transitions( # ------------------------------------------------------------------ def test_show_functions_returns_created_function( - self, fs_client: FsClient, function_registry: List[str] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str ): """show_functions must contain the newly created function.""" fn_name = _unique("show") function_registry.append(fn_name) - config = _build_counter_config(fn_name).build() + config = _build_counter_config(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) listing = fs_client.show_functions() @@ -118,13 +118,13 @@ def test_show_functions_returns_created_function( assert fn_name in names def test_show_functions_result_fields( - self, fs_client: FsClient, function_registry: List[str] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str ): """Each FunctionInfo must carry name, task_type, and status.""" fn_name = _unique("fields") function_registry.append(fn_name) - config = _build_counter_config(fn_name).build() + config = _build_counter_config(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) listing = fs_client.show_functions() @@ -139,14 +139,14 @@ def test_show_functions_result_fields( # ------------------------------------------------------------------ def test_multiple_functions_coexist( - self, fs_client: FsClient, function_registry: List[str] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str ): """Several independently created functions must all be listed.""" names = [_unique("multi") for _ in range(3)] function_registry.extend(names) for name in names: - config = _build_counter_config(name).build() + config = _build_counter_config(name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) listing = fs_client.show_functions() @@ -159,13 +159,13 @@ def test_multiple_functions_coexist( # ------------------------------------------------------------------ def test_rapid_create_drop_cycle( - self, fs_client: FsClient, function_registry: List[str] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str ): """Rapidly creating and dropping functions must not corrupt server state.""" for i in range(5): fn_name = _unique(f"rapid-{i}") - config = _build_counter_config(fn_name).build() + config = _build_counter_config(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) fs_client.stop_function(fn_name) @@ -180,13 +180,13 @@ def test_rapid_create_drop_cycle( # ------------------------------------------------------------------ def test_restart_preserves_identity( - self, fs_client: FsClient, function_registry: List[str] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str ): """Stopping and restarting a function should keep its name and type.""" fn_name = _unique("restart") function_registry.append(fn_name) - config = _build_counter_config(fn_name).build() + config = _build_counter_config(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) fs_client.stop_function(fn_name) @@ -202,13 +202,13 @@ def test_restart_preserves_identity( # ------------------------------------------------------------------ def test_stop_then_drop( - self, fs_client: FsClient, function_registry: List[str] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str ): """Explicitly stopping, then dropping must always succeed.""" fn_name = _unique("stop-drop") function_registry.append(fn_name) - config = _build_counter_config(fn_name).build() + config = _build_counter_config(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) assert fs_client.stop_function(fn_name) is True From e268a4ab3c431a538d38192af3b548c3333ac7d4 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 16 Apr 2026 23:18:33 +0800 Subject: [PATCH 12/15] update --- tests/integration/framework/config.py | 132 ++++++-- tests/integration/framework/instance.py | 146 ++++++--- tests/integration/framework/kafka_manager.py | 305 +++++++++++------- tests/integration/framework/process.py | 147 ++++++--- tests/integration/framework/utils.py | 108 +++++-- tests/integration/framework/workspace.py | 73 ++++- .../test/wasm/python_sdk/conftest.py | 115 +++---- .../test/wasm/python_sdk/test_data_flow.py | 242 ++++++++++++++ .../test/wasm/python_sdk/test_lifecycle.py | 190 ++++------- 9 files changed, 1000 insertions(+), 458 deletions(-) create mode 100644 tests/integration/test/wasm/python_sdk/test_data_flow.py diff --git a/tests/integration/framework/config.py b/tests/integration/framework/config.py index 51effad5..d702ecdc 100644 --- a/tests/integration/framework/config.py +++ b/tests/integration/framework/config.py @@ -15,35 +15,86 @@ the FunctionStream binary via FUNCTION_STREAM_CONF. """ +import logging +import os +import tempfile from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List import yaml from .workspace import InstanceWorkspace -_INTEGRATION_DIR = Path(__file__).resolve().parents[1] -_PROJECT_ROOT = _INTEGRATION_DIR.parents[1] -_SHARED_CACHE_DIR = _INTEGRATION_DIR / "target" / ".shared_cache" +logger = logging.getLogger(__name__) -def _find_python_wasm() -> str: - """Locate the Python WASM runtime for server initialisation.""" - candidates = [ - _PROJECT_ROOT / "python" / "functionstream-runtime" / "target" / "functionstream-python-runtime.wasm", - _PROJECT_ROOT / "dist" / "function-stream" / "data" / "cache" / "python-runner" / "functionstream-python-runtime.wasm", - ] - for c in candidates: - if c.exists(): - return str(c.resolve()) - return str(candidates[0]) +class ConfigurationError(Exception): + """Base exception for all configuration-related errors.""" + pass + + +class WasmRuntimeNotFoundError(ConfigurationError): + """Raised when the required Python WASM runtime cannot be located.""" + pass + + +class PathManager: + """ + Utility class for resolving and managing cross-platform paths. + All outputs are standardized to POSIX format for safe YAML serialization. + """ + + _INTEGRATION_DIR = Path(__file__).resolve().parents[1] + _PROJECT_ROOT = _INTEGRATION_DIR.parents[1] + + @classmethod + def get_shared_cache_dir(cls) -> Path: + return cls._INTEGRATION_DIR / "target" / ".shared_cache" + + @classmethod + def build_posix_path(cls, base: Path, *segments: str) -> str: + """ + Safely joins a base Path with string segments and returns a POSIX-compliant string. + Eliminates manual string concatenation and Windows backslash issues. + """ + target_path = base + for segment in segments: + target_path = target_path / segment + return target_path.as_posix() + + @classmethod + def find_python_wasm_posix(cls) -> str: + """Locates the Python WASM runtime and returns its POSIX path.""" + candidates: List[Path] = [ + cls._PROJECT_ROOT / "python" / "functionstream-runtime" / "target" / "functionstream-python-runtime.wasm", + cls._PROJECT_ROOT / "dist" / "function-stream" / "data" / "cache" / "python-runner" / "functionstream-python-runtime.wasm", + ] + + for candidate in candidates: + if candidate.is_file(): + logger.debug("Found Python WASM runtime at: %s", candidate) + return candidate.resolve().as_posix() + + raise WasmRuntimeNotFoundError( + "Could not find python-runtime.wasm. Checked locations:\n" + + "\n".join(f" - {c}" for c in candidates) + ) class InstanceConfig: - """Generates and persists config.yaml for one FunctionStream instance.""" + """ + Generates and persists config.yaml for one FunctionStream instance. + """ def __init__(self, host: str, port: int, workspace: InstanceWorkspace): self._workspace = workspace + + wasm_path = PathManager.find_python_wasm_posix() + cache_dir = PathManager.build_posix_path(PathManager.get_shared_cache_dir(), "python-runner") + log_file = PathManager.build_posix_path(workspace.log_dir, "app.log") + task_db_path = PathManager.build_posix_path(workspace.data_dir, "task") + catalog_db_path = PathManager.build_posix_path(workspace.data_dir, "stream_catalog") + self._config: Dict[str, Any] = { "service": { "service_id": f"it-{port}", @@ -56,13 +107,13 @@ def __init__(self, host: str, port: int, workspace: InstanceWorkspace): "logging": { "level": "info", "format": "json", - "file_path": str(workspace.log_dir / "app.log"), + "file_path": log_file, "max_file_size": 50, "max_files": 3, }, "python": { - "wasm_path": _find_python_wasm(), - "cache_dir": str(_SHARED_CACHE_DIR / "python-runner"), + "wasm_path": wasm_path, + "cache_dir": cache_dir, "enable_cache": True, }, "state_storage": { @@ -70,32 +121,55 @@ def __init__(self, host: str, port: int, workspace: InstanceWorkspace): }, "task_storage": { "storage_type": "rocksdb", - "db_path": str(workspace.data_dir / "task"), + "db_path": task_db_path, }, "stream_catalog": { "persist": True, - "db_path": str(workspace.data_dir / "stream_catalog"), + "db_path": catalog_db_path, }, } @property def raw(self) -> Dict[str, Any]: - return self._config + return self._config.copy() def override(self, overrides: Dict[str, Any]) -> None: - """ - Apply overrides using dot-separated keys. - Example: {"service.debug": True, "logging.level": "debug"} - """ + """Apply overrides using dot-separated keys.""" for dotted_key, value in overrides.items(): keys = dotted_key.split(".") target = self._config + for k in keys[:-1]: target = target.setdefault(k, {}) + if not isinstance(target, dict): + raise ConfigurationError(f"Cannot override key '{dotted_key}': '{k}' is not a dictionary.") + target[keys[-1]] = value def write_to_workspace(self) -> Path: - """Serialize config to the workspace config.yaml and return its path.""" - with open(self._workspace.config_file, "w", encoding="utf-8") as f: - yaml.dump(self._config, f, default_flow_style=False, sort_keys=False) - return self._workspace.config_file + """Serialize config to the workspace config.yaml safely.""" + target_file = Path(self._workspace.config_file) + target_file.parent.mkdir(parents=True, exist_ok=True) + + fd, temp_path = tempfile.mkstemp( + dir=target_file.parent, + prefix=".config-", + suffix=".yaml", + text=True + ) + + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + yaml.safe_dump( + self._config, + f, + default_flow_style=False, + sort_keys=False + ) + os.replace(temp_path, target_file) + except Exception as e: + Path(temp_path).unlink(missing_ok=True) + logger.error("Failed to write configuration file: %s", e) + raise ConfigurationError(f"Configuration write failed: {e}") from e + + return target_file \ No newline at end of file diff --git a/tests/integration/framework/instance.py b/tests/integration/framework/instance.py index be22d9db..b18ca737 100644 --- a/tests/integration/framework/instance.py +++ b/tests/integration/framework/instance.py @@ -17,47 +17,84 @@ import logging from pathlib import Path -from typing import Any, Optional +from types import TracebackType +from typing import TYPE_CHECKING, Any, Optional, Type +from .paths import PathManager from .config import InstanceConfig from .process import FunctionStreamProcess from .utils import find_free_port, wait_for_port from .workspace import InstanceWorkspace -logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from fs_client import FsClient + from fs_client._proto.function_stream_pb2 import SqlResponse -_INTEGRATION_DIR = Path(__file__).resolve().parents[1] -_PROJECT_ROOT = _INTEGRATION_DIR.parents[1] -_TARGET_DIR = _INTEGRATION_DIR / "target" -_BINARY_PATH = _PROJECT_ROOT / "target" / "release" / "function-stream" + +class FunctionStreamInstanceError(Exception): + """Base exception for FunctionStream instance errors.""" + pass + + +class ServerStartupError(FunctionStreamInstanceError): + """Raised when the server fails to start or bind to the port within the timeout.""" + pass + + +class MissingDependencyError(FunctionStreamInstanceError): + """Raised when an optional client dependency (e.g., grpc) is missing.""" + pass class FunctionStreamInstance: """ Facade for a single FunctionStream server used in integration tests. + Designed to be used as a context manager to guarantee resource cleanup. + Usage: - inst = FunctionStreamInstance("my_test") - inst.configure(**{"service.debug": True}).start() - client = inst.get_client() - ... - inst.kill() + with FunctionStreamInstance("my_test").configure(**{"service.debug": True}) as inst: + client = inst.get_client() + inst.execute_sql("SELECT 1;") """ def __init__( - self, - test_name: str = "unnamed", - host: str = "127.0.0.1", - binary_path: Optional[Path] = None, + self, + test_name: str = "unnamed", + host: str = "127.0.0.1", + binary_path: Optional[Path] = None, ): + self.test_name = test_name self.host = host self.port = find_free_port() - binary = binary_path or _BINARY_PATH + self._logger = logging.getLogger(f"{__name__}.{self.test_name}-{self.port}") + + actual_binary = binary_path or PathManager.get_binary_path() + target_dir = PathManager.get_target_dir() - self.workspace = InstanceWorkspace(_TARGET_DIR, test_name, self.port) + self.workspace = InstanceWorkspace(target_dir, test_name, self.port) self.config = InstanceConfig(self.host, self.port, self.workspace) - self.process = FunctionStreamProcess(binary, self.workspace) + self.process = FunctionStreamProcess(actual_binary, self.workspace) + + # ------------------------------------------------------------------ + # Context Manager Protocol (Replaces __del__) + # ------------------------------------------------------------------ + + def __enter__(self) -> "FunctionStreamInstance": + """Start the instance when entering the 'with' block.""" + self.start() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + """Ensure absolute cleanup when exiting the 'with' block.""" + self._logger.debug("Tearing down instance due to context exit.") + self.kill() # ------------------------------------------------------------------ # Properties @@ -68,7 +105,7 @@ def is_running(self) -> bool: return self.process.is_running @property - def pid(self): + def pid(self) -> Optional[int]: return self.process.pid # ------------------------------------------------------------------ @@ -76,7 +113,10 @@ def pid(self): # ------------------------------------------------------------------ def configure(self, **overrides: Any) -> "FunctionStreamInstance": - """Apply config overrides. Chainable: inst.configure(k=v).start().""" + """ + Apply config overrides. Chainable. + Example: inst.configure(service__debug=True).start() + """ self.config.override(overrides) return self @@ -87,12 +127,13 @@ def configure(self, **overrides: Any) -> "FunctionStreamInstance": def start(self, timeout: float = 30.0) -> "FunctionStreamInstance": """Prepare workspace, write config, launch binary, wait for ready.""" if self.is_running: + self._logger.debug("Instance is already running. Skipping start.") return self self.workspace.setup() self.config.write_to_workspace() - logger.info( + self._logger.info( "Starting FunctionStream [port=%d, dir=%s]", self.port, self.workspace.root_dir, @@ -102,30 +143,31 @@ def start(self, timeout: float = 30.0) -> "FunctionStreamInstance": if not wait_for_port(self.host, self.port, timeout): stderr_tail = self._read_tail(self.workspace.stderr_file) self.process.kill() - raise RuntimeError( + raise ServerStartupError( f"Server did not become ready within {timeout}s on port {self.port}.\n" - f"stderr tail:\n{stderr_tail}" + f"Process stderr tail:\n{stderr_tail}" ) - logger.info( - "FunctionStream ready [port=%d, pid=%d]", - self.port, - self.process.pid, - ) + self._logger.info("FunctionStream ready [pid=%s]", self.pid) return self def stop(self, timeout: float = 10.0) -> None: """Graceful SIGTERM shutdown, then remove everything except logs.""" - self.process.stop(timeout=timeout) + if self.is_running: + self._logger.debug("Stopping instance gracefully...") + self.process.stop(timeout=timeout) self.workspace.cleanup() def kill(self) -> None: """Immediate SIGKILL, then remove everything except logs.""" - self.process.kill() + if self.is_running: + self._logger.debug("Killing instance immediately...") + self.process.kill() self.workspace.cleanup() def restart(self, timeout: float = 10.0) -> "FunctionStreamInstance": """Stop then start (same port, same workspace).""" + self._logger.info("Restarting instance...") self.stop(timeout=timeout) return self.start() @@ -133,35 +175,33 @@ def restart(self, timeout: float = 10.0) -> "FunctionStreamInstance": # Client access # ------------------------------------------------------------------ - def get_client(self): + def get_client(self) -> "FsClient": """Return a connected FsClient. Caller should close it when done.""" try: from fs_client import FsClient - except ImportError: - raise ImportError( + except ImportError as e: + raise MissingDependencyError( "functionstream-client is not installed. " "Run: pip install -e python/functionstream-client" - ) + ) from e return FsClient(host=self.host, port=self.port) - def execute_sql(self, sql: str, timeout: float = 30.0): + def execute_sql(self, sql: str, timeout: float = 30.0) -> "SqlResponse": """Convenience helper: send a SQL statement via gRPC ExecuteSql.""" try: import grpc from fs_client._proto import function_stream_pb2, function_stream_pb2_grpc - except ImportError: - raise ImportError( - "functionstream-client is not installed. " - "Run: pip install -e python/functionstream-client" - ) + except ImportError as e: + raise MissingDependencyError( + "gRPC dependencies or proto definitions are not available." + ) from e - channel = grpc.insecure_channel(f"{self.host}:{self.port}") - try: + # Use context manager for channel to prevent socket leaks + with grpc.insecure_channel(f"{self.host}:{self.port}") as channel: stub = function_stream_pb2_grpc.FunctionStreamServiceStub(channel) request = function_stream_pb2.SqlRequest(sql=sql) + self._logger.debug("Executing SQL: %s", sql) return stub.ExecuteSql(request, timeout=timeout) - finally: - channel.close() # ------------------------------------------------------------------ # Internals @@ -169,15 +209,15 @@ def execute_sql(self, sql: str, timeout: float = 30.0): @staticmethod def _read_tail(path: Path, chars: int = 2000) -> str: - if not path.exists(): - return "" - text = path.read_text(errors="replace") - return text[-chars:] if len(text) > chars else text + """Reads the end of a file safely, useful for pulling crash logs.""" + if not path.is_file(): + return "" + try: + text = path.read_text(errors="replace") + return text[-chars:] if len(text) > chars else text + except Exception as e: + return f"" def __repr__(self) -> str: status = "running" if self.is_running else "stopped" - return f"" - - def __del__(self): - if hasattr(self, "process") and self.process.is_running: - self.process.kill() + return f"" \ No newline at end of file diff --git a/tests/integration/framework/kafka_manager.py b/tests/integration/framework/kafka_manager.py index 071d90d2..e495f638 100644 --- a/tests/integration/framework/kafka_manager.py +++ b/tests/integration/framework/kafka_manager.py @@ -15,51 +15,112 @@ Provides automated image pull, idempotent container start, health check, topic lifecycle management, and data cleanup via KRaft-mode single-node Kafka. - -Usage:: - - mgr = KafkaDockerManager() - mgr.setup_kafka() - mgr.create_topics_if_not_exist(["input-topic", "output-topic"]) - ... - mgr.clear_all_topics() - mgr.teardown_kafka() """ import logging import time -from typing import List +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, TypeVar import docker -from docker.errors import APIError, NotFound +from docker.errors import APIError, DockerException, NotFound +from confluent_kafka import Consumer, KafkaError, KafkaException +from confluent_kafka import TopicPartition as _new_topic_partition from confluent_kafka.admin import AdminClient, NewTopic logger = logging.getLogger(__name__) -_DEFAULT_IMAGE = "apache/kafka:3.7.0" -_DEFAULT_CONTAINER = "fs-integration-kafka-broker" -_DEFAULT_BOOTSTRAP = "127.0.0.1:9092" +T = TypeVar("T") + + +class KafkaDockerManagerError(Exception): + """Base exception for KafkaDockerManager errors.""" + pass + + +class KafkaReadinessTimeoutError(KafkaDockerManagerError): + """Raised when Kafka fails to become ready within the timeout.""" + pass + + +@dataclass(frozen=True) +class KafkaConfig: + """Configuration for the Kafka Docker Container.""" + image: str = "apache/kafka:3.7.0" + container_name: str = "fs-integration-kafka-broker" + bootstrap_servers: str = "127.0.0.1:9092" + internal_port: int = 9092 + controller_port: int = 9093 + cluster_id: str = "fs-integration-test-cluster-01" + readiness_timeout_sec: int = 60 + + @property + def environment_vars(self) -> Dict[str, str]: + """Generate KRaft environment variables.""" + return { + "KAFKA_NODE_ID": "1", + "KAFKA_PROCESS_ROLES": "broker,controller", + "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", + "KAFKA_LISTENERS": ( + f"PLAINTEXT://0.0.0.0:{self.internal_port}," + f"CONTROLLER://0.0.0.0:{self.controller_port}" + ), + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": ( + "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + ), + "KAFKA_ADVERTISED_LISTENERS": f"PLAINTEXT://{self.bootstrap_servers}", + "KAFKA_CONTROLLER_QUORUM_VOTERS": f"1@localhost:{self.controller_port}", + "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", + "KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR": "1", + "KAFKA_TRANSACTION_STATE_LOG_MIN_ISR": "1", + "CLUSTER_ID": self.cluster_id, + } class KafkaDockerManager: """ Manages a single-node Kafka broker inside a Docker container (KRaft mode). - The class is intentionally stateless with respect to topics: every public - method is idempotent so that tests can call ``setup_kafka()`` multiple - times without side-effects. + Designed to be stateless with respect to topics. Highly recommended to use + as a context manager to ensure proper cleanup. + + Usage:: + + with KafkaDockerManager() as mgr: + mgr.create_topics_if_not_exist(["input-topic", "output-topic"]) + # Run tests... + mgr.clear_all_topics() """ def __init__( - self, - image: str = _DEFAULT_IMAGE, - container_name: str = _DEFAULT_CONTAINER, - bootstrap_servers: str = _DEFAULT_BOOTSTRAP, + self, + config: Optional[KafkaConfig] = None, + docker_client: Optional[docker.DockerClient] = None, ) -> None: - self.docker_client = docker.from_env() - self.image_name = image - self.container_name = container_name - self.bootstrap_servers = bootstrap_servers + self.config = config or KafkaConfig() + # Dependency Injection: Allow passing an existing client, or create a lazy one. + self._docker_client = docker_client + + @property + def docker_client(self) -> docker.DockerClient: + """Lazy initialization of the Docker client.""" + if self._docker_client is None: + try: + self._docker_client = docker.from_env() + except DockerException as e: + raise KafkaDockerManagerError(f"Failed to connect to Docker daemon: {e}") from e + return self._docker_client + + # ------------------------------------------------------------------ + # Context Manager Protocol + # ------------------------------------------------------------------ + + def __enter__(self) -> "KafkaDockerManager": + self.setup_kafka() + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.teardown_kafka() # ------------------------------------------------------------------ # Full setup / teardown @@ -67,138 +128,152 @@ def __init__( def setup_kafka(self) -> None: """Pull image -> start container -> wait for readiness.""" + logger.info("Setting up Kafka broker (KRaft)...") self._ensure_image() self._ensure_container() self._wait_for_readiness() + logger.info("Kafka setup complete. Broker is ready.") def teardown_kafka(self) -> None: - """Stop and remove the Kafka container.""" + """Stop and gracefully remove the Kafka container.""" try: - container = self.docker_client.containers.get(self.container_name) - logger.info("Stopping Kafka container '%s' ...", self.container_name) - container.stop() + container = self.docker_client.containers.get(self.config.container_name) + logger.info("Stopping Kafka container '%s'...", self.config.container_name) + container.stop(timeout=5) # Give Kafka a few seconds for graceful shutdown except NotFound: - pass + logger.debug("Container '%s' not found during teardown.", self.config.container_name) except APIError as exc: - logger.warning("Error while stopping Kafka: %s", exc) + logger.warning("Docker API error while stopping Kafka: %s", exc) + except Exception as exc: + logger.error("Unexpected error during teardown: %s", exc) # ------------------------------------------------------------------ - # Image management + # Docker Operations # ------------------------------------------------------------------ def _ensure_image(self) -> None: try: - self.docker_client.images.get(self.image_name) - logger.info("Image '%s' already present locally.", self.image_name) + self.docker_client.images.get(self.config.image) + logger.debug("Image '%s' already present locally.", self.config.image) except NotFound: - logger.info("Pulling Kafka image '%s' ...", self.image_name) - self.docker_client.images.pull(self.image_name) + logger.info("Pulling Kafka image '%s' (this may take a while)...", self.config.image) + self.docker_client.images.pull(self.config.image) logger.info("Image pulled successfully.") - # ------------------------------------------------------------------ - # Container management (KRaft single-node, apache/kafka official image) - # ------------------------------------------------------------------ - def _ensure_container(self) -> None: try: - container = self.docker_client.containers.get(self.container_name) + container = self.docker_client.containers.get(self.config.container_name) if container.status != "running": - logger.info( - "Container '%s' exists but is not running; starting ...", - self.container_name, - ) + logger.info("Starting existing container '%s'...", self.config.container_name) container.start() else: - logger.info( - "Container '%s' is already running.", self.container_name - ) + logger.debug("Container '%s' is already running.", self.config.container_name) except NotFound: - logger.info( - "Creating Kafka container '%s' ...", self.container_name - ) - env = { - "KAFKA_NODE_ID": "1", - "KAFKA_PROCESS_ROLES": "broker,controller", - "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", - "KAFKA_LISTENERS": ( - "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093" - ), - "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": ( - "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" - ), - "KAFKA_ADVERTISED_LISTENERS": ( - f"PLAINTEXT://{self.bootstrap_servers}" - ), - "KAFKA_CONTROLLER_QUORUM_VOTERS": "1@localhost:9093", - "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", - "KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR": "1", - "KAFKA_TRANSACTION_STATE_LOG_MIN_ISR": "1", - "CLUSTER_ID": "fs-integration-test-cluster-01", - } + logger.info("Creating and starting new Kafka container '%s'...", self.config.container_name) self.docker_client.containers.run( - image=self.image_name, - name=self.container_name, - ports={"9092/tcp": 9092}, - environment=env, + image=self.config.image, + name=self.config.container_name, + ports={f"{self.config.internal_port}/tcp": self.config.internal_port}, + environment=self.config.environment_vars, detach=True, - remove=True, + remove=True, # Auto-remove on stop ) - def _wait_for_readiness(self, timeout: int = 60) -> None: - logger.info( - "Waiting for Kafka to become ready at %s ...", - self.bootstrap_servers, - ) - deadline = time.time() + timeout + # ------------------------------------------------------------------ + # Readiness Probes + # ------------------------------------------------------------------ + + def _retry_until_ready( + self, + action: Callable[[], bool], + timeout_msg: str, + interval: float = 1.0 + ) -> None: + """Generic polling mechanism with timeout.""" + deadline = time.time() + self.config.readiness_timeout_sec while time.time() < deadline: + if action(): + return + time.sleep(interval) + raise KafkaReadinessTimeoutError(f"{timeout_msg} after {self.config.readiness_timeout_sec}s.") + + def _wait_for_readiness(self) -> None: + """Wait for AdminClient to list topics and Coordinator to be ready.""" + logger.info("Waiting for Kafka API to become responsive at %s...", self.config.bootstrap_servers) + + def _is_api_ready() -> bool: try: - admin = AdminClient( - {"bootstrap.servers": self.bootstrap_servers} - ) + admin = AdminClient({"bootstrap.servers": self.config.bootstrap_servers}) admin.list_topics(timeout=2) - logger.info("Kafka is ready.") - return - except Exception: - time.sleep(1) - raise TimeoutError( - f"Kafka did not become ready within {timeout}s. " - "Check Docker logs for details." - ) + return True + except KafkaException: + return False + + self._retry_until_ready(_is_api_ready, "Kafka API did not become responsive") + + logger.info("Broker API is up. Verifying group coordinator...") + self._wait_for_group_coordinator() + + def _wait_for_group_coordinator(self) -> None: + """Ensures __consumer_offsets is initialized and coordinator is ready.""" + def _is_coordinator_ready() -> bool: + consumer = None + try: + consumer = Consumer({ + "bootstrap.servers": self.config.bootstrap_servers, + "group.id": "__readiness_probe__", + "session.timeout.ms": "6000", + }) + # Attempt to read committed offsets to trigger coordinator interaction + consumer.committed([_new_topic_partition("__consumer_offsets", 0)], timeout=2) + return True + except KafkaException: + return False + except Exception as e: + logger.debug("Unexpected error during coordinator probe: %s", e) + return False + finally: + if consumer is not None: + consumer.close() + + self._retry_until_ready(_is_coordinator_ready, "Group coordinator did not become ready") # ------------------------------------------------------------------ # Topic management # ------------------------------------------------------------------ def create_topic( - self, - topic_name: str, - num_partitions: int = 1, - replication_factor: int = 1, + self, + topic_name: str, + num_partitions: int = 1, + replication_factor: int = 1, ) -> None: - """ - Create a Kafka topic idempotently. - - If the topic already exists the call succeeds silently. - """ - admin = AdminClient({"bootstrap.servers": self.bootstrap_servers}) + """Create a Kafka topic idempotently.""" + admin = AdminClient({"bootstrap.servers": self.config.bootstrap_servers}) new_topic = NewTopic( topic_name, num_partitions=num_partitions, replication_factor=replication_factor, ) + futures = admin.create_topics([new_topic], operation_timeout=5) for topic, future in futures.items(): try: future.result() logger.info("Created topic '%s'.", topic) - except Exception as exc: - if "TOPIC_ALREADY_EXISTS" in str(exc): + except KafkaException as exc: + kafka_error = exc.args[0] + if kafka_error.code() == KafkaError.TOPIC_ALREADY_EXISTS: logger.debug("Topic '%s' already exists; skipping.", topic) else: + logger.error("Failed to create topic '%s': %s", topic, kafka_error) raise + except Exception as exc: + logger.error("Unexpected error creating topic '%s': %s", topic, exc) + raise def create_topics_if_not_exist( - self, topic_names: List[str], num_partitions: int = 1 + self, topic_names: List[str], num_partitions: int = 1 ) -> None: """Batch-create topics idempotently.""" for topic in topic_names: @@ -206,16 +281,24 @@ def create_topics_if_not_exist( def clear_all_topics(self) -> None: """Delete every non-internal topic (fast data reset between tests).""" - admin = AdminClient({"bootstrap.servers": self.bootstrap_servers}) + admin = AdminClient({"bootstrap.servers": self.config.bootstrap_servers}) try: metadata = admin.list_topics(timeout=5) to_delete = [ t for t in metadata.topics if not t.startswith("__") ] - if to_delete: - logger.debug("Deleting leftover topics: %s", to_delete) - futures = admin.delete_topics(to_delete, operation_timeout=5) - for _topic, fut in futures.items(): + if not to_delete: + logger.debug("No user topics to clean up.") + return + + logger.info("Deleting topics: %s", to_delete) + futures = admin.delete_topics(to_delete, operation_timeout=5) + + for topic, fut in futures.items(): + try: fut.result() + logger.debug("Deleted topic '%s'.", topic) + except KafkaException as exc: + logger.warning("Failed to delete topic '%s': %s", topic, exc.args[0]) except Exception as exc: - logger.warning("Topic cleanup failed: %s", exc) + logger.warning("Topic cleanup process encountered an error: %s", exc) \ No newline at end of file diff --git a/tests/integration/framework/process.py b/tests/integration/framework/process.py index 6b04de67..1c81ba02 100644 --- a/tests/integration/framework/process.py +++ b/tests/integration/framework/process.py @@ -19,23 +19,42 @@ import os import signal import subprocess +import sys from pathlib import Path -from typing import Optional +from typing import IO, Any, Dict, Optional from .workspace import InstanceWorkspace logger = logging.getLogger(__name__) +class ProcessManagerError(Exception): + """Base exception for process management errors.""" + pass + + +class BinaryNotFoundError(ProcessManagerError): + """Raised when the target binary executable does not exist.""" + pass + + +class ProcessAlreadyRunningError(ProcessManagerError): + """Raised when attempting to start a process that is already running.""" + pass + + class FunctionStreamProcess: - """Manages the lifecycle of a single FunctionStream OS process.""" + """ + Manages the lifecycle of a single FunctionStream OS process safely. + Fully cross-platform compatible (Windows, macOS, Linux). + """ def __init__(self, binary_path: Path, workspace: InstanceWorkspace): - self._binary = binary_path + self._binary = binary_path.resolve() self._workspace = workspace - self._process: Optional[subprocess.Popen] = None - self._stdout_fh = None - self._stderr_fh = None + self._process: Optional[subprocess.Popen[Any]] = None + self._stdout_fh: Optional[IO[Any]] = None + self._stderr_fh: Optional[IO[Any]] = None @property def pid(self) -> Optional[int]: @@ -43,74 +62,124 @@ def pid(self) -> Optional[int]: @property def is_running(self) -> bool: - return self._process is not None and self._process.poll() is None + """Check if the process is currently running without blocking.""" + if self._process is None: + return False + # poll() returns None if process is still running, else returns exit code + return self._process.poll() is None def start(self) -> None: - """Launch the binary, redirecting stdout/stderr to log files.""" - if not self._binary.exists(): - raise FileNotFoundError( + """Launch the binary, redirecting stdout/stderr to log files safely.""" + if self.is_running: + raise ProcessAlreadyRunningError( + f"Process is already running with PID {self.pid}." + ) + + if not self._binary.is_file(): + raise BinaryNotFoundError( f"Binary not found: {self._binary}. " "Run 'make integration-build' first." ) - env = os.environ.copy() + env: Dict[str, str] = os.environ.copy() env["FUNCTION_STREAM_CONF"] = str(self._workspace.config_file) env["FUNCTION_STREAM_HOME"] = str(self._workspace.root_dir) - self._stdout_fh = open(self._workspace.stdout_file, "w") - self._stderr_fh = open(self._workspace.stderr_file, "w") + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + popen_kwargs["preexec_fn"] = os.setsid + try: + self._stdout_fh = open(self._workspace.stdout_file, "w", encoding="utf-8") + self._stderr_fh = open(self._workspace.stderr_file, "w", encoding="utf-8") + + self._process = subprocess.Popen( + [str(self._binary)], + env=env, + cwd=str(self._workspace.root_dir), + stdout=self._stdout_fh, + stderr=self._stderr_fh, + **popen_kwargs, + ) + logger.info("Process started successfully [pid=%d]", self._process.pid) - self._process = subprocess.Popen( - [str(self._binary)], - env=env, - cwd=str(self._workspace.root_dir), - stdout=self._stdout_fh, - stderr=self._stderr_fh, - preexec_fn=os.setsid if os.name != "nt" else None, - ) - logger.info("Process started [pid=%d]", self._process.pid) + except Exception as e: + self._close_handles() + logger.error("Failed to start process: %s", e) + raise ProcessManagerError(f"Process initialization failed: {e}") from e def stop(self, timeout: float = 10.0) -> None: - """Graceful shutdown via SIGTERM, falls back to SIGKILL on timeout.""" - if not self.is_running: + """ + Graceful shutdown via SIGTERM (POSIX) or CTRL_BREAK (Windows). + Falls back to immediate Kill on timeout. + """ + if not self.is_running or self._process is None: self._close_handles() return - logger.info("Sending SIGTERM [pid=%d]", self._process.pid) + logger.debug("Attempting graceful shutdown [pid=%d]...", self._process.pid) try: - os.killpg(os.getpgid(self._process.pid), signal.SIGTERM) + self._send_termination_signal() self._process.wait(timeout=timeout) + logger.info("Process stopped gracefully [pid=%d].", self._process.pid) except (ProcessLookupError, PermissionError): + # Process already died or OS denied access pass except subprocess.TimeoutExpired: - logger.warning("SIGTERM timed out, escalating to SIGKILL [pid=%d]", self._process.pid) + logger.warning("Graceful shutdown timed out (%.1fs). Escalating to Kill [pid=%d].", + timeout, self._process.pid) self.kill() return - - self._close_handles() + finally: + self._close_handles() def kill(self) -> None: - """Immediately SIGKILL the process group.""" - if not self.is_running: + """Immediately terminate the process group across all platforms.""" + if not self.is_running or self._process is None: self._close_handles() return - logger.info("Sending SIGKILL [pid=%d]", self._process.pid) + logger.debug("Executing force kill [pid=%d]...", self._process.pid) try: - os.killpg(os.getpgid(self._process.pid), signal.SIGKILL) - self._process.wait(timeout=5) + self._send_kill_signal() + self._process.wait(timeout=5.0) + logger.info("Process forcefully killed [pid=%d].", self._process.pid) except (ProcessLookupError, PermissionError): pass except subprocess.TimeoutExpired: + logger.error("Process group kill timed out. Falling back to base kill.") self._process.kill() - self._process.wait(timeout=5) + self._process.wait(timeout=2.0) + finally: + self._close_handles() - self._close_handles() + # ------------------------------------------------------------------ + # OS-Specific Signal Implementations + # ------------------------------------------------------------------ + + def _send_termination_signal(self) -> None: + """Cross-platform implementation for graceful termination.""" + if sys.platform == "win32": + self._process.send_signal(signal.CTRL_BREAK_EVENT) + else: + os.killpg(os.getpgid(self._process.pid), signal.SIGTERM) + + def _send_kill_signal(self) -> None: + if sys.platform == "win32": + self._process.kill() + else: + os.killpg(os.getpgid(self._process.pid), signal.SIGKILL) def _close_handles(self) -> None: + """Safely close open file handles and clear state.""" for fh in (self._stdout_fh, self._stderr_fh): - if fh and not fh.closed: - fh.close() + if fh is not None and not fh.closed: + try: + fh.close() + except Exception as e: + logger.debug("Failed to close file handle: %s", e) + self._stdout_fh = None self._stderr_fh = None - self._process = None + self._process = None \ No newline at end of file diff --git a/tests/integration/framework/utils.py b/tests/integration/framework/utils.py index d4783a81..2ec125ed 100644 --- a/tests/integration/framework/utils.py +++ b/tests/integration/framework/utils.py @@ -11,34 +11,102 @@ # limitations under the License. """ -Stateless utility functions: port allocation, health checks. +Stateless utility functions: resilient port allocation and health checks. """ -import random +import logging import socket import time +from typing import Optional +logger = logging.getLogger(__name__) -def find_free_port(start: int = 18000, end: int = 28000) -> int: - """Find a random available TCP port in the given range.""" - for _ in range(200): - port = random.randint(start, end) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("127.0.0.1", port)) - return port - except OSError: - continue - raise RuntimeError(f"Could not find a free port in [{start}, {end}]") +class NetworkUtilityError(Exception): + """Base exception for network utility functions.""" + pass -def wait_for_port(host: str, port: int, timeout: float = 30.0) -> bool: - """Block until the given TCP port is accepting connections.""" + +class PortAllocationError(NetworkUtilityError): + """Raised when a free port cannot be allocated.""" + pass + + +def find_free_port(host: str = "127.0.0.1", port_range: Optional[tuple[int, int]] = None) -> int: + """ + Finds a reliable, available TCP port. + + By default, relies on the OS kernel to allocate an ephemeral port (binding to port 0). + This guarantees no immediate collision and is extremely fast. + + Args: + host: The interface IP to bind against. + port_range: Optional tuple of (start, end). If provided, falls back to scanning + this specific range instead of OS assignment. + + Raises: + PortAllocationError: If binding fails or no ports are available in the range. + """ + if port_range is not None: + start, end = port_range + import random + candidates = list(range(start, end + 1)) + random.shuffle(candidates) + + for port in candidates: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + s.bind((host, port)) + return port + except OSError: + continue + + raise PortAllocationError(f"Exhausted all ports in requested range [{start}, {end}].") + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + s.bind((host, 0)) + allocated_port = s.getsockname()[1] + return allocated_port + except OSError as e: + raise PortAllocationError(f"OS failed to allocate an ephemeral port on {host}: {e}") from e + + +def wait_for_port( + host: str, + port: int, + timeout: float = 30.0, + poll_interval: float = 0.5 +) -> bool: + """ + Blocks until the given TCP port is accepting connections. + + Uses `socket.create_connection` for robust DNS resolution and + transparent IPv4/IPv6 dual-stack support. + + Args: + host: Hostname or IP address. + port: Target TCP port. + timeout: Max seconds to wait. + poll_interval: Seconds to wait between retries. + + Returns: + True if connection succeeded within timeout, False otherwise. + """ + logger.debug("Waiting up to %.1fs for %s:%d to become responsive...", timeout, host, port) deadline = time.monotonic() + timeout + while time.monotonic() < deadline: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(0.5) - if s.connect_ex((host, port)) == 0: + try: + with socket.create_connection((host, port), timeout=poll_interval): + logger.debug("Successfully connected to %s:%d", host, port) return True - time.sleep(0.3) - return False + except (OSError, ConnectionRefusedError): + pass + + time.sleep(poll_interval) + + logger.error("Timeout reached: %s:%d did not accept connections after %.1fs.", host, port, timeout) + return False \ No newline at end of file diff --git a/tests/integration/framework/workspace.py b/tests/integration/framework/workspace.py index b96ebb96..515a6918 100644 --- a/tests/integration/framework/workspace.py +++ b/tests/integration/framework/workspace.py @@ -15,7 +15,7 @@ FunctionStream test instance. Layout (during test run): - tests/integration/target/// + tests/integration/target//--/ conf/config.yaml data/ logs/stdout.log, stderr.log, app.log @@ -23,23 +23,38 @@ After cleanup only ``logs/`` is retained. """ +import logging import shutil +import time +import uuid from datetime import datetime from pathlib import Path +logger = logging.getLogger(__name__) + + +class WorkspaceError(Exception): + """Base exception for workspace-related errors.""" + pass + class InstanceWorkspace: - """Owns the on-disk directory environment for one FunctionStream instance.""" + """ + Owns the on-disk directory environment for one FunctionStream instance. + Designed for safe parallel execution and cross-platform robustness. + """ def __init__(self, target_dir: Path, test_name: str, port: int): - self.target_dir = target_dir + self.target_dir = target_dir.resolve() self.test_name = test_name self.port = port - self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - self.root_dir = ( - self.target_dir / self.test_name / self.timestamp - ) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + unique_id = uuid.uuid4().hex[:6] + self.instance_id = f"{timestamp}-{port}-{unique_id}" + + self.root_dir = self.target_dir / self.test_name / self.instance_id + self.conf_dir = self.root_dir / "conf" self.data_dir = self.root_dir / "data" self.log_dir = self.root_dir / "logs" @@ -49,12 +64,46 @@ def __init__(self, target_dir: Path, test_name: str, port: int): self.stderr_file = self.log_dir / "stderr.log" def setup(self) -> None: - """Create the full directory tree.""" - for d in (self.conf_dir, self.data_dir, self.log_dir): - d.mkdir(parents=True, exist_ok=True) + """Create the full directory tree safely.""" + logger.debug("Setting up workspace at %s", self.root_dir) + try: + for d in (self.conf_dir, self.data_dir, self.log_dir): + d.mkdir(parents=True, exist_ok=True) + except OSError as e: + logger.error("Failed to create workspace directories: %s", e) + raise WorkspaceError(f"Workspace setup failed: {e}") from e def cleanup(self) -> None: - """Remove everything except logs/ so only diagnostic output remains.""" + """ + Remove everything except logs/ so only diagnostic output remains. + Uses a robust deletion strategy to handle temporary OS file locks. + """ + logger.debug("Cleaning up workspace (retaining logs) at %s", self.root_dir) for d in (self.conf_dir, self.data_dir): if d.exists(): - shutil.rmtree(d) + self._safe_rmtree(d) + + def _safe_rmtree(self, path: Path, retries: int = 3, delay: float = 0.5) -> None: + """ + Robust directory removal with backoff retries. + Crucial for Windows where RocksDB/LevelDB files might hold residual locks + for a few milliseconds after the parent process dies. + """ + for attempt in range(1, retries + 1): + try: + shutil.rmtree(path, ignore_errors=False) + return + except OSError as e: + if attempt < retries: + logger.debug( + "File lock detected during cleanup of %s (attempt %d/%d). Retrying in %ss...", + path, attempt, retries, delay + ) + time.sleep(delay) + else: + logger.warning( + "Failed to completely remove %s after %d attempts. " + "Leaving residual files. Error: %s", + path, retries, e + ) + shutil.rmtree(path, ignore_errors=True) \ No newline at end of file diff --git a/tests/integration/test/wasm/python_sdk/conftest.py b/tests/integration/test/wasm/python_sdk/conftest.py index 264484d6..aa5d60c6 100644 --- a/tests/integration/test/wasm/python_sdk/conftest.py +++ b/tests/integration/test/wasm/python_sdk/conftest.py @@ -10,104 +10,71 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Global pytest fixtures for the FunctionStream Python SDK integration tests. -Provides managed Kafka broker, server instances, client connections, -and automated resource cleanup. -""" - +import logging +import re import sys from pathlib import Path from typing import Generator, List -import pytest - -# tests/integration/test/wasm/python_sdk -> parents[3] = tests/integration -sys.path.insert(0, str(Path(__file__).resolve().parents[3])) -# Make the ``processors`` package importable from this directory -sys.path.insert(0, str(Path(__file__).resolve().parent)) +_CURRENT_DIR = Path(__file__).resolve().parent +_INTEGRATION_ROOT = Path(__file__).resolve().parents[3] -from framework import FunctionStreamInstance # noqa: E402 -from fs_client.client import FsClient # noqa: E402 +if str(_INTEGRATION_ROOT) not in sys.path: + sys.path.insert(0, str(_INTEGRATION_ROOT)) +if str(_CURRENT_DIR) not in sys.path: + sys.path.insert(0, str(_CURRENT_DIR)) -PROJECT_ROOT = Path(__file__).resolve().parents[5] +import pytest +from framework import FunctionStreamInstance, KafkaDockerManager +from fs_client.client import FsClient +logger = logging.getLogger(__name__) -# ====================================================================== -# Kafka broker (opt-in, independent of fs_server) -# ====================================================================== @pytest.fixture(scope="session") -def kafka(): +def kafka() -> Generator[KafkaDockerManager, None, None]: """ - Session-scoped fixture: start a Docker-managed Kafka broker once - for the entire test session. - - Tests that need a live Kafka broker should declare this fixture - as a parameter. Tests that only register functions (without - producing / consuming data) do NOT need this fixture. + Session-scoped Kafka broker manager. + Leverages Context Manager for guaranteed teardown. """ - from framework import KafkaDockerManager - - mgr = KafkaDockerManager() - mgr.setup_kafka() - yield mgr - mgr.clear_all_topics() - mgr.teardown_kafka() - + with KafkaDockerManager() as mgr: + yield mgr + try: + mgr.clear_all_topics() + except Exception as e: + logger.warning("Failed to clear topics during Kafka teardown: %s", e) -# ====================================================================== -# Kafka topics (depends on kafka broker) -# ====================================================================== @pytest.fixture(scope="session") -def kafka_topics(kafka): +def kafka_topics(kafka: KafkaDockerManager) -> str: """ - Session-scoped fixture: ensure the Kafka broker is running and - pre-create the standard input/output topics used by tests. - - Returns the bootstrap servers string for use in WasmTaskBuilder configs. + Pre-creates standard topics and returns the bootstrap server address. """ kafka.create_topics_if_not_exist(["in", "out", "events", "counts"]) - return kafka.bootstrap_servers + return kafka.config.bootstrap_servers -# ====================================================================== -# FunctionStream server (per-test instance with isolated logs) -# ====================================================================== - -def _derive_test_name(request: pytest.FixtureRequest) -> str: - """Build a human-readable path from the pytest node: wasm/python_sdk//.""" - cls = request.node.cls - parts = ["wasm", "python_sdk"] - if cls is not None: - parts.append(cls.__name__) - parts.append(request.node.name) - return "/".join(parts) +def _sanitize_node_id(nodeid: str) -> str: + """Converts a pytest nodeid into a safe directory name.""" + clean_name = re.sub(r"[^\w\-]+", "-", nodeid) + return clean_name.strip("-") @pytest.fixture def fs_server(request: pytest.FixtureRequest) -> Generator[FunctionStreamInstance, None, None]: """ - Function-scoped fixture: each test gets its own FunctionStream server - with isolated log directory named after the test class and method. + Function-scoped FunctionStream instance. + Uses Context Manager to ensure SIGKILL and workspace cleanup. """ - test_name = _derive_test_name(request) - instance = FunctionStreamInstance(test_name=test_name) - instance.start() - yield instance - instance.kill() - + test_name = _sanitize_node_id(request.node.nodeid) + with FunctionStreamInstance(test_name=test_name) as instance: + yield instance -# ====================================================================== -# Client & resource tracking -# ====================================================================== @pytest.fixture def fs_client(fs_server: FunctionStreamInstance) -> Generator[FsClient, None, None]: """ - Function-scoped fixture: provide a fresh client connected to the server. - The connection is automatically closed after each test. + Function-scoped FsClient connected to the isolated fs_server. """ with fs_server.get_client() as client: yield client @@ -116,9 +83,8 @@ def fs_client(fs_server: FunctionStreamInstance) -> Generator[FsClient, None, No @pytest.fixture def function_registry(fs_client: FsClient) -> Generator[List[str], None, None]: """ - RAII Resource Manager: tracks registered function names. - Automatically stops and drops every tracked function after each test, - guaranteeing environment idempotency regardless of assertion failures. + RAII-style registry for FunctionStream tasks. + Ensures absolute teardown of functions to prevent state leakage. """ registered_names: List[str] = [] @@ -127,9 +93,10 @@ def function_registry(fs_client: FsClient) -> Generator[List[str], None, None]: for name in registered_names: try: fs_client.stop_function(name) - except Exception: - pass + except Exception as e: + logger.debug("Failed to stop function '%s' during cleanup: %s", name, e) + try: fs_client.drop_function(name) - except Exception: - pass + except Exception as e: + logger.error("Failed to drop function '%s' during cleanup: %s", name, e) \ No newline at end of file diff --git a/tests/integration/test/wasm/python_sdk/test_data_flow.py b/tests/integration/test/wasm/python_sdk/test_data_flow.py new file mode 100644 index 00000000..9e9532a2 --- /dev/null +++ b/tests/integration/test/wasm/python_sdk/test_data_flow.py @@ -0,0 +1,242 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import time +import uuid +from dataclasses import dataclass +from typing import Any, Dict, List + +from confluent_kafka import Consumer, KafkaError, Producer +from fs_client.client import FsClient +from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder +from processors.counter_processor import CounterProcessor + +logger = logging.getLogger(__name__) + +CONSUME_TIMEOUT_S = 60.0 +POLL_INTERVAL_S = 0.5 +CONSUMER_WARMUP_S = 3.0 + + +@dataclass(frozen=True) +class FlowContext: + fn_name: str + in_topic: str + out_topic: str + + +def _unique_id(prefix: str) -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +def _build_earliest_input(bootstrap: str, topic: str, group: str) -> KafkaInput: + ki = KafkaInput(bootstrap, topic, group, 0) + ki.data["auto.offset.reset"] = "earliest" + return ki + + +def produce_messages(bootstrap: str, topic: str, messages: List[str], timeout: float = 10.0) -> None: + producer = Producer({"bootstrap.servers": bootstrap}) + try: + for msg in messages: + producer.produce(topic, value=msg.encode("utf-8")) + finally: + remaining = producer.flush(timeout=timeout) + if remaining > 0: + raise RuntimeError(f"Producer failed to flush {remaining} messages within {timeout}s") + + +def consume_messages( + bootstrap: str, + topic: str, + expected_count: int, + timeout: float = CONSUME_TIMEOUT_S, +) -> List[Dict[str, Any]]: + consumer = Consumer({ + "bootstrap.servers": bootstrap, + "group.id": _unique_id("test-consumer"), + "auto.offset.reset": "earliest", + "enable.auto.commit": "false", + }) + consumer.subscribe([topic]) + collected: List[Dict[str, Any]] = [] + deadline = time.time() + timeout + + try: + while len(collected) < expected_count and time.time() < deadline: + msg = consumer.poll(timeout=POLL_INTERVAL_S) + if msg is None: + continue + if msg.error(): + if msg.error().code() != KafkaError._PARTITION_EOF: + logger.error("Kafka consumer error: %s", msg.error()) + continue + + payload = msg.value().decode("utf-8") + collected.append(json.loads(payload)) + finally: + consumer.close() + + if len(collected) < expected_count: + raise TimeoutError(f"Expected {expected_count} messages, received {len(collected)}") + + return collected + + +def deploy_function( + fs_client: FsClient, + fn_name: str, + bootstrap: str, + in_topic: str, + out_topic: str, +) -> None: + config = ( + WasmTaskBuilder() + .set_name(fn_name) + .add_init_config("class_name", "CounterProcessor") + .add_input_group([_build_earliest_input(bootstrap, in_topic, fn_name)]) + .add_output(KafkaOutput(bootstrap, out_topic, 0)) + .build() + ) + + success = fs_client.create_python_function_from_config(config, CounterProcessor) + if not success: + raise RuntimeError(f"Failed to deploy function: {fn_name}") + + time.sleep(CONSUMER_WARMUP_S) + + +class TestDataFlow: + + def _setup_flow(self, function_registry: List[str], kafka: Any, prefix: str) -> FlowContext: + fn_name = _unique_id(prefix) + ctx = FlowContext( + fn_name=fn_name, + in_topic=f"{fn_name}-in", + out_topic=f"{fn_name}-out" + ) + function_registry.append(ctx.fn_name) + kafka.create_topics_if_not_exist([ctx.in_topic, ctx.out_topic]) + return ctx + + def test_single_word_counting( + self, + fs_client: FsClient, + function_registry: List[str], + kafka: Any, + kafka_topics: str, + ): + ctx = self._setup_flow(function_registry, kafka, "flow-single") + word = "hello" + n = 10 + + produce_messages(kafka_topics, ctx.in_topic, [word] * n) + deploy_function(fs_client, ctx.fn_name, kafka_topics, ctx.in_topic, ctx.out_topic) + records = consume_messages(kafka_topics, ctx.out_topic, n) + + for i, rec in enumerate(records, start=1): + assert rec["word"] == word + assert rec["count"] == i + assert rec["total"] == i + + def test_multiple_distinct_words( + self, + fs_client: FsClient, + function_registry: List[str], + kafka: Any, + kafka_topics: str, + ): + ctx = self._setup_flow(function_registry, kafka, "flow-multi") + messages = ["apple", "banana", "apple", "cherry", "banana", "apple"] + + produce_messages(kafka_topics, ctx.in_topic, messages) + deploy_function(fs_client, ctx.fn_name, kafka_topics, ctx.in_topic, ctx.out_topic) + records = consume_messages(kafka_topics, ctx.out_topic, len(messages)) + + per_word_counts: Dict[str, int] = {} + for rec in records: + word = rec["word"] + per_word_counts[word] = per_word_counts.get(word, 0) + 1 + assert rec["count"] == per_word_counts[word] + + assert per_word_counts == {"apple": 3, "banana": 2, "cherry": 1} + + def test_large_batch_throughput( + self, + fs_client: FsClient, + function_registry: List[str], + kafka: Any, + kafka_topics: str, + ): + ctx = self._setup_flow(function_registry, kafka, "flow-batch") + batch_size = 500 + messages = [f"item-{i % 50}" for i in range(batch_size)] + + produce_messages(kafka_topics, ctx.in_topic, messages) + deploy_function(fs_client, ctx.fn_name, kafka_topics, ctx.in_topic, ctx.out_topic) + records = consume_messages(kafka_topics, ctx.out_topic, batch_size) + + assert len(records) == batch_size + totals = [r["total"] for r in records] + assert totals == list(range(1, batch_size + 1)) + + per_word: Dict[str, int] = {} + for rec in records: + word = rec["word"] + per_word[word] = per_word.get(word, 0) + 1 + assert rec["count"] == per_word[word] + + def test_empty_messages_are_skipped( + self, + fs_client: FsClient, + function_registry: List[str], + kafka: Any, + kafka_topics: str, + ): + ctx = self._setup_flow(function_registry, kafka, "flow-empty") + messages = ["foo", "", "bar", " ", "foo", ""] + + produce_messages(kafka_topics, ctx.in_topic, messages) + deploy_function(fs_client, ctx.fn_name, kafka_topics, ctx.in_topic, ctx.out_topic) + + expected_outputs = 3 + records = consume_messages(kafka_topics, ctx.out_topic, expected_outputs) + + words = [r["word"] for r in records] + assert words == ["foo", "bar", "foo"] + assert records[0]["count"] == 1 + assert records[1]["count"] == 1 + assert records[2]["count"] == 2 + + def test_output_json_schema( + self, + fs_client: FsClient, + function_registry: List[str], + kafka: Any, + kafka_topics: str, + ): + ctx = self._setup_flow(function_registry, kafka, "flow-schema") + test_words = ["alpha", "beta", "gamma"] + + produce_messages(kafka_topics, ctx.in_topic, test_words) + deploy_function(fs_client, ctx.fn_name, kafka_topics, ctx.in_topic, ctx.out_topic) + records = consume_messages(kafka_topics, ctx.out_topic, len(test_words)) + + for rec in records: + assert set(rec.keys()) == {"word", "count", "total"} + assert isinstance(rec["word"], str) + assert isinstance(rec["count"], int) + assert isinstance(rec["total"], int) + assert rec["count"] >= 1 + assert rec["total"] >= 1 \ No newline at end of file diff --git a/tests/integration/test/wasm/python_sdk/test_lifecycle.py b/tests/integration/test/wasm/python_sdk/test_lifecycle.py index 188bb0f6..34c8dfbe 100644 --- a/tests/integration/test/wasm/python_sdk/test_lifecycle.py +++ b/tests/integration/test/wasm/python_sdk/test_lifecycle.py @@ -10,31 +10,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Integration tests for basic CRUD operations and status state-machine transitions. - -Covers: - - Full lifecycle: Create -> Show -> Stop -> Start -> Stop -> Drop - - Multiple function coexistence - - Rapid create / drop cycling - - show_functions listing correctness -""" - import uuid -from typing import List +from typing import Any, List, Optional from fs_client.client import FsClient -from fs_client.config import WasmTaskBuilder, KafkaInput, KafkaOutput - +from fs_client.config import KafkaInput, KafkaOutput, WasmTaskBuilder from processors.counter_processor import CounterProcessor +EXPECTED_STOPPED_STATES = frozenset(["STOPPED", "PAUSED", "INITIALIZED"]) +EXPECTED_RUNNING_STATES = frozenset(["RUNNING"]) -def _unique(prefix: str) -> str: + +def _generate_unique_name(prefix: str) -> str: return f"{prefix}-{uuid.uuid4().hex[:8]}" -def _build_counter_config(fn_name: str, bootstrap: str) -> "WasmTaskBuilder": - """Return a ready-to-build builder pre-configured for CounterProcessor.""" +def _create_counter_task_builder(fn_name: str, bootstrap: str) -> WasmTaskBuilder: return ( WasmTaskBuilder() .set_name(fn_name) @@ -44,176 +35,135 @@ def _build_counter_config(fn_name: str, bootstrap: str) -> "WasmTaskBuilder": ) -class TestFunctionLifecycle: - """ - Integration tests for basic CRUD operations and status state machine - transitions. - """ +def _get_function_info(fs_client: FsClient, fn_name: str) -> Optional[Any]: + listing = fs_client.show_functions() + for fn in listing.functions: + if fn.name == fn_name: + return fn + return None + - # ------------------------------------------------------------------ - # Golden-path lifecycle - # ------------------------------------------------------------------ +class TestFunctionLifecycle: def test_full_lifecycle_transitions( - self, fs_client: FsClient, function_registry: List[str], kafka_topics: str - ): - """Test the golden path: Create -> Show -> Stop -> Start -> Stop -> Drop.""" - fn_name = _unique("lifecycle") + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str + ) -> None: + fn_name = _generate_unique_name("lifecycle") function_registry.append(fn_name) - config = _build_counter_config(fn_name, kafka_topics).add_init_config("test_mode", "true").build() + config = _create_counter_task_builder(fn_name, kafka_topics).add_init_config("test_mode", "true").build() - # 1. Create assert fs_client.create_python_function_from_config(config, CounterProcessor) is True - # 2. Verify visibility - listing = fs_client.show_functions() - fn_info = next((f for f in listing.functions if f.name == fn_name), None) - assert fn_info is not None, "Function must be listed after creation" - assert fn_info.status, "Status must be a non-empty string" + fn_info = _get_function_info(fs_client, fn_name) + assert fn_info is not None + assert bool(fn_info.status) - # 3. Stop (server may auto-start on creation; stop first to get a known state) assert fs_client.stop_function(fn_name) is True - # 4. Start assert fs_client.start_function(fn_name) is True - listing = fs_client.show_functions() - fn_info = next(f for f in listing.functions if f.name == fn_name) - assert fn_info.status.upper() == "RUNNING", ( - f"Expected RUNNING after start, got {fn_info.status}" - ) + fn_info = _get_function_info(fs_client, fn_name) + assert fn_info is not None + assert fn_info.status.upper() in EXPECTED_RUNNING_STATES - # 5. Stop again assert fs_client.stop_function(fn_name) is True - listing = fs_client.show_functions() - fn_info = next(f for f in listing.functions if f.name == fn_name) - assert fn_info.status.upper() in ("STOPPED", "PAUSED", "INITIALIZED"), ( - f"Expected STOPPED/PAUSED after stop, got {fn_info.status}" - ) + fn_info = _get_function_info(fs_client, fn_name) + assert fn_info is not None + assert fn_info.status.upper() in EXPECTED_STOPPED_STATES - # 6. Drop assert fs_client.drop_function(fn_name) is True - listing = fs_client.show_functions() - assert fn_name not in [f.name for f in listing.functions], ( - "Function must be removed from registry after drop" - ) - function_registry.remove(fn_name) + assert _get_function_info(fs_client, fn_name) is None - # ------------------------------------------------------------------ - # show_functions consistency - # ------------------------------------------------------------------ + if fn_name in function_registry: + function_registry.remove(fn_name) def test_show_functions_returns_created_function( - self, fs_client: FsClient, function_registry: List[str], kafka_topics: str - ): - """show_functions must contain the newly created function.""" - fn_name = _unique("show") + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str + ) -> None: + fn_name = _generate_unique_name("show") function_registry.append(fn_name) - config = _build_counter_config(fn_name, kafka_topics).build() + config = _create_counter_task_builder(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) - listing = fs_client.show_functions() - names = [f.name for f in listing.functions] - assert fn_name in names + assert _get_function_info(fs_client, fn_name) is not None def test_show_functions_result_fields( - self, fs_client: FsClient, function_registry: List[str], kafka_topics: str - ): - """Each FunctionInfo must carry name, task_type, and status.""" - fn_name = _unique("fields") + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str + ) -> None: + fn_name = _generate_unique_name("fields") function_registry.append(fn_name) - config = _build_counter_config(fn_name, kafka_topics).build() + config = _create_counter_task_builder(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) - listing = fs_client.show_functions() - fn_info = next(f for f in listing.functions if f.name == fn_name) - + fn_info = _get_function_info(fs_client, fn_name) + assert fn_info is not None assert fn_info.name == fn_name - assert fn_info.task_type, "task_type must not be empty" - assert fn_info.status, "status must not be empty" - - # ------------------------------------------------------------------ - # Multiple function coexistence - # ------------------------------------------------------------------ + assert bool(fn_info.task_type) + assert bool(fn_info.status) def test_multiple_functions_coexist( - self, fs_client: FsClient, function_registry: List[str], kafka_topics: str - ): - """Several independently created functions must all be listed.""" - names = [_unique("multi") for _ in range(3)] + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str + ) -> None: + names = [_generate_unique_name("multi") for _ in range(3)] function_registry.extend(names) for name in names: - config = _build_counter_config(name, kafka_topics).build() + config = _create_counter_task_builder(name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) listing = fs_client.show_functions() listed_names = {f.name for f in listing.functions} - for name in names: - assert name in listed_names, f"{name} missing from listing" - # ------------------------------------------------------------------ - # Rapid create / drop cycling - # ------------------------------------------------------------------ + for name in names: + assert name in listed_names def test_rapid_create_drop_cycle( - self, fs_client: FsClient, function_registry: List[str], kafka_topics: str - ): - """Rapidly creating and dropping functions must not corrupt server state.""" + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str + ) -> None: for i in range(5): - fn_name = _unique(f"rapid-{i}") + fn_name = _generate_unique_name(f"rapid-{i}") + config = _create_counter_task_builder(fn_name, kafka_topics).build() - config = _build_counter_config(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) - fs_client.stop_function(fn_name) assert fs_client.drop_function(fn_name) is True listing = fs_client.show_functions() remaining = [f.name for f in listing.functions if f.name.startswith("rapid-")] - assert remaining == [], f"Stale functions remain: {remaining}" - - # ------------------------------------------------------------------ - # Restart (stop + start) resilience - # ------------------------------------------------------------------ + assert not remaining def test_restart_preserves_identity( - self, fs_client: FsClient, function_registry: List[str], kafka_topics: str - ): - """Stopping and restarting a function should keep its name and type.""" - fn_name = _unique("restart") + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str + ) -> None: + fn_name = _generate_unique_name("restart") function_registry.append(fn_name) - config = _build_counter_config(fn_name, kafka_topics).build() + config = _create_counter_task_builder(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) fs_client.stop_function(fn_name) fs_client.start_function(fn_name) - listing = fs_client.show_functions() - fn_info = next(f for f in listing.functions if f.name == fn_name) + fn_info = _get_function_info(fs_client, fn_name) + assert fn_info is not None assert fn_info.name == fn_name - assert fn_info.status.upper() == "RUNNING" - - # ------------------------------------------------------------------ - # Drop after stop (explicit two-phase teardown) - # ------------------------------------------------------------------ + assert fn_info.status.upper() in EXPECTED_RUNNING_STATES def test_stop_then_drop( - self, fs_client: FsClient, function_registry: List[str], kafka_topics: str - ): - """Explicitly stopping, then dropping must always succeed.""" - fn_name = _unique("stop-drop") + self, fs_client: FsClient, function_registry: List[str], kafka_topics: str + ) -> None: + fn_name = _generate_unique_name("stop-drop") function_registry.append(fn_name) - config = _build_counter_config(fn_name, kafka_topics).build() + config = _create_counter_task_builder(fn_name, kafka_topics).build() fs_client.create_python_function_from_config(config, CounterProcessor) assert fs_client.stop_function(fn_name) is True assert fs_client.drop_function(fn_name) is True - listing = fs_client.show_functions() - assert fn_name not in [f.name for f in listing.functions] - function_registry.remove(fn_name) + assert _get_function_info(fs_client, fn_name) is None + + if fn_name in function_registry: + function_registry.remove(fn_name) \ No newline at end of file From cb3a53b6b3b0901beb0c5d01b8d90c692879b9ea Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 16 Apr 2026 23:28:53 +0800 Subject: [PATCH 13/15] update --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 82d9ce66..2b8dc574 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -59,7 +59,7 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Build Release Binary - run: cargo build --release --features python + run: make env && make dist - name: Pre-pull Kafka Image run: docker pull apache/kafka:3.7.0 From 96638e37780af34cd26ab3ad50eec83023c21d01 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 17 Apr 2026 00:45:31 +0800 Subject: [PATCH 14/15] update --- .../operators/grouping/incremental_aggregate.rs | 6 +++--- src/sql/logical_planner/streaming_planner.rs | 2 +- src/sql/types/data_type.rs | 11 ++++------- tests/integration/framework/instance.py | 3 +-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/runtime/streaming/operators/grouping/incremental_aggregate.rs b/src/runtime/streaming/operators/grouping/incremental_aggregate.rs index 625cdee5..43e0e657 100644 --- a/src/runtime/streaming/operators/grouping/incremental_aggregate.rs +++ b/src/runtime/streaming/operators/grouping/incremental_aggregate.rs @@ -465,7 +465,7 @@ impl IncrementalAggregatingFunc { state }); - for (idx, v) in agg.state_cols.iter().zip(state.into_iter()) { + for (idx, v) in agg.state_cols.iter().zip(state) { states[*idx].push(v); } } @@ -543,7 +543,7 @@ impl IncrementalAggregatingFunc { } } - let mut cols = self.key_converter.convert_rows(rows.into_iter())?; + let mut cols = self.key_converter.convert_rows(rows)?; cols.push(Arc::new(accumulator_builder.finish())); cols.push(Arc::new(args_row_builder.finish())); cols.push(Arc::new(count_builder.finish())); @@ -612,7 +612,7 @@ impl IncrementalAggregatingFunc { mem::take(&mut self.updated_keys).into_iter().unzip(); let mut deleted_keys = vec![]; - for (k, retract) in updated_keys.iter().zip(updated_values.into_iter()) { + for (k, retract) in updated_keys.iter().zip(updated_values) { let append = self.evaluate(&k.0)?; if let Some(v) = retract { diff --git a/src/sql/logical_planner/streaming_planner.rs b/src/sql/logical_planner/streaming_planner.rs index e501695d..4619fb3f 100644 --- a/src/sql/logical_planner/streaming_planner.rs +++ b/src/sql/logical_planner/streaming_planner.rs @@ -341,7 +341,7 @@ impl PlanToGraphVisitor<'_> { let node_index = self.graph.add_node(execution_unit); self.add_index_to_traversal(node_index); - for (source, edge) in input_nodes.into_iter().zip(routing_edges.into_iter()) { + for (source, edge) in input_nodes.into_iter().zip(routing_edges) { self.graph.add_edge(source, node_index, edge); } diff --git a/src/sql/types/data_type.rs b/src/sql/types/data_type.rs index 070324d5..387a4190 100644 --- a/src/sql/types/data_type.rs +++ b/src/sql/types/data_type.rs @@ -98,14 +98,11 @@ fn convert_simple_data_type( } }, SQLDataType::Date => Ok(DataType::Date32), - SQLDataType::Time(None, tz_info) => { + SQLDataType::Time(None, tz_info) if matches!(tz_info, TimezoneInfo::None) - || matches!(tz_info, TimezoneInfo::WithoutTimeZone) - { - Ok(DataType::Time64(TimeUnit::Nanosecond)) - } else { - return plan_err!("Unsupported SQL type {sql_type:?}"); - } + || matches!(tz_info, TimezoneInfo::WithoutTimeZone) => + { + Ok(DataType::Time64(TimeUnit::Nanosecond)) } SQLDataType::Numeric(exact_number_info) | SQLDataType::Decimal(exact_number_info) => { let (precision, scale) = match *exact_number_info { diff --git a/tests/integration/framework/instance.py b/tests/integration/framework/instance.py index b18ca737..e30415db 100644 --- a/tests/integration/framework/instance.py +++ b/tests/integration/framework/instance.py @@ -20,8 +20,7 @@ from types import TracebackType from typing import TYPE_CHECKING, Any, Optional, Type -from .paths import PathManager -from .config import InstanceConfig +from .config import InstanceConfig, PathManager from .process import FunctionStreamProcess from .utils import find_free_port, wait_for_port from .workspace import InstanceWorkspace From 29eb766a7e65ef3bc8c0bc670e0ef809312b8919 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 17 Apr 2026 00:52:18 +0800 Subject: [PATCH 15/15] update --- tests/integration/framework/config.py | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/integration/framework/config.py b/tests/integration/framework/config.py index d702ecdc..e086a4b0 100644 --- a/tests/integration/framework/config.py +++ b/tests/integration/framework/config.py @@ -47,6 +47,48 @@ class PathManager: _INTEGRATION_DIR = Path(__file__).resolve().parents[1] _PROJECT_ROOT = _INTEGRATION_DIR.parents[1] + @classmethod + def get_target_dir(cls) -> Path: + """Return the integration-test workspace root (tests/integration/target).""" + return cls._INTEGRATION_DIR / "target" + + @classmethod + def get_binary_path(cls) -> Path: + """Locate the compiled function-stream binary under the project target dir.""" + import platform + import struct + + system = platform.system() + machine = platform.machine().lower() + arch_map = {"arm64": "aarch64", "amd64": "x86_64"} + arch = arch_map.get(machine, machine) + + if system == "Linux": + triple = f"{arch}-unknown-linux-gnu" + elif system == "Darwin": + triple = f"{arch}-apple-darwin" + elif system == "Windows": + triple = f"{arch}-pc-windows-msvc" + else: + triple = f"{arch}-unknown-linux-gnu" + + binary_name = "function-stream.exe" if system == "Windows" else "function-stream" + candidate = cls._PROJECT_ROOT / "target" / triple / "release" / binary_name + + if candidate.is_file(): + return candidate + + fallback = cls._PROJECT_ROOT / "target" / "release" / binary_name + if fallback.is_file(): + return fallback + + raise FileNotFoundError( + f"Cannot find function-stream binary. Checked:\n" + f" - {candidate}\n" + f" - {fallback}\n" + f"Build first with: make build (or make build-lite)" + ) + @classmethod def get_shared_cache_dir(cls) -> Path: return cls._INTEGRATION_DIR / "target" / ".shared_cache"