diff --git a/application/single_app/config.py b/application/single_app/config.py index 3661cf2b0..7adc43bf2 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.215" +VERSION = "0.250.216" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/route_backend_plugins.py b/application/single_app/route_backend_plugins.py index 7e0ae59cb..234dd62fa 100644 --- a/application/single_app/route_backend_plugins.py +++ b/application/single_app/route_backend_plugins.py @@ -17,12 +17,21 @@ build_sql_server_odbc_connection_string, connect_with_sql_server_odbc_fallback, ) +from semantic_kernel_plugins.rocksdb_plugin import ( + AUTH_SCHEME_API_KEY, + AUTH_SCHEME_BEARER, + AUTH_SCHEME_NONE, + DEFAULT_API_KEY_HEADER, + SUPPORTED_AUTH_SCHEMES, + normalize_rocksdb_base_url, +) from functions_settings import get_settings, is_tabular_processing_enabled, update_settings from functions_authentication import * from functions_appinsights import log_event from swagger_wrapper import swagger_route, get_auth_security import logging import os +import requests from functions_debug import debug_print import importlib.util from functions_plugins import get_merged_plugin_settings @@ -456,6 +465,23 @@ def get_plugin_types(allowed_type_filter=None): }, 'metadata': {'description': 'Example Cosmos query plugin'} } + elif 'rocksdb' in module_name.lower(): + safe_manifest = { + 'endpoint': 'https://rocksdb.example.com/api', + 'auth': {'type': 'NoAuth'}, + 'additionalFields': { + 'base_url': 'https://rocksdb.example.com/api', + 'auth_scheme': AUTH_SCHEME_NONE, + 'column_family': 'default', + 'key_encoding': 'utf8', + 'value_encoding': 'utf8', + 'read_only': True, + 'max_results': 100, + 'max_value_bytes': 32768, + 'timeout': 30, + }, + 'metadata': {'description': 'Example RocksDB key-value action'} + } elif 'blob_storage' in module_name.lower(): safe_manifest = { 'endpoint': 'https://example.blob.core.windows.net', @@ -2702,3 +2728,143 @@ def request_with_timeout(*args, **kwargs): client.close() except Exception: pass + + +@bpap.route('/api/plugins/test-rocksdb-connection', methods=['POST']) +@swagger_route(security=get_auth_security()) +@login_required +@user_required +def test_rocksdb_connection(): + """Probe a RocksDB HTTP service and confirm it answers with the expected contract.""" + data = request.get_json(silent=True) or {} + user_id = get_current_user_id() + auth_scheme = (data.get('auth_scheme') or AUTH_SCHEME_NONE).strip().lower() + api_key_header = (data.get('api_key_header') or '').strip() or DEFAULT_API_KEY_HEADER + auth_key = (data.get('auth_key') or '').strip() + timeout = min(max(int(data.get('timeout', 10) or 10), 1), 30) + + try: + base_url = normalize_rocksdb_base_url(data.get('base_url') or '') + except ValueError: + return jsonify({ + 'success': False, + 'error': 'The RocksDB service base URL is invalid. Use an http or https URL that includes a host name.' + }), 400 + + if not base_url: + return jsonify({'success': False, 'error': 'A RocksDB service base URL is required.'}), 400 + if auth_scheme not in SUPPORTED_AUTH_SCHEMES: + return jsonify({'success': False, 'error': "Auth scheme must be one of 'none', 'bearer', or 'api_key'."}), 400 + + try: + existing_plugin = _load_existing_plugin_for_test(data.get('existing_plugin'), user_id) + except PermissionError: + return jsonify({'success': False, 'error': 'You do not have access to the selected action.'}), 403 + except LookupError: + return jsonify({'success': False, 'error': 'The selected action could not be found.'}), 404 + except ValueError: + return jsonify({'success': False, 'error': 'The selected action reference is invalid.'}), 400 + + if auth_scheme != AUTH_SCHEME_NONE: + existing_auth = {} + if isinstance(existing_plugin, dict) and isinstance(existing_plugin.get('auth'), dict): + existing_auth = existing_plugin['auth'] + + if auth_key == ui_trigger_word: + auth_key = existing_auth.get('key', '') + if auth_key == ui_trigger_word: + return jsonify({ + 'success': False, + 'error': 'The stored RocksDB service token could not be resolved for testing. Re-enter the token.' + }), 400 + + try: + auth_key = _resolve_secret_value_for_plugin_test(auth_key, 'auth.key', plugin_label='RocksDB') + except ValueError: + return jsonify({ + 'success': False, + 'error': 'The stored RocksDB service token could not be resolved. Re-enter the token.' + }), 400 + + if not auth_key: + return jsonify({ + 'success': False, + 'error': 'A service token is required when the auth scheme is bearer or api_key.' + }), 400 + + headers = {'Accept': 'application/json'} + if auth_scheme == AUTH_SCHEME_BEARER: + headers['Authorization'] = f'Bearer {auth_key}' + elif auth_scheme == AUTH_SCHEME_API_KEY: + headers[api_key_header] = auth_key + + try: + response = requests.get( + f'{base_url}/health', + headers=headers, + timeout=timeout, + ) + + if response.status_code == 404: + # Minimal services may not implement /health, so fall back to a single-item scan. + response = requests.post( + f'{base_url}/scan', + json={'limit': 1}, + headers=headers, + timeout=timeout, + ) + + if response.status_code in (401, 403): + return jsonify({ + 'success': False, + 'error': 'The RocksDB service rejected the supplied credentials. Verify the auth scheme and token.' + }), 403 + + if response.status_code >= 400: + return jsonify({ + 'success': False, + 'error': f'The RocksDB service returned HTTP {response.status_code}.' + }), 400 + + try: + response.json() + except ValueError: + return jsonify({ + 'success': False, + 'error': 'The RocksDB service returned a non-JSON response. Confirm the base URL points at the service API root.' + }), 400 + + log_event( + '[PLUGINS] RocksDB connection test succeeded', + extra={'user_id': user_id, 'auth_scheme': auth_scheme, 'status_code': response.status_code}, + level=logging.INFO, + ) + return jsonify({ + 'success': True, + 'message': f'Successfully reached the RocksDB service at {base_url}.' + }) + except requests.exceptions.SSLError: + return jsonify({ + 'success': False, + 'error': 'TLS verification failed for the RocksDB service. Install the issuing certificate authority in the application trust store.' + }), 400 + except requests.exceptions.Timeout: + return jsonify({ + 'success': False, + 'error': f'The RocksDB service did not respond within {timeout} seconds.' + }), 400 + except Exception as exc: + log_event( + '[PLUGINS] RocksDB connection test failed', + extra={ + 'user_id': user_id, + 'auth_scheme': auth_scheme, + 'error_type': type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return jsonify({ + 'success': False, + 'error': 'The RocksDB service could not be reached. Verify the base URL and that the service is running.' + }), 400 diff --git a/application/single_app/semantic_kernel_plugins/plugin_health_checker.py b/application/single_app/semantic_kernel_plugins/plugin_health_checker.py index b3850004f..358f13d43 100644 --- a/application/single_app/semantic_kernel_plugins/plugin_health_checker.py +++ b/application/single_app/semantic_kernel_plugins/plugin_health_checker.py @@ -73,6 +73,17 @@ validate_mcp_endpoint_for_transport, ) from functions_simplechat_operations import SIMPLECHAT_DEFAULT_ENDPOINT +from semantic_kernel_plugins.rocksdb_plugin import ( + AUTH_SCHEME_NONE, + MAX_RESULTS_CEILING, + MAX_TIMEOUT, + MAX_VALUE_BYTES_CEILING, + ROCKSDB_PLUGIN_TYPE, + SUPPORTED_AUTH_SCHEMES, + SUPPORTED_KEY_ENCODINGS, + SUPPORTED_VALUE_ENCODINGS, + normalize_rocksdb_base_url, +) class PluginHealthChecker: @@ -346,7 +357,10 @@ def validate_plugin_manifest(manifest: Dict[str, Any], plugin_type: str) -> Tupl errors.append("Cosmos plugin only supports auth.type values 'identity' and 'key'") if auth_type == 'key' and not auth.get('key'): errors.append("Cosmos plugin requires auth.key when auth.type='key'") - + + elif plugin_type == ROCKSDB_PLUGIN_TYPE: + errors.extend(PluginHealthChecker._validate_rocksdb_manifest(manifest)) + elif plugin_type == 'log_analytics': additional_fields = manifest.get('additionalFields', {}) if 'workspaceId' not in additional_fields: @@ -441,7 +455,60 @@ def validate_plugin_manifest(manifest: Dict[str, Any], plugin_type: str) -> Tupl errors.append("Azure Maps plugin requires auth.key with an Azure Maps subscription key") return len(errors) == 0, errors - + + @staticmethod + def _validate_rocksdb_manifest(manifest: Dict[str, Any]) -> List[str]: + """Validate a RocksDB action manifest against the RocksDB HTTP service contract.""" + errors: List[str] = [] + additional_fields = manifest.get('additionalFields', {}) + if not isinstance(additional_fields, dict): + additional_fields = {} + + auth = manifest.get('auth', {}) if isinstance(manifest.get('auth'), dict) else {} + + key_encoding = str(additional_fields.get('key_encoding') or 'utf8').strip().lower() + if key_encoding not in SUPPORTED_KEY_ENCODINGS: + errors.append("RocksDB plugin key_encoding must be either 'utf8' or 'base64'") + + value_encoding = str(additional_fields.get('value_encoding') or 'utf8').strip().lower() + if value_encoding not in SUPPORTED_VALUE_ENCODINGS: + errors.append("RocksDB plugin value_encoding must be one of 'utf8', 'base64', or 'json'") + + for field_name, minimum, maximum in ( + ('max_results', 1, MAX_RESULTS_CEILING), + ('max_value_bytes', 1, MAX_VALUE_BYTES_CEILING), + ('timeout', 1, MAX_TIMEOUT), + ): + raw_value = additional_fields.get(field_name) + if raw_value in (None, ''): + continue + try: + parsed_value = int(raw_value) + except (TypeError, ValueError): + errors.append(f"RocksDB plugin additionalFields.{field_name} must be an integer") + continue + if parsed_value < minimum or parsed_value > maximum: + errors.append( + f"RocksDB plugin additionalFields.{field_name} must be between {minimum} and {maximum}" + ) + + base_url = str(additional_fields.get('base_url') or manifest.get('endpoint') or '').strip() + if not base_url: + errors.append("RocksDB plugin requires 'base_url' in additionalFields") + else: + try: + normalize_rocksdb_base_url(base_url) + except ValueError as validation_error: + errors.append(f"RocksDB plugin base_url is invalid: {validation_error}") + + auth_scheme = str(additional_fields.get('auth_scheme') or AUTH_SCHEME_NONE).strip().lower() + if auth_scheme not in SUPPORTED_AUTH_SCHEMES: + errors.append("RocksDB plugin auth_scheme must be one of 'none', 'bearer', or 'api_key'") + elif auth_scheme != AUTH_SCHEME_NONE and not auth.get('key'): + errors.append("RocksDB plugin requires auth.key when auth_scheme is 'bearer' or 'api_key'") + + return errors + @staticmethod def check_plugin_health(plugin_instance: BasePlugin, plugin_name: str) -> Dict[str, Any]: """ diff --git a/application/single_app/semantic_kernel_plugins/rocksdb_plugin.py b/application/single_app/semantic_kernel_plugins/rocksdb_plugin.py new file mode 100644 index 000000000..2db297b40 --- /dev/null +++ b/application/single_app/semantic_kernel_plugins/rocksdb_plugin.py @@ -0,0 +1,902 @@ +# rocksdb_plugin.py +""" +RocksDB key-value store plugin for Semantic Kernel. + +RocksDB is an embedded C++ library with no native network protocol, so this plugin talks to a +RocksDB-backed HTTP/JSON service using the SimpleChat RocksDB service contract documented in +``docs/explanation/features/v0.250.216/ROCKSDB_ACTION.md``. SimpleChat never opens a RocksDB +database directory itself. + +Read operations are always available. Write operations are refused unless the action is +explicitly configured with ``read_only`` disabled. +""" + +import json +import logging +from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.parse import urlparse + +import requests +from semantic_kernel.functions import kernel_function + +from functions_appinsights import log_event, sanitize_log_message +from semantic_kernel_plugins.base_plugin import BasePlugin +from semantic_kernel_plugins.plugin_invocation_logger import plugin_function_logger + + +ROCKSDB_PLUGIN_TYPE = "rocksdb" + +AUTH_SCHEME_NONE = "none" +AUTH_SCHEME_BEARER = "bearer" +AUTH_SCHEME_API_KEY = "api_key" +SUPPORTED_AUTH_SCHEMES = (AUTH_SCHEME_NONE, AUTH_SCHEME_BEARER, AUTH_SCHEME_API_KEY) + +KEY_ENCODING_UTF8 = "utf8" +KEY_ENCODING_BASE64 = "base64" +SUPPORTED_KEY_ENCODINGS = (KEY_ENCODING_UTF8, KEY_ENCODING_BASE64) + +VALUE_ENCODING_UTF8 = "utf8" +VALUE_ENCODING_BASE64 = "base64" +VALUE_ENCODING_JSON = "json" +SUPPORTED_VALUE_ENCODINGS = (VALUE_ENCODING_UTF8, VALUE_ENCODING_BASE64, VALUE_ENCODING_JSON) + +DEFAULT_COLUMN_FAMILY = "default" +DEFAULT_API_KEY_HEADER = "X-API-Key" +DEFAULT_MAX_RESULTS = 100 +MAX_RESULTS_CEILING = 1000 +DEFAULT_MAX_VALUE_BYTES = 32768 +MAX_VALUE_BYTES_CEILING = 1048576 +DEFAULT_TIMEOUT = 30 +MAX_TIMEOUT = 300 +ALLOWED_REMOTE_SCHEMES = ("http", "https") + + +def normalize_rocksdb_base_url(base_url: str) -> str: + """Normalize a RocksDB service base URL and reject unsupported schemes.""" + cleaned_url = (base_url or "").strip().rstrip("/") + if not cleaned_url: + return "" + + parsed_url = urlparse(cleaned_url) + if parsed_url.scheme.lower() not in ALLOWED_REMOTE_SCHEMES: + raise ValueError("RocksDB service URLs must use the http or https scheme.") + if not parsed_url.netloc: + raise ValueError("RocksDB service URLs must include a host name.") + return cleaned_url + + +class RocksDbResult: + """Structured RocksDB plugin result paired with the plugin metadata.""" + + def __init__(self, data: Any, metadata: Dict[str, Any]): + self.data = data + self.metadata = metadata + + def __str__(self) -> str: + return str(self.data) + + def __repr__(self) -> str: + return f"RocksDbResult(data={self.data!r}, metadata={self.metadata!r})" + + +class RocksDbPlugin(BasePlugin): + """RocksDB key-value plugin backed by a RocksDB HTTP/JSON service.""" + + def __init__(self, manifest: Optional[Dict[str, Any]] = None): + super().__init__(manifest) + self.manifest = manifest or {} + additional_fields = self.manifest.get("additionalFields", {}) or {} + auth = self.manifest.get("auth", {}) or {} + + self.base_url = str( + additional_fields.get("base_url") or self.manifest.get("endpoint") or "" + ).strip().rstrip("/") + self.auth_scheme = self._normalize_choice( + additional_fields.get("auth_scheme"), AUTH_SCHEME_NONE + ) + self.api_key_header = ( + str(additional_fields.get("api_key_header") or "").strip() or DEFAULT_API_KEY_HEADER + ) + + self.column_family = ( + str(additional_fields.get("column_family") or "").strip() or DEFAULT_COLUMN_FAMILY + ) + self.key_encoding = self._normalize_choice( + additional_fields.get("key_encoding"), KEY_ENCODING_UTF8 + ) + self.value_encoding = self._normalize_choice( + additional_fields.get("value_encoding"), VALUE_ENCODING_UTF8 + ) + self.key_prefix_hints = self._normalize_prefix_hints( + additional_fields.get("key_prefix_hints") + ) + self.read_only = self._coerce_bool(additional_fields.get("read_only"), True) + self.max_results = self._coerce_int( + additional_fields.get("max_results"), DEFAULT_MAX_RESULTS, "max_results" + ) + self.max_value_bytes = self._coerce_int( + additional_fields.get("max_value_bytes"), DEFAULT_MAX_VALUE_BYTES, "max_value_bytes" + ) + self.timeout = self._coerce_int(additional_fields.get("timeout"), DEFAULT_TIMEOUT, "timeout") + + self.auth_type = str(auth.get("type") or "NoAuth").strip() + self.auth_key = str(auth.get("key") or "").strip() + self._metadata = self.manifest.get("metadata", {}) or {} + self._http_session = None + + self._validate_configuration() + + log_event( + "[ROCKSDB_PLUGIN] Initialized plugin", + extra={ + "column_family": self.column_family, + "key_encoding": self.key_encoding, + "value_encoding": self.value_encoding, + "read_only": self.read_only, + "max_results": self.max_results, + "max_value_bytes": self.max_value_bytes, + "timeout": self.timeout, + "auth_scheme": self.auth_scheme, + "prefix_hint_count": len(self.key_prefix_hints), + }, + level=logging.INFO, + ) + + @property + def display_name(self) -> str: + return "RocksDB" + + @property + def metadata(self) -> Dict[str, Any]: + user_desc = self._metadata.get( + "description", + "RocksDB key-value store plugin backed by a RocksDB HTTP service.", + ) + write_desc = ( + "Write operations are blocked because this action is configured as read-only." + if self.read_only + else "Write operations are enabled for this action; confirm intent before mutating data." + ) + api_desc = ( + "This plugin reads an ordered RocksDB key-value store through a RocksDB HTTP service. " + "Keys are byte strings sorted lexicographically, so prefix and range scans are the " + "efficient access patterns; avoid scanning the whole keyspace. Use get_value for a " + "single key, get_values for a batch of keys, and scan_prefix or scan_range to walk a " + f"key range. {write_desc}" + ) + return { + "name": self._metadata.get("name", "rocksdb_plugin"), + "type": ROCKSDB_PLUGIN_TYPE, + "description": f"{user_desc}\n\n{api_desc}", + "methods": [ + { + "name": "get_value", + "description": "Read a single value by exact key.", + "parameters": [ + {"name": "key", "type": "str", "description": "Exact key to read.", "required": True}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "The value plus lookup metadata."}, + }, + { + "name": "get_values", + "description": "Read several values in one call using exact keys.", + "parameters": [ + {"name": "keys", "type": "List[str] | str", "description": "Keys to read, as a list or JSON array string.", "required": True}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "One result entry per requested key."}, + }, + { + "name": "key_exists", + "description": "Check whether a key is present without returning its value.", + "parameters": [ + {"name": "key", "type": "str", "description": "Exact key to test.", "required": True}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "Existence flag for the key."}, + }, + { + "name": "scan_prefix", + "description": "List key-value pairs whose keys start with a prefix.", + "parameters": [ + {"name": "prefix", "type": "str", "description": "Key prefix to scan.", "required": True}, + {"name": "limit", "type": "int", "description": "Optional per-call result cap.", "required": False}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "Matching key-value pairs and truncation state."}, + }, + { + "name": "scan_range", + "description": "List key-value pairs between an inclusive start key and an exclusive end key.", + "parameters": [ + {"name": "start_key", "type": "str", "description": "Inclusive first key of the range.", "required": False}, + {"name": "end_key", "type": "str", "description": "Exclusive last key of the range.", "required": False}, + {"name": "limit", "type": "int", "description": "Optional per-call result cap.", "required": False}, + {"name": "reverse", "type": "bool", "description": "Walk the range in descending key order.", "required": False}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "Matching key-value pairs and truncation state."}, + }, + { + "name": "list_column_families", + "description": "List the column families available in the configured database.", + "parameters": [], + "returns": {"type": "RocksDbResult", "description": "Available column family names."}, + }, + { + "name": "get_database_stats", + "description": "Report size and key-count estimates for the configured database.", + "parameters": [], + "returns": {"type": "RocksDbResult", "description": "Database statistics."}, + }, + { + "name": "put_value", + "description": "Write a single key-value pair. Requires the action to allow writes.", + "parameters": [ + {"name": "key", "type": "str", "description": "Key to write.", "required": True}, + {"name": "value", "type": "Any", "description": "Value to store.", "required": True}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "Write confirmation."}, + }, + { + "name": "delete_value", + "description": "Delete a single key. Requires the action to allow writes.", + "parameters": [ + {"name": "key", "type": "str", "description": "Key to delete.", "required": True}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "Delete confirmation."}, + }, + { + "name": "write_batch", + "description": "Apply several put and delete operations atomically. Requires the action to allow writes.", + "parameters": [ + {"name": "operations", "type": "List[Dict[str, Any]] | str", "description": "Operations as a list or JSON array string, each with op, key, and value.", "required": True}, + {"name": "column_family", "type": "str", "description": "Optional column family override.", "required": False}, + ], + "returns": {"type": "RocksDbResult", "description": "Batch write confirmation."}, + }, + ], + } + + def get_functions(self) -> List[str]: + return [ + "get_value", + "get_values", + "key_exists", + "scan_prefix", + "scan_range", + "list_column_families", + "get_database_stats", + "put_value", + "delete_value", + "write_batch", + ] + + def set_http_session(self, session: Any) -> None: + """Override the HTTP session used for RocksDB service calls.""" + self._http_session = session + + def build_instruction_context(self) -> str: + """Return agent instruction text describing the configured RocksDB store.""" + hint_lines = [f"- {prefix_hint}" for prefix_hint in self.key_prefix_hints] or [ + "- No key prefix hints were configured." + ] + access_line = ( + "- Access: read-only" + if self.read_only + else "- Access: reads and writes are enabled" + ) + return ( + f"### RocksDB Store: {self.base_url}\n" + f"- Column family: {self.column_family}\n" + f"{access_line}\n" + f"- Max results per call: {self.max_results}\n" + f"- Key encoding: {self.key_encoding}, value encoding: {self.value_encoding}\n" + "- Keys are byte strings sorted lexicographically; prefer prefix and range scans.\n" + "- Configured key prefix hints:\n" + + "\n".join(hint_lines) + ) + + @kernel_function(description="Read a single RocksDB value by its exact key. Use this when you already know the full key; use scan_prefix when you only know how the key starts.") + @plugin_function_logger("RocksDbPlugin") + def get_value(self, key: str, column_family: Optional[str] = None) -> RocksDbResult: + try: + resolved_family = self._resolve_column_family(column_family) + payload = self._remote_request( + "POST", "/get", {"key": key, "column_family": resolved_family} + ) + found = bool(payload.get("found", payload.get("value") is not None)) + value, truncated, byte_length = self._normalize_value(payload.get("value")) + + return self._success( + { + "key": key, + "column_family": resolved_family, + "found": found, + "value": value if found else None, + "value_bytes": byte_length, + "value_truncated": truncated, + } + ) + except Exception as exc: + return self._failure("get_value", exc, {"key": key}) + + @kernel_function(description="Read several RocksDB values in one call using their exact keys. Pass the keys as a list or a JSON array string.") + @plugin_function_logger("RocksDbPlugin") + def get_values( + self, + keys: Union[str, List[str]], + column_family: Optional[str] = None, + ) -> RocksDbResult: + try: + resolved_family = self._resolve_column_family(column_family) + normalized_keys = self._normalize_key_list(keys) + if not normalized_keys: + raise ValueError("At least one key is required.") + if len(normalized_keys) > self.max_results: + normalized_keys = normalized_keys[: self.max_results] + + payload = self._remote_request( + "POST", + "/multi_get", + {"keys": normalized_keys, "column_family": resolved_family}, + ) + remote_results = payload.get("results") + if not isinstance(remote_results, list): + raise RuntimeError("RocksDB service returned an unexpected multi_get payload.") + + results = [] + for entry in remote_results: + entry_dict = entry if isinstance(entry, dict) else {} + found = bool(entry_dict.get("found", entry_dict.get("value") is not None)) + value, truncated, byte_length = self._normalize_value(entry_dict.get("value")) + results.append( + { + "key": entry_dict.get("key"), + "found": found, + "value": value if found else None, + "value_bytes": byte_length, + "value_truncated": truncated, + } + ) + + return self._success( + { + "column_family": resolved_family, + "requested_key_count": len(normalized_keys), + "found_count": sum(1 for entry in results if entry.get("found")), + "results": results, + } + ) + except Exception as exc: + return self._failure("get_values", exc, {"key_count": self._safe_length(keys)}) + + @kernel_function(description="Check whether a RocksDB key exists without returning its value. Use this for cheap existence checks on large values.") + @plugin_function_logger("RocksDbPlugin") + def key_exists(self, key: str, column_family: Optional[str] = None) -> RocksDbResult: + try: + resolved_family = self._resolve_column_family(column_family) + payload = self._remote_request( + "POST", "/exists", {"key": key, "column_family": resolved_family} + ) + exists = bool(payload.get("exists", payload.get("found", False))) + return self._success( + {"key": key, "column_family": resolved_family, "exists": exists} + ) + except Exception as exc: + return self._failure("key_exists", exc, {"key": key}) + + @kernel_function(description="List RocksDB key-value pairs whose keys start with a prefix. This is the efficient way to explore a RocksDB store because keys are stored in sorted order.") + @plugin_function_logger("RocksDbPlugin") + def scan_prefix( + self, + prefix: str, + limit: Optional[int] = None, + column_family: Optional[str] = None, + ) -> RocksDbResult: + try: + resolved_family = self._resolve_column_family(column_family) + effective_limit = self._resolve_limit(limit) + items, truncated = self._scan( + column_family=resolved_family, + prefix=prefix, + start_key=None, + end_key=None, + limit=effective_limit, + reverse=False, + ) + return self._success( + { + "column_family": resolved_family, + "prefix": prefix, + "limit": effective_limit, + "item_count": len(items), + "is_truncated": truncated, + "items": items, + } + ) + except Exception as exc: + return self._failure("scan_prefix", exc, {"prefix": prefix}) + + @kernel_function(description="List RocksDB key-value pairs between an inclusive start key and an exclusive end key. Set reverse to walk the range from the highest key downwards.") + @plugin_function_logger("RocksDbPlugin") + def scan_range( + self, + start_key: Optional[str] = None, + end_key: Optional[str] = None, + limit: Optional[int] = None, + reverse: bool = False, + column_family: Optional[str] = None, + ) -> RocksDbResult: + try: + resolved_family = self._resolve_column_family(column_family) + effective_limit = self._resolve_limit(limit) + normalized_reverse = self._coerce_bool(reverse, False) + items, truncated = self._scan( + column_family=resolved_family, + prefix=None, + start_key=start_key, + end_key=end_key, + limit=effective_limit, + reverse=normalized_reverse, + ) + return self._success( + { + "column_family": resolved_family, + "start_key": start_key, + "end_key": end_key, + "reverse": normalized_reverse, + "limit": effective_limit, + "item_count": len(items), + "is_truncated": truncated, + "items": items, + } + ) + except Exception as exc: + return self._failure( + "scan_range", exc, {"start_key": start_key, "end_key": end_key} + ) + + @kernel_function(description="List the column families available in the configured RocksDB database.") + @plugin_function_logger("RocksDbPlugin") + def list_column_families(self) -> RocksDbResult: + try: + payload = self._remote_request("GET", "/column_families") + column_families = payload.get("column_families") + if not isinstance(column_families, list): + raise RuntimeError( + "RocksDB service returned an unexpected column_families payload." + ) + + return self._success( + { + "column_families": column_families, + "configured_column_family": self.column_family, + } + ) + except Exception as exc: + return self._failure("list_column_families", exc) + + @kernel_function(description="Report size and key-count estimates for the configured RocksDB database.") + @plugin_function_logger("RocksDbPlugin") + def get_database_stats(self) -> RocksDbResult: + try: + payload = self._remote_request("GET", "/stats") + statistics = payload.get("stats", payload) + return self._success( + { + "column_family": self.column_family, + "read_only": self.read_only, + "stats": statistics, + } + ) + except Exception as exc: + return self._failure("get_database_stats", exc) + + @kernel_function(description="Write a single RocksDB key-value pair. This is refused when the action is configured as read-only.") + @plugin_function_logger("RocksDbPlugin") + def put_value( + self, + key: str, + value: Any, + column_family: Optional[str] = None, + ) -> RocksDbResult: + blocked_result = self._write_guard("put_value") + if blocked_result is not None: + return blocked_result + + try: + resolved_family = self._resolve_column_family(column_family) + self._remote_request( + "POST", + "/put", + {"key": key, "value": value, "column_family": resolved_family}, + ) + + log_event( + "[ROCKSDB_PLUGIN] Key written", + extra={"column_family": resolved_family}, + level=logging.INFO, + ) + return self._success( + {"key": key, "column_family": resolved_family, "written": True} + ) + except Exception as exc: + return self._failure("put_value", exc, {"key": key}) + + @kernel_function(description="Delete a single RocksDB key. This is refused when the action is configured as read-only.") + @plugin_function_logger("RocksDbPlugin") + def delete_value(self, key: str, column_family: Optional[str] = None) -> RocksDbResult: + blocked_result = self._write_guard("delete_value") + if blocked_result is not None: + return blocked_result + + try: + resolved_family = self._resolve_column_family(column_family) + self._remote_request( + "POST", "/delete", {"key": key, "column_family": resolved_family} + ) + + log_event( + "[ROCKSDB_PLUGIN] Key deleted", + extra={"column_family": resolved_family}, + level=logging.INFO, + ) + return self._success( + {"key": key, "column_family": resolved_family, "deleted": True} + ) + except Exception as exc: + return self._failure("delete_value", exc, {"key": key}) + + @kernel_function(description="Apply several RocksDB put and delete operations atomically. Each operation needs an op of put or delete, a key, and a value for puts. This is refused when the action is configured as read-only.") + @plugin_function_logger("RocksDbPlugin") + def write_batch( + self, + operations: Union[str, List[Dict[str, Any]]], + column_family: Optional[str] = None, + ) -> RocksDbResult: + blocked_result = self._write_guard("write_batch") + if blocked_result is not None: + return blocked_result + + try: + resolved_family = self._resolve_column_family(column_family) + normalized_operations = self._normalize_batch_operations(operations) + if not normalized_operations: + raise ValueError("At least one batch operation is required.") + if len(normalized_operations) > self.max_results: + raise ValueError( + f"Batch operations are limited to {self.max_results} entries per call." + ) + + self._remote_request( + "POST", + "/batch", + {"operations": normalized_operations, "column_family": resolved_family}, + ) + + log_event( + "[ROCKSDB_PLUGIN] Batch applied", + extra={ + "column_family": resolved_family, + "operation_count": len(normalized_operations), + }, + level=logging.INFO, + ) + return self._success( + { + "column_family": resolved_family, + "operation_count": len(normalized_operations), + "applied": True, + } + ) + except Exception as exc: + return self._failure("write_batch", exc) + + def _validate_configuration(self) -> None: + if self.key_encoding not in SUPPORTED_KEY_ENCODINGS: + raise ValueError("RocksDbPlugin key_encoding must be either 'utf8' or 'base64'.") + if self.value_encoding not in SUPPORTED_VALUE_ENCODINGS: + raise ValueError( + "RocksDbPlugin value_encoding must be one of 'utf8', 'base64', or 'json'." + ) + if self.max_results < 1 or self.max_results > MAX_RESULTS_CEILING: + raise ValueError( + f"RocksDbPlugin max_results must be between 1 and {MAX_RESULTS_CEILING}." + ) + if self.max_value_bytes < 1 or self.max_value_bytes > MAX_VALUE_BYTES_CEILING: + raise ValueError( + f"RocksDbPlugin max_value_bytes must be between 1 and {MAX_VALUE_BYTES_CEILING}." + ) + if self.timeout < 1 or self.timeout > MAX_TIMEOUT: + raise ValueError( + f"RocksDbPlugin timeout must be between 1 and {MAX_TIMEOUT} seconds." + ) + + self.base_url = normalize_rocksdb_base_url(self.base_url) + if not self.base_url: + raise ValueError("RocksDbPlugin requires base_url for the RocksDB service.") + if self.auth_scheme not in SUPPORTED_AUTH_SCHEMES: + raise ValueError( + "RocksDbPlugin auth_scheme must be one of 'none', 'bearer', or 'api_key'." + ) + if self.auth_scheme != AUTH_SCHEME_NONE and not self.auth_key: + raise ValueError( + "RocksDbPlugin requires auth.key when auth_scheme is 'bearer' or 'api_key'." + ) + + def _success(self, payload: Dict[str, Any]) -> RocksDbResult: + return RocksDbResult(dict(payload), self.metadata) + + def _failure( + self, + operation: str, + error: Exception, + context: Optional[Dict[str, Any]] = None, + ) -> RocksDbResult: + # Log the error type and a sanitized detail rather than interpolating the raw + # exception, because transport exceptions can embed request details. + log_event( + f"[ROCKSDB_PLUGIN] {operation} failed", + extra={ + "operation": operation, + "column_family": self.column_family, + "error_type": type(error).__name__, + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + payload = {"operation": operation, "error": sanitize_log_message(error)} + if context: + payload.update(context) + return RocksDbResult(payload, self.metadata) + + def _write_guard(self, operation: str) -> Optional[RocksDbResult]: + if not self.read_only: + return None + return RocksDbResult( + { + "operation": operation, + "error": ( + f"The '{operation}' operation is blocked because this RocksDB action is " + "configured as read-only." + ), + "read_only": True, + }, + self.metadata, + ) + + def _resolve_column_family(self, column_family: Optional[str]) -> str: + candidate = str(column_family or "").strip() + return candidate or self.column_family + + def _resolve_limit(self, limit: Optional[int]) -> int: + if limit in (None, ""): + return self.max_results + try: + requested_limit = int(limit) + except (TypeError, ValueError): + return self.max_results + if requested_limit < 1: + return self.max_results + return min(requested_limit, self.max_results) + + def _scan( + self, + column_family: str, + prefix: Optional[str], + start_key: Optional[str], + end_key: Optional[str], + limit: int, + reverse: bool, + ) -> Tuple[List[Dict[str, Any]], bool]: + request_payload = { + "column_family": column_family, + "limit": limit, + "reverse": reverse, + } + if prefix not in (None, ""): + request_payload["prefix"] = prefix + if start_key not in (None, ""): + request_payload["start_key"] = start_key + if end_key not in (None, ""): + request_payload["end_key"] = end_key + + payload = self._remote_request("POST", "/scan", request_payload) + remote_items = payload.get("items") + if not isinstance(remote_items, list): + raise RuntimeError("RocksDB service returned an unexpected scan payload.") + + items = [] + for entry in remote_items[:limit]: + entry_dict = entry if isinstance(entry, dict) else {} + value, truncated, byte_length = self._normalize_value(entry_dict.get("value")) + items.append( + { + "key": entry_dict.get("key"), + "value": value, + "value_bytes": byte_length, + "value_truncated": truncated, + } + ) + + is_truncated = bool(payload.get("truncated", len(remote_items) > limit)) + return items, is_truncated + + def _remote_request( + self, + method: str, + path: str, + payload: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + session = self._http_session + if session is None: + session = requests.Session() + self._http_session = session + + request_payload = None + if payload is not None: + # The service needs the wire encoding to interpret keys and values correctly. + request_payload = dict(payload) + request_payload["key_encoding"] = self.key_encoding + request_payload["value_encoding"] = self.value_encoding + + response = session.request( + method, + f"{self.base_url}{path}", + json=request_payload, + headers=self._build_remote_headers(), + timeout=self.timeout, + ) + + status_code = getattr(response, "status_code", None) + if status_code is None or int(status_code) >= 400: + raise RuntimeError( + f"RocksDB service request to {path} failed with HTTP {status_code}." + ) + + try: + body = response.json() + except Exception as decode_error: + raise RuntimeError( + f"RocksDB service request to {path} returned a non-JSON response." + ) from decode_error + + if not isinstance(body, dict): + raise RuntimeError( + f"RocksDB service request to {path} returned an unexpected payload shape." + ) + return body + + def _build_remote_headers(self) -> Dict[str, str]: + headers = {"Accept": "application/json"} + if self.auth_scheme == AUTH_SCHEME_BEARER and self.auth_key: + headers["Authorization"] = f"Bearer {self.auth_key}" + elif self.auth_scheme == AUTH_SCHEME_API_KEY and self.auth_key: + headers[self.api_key_header] = self.auth_key + return headers + + def _normalize_value(self, value: Any) -> Tuple[Any, bool, int]: + """Cap an oversized service value so one record cannot flood the model context.""" + if value is None: + return None, False, 0 + + if isinstance(value, str): + raw_bytes = value.encode("utf-8", errors="replace") + else: + raw_bytes = json.dumps(value, default=str).encode("utf-8") + + byte_length = len(raw_bytes) + if byte_length <= self.max_value_bytes: + return value, False, byte_length + return ( + raw_bytes[: self.max_value_bytes].decode("utf-8", errors="replace"), + True, + byte_length, + ) + + def _normalize_key_list(self, keys: Union[str, List[Any]]) -> List[str]: + candidate_keys = keys + if isinstance(candidate_keys, str): + stripped_keys = candidate_keys.strip() + if stripped_keys.startswith("["): + try: + candidate_keys = json.loads(stripped_keys) + except (TypeError, ValueError) as decode_error: + raise ValueError( + "Keys must be a list or a valid JSON array string." + ) from decode_error + else: + candidate_keys = [ + part.strip() for part in stripped_keys.split(",") if part.strip() + ] + + if not isinstance(candidate_keys, (list, tuple)): + raise ValueError("Keys must be a list or a valid JSON array string.") + return [str(single_key) for single_key in candidate_keys if str(single_key).strip()] + + def _normalize_batch_operations( + self, operations: Union[str, List[Dict[str, Any]]] + ) -> List[Dict[str, Any]]: + candidate_operations = operations + if isinstance(candidate_operations, str): + try: + candidate_operations = json.loads(candidate_operations) + except (TypeError, ValueError) as decode_error: + raise ValueError( + "Batch operations must be a list or a valid JSON array string." + ) from decode_error + + if isinstance(candidate_operations, dict): + candidate_operations = [candidate_operations] + if not isinstance(candidate_operations, (list, tuple)): + raise ValueError("Batch operations must be a list or a valid JSON array string.") + + normalized_operations = [] + for operation in candidate_operations: + if not isinstance(operation, dict): + raise ValueError("Each batch operation must be an object.") + operation_name = str(operation.get("op") or "").strip().lower() + if operation_name not in ("put", "delete"): + raise ValueError("Batch operations must use an op of 'put' or 'delete'.") + if str(operation.get("key") or "").strip() == "": + raise ValueError("Each batch operation requires a key.") + normalized_operations.append( + { + "op": operation_name, + "key": operation.get("key"), + "value": operation.get("value"), + } + ) + return normalized_operations + + @staticmethod + def _normalize_choice(value: Any, default_value: str) -> str: + normalized_value = str(value or "").strip().lower() + return normalized_value or default_value + + @staticmethod + def _normalize_prefix_hints(prefix_hints: Any) -> List[str]: + if prefix_hints in (None, ""): + return [] + if isinstance(prefix_hints, str): + candidates = [part.strip() for part in prefix_hints.replace("\n", ",").split(",")] + elif isinstance(prefix_hints, (list, tuple)): + candidates = [str(part).strip() for part in prefix_hints] + else: + return [] + return [candidate for candidate in candidates if candidate] + + @staticmethod + def _coerce_bool(value: Any, default_value: bool) -> bool: + if value in (None, ""): + return default_value + if isinstance(value, bool): + return value + normalized_value = str(value).strip().lower() + if normalized_value in ("true", "1", "yes", "on"): + return True + if normalized_value in ("false", "0", "no", "off"): + return False + return default_value + + @staticmethod + def _coerce_int(value: Any, default_value: int, field_name: str) -> int: + if value in (None, ""): + return default_value + try: + return int(value) + except (TypeError, ValueError) as convert_error: + raise ValueError( + f"RocksDbPlugin {field_name} must be an integer." + ) from convert_error + + @staticmethod + def _safe_length(value: Any) -> Optional[int]: + try: + return len(value) + except TypeError: + return None diff --git a/application/single_app/static/js/plugin_modal_stepper.js b/application/single_app/static/js/plugin_modal_stepper.js index 4e90e890c..6fd0a7b99 100644 --- a/application/single_app/static/js/plugin_modal_stepper.js +++ b/application/single_app/static/js/plugin_modal_stepper.js @@ -53,6 +53,15 @@ const MCP_RESERVED_CUSTOM_HEADER_NAMES = new Set([ ]); const AZURE_MAPS_PLUGIN_TYPE = 'azure_maps_openlayers'; const AZURE_MAPS_DEFAULT_ENDPOINT = 'https://atlas.microsoft.com'; +const ROCKSDB_PLUGIN_TYPE = 'rocksdb'; +const ROCKSDB_AUTH_SCHEME_NONE = 'none'; +const ROCKSDB_AUTH_SCHEME_BEARER = 'bearer'; +const ROCKSDB_AUTH_SCHEME_API_KEY = 'api_key'; +const ROCKSDB_DEFAULT_API_KEY_HEADER = 'X-API-Key'; +const ROCKSDB_DEFAULT_COLUMN_FAMILY = 'default'; +const ROCKSDB_DEFAULT_MAX_RESULTS = 100; +const ROCKSDB_DEFAULT_MAX_VALUE_BYTES = 32768; +const ROCKSDB_DEFAULT_TIMEOUT = 30; const CHART_DEFAULT_ENDPOINT = 'chart://internal'; const INTERNAL_DOCUMENT_SEARCH_ENDPOINT = 'internal://document-search'; const MSGRAPH_DEFAULT_ENDPOINT = 'https://graph.microsoft.com'; @@ -726,6 +735,11 @@ export class PluginModalStepper { document.getElementById('sql-identity-select').addEventListener('change', () => this.handleActionIdentityChange('sql')); document.getElementById('cosmos-auth-type').addEventListener('change', () => this.handleCosmosAuthTypeChange()); + const rocksDbAuthSchemeSelect = document.getElementById('rocksdb-auth-scheme'); + if (rocksDbAuthSchemeSelect) { + rocksDbAuthSchemeSelect.addEventListener('change', () => this.handleRocksDbAuthSchemeChange()); + } + // Test SQL connection button const testConnBtn = document.getElementById('sql-test-connection-btn'); if (testConnBtn) { @@ -737,6 +751,11 @@ export class PluginModalStepper { testCosmosBtn.addEventListener('click', () => this.testCosmosConnection()); } + const testRocksDbBtn = document.getElementById('rocksdb-test-connection-btn'); + if (testRocksDbBtn) { + testRocksDbBtn.addEventListener('click', () => this.testRocksDbConnection()); + } + const testYamcsBtn = document.getElementById('yamcs-test-connection-btn'); if (testYamcsBtn) { testYamcsBtn.addEventListener('click', () => this.testYamcsConnection()); @@ -1268,6 +1287,10 @@ export class PluginModalStepper { return !!(type && type.toLowerCase() === 'cosmos_query'); } + isRocksDbType(type = this.selectedType) { + return !!(type && type.toLowerCase() === ROCKSDB_PLUGIN_TYPE); + } + isDocumentSearchType(type = this.selectedType) { return !!(type && ['search', 'document_search'].includes(type.toLowerCase())); } @@ -3414,7 +3437,7 @@ export class PluginModalStepper { } isStructuredConfigType(type = this.selectedType) { - return this.isSqlType(type) || this.isCosmosType(type) || this.isDocumentSearchType(type) || this.isBlobStorageType(type) || this.isDatabricksType(type) || this.isSnowflakeType(type) || this.isTableauType(type) || this.isYamcsType(type) || this.isMcpType(type) || this.isSimpleChatType(type) || this.isMsGraphType(type) || this.isAzureMapsType(type) || this.isChartType(type); + return this.isSqlType(type) || this.isCosmosType(type) || this.isRocksDbType(type) || this.isDocumentSearchType(type) || this.isBlobStorageType(type) || this.isDatabricksType(type) || this.isSnowflakeType(type) || this.isTableauType(type) || this.isYamcsType(type) || this.isMcpType(type) || this.isSimpleChatType(type) || this.isMsGraphType(type) || this.isAzureMapsType(type) || this.isChartType(type); } showConfigSectionForType() { @@ -3423,6 +3446,7 @@ export class PluginModalStepper { generic: document.getElementById('generic-config-section'), sql: document.getElementById('sql-config-section'), cosmos: document.getElementById('cosmos-config-section'), + rocksdb: document.getElementById('rocksdb-config-section'), documentSearch: document.getElementById('document-search-config-section'), blobStorage: document.getElementById('blob-storage-config-section'), databricks: document.getElementById('databricks-config-section'), @@ -3453,6 +3477,9 @@ export class PluginModalStepper { } else if (this.isCosmosType()) { showOnly('cosmos'); this.initializeCosmosConfiguration(); + } else if (this.isRocksDbType()) { + showOnly('rocksdb'); + this.initializeRocksDbConfiguration(); } else if (this.isDocumentSearchType()) { showOnly('documentSearch'); this.initializeDocumentSearchConfiguration(); @@ -3530,6 +3557,8 @@ export class PluginModalStepper { titleEl.textContent = 'Database Configuration'; } else if (isCosmosType) { titleEl.textContent = 'Cosmos Configuration'; + } else if (this.isRocksDbType()) { + titleEl.textContent = 'RocksDB Configuration'; } else if (isDocumentSearchType) { titleEl.textContent = 'Document Search Configuration'; } else if (isBlobStorageType) { @@ -3727,6 +3756,7 @@ export class PluginModalStepper { const openApiSection = document.getElementById('openapi-config-section'); const sqlSection = document.getElementById('sql-config-section'); const cosmosSection = document.getElementById('cosmos-config-section'); + const rocksDbSection = document.getElementById('rocksdb-config-section'); const documentSearchSection = document.getElementById('document-search-config-section'); const blobStorageSection = document.getElementById('blob-storage-config-section'); const databricksSection = document.getElementById('databricks-config-section'); @@ -3741,6 +3771,7 @@ export class PluginModalStepper { const isOpenApiVisible = !openApiSection.classList.contains('d-none'); const isSqlVisible = !sqlSection.classList.contains('d-none'); const isCosmosVisible = !cosmosSection.classList.contains('d-none'); + const isRocksDbVisible = !!rocksDbSection && !rocksDbSection.classList.contains('d-none'); const isDocumentSearchVisible = !documentSearchSection.classList.contains('d-none'); const isBlobStorageVisible = !blobStorageSection.classList.contains('d-none'); const isDatabricksVisible = !databricksSection.classList.contains('d-none'); @@ -3901,6 +3932,12 @@ export class PluginModalStepper { this.showError('Timeout must be between 1 and 120 seconds.'); return false; } + } else if (isRocksDbVisible) { + const rocksDbError = this.getRocksDbValidationError(); + if (rocksDbError) { + this.showError(rocksDbError); + return false; + } } else if (isBlobStorageVisible) { const connectionString = document.getElementById('blob-storage-connection-string').value.trim(); const containerName = document.getElementById('blob-storage-container-name').value.trim(); @@ -4686,6 +4723,282 @@ export class PluginModalStepper { } } + initializeRocksDbConfiguration() { + const defaultValues = [ + ['rocksdb-auth-scheme', ROCKSDB_AUTH_SCHEME_NONE], + ['rocksdb-read-only', 'true'], + ['rocksdb-key-encoding', 'utf8'], + ['rocksdb-value-encoding', 'utf8'], + ['rocksdb-column-family', ROCKSDB_DEFAULT_COLUMN_FAMILY], + ['rocksdb-api-key-header', ROCKSDB_DEFAULT_API_KEY_HEADER], + ['rocksdb-max-results', String(ROCKSDB_DEFAULT_MAX_RESULTS)], + ['rocksdb-max-value-bytes', String(ROCKSDB_DEFAULT_MAX_VALUE_BYTES)], + ['rocksdb-timeout', String(ROCKSDB_DEFAULT_TIMEOUT)] + ]; + + defaultValues.forEach(([fieldId, defaultValue]) => { + const field = document.getElementById(fieldId); + if (field && !field.value) { + field.value = defaultValue; + } + }); + + const resultDiv = document.getElementById('rocksdb-test-connection-result'); + if (resultDiv) { + resultDiv.classList.add('d-none'); + } + + this.handleRocksDbAuthSchemeChange(); + } + + setRocksDbFieldValue(fieldId, value) { + const field = document.getElementById(fieldId); + if (field) { + field.value = value === null || value === undefined ? '' : value; + } + } + + handleRocksDbAuthSchemeChange() { + const authScheme = document.getElementById('rocksdb-auth-scheme')?.value || ROCKSDB_AUTH_SCHEME_NONE; + const requiresToken = authScheme === ROCKSDB_AUTH_SCHEME_BEARER || authScheme === ROCKSDB_AUTH_SCHEME_API_KEY; + + document.getElementById('rocksdb-auth-key-group')?.classList.toggle('d-none', !requiresToken); + document.getElementById('rocksdb-api-key-header-group')?.classList.toggle( + 'd-none', + authScheme !== ROCKSDB_AUTH_SCHEME_API_KEY + ); + + const keyInput = document.getElementById('rocksdb-auth-key'); + if (keyInput) { + keyInput.required = requiresToken; + } + } + + getRocksDbKeyPrefixHints() { + const rawValue = document.getElementById('rocksdb-key-prefix-hints')?.value || ''; + return rawValue + .split(/[,\n]/) + .map(value => value.trim()) + .filter(Boolean); + } + + getRocksDbEndpointValue() { + return (document.getElementById('rocksdb-base-url')?.value || '').trim().replace(/\/+$/, ''); + } + + getRocksDbAuthLabel() { + const authScheme = document.getElementById('rocksdb-auth-scheme')?.value || ROCKSDB_AUTH_SCHEME_NONE; + if (authScheme === ROCKSDB_AUTH_SCHEME_BEARER) { + return 'Bearer Token'; + } + if (authScheme === ROCKSDB_AUTH_SCHEME_API_KEY) { + return 'API Key Header'; + } + return 'No Authentication'; + } + + getRocksDbValidationError() { + const baseUrl = (document.getElementById('rocksdb-base-url')?.value || '').trim(); + if (!baseUrl) { + return 'RocksDB service base URL is required.'; + } + if (!/^https?:\/\/.+/i.test(baseUrl)) { + return 'RocksDB service base URL must start with http:// or https://.'; + } + + const authScheme = document.getElementById('rocksdb-auth-scheme')?.value || ROCKSDB_AUTH_SCHEME_NONE; + if (authScheme !== ROCKSDB_AUTH_SCHEME_NONE && !(document.getElementById('rocksdb-auth-key')?.value || '').trim()) { + return 'A service token is required when the RocksDB authentication scheme is Bearer Token or API Key Header.'; + } + + const maxResults = parseInt(document.getElementById('rocksdb-max-results')?.value, 10); + if (Number.isNaN(maxResults) || maxResults < 1 || maxResults > 1000) { + return 'Max results must be between 1 and 1000.'; + } + + const maxValueBytes = parseInt(document.getElementById('rocksdb-max-value-bytes')?.value, 10); + if (Number.isNaN(maxValueBytes) || maxValueBytes < 1 || maxValueBytes > 1048576) { + return 'Max value bytes must be between 1 and 1048576.'; + } + + const timeout = parseInt(document.getElementById('rocksdb-timeout')?.value, 10); + if (Number.isNaN(timeout) || timeout < 1 || timeout > 300) { + return 'Timeout must be between 1 and 300 seconds.'; + } + + return null; + } + + getRocksDbConfiguration() { + const endpoint = this.getRocksDbEndpointValue(); + const authScheme = document.getElementById('rocksdb-auth-scheme')?.value || ROCKSDB_AUTH_SCHEME_NONE; + + const additionalFields = { + base_url: endpoint, + auth_scheme: authScheme, + column_family: (document.getElementById('rocksdb-column-family')?.value || '').trim() || ROCKSDB_DEFAULT_COLUMN_FAMILY, + key_encoding: document.getElementById('rocksdb-key-encoding')?.value || 'utf8', + value_encoding: document.getElementById('rocksdb-value-encoding')?.value || 'utf8', + key_prefix_hints: this.getRocksDbKeyPrefixHints(), + read_only: (document.getElementById('rocksdb-read-only')?.value || 'true') === 'true', + max_results: parseInt(document.getElementById('rocksdb-max-results')?.value, 10) || ROCKSDB_DEFAULT_MAX_RESULTS, + max_value_bytes: parseInt(document.getElementById('rocksdb-max-value-bytes')?.value, 10) || ROCKSDB_DEFAULT_MAX_VALUE_BYTES, + timeout: parseInt(document.getElementById('rocksdb-timeout')?.value, 10) || ROCKSDB_DEFAULT_TIMEOUT + }; + + if (authScheme === ROCKSDB_AUTH_SCHEME_API_KEY) { + additionalFields.api_key_header = (document.getElementById('rocksdb-api-key-header')?.value || '').trim() + || ROCKSDB_DEFAULT_API_KEY_HEADER; + } + + const auth = {}; + if (authScheme === ROCKSDB_AUTH_SCHEME_NONE) { + auth.type = 'NoAuth'; + } else { + auth.type = 'key'; + auth.key = (document.getElementById('rocksdb-auth-key')?.value || '').trim(); + } + + return { endpoint, auth, additionalFields }; + } + + populateRocksDbForm(plugin) { + const additionalFields = plugin.additionalFields || plugin.additional_fields || {}; + const auth = plugin.auth || {}; + const prefixHints = Array.isArray(additionalFields.key_prefix_hints) + ? additionalFields.key_prefix_hints.join('\n') + : (additionalFields.key_prefix_hints || ''); + + this.setRocksDbFieldValue('rocksdb-base-url', additionalFields.base_url || plugin.endpoint || ''); + this.setRocksDbFieldValue('rocksdb-auth-scheme', additionalFields.auth_scheme || ROCKSDB_AUTH_SCHEME_NONE); + this.setRocksDbFieldValue('rocksdb-auth-key', auth.key || ''); + this.setRocksDbFieldValue('rocksdb-api-key-header', additionalFields.api_key_header || ROCKSDB_DEFAULT_API_KEY_HEADER); + this.setRocksDbFieldValue('rocksdb-column-family', additionalFields.column_family || ROCKSDB_DEFAULT_COLUMN_FAMILY); + this.setRocksDbFieldValue('rocksdb-read-only', additionalFields.read_only === false ? 'false' : 'true'); + this.setRocksDbFieldValue('rocksdb-key-encoding', additionalFields.key_encoding || 'utf8'); + this.setRocksDbFieldValue('rocksdb-value-encoding', additionalFields.value_encoding || 'utf8'); + this.setRocksDbFieldValue('rocksdb-key-prefix-hints', prefixHints); + this.setRocksDbFieldValue('rocksdb-max-results', additionalFields.max_results || ROCKSDB_DEFAULT_MAX_RESULTS); + this.setRocksDbFieldValue('rocksdb-max-value-bytes', additionalFields.max_value_bytes || ROCKSDB_DEFAULT_MAX_VALUE_BYTES); + this.setRocksDbFieldValue('rocksdb-timeout', additionalFields.timeout || ROCKSDB_DEFAULT_TIMEOUT); + + this.handleRocksDbAuthSchemeChange(); + } + + setRocksDbSummaryText(elementId, value) { + const element = document.getElementById(elementId); + if (element) { + element.textContent = value; + } + } + + populateRocksDbSummary() { + const rocksDbSection = document.getElementById('summary-rocksdb-section'); + if (!rocksDbSection) { + return; + } + + if (!this.isRocksDbType()) { + rocksDbSection.classList.add('d-none'); + return; + } + + const readOnly = (document.getElementById('rocksdb-read-only')?.value || 'true') === 'true'; + const prefixHints = this.getRocksDbKeyPrefixHints(); + const timeoutValue = (document.getElementById('rocksdb-timeout')?.value || '').trim(); + const keyEncoding = document.getElementById('rocksdb-key-encoding')?.value || 'utf8'; + const valueEncoding = document.getElementById('rocksdb-value-encoding')?.value || 'utf8'; + + this.setRocksDbSummaryText('summary-rocksdb-target', this.getRocksDbEndpointValue() || '-'); + this.setRocksDbSummaryText('summary-rocksdb-auth-scheme', this.getRocksDbAuthLabel()); + this.setRocksDbSummaryText('summary-rocksdb-access', readOnly ? 'Read-only' : 'Reads and writes'); + this.setRocksDbSummaryText( + 'summary-rocksdb-column-family', + (document.getElementById('rocksdb-column-family')?.value || '').trim() || ROCKSDB_DEFAULT_COLUMN_FAMILY + ); + this.setRocksDbSummaryText('summary-rocksdb-encoding', `Keys: ${keyEncoding}, Values: ${valueEncoding}`); + this.setRocksDbSummaryText( + 'summary-rocksdb-max-results', + (document.getElementById('rocksdb-max-results')?.value || '').trim() || '-' + ); + this.setRocksDbSummaryText( + 'summary-rocksdb-max-value-bytes', + (document.getElementById('rocksdb-max-value-bytes')?.value || '').trim() || '-' + ); + this.setRocksDbSummaryText('summary-rocksdb-timeout', timeoutValue ? `${timeoutValue} seconds` : '-'); + this.setRocksDbSummaryText( + 'summary-rocksdb-key-prefix-hints', + prefixHints.length ? prefixHints.join(', ') : 'None configured' + ); + + rocksDbSection.classList.remove('d-none'); + } + + async testRocksDbConnection() { + const btn = document.getElementById('rocksdb-test-connection-btn'); + const resultDiv = document.getElementById('rocksdb-test-connection-result'); + const alertDiv = document.getElementById('rocksdb-test-connection-alert'); + if (!btn || !resultDiv || !alertDiv) return; + + const validationError = this.getRocksDbValidationError(); + if (validationError) { + resultDiv.classList.remove('d-none'); + alertDiv.className = 'alert alert-warning mb-0 py-2 px-3 small'; + alertDiv.textContent = validationError; + return; + } + + const authScheme = document.getElementById('rocksdb-auth-scheme')?.value || ROCKSDB_AUTH_SCHEME_NONE; + const payload = { + base_url: this.getRocksDbEndpointValue(), + auth_scheme: authScheme, + timeout: Math.min(parseInt(document.getElementById('rocksdb-timeout')?.value, 10) || 10, 30) + }; + + if (authScheme === ROCKSDB_AUTH_SCHEME_API_KEY) { + payload.api_key_header = (document.getElementById('rocksdb-api-key-header')?.value || '').trim() + || ROCKSDB_DEFAULT_API_KEY_HEADER; + } + if (authScheme !== ROCKSDB_AUTH_SCHEME_NONE) { + payload.auth_key = (document.getElementById('rocksdb-auth-key')?.value || '').trim(); + } + + const existingPluginContext = this.getTestPluginContext(); + if (existingPluginContext) { + payload.existing_plugin = existingPluginContext; + } + + const originalText = btn.innerHTML; + btn.innerHTML = 'Testing...'; + btn.disabled = true; + resultDiv.classList.add('d-none'); + + try { + const response = await fetch('/api/plugins/test-rocksdb-connection', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + const data = await response.json(); + + resultDiv.classList.remove('d-none'); + if (data.success) { + alertDiv.className = 'alert alert-success mb-0 py-2 px-3 small'; + alertDiv.innerHTML = '' + this.escapeHtml(data.message || 'Connection successful!'); + } else { + alertDiv.className = 'alert alert-danger mb-0 py-2 px-3 small'; + alertDiv.innerHTML = '' + this.escapeHtml(data.error || 'Connection failed.'); + } + } catch (error) { + resultDiv.classList.remove('d-none'); + alertDiv.className = 'alert alert-danger mb-0 py-2 px-3 small'; + alertDiv.innerHTML = 'Test failed: ' + this.escapeHtml(error.message || 'Network error'); + } finally { + btn.innerHTML = originalText; + btn.disabled = false; + } + } + async testYamcsConnection() { const btn = document.getElementById('yamcs-test-connection-btn'); const resultDiv = document.getElementById('yamcs-test-connection-result'); @@ -5341,6 +5654,8 @@ export class PluginModalStepper { document.getElementById('cosmos-auth-type').value = auth.type || 'identity'; document.getElementById('cosmos-auth-key').value = auth.key || ''; this.initializeCosmosConfiguration(); + } else if (this.isRocksDbType(plugin.type)) { + this.populateRocksDbForm(plugin); } else if (this.isDocumentSearchType(plugin.type)) { this.populateDocumentSearchForm(plugin.additionalFields || {}); this.initializeDocumentSearchConfiguration(); @@ -5416,6 +5731,7 @@ export class PluginModalStepper { const openApiSection = document.getElementById('openapi-config-section'); const sqlSection = document.getElementById('sql-config-section'); const cosmosSection = document.getElementById('cosmos-config-section'); + const rocksDbSection = document.getElementById('rocksdb-config-section'); const documentSearchSection = document.getElementById('document-search-config-section'); const blobStorageSection = document.getElementById('blob-storage-config-section'); const databricksSection = document.getElementById('databricks-config-section'); @@ -5427,6 +5743,7 @@ export class PluginModalStepper { const isOpenApiVisible = !openApiSection.classList.contains('d-none'); const isSqlVisible = !sqlSection.classList.contains('d-none'); const isCosmosVisible = !cosmosSection.classList.contains('d-none'); + const isRocksDbVisible = !!rocksDbSection && !rocksDbSection.classList.contains('d-none'); const isDocumentSearchVisible = !documentSearchSection.classList.contains('d-none'); const isBlobStorageVisible = !blobStorageSection.classList.contains('d-none'); const isDatabricksVisible = !databricksSection.classList.contains('d-none'); @@ -5658,6 +5975,16 @@ export class PluginModalStepper { additionalFields.field_hints = this.getCosmosFieldHints(); additionalFields.max_items = parseInt(document.getElementById('cosmos-max-items').value, 10) || 100; additionalFields.timeout = parseInt(document.getElementById('cosmos-timeout').value, 10) || 30; + } else if (isRocksDbVisible) { + const rocksDbError = this.getRocksDbValidationError(); + if (rocksDbError) { + throw new Error(rocksDbError); + } + + const rocksDbConfig = this.getRocksDbConfiguration(); + endpoint = rocksDbConfig.endpoint; + auth = rocksDbConfig.auth; + additionalFields = rocksDbConfig.additionalFields; } else if (isDocumentSearchVisible) { endpoint = INTERNAL_DOCUMENT_SEARCH_ENDPOINT; auth.type = 'NoAuth'; @@ -5833,6 +6160,7 @@ export class PluginModalStepper { const endpointRow = document.getElementById('summary-plugin-endpoint-row'); const databaseTypeRow = document.getElementById('summary-plugin-database-type-row'); + const isRocksDbType = this.isRocksDbType(); if (isSqlType) { // Hide endpoint for SQL plugins since they don't use endpoints @@ -5845,6 +6173,12 @@ export class PluginModalStepper { endpointRow.style.display = ''; document.getElementById('summary-plugin-database-type').textContent = 'Cosmos DB for NoSQL'; databaseTypeRow.style.display = ''; + } else if (isRocksDbType) { + const endpoint = this.getEndpointValue(); + document.getElementById('summary-plugin-endpoint').textContent = endpoint || '-'; + endpointRow.style.display = ''; + document.getElementById('summary-plugin-database-type').textContent = 'RocksDB HTTP service'; + databaseTypeRow.style.display = ''; } else if (isDocumentSearchType) { endpointRow.style.display = 'none'; document.getElementById('summary-plugin-database-type').textContent = 'Internal document search'; @@ -5915,10 +6249,10 @@ export class PluginModalStepper { } const databaseType = this.getSqlDatabaseType(); - if (!isSqlType && !isCosmosType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType && databaseType) { + if (!isSqlType && !isCosmosType && !isRocksDbType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType && databaseType) { document.getElementById('summary-plugin-database-type').textContent = databaseType; databaseTypeRow.style.display = ''; - } else if (!isSqlType && !isCosmosType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType) { + } else if (!isSqlType && !isCosmosType && !isRocksDbType && !isDocumentSearchType && !isBlobStorageType && !isDatabricksType && !isSnowflakeType && !isTableauType && !isYamcsType && !isMcpType && !isSimpleChatType && !isMsGraphType && !isAzureMapsType && !isChartType) { databaseTypeRow.style.display = 'none'; } @@ -5926,6 +6260,7 @@ export class PluginModalStepper { this.populateOpenApiSummary(); this.populateSqlSummary(); this.populateCosmosSummary(); + this.populateRocksDbSummary(); this.populateDocumentSearchSummary(); this.populateBlobStorageSummary(); this.populateDatabricksSummary(); @@ -5962,6 +6297,8 @@ export class PluginModalStepper { return document.getElementById('sql-connection-string').value.trim(); } else if (isCosmosType) { return document.getElementById('cosmos-endpoint').value.trim(); + } else if (this.isRocksDbType()) { + return this.getRocksDbEndpointValue(); } else if (isDocumentSearchType) { return INTERNAL_DOCUMENT_SEARCH_ENDPOINT; } else if (isBlobStorageType) { @@ -6014,6 +6351,8 @@ export class PluginModalStepper { } else if (isCosmosType) { const authType = document.getElementById('cosmos-auth-type')?.value || 'identity'; return authType === 'key' ? 'Account Key' : 'Managed Identity'; + } else if (this.isRocksDbType()) { + return this.getRocksDbAuthLabel(); } else if (isDocumentSearchType) { return 'Internal user context'; } else if (isBlobStorageType) { @@ -6615,6 +6954,8 @@ export class PluginModalStepper { currentEndpoint = document.getElementById('sql-connection-string')?.value || ''; } else if (isCosmosType) { currentEndpoint = document.getElementById('cosmos-endpoint')?.value || ''; + } else if (this.isRocksDbType()) { + currentEndpoint = this.getRocksDbEndpointValue(); } else if (isDocumentSearchType) { currentEndpoint = INTERNAL_DOCUMENT_SEARCH_ENDPOINT; } else if (isDatabricksType) { @@ -6659,6 +7000,12 @@ export class PluginModalStepper { if (currentAuthType === 'key') { currentAuthKey = document.getElementById('cosmos-auth-key')?.value || ''; } + } else if (this.isRocksDbType()) { + const rocksDbAuthScheme = document.getElementById('rocksdb-auth-scheme')?.value || ROCKSDB_AUTH_SCHEME_NONE; + currentAuthType = rocksDbAuthScheme !== ROCKSDB_AUTH_SCHEME_NONE ? 'key' : 'NoAuth'; + if (currentAuthType === 'key') { + currentAuthKey = document.getElementById('rocksdb-auth-key')?.value || ''; + } } else if (isDocumentSearchType) { currentAuthType = 'NoAuth'; } else if (isTableauType) { @@ -6717,6 +7064,8 @@ export class PluginModalStepper { max_items: parseInt(document.getElementById('cosmos-max-items')?.value, 10) || 100, timeout: parseInt(document.getElementById('cosmos-timeout')?.value, 10) || 30 }, null, 2); + } else if (this.isRocksDbType()) { + currentAdditionalFields = JSON.stringify(this.getRocksDbConfiguration().additionalFields, null, 2); } else if (isDocumentSearchType) { currentAdditionalFields = JSON.stringify(this.getDocumentSearchAdditionalFields(), null, 2); } else if (isTableauType) { diff --git a/application/single_app/static/js/workspace/view-utils.js b/application/single_app/static/js/workspace/view-utils.js index 2dee40f19..aa62d9224 100644 --- a/application/single_app/static/js/workspace/view-utils.js +++ b/application/single_app/static/js/workspace/view-utils.js @@ -53,6 +53,7 @@ export function getTypeIcon(type) { const t = type.toLowerCase(); if (t.includes("sql")) return "bi-database"; if (t.includes("cosmos")) return "bi-database-fill-gear"; + if (t.includes("rocksdb")) return "bi-hdd-stack"; if (t.includes("chart")) return "bi-bar-chart-line"; if (t.includes("simplechat")) return "bi-chat-square-dots"; if (t === "mcp" || t.includes("model_context_protocol")) return "bi-diagram-3"; diff --git a/application/single_app/static/json/schemas/rocksdb.definition.json b/application/single_app/static/json/schemas/rocksdb.definition.json new file mode 100644 index 000000000..83dfd04da --- /dev/null +++ b/application/single_app/static/json/schemas/rocksdb.definition.json @@ -0,0 +1,7 @@ +{ + "$schema": "./plugin.definition.schema.json", + "allowedAuthTypes": [ + "NoAuth", + "key" + ] +} diff --git a/application/single_app/templates/_plugin_modal.html b/application/single_app/templates/_plugin_modal.html index bf57cee0f..74988d498 100644 --- a/application/single_app/templates/_plugin_modal.html +++ b/application/single_app/templates/_plugin_modal.html @@ -1352,6 +1352,121 @@
X-API-Key.