From eff6f828ca02e2e16a51f3e516d39a5f4e6b55e4 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Wed, 30 Sep 2026 11:48:54 +0800 Subject: [PATCH 01/16] feat(account-store): add MySQL and Redis providers --- docs/en/guides/01-configuration.md | 88 +- docs/zh/guides/01-configuration.md | 75 +- openviking/server/account_stores/mysql.py | 1261 +++++++++++++++++ .../server/account_stores/mysql_schema.py | 276 ++++ .../server/account_stores/redis_mysql.py | 831 +++++++++++ .../sql/changes/0000_template.sql | 5 + openviking/server/account_stores/sql/init.sql | 99 ++ openviking/server/store_assembly.py | 20 +- pyproject.toml | 10 + tests/server/test_account_services.py | 137 ++ .../server/test_redis_mysql_account_store.py | 524 +++++++ uv.lock | 62 +- 12 files changed, 3384 insertions(+), 4 deletions(-) create mode 100644 openviking/server/account_stores/mysql.py create mode 100644 openviking/server/account_stores/mysql_schema.py create mode 100644 openviking/server/account_stores/redis_mysql.py create mode 100644 openviking/server/account_stores/sql/changes/0000_template.sql create mode 100644 openviking/server/account_stores/sql/init.sql create mode 100644 tests/server/test_redis_mysql_account_store.py diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index d6bd298618..b8fb113bea 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1898,7 +1898,7 @@ When running OpenViking as an HTTP service, add a `server` section to `ov.conf`: | `port` | int | Bind port | `1933` | | `auth_mode` | str / null | Built-in modes: `"dev"`, `"api_key"`, `"trusted"`, `"oidc"`, `"ldap"`. When omitted/null, infer `api_key` from a non-empty `root_api_key`; otherwise infer `dev`. | `null` | | `root_api_key` | str | Root API key for multi-tenant auth in `api_key` mode. In `trusted` mode it is required on every deployment, including localhost; it does not become the source of user identity | `null` | -| `account_store.provider` | str | Complete Account store provider. The built-in `"file"` provider owns account, user, group, and API-key records while preserving the existing AGFS JSON registry and process-local lookup indexes. | `"file"` | +| `account_store.provider` | str | Complete Account store provider. Built-in values are `"file"`, `"mysql"`, and `"redis_mysql"`; it owns account, user, group, and API-key records. `"file"` preserves the existing AGFS JSON registry and owns its process-local lookup indexes. `"redis_mysql"` keeps MySQL authoritative and adds a shared Redis cache for authentication and request identity reads. | `"file"` | | `account_store.params` | object | Provider-specific startup parameters. Framework-owned filesystem and File-provider API-key hashing dependencies cannot be overridden here. | `{}` | | `profile_enabled` | bool | Whether to allow request-scoped cProfile via `profile=1` on HTTP requests. When disabled, the server ignores that query parameter. When enabled, the CLI can display the returned `profile`, while the Python HTTP client currently triggers profiling but does not automatically attach the top-level `profile` field to most SDK return values. | `false` | | `cors_origins` | list | Allowed CORS origins | `["*"]` | @@ -1913,6 +1913,92 @@ When running OpenViking as an HTTP service, add a `server` section to `ov.conf`: | `user_config_defaults.auto_commit_policy` | object | Deployment default auto-commit policy for newly created sessions without an explicit policy. | `null` | | `agent_evolution.enabled` | bool | Startup cluster default for Agent Evolution. Account and Cluster Admin settings may override it at runtime. When enabled, session commits may generate or update cases, trajectories, and experiences according to the session `memory_policy`. Existing memories remain readable and searchable when disabled. | `false` | +The MySQL provider stores a versioned SHA-256 digest of each high-entropy API +key by default and uses a separate digest index for constant-cost lookup. Set +`credential_storage` to `"encrypted"` to retain keys in AES-GCM encrypted form +instead; this requires `credential_encryption_key`, a base64-encoded 32-byte +key. MySQL never persists API-key plaintext. Hash mode exposes only key prefixes +in user listings, while encrypted mode can decrypt the key for the same listing +workflow. `credential_storage` controls the format of newly written credentials; +existing rows are decoded from their `material` prefix. When migrating from +encrypted storage to hash storage, retain `credential_encryption_key` until all +encrypted credentials have been rotated. Replacing that encryption key directly +is not supported because `aesgcm:v1` records do not carry a key identifier. + +Install the MySQL extra and set a non-empty `OV_RESOURCE_ID` before running the +server. This value is stored as `resource_id` on every account-store table and +is the deployment boundary when multiple OpenViking resources share a database. +The provider rejects startup when it is absent. + +The provider never creates or migrates database tables. Create the normalized +tables in `openviking/server/account_stores/sql/init.sql` through your normal +DBA or deployment process before starting the server. Future reviewed schema +changes are stored as ordered SQL files in +`openviking/server/account_stores/sql/changes/`. The initialization script uses +the default `ov_accounts` prefix. When using `account_store.params.table`, +replace only the backtick-quoted `ov_accounts` table identifiers in the +initialization script. Constraint and index names are stable and independent of +the configured table names. + +```json +{ + "server": { + "account_store": { + "provider": "mysql", + "params": { + "host": "db.example.com", + "user": "openviking", + "password": "secret", + "database": "openviking", + "credential_storage": "encrypted", + "credential_encryption_key": "base64-encoded-32-byte-key" + } + } + } +} +``` + +`redis_mysql` requires the `redis-mysql` extra and wraps the same MySQL +parameters. It caches API-key bindings, users, per-user group IDs, and deletion +records for a fixed 60 seconds by default. Cache fills and account mutations +share an account-scoped Redis lease; normal cache hits do not acquire that +lease. Mutations commit to MySQL first, then wait up to 30 seconds for the lease +and invalidate affected cache entries before returning. If that wait times out, +the provider revokes the current lease, reacquires it, and invalidates before +returning the committed MySQL result. Cache fills also maintain an +account-scoped, expiry-scored key index. Each indexed cache fill removes expired +index members, allowing account deletion or recreation to clear only that +account's current cached users, groups, deletion records, and credential data. +API-key bindings are checked against account and user credential fences, so +normal key replacement, revocation, and user deletion invalidate old bindings. +If Redis is unavailable after a MySQL mutation, any stale cache entry remains +valid only until its fixed TTL expires. + +```json +{ + "server": { + "account_store": { + "provider": "redis_mysql", + "params": { + "mysql": { + "host": "db.example.com", + "user": "openviking", + "password": "secret", + "database": "openviking" + }, + "redis": { + "url": "rediss://redis.example.com:6379/0", + "key_prefix": "ov:account-auth", + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + "lock_wait_seconds": 30 + } + } + } + } +} +``` + Omitting `auth_mode` (or setting it to `null`) selects `api_key` when a non-empty `root_api_key` is configured, and `dev` otherwise. `dev` is allowed only on localhost and accepts requests without authentication. An empty-string `root_api_key` is invalid. Explicit `auth_mode: "api_key"` requires a non-empty `root_api_key`, including on localhost; without it, startup fails instead of falling back to development mode. Use the root key with the Admin API to create accounts and user/admin keys; use those tenant-bound keys for data access. `trusted` mode accepts account/user identity headers from a trusted gateway and does not require user-key provisioning first. Its root key is required for startup and every authenticated request, including on localhost. For role resolution, OIDC/LDAP setup, and gateway requirements, see [Authentication](04-authentication.md). diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index 8f06b0be98..1e859f5ffd 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1885,7 +1885,7 @@ ov add-resource ./docs --exclude "*.tmp" | `port` | int | 绑定端口 | `1933` | | `auth_mode` | str / null | 内置模式：`"dev"`、`"api_key"`、`"trusted"`、`"oidc"`、`"ldap"`。省略或设为 null 时，有非空 `root_api_key` 则推导为 `api_key`，否则为 `dev`。 | `null` | | `root_api_key` | str | `api_key` 模式必填的 Root API Key；`trusted` 模式在所有部署中均必填，包括 localhost，不负责解析普通用户身份 | `null` | -| `account_store.provider` | str | 完整 Account Store Provider。内置 `"file"` Provider，统一持久化 Account、User、Group 和 API Key，兼容现有 AGFS JSON 注册表并维护进程内查询索引。 | `"file"` | +| `account_store.provider` | str | 完整 Account Store Provider。内置 `"file"`、`"mysql"` 和 `"redis_mysql"`；统一持久化 Account、User、Group 和 API Key。`"file"` 兼容现有 AGFS JSON 注册表，并由 Provider 自身维护进程内查询索引。`"redis_mysql"` 保持 MySQL 为权威存储，并增加共享 Redis 认证和请求身份读取缓存。 | `"file"` | | `account_store.params` | object | Provider 启动参数。不能通过这里覆盖框架注入的 VikingFS 和 File Provider API Key Hash 配置。 | `{}` | | `profile_enabled` | bool | 是否允许 HTTP 请求通过 `profile=1` 开启请求级 cProfile。关闭时服务端会忽略该请求参数；开启后，CLI 可以显示返回的 `profile`，而 Python HTTP client 默认只触发服务端 profile，不会把顶层 `profile` 字段自动附着到大多数 SDK 返回值上。 | `false` | | `cors_origins` | list | CORS 允许的来源 | `["*"]` | @@ -1900,6 +1900,79 @@ ov add-resource ./docs --exclude "*.tmp" | `user_config_defaults.auto_commit_policy` | object | 新建 Session 未显式指定策略时使用的部署级自动 Commit 默认策略。 | `null` | | `agent_evolution.enabled` | bool | Agent 进化的集群启动默认值，运行时可由 Account 或 Cluster Admin settings 覆盖。开启时，session commit 可按 session `memory_policy` 生成或更新 cases、trajectories 和 experiences；关闭后已有记忆仍可读取和检索。 | `false` | +MySQL Provider 默认保存高熵 API Key 的带版本 SHA-256 摘要，并通过独立摘要索引 +进行固定成本查询。将 `credential_storage` 设为 `"encrypted"` 可改用 AES-GCM +可逆加密；此时必须提供 `credential_encryption_key`，其值为 base64 编码的 +32 字节密钥。MySQL 不会持久化 API Key 原文。哈希模式的用户列表只展示 key +prefix；加密模式可为同一列表工作流解密展示 key。`credential_storage` 仅决定 +新写入凭证的格式，读取时根据每条记录的 `material` 前缀选择解码方式。从加密模式 +迁移到哈希模式时，应保留 `credential_encryption_key`，直到所有旧加密凭证完成 +轮换。`aesgcm:v1` 记录不包含 key id，因此不支持直接替换该加密密钥。 + +安装 MySQL extra 后，启动服务前必须设置非空的 `OV_RESOURCE_ID`。该值会作为 +每张 Account Store 表上的 `resource_id` 写入，并在多个 OpenViking resource +共用一个数据库时充当部署隔离边界。未设置时 Provider 会拒绝启动。 + +Provider 不会创建或迁移数据库表。启动服务前，应通过现有 DBA 或部署流程手工创建 +`openviking/server/account_stores/sql/init.sql` 中定义的规范化表。后续经评审的 +schema 变更按顺序存放于 `openviking/server/account_stores/sql/changes/`。初始化 +脚本使用默认表前缀 `ov_accounts`；如配置了 +`account_store.params.table`，执行前只替换初始化脚本中反引号包裹的 +`ov_accounts` 表标识符。约束名和索引名保持固定，不依赖配置的表名。 + +```json +{ + "server": { + "account_store": { + "provider": "mysql", + "params": { + "host": "db.example.com", + "user": "openviking", + "password": "secret", + "database": "openviking", + "credential_storage": "encrypted", + "credential_encryption_key": "base64-encoded-32-byte-key" + } + } + } +} +``` + +`redis_mysql` 需要安装 `redis-mysql` extra，并在参数中嵌套同一份 MySQL 配置。 +它默认以固定 60 秒 TTL 缓存 API Key 绑定、User、每个 User 的 Group ID 和删除记录。 +缓存重建与同一 Account 的修改共用 Redis 租约，正常缓存命中不获取该租约。修改会先 +提交 MySQL，再最多等待 30 秒获取租约并在返回前失效相关缓存；等待超时后会撤销当前 +租约、重新加锁并完成失效，然后正常返回已提交的 MySQL 结果。缓存回填同时维护 Account +级、按过期时间排序的 key 索引；每次写入已索引缓存时会清理过期成员，因此删除或重建 +Account 时只会清理该 Account 当前的 User、Group、删除记录和凭据缓存。API Key 绑定会 +校验 Account 和 User 的凭据 fence，所以正常的换 Key、吊销和删 User 会使旧绑定失效。 +若 MySQL 修改后 Redis 不可用，遗留缓存最多保持到固定 TTL 到期。 + +```json +{ + "server": { + "account_store": { + "provider": "redis_mysql", + "params": { + "mysql": { + "host": "db.example.com", + "user": "openviking", + "password": "secret", + "database": "openviking" + }, + "redis": { + "url": "rediss://redis.example.com:6379/0", + "key_prefix": "ov:account-auth", + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + "lock_wait_seconds": 30 + } + } + } + } +} +``` + 省略 `auth_mode`（或设为 `null`）时，配置了非空 `root_api_key` 则选择 `api_key`，否则选择 `dev`。`dev` 仅允许监听 localhost，不进行身份认证。`root_api_key` 不能配置为空字符串。 显式设置 `auth_mode: "api_key"` 时，包括 localhost 在内都必须提供非空 `root_api_key`；缺少该 key 会导致启动失败，不会回退到开发模式。使用 root key 调用 Admin API 创建 account 和 user/admin key，数据访问使用这些绑定租户身份的 key。`trusted` 模式接受可信网关注入的 account/user 身份头，无需预先创建 user key；其 root key 在启动和每个认证请求中均必填，包括 localhost。角色解析、OIDC/LDAP 配置与网关要求参见 [身份认证](04-authentication.md)。 diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py new file mode 100644 index 0000000000..22cc9682fe --- /dev/null +++ b/openviking/server/account_stores/mysql.py @@ -0,0 +1,1261 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""SQLAlchemy-backed MySQL implementation of the account and API-key store.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import os +import secrets +import ssl as ssl_module +from contextlib import asynccontextmanager +from datetime import datetime +from typing import AsyncGenerator +from uuid import uuid4 + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from sqlalchemy import and_, case, func, select, update +from sqlalchemy.engine import URL +from sqlalchemy.exc import DBAPIError, IntegrityError +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from openviking.server.account_stores.base import AccountStore +from openviking.server.account_stores.models import ( + AccountSummary, + DeletionRecord, + GroupSummary, + UsersPage, + UserSummary, +) +from openviking.server.account_stores.mysql_schema import ( + AccountStoreModels, + make_account_store_models, + validate_table_prefix, +) +from openviking.server.identity import Role +from openviking_cli.exceptions import ( + AlreadyExistsError, + FailedPreconditionError, + InvalidArgumentError, + NotFoundError, + PermissionDeniedError, +) +from openviking_cli.session.user_id import ( + validate_account_id, + validate_identifier_part, + validate_user_id, +) + + +def _validate(value: str, validator) -> None: + if error := validator(value): + raise InvalidArgumentError(error) + + +def _validate_role(role: str) -> Role: + if role == Role.ROOT: + raise PermissionDeniedError("Account users cannot be assigned ROOT") + if role not in (Role.ADMIN, Role.USER): + raise InvalidArgumentError("Account user role must be user or admin.") + return Role(role) + + +def _like_pattern(pattern: str) -> str: + escaped = [] + for character in pattern: + if character == "*": + escaped.append("%") + elif character == "?": + escaped.append("_") + elif character in {"%", "_", "="}: + escaped.append("=" + character) + else: + escaped.append(character) + return "".join(escaped) + + +def _timestamp(value: datetime) -> str: + return value.isoformat() + + +def _credential_digest(api_key: str) -> str: + """Return the lookup digest for a high-entropy API key.""" + return hashlib.sha256(api_key.encode()).hexdigest() + + +_DIGEST_CREDENTIAL_PREFIX = "sha256:v1:" +_ENCRYPTED_CREDENTIAL_PREFIX = "aesgcm:v1:" + + +def _deletion(record) -> DeletionRecord | None: + if record.deletion_task_id is None: + return None + return DeletionRecord( + task_id=record.deletion_task_id, + owner_account_id=record.deletion_owner_account_id, + owner_user_id=record.deletion_owner_user_id, + ) + + +class MySQLAccountStore(AccountStore): + """A manually provisioned relational account store scoped by ``resource_id``.""" + + def __init__(self, *, params: dict[str, object] | None = None) -> None: + values = dict(params or {}) + try: + table_prefix = validate_table_prefix(values.pop("table", "ov_accounts")) + except ValueError as exc: + raise InvalidArgumentError(str(exc)) from None + self._models: AccountStoreModels = make_account_store_models(table_prefix) + self._tables = self._models.table_names + self._resource_id = os.environ.get("OV_RESOURCE_ID", "").strip() + if not self._resource_id: + raise InvalidArgumentError("OV_RESOURCE_ID is required for MySQL account storage") + + credential_storage = values.pop("credential_storage", "hash") + encryption_key = values.pop("credential_encryption_key", None) + self._credential_storage = self._validate_credential_storage(credential_storage) + self._credential_cipher = self._new_credential_cipher(encryption_key) + if self._credential_storage == "encrypted" and self._credential_cipher is None: + raise InvalidArgumentError( + "encrypted MySQL credential storage requires credential_encryption_key" + ) + self._url, engine_options = self._connection_options(values) + self._engine_options = engine_options + self._engine: AsyncEngine | None = None + self._sessions: async_sessionmaker[AsyncSession] | None = None + + @staticmethod + def _validate_credential_storage(mode: object) -> str: + if mode not in {"hash", "encrypted"}: + raise InvalidArgumentError("credential_storage must be 'hash' or 'encrypted'") + return str(mode) + + @staticmethod + def _new_credential_cipher(encryption_key: object) -> AESGCM | None: + if encryption_key is None: + return None + if not isinstance(encryption_key, str): + raise InvalidArgumentError("credential_encryption_key must be base64-encoded") + try: + key = base64.b64decode(encryption_key.encode(), altchars=b"-_", validate=True) + except (binascii.Error, ValueError, UnicodeError) as exc: + raise InvalidArgumentError("credential_encryption_key must be base64-encoded") from exc + if len(key) != 32: + raise InvalidArgumentError("credential_encryption_key must decode to exactly 32 bytes") + return AESGCM(key) + + @staticmethod + def _credential_associated_data(account_id: str, user_id: str) -> bytes: + return f"{account_id}\0{user_id}".encode() + + def _encode_credential(self, account_id: str, user_id: str, api_key: str) -> str: + if self._credential_storage == "hash": + return _DIGEST_CREDENTIAL_PREFIX + _credential_digest(api_key) + assert self._credential_cipher is not None + nonce = secrets.token_bytes(12) + ciphertext = self._credential_cipher.encrypt( + nonce, + api_key.encode(), + self._credential_associated_data(account_id, user_id), + ) + return _ENCRYPTED_CREDENTIAL_PREFIX + base64.urlsafe_b64encode(nonce + ciphertext).decode() + + def _decrypt_credential(self, account_id: str, user_id: str, material: str) -> str | None: + if self._credential_cipher is None or not material.startswith(_ENCRYPTED_CREDENTIAL_PREFIX): + return None + try: + payload = base64.urlsafe_b64decode( + material[len(_ENCRYPTED_CREDENTIAL_PREFIX) :].encode() + ) + return self._credential_cipher.decrypt( + payload[:12], + payload[12:], + self._credential_associated_data(account_id, user_id), + ).decode() + except (InvalidTag, ValueError, UnicodeError): + return None + + def _display_credential(self, account_id: str, user_id: str, material: str) -> str | None: + if material.startswith(_DIGEST_CREDENTIAL_PREFIX): + return None + return self._decrypt_credential(account_id, user_id, material) + + def _matches_credential( + self, account_id: str, user_id: str, material: str, api_key: str + ) -> bool: + if material.startswith(_DIGEST_CREDENTIAL_PREFIX): + expected = _DIGEST_CREDENTIAL_PREFIX + _credential_digest(api_key) + return hmac.compare_digest(material, expected) + plaintext = self._decrypt_credential(account_id, user_id, material) + return plaintext is not None and hmac.compare_digest(plaintext, api_key) + + def _new_credential( + self, + account_id: str, + user_id: str, + user_pk: int, + api_key: str, + *, + sequence: int, + ): + Credential = self._models.Credential + return Credential( + credential_id=uuid4().hex, + resource_id=self._resource_id, + user_pk=user_pk, + active_user_pk=user_pk, + material=self._encode_credential(account_id, user_id, api_key), + lookup_digest=_credential_digest(api_key), + prefix=api_key[:8], + sequence=sequence, + ) + + @staticmethod + def _connection_options( + values: dict[str, object], + ) -> tuple[URL, dict[str, object]]: + allowed = { + "user", + "password", + "database", + "host", + "port", + "minsize", + "maxsize", + "connect_timeout", + "ssl", + "unix_socket", + } + unknown = values.keys() - allowed + if unknown: + raise InvalidArgumentError( + f"Unknown MySQL account store parameters: {', '.join(sorted(unknown))}" + ) + missing = [name for name in ("user", "password", "database") if name not in values] + if missing: + raise InvalidArgumentError(f"MySQL account store requires: {', '.join(missing)}") + + connect_args: dict[str, object] = { + "connect_timeout": values.get("connect_timeout", 10), + } + if socket := values.get("unix_socket"): + connect_args["unix_socket"] = socket + ssl = values.get("ssl") + if isinstance(ssl, dict): + unknown_ssl = ssl.keys() - {"ca", "cert", "key", "check_hostname"} + if unknown_ssl: + raise InvalidArgumentError(f"Unknown MySQL TLS options: {sorted(unknown_ssl)}") + context = ssl_module.create_default_context(cafile=ssl.get("ca")) + context.check_hostname = bool(ssl.get("check_hostname", True)) + if ssl.get("cert"): + context.load_cert_chain(ssl["cert"], ssl.get("key")) + connect_args["ssl"] = context + elif ssl is True: + connect_args["ssl"] = ssl_module.create_default_context() + elif ssl not in (None, False): + raise InvalidArgumentError("ssl must be a boolean or TLS options mapping") + + url = URL.create( + "mysql+aiomysql", + username=str(values["user"]), + password=str(values["password"]), + host=str(values.get("host", "127.0.0.1")), + port=int(values.get("port", 3306)), + database=str(values["database"]), + query={"charset": "utf8mb4"}, + ) + return url, { + "pool_size": int(values.get("maxsize", 10)), + "max_overflow": 0, + "pool_pre_ping": True, + "pool_recycle": 3600, + "connect_args": connect_args, + "isolation_level": "READ COMMITTED", + } + + async def load(self) -> None: + if self._engine is not None: + return + try: + engine = create_async_engine(self._url, **self._engine_options) + sessions = async_sessionmaker(engine, expire_on_commit=False) + async with sessions() as session: + for model in ( + self._models.Account, + self._models.User, + self._models.Group, + self._models.GroupMember, + self._models.Credential, + ): + await session.execute(select(*model.__table__.columns).limit(1)) + except ImportError: + raise RuntimeError("MySQL account storage requires openviking[mysql]") from None + except DBAPIError as exc: + await engine.dispose() + original_args = getattr(exc.orig, "args", ()) + error_code = original_args[0] if original_args else None + message = str(exc).lower() + if error_code in {1054, 1146} or any( + marker in message + for marker in ("unknown column", "doesn't exist", "does not exist") + ): + raise RuntimeError( + "MySQL account schema is missing or incompatible; create " + "all configured relational tables before starting" + ) from exc + raise + except BaseException: + if "engine" in locals(): + await engine.dispose() + raise + self._engine = engine + self._sessions = sessions + + async def close(self) -> None: + if self._engine is not None: + await self._engine.dispose() + self._engine = None + self._sessions = None + + @property + def _session_factory(self) -> async_sessionmaker[AsyncSession]: + assert self._sessions is not None + return self._sessions + + @asynccontextmanager + async def _transaction(self) -> AsyncGenerator[AsyncSession, None]: + async with self._session_factory() as session: + async with session.begin(): + yield session + + async def _lock_account(self, session: AsyncSession, account_id: str): + Account = self._models.Account + account = await session.scalar( + select(Account) + .where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, + ) + .with_for_update() + ) + if account is None: + raise NotFoundError(account_id, "account") + return account + + async def _lock_user(self, session: AsyncSession, account, user_id: str): + User = self._models.User + user = await session.scalar( + select(User) + .where( + User.resource_id == self._resource_id, + User.account_pk == account.account_pk, + User.user_id == user_id, + ) + .with_for_update() + ) + if user is None: + raise NotFoundError(user_id, "user") + return user + + @staticmethod + def _require_active(record) -> None: + deletion = _deletion(record) + if deletion: + raise FailedPreconditionError( + "Identity deletion is in progress", details={"task_id": deletion["task_id"]} + ) + + async def _protect_last_admin(self, session: AsyncSession, account, user) -> None: + if user.role != Role.ADMIN or user.deletion_task_id: + return + User = self._models.User + count = await session.scalar( + select(func.count()) + .select_from(User) + .where( + User.resource_id == self._resource_id, + User.account_pk == account.account_pk, + User.role == Role.ADMIN, + User.deletion_task_id.is_(None), + ) + ) + if count <= 1: + raise FailedPreconditionError("Cannot remove the last active account admin") + + @staticmethod + def _account_summary(account, user_count: int) -> AccountSummary: + return { + "account_id": account.account_id, + "created_at": _timestamp(account.created_at), + "user_count": user_count, + "status": "deleting" if account.deletion_task_id else "active", + **({"task_id": account.deletion_task_id} if account.deletion_task_id else {}), + } + + async def get_account(self, account_id: str) -> AccountSummary | None: + Account, User = self._models.Account, self._models.User + async with self._session_factory() as session: + row = ( + await session.execute( + select(Account, func.count(User.user_pk)) + .outerjoin( + User, + and_( + User.resource_id == Account.resource_id, + User.account_pk == Account.account_pk, + ), + ) + .where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, + ) + .group_by(Account.account_pk) + ) + ).one_or_none() + return self._account_summary(*row) if row else None + + async def get_user(self, account_id: str, user_id: str) -> UserSummary | None: + Account, User = self._models.Account, self._models.User + async with self._session_factory() as session: + user = await session.scalar( + select(User) + .join( + Account, + and_( + Account.resource_id == User.resource_id, + Account.account_pk == User.account_pk, + ), + ) + .where( + User.resource_id == self._resource_id, + Account.account_id == account_id, + User.user_id == user_id, + ) + ) + return UserSummary(user_id=user.user_id, role=user.role) if user else None + + async def create_account_with_api_key( + self, + account_id: str, + admin_user_id: str, + api_key: str, + ) -> None: + _validate(account_id, validate_account_id) + _validate(admin_user_id, validate_user_id) + Account, User = self._models.Account, self._models.User + try: + async with self._transaction() as session: + account = Account( + resource_id=self._resource_id, account_id=account_id + ) + session.add(account) + await session.flush() + user = User( + resource_id=self._resource_id, + account_pk=account.account_pk, + user_id=admin_user_id, + role=Role.ADMIN, + ) + session.add(user) + await session.flush() + session.add( + self._new_credential( + account_id, + admin_user_id, + user.user_pk, + api_key, + sequence=1, + ) + ) + except IntegrityError as exc: + raise AlreadyExistsError(account_id, "account") from exc + + async def create_user(self, account_id: str, user_id: str, role: str) -> None: + _validate(user_id, validate_user_id) + role = _validate_role(role) + User = self._models.User + try: + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + session.add( + User( + resource_id=self._resource_id, + account_pk=account.account_pk, + user_id=user_id, + role=role, + ) + ) + await session.flush() + except IntegrityError as exc: + raise AlreadyExistsError(user_id, "user") from exc + + async def create_user_with_api_key( + self, + account_id: str, + user_id: str, + role: str, + api_key: str, + ) -> None: + _validate(user_id, validate_user_id) + role = _validate_role(role) + User = self._models.User + try: + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + user = User( + resource_id=self._resource_id, + account_pk=account.account_pk, + user_id=user_id, + role=role, + ) + session.add(user) + await session.flush() + session.add( + self._new_credential( + account_id, + user_id, + user.user_pk, + api_key, + sequence=1, + ) + ) + except IntegrityError as exc: + raise AlreadyExistsError(user_id, "user") from exc + + async def delete_account(self, account_id: str) -> None: + async with self._transaction() as session: + await session.delete(await self._lock_account(session, account_id)) + + async def list_accounts( + self, + name_filter: str | None = None, + limit: int | None = None, + page: int = 1, + query_filter: str | None = None, + ) -> list[AccountSummary]: + Account, User = self._models.Account, self._models.User + predicates = [Account.resource_id == self._resource_id] + if name_filter: + predicates.append(Account.account_id.like(_like_pattern(name_filter), escape="=")) + if query := (query_filter or "").strip(): + predicates.append(func.lower(Account.account_id).like(f"%{query.casefold()}%")) + statement = ( + select(Account, func.count(User.user_pk)) + .outerjoin( + User, + and_( + User.resource_id == Account.resource_id, + User.account_pk == Account.account_pk, + ), + ) + .where(*predicates) + .group_by(Account.account_pk) + .order_by(Account.created_at, Account.account_id) + ) + if limit is not None: + statement = statement.limit(limit).offset((max(1, page) - 1) * limit) + async with self._session_factory() as session: + rows = (await session.execute(statement)).all() + return [self._account_summary(account, count) for account, count in rows] + + async def list_users_page( + self, + account_id: str, + limit: int | None = 100, + name_filter: str | None = None, + role_filter: str | None = None, + page: int = 1, + query_filter: str | None = None, + ) -> UsersPage: + User, Credential = self._models.User, self._models.Credential + async with self._session_factory() as session: + # One HTTP response must derive its rows and summary from one MVCC snapshot. + await session.connection( + execution_options={"isolation_level": "REPEATABLE READ"} + ) + account = await session.scalar( + select(self._models.Account).where( + self._models.Account.resource_id == self._resource_id, + self._models.Account.account_id == account_id, + ) + ) + if account is None: + raise NotFoundError(account_id, "account") + base = [ + User.resource_id == self._resource_id, + User.account_pk == account.account_pk, + User.deletion_task_id.is_(None), + ] + account_total, manager_count = ( + await session.execute( + select( + func.count(), + func.coalesce(func.sum(case((User.role == Role.ADMIN, 1), else_=0)), 0), + ).where(*base) + ) + ).one() + key_count = await session.scalar( + select(func.count()) + .select_from(Credential) + .join( + User, + and_( + User.resource_id == Credential.resource_id, + User.user_pk == Credential.user_pk, + ), + ) + .where(*base, Credential.revoked.is_(False)) + ) + filters = list(base) + if name_filter: + filters.append(User.user_id.like(_like_pattern(name_filter), escape="=")) + if role_filter: + filters.append(User.role == role_filter) + if query := (query_filter or "").strip(): + filters.append(func.lower(User.user_id).like(f"%{query.casefold()}%")) + total = await session.scalar(select(func.count()).select_from(User).where(*filters)) + statement = select(User).where(*filters).order_by(User.created_at, User.user_pk) + if limit is not None: + statement = statement.limit(limit).offset((max(1, page) - 1) * limit) + users = list((await session.scalars(statement)).all()) + credentials = {} + if users: + credentials = { + credential.user_pk: credential + for credential in ( + await session.scalars( + select(Credential).where( + Credential.resource_id == self._resource_id, + Credential.user_pk.in_([user.user_pk for user in users]), + Credential.revoked.is_(False), + ) + ) + ) + } + result: list[UserSummary] = [] + for user in users: + row: UserSummary = {"user_id": user.user_id, "role": user.role} + if credential := credentials.get(user.user_pk): + api_key = self._display_credential(account_id, user.user_id, credential.material) + if api_key is None: + row["key_prefix"] = credential.prefix + else: + row["api_key"] = api_key + result.append(row) + return { + "users": result, + "total": total, + "account_total": account_total, + "manager_count": manager_count, + "key_count": key_count, + } + + async def set_role(self, account_id: str, user_id: str, role: str) -> None: + role = _validate_role(role) + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + user = await self._lock_user(session, account, user_id) + self._require_active(user) + if role != Role.ADMIN: + await self._protect_last_admin(session, account, user) + user.role = role + + async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> dict[str, int]: + for account_id, user_ids in identities.items(): + _validate(account_id, validate_account_id) + for user_id in user_ids: + _validate(user_id, validate_user_id) + created_accounts = 0 + created_users = 0 + Account, User = self._models.Account, self._models.User + for account_id, user_ids in identities.items(): + if not user_ids: + continue + async with self._transaction() as session: + account = await session.scalar( + select(Account).where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, + ) + ) + if account is None: + try: + async with session.begin_nested(): + account = Account( + resource_id=self._resource_id, + account_id=account_id, + ) + session.add(account) + await session.flush() + created_accounts += 1 + except IntegrityError: + account = await self._lock_account(session, account_id) + else: + account = await self._lock_account(session, account_id) + if account.deletion_task_id: + continue + existing = set( + ( + await session.scalars( + select(User.user_id).where( + User.resource_id == self._resource_id, + User.account_pk == account.account_pk, + User.user_id.in_(user_ids), + ) + ) + ).all() + ) + for user_id in sorted(user_ids - existing): + session.add( + User( + resource_id=self._resource_id, + account_pk=account.account_pk, + user_id=user_id, + role=Role.USER, + ) + ) + created_users += 1 + return {"created_accounts": created_accounts, "created_users": created_users} + + async def begin_deletion( + self, + account_id: str, + user_id: str | None, + *, + task_id: str, + owner_account_id: str, + owner_user_id: str, + ) -> tuple[DeletionRecord, bool]: + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + record = account + if user_id is not None: + self._require_active(account) + record = await self._lock_user(session, account, user_id) + await self._protect_last_admin(session, account, record) + if current := _deletion(record): + return current, False + record.deletion_task_id = task_id + record.deletion_owner_account_id = owner_account_id + record.deletion_owner_user_id = owner_user_id + return DeletionRecord( + task_id=task_id, + owner_account_id=owner_account_id, + owner_user_id=owner_user_id, + ), True + + async def replace_deletion_task( + self, + account_id: str, + user_id: str | None, + *, + expected_task_id: str, + task_id: str, + owner_account_id: str, + owner_user_id: str, + ) -> DeletionRecord: + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + record = account + if user_id is not None: + self._require_active(account) + record = await self._lock_user(session, account, user_id) + current = _deletion(record) + if current is None or current["task_id"] != expected_task_id: + return current or DeletionRecord() + record.deletion_task_id = task_id + record.deletion_owner_account_id = owner_account_id + record.deletion_owner_user_id = owner_user_id + return DeletionRecord( + task_id=task_id, + owner_account_id=owner_account_id, + owner_user_id=owner_user_id, + ) + + async def finish_deletion(self, account_id: str, user_id: str | None, task_id: str) -> bool: + async with self._transaction() as session: + try: + account = await self._lock_account(session, account_id) + except NotFoundError: + return False + if user_id is None: + if account.deletion_task_id != task_id: + return False + await session.delete(account) + return True + if account.deletion_task_id: + return False + try: + user = await self._lock_user(session, account, user_id) + except NotFoundError: + return False + if user.deletion_task_id != task_id: + return False + await session.delete(user) + return True + + async def get_deletion( + self, account_id: str, user_id: str | None = None + ) -> DeletionRecord | None: + if user_id is None: + async with self._session_factory() as session: + account = await session.scalar( + select(self._models.Account).where( + self._models.Account.resource_id == self._resource_id, + self._models.Account.account_id == account_id, + ) + ) + return _deletion(account) if account else None + + Account, User = self._models.Account, self._models.User + async with self._session_factory() as session: + row = ( + await session.execute( + select(Account, User) + .outerjoin( + User, + and_( + User.resource_id == Account.resource_id, + User.account_pk == Account.account_pk, + User.user_id == user_id, + ), + ) + .where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, + ) + ) + ).one_or_none() + if row is None: + return None + account, user = row + return _deletion(account) or (_deletion(user) if user else None) + + async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: + Account, User = self._models.Account, self._models.User + async with self._session_factory() as session: + accounts = ( + await session.scalars( + select(Account).where( + Account.resource_id == self._resource_id, + Account.deletion_task_id.is_not(None), + ) + ) + ).all() + users = ( + await session.execute( + select(Account.account_id, User) + .join( + User, + and_( + User.resource_id == Account.resource_id, + User.account_pk == Account.account_pk, + ), + ) + .where( + Account.resource_id == self._resource_id, + Account.deletion_task_id.is_(None), + User.deletion_task_id.is_not(None), + ) + ) + ).all() + return [(account.account_id, None, _deletion(account)) for account in accounts] + [ + (account_id, user.user_id, _deletion(user)) for account_id, user in users + ] + + async def create_group(self, account_id: str, group_id: str) -> GroupSummary: + _validate(group_id, lambda value: validate_identifier_part(value, "group_id")) + Group = self._models.Group + try: + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + session.add( + Group( + resource_id=self._resource_id, + account_pk=account.account_pk, + group_id=group_id, + ) + ) + await session.flush() + except IntegrityError as exc: + raise AlreadyExistsError(group_id, "group") from exc + return GroupSummary(group_id=group_id, member_count=0) + + async def _account_for_read(self, session: AsyncSession, account_id: str): + account = await session.scalar( + select(self._models.Account).where( + self._models.Account.resource_id == self._resource_id, + self._models.Account.account_id == account_id, + ) + ) + if account is None: + raise NotFoundError(account_id, "account") + return account + + async def get_groups(self, account_id: str) -> list[GroupSummary]: + Group, GroupMember = self._models.Group, self._models.GroupMember + async with self._session_factory() as session: + account = await self._account_for_read(session, account_id) + rows = ( + await session.execute( + select(Group.group_id, func.count(GroupMember.user_pk)) + .outerjoin( + GroupMember, + and_( + GroupMember.resource_id == Group.resource_id, + GroupMember.group_pk == Group.group_pk, + ), + ) + .where( + Group.resource_id == self._resource_id, + Group.account_pk == account.account_pk, + ) + .group_by(Group.group_pk) + .order_by(Group.group_id) + ) + ).all() + return [GroupSummary(group_id=group_id, member_count=count) for group_id, count in rows] + + async def get_group_members(self, account_id: str, group_id: str) -> list[str]: + Group, GroupMember, User = (self._models.Group, self._models.GroupMember, self._models.User) + async with self._session_factory() as session: + account = await self._account_for_read(session, account_id) + group = await session.scalar( + select(Group).where( + Group.resource_id == self._resource_id, + Group.account_pk == account.account_pk, + Group.group_id == group_id, + ) + ) + if group is None: + raise NotFoundError(group_id, "group") + return list( + ( + await session.scalars( + select(User.user_id) + .join( + GroupMember, + and_( + GroupMember.resource_id == User.resource_id, + GroupMember.user_pk == User.user_pk, + ), + ) + .where( + GroupMember.resource_id == self._resource_id, + GroupMember.group_pk == group.group_pk, + ) + .order_by(User.user_id) + ) + ).all() + ) + + async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, ...]: + Account, User, Group, GroupMember = ( + self._models.Account, + self._models.User, + self._models.Group, + self._models.GroupMember, + ) + async with self._session_factory() as session: + group_ids = ( + await session.scalars( + select(Group.group_id) + .select_from(Account) + .join( + User, + and_( + User.resource_id == Account.resource_id, + User.account_pk == Account.account_pk, + ), + ) + .outerjoin( + GroupMember, + and_( + GroupMember.resource_id == User.resource_id, + GroupMember.user_pk == User.user_pk, + ), + ) + .outerjoin( + Group, + and_( + Group.resource_id == GroupMember.resource_id, + Group.group_pk == GroupMember.group_pk, + ), + ) + .where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, + User.user_id == user_id, + ) + .order_by(Group.group_id) + ) + ).all() + return tuple(group_id for group_id in group_ids if group_id is not None) + + async def add_group_member(self, account_id: str, group_id: str, user_id: str) -> bool: + Group, GroupMember = self._models.Group, self._models.GroupMember + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + group = await session.scalar( + select(Group) + .where( + Group.resource_id == self._resource_id, + Group.account_pk == account.account_pk, + Group.group_id == group_id, + ) + .with_for_update() + ) + if group is None: + raise NotFoundError(group_id, "group") + user = await self._lock_user(session, account, user_id) + self._require_active(user) + if ( + await session.scalar( + select(GroupMember.group_pk).where( + GroupMember.resource_id == self._resource_id, + GroupMember.group_pk == group.group_pk, + GroupMember.user_pk == user.user_pk, + ) + ) + is None + ): + session.add( + GroupMember( + resource_id=self._resource_id, + group_pk=group.group_pk, + user_pk=user.user_pk, + ) + ) + return True + + async def remove_group_member(self, account_id: str, group_id: str, user_id: str) -> bool: + Group, GroupMember, User = (self._models.Group, self._models.GroupMember, self._models.User) + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + group = await session.scalar( + select(Group) + .where( + Group.resource_id == self._resource_id, + Group.account_pk == account.account_pk, + Group.group_id == group_id, + ) + .with_for_update() + ) + if group is None: + raise NotFoundError(group_id, "group") + membership = await session.scalar( + select(GroupMember) + .join( + User, + and_( + User.resource_id == GroupMember.resource_id, + User.user_pk == GroupMember.user_pk, + ), + ) + .where( + GroupMember.resource_id == self._resource_id, + GroupMember.group_pk == group.group_pk, + User.account_pk == account.account_pk, + User.user_id == user_id, + ) + ) + if membership is None: + return False + await session.delete(membership) + return True + + async def delete_group(self, account_id: str, group_id: str) -> None: + Group, GroupMember = self._models.Group, self._models.GroupMember + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + group = await session.scalar( + select(Group) + .where( + Group.resource_id == self._resource_id, + Group.account_pk == account.account_pk, + Group.group_id == group_id, + ) + .with_for_update() + ) + if group is None: + raise NotFoundError(group_id, "group") + if await session.scalar( + select(func.count()) + .select_from(GroupMember) + .where( + GroupMember.resource_id == self._resource_id, + GroupMember.group_pk == group.group_pk, + ) + ): + raise FailedPreconditionError("Group must be empty before deletion") + await session.delete(group) + + async def replace_active_user_api_key( + self, account_id: str, user_id: str, api_key: str + ) -> None: + Credential = self._models.Credential + async with self._transaction() as session: + account = await self._lock_account(session, account_id) + self._require_active(account) + user = await self._lock_user(session, account, user_id) + self._require_active(user) + sequence = await session.scalar( + select(func.coalesce(func.max(Credential.sequence), 0)).where( + Credential.resource_id == self._resource_id, + Credential.user_pk == user.user_pk, + ) + ) + await session.execute( + update(Credential) + .where( + Credential.resource_id == self._resource_id, + Credential.user_pk == user.user_pk, + Credential.revoked.is_(False), + ) + .values( + revoked=True, + active_user_pk=None, + version=Credential.version + 1, + revoked_at=func.utc_timestamp(6), + ) + ) + session.add( + self._new_credential( + account_id, + user_id, + user.user_pk, + api_key, + sequence=sequence + 1, + ) + ) + + async def verify_api_key( + self, + api_key: str, + *, + account_id_hint: str | None = None, + user_id_hint: str | None = None, + ) -> tuple[str, str] | None: + Account, User, Credential = ( + self._models.Account, + self._models.User, + self._models.Credential, + ) + predicates = [ + Credential.resource_id == self._resource_id, + Credential.lookup_digest == _credential_digest(api_key), + Credential.revoked.is_(False), + ] + if account_id_hint is not None: + predicates.append(Account.account_id == account_id_hint) + if user_id_hint is not None: + predicates.append(User.user_id == user_id_hint) + async with self._session_factory() as session: + candidates = ( + await session.execute( + select(Credential.material, Account.account_id, User.user_id) + .join( + User, + and_( + User.resource_id == Credential.resource_id, + User.user_pk == Credential.user_pk, + ), + ) + .join( + Account, + and_( + Account.resource_id == User.resource_id, + Account.account_pk == User.account_pk, + ), + ) + .where(*predicates) + ) + ).all() + for material, candidate_account_id, candidate_user_id in candidates: + if self._matches_credential(candidate_account_id, candidate_user_id, material, api_key): + return candidate_account_id, candidate_user_id + return None + + async def revoke_active_api_keys(self, account_id: str, user_id: str | None = None) -> None: + User, Credential = self._models.User, self._models.Credential + async with self._transaction() as session: + account = await session.scalar( + select(self._models.Account) + .where( + self._models.Account.resource_id == self._resource_id, + self._models.Account.account_id == account_id, + ) + .with_for_update() + ) + if account is None: + return + user_ids = select(User.user_pk).where( + User.resource_id == self._resource_id, + User.account_pk == account.account_pk, + *([User.user_id == user_id] if user_id is not None else []), + ) + await session.execute( + update(Credential) + .where( + Credential.resource_id == self._resource_id, + Credential.user_pk.in_(user_ids), + Credential.revoked.is_(False), + ) + .values( + revoked=True, + active_user_pk=None, + version=Credential.version + 1, + revoked_at=func.utc_timestamp(6), + ) + ) + + async def get_user_key_fingerprint(self, account_id: str, user_id: str) -> str | None: + Account, User, Credential = ( + self._models.Account, + self._models.User, + self._models.Credential, + ) + async with self._session_factory() as session: + material = await session.scalar( + select(Credential.material) + .join( + User, + and_( + User.resource_id == Credential.resource_id, + User.user_pk == Credential.user_pk, + ), + ) + .join( + Account, + and_( + Account.resource_id == User.resource_id, + Account.account_pk == User.account_pk, + ), + ) + .where( + Credential.resource_id == self._resource_id, + Credential.revoked.is_(False), + Account.account_id == account_id, + Account.deletion_task_id.is_(None), + User.user_id == user_id, + User.deletion_task_id.is_(None), + ) + ) + return hashlib.sha256(material.encode()).hexdigest() if material else None diff --git a/openviking/server/account_stores/mysql_schema.py b/openviking/server/account_stores/mysql_schema.py new file mode 100644 index 0000000000..11363a1ced --- /dev/null +++ b/openviking/server/account_stores/mysql_schema.py @@ -0,0 +1,276 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""SQLAlchemy schema for the MySQL account-store provider. + +The table prefix is an operational namespace, while ``resource_id`` is the +tenant boundary. The latter is deliberately present in every primary lookup, +unique key, and foreign-key relationship so one database can hold many +OpenViking deployments safely. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Optional + +from sqlalchemy import ( + BigInteger, + Boolean, + ForeignKeyConstraint, + Index, + PrimaryKeyConstraint, + String, + Text, + UniqueConstraint, + text, +) +from sqlalchemy.dialects.mysql import DATETIME +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +def _schema_object_name(kind: str, semantic_name: str) -> str: + """Return a stable schema-object name independent of configured table names.""" + return f"{kind}_ov_as_{semantic_name}" + + +def validate_table_prefix(table_prefix: object) -> str: + if not isinstance(table_prefix, str) or not re.fullmatch( + r"[A-Za-z][A-Za-z0-9_]{0,49}", table_prefix + ): + raise ValueError( + "MySQL table must start with a letter, contain only letters, " + "digits, or underscores, and be at most 50 characters" + ) + return table_prefix + + +@dataclass(frozen=True) +class AccountStoreModels: + Base: type[DeclarativeBase] + Account: type + User: type + Group: type + GroupMember: type + Credential: type + table_prefix: str + + @property + def table_names(self) -> dict[str, str]: + return { + "accounts": self.Account.__tablename__, + "users": self.User.__tablename__, + "groups": self.Group.__tablename__, + "memberships": self.GroupMember.__tablename__, + "credentials": self.Credential.__tablename__, + } + + +def make_account_store_models(table_prefix: str) -> AccountStoreModels: + """Build a mapper set for one configured account-store table prefix.""" + table_prefix = validate_table_prefix(table_prefix) + accounts = table_prefix + users = f"{table_prefix}_users" + groups = f"{table_prefix}_groups" + members = f"{table_prefix}_group_members" + credentials = f"{table_prefix}_credentials" + + class Base(DeclarativeBase): + pass + + class Account(Base): + __tablename__ = accounts + __table_args__ = ( + UniqueConstraint( + "resource_id", + "account_id", + name=_schema_object_name("uq", "account_identity"), + ), + UniqueConstraint( + "resource_id", + "account_pk", + name=_schema_object_name("uq", "account_resource_pk"), + ), + Index( + _schema_object_name("ix", "account_deletion"), + "resource_id", + "deletion_task_id", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + account_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + account_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + deletion_task_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + deletion_owner_account_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + deletion_owner_user_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + + class User(Base): + __tablename__ = users + __table_args__ = ( + ForeignKeyConstraint( + ["resource_id", "account_pk"], + [f"{accounts}.resource_id", f"{accounts}.account_pk"], + ondelete="CASCADE", + name=_schema_object_name("fk", "user_account"), + ), + UniqueConstraint( + "resource_id", + "account_pk", + "user_id", + name=_schema_object_name("uq", "user_identity"), + ), + UniqueConstraint( + "resource_id", + "user_pk", + name=_schema_object_name("uq", "user_resource_pk"), + ), + Index( + _schema_object_name("ix", "user_admin"), + "resource_id", + "account_pk", + "role", + ), + Index( + _schema_object_name("ix", "user_deletion"), + "resource_id", + "deletion_task_id", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + user_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + account_pk: Mapped[int] = mapped_column(BigInteger) + user_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + role: Mapped[str] = mapped_column(String(16)) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + deletion_task_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + deletion_owner_account_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + deletion_owner_user_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + + class Group(Base): + __tablename__ = groups + __table_args__ = ( + ForeignKeyConstraint( + ["resource_id", "account_pk"], + [f"{accounts}.resource_id", f"{accounts}.account_pk"], + ondelete="CASCADE", + name=_schema_object_name("fk", "group_account"), + ), + UniqueConstraint( + "resource_id", + "account_pk", + "group_id", + name=_schema_object_name("uq", "group_identity"), + ), + UniqueConstraint( + "resource_id", + "group_pk", + name=_schema_object_name("uq", "group_resource_pk"), + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + group_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + account_pk: Mapped[int] = mapped_column(BigInteger) + group_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + + class GroupMember(Base): + __tablename__ = members + __table_args__ = ( + PrimaryKeyConstraint("resource_id", "group_pk", "user_pk"), + ForeignKeyConstraint( + ["resource_id", "group_pk"], + [f"{groups}.resource_id", f"{groups}.group_pk"], + ondelete="CASCADE", + name=_schema_object_name("fk", "member_group"), + ), + ForeignKeyConstraint( + ["resource_id", "user_pk"], + [f"{users}.resource_id", f"{users}.user_pk"], + ondelete="CASCADE", + name=_schema_object_name("fk", "member_user"), + ), + Index( + _schema_object_name("ix", "member_user"), + "resource_id", + "user_pk", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + group_pk: Mapped[int] = mapped_column(BigInteger) + user_pk: Mapped[int] = mapped_column(BigInteger) + + class Credential(Base): + __tablename__ = credentials + __table_args__ = ( + ForeignKeyConstraint( + ["resource_id", "user_pk"], + [f"{users}.resource_id", f"{users}.user_pk"], + ondelete="CASCADE", + name=_schema_object_name("fk", "credential_user"), + ), + UniqueConstraint( + "resource_id", + "active_user_pk", + name=_schema_object_name("uq", "active_credential"), + ), + Index( + _schema_object_name("ix", "credential_lookup"), + "resource_id", + "lookup_digest", + "revoked", + "user_pk", + ), + Index( + _schema_object_name("ix", "credential_history"), + "resource_id", + "user_pk", + "revoked", + "sequence", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + credential_id: Mapped[str] = mapped_column( + String(32, collation="ascii_bin"), primary_key=True + ) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + user_pk: Mapped[int] = mapped_column(BigInteger) + material: Mapped[str] = mapped_column(Text(collation="ascii_bin")) + lookup_digest: Mapped[str] = mapped_column(String(64, collation="ascii_bin")) + prefix: Mapped[str] = mapped_column(String(8, collation="ascii_bin")) + revoked: Mapped[bool] = mapped_column(Boolean, server_default=text("0")) + version: Mapped[int] = mapped_column(BigInteger, server_default=text("1")) + sequence: Mapped[int] = mapped_column(BigInteger) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + revoked_at: Mapped[Optional[object]] = mapped_column(DATETIME(fsp=6), nullable=True) + active_user_pk: Mapped[Optional[int]] = mapped_column( + BigInteger, + nullable=True, + ) + + return AccountStoreModels(Base, Account, User, Group, GroupMember, Credential, table_prefix) diff --git a/openviking/server/account_stores/redis_mysql.py b/openviking/server/account_stores/redis_mysql.py new file mode 100644 index 0000000000..3fe633b542 --- /dev/null +++ b/openviking/server/account_stores/redis_mysql.py @@ -0,0 +1,831 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Redis read-through cache over the authoritative MySQL account store.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +import secrets +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from typing import Any, TypeVar + +from openviking.server.account_stores.base import AccountStore +from openviking.server.account_stores.models import ( + AccountSummary, + DeletionRecord, + GroupSummary, + UsersPage, + UserSummary, +) +from openviking.server.account_stores.mysql import MySQLAccountStore +from openviking_cli.exceptions import InvalidArgumentError +from openviking_cli.utils import get_logger + +logger = get_logger(__name__) +T = TypeVar("T") + +_RELEASE_LOCK_SCRIPT = """ +-- release-lock +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +""" + +_GET_IF_LOCKED_SCRIPT = """ +-- get-if-locked +if redis.call('GET', KEYS[1]) ~= ARGV[1] then + return {0} +end +local value = redis.call('GET', KEYS[2]) +if value then + return {2, value} +end +return {1} +""" + +_SET_IF_LOCKED_SCRIPT = """ +-- set-if-locked +if redis.call('GET', KEYS[1]) ~= ARGV[1] then + return 0 +end +redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3]) +if ARGV[5] == '1' then + local ttl = redis.call('PTTL', KEYS[2]) + if ttl <= 0 then + redis.call('DEL', KEYS[2]) + return 0 + end + local now = redis.call('TIME') + local now_ms = now[1] * 1000 + math.floor(now[2] / 1000) + redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', now_ms) + redis.call('ZADD', KEYS[3], now_ms + ttl, KEYS[2]) + redis.call('EXPIRE', KEYS[3], ARGV[4]) +end +return 1 +""" + +_DELETE_AND_UNINDEX_SCRIPT = """ +-- delete-and-unindex +if #KEYS == 1 then + return 0 +end +local deleted = redis.call('DEL', unpack(KEYS, 2)) +redis.call('ZREM', KEYS[1], unpack(KEYS, 2)) +return deleted +""" + +_PRUNE_CACHE_INDEX_SCRIPT = """ +-- prune-cache-index +local now = redis.call('TIME') +local now_ms = now[1] * 1000 + math.floor(now[2] / 1000) +return redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms) +""" + +_ACCOUNT_CACHE_CLEAR_BATCH_SIZE = 500 + + +class _CacheLockUnavailable(Exception): + """Redis could not provide the lock needed to coordinate a cache fill.""" + + +def _digest(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +def _key_part(value: str) -> str: + return base64.urlsafe_b64encode(value.encode()).decode().rstrip("=") + + +def _decode(raw: str | bytes | None) -> dict[str, Any] | None: + if raw is None: + return None + try: + value = json.loads(raw) + except (TypeError, ValueError, UnicodeError): + return None + return value if isinstance(value, dict) else None + + +class RedisMySQLAccountStore(AccountStore): + """Cache hot account reads while preserving MySQL as the authority. + + A Redis lease coordinates cache fills and mutations per account. If Redis + is unavailable, MySQL operations still proceed and cache invalidation is + best-effort; any stale value is bounded by the fixed cache TTL. + """ + + def __init__( + self, + *, + params: dict[str, object] | None = None, + mysql_store: MySQLAccountStore | None = None, + redis_client: Any | None = None, + ) -> None: + values = dict(params or {}) + mysql_params = values.pop("mysql", None) + redis_params = values.pop("redis", None) + if values: + names = ", ".join(sorted(values)) + raise InvalidArgumentError(f"Unknown Redis MySQL account store parameters: {names}") + if mysql_store is None: + if not isinstance(mysql_params, dict): + raise InvalidArgumentError("redis_mysql account storage requires mysql parameters") + mysql_store = MySQLAccountStore(params=mysql_params) + if not isinstance(redis_params, dict): + raise InvalidArgumentError("redis_mysql account storage requires redis parameters") + + redis_values = dict(redis_params) + url = redis_values.pop("url", None) + self._key_prefix = redis_values.pop("key_prefix", "ov:account-cache") + self._ttl_seconds = redis_values.pop("ttl_seconds", 60) + self._lock_ttl_seconds = redis_values.pop("lock_ttl_seconds", 60) + self._lock_wait_seconds = redis_values.pop("lock_wait_seconds", 30) + if redis_values: + names = ", ".join(sorted(redis_values)) + raise InvalidArgumentError(f"Unknown Redis cache parameters: {names}") + if not isinstance(url, str) or not url: + raise InvalidArgumentError("redis.url must be a non-empty string") + if not isinstance(self._key_prefix, str) or not self._key_prefix: + raise InvalidArgumentError("redis.key_prefix must be a non-empty string") + if "{" in self._key_prefix or "}" in self._key_prefix: + raise InvalidArgumentError( + "redis.key_prefix must not contain Redis hash-tag characters" + ) + for name in ("ttl_seconds", "lock_ttl_seconds", "lock_wait_seconds"): + value = getattr(self, f"_{name}") + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise InvalidArgumentError(f"redis.{name} must be a positive integer") + if self._lock_ttl_seconds < self._ttl_seconds: + raise InvalidArgumentError("redis.lock_ttl_seconds must be at least redis.ttl_seconds") + + self._mysql = mysql_store + self._index_ttl_seconds = self._ttl_seconds + self._lock_ttl_seconds + self._resource_id = mysql_store._resource_id + self._redis = redis_client + self._redis_url = url + self._owns_redis = redis_client is None + + async def load(self) -> None: + await self._mysql.load() + try: + if self._redis is None: + try: + from redis.asyncio import Redis + except ImportError: + raise RuntimeError( + "Redis MySQL account storage requires openviking[redis-mysql]" + ) from None + self._redis = Redis.from_url(self._redis_url, decode_responses=True) + await self._redis.ping() + except BaseException: + await self.close() + raise + + async def close(self) -> None: + try: + if self._redis is not None and self._owns_redis: + close = getattr(self._redis, "aclose", None) + if close is not None: + await close() + finally: + self._redis = None + await self._mysql.close() + + def _account_tag(self, account_id: str) -> str: + return _key_part(f"{self._resource_id}\0{account_id}") + + def _key(self, account_id: str, suffix: str) -> str: + return f"{self._key_prefix}:v1:{{{self._account_tag(account_id)}}}:{suffix}" + + def _lock_key(self, account_id: str) -> str: + return self._key(account_id, "lock") + + def _cache_keys_index_key(self, account_id: str) -> str: + return self._key(account_id, "cache-keys") + + def _user_key(self, account_id: str, user_id: str) -> str: + return self._key(account_id, f"user:{_key_part(user_id)}") + + def _group_ids_key(self, account_id: str, user_id: str) -> str: + return self._key(account_id, f"groups:{_key_part(user_id)}") + + def _account_deletion_key(self, account_id: str) -> str: + return self._key(account_id, "deletion:account") + + def _user_deletion_key(self, account_id: str, user_id: str) -> str: + return self._key(account_id, f"deletion:user:{_key_part(user_id)}") + + def _account_credential_fence_key(self, account_id: str) -> str: + return self._key(account_id, "credential-fence:account") + + def _user_credential_fence_key(self, account_id: str, user_id: str) -> str: + return self._key(account_id, f"credential-fence:user:{_key_part(user_id)}") + + def _binding_key(self, account_id: str, api_key: str) -> str: + digest_input = self._resource_id + "\0" + api_key + return self._key(account_id, f"binding:{_digest(digest_input)}") + + async def _delete(self, *keys: str) -> None: + assert self._redis is not None + if keys: + await self._redis.delete(*keys) + + @asynccontextmanager + async def _account_lock(self, account_id: str) -> AsyncIterator[str]: + assert self._redis is not None + key = self._lock_key(account_id) + token = secrets.token_urlsafe(24) + deadline = time.monotonic() + self._lock_wait_seconds + try: + while True: + if await self._redis.set( + key, + token, + nx=True, + px=self._lock_ttl_seconds * 1000, + ): + break + if time.monotonic() >= deadline: + raise _CacheLockUnavailable( + f"timed out waiting for account cache lock: {account_id}" + ) + await asyncio.sleep(0.02 + secrets.randbelow(20) / 1000) + except _CacheLockUnavailable: + raise + except Exception as exc: + raise _CacheLockUnavailable( + f"failed to acquire account cache lock: {account_id}" + ) from exc + try: + yield token + finally: + try: + await self._redis.eval(_RELEASE_LOCK_SCRIPT, 1, key, token) + except Exception: + logger.exception("Failed to release account cache lock") + + @staticmethod + def _nullable_payload(value: dict[str, Any] | tuple[str, ...] | None) -> dict[str, Any]: + return {"found": value is not None, "value": value} + + @staticmethod + def _read_nullable(payload: dict[str, Any] | None) -> tuple[bool, Any]: + if payload is None or not isinstance(payload.get("found"), bool): + return False, None + return True, payload.get("value") + + async def _get_if_locked( + self, account_id: str, token: str, key: str + ) -> tuple[bool, dict[str, Any] | None]: + assert self._redis is not None + result = await self._redis.eval( + _GET_IF_LOCKED_SCRIPT, + 2, + self._lock_key(account_id), + key, + token, + ) + if not isinstance(result, (list, tuple)) or not result: + return False, None + status = int(result[0]) + if status == 0: + return False, None + if status == 1: + return True, None + return True, _decode(result[1]) + + async def _set_if_locked( + self, + account_id: str, + token: str, + key: str, + value: dict[str, Any], + *, + index: bool = True, + ) -> bool: + assert self._redis is not None + result = await self._redis.eval( + _SET_IF_LOCKED_SCRIPT, + 3, + self._lock_key(account_id), + key, + self._cache_keys_index_key(account_id), + token, + json.dumps(value, separators=(",", ":")), + self._ttl_seconds, + self._index_ttl_seconds, + int(index), + ) + return bool(result) + + async def _read_through( + self, + account_id: str, + key: str, + loader: Callable[[], Awaitable[T]], + ) -> T: + try: + hit, value = self._read_nullable(_decode(await self._redis.get(key))) + if hit: + return value + except Exception: + logger.exception("Redis cache read failed; reading %s from MySQL", key) + return await loader() + + try: + async with self._account_lock(account_id) as token: + owned, payload = await self._get_if_locked(account_id, token, key) + if not owned: + logger.warning("Redis cache lease expired before reading %s", key) + return await loader() + hit, value = self._read_nullable(payload) + if hit: + return value + value = await loader() + try: + if not await self._set_if_locked( + account_id, + token, + key, + self._nullable_payload(value), + ): + logger.warning("Redis cache lease expired before writing %s", key) + except Exception: + logger.exception("Failed to populate Redis account cache for %s", key) + return value + except _CacheLockUnavailable: + logger.warning("Redis cache lock unavailable; reading %s from MySQL", key) + except Exception: + logger.exception("Redis cache read failed; reading %s from MySQL", key) + return await loader() + + async def _delete_and_unindex(self, account_id: str, *keys: str) -> None: + assert self._redis is not None + if keys: + await self._redis.eval( + _DELETE_AND_UNINDEX_SCRIPT, + len(keys) + 1, + self._cache_keys_index_key(account_id), + *keys, + ) + + async def _invalidate(self, account_id: str, *keys: str) -> None: + try: + await self._delete_and_unindex(account_id, *keys) + except Exception: + logger.exception("Failed to invalidate Redis account cache") + + async def _prune_cache_index(self, account_id: str) -> None: + assert self._redis is not None + await self._redis.eval( + _PRUNE_CACHE_INDEX_SCRIPT, + 1, + self._cache_keys_index_key(account_id), + ) + + async def _clear_account_cache(self, account_id: str) -> None: + assert self._redis is not None + index_key = self._cache_keys_index_key(account_id) + try: + await self._prune_cache_index(account_id) + while True: + keys = await self._redis.zrange( + index_key, + 0, + _ACCOUNT_CACHE_CLEAR_BATCH_SIZE - 1, + ) + if not keys: + break + await self._delete_and_unindex(account_id, *keys) + await self._delete(index_key) + except Exception: + logger.exception("Failed to clear Redis account cache for %s", account_id) + + async def _invalidate_after_lock_timeout( + self, + account_id: str, + keys: tuple[str, ...], + *, + account_wide: bool, + ) -> None: + try: + await self._delete(self._lock_key(account_id)) + except Exception: + logger.exception("Failed to revoke Redis account cache lock") + try: + async with self._account_lock(account_id): + if account_wide: + await self._clear_account_cache(account_id) + if keys: + await self._invalidate(account_id, *keys) + except _CacheLockUnavailable: + logger.warning( + "Redis cache lock remained unavailable after revoking the lease for %s", + account_id, + ) + if account_wide: + await self._clear_account_cache(account_id) + if keys: + await self._invalidate(account_id, *keys) + + async def _mutate_account( + self, + account_id: str, + operation: Callable[[], Awaitable[T]], + invalidation_keys: Callable[[], tuple[str, ...]], + *, + account_wide: bool = False, + ) -> T: + result = await operation() + keys = invalidation_keys() + try: + async with self._account_lock(account_id): + if account_wide: + await self._clear_account_cache(account_id) + if keys: + await self._invalidate(account_id, *keys) + except _CacheLockUnavailable: + logger.warning( + "Redis cache lock unavailable after mutating %s; " + "revoking the lease and invalidating without the lock", + account_id, + ) + await self._invalidate_after_lock_timeout( + account_id, + keys, + account_wide=account_wide, + ) + return result + + def _user_cache_keys( + self, account_id: str, user_id: str, *, credentials: bool = False + ) -> tuple[str, ...]: + keys = ( + self._user_key(account_id, user_id), + self._group_ids_key(account_id, user_id), + self._user_deletion_key(account_id, user_id), + ) + if credentials: + return keys + (self._user_credential_fence_key(account_id, user_id),) + return keys + + async def _cached_binding( + self, account_id: str, api_key: str + ) -> tuple[bool, tuple[str, str] | None]: + binding = _decode(await self._redis.get(self._binding_key(account_id, api_key))) + hit, value = self._read_nullable(binding) + if not hit: + return False, None + if value is None: + return True, None + if not isinstance(value, dict): + return False, None + try: + account_id = str(value["account_id"]) + user_id = str(value["user_id"]) + account_fence = str(value["account_fence"]) + user_fence = str(value["user_fence"]) + except (KeyError, TypeError, ValueError): + return False, None + current_account_fence = _decode( + await self._redis.get(self._account_credential_fence_key(account_id)) + ) + current_user_fence = _decode( + await self._redis.get(self._user_credential_fence_key(account_id, user_id)) + ) + if ( + current_account_fence is None + or current_user_fence is None + or current_account_fence.get("value") != account_fence + or current_user_fence.get("value") != user_fence + ): + return False, None + return True, (account_id, user_id) + + async def _cache_binding( + self, + account_id: str, + token: str, + api_key: str, + binding: tuple[str, str] | None, + ) -> None: + if binding is None: + await self._set_if_locked( + account_id, + token, + self._binding_key(account_id, api_key), + self._nullable_payload(None), + index=False, + ) + return + binding_account_id, user_id = binding + if binding_account_id != account_id: + return + account_fence = secrets.token_urlsafe(18) + user_fence = secrets.token_urlsafe(18) + account_fence_key = self._account_credential_fence_key(account_id) + user_fence_key = self._user_credential_fence_key(account_id, user_id) + existing_account_fence = _decode(await self._redis.get(account_fence_key)) + existing_user_fence = _decode(await self._redis.get(user_fence_key)) + if existing_account_fence is not None: + account_fence = str(existing_account_fence.get("value", account_fence)) + elif not await self._set_if_locked( + account_id, + token, + account_fence_key, + self._nullable_payload(account_fence), + ): + return + if existing_user_fence is not None: + user_fence = str(existing_user_fence.get("value", user_fence)) + elif not await self._set_if_locked( + account_id, + token, + user_fence_key, + self._nullable_payload(user_fence), + ): + return + await self._set_if_locked( + account_id, + token, + self._binding_key(account_id, api_key), + self._nullable_payload( + { + "account_id": account_id, + "user_id": user_id, + "account_fence": account_fence, + "user_fence": user_fence, + } + ), + ) + + async def verify_api_key( + self, + api_key: str, + *, + account_id_hint: str | None = None, + user_id_hint: str | None = None, + ) -> tuple[str, str] | None: + if account_id_hint is None: + return await self._mysql.verify_api_key( + api_key, account_id_hint=None, user_id_hint=user_id_hint + ) + try: + hit, binding = await self._cached_binding(account_id_hint, api_key) + if hit: + return binding + except Exception: + logger.exception("Redis API-key cache read failed; falling back to MySQL") + try: + async with self._account_lock(account_id_hint) as token: + hit, binding = await self._cached_binding(account_id_hint, api_key) + if hit: + return binding + binding = await self._mysql.verify_api_key( + api_key, + account_id_hint=account_id_hint, + user_id_hint=user_id_hint, + ) + try: + await self._cache_binding( + account_id_hint, + token, + api_key, + binding, + ) + except Exception: + logger.exception("Failed to populate Redis API-key cache") + return binding + except _CacheLockUnavailable: + logger.warning("Redis cache lock unavailable; verifying API key in MySQL") + except Exception: + logger.exception("Redis API-key cache write failed; verifying in MySQL") + return await self._mysql.verify_api_key( + api_key, account_id_hint=account_id_hint, user_id_hint=user_id_hint + ) + + async def get_user(self, account_id: str, user_id: str) -> UserSummary | None: + value = await self._read_through( + account_id, + self._user_key(account_id, user_id), + lambda: self._mysql.get_user(account_id, user_id), + ) + return value if value is None else UserSummary(value) + + async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, ...]: + value = await self._read_through( + account_id, + self._group_ids_key(account_id, user_id), + lambda: self._mysql.get_user_group_ids(account_id, user_id), + ) + return tuple(value or ()) + + async def get_deletion( + self, account_id: str, user_id: str | None = None + ) -> DeletionRecord | None: + account_deletion = await self._read_through( + account_id, + self._account_deletion_key(account_id), + lambda: self._mysql.get_deletion(account_id), + ) + if account_deletion is not None or user_id is None: + return account_deletion + value = await self._read_through( + account_id, + self._user_deletion_key(account_id, user_id), + lambda: self._mysql.get_deletion(account_id, user_id), + ) + return value + + async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: + return await self._mysql.iter_deletions() + + async def get_account(self, account_id: str) -> AccountSummary | None: + return await self._mysql.get_account(account_id) + + async def create_account_with_api_key( + self, + account_id: str, + admin_user_id: str, + api_key: str, + ) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.create_account_with_api_key( + account_id, + admin_user_id, + api_key, + ), + lambda: (self._binding_key(account_id, api_key),), + account_wide=True, + ) + + async def delete_account(self, account_id: str) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.delete_account(account_id), + lambda: (), + account_wide=True, + ) + + async def list_accounts(self, **kwargs) -> list[AccountSummary]: + return await self._mysql.list_accounts(**kwargs) + + async def create_user(self, account_id: str, user_id: str, role: str) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.create_user(account_id, user_id, role), + lambda: self._user_cache_keys(account_id, user_id, credentials=True), + ) + + async def create_user_with_api_key( + self, + account_id: str, + user_id: str, + role: str, + api_key: str, + ) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.create_user_with_api_key( + account_id, + user_id, + role, + api_key, + ), + lambda: self._user_cache_keys( + account_id, + user_id, + credentials=True, + ) + + (self._binding_key(account_id, api_key),), + ) + + async def list_users_page(self, account_id: str, **kwargs) -> UsersPage: + return await self._mysql.list_users_page(account_id, **kwargs) + + async def set_role(self, account_id: str, user_id: str, role: str) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.set_role(account_id, user_id, role), + lambda: (self._user_key(account_id, user_id),), + ) + + async def create_group(self, account_id: str, group_id: str) -> GroupSummary: + return await self._mutate_account( + account_id, + lambda: self._mysql.create_group(account_id, group_id), + lambda: (), + ) + + async def get_groups(self, account_id: str) -> list[GroupSummary]: + return await self._mysql.get_groups(account_id) + + async def get_group_members(self, account_id: str, group_id: str) -> list[str]: + return await self._mysql.get_group_members(account_id, group_id) + + async def add_group_member(self, account_id: str, group_id: str, user_id: str) -> bool: + return await self._mutate_account( + account_id, + lambda: self._mysql.add_group_member(account_id, group_id, user_id), + lambda: (self._group_ids_key(account_id, user_id),), + ) + + async def remove_group_member(self, account_id: str, group_id: str, user_id: str) -> bool: + return await self._mutate_account( + account_id, + lambda: self._mysql.remove_group_member(account_id, group_id, user_id), + lambda: (self._group_ids_key(account_id, user_id),), + ) + + async def delete_group(self, account_id: str, group_id: str) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.delete_group(account_id, group_id), + lambda: (), + ) + + async def ensure_trusted_identities( + self, identities: dict[str, set[str]] + ) -> dict[str, int]: + result = {"created_accounts": 0, "created_users": 0} + for account_id, user_ids in identities.items(): + created = await self._mutate_account( + account_id, + lambda account_id=account_id, user_ids=user_ids: ( + self._mysql.ensure_trusted_identities({account_id: user_ids}) + ), + lambda: (), + account_wide=True, + ) + result["created_accounts"] += created["created_accounts"] + result["created_users"] += created["created_users"] + return result + + async def begin_deletion( + self, account_id: str, user_id: str | None, **kwargs + ) -> tuple[DeletionRecord, bool]: + return await self._mutate_account( + account_id, + lambda: self._mysql.begin_deletion(account_id, user_id, **kwargs), + lambda: ( + self._account_deletion_key(account_id) + if user_id is None + else self._user_deletion_key(account_id, user_id), + ), + ) + + async def replace_deletion_task( + self, account_id: str, user_id: str | None, **kwargs + ) -> DeletionRecord: + return await self._mutate_account( + account_id, + lambda: self._mysql.replace_deletion_task(account_id, user_id, **kwargs), + lambda: ( + self._account_deletion_key(account_id) + if user_id is None + else self._user_deletion_key(account_id, user_id), + ), + ) + + async def finish_deletion(self, account_id: str, user_id: str | None, task_id: str) -> bool: + return await self._mutate_account( + account_id, + lambda: self._mysql.finish_deletion(account_id, user_id, task_id), + lambda: () + if user_id is None + else self._user_cache_keys(account_id, user_id, credentials=True), + account_wide=user_id is None, + ) + + async def replace_active_user_api_key( + self, account_id: str, user_id: str, api_key: str + ) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.replace_active_user_api_key(account_id, user_id, api_key), + lambda: ( + self._user_credential_fence_key(account_id, user_id), + self._binding_key(account_id, api_key), + ), + ) + + async def revoke_active_api_keys( + self, account_id: str, user_id: str | None = None + ) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.revoke_active_api_keys(account_id, user_id), + lambda: (self._account_credential_fence_key(account_id),) + if user_id is None + else (self._user_credential_fence_key(account_id, user_id),), + ) + + async def get_user_key_fingerprint(self, account_id: str, user_id: str) -> str | None: + return await self._mysql.get_user_key_fingerprint(account_id, user_id) diff --git a/openviking/server/account_stores/sql/changes/0000_template.sql b/openviking/server/account_stores/sql/changes/0000_template.sql new file mode 100644 index 0000000000..414ec0cd3c --- /dev/null +++ b/openviking/server/account_stores/sql/changes/0000_template.sql @@ -0,0 +1,5 @@ +-- Copy this file as YYYYMMDD_description.sql for each reviewed schema change. +-- Keep changes forward-only. Do not modify init.sql after it is released. +-- MySQL DDL may commit implicitly; review rollback and rollout separately. + +-- ALTER TABLE `ov_accounts` ADD COLUMN example VARCHAR(255) NULL; diff --git a/openviking/server/account_stores/sql/init.sql b/openviking/server/account_stores/sql/init.sql new file mode 100644 index 0000000000..663911f304 --- /dev/null +++ b/openviking/server/account_stores/sql/init.sql @@ -0,0 +1,99 @@ +-- OpenViking MySQL account store initial schema. +-- Table prefix: ov_accounts (the default account_store.params.table value). +-- Execute manually before starting a MySQL account-store deployment. +-- For a custom table prefix, replace only backtick-quoted ov_accounts table +-- identifiers. Constraint and index names are stable and table-name independent. + +CREATE TABLE `ov_accounts` ( + account_pk BIGINT NOT NULL AUTO_INCREMENT, + resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + account_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deletion_task_id VARCHAR(64) NULL, + deletion_owner_account_id VARCHAR(255) COLLATE utf8mb4_bin NULL, + deletion_owner_user_id VARCHAR(255) COLLATE utf8mb4_bin NULL, + PRIMARY KEY (account_pk), + CONSTRAINT uq_ov_as_account_identity UNIQUE (resource_id, account_id), + CONSTRAINT uq_ov_as_account_resource_pk UNIQUE (resource_id, account_pk), + INDEX ix_ov_as_account_deletion (resource_id, deletion_task_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE `ov_accounts_users` ( + user_pk BIGINT NOT NULL AUTO_INCREMENT, + resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + account_pk BIGINT NOT NULL, + user_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + `role` VARCHAR(16) NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + deletion_task_id VARCHAR(64) NULL, + deletion_owner_account_id VARCHAR(255) COLLATE utf8mb4_bin NULL, + deletion_owner_user_id VARCHAR(255) COLLATE utf8mb4_bin NULL, + PRIMARY KEY (user_pk), + CONSTRAINT fk_ov_as_user_account + FOREIGN KEY (resource_id, account_pk) + REFERENCES `ov_accounts` (resource_id, account_pk) + ON DELETE CASCADE, + CONSTRAINT uq_ov_as_user_identity + UNIQUE (resource_id, account_pk, user_id), + CONSTRAINT uq_ov_as_user_resource_pk UNIQUE (resource_id, user_pk), + INDEX ix_ov_as_user_admin (resource_id, account_pk, `role`), + INDEX ix_ov_as_user_deletion (resource_id, deletion_task_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE `ov_accounts_groups` ( + group_pk BIGINT NOT NULL AUTO_INCREMENT, + resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + account_pk BIGINT NOT NULL, + group_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (group_pk), + CONSTRAINT fk_ov_as_group_account + FOREIGN KEY (resource_id, account_pk) + REFERENCES `ov_accounts` (resource_id, account_pk) + ON DELETE CASCADE, + CONSTRAINT uq_ov_as_group_identity + UNIQUE (resource_id, account_pk, group_id), + CONSTRAINT uq_ov_as_group_resource_pk UNIQUE (resource_id, group_pk) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE `ov_accounts_group_members` ( + resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + group_pk BIGINT NOT NULL, + user_pk BIGINT NOT NULL, + PRIMARY KEY (resource_id, group_pk, user_pk), + CONSTRAINT fk_ov_as_member_group + FOREIGN KEY (resource_id, group_pk) + REFERENCES `ov_accounts_groups` (resource_id, group_pk) + ON DELETE CASCADE, + CONSTRAINT fk_ov_as_member_user + FOREIGN KEY (resource_id, user_pk) + REFERENCES `ov_accounts_users` (resource_id, user_pk) + ON DELETE CASCADE, + INDEX ix_ov_as_member_user (resource_id, user_pk) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE `ov_accounts_credentials` ( + credential_id VARCHAR(32) COLLATE ascii_bin NOT NULL, + resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + user_pk BIGINT NOT NULL, + material TEXT COLLATE ascii_bin NOT NULL, + lookup_digest VARCHAR(64) COLLATE ascii_bin NOT NULL, + prefix VARCHAR(8) COLLATE ascii_bin NOT NULL, + revoked BOOLEAN NOT NULL DEFAULT 0, + version BIGINT NOT NULL DEFAULT 1, + sequence BIGINT NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + revoked_at DATETIME(6) NULL, + active_user_pk BIGINT NULL, + PRIMARY KEY (credential_id), + CONSTRAINT fk_ov_as_credential_user + FOREIGN KEY (resource_id, user_pk) + REFERENCES `ov_accounts_users` (resource_id, user_pk) + ON DELETE CASCADE, + CONSTRAINT uq_ov_as_active_credential + UNIQUE (resource_id, active_user_pk), + INDEX ix_ov_as_credential_lookup + (resource_id, lookup_digest, revoked, user_pk), + INDEX ix_ov_as_credential_history + (resource_id, user_pk, revoked, sequence) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/openviking/server/store_assembly.py b/openviking/server/store_assembly.py index 64741d1894..0e636acd97 100644 --- a/openviking/server/store_assembly.py +++ b/openviking/server/store_assembly.py @@ -67,7 +67,25 @@ def build_account_store( watch_enabled=account_store_watch_enabled, watch_interval_seconds=account_store_watch_interval_seconds, ) + if provider == "mysql": + try: + from openviking.server.account_stores.mysql import MySQLAccountStore + except ImportError: + raise RuntimeError( + "MySQL account storage requires openviking[mysql]" + ) from None + return MySQLAccountStore(params=account_store_params) + if provider == "redis_mysql": + try: + from openviking.server.account_stores.redis_mysql import ( + RedisMySQLAccountStore, + ) + except ImportError: + raise RuntimeError( + "Redis MySQL account storage requires openviking[redis-mysql]" + ) from None + return RedisMySQLAccountStore(params=account_store_params) raise InvalidArgumentError( f"Unknown account store provider '{account_store_provider}'. " - "Available providers: file" + "Available providers: file, mysql, redis_mysql" ) diff --git a/pyproject.toml b/pyproject.toml index 4d8cdf3165..15d86fc803 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,17 @@ gateway-fast = [ "uvloop>=0.21; sys_platform != 'win32'", "httptools>=0.6", ] +mysql = [ + "aiomysql>=0.2.0,<1", + "SQLAlchemy[asyncio]>=2.0,<2.1", +] +redis-mysql = [ + "aiomysql>=0.2.0,<1", + "redis>=5.0,<7", + "SQLAlchemy[asyncio]>=2.0,<2.1", +] test = [ + "SQLAlchemy[asyncio]>=2.0,<2.1", "pytest>=7.0.0", "pytest-asyncio>=0.21.0", "pytest-xdist>=3.5.0", diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index 54a2af62db..6dfa6965f2 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -2,12 +2,19 @@ # SPDX-License-Identifier: AGPL-3.0 import asyncio +import base64 import json +import os import threading from types import SimpleNamespace +from uuid import uuid4 import pytest +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import create_async_engine +from openviking.server.account_stores import mysql as mysql_store_module +from openviking.server.account_stores.mysql import MySQLAccountStore from openviking.server.api_keys import APIKeyManager from openviking.server.api_keys.legacy import FileStore from openviking.server.identity import Role @@ -16,6 +23,7 @@ FailedPreconditionError, InvalidArgumentError, NotFoundError, + UnauthenticatedError, ) from tests.server.account_store_fakes import InMemoryAGFS @@ -299,6 +307,53 @@ def read(self, path, ctx=None): assert (await store.get_deletion("acme", "bob"))["task_id"] == "delete" +def test_mysql_requires_resource_id(monkeypatch): + monkeypatch.delenv("OV_RESOURCE_ID", raising=False) + with pytest.raises(InvalidArgumentError, match="OV_RESOURCE_ID is required"): + MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) + + +async def test_mysql_load_rejects_incomplete_schema_and_disposes_engine(monkeypatch): + monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") + store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) + + class FakeEngine: + disposed = False + + async def dispose(self): + self.disposed = True + + engine = FakeEngine() + + class FakeSession: + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + async def execute(self, statement): + table = statement.get_final_froms()[0].name + if table == store._tables["groups"]: + raise OperationalError( + str(statement), + {}, + Exception(1146, f"Table '{table}' doesn't exist"), + ) + + monkeypatch.setattr(mysql_store_module, "create_async_engine", lambda *args, **kwargs: engine) + monkeypatch.setattr( + mysql_store_module, + "async_sessionmaker", + lambda *args, **kwargs: FakeSession, + ) + + with pytest.raises(RuntimeError, match="schema is missing or incompatible"): + await store.load() + + assert engine.disposed + + async def test_file_store_rejects_deleting_nonempty_group(): store = _file_store() await store.load() @@ -309,3 +364,85 @@ async def test_file_store_rejects_deleting_nonempty_group(): with pytest.raises(FailedPreconditionError, match="Group must be empty"): await store.delete_group("acme", "engineering") assert await store.get_group_members("acme", "engineering") == ["alice"] + + +async def test_mysql_managers_share_authoritative_state(monkeypatch): + raw = os.environ.get("OV_TEST_MYSQL") + if not raw: + pytest.skip("Set OV_TEST_MYSQL to connection JSON for the MySQL test") + monkeypatch.setenv("OV_RESOURCE_ID", f"test-{uuid4().hex}") + encryption_key = base64.urlsafe_b64encode(b"k" * 32).decode() + params = { + **json.loads(raw), + "table": f"ov_test_{uuid4().hex}", + "credential_encryption_key": encryption_key, + } + first = build_api_key_manager( + "root", + None, + account_store_provider="mysql", + account_store_params={**params, "credential_storage": "encrypted"}, + ) + second = build_api_key_manager( + "root", + None, + account_store_provider="mysql", + account_store_params={**params, "credential_storage": "hash"}, + ) + setup_engine = create_async_engine(first._store._url, **first._store._engine_options) + async with setup_engine.begin() as connection: + await connection.run_sync(first._store._models.Base.metadata.create_all) + await setup_engine.dispose() + await first.load() + await second.load() + try: + trusted_results = await asyncio.gather( + first.ensure_trusted_identities({"trusted": {"alice", "bob"}}), + second.ensure_trusted_identities({"trusted": {"bob", "carol"}}), + ) + assert sum(result["created_accounts"] for result in trusted_results) == 1 + assert sum(result["created_users"] for result in trusted_results) == 3 + assert await first.get_registered_user_role("trusted", "alice") == Role.USER + assert await second.get_registered_user_role("trusted", "bob") == Role.USER + assert await first.get_registered_user_role("trusted", "carol") == Role.USER + assert (await first.get_account("trusted"))["user_count"] == 3 + repeat = await second.ensure_trusted_identities({"trusted": {"alice", "bob", "carol"}}) + assert repeat == {"created_accounts": 0, "created_users": 0} + + deletion, created = await first.begin_deletion( + "trusted", + "carol", + task_id="delete-carol", + owner_account_id="system", + owner_user_id="system", + ) + assert created + assert await second.get_deletion("trusted", "carol") == deletion + account_deletion, account_created = await first.begin_deletion( + "trusted", + None, + task_id="delete-trusted", + owner_account_id="system", + owner_user_id="system", + ) + assert account_created + assert await second.get_deletion("trusted", "carol") == account_deletion + + key = await first.create_account("acme", "alice") + assert (await second.resolve_identity(key)).role == Role.ADMIN + user_key = await first.register_user("acme", "dave") + assert (await second.resolve_identity(user_key)).role == Role.USER + hashed_key = await second.create_account("hashed", "bob") + assert (await first.resolve_identity(hashed_key)).role == Role.ADMIN + rotated = await first.regenerate_key("acme", "alice") + assert (await second.resolve_identity(rotated)).role == Role.ADMIN + with pytest.raises(UnauthenticatedError): + await second.resolve_identity(key) + await first.create_group("acme", "engineering") + await first.add_group_member("acme", "engineering", "alice") + assert await second.get_user_group_ids("acme", "alice") == ("engineering",) + finally: + async with first._store._engine.begin() as connection: + await connection.run_sync(first._store._models.Base.metadata.drop_all) + await second.close() + await first.close() diff --git a/tests/server/test_redis_mysql_account_store.py b/tests/server/test_redis_mysql_account_store.py new file mode 100644 index 0000000000..9472ddf30d --- /dev/null +++ b/tests/server/test_redis_mysql_account_store.py @@ -0,0 +1,524 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +import asyncio +import base64 + +import pytest + +from openviking.server.account_stores.redis_mysql import RedisMySQLAccountStore +from openviking.server.api_keys.new import APIKeyManager +from openviking.server.identity import Role +from openviking_cli.exceptions import InvalidArgumentError + + +class FakeRedis: + def __init__(self): + self.values: dict[str, str] = {} + self.zsets: dict[str, dict[str, int]] = {} + self.now_ms = 0 + + async def ping(self): + return True + + async def get(self, key: str): + return self.values.get(key) + + async def set(self, key: str, value, *, ex=None, nx=False, px=None): + del ex, px + if nx and key in self.values: + return False + self.values[key] = str(value) + return True + + async def delete(self, *keys: str): + removed = 0 + for key in keys: + removed += self.values.pop(key, None) is not None + removed += self.zsets.pop(key, None) is not None + return removed + + async def zrange(self, key: str, start: int, end: int): + members = sorted(self.zsets.get(key, {}).items(), key=lambda item: (item[1], item[0])) + if end == -1: + return [member for member, _ in members[start:]] + return [member for member, _ in members[start : end + 1]] + + def _prune_zset(self, key: str): + members = self.zsets.get(key, {}) + for member, score in tuple(members.items()): + if score <= self.now_ms: + del members[member] + + async def eval(self, script: str, numkeys: int, *args): + keys = args[:numkeys] + values = args[numkeys:] + if "-- release-lock" in script: + if self.values.get(keys[0]) == values[0]: + return await self.delete(keys[0]) + return 0 + if "-- get-if-locked" in script: + if self.values.get(keys[0]) != values[0]: + return [0] + value = self.values.get(keys[1]) + return [1] if value is None else [2, value] + if "-- set-if-locked" in script: + if self.values.get(keys[0]) != values[0]: + return 0 + self.values[keys[1]] = values[1] + if str(values[4]) == "1": + self._prune_zset(keys[2]) + self.zsets.setdefault(keys[2], {})[keys[1]] = ( + self.now_ms + int(values[2]) * 1000 + ) + return 1 + if "-- delete-and-unindex" in script: + await self.delete(*keys[1:]) + members = self.zsets.get(keys[0], {}) + for key in keys[1:]: + members.pop(key, None) + return len(keys) - 1 + if "-- prune-cache-index" in script: + self._prune_zset(keys[0]) + return 0 + raise AssertionError("Unknown Lua script") + + +class FakeMySQLAuthority: + _resource_id = "test-resource" + + def __init__(self, api_key: str): + self.api_key = api_key + self.account_exists = True + self.user_exists = True + self.user = {"user_id": "alice", "role": Role.ADMIN} + self.group_ids = ("engineering",) + self.account_deletion = None + self.user_deletion = None + self.calls = { + "verify": 0, + "user": 0, + "groups": 0, + "account_deletion": 0, + "user_deletion": 0, + } + + async def load(self): + return None + + async def close(self): + return None + + async def verify_api_key(self, api_key: str, **_kwargs): + self.calls["verify"] += 1 + return ( + ("acme", "alice") + if self.account_exists and self.user_exists and api_key == self.api_key + else None + ) + + async def get_user(self, account_id: str, user_id: str): + self.calls["user"] += 1 + if ( + not self.account_exists + or not self.user_exists + or (account_id, user_id) != ("acme", "alice") + ): + return None + return dict(self.user) + + async def get_user_group_ids(self, account_id: str, user_id: str): + self.calls["groups"] += 1 + return ( + self.group_ids + if self.account_exists and (account_id, user_id) == ("acme", "alice") + else () + ) + + async def get_deletion(self, account_id: str, user_id: str | None = None): + if not self.account_exists or account_id != "acme": + return None + if user_id is None: + self.calls["account_deletion"] += 1 + return self.account_deletion + self.calls["user_deletion"] += 1 + return self.account_deletion or self.user_deletion + + async def replace_active_user_api_key(self, account_id: str, user_id: str, api_key: str): + assert (account_id, user_id) == ("acme", "alice") + self.api_key = api_key + + async def create_account_with_api_key( + self, + account_id: str, + admin_user_id: str, + api_key: str, + ): + assert (account_id, admin_user_id) == ("acme", "alice") + self.account_exists = True + self.user_exists = True + self.api_key = api_key + self.user = {"user_id": "alice", "role": Role.ADMIN} + self.group_ids = () + self.account_deletion = None + self.user_deletion = None + + async def create_user_with_api_key( + self, + account_id: str, + user_id: str, + role: str, + api_key: str, + ): + assert (account_id, user_id) == ("acme", "alice") + self.user_exists = True + self.api_key = api_key + self.user = {"user_id": user_id, "role": role} + + async def delete_account(self, account_id: str): + assert account_id == "acme" + self.account_exists = False + + async def revoke_active_api_keys(self, account_id: str, user_id: str | None = None): + assert account_id == "acme" + self.api_key = "" + + async def set_role(self, account_id: str, user_id: str, role: str): + assert (account_id, user_id) == ("acme", "alice") + self.user["role"] = role + + async def add_group_member(self, account_id: str, group_id: str, user_id: str): + assert (account_id, user_id) == ("acme", "alice") + self.group_ids = tuple(sorted(set(self.group_ids) | {group_id})) + return True + + async def begin_deletion(self, account_id: str, user_id: str | None, **_kwargs): + assert (account_id, user_id) == ("acme", "alice") + self.user_deletion = {"task_id": "delete-1"} + return self.user_deletion, True + + +def _store(api_key: str) -> tuple[RedisMySQLAccountStore, FakeMySQLAuthority]: + mysql = FakeMySQLAuthority(api_key) + store = RedisMySQLAccountStore( + params={ + "redis": { + "url": "redis://unused", + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + } + }, + mysql_store=mysql, + redis_client=FakeRedis(), + ) + return store, mysql + + +def _api_key(secret: str = "secret") -> str: + encoded_account = base64.urlsafe_b64encode(b"acme").decode().rstrip("=") + encoded_user = base64.urlsafe_b64encode(b"alice").decode().rstrip("=") + encoded_secret = base64.urlsafe_b64encode(secret.encode()).decode().rstrip("=") + return f"{encoded_account}.{encoded_user}.{encoded_secret}" + + +def test_account_keys_include_resource_in_hash_tag(): + store, _ = _store(_api_key()) + + assert "{dGVzdC1yZXNvdXJjZQBhY21l}" in store._user_key("acme", "alice") + assert "{dGVzdC1yZXNvdXJjZQBhY21l}" in store._binding_key("acme", _api_key()) + assert store._lock_wait_seconds == 30 + + +def test_redis_timeouts_must_be_positive_integers(): + mysql = FakeMySQLAuthority(_api_key()) + for name, value in ( + ("ttl_seconds", 1.5), + ("lock_ttl_seconds", 1.5), + ("lock_wait_seconds", 1.5), + ("ttl_seconds", True), + ("lock_ttl_seconds", 0), + ("lock_wait_seconds", -1), + ): + with pytest.raises(InvalidArgumentError, match=f"redis.{name} must be a positive integer"): + RedisMySQLAccountStore( + params={"redis": {"url": "redis://unused", name: value}}, + mysql_store=mysql, + redis_client=FakeRedis(), + ) + + +async def test_verify_api_key_caches_public_binding_read(): + api_key = _api_key() + store, mysql = _store(api_key) + await store.load() + + assert await store.verify_api_key(api_key, account_id_hint="acme", user_id_hint="alice") == ( + "acme", + "alice", + ) + assert await store.verify_api_key(api_key, account_id_hint="acme", user_id_hint="alice") == ( + "acme", + "alice", + ) + + assert mysql.calls["verify"] == 1 + + +async def test_concurrent_user_cache_misses_query_mysql_once(): + store, mysql = _store(_api_key()) + await store.load() + read_started = asyncio.Event() + allow_read = asyncio.Event() + + async def delayed_get_user(account_id: str, user_id: str): + mysql.calls["user"] += 1 + read_started.set() + await allow_read.wait() + return {"user_id": user_id, "role": Role.ADMIN} + + mysql.get_user = delayed_get_user + reads = [ + asyncio.create_task(store.get_user("acme", "alice")) + for _ in range(5) + ] + await read_started.wait() + await asyncio.sleep(0.05) + assert mysql.calls["user"] == 1 + + allow_read.set() + assert await asyncio.gather(*reads) == [ + {"user_id": "alice", "role": Role.ADMIN} + ] * 5 + assert mysql.calls["user"] == 1 + + +async def test_expired_cache_lease_cannot_refill_cache(): + store, _ = _store(_api_key()) + await store.load() + redis = store._redis + key = store._user_key("acme", "alice") + redis.values[store._lock_key("acme")] = "new-owner" + + written = await store._set_if_locked( + "acme", + "expired-owner", + key, + {"found": True, "value": {"user_id": "alice", "role": Role.ADMIN}}, + ) + + assert written is False + assert key not in redis.values + + +async def test_write_lock_timeout_revokes_stale_lease_before_invalidation(): + store, mysql = _store(_api_key()) + store._lock_wait_seconds = 0.05 + await store.load() + assert await store.get_user("acme", "alice") is not None + redis = store._redis + redis.values[store._lock_key("acme")] = "other-owner" + + mutation = asyncio.create_task(store.set_role("acme", "alice", Role.USER)) + await asyncio.sleep(0.01) + assert mysql.user["role"] == Role.USER + assert not mutation.done() + await mutation + + assert store._lock_key("acme") not in redis.values + assert store._user_key("acme", "alice") not in redis.values + assert not await store._set_if_locked( + "acme", + "other-owner", + store._user_key("acme", "alice"), + {"found": True, "value": {"user_id": "alice", "role": Role.ADMIN}}, + ) + + +async def test_api_key_manager_uses_cached_public_store_reads(): + api_key = _api_key() + store, mysql = _store(api_key) + await store.load() + manager = APIKeyManager("root", store) + + first = await manager.resolve_identity(api_key) + second = await manager.resolve_identity(api_key) + + assert (first.account_id, first.user_id, first.role) == ("acme", "alice", Role.ADMIN) + assert second == first + assert mysql.calls == { + "verify": 1, + "user": 1, + "groups": 0, + "account_deletion": 1, + "user_deletion": 1, + } + + +async def test_replacing_key_invalidates_cached_binding_and_prior_negative(): + old_key = _api_key("old") + new_key = _api_key("new") + store, mysql = _store(old_key) + await store.load() + assert await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") + assert await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint="alice") is None + assert store._binding_key("acme", new_key) not in await store._redis.zrange( + store._cache_keys_index_key("acme"), + 0, + -1, + ) + + await store.replace_active_user_api_key("acme", "alice", new_key) + + assert await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") is None + assert await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint="alice") == ( + "acme", + "alice", + ) + assert mysql.calls["verify"] == 4 + + +async def test_creating_account_invalidates_prior_negative_key_binding(): + api_key = _api_key("new-account") + store, mysql = _store(api_key) + mysql.account_exists = False + mysql.user_exists = False + await store.load() + + assert await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="alice", + ) is None + + await store.create_account_with_api_key("acme", "alice", api_key) + + assert await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="alice", + ) == ("acme", "alice") + + +async def test_creating_user_invalidates_prior_negative_key_binding(): + api_key = _api_key("new-user") + store, mysql = _store(api_key) + mysql.user_exists = False + await store.load() + + assert await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="alice", + ) is None + + await store.create_user_with_api_key("acme", "alice", Role.USER, api_key) + + assert await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="alice", + ) == ("acme", "alice") + + +async def test_account_delete_and_recreate_clear_all_indexed_cache_keys(): + api_key = _api_key() + store, mysql = _store(api_key) + await store.load() + redis = store._redis + + assert await store.get_user("acme", "alice") + assert await store.get_user_group_ids("acme", "alice") == ("engineering",) + assert await store.get_deletion("acme", "alice") is None + assert await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="alice", + ) + index_key = store._cache_keys_index_key("acme") + indexed_keys = set(await redis.zrange(index_key, 0, -1)) + assert store._user_key("acme", "alice") in indexed_keys + assert store._group_ids_key("acme", "alice") in indexed_keys + assert store._account_deletion_key("acme") in indexed_keys + assert store._user_deletion_key("acme", "alice") in indexed_keys + assert store._binding_key("acme", api_key) in indexed_keys + assert store._account_credential_fence_key("acme") in indexed_keys + assert store._user_credential_fence_key("acme", "alice") in indexed_keys + + await store.delete_account("acme") + + assert index_key not in redis.zsets + assert not indexed_keys.intersection(redis.values) + assert await store.get_user("acme", "alice") is None + + await store.create_account_with_api_key("acme", "alice", api_key) + + assert await store.get_user("acme", "alice") == { + "user_id": "alice", + "role": Role.ADMIN, + } + + +async def test_refill_prunes_expired_cache_index_members(): + store, _ = _store(_api_key()) + await store.load() + redis = store._redis + index_key = store._cache_keys_index_key("acme") + user_key = store._user_key("acme", "alice") + group_key = store._group_ids_key("acme", "alice") + + assert await store.get_user("acme", "alice") + redis.now_ms += 60_001 + assert await store.get_user_group_ids("acme", "alice") == ("engineering",) + + assert await redis.zrange(index_key, 0, -1) == [group_key] + assert user_key not in redis.zsets[index_key] + + +async def test_targeted_invalidation_removes_cache_index_member(): + store, _ = _store(_api_key()) + await store.load() + redis = store._redis + index_key = store._cache_keys_index_key("acme") + user_key = store._user_key("acme", "alice") + + assert await store.get_user("acme", "alice") + assert user_key in await redis.zrange(index_key, 0, -1) + + await store.set_role("acme", "alice", Role.USER) + + assert user_key not in await redis.zrange(index_key, 0, -1) + + +async def test_user_group_and_deletion_reads_are_cached_and_invalidated(): + api_key = _api_key() + store, mysql = _store(api_key) + await store.load() + + assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.ADMIN} + assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.ADMIN} + await store.set_role("acme", "alice", Role.USER) + assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.USER} + + assert await store.get_user_group_ids("acme", "alice") == ("engineering",) + assert await store.get_user_group_ids("acme", "alice") == ("engineering",) + await store.add_group_member("acme", "product", "alice") + assert await store.get_user_group_ids("acme", "alice") == ("engineering", "product") + + assert await store.get_deletion("acme", "alice") is None + assert await store.get_deletion("acme", "alice") is None + await store.begin_deletion( + "acme", + "alice", + task_id="delete-1", + owner_account_id="acme", + owner_user_id="alice", + ) + assert await store.get_deletion("acme", "alice") == {"task_id": "delete-1"} + + assert mysql.calls == { + "verify": 0, + "user": 2, + "groups": 2, + "account_deletion": 1, + "user_deletion": 2, + } diff --git a/uv.lock b/uv.lock index 7eb82a11fb..9419799594 100644 --- a/uv.lock +++ b/uv.lock @@ -160,6 +160,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" }, ] +[[package]] +name = "aiomysql" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pymysql" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/e0/302aeffe8d90853556f47f3106b89c16cc2ec2a4d269bdfd82e3f4ae12cc/aiomysql-0.3.2.tar.gz", hash = "sha256:72d15ef5cfc34c03468eb41e1b90adb9fd9347b0b589114bd23ead569a02ac1a", size = 108311, upload-time = "2025-10-22T00:15:21.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/af/aae0153c3e28712adaf462328f6c7a3c196a1c1c27b491de4377dd3e6b52/aiomysql-0.3.2-py3-none-any.whl", hash = "sha256:c82c5ba04137d7afd5c693a258bea8ead2aad77101668044143a991e04632eb2", size = 71834, upload-time = "2025-10-22T00:15:15.905Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -1439,6 +1451,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/3f/9859f655d11901e7b2996c6e3d33e0caa9a1d4572c3bc61ed0faa64b2f4c/greenlet-3.3.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9bc885b89709d901859cf95179ec9f6bb67a3d2bb1f0e88456461bd4b7f8fd0d", size = 277747, upload-time = "2026-02-20T20:16:21.325Z" }, { url = "https://files.pythonhosted.org/packages/fb/07/cb284a8b5c6498dbd7cba35d31380bb123d7dceaa7907f606c8ff5993cbf/greenlet-3.3.2-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b568183cf65b94919be4438dc28416b234b678c608cafac8874dfeeb2a9bbe13", size = 579202, upload-time = "2026-02-20T20:47:28.955Z" }, { url = "https://files.pythonhosted.org/packages/ed/45/67922992b3a152f726163b19f890a85129a992f39607a2a53155de3448b8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:527fec58dc9f90efd594b9b700662ed3fb2493c2122067ac9c740d98080a620e", size = 590620, upload-time = "2026-02-20T20:55:55.581Z" }, + { url = "https://files.pythonhosted.org/packages/03/5f/6e2a7d80c353587751ef3d44bb947f0565ec008a2e0927821c007e96d3a7/greenlet-3.3.2-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508c7f01f1791fbc8e011bd508f6794cb95397fdb198a46cb6635eb5b78d85a7", size = 602132, upload-time = "2026-02-20T21:02:43.261Z" }, { url = "https://files.pythonhosted.org/packages/ad/55/9f1ebb5a825215fadcc0f7d5073f6e79e3007e3282b14b22d6aba7ca6cb8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad0c8917dd42a819fe77e6bdfcb84e3379c0de956469301d9fd36427a1ca501f", size = 591729, upload-time = "2026-02-20T20:20:58.395Z" }, { url = "https://files.pythonhosted.org/packages/24/b4/21f5455773d37f94b866eb3cf5caed88d6cea6dd2c6e1f9c34f463cba3ec/greenlet-3.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97245cc10e5515dbc8c3104b2928f7f02b6813002770cfaffaf9a6e0fc2b94ef", size = 1551946, upload-time = "2026-02-20T20:49:31.102Z" }, { url = "https://files.pythonhosted.org/packages/00/68/91f061a926abead128fe1a87f0b453ccf07368666bd59ffa46016627a930/greenlet-3.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8c1fdd7d1b309ff0da81d60a9688a8bd044ac4e18b250320a96fc68d31c209ca", size = 1618494, upload-time = "2026-02-20T20:21:06.541Z" }, @@ -1446,6 +1459,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/47/16400cb42d18d7a6bb46f0626852c1718612e35dcb0dffa16bbaffdf5dd2/greenlet-3.3.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:c56692189a7d1c7606cb794be0a8381470d95c57ce5be03fb3d0ef57c7853b86", size = 278890, upload-time = "2026-02-20T20:19:39.263Z" }, { url = "https://files.pythonhosted.org/packages/a3/90/42762b77a5b6aa96cd8c0e80612663d39211e8ae8a6cd47c7f1249a66262/greenlet-3.3.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ebd458fa8285960f382841da585e02201b53a5ec2bac6b156fc623b5ce4499f", size = 581120, upload-time = "2026-02-20T20:47:30.161Z" }, { url = "https://files.pythonhosted.org/packages/bf/6f/f3d64f4fa0a9c7b5c5b3c810ff1df614540d5aa7d519261b53fba55d4df9/greenlet-3.3.2-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a443358b33c4ec7b05b79a7c8b466f5d275025e750298be7340f8fc63dff2a55", size = 594363, upload-time = "2026-02-20T20:55:56.965Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8b/1430a04657735a3f23116c2e0d5eb10220928846e4537a938a41b350bed6/greenlet-3.3.2-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4375a58e49522698d3e70cc0b801c19433021b5c37686f7ce9c65b0d5c8677d2", size = 605046, upload-time = "2026-02-20T21:02:45.234Z" }, { url = "https://files.pythonhosted.org/packages/72/83/3e06a52aca8128bdd4dcd67e932b809e76a96ab8c232a8b025b2850264c5/greenlet-3.3.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e2cd90d413acbf5e77ae41e5d3c9b3ac1d011a756d7284d7f3f2b806bbd6358", size = 594156, upload-time = "2026-02-20T20:20:59.955Z" }, { url = "https://files.pythonhosted.org/packages/70/79/0de5e62b873e08fe3cef7dbe84e5c4bc0e8ed0c7ff131bccb8405cd107c8/greenlet-3.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:442b6057453c8cb29b4fb36a2ac689382fc71112273726e2423f7f17dc73bf99", size = 1554649, upload-time = "2026-02-20T20:49:32.293Z" }, { url = "https://files.pythonhosted.org/packages/5a/00/32d30dee8389dc36d42170a9c66217757289e2afb0de59a3565260f38373/greenlet-3.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45abe8eb6339518180d5a7fa47fa01945414d7cca5ecb745346fc6a87d2750be", size = 1619472, upload-time = "2026-02-20T20:21:07.966Z" }, @@ -1454,6 +1468,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd", size = 280358, upload-time = "2026-02-20T20:17:43.971Z" }, { url = "https://files.pythonhosted.org/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd", size = 601217, upload-time = "2026-02-20T20:47:31.462Z" }, { url = "https://files.pythonhosted.org/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac", size = 611792, upload-time = "2026-02-20T20:55:58.423Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb", size = 626250, upload-time = "2026-02-20T21:02:46.596Z" }, { url = "https://files.pythonhosted.org/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070", size = 613875, upload-time = "2026-02-20T20:21:01.102Z" }, { url = "https://files.pythonhosted.org/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79", size = 1571467, upload-time = "2026-02-20T20:49:33.495Z" }, { url = "https://files.pythonhosted.org/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395", size = 1640001, upload-time = "2026-02-20T20:21:09.154Z" }, @@ -1462,6 +1477,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/48/f8b875fa7dea7dd9b33245e37f065af59df6a25af2f9561efa8d822fde51/greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4", size = 279120, upload-time = "2026-02-20T20:19:01.9Z" }, { url = "https://files.pythonhosted.org/packages/49/8d/9771d03e7a8b1ee456511961e1b97a6d77ae1dea4a34a5b98eee706689d3/greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986", size = 603238, upload-time = "2026-02-20T20:47:32.873Z" }, { url = "https://files.pythonhosted.org/packages/59/0e/4223c2bbb63cd5c97f28ffb2a8aee71bdfb30b323c35d409450f51b91e3e/greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92", size = 614219, upload-time = "2026-02-20T20:55:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/94/2b/4d012a69759ac9d77210b8bfb128bc621125f5b20fc398bce3940d036b1c/greenlet-3.3.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccd21bb86944ca9be6d967cf7691e658e43417782bce90b5d2faeda0ff78a7dd", size = 628268, upload-time = "2026-02-20T21:02:48.024Z" }, { url = "https://files.pythonhosted.org/packages/7a/34/259b28ea7a2a0c904b11cd36c79b8cef8019b26ee5dbe24e73b469dea347/greenlet-3.3.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6997d360a4e6a4e936c0f9625b1c20416b8a0ea18a8e19cabbefc712e7397ab", size = 616774, upload-time = "2026-02-20T20:21:02.454Z" }, { url = "https://files.pythonhosted.org/packages/0a/03/996c2d1689d486a6e199cb0f1cf9e4aa940c500e01bdf201299d7d61fa69/greenlet-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64970c33a50551c7c50491671265d8954046cb6e8e2999aacdd60e439b70418a", size = 1571277, upload-time = "2026-02-20T20:49:34.795Z" }, { url = "https://files.pythonhosted.org/packages/d9/c4/2570fc07f34a39f2caf0bf9f24b0a1a0a47bc2e8e465b2c2424821389dfc/greenlet-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a9172f5bf6bd88e6ba5a84e0a68afeac9dc7b6b412b245dd64f52d83c81e55b", size = 1640455, upload-time = "2026-02-20T20:21:10.261Z" }, @@ -1470,6 +1486,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/ae/8bffcbd373b57a5992cd077cbe8858fff39110480a9d50697091faea6f39/greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab", size = 279650, upload-time = "2026-02-20T20:18:00.783Z" }, { url = "https://files.pythonhosted.org/packages/d1/c0/45f93f348fa49abf32ac8439938726c480bd96b2a3c6f4d949ec0124b69f/greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082", size = 650295, upload-time = "2026-02-20T20:47:34.036Z" }, { url = "https://files.pythonhosted.org/packages/b3/de/dd7589b3f2b8372069ab3e4763ea5329940fc7ad9dcd3e272a37516d7c9b/greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9", size = 662163, upload-time = "2026-02-20T20:56:01.295Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ac/85804f74f1ccea31ba518dcc8ee6f14c79f73fe36fa1beba38930806df09/greenlet-3.3.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9", size = 675371, upload-time = "2026-02-20T21:02:49.664Z" }, { url = "https://files.pythonhosted.org/packages/d2/d8/09bfa816572a4d83bccd6750df1926f79158b1c36c5f73786e26dbe4ee38/greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506", size = 664160, upload-time = "2026-02-20T20:21:04.015Z" }, { url = "https://files.pythonhosted.org/packages/48/cf/56832f0c8255d27f6c35d41b5ec91168d74ec721d85f01a12131eec6b93c/greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce", size = 1619181, upload-time = "2026-02-20T20:49:36.052Z" }, { url = "https://files.pythonhosted.org/packages/0a/23/b90b60a4aabb4cec0796e55f25ffbfb579a907c3898cd2905c8918acaa16/greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5", size = 1687713, upload-time = "2026-02-20T20:21:11.684Z" }, @@ -1478,6 +1495,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/6d/8f2ef704e614bcf58ed43cfb8d87afa1c285e98194ab2cfad351bf04f81e/greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54", size = 286617, upload-time = "2026-02-20T20:19:29.856Z" }, { url = "https://files.pythonhosted.org/packages/5e/0d/93894161d307c6ea237a43988f27eba0947b360b99ac5239ad3fe09f0b47/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4", size = 655189, upload-time = "2026-02-20T20:47:35.742Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/d2d506ebd8abcb57386ec4f7ba20f4030cbe56eae541bc6fd6ef399c0b41/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff", size = 658225, upload-time = "2026-02-20T20:56:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/d1/67/8197b7e7e602150938049d8e7f30de1660cfb87e4c8ee349b42b67bdb2e1/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:59b3e2c40f6706b05a9cd299c836c6aa2378cabe25d021acd80f13abf81181cf", size = 666581, upload-time = "2026-02-20T21:02:51.526Z" }, { url = "https://files.pythonhosted.org/packages/8e/30/3a09155fbf728673a1dea713572d2d31159f824a37c22da82127056c44e4/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4", size = 657907, upload-time = "2026-02-20T20:21:05.259Z" }, { url = "https://files.pythonhosted.org/packages/f3/fd/d05a4b7acd0154ed758797f0a43b4c0962a843bedfe980115e842c5b2d08/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727", size = 1618857, upload-time = "2026-02-20T20:49:37.309Z" }, { url = "https://files.pythonhosted.org/packages/6f/e1/50ee92a5db521de8f35075b5eff060dd43d39ebd46c2181a2042f7070385/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e", size = 1680010, upload-time = "2026-02-20T20:21:13.427Z" }, @@ -3609,12 +3627,21 @@ gemini-async = [ local-embed = [ { name = "llama-cpp-python" }, ] +mysql = [ + { name = "aiomysql" }, + { name = "sqlalchemy", extra = ["asyncio"] }, +] ocr = [ { name = "pytesseract" }, ] opengauss = [ { name = "psycopg2-binary" }, ] +redis-mysql = [ + { name = "aiomysql" }, + { name = "redis" }, + { name = "sqlalchemy", extra = ["asyncio"] }, +] test = [ { name = "boto3" }, { name = "datasets" }, @@ -3627,6 +3654,7 @@ test = [ { name = "pytest-cov" }, { name = "pytest-xdist" }, { name = "ragas" }, + { name = "sqlalchemy", extra = ["asyncio"] }, ] [package.dev-dependencies] @@ -3638,6 +3666,8 @@ dev = [ requires-dist = [ { name = "agent-sandbox", marker = "extra == 'bot'", specifier = ">=0.0.23" }, { name = "aiohttp", specifier = ">=3.13.3" }, + { name = "aiomysql", marker = "extra == 'mysql'", specifier = ">=0.2.0,<1" }, + { name = "aiomysql", marker = "extra == 'redis-mysql'", specifier = ">=0.2.0,<1" }, { name = "anyio", marker = "extra == 'gemini-async'", specifier = ">=4.0.0" }, { name = "apscheduler", specifier = ">=3.11.0" }, { name = "argon2-cffi", specifier = ">=23.0.0" }, @@ -3730,6 +3760,7 @@ requires-dist = [ { name = "ragas", marker = "extra == 'test'", specifier = ">=0.1.0" }, { name = "rapidfuzz", specifier = ">=3.14,<4" }, { name = "readability-lxml", marker = "extra == 'bot'", specifier = ">=0.8.0" }, + { name = "redis", marker = "extra == 'redis-mysql'", specifier = ">=5.0,<7" }, { name = "requests", specifier = ">=2.33.0" }, { name = "rich", marker = "extra == 'bot'", specifier = ">=13.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, @@ -3741,6 +3772,9 @@ requires-dist = [ { name = "socksio", marker = "extra == 'bot'", specifier = ">=1.0.0" }, { name = "sphinx", marker = "extra == 'doc'", specifier = ">=7.0.0" }, { name = "sphinx-rtd-theme", marker = "extra == 'doc'", specifier = ">=1.3.0" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'mysql'", specifier = ">=2.0,<2.1" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'redis-mysql'", specifier = ">=2.0,<2.1" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'test'", specifier = ">=2.0,<2.1" }, { name = "tabulate", specifier = ">=0.9.0" }, { name = "tavily-python", marker = "extra == 'bot'", specifier = ">=0.5.0" }, { name = "tiktoken", marker = "extra == 'benchmark'", specifier = ">=0.5.0" }, @@ -3769,7 +3803,7 @@ requires-dist = [ { name = "wheel", marker = "extra == 'build'" }, { name = "xxhash", specifier = ">=3.0.0" }, ] -provides-extras = ["gateway-fast", "test", "auth", "dev", "doc", "eval", "gemini", "gemini-async", "ocr", "build", "bot", "benchmark", "local-embed", "opengauss"] +provides-extras = ["gateway-fast", "mysql", "redis-mysql", "test", "auth", "dev", "doc", "eval", "gemini", "gemini-async", "ocr", "build", "bot", "benchmark", "local-embed", "opengauss"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.2" }] @@ -4805,6 +4839,15 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pymysql" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/d4/c15b459e25a23767d2f4065ef40968920320f04e302889574310c21c96a3/pymysql-1.2.3.tar.gz", hash = "sha256:d5b288529782e536ae171866df3ca9dc4f6cbfb3cc2f18e6f837fbb90dbc262b", size = 50629, upload-time = "2026-09-17T12:22:49.146Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/4b/0a906d8184f011ff8dbd4722743783867589b33269d2c5fff238d636fdcb/pymysql-1.2.3-py3-none-any.whl", hash = "sha256:14f1c68e2ed859243ae5ca41ffbe677027fc46bc136a9f0be8a4e928e5e7415a", size = 46740, upload-time = "2026-09-17T12:22:47.826Z" }, +] + [[package]] name = "pyopenssl" version = "26.3.0" @@ -5438,6 +5481,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/75/2cc58965097e351415af420be81c4665cf80da52a17ef43c01ffbe2caf91/readability_lxml-0.8.4.1-py3-none-any.whl", hash = "sha256:874c0cea22c3bf2b78c7f8df831bfaad3c0a89b7301d45a188db581652b4b465", size = 19912, upload-time = "2025-05-03T21:11:43.993Z" }, ] +[[package]] +name = "redis" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -6228,6 +6283,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/2c/9664130905f03db57961b8980b05cab624afd114bf2be2576628a9f22da4/sqlalchemy-2.0.48-py3-none-any.whl", hash = "sha256:a66fe406437dd65cacd96a72689a3aaaecaebbcd62d81c5ac1c0fdbeac835096", size = 1940202, upload-time = "2026-03-02T15:52:43.285Z" }, ] +[package.optional-dependencies] +asyncio = [ + { name = "greenlet" }, +] + [[package]] name = "sse-starlette" version = "3.3.4" From d91d329f9a497d549a81306882407510c698c600 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Wed, 30 Sep 2026 19:00:02 +0800 Subject: [PATCH 02/16] fix(account-store): harden distributed providers --- docs/en/guides/01-configuration.md | 18 +- docs/zh/guides/01-configuration.md | 12 +- openviking/server/account_stores/mysql.py | 125 ++--- .../server/account_stores/mysql_schema.py | 475 ++++++++---------- .../server/account_stores/redis_mysql.py | 100 ++-- openviking/server/account_stores/sql/init.sql | 55 +- openviking/server/config.py | 7 +- tests/server/test_account_services.py | 126 ++++- tests/server/test_auth.py | 7 +- .../server/test_redis_mysql_account_store.py | 76 ++- 10 files changed, 588 insertions(+), 413 deletions(-) diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index b8fb113bea..e8a242861a 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1934,11 +1934,9 @@ The provider never creates or migrates database tables. Create the normalized tables in `openviking/server/account_stores/sql/init.sql` through your normal DBA or deployment process before starting the server. Future reviewed schema changes are stored as ordered SQL files in -`openviking/server/account_stores/sql/changes/`. The initialization script uses -the default `ov_accounts` prefix. When using `account_store.params.table`, -replace only the backtick-quoted `ov_accounts` table identifiers in the -initialization script. Constraint and index names are stable and independent of -the configured table names. +`openviking/server/account_stores/sql/changes/`. The five `ov_accounts*` table +names are fixed; use `OV_RESOURCE_ID` rather than separate table prefixes to +isolate OpenViking deployments that share a database. ```json { @@ -1962,17 +1960,15 @@ the configured table names. parameters. It caches API-key bindings, users, per-user group IDs, and deletion records for a fixed 60 seconds by default. Cache fills and account mutations share an account-scoped Redis lease; normal cache hits do not acquire that -lease. Mutations commit to MySQL first, then wait up to 30 seconds for the lease -and invalidate affected cache entries before returning. If that wait times out, -the provider revokes the current lease, reacquires it, and invalidates before -returning the committed MySQL result. Cache fills also maintain an +lease. Mutations commit to MySQL first, then invalidate affected cache entries. +If Redis is unavailable, the MySQL mutation still succeeds and stale cache +entries remain valid only until their fixed TTL expires. Cache fills also +maintain an account-scoped, expiry-scored key index. Each indexed cache fill removes expired index members, allowing account deletion or recreation to clear only that account's current cached users, groups, deletion records, and credential data. API-key bindings are checked against account and user credential fences, so normal key replacement, revocation, and user deletion invalidate old bindings. -If Redis is unavailable after a MySQL mutation, any stale cache entry remains -valid only until its fixed TTL expires. ```json { diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index 1e859f5ffd..c761abf92c 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1915,10 +1915,9 @@ prefix；加密模式可为同一列表工作流解密展示 key。`credential_s Provider 不会创建或迁移数据库表。启动服务前，应通过现有 DBA 或部署流程手工创建 `openviking/server/account_stores/sql/init.sql` 中定义的规范化表。后续经评审的 -schema 变更按顺序存放于 `openviking/server/account_stores/sql/changes/`。初始化 -脚本使用默认表前缀 `ov_accounts`；如配置了 -`account_store.params.table`，执行前只替换初始化脚本中反引号包裹的 -`ov_accounts` 表标识符。约束名和索引名保持固定，不依赖配置的表名。 +schema 变更按顺序存放于 `openviking/server/account_stores/sql/changes/`。 +五张 `ov_accounts*` 表的名称固定；多个 OpenViking resource 共用数据库时， +通过 `OV_RESOURCE_ID` 隔离部署，不使用不同表前缀。 ```json { @@ -1941,12 +1940,11 @@ schema 变更按顺序存放于 `openviking/server/account_stores/sql/changes/` `redis_mysql` 需要安装 `redis-mysql` extra，并在参数中嵌套同一份 MySQL 配置。 它默认以固定 60 秒 TTL 缓存 API Key 绑定、User、每个 User 的 Group ID 和删除记录。 缓存重建与同一 Account 的修改共用 Redis 租约，正常缓存命中不获取该租约。修改会先 -提交 MySQL，再最多等待 30 秒获取租约并在返回前失效相关缓存；等待超时后会撤销当前 -租约、重新加锁并完成失效，然后正常返回已提交的 MySQL 结果。缓存回填同时维护 Account +提交 MySQL，再失效相关缓存。Redis 不可用时 MySQL 修改仍会成功，遗留缓存最多保持到 +固定 TTL 到期。缓存回填同时维护 Account 级、按过期时间排序的 key 索引；每次写入已索引缓存时会清理过期成员，因此删除或重建 Account 时只会清理该 Account 当前的 User、Group、删除记录和凭据缓存。API Key 绑定会 校验 Account 和 User 的凭据 fence，所以正常的换 Key、吊销和删 User 会使旧绑定失效。 -若 MySQL 修改后 Redis 不可用，遗留缓存最多保持到固定 TTL 到期。 ```json { diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index 22cc9682fe..5db4d57cf4 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -37,9 +37,11 @@ UserSummary, ) from openviking.server.account_stores.mysql_schema import ( - AccountStoreModels, - make_account_store_models, - validate_table_prefix, + Account, + Credential, + Group, + GroupMember, + User, ) from openviking.server.identity import Role from openviking_cli.exceptions import ( @@ -83,6 +85,13 @@ def _like_pattern(pattern: str) -> str: return "".join(escaped) +def _substring_pattern(value: str) -> str: + escaped = "".join( + "=" + character if character in {"%", "_", "="} else character for character in value + ) + return f"%{escaped}%" + + def _timestamp(value: datetime) -> str: return value.isoformat() @@ -111,12 +120,6 @@ class MySQLAccountStore(AccountStore): def __init__(self, *, params: dict[str, object] | None = None) -> None: values = dict(params or {}) - try: - table_prefix = validate_table_prefix(values.pop("table", "ov_accounts")) - except ValueError as exc: - raise InvalidArgumentError(str(exc)) from None - self._models: AccountStoreModels = make_account_store_models(table_prefix) - self._tables = self._models.table_names self._resource_id = os.environ.get("OV_RESOURCE_ID", "").strip() if not self._resource_id: raise InvalidArgumentError("OV_RESOURCE_ID is required for MySQL account storage") @@ -208,7 +211,6 @@ def _new_credential( *, sequence: int, ): - Credential = self._models.Credential return Credential( credential_id=uuid4().hex, resource_id=self._resource_id, @@ -290,13 +292,7 @@ async def load(self) -> None: engine = create_async_engine(self._url, **self._engine_options) sessions = async_sessionmaker(engine, expire_on_commit=False) async with sessions() as session: - for model in ( - self._models.Account, - self._models.User, - self._models.Group, - self._models.GroupMember, - self._models.Credential, - ): + for model in (Account, User, Group, GroupMember, Credential): await session.execute(select(*model.__table__.columns).limit(1)) except ImportError: raise RuntimeError("MySQL account storage requires openviking[mysql]") from None @@ -339,7 +335,6 @@ async def _transaction(self) -> AsyncGenerator[AsyncSession, None]: yield session async def _lock_account(self, session: AsyncSession, account_id: str): - Account = self._models.Account account = await session.scalar( select(Account) .where( @@ -353,7 +348,6 @@ async def _lock_account(self, session: AsyncSession, account_id: str): return account async def _lock_user(self, session: AsyncSession, account, user_id: str): - User = self._models.User user = await session.scalar( select(User) .where( @@ -378,7 +372,6 @@ def _require_active(record) -> None: async def _protect_last_admin(self, session: AsyncSession, account, user) -> None: if user.role != Role.ADMIN or user.deletion_task_id: return - User = self._models.User count = await session.scalar( select(func.count()) .select_from(User) @@ -403,7 +396,6 @@ def _account_summary(account, user_count: int) -> AccountSummary: } async def get_account(self, account_id: str) -> AccountSummary | None: - Account, User = self._models.Account, self._models.User async with self._session_factory() as session: row = ( await session.execute( @@ -425,7 +417,6 @@ async def get_account(self, account_id: str) -> AccountSummary | None: return self._account_summary(*row) if row else None async def get_user(self, account_id: str, user_id: str) -> UserSummary | None: - Account, User = self._models.Account, self._models.User async with self._session_factory() as session: user = await session.scalar( select(User) @@ -452,12 +443,9 @@ async def create_account_with_api_key( ) -> None: _validate(account_id, validate_account_id) _validate(admin_user_id, validate_user_id) - Account, User = self._models.Account, self._models.User try: async with self._transaction() as session: - account = Account( - resource_id=self._resource_id, account_id=account_id - ) + account = Account(resource_id=self._resource_id, account_id=account_id) session.add(account) await session.flush() user = User( @@ -483,7 +471,6 @@ async def create_account_with_api_key( async def create_user(self, account_id: str, user_id: str, role: str) -> None: _validate(user_id, validate_user_id) role = _validate_role(role) - User = self._models.User try: async with self._transaction() as session: account = await self._lock_account(session, account_id) @@ -509,7 +496,6 @@ async def create_user_with_api_key( ) -> None: _validate(user_id, validate_user_id) role = _validate_role(role) - User = self._models.User try: async with self._transaction() as session: account = await self._lock_account(session, account_id) @@ -545,12 +531,16 @@ async def list_accounts( page: int = 1, query_filter: str | None = None, ) -> list[AccountSummary]: - Account, User = self._models.Account, self._models.User predicates = [Account.resource_id == self._resource_id] if name_filter: predicates.append(Account.account_id.like(_like_pattern(name_filter), escape="=")) if query := (query_filter or "").strip(): - predicates.append(func.lower(Account.account_id).like(f"%{query.casefold()}%")) + predicates.append( + func.lower(Account.account_id).like( + _substring_pattern(query.casefold()), + escape="=", + ) + ) statement = ( select(Account, func.count(User.user_pk)) .outerjoin( @@ -579,16 +569,13 @@ async def list_users_page( page: int = 1, query_filter: str | None = None, ) -> UsersPage: - User, Credential = self._models.User, self._models.Credential async with self._session_factory() as session: # One HTTP response must derive its rows and summary from one MVCC snapshot. - await session.connection( - execution_options={"isolation_level": "REPEATABLE READ"} - ) + await session.connection(execution_options={"isolation_level": "REPEATABLE READ"}) account = await session.scalar( - select(self._models.Account).where( - self._models.Account.resource_id == self._resource_id, - self._models.Account.account_id == account_id, + select(Account).where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, ) ) if account is None: @@ -624,7 +611,12 @@ async def list_users_page( if role_filter: filters.append(User.role == role_filter) if query := (query_filter or "").strip(): - filters.append(func.lower(User.user_id).like(f"%{query.casefold()}%")) + filters.append( + func.lower(User.user_id).like( + _substring_pattern(query.casefold()), + escape="=", + ) + ) total = await session.scalar(select(func.count()).select_from(User).where(*filters)) statement = select(User).where(*filters).order_by(User.created_at, User.user_pk) if limit is not None: @@ -680,7 +672,6 @@ async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> di _validate(user_id, validate_user_id) created_accounts = 0 created_users = 0 - Account, User = self._models.Account, self._models.User for account_id, user_ids in identities.items(): if not user_ids: continue @@ -813,14 +804,13 @@ async def get_deletion( if user_id is None: async with self._session_factory() as session: account = await session.scalar( - select(self._models.Account).where( - self._models.Account.resource_id == self._resource_id, - self._models.Account.account_id == account_id, + select(Account).where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, ) ) return _deletion(account) if account else None - Account, User = self._models.Account, self._models.User async with self._session_factory() as session: row = ( await session.execute( @@ -845,7 +835,6 @@ async def get_deletion( return _deletion(account) or (_deletion(user) if user else None) async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: - Account, User = self._models.Account, self._models.User async with self._session_factory() as session: accounts = ( await session.scalars( @@ -878,7 +867,6 @@ async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: async def create_group(self, account_id: str, group_id: str) -> GroupSummary: _validate(group_id, lambda value: validate_identifier_part(value, "group_id")) - Group = self._models.Group try: async with self._transaction() as session: account = await self._lock_account(session, account_id) @@ -897,9 +885,9 @@ async def create_group(self, account_id: str, group_id: str) -> GroupSummary: async def _account_for_read(self, session: AsyncSession, account_id: str): account = await session.scalar( - select(self._models.Account).where( - self._models.Account.resource_id == self._resource_id, - self._models.Account.account_id == account_id, + select(Account).where( + Account.resource_id == self._resource_id, + Account.account_id == account_id, ) ) if account is None: @@ -907,7 +895,6 @@ async def _account_for_read(self, session: AsyncSession, account_id: str): return account async def get_groups(self, account_id: str) -> list[GroupSummary]: - Group, GroupMember = self._models.Group, self._models.GroupMember async with self._session_factory() as session: account = await self._account_for_read(session, account_id) rows = ( @@ -917,6 +904,7 @@ async def get_groups(self, account_id: str) -> list[GroupSummary]: GroupMember, and_( GroupMember.resource_id == Group.resource_id, + GroupMember.account_pk == Group.account_pk, GroupMember.group_pk == Group.group_pk, ), ) @@ -931,7 +919,6 @@ async def get_groups(self, account_id: str) -> list[GroupSummary]: return [GroupSummary(group_id=group_id, member_count=count) for group_id, count in rows] async def get_group_members(self, account_id: str, group_id: str) -> list[str]: - Group, GroupMember, User = (self._models.Group, self._models.GroupMember, self._models.User) async with self._session_factory() as session: account = await self._account_for_read(session, account_id) group = await session.scalar( @@ -951,11 +938,13 @@ async def get_group_members(self, account_id: str, group_id: str) -> list[str]: GroupMember, and_( GroupMember.resource_id == User.resource_id, + GroupMember.account_pk == User.account_pk, GroupMember.user_pk == User.user_pk, ), ) .where( GroupMember.resource_id == self._resource_id, + GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, ) .order_by(User.user_id) @@ -964,12 +953,6 @@ async def get_group_members(self, account_id: str, group_id: str) -> list[str]: ) async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, ...]: - Account, User, Group, GroupMember = ( - self._models.Account, - self._models.User, - self._models.Group, - self._models.GroupMember, - ) async with self._session_factory() as session: group_ids = ( await session.scalars( @@ -986,6 +969,7 @@ async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, GroupMember, and_( GroupMember.resource_id == User.resource_id, + GroupMember.account_pk == User.account_pk, GroupMember.user_pk == User.user_pk, ), ) @@ -993,6 +977,7 @@ async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, Group, and_( Group.resource_id == GroupMember.resource_id, + Group.account_pk == GroupMember.account_pk, Group.group_pk == GroupMember.group_pk, ), ) @@ -1007,7 +992,6 @@ async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, return tuple(group_id for group_id in group_ids if group_id is not None) async def add_group_member(self, account_id: str, group_id: str, user_id: str) -> bool: - Group, GroupMember = self._models.Group, self._models.GroupMember async with self._transaction() as session: account = await self._lock_account(session, account_id) self._require_active(account) @@ -1028,6 +1012,7 @@ async def add_group_member(self, account_id: str, group_id: str, user_id: str) - await session.scalar( select(GroupMember.group_pk).where( GroupMember.resource_id == self._resource_id, + GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, GroupMember.user_pk == user.user_pk, ) @@ -1037,6 +1022,7 @@ async def add_group_member(self, account_id: str, group_id: str, user_id: str) - session.add( GroupMember( resource_id=self._resource_id, + account_pk=account.account_pk, group_pk=group.group_pk, user_pk=user.user_pk, ) @@ -1044,7 +1030,6 @@ async def add_group_member(self, account_id: str, group_id: str, user_id: str) - return True async def remove_group_member(self, account_id: str, group_id: str, user_id: str) -> bool: - Group, GroupMember, User = (self._models.Group, self._models.GroupMember, self._models.User) async with self._transaction() as session: account = await self._lock_account(session, account_id) self._require_active(account) @@ -1065,11 +1050,13 @@ async def remove_group_member(self, account_id: str, group_id: str, user_id: str User, and_( User.resource_id == GroupMember.resource_id, + User.account_pk == GroupMember.account_pk, User.user_pk == GroupMember.user_pk, ), ) .where( GroupMember.resource_id == self._resource_id, + GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, User.account_pk == account.account_pk, User.user_id == user_id, @@ -1081,7 +1068,6 @@ async def remove_group_member(self, account_id: str, group_id: str, user_id: str return True async def delete_group(self, account_id: str, group_id: str) -> None: - Group, GroupMember = self._models.Group, self._models.GroupMember async with self._transaction() as session: account = await self._lock_account(session, account_id) self._require_active(account) @@ -1101,6 +1087,7 @@ async def delete_group(self, account_id: str, group_id: str) -> None: .select_from(GroupMember) .where( GroupMember.resource_id == self._resource_id, + GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, ) ): @@ -1110,7 +1097,6 @@ async def delete_group(self, account_id: str, group_id: str) -> None: async def replace_active_user_api_key( self, account_id: str, user_id: str, api_key: str ) -> None: - Credential = self._models.Credential async with self._transaction() as session: account = await self._lock_account(session, account_id) self._require_active(account) @@ -1153,15 +1139,12 @@ async def verify_api_key( account_id_hint: str | None = None, user_id_hint: str | None = None, ) -> tuple[str, str] | None: - Account, User, Credential = ( - self._models.Account, - self._models.User, - self._models.Credential, - ) predicates = [ Credential.resource_id == self._resource_id, Credential.lookup_digest == _credential_digest(api_key), Credential.revoked.is_(False), + Account.deletion_task_id.is_(None), + User.deletion_task_id.is_(None), ] if account_id_hint is not None: predicates.append(Account.account_id == account_id_hint) @@ -1194,13 +1177,12 @@ async def verify_api_key( return None async def revoke_active_api_keys(self, account_id: str, user_id: str | None = None) -> None: - User, Credential = self._models.User, self._models.Credential async with self._transaction() as session: account = await session.scalar( - select(self._models.Account) + select(Account) .where( - self._models.Account.resource_id == self._resource_id, - self._models.Account.account_id == account_id, + Account.resource_id == self._resource_id, + Account.account_id == account_id, ) .with_for_update() ) @@ -1227,11 +1209,6 @@ async def revoke_active_api_keys(self, account_id: str, user_id: str | None = No ) async def get_user_key_fingerprint(self, account_id: str, user_id: str) -> str | None: - Account, User, Credential = ( - self._models.Account, - self._models.User, - self._models.Credential, - ) async with self._session_factory() as session: material = await session.scalar( select(Credential.material) diff --git a/openviking/server/account_stores/mysql_schema.py b/openviking/server/account_stores/mysql_schema.py index 11363a1ced..9968b3496e 100644 --- a/openviking/server/account_stores/mysql_schema.py +++ b/openviking/server/account_stores/mysql_schema.py @@ -1,17 +1,14 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -"""SQLAlchemy schema for the MySQL account-store provider. +"""Fixed SQLAlchemy schema for the MySQL account-store provider. -The table prefix is an operational namespace, while ``resource_id`` is the -tenant boundary. The latter is deliberately present in every primary lookup, -unique key, and foreign-key relationship so one database can hold many -OpenViking deployments safely. +``resource_id`` is the tenant boundary. It is deliberately present in every +primary lookup, unique key, and foreign-key relationship so one database can +hold many OpenViking deployments safely. """ from __future__ import annotations -import re -from dataclasses import dataclass from typing import Optional from sqlalchemy import ( @@ -29,248 +26,222 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column -def _schema_object_name(kind: str, semantic_name: str) -> str: - """Return a stable schema-object name independent of configured table names.""" - return f"{kind}_ov_as_{semantic_name}" - - -def validate_table_prefix(table_prefix: object) -> str: - if not isinstance(table_prefix, str) or not re.fullmatch( - r"[A-Za-z][A-Za-z0-9_]{0,49}", table_prefix - ): - raise ValueError( - "MySQL table must start with a letter, contain only letters, " - "digits, or underscores, and be at most 50 characters" - ) - return table_prefix - - -@dataclass(frozen=True) -class AccountStoreModels: - Base: type[DeclarativeBase] - Account: type - User: type - Group: type - GroupMember: type - Credential: type - table_prefix: str - - @property - def table_names(self) -> dict[str, str]: - return { - "accounts": self.Account.__tablename__, - "users": self.User.__tablename__, - "groups": self.Group.__tablename__, - "memberships": self.GroupMember.__tablename__, - "credentials": self.Credential.__tablename__, - } - - -def make_account_store_models(table_prefix: str) -> AccountStoreModels: - """Build a mapper set for one configured account-store table prefix.""" - table_prefix = validate_table_prefix(table_prefix) - accounts = table_prefix - users = f"{table_prefix}_users" - groups = f"{table_prefix}_groups" - members = f"{table_prefix}_group_members" - credentials = f"{table_prefix}_credentials" - - class Base(DeclarativeBase): - pass - - class Account(Base): - __tablename__ = accounts - __table_args__ = ( - UniqueConstraint( - "resource_id", - "account_id", - name=_schema_object_name("uq", "account_identity"), - ), - UniqueConstraint( - "resource_id", - "account_pk", - name=_schema_object_name("uq", "account_resource_pk"), - ), - Index( - _schema_object_name("ix", "account_deletion"), - "resource_id", - "deletion_task_id", - ), - {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, - ) - - account_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - account_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - created_at: Mapped[object] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") - ) - deletion_task_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) - deletion_owner_account_id: Mapped[Optional[str]] = mapped_column( - String(255, collation="utf8mb4_bin"), nullable=True - ) - deletion_owner_user_id: Mapped[Optional[str]] = mapped_column( - String(255, collation="utf8mb4_bin"), nullable=True - ) - - class User(Base): - __tablename__ = users - __table_args__ = ( - ForeignKeyConstraint( - ["resource_id", "account_pk"], - [f"{accounts}.resource_id", f"{accounts}.account_pk"], - ondelete="CASCADE", - name=_schema_object_name("fk", "user_account"), - ), - UniqueConstraint( - "resource_id", - "account_pk", - "user_id", - name=_schema_object_name("uq", "user_identity"), - ), - UniqueConstraint( - "resource_id", - "user_pk", - name=_schema_object_name("uq", "user_resource_pk"), - ), - Index( - _schema_object_name("ix", "user_admin"), - "resource_id", - "account_pk", - "role", - ), - Index( - _schema_object_name("ix", "user_deletion"), - "resource_id", - "deletion_task_id", - ), - {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, - ) - - user_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - account_pk: Mapped[int] = mapped_column(BigInteger) - user_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - role: Mapped[str] = mapped_column(String(16)) - created_at: Mapped[object] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") - ) - deletion_task_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) - deletion_owner_account_id: Mapped[Optional[str]] = mapped_column( - String(255, collation="utf8mb4_bin"), nullable=True - ) - deletion_owner_user_id: Mapped[Optional[str]] = mapped_column( - String(255, collation="utf8mb4_bin"), nullable=True - ) - - class Group(Base): - __tablename__ = groups - __table_args__ = ( - ForeignKeyConstraint( - ["resource_id", "account_pk"], - [f"{accounts}.resource_id", f"{accounts}.account_pk"], - ondelete="CASCADE", - name=_schema_object_name("fk", "group_account"), - ), - UniqueConstraint( - "resource_id", - "account_pk", - "group_id", - name=_schema_object_name("uq", "group_identity"), - ), - UniqueConstraint( - "resource_id", - "group_pk", - name=_schema_object_name("uq", "group_resource_pk"), - ), - {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, - ) - - group_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - account_pk: Mapped[int] = mapped_column(BigInteger) - group_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - created_at: Mapped[object] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") - ) - - class GroupMember(Base): - __tablename__ = members - __table_args__ = ( - PrimaryKeyConstraint("resource_id", "group_pk", "user_pk"), - ForeignKeyConstraint( - ["resource_id", "group_pk"], - [f"{groups}.resource_id", f"{groups}.group_pk"], - ondelete="CASCADE", - name=_schema_object_name("fk", "member_group"), - ), - ForeignKeyConstraint( - ["resource_id", "user_pk"], - [f"{users}.resource_id", f"{users}.user_pk"], - ondelete="CASCADE", - name=_schema_object_name("fk", "member_user"), - ), - Index( - _schema_object_name("ix", "member_user"), - "resource_id", - "user_pk", - ), - {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, - ) - - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - group_pk: Mapped[int] = mapped_column(BigInteger) - user_pk: Mapped[int] = mapped_column(BigInteger) - - class Credential(Base): - __tablename__ = credentials - __table_args__ = ( - ForeignKeyConstraint( - ["resource_id", "user_pk"], - [f"{users}.resource_id", f"{users}.user_pk"], - ondelete="CASCADE", - name=_schema_object_name("fk", "credential_user"), - ), - UniqueConstraint( - "resource_id", - "active_user_pk", - name=_schema_object_name("uq", "active_credential"), - ), - Index( - _schema_object_name("ix", "credential_lookup"), - "resource_id", - "lookup_digest", - "revoked", - "user_pk", - ), - Index( - _schema_object_name("ix", "credential_history"), - "resource_id", - "user_pk", - "revoked", - "sequence", - ), - {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, - ) - - credential_id: Mapped[str] = mapped_column( - String(32, collation="ascii_bin"), primary_key=True - ) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) - user_pk: Mapped[int] = mapped_column(BigInteger) - material: Mapped[str] = mapped_column(Text(collation="ascii_bin")) - lookup_digest: Mapped[str] = mapped_column(String(64, collation="ascii_bin")) - prefix: Mapped[str] = mapped_column(String(8, collation="ascii_bin")) - revoked: Mapped[bool] = mapped_column(Boolean, server_default=text("0")) - version: Mapped[int] = mapped_column(BigInteger, server_default=text("1")) - sequence: Mapped[int] = mapped_column(BigInteger) - created_at: Mapped[object] = mapped_column( - DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") - ) - revoked_at: Mapped[Optional[object]] = mapped_column(DATETIME(fsp=6), nullable=True) - active_user_pk: Mapped[Optional[int]] = mapped_column( - BigInteger, - nullable=True, - ) - - return AccountStoreModels(Base, Account, User, Group, GroupMember, Credential, table_prefix) +class Base(DeclarativeBase): + pass + + +class Account(Base): + __tablename__ = "ov_accounts" + __table_args__ = ( + UniqueConstraint( + "resource_id", + "account_id", + name="uq_ov_accounts_account_identity", + ), + UniqueConstraint( + "resource_id", + "account_pk", + name="uq_ov_accounts_account_resource_pk", + ), + Index( + "ix_ov_accounts_account_deletion", + "resource_id", + "deletion_task_id", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + account_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + account_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + deletion_task_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + deletion_owner_account_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + deletion_owner_user_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + + +class User(Base): + __tablename__ = "ov_accounts_users" + __table_args__ = ( + ForeignKeyConstraint( + ["resource_id", "account_pk"], + ["ov_accounts.resource_id", "ov_accounts.account_pk"], + ondelete="CASCADE", + name="fk_ov_accounts_user_account", + ), + UniqueConstraint( + "resource_id", + "account_pk", + "user_id", + name="uq_ov_accounts_user_identity", + ), + UniqueConstraint( + "resource_id", + "user_pk", + name="uq_ov_accounts_user_resource_pk", + ), + UniqueConstraint( + "resource_id", + "account_pk", + "user_pk", + name="uq_ov_accounts_user_account_pk", + ), + Index( + "ix_ov_accounts_user_admin", + "resource_id", + "account_pk", + "role", + ), + Index( + "ix_ov_accounts_user_deletion", + "resource_id", + "deletion_task_id", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + user_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + account_pk: Mapped[int] = mapped_column(BigInteger) + user_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + role: Mapped[str] = mapped_column(String(16)) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + deletion_task_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + deletion_owner_account_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + deletion_owner_user_id: Mapped[Optional[str]] = mapped_column( + String(255, collation="utf8mb4_bin"), nullable=True + ) + + +class Group(Base): + __tablename__ = "ov_accounts_groups" + __table_args__ = ( + ForeignKeyConstraint( + ["resource_id", "account_pk"], + ["ov_accounts.resource_id", "ov_accounts.account_pk"], + ondelete="CASCADE", + name="fk_ov_accounts_group_account", + ), + UniqueConstraint( + "resource_id", + "account_pk", + "group_id", + name="uq_ov_accounts_group_identity", + ), + UniqueConstraint( + "resource_id", + "group_pk", + name="uq_ov_accounts_group_resource_pk", + ), + UniqueConstraint( + "resource_id", + "account_pk", + "group_pk", + name="uq_ov_accounts_group_account_pk", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + group_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + account_pk: Mapped[int] = mapped_column(BigInteger) + group_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + + +class GroupMember(Base): + __tablename__ = "ov_accounts_group_members" + __table_args__ = ( + PrimaryKeyConstraint("resource_id", "account_pk", "group_pk", "user_pk"), + ForeignKeyConstraint( + ["resource_id", "account_pk", "group_pk"], + [ + "ov_accounts_groups.resource_id", + "ov_accounts_groups.account_pk", + "ov_accounts_groups.group_pk", + ], + ondelete="CASCADE", + name="fk_ov_accounts_member_group", + ), + ForeignKeyConstraint( + ["resource_id", "account_pk", "user_pk"], + [ + "ov_accounts_users.resource_id", + "ov_accounts_users.account_pk", + "ov_accounts_users.user_pk", + ], + ondelete="CASCADE", + name="fk_ov_accounts_member_user", + ), + Index( + "ix_ov_accounts_member_user", + "resource_id", + "account_pk", + "user_pk", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + account_pk: Mapped[int] = mapped_column(BigInteger) + group_pk: Mapped[int] = mapped_column(BigInteger) + user_pk: Mapped[int] = mapped_column(BigInteger) + + +class Credential(Base): + __tablename__ = "ov_accounts_credentials" + __table_args__ = ( + ForeignKeyConstraint( + ["resource_id", "user_pk"], + ["ov_accounts_users.resource_id", "ov_accounts_users.user_pk"], + ondelete="CASCADE", + name="fk_ov_accounts_credential_user", + ), + UniqueConstraint( + "resource_id", + "active_user_pk", + name="uq_ov_accounts_active_credential", + ), + Index( + "ix_ov_accounts_credential_lookup", + "resource_id", + "lookup_digest", + "revoked", + "user_pk", + ), + Index( + "ix_ov_accounts_credential_history", + "resource_id", + "user_pk", + "revoked", + "sequence", + ), + {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, + ) + + credential_id: Mapped[str] = mapped_column(String(32, collation="ascii_bin"), primary_key=True) + resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + user_pk: Mapped[int] = mapped_column(BigInteger) + material: Mapped[str] = mapped_column(Text(collation="ascii_bin")) + lookup_digest: Mapped[str] = mapped_column(String(64, collation="ascii_bin")) + prefix: Mapped[str] = mapped_column(String(8, collation="ascii_bin")) + revoked: Mapped[bool] = mapped_column(Boolean, server_default=text("0")) + version: Mapped[int] = mapped_column(BigInteger, server_default=text("1")) + sequence: Mapped[int] = mapped_column(BigInteger) + created_at: Mapped[object] = mapped_column( + DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") + ) + revoked_at: Mapped[Optional[object]] = mapped_column(DATETIME(fsp=6), nullable=True) + active_user_pk: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True) diff --git a/openviking/server/account_stores/redis_mysql.py b/openviking/server/account_stores/redis_mysql.py index 3fe633b542..ec1380a1b8 100644 --- a/openviking/server/account_stores/redis_mysql.py +++ b/openviking/server/account_stores/redis_mysql.py @@ -231,6 +231,18 @@ def _binding_key(self, account_id: str, api_key: str) -> str: digest_input = self._resource_id + "\0" + api_key return self._key(account_id, f"binding:{_digest(digest_input)}") + def _negative_binding_key(self, account_id: str, api_key: str, user_id_hint: str | None) -> str: + digest_input = self._resource_id + "\0" + api_key + "\0" + (user_id_hint or "") + return self._key(account_id, f"binding-miss:{_digest(digest_input)}") + + def _binding_cache_keys( + self, account_id: str, api_key: str, user_id: str + ) -> tuple[str, str]: + return ( + self._binding_key(account_id, api_key), + self._negative_binding_key(account_id, api_key, user_id), + ) + async def _delete(self, *keys: str) -> None: assert self._redis is not None if keys: @@ -476,37 +488,49 @@ def _user_cache_keys( return keys async def _cached_binding( - self, account_id: str, api_key: str + self, + account_id: str, + api_key: str, + user_id_hint: str | None, ) -> tuple[bool, tuple[str, str] | None]: binding = _decode(await self._redis.get(self._binding_key(account_id, api_key))) hit, value = self._read_nullable(binding) - if not hit: - return False, None - if value is None: + if hit and value is None: return True, None - if not isinstance(value, dict): - return False, None - try: - account_id = str(value["account_id"]) - user_id = str(value["user_id"]) - account_fence = str(value["account_fence"]) - user_fence = str(value["user_fence"]) - except (KeyError, TypeError, ValueError): - return False, None - current_account_fence = _decode( - await self._redis.get(self._account_credential_fence_key(account_id)) - ) - current_user_fence = _decode( - await self._redis.get(self._user_credential_fence_key(account_id, user_id)) + if isinstance(value, dict): + try: + binding_account_id = str(value["account_id"]) + user_id = str(value["user_id"]) + account_fence = str(value["account_fence"]) + user_fence = str(value["user_fence"]) + except (KeyError, TypeError, ValueError): + pass + else: + if user_id_hint is None or user_id == user_id_hint: + current_account_fence = _decode( + await self._redis.get( + self._account_credential_fence_key(binding_account_id) + ) + ) + current_user_fence = _decode( + await self._redis.get( + self._user_credential_fence_key(binding_account_id, user_id) + ) + ) + if ( + current_account_fence is not None + and current_user_fence is not None + and current_account_fence.get("value") == account_fence + and current_user_fence.get("value") == user_fence + ): + return True, (binding_account_id, user_id) + negative = _decode( + await self._redis.get( + self._negative_binding_key(account_id, api_key, user_id_hint) + ) ) - if ( - current_account_fence is None - or current_user_fence is None - or current_account_fence.get("value") != account_fence - or current_user_fence.get("value") != user_fence - ): - return False, None - return True, (account_id, user_id) + negative_hit, negative_value = self._read_nullable(negative) + return (True, None) if negative_hit and negative_value is None else (False, None) async def _cache_binding( self, @@ -514,12 +538,13 @@ async def _cache_binding( token: str, api_key: str, binding: tuple[str, str] | None, + user_id_hint: str | None, ) -> None: if binding is None: await self._set_if_locked( account_id, token, - self._binding_key(account_id, api_key), + self._negative_binding_key(account_id, api_key, user_id_hint), self._nullable_payload(None), index=False, ) @@ -577,14 +602,22 @@ async def verify_api_key( api_key, account_id_hint=None, user_id_hint=user_id_hint ) try: - hit, binding = await self._cached_binding(account_id_hint, api_key) + hit, binding = await self._cached_binding( + account_id_hint, + api_key, + user_id_hint, + ) if hit: return binding except Exception: logger.exception("Redis API-key cache read failed; falling back to MySQL") try: async with self._account_lock(account_id_hint) as token: - hit, binding = await self._cached_binding(account_id_hint, api_key) + hit, binding = await self._cached_binding( + account_id_hint, + api_key, + user_id_hint, + ) if hit: return binding binding = await self._mysql.verify_api_key( @@ -598,6 +631,7 @@ async def verify_api_key( token, api_key, binding, + user_id_hint, ) except Exception: logger.exception("Failed to populate Redis API-key cache") @@ -662,7 +696,7 @@ async def create_account_with_api_key( admin_user_id, api_key, ), - lambda: (self._binding_key(account_id, api_key),), + lambda: self._binding_cache_keys(account_id, api_key, admin_user_id), account_wide=True, ) @@ -704,7 +738,7 @@ async def create_user_with_api_key( user_id, credentials=True, ) - + (self._binding_key(account_id, api_key),), + + self._binding_cache_keys(account_id, api_key, user_id), ) async def list_users_page(self, account_id: str, **kwargs) -> UsersPage: @@ -812,8 +846,8 @@ async def replace_active_user_api_key( lambda: self._mysql.replace_active_user_api_key(account_id, user_id, api_key), lambda: ( self._user_credential_fence_key(account_id, user_id), - self._binding_key(account_id, api_key), - ), + ) + + self._binding_cache_keys(account_id, api_key, user_id), ) async def revoke_active_api_keys( diff --git a/openviking/server/account_stores/sql/init.sql b/openviking/server/account_stores/sql/init.sql index 663911f304..f179ebe0b4 100644 --- a/openviking/server/account_stores/sql/init.sql +++ b/openviking/server/account_stores/sql/init.sql @@ -1,8 +1,6 @@ -- OpenViking MySQL account store initial schema. --- Table prefix: ov_accounts (the default account_store.params.table value). -- Execute manually before starting a MySQL account-store deployment. --- For a custom table prefix, replace only backtick-quoted ov_accounts table --- identifiers. Constraint and index names are stable and table-name independent. +-- Table names are fixed and resource_id isolates OpenViking deployments. CREATE TABLE `ov_accounts` ( account_pk BIGINT NOT NULL AUTO_INCREMENT, @@ -13,9 +11,9 @@ CREATE TABLE `ov_accounts` ( deletion_owner_account_id VARCHAR(255) COLLATE utf8mb4_bin NULL, deletion_owner_user_id VARCHAR(255) COLLATE utf8mb4_bin NULL, PRIMARY KEY (account_pk), - CONSTRAINT uq_ov_as_account_identity UNIQUE (resource_id, account_id), - CONSTRAINT uq_ov_as_account_resource_pk UNIQUE (resource_id, account_pk), - INDEX ix_ov_as_account_deletion (resource_id, deletion_task_id) + CONSTRAINT uq_ov_accounts_account_identity UNIQUE (resource_id, account_id), + CONSTRAINT uq_ov_accounts_account_resource_pk UNIQUE (resource_id, account_pk), + INDEX ix_ov_accounts_account_deletion (resource_id, deletion_task_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_users` ( @@ -29,15 +27,17 @@ CREATE TABLE `ov_accounts_users` ( deletion_owner_account_id VARCHAR(255) COLLATE utf8mb4_bin NULL, deletion_owner_user_id VARCHAR(255) COLLATE utf8mb4_bin NULL, PRIMARY KEY (user_pk), - CONSTRAINT fk_ov_as_user_account + CONSTRAINT fk_ov_accounts_user_account FOREIGN KEY (resource_id, account_pk) REFERENCES `ov_accounts` (resource_id, account_pk) ON DELETE CASCADE, - CONSTRAINT uq_ov_as_user_identity + CONSTRAINT uq_ov_accounts_user_identity UNIQUE (resource_id, account_pk, user_id), - CONSTRAINT uq_ov_as_user_resource_pk UNIQUE (resource_id, user_pk), - INDEX ix_ov_as_user_admin (resource_id, account_pk, `role`), - INDEX ix_ov_as_user_deletion (resource_id, deletion_task_id) + CONSTRAINT uq_ov_accounts_user_resource_pk UNIQUE (resource_id, user_pk), + CONSTRAINT uq_ov_accounts_user_account_pk + UNIQUE (resource_id, account_pk, user_pk), + INDEX ix_ov_accounts_user_admin (resource_id, account_pk, `role`), + INDEX ix_ov_accounts_user_deletion (resource_id, deletion_task_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_groups` ( @@ -47,29 +47,32 @@ CREATE TABLE `ov_accounts_groups` ( group_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), PRIMARY KEY (group_pk), - CONSTRAINT fk_ov_as_group_account + CONSTRAINT fk_ov_accounts_group_account FOREIGN KEY (resource_id, account_pk) REFERENCES `ov_accounts` (resource_id, account_pk) ON DELETE CASCADE, - CONSTRAINT uq_ov_as_group_identity + CONSTRAINT uq_ov_accounts_group_identity UNIQUE (resource_id, account_pk, group_id), - CONSTRAINT uq_ov_as_group_resource_pk UNIQUE (resource_id, group_pk) + CONSTRAINT uq_ov_accounts_group_resource_pk UNIQUE (resource_id, group_pk), + CONSTRAINT uq_ov_accounts_group_account_pk + UNIQUE (resource_id, account_pk, group_pk) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_group_members` ( resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + account_pk BIGINT NOT NULL, group_pk BIGINT NOT NULL, user_pk BIGINT NOT NULL, - PRIMARY KEY (resource_id, group_pk, user_pk), - CONSTRAINT fk_ov_as_member_group - FOREIGN KEY (resource_id, group_pk) - REFERENCES `ov_accounts_groups` (resource_id, group_pk) + PRIMARY KEY (resource_id, account_pk, group_pk, user_pk), + CONSTRAINT fk_ov_accounts_member_group + FOREIGN KEY (resource_id, account_pk, group_pk) + REFERENCES `ov_accounts_groups` (resource_id, account_pk, group_pk) ON DELETE CASCADE, - CONSTRAINT fk_ov_as_member_user - FOREIGN KEY (resource_id, user_pk) - REFERENCES `ov_accounts_users` (resource_id, user_pk) + CONSTRAINT fk_ov_accounts_member_user + FOREIGN KEY (resource_id, account_pk, user_pk) + REFERENCES `ov_accounts_users` (resource_id, account_pk, user_pk) ON DELETE CASCADE, - INDEX ix_ov_as_member_user (resource_id, user_pk) + INDEX ix_ov_accounts_member_user (resource_id, account_pk, user_pk) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_credentials` ( @@ -86,14 +89,14 @@ CREATE TABLE `ov_accounts_credentials` ( revoked_at DATETIME(6) NULL, active_user_pk BIGINT NULL, PRIMARY KEY (credential_id), - CONSTRAINT fk_ov_as_credential_user + CONSTRAINT fk_ov_accounts_credential_user FOREIGN KEY (resource_id, user_pk) REFERENCES `ov_accounts_users` (resource_id, user_pk) ON DELETE CASCADE, - CONSTRAINT uq_ov_as_active_credential + CONSTRAINT uq_ov_accounts_active_credential UNIQUE (resource_id, active_user_pk), - INDEX ix_ov_as_credential_lookup + INDEX ix_ov_accounts_credential_lookup (resource_id, lookup_digest, revoked, user_pk), - INDEX ix_ov_as_credential_history + INDEX ix_ov_accounts_credential_history (resource_id, user_pk, revoked, sequence) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/openviking/server/config.py b/openviking/server/config.py index 4f07f587a9..7759acd0fd 100644 --- a/openviking/server/config.py +++ b/openviking/server/config.py @@ -313,8 +313,11 @@ class AccountStoreConfig(BaseModel): @classmethod def _validate_provider(cls, value: str) -> str: normalized = value.strip().lower() - if not normalized: - raise ValueError("account_store.provider must not be empty") + supported = {"file", "mysql", "redis_mysql"} + if normalized not in supported: + raise ValueError( + "account_store.provider must be one of: " + ", ".join(sorted(supported)) + ) return normalized diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index 6dfa6965f2..8b2a4d1497 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -10,11 +10,14 @@ from uuid import uuid4 import pytest +from sqlalchemy import delete +from sqlalchemy.dialects import mysql from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import create_async_engine from openviking.server.account_stores import mysql as mysql_store_module -from openviking.server.account_stores.mysql import MySQLAccountStore +from openviking.server.account_stores.mysql import MySQLAccountStore, _substring_pattern +from openviking.server.account_stores.mysql_schema import Account, Base, Group, GroupMember from openviking.server.api_keys import APIKeyManager from openviking.server.api_keys.legacy import FileStore from openviking.server.identity import Role @@ -241,6 +244,45 @@ async def test_file_store_allows_only_one_active_credential_per_user(): assert active == ("acme", "alice") +async def test_file_hashing_never_persists_plaintext_credentials(): + agfs = InMemoryAGFS() + manager = build_api_key_manager( + "root", + SimpleNamespace(agfs=agfs), + api_key_hashing_enabled=True, + ) + await manager.load() + account_key = await manager.create_account("acme", "alice") + user_key = await manager.register_user("acme", "bob") + rotated_key = await manager.regenerate_key("acme", "bob") + + users = json.loads(agfs._files["/local/acme/_system/users.json"])["users"] + assert users["alice"]["key"].startswith("$argon2") + assert users["bob"]["key"].startswith("$argon2") + assert account_key.encode() not in agfs._files["/local/acme/_system/users.json"] + assert user_key.encode() not in agfs._files["/local/acme/_system/users.json"] + assert rotated_key.encode() not in agfs._files["/local/acme/_system/users.json"] + + +async def test_file_load_migrates_plaintext_credential_to_argon2(): + agfs = InMemoryAGFS() + plaintext = build_api_key_manager("root", SimpleNamespace(agfs=agfs)) + await plaintext.load() + key = await plaintext.create_account("acme", "alice") + + hashed = build_api_key_manager( + "root", + SimpleNamespace(agfs=agfs), + api_key_hashing_enabled=True, + ) + await hashed.load() + + users = json.loads(agfs._files["/local/acme/_system/users.json"])["users"] + assert users["alice"]["key"].startswith("$argon2") + assert key.encode() not in agfs._files["/local/acme/_system/users.json"] + assert (await hashed.resolve_identity(key)).user_id == "alice" + + async def test_file_management_user_read_refreshes_only_explicitly(): fs = SimpleNamespace(agfs=InMemoryAGFS()) writer = build_api_key_manager("root", fs) @@ -313,6 +355,76 @@ def test_mysql_requires_resource_id(monkeypatch): MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) +def test_mysql_substring_filter_escapes_sql_wildcards(): + assert _substring_pattern("alice") == "%alice%" + assert _substring_pattern("a_b%=c") == "%a=_b=%==c%" + + +def test_mysql_schema_uses_fixed_tables_and_scoped_memberships(): + assert set(Base.metadata.tables) == { + "ov_accounts", + "ov_accounts_users", + "ov_accounts_groups", + "ov_accounts_group_members", + "ov_accounts_credentials", + } + membership = GroupMember.__table__ + assert "account_pk" in membership.c + assert all( + {"resource_id", "account_pk"}.issubset( + {element.parent.name for element in constraint.elements} + ) + for constraint in membership.foreign_key_constraints + ) + + +def test_mysql_rejects_removed_table_parameter(monkeypatch): + monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") + with pytest.raises(InvalidArgumentError, match="Unknown MySQL account store parameters: table"): + MySQLAccountStore( + params={ + "user": "unused", + "password": "unused", + "database": "unused", + "table": "tenant_accounts", + } + ) + + +async def test_mysql_verification_excludes_deleting_identities(monkeypatch): + monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") + store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) + statements = [] + + class Result: + def all(self): + return [] + + class Session: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def execute(self, statement): + statements.append( + str( + statement.compile( + dialect=mysql.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ) + ) + return Result() + + store._sessions = lambda: Session() + + assert await store.verify_api_key("key", account_id_hint="acme") is None + statement = statements[0] + assert statement.count("deletion_task_id IS NULL") == 2 + + async def test_mysql_load_rejects_incomplete_schema_and_disposes_engine(monkeypatch): monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) @@ -334,7 +446,7 @@ async def __aexit__(self, *args): async def execute(self, statement): table = statement.get_final_froms()[0].name - if table == store._tables["groups"]: + if table == Group.__tablename__: raise OperationalError( str(statement), {}, @@ -374,7 +486,6 @@ async def test_mysql_managers_share_authoritative_state(monkeypatch): encryption_key = base64.urlsafe_b64encode(b"k" * 32).decode() params = { **json.loads(raw), - "table": f"ov_test_{uuid4().hex}", "credential_encryption_key": encryption_key, } first = build_api_key_manager( @@ -391,7 +502,10 @@ async def test_mysql_managers_share_authoritative_state(monkeypatch): ) setup_engine = create_async_engine(first._store._url, **first._store._engine_options) async with setup_engine.begin() as connection: - await connection.run_sync(first._store._models.Base.metadata.create_all) + await connection.run_sync(Base.metadata.create_all) + await connection.execute( + delete(Account).where(Account.resource_id == first._store._resource_id) + ) await setup_engine.dispose() await first.load() await second.load() @@ -443,6 +557,8 @@ async def test_mysql_managers_share_authoritative_state(monkeypatch): assert await second.get_user_group_ids("acme", "alice") == ("engineering",) finally: async with first._store._engine.begin() as connection: - await connection.run_sync(first._store._models.Base.metadata.drop_all) + await connection.execute( + delete(Account).where(Account.resource_id == first._store._resource_id) + ) await second.close() await first.close() diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index dfe92bb2b7..21d84995c2 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -1286,13 +1286,18 @@ async def test_trusted_mode_with_root_api_key_accepts_matching_api_key(): assert identity.user_id == "alice" -async def test_trusted_mode_root_key_allows_admin_role_header(auth_app): +async def test_trusted_mode_root_key_allows_admin_role_header(auth_app, monkeypatch): """Trusted upstreams with the root key may assert ADMIN role for tenant routes.""" manager = auth_app.state.api_key_manager account_id = _uid() await manager.create_account(account_id, "owner") await manager.register_user(account_id, "alice", "user") + async def fail_role_lookup(*_args): + raise AssertionError("asserted roles must not read the account store") + + monkeypatch.setattr(manager, "get_registered_user_role", fail_role_lookup) + request = _make_request( "/api/v1/resources", headers={ diff --git a/tests/server/test_redis_mysql_account_store.py b/tests/server/test_redis_mysql_account_store.py index 9472ddf30d..d1cc0c15bd 100644 --- a/tests/server/test_redis_mysql_account_store.py +++ b/tests/server/test_redis_mysql_account_store.py @@ -109,11 +109,23 @@ async def load(self): async def close(self): return None - async def verify_api_key(self, api_key: str, **_kwargs): + async def verify_api_key( + self, + api_key: str, + *, + account_id_hint: str | None = None, + user_id_hint: str | None = None, + ): self.calls["verify"] += 1 return ( ("acme", "alice") - if self.account_exists and self.user_exists and api_key == self.api_key + if ( + self.account_exists + and self.user_exists + and api_key == self.api_key + and account_id_hint in (None, "acme") + and user_id_hint in (None, "alice") + ) else None ) @@ -264,6 +276,32 @@ async def test_verify_api_key_caches_public_binding_read(): assert mysql.calls["verify"] == 1 +async def test_cached_binding_respects_user_id_hint(): + api_key = _api_key() + store, mysql = _store(api_key) + await store.load() + + assert await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="alice", + ) == ("acme", "alice") + assert ( + await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="bob", + ) + is None + ) + assert await store.verify_api_key( + api_key, + account_id_hint="acme", + user_id_hint="alice", + ) == ("acme", "alice") + assert mysql.calls["verify"] == 2 + + async def test_concurrent_user_cache_misses_query_mysql_once(): store, mysql = _store(_api_key()) await store.load() @@ -334,6 +372,38 @@ async def test_write_lock_timeout_revokes_stale_lease_before_invalidation(): ) +async def test_invalidation_failure_does_not_block_mysql_mutation(): + store, mysql = _store(_api_key()) + await store.load() + assert await store.get_user("acme", "alice") is not None + + async def fail_invalidation(*_args): + raise ConnectionError("redis unavailable") + + store._delete_and_unindex = fail_invalidation + + await store.set_role("acme", "alice", Role.USER) + + assert mysql.user["role"] == Role.USER + assert store._user_key("acme", "alice") in store._redis.values + + +async def test_redis_outage_does_not_block_mysql_mutation(): + store, mysql = _store(_api_key()) + await store.load() + + async def unavailable(*_args, **_kwargs): + raise ConnectionError("redis unavailable") + + store._redis.set = unavailable + store._redis.delete = unavailable + store._redis.eval = unavailable + + await store.set_role("acme", "alice", Role.USER) + + assert mysql.user["role"] == Role.USER + + async def test_api_key_manager_uses_cached_public_store_reads(): api_key = _api_key() store, mysql = _store(api_key) @@ -361,6 +431,8 @@ async def test_replacing_key_invalidates_cached_binding_and_prior_negative(): await store.load() assert await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") assert await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint="alice") is None + assert await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint="alice") is None + assert mysql.calls["verify"] == 2 assert store._binding_key("acme", new_key) not in await store._redis.zrange( store._cache_keys_index_key("acme"), 0, From da17bfaa49251ff50d3fca5bbca97964f5fd2fd4 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Wed, 30 Sep 2026 19:25:05 +0800 Subject: [PATCH 03/16] fix(mysql): require compatible SQLAlchemy version --- pyproject.toml | 4 +- uv.lock | 101 +++++++++++++++++++++++-------------------------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 15d86fc803..37ad2c6a1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,12 +99,12 @@ gateway-fast = [ ] mysql = [ "aiomysql>=0.2.0,<1", - "SQLAlchemy[asyncio]>=2.0,<2.1", + "SQLAlchemy[asyncio]>=2.0.54,<2.1", ] redis-mysql = [ "aiomysql>=0.2.0,<1", "redis>=5.0,<7", - "SQLAlchemy[asyncio]>=2.0,<2.1", + "SQLAlchemy[asyncio]>=2.0.54,<2.1", ] test = [ "SQLAlchemy[asyncio]>=2.0,<2.1", diff --git a/uv.lock b/uv.lock index 9419799594..0525c334c7 100644 --- a/uv.lock +++ b/uv.lock @@ -3772,8 +3772,8 @@ requires-dist = [ { name = "socksio", marker = "extra == 'bot'", specifier = ">=1.0.0" }, { name = "sphinx", marker = "extra == 'doc'", specifier = ">=7.0.0" }, { name = "sphinx-rtd-theme", marker = "extra == 'doc'", specifier = ">=1.3.0" }, - { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'mysql'", specifier = ">=2.0,<2.1" }, - { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'redis-mysql'", specifier = ">=2.0,<2.1" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'mysql'", specifier = ">=2.0.54,<2.1" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'redis-mysql'", specifier = ">=2.0.54,<2.1" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'test'", specifier = ">=2.0,<2.1" }, { name = "tabulate", specifier = ">=0.9.0" }, { name = "tavily-python", marker = "extra == 'bot'", specifier = ">=0.5.0" }, @@ -6225,62 +6225,57 @@ wheels = [ [[package]] name = "sqlalchemy" -version = "2.0.48" +version = "2.0.54" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/73/b4a9737255583b5fa858e0bb8e116eb94b88c910164ed2ed719147bde3de/sqlalchemy-2.0.48.tar.gz", hash = "sha256:5ca74f37f3369b45e1f6b7b06afb182af1fd5dde009e4ffd831830d98cbe5fe7", size = 9886075, upload-time = "2026-03-02T15:28:51.474Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/67/1235676e93dd3b742a4a8eddfae49eea46c85e3eed29f0da446a8dd57500/sqlalchemy-2.0.48-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7001dc9d5f6bb4deb756d5928eaefe1930f6f4179da3924cbd95ee0e9f4dce89", size = 2157384, upload-time = "2026-03-02T15:38:26.781Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d7/fa728b856daa18c10e1390e76f26f64ac890c947008284387451d56ca3d0/sqlalchemy-2.0.48-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a89ce07ad2d4b8cfc30bd5889ec40613e028ed80ef47da7d9dd2ce969ad30e0", size = 3236981, upload-time = "2026-03-02T15:58:53.53Z" }, - { url = "https://files.pythonhosted.org/packages/5c/ad/6c4395649a212a6c603a72c5b9ab5dce3135a1546cfdffa3c427e71fd535/sqlalchemy-2.0.48-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10853a53a4a00417a00913d270dddda75815fcb80675874285f41051c094d7dd", size = 3235232, upload-time = "2026-03-02T15:52:25.654Z" }, - { url = "https://files.pythonhosted.org/packages/01/f4/58f845e511ac0509765a6f85eb24924c1ef0d54fb50de9d15b28c3601458/sqlalchemy-2.0.48-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fac0fa4e4f55f118fd87177dacb1c6522fe39c28d498d259014020fec9164c29", size = 3188106, upload-time = "2026-03-02T15:58:55.193Z" }, - { url = "https://files.pythonhosted.org/packages/3f/f9/6dcc7bfa5f5794c3a095e78cd1de8269dfb5584dfd4c2c00a50d3c1ade44/sqlalchemy-2.0.48-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3713e21ea67bca727eecd4a24bf68bcd414c403faae4989442be60994301ded0", size = 3209522, upload-time = "2026-03-02T15:52:27.407Z" }, - { url = "https://files.pythonhosted.org/packages/d7/5a/b632875ab35874d42657f079529f0745410604645c269a8c21fb4272ff7a/sqlalchemy-2.0.48-cp310-cp310-win32.whl", hash = "sha256:d404dc897ce10e565d647795861762aa2d06ca3f4a728c5e9a835096c7059018", size = 2117695, upload-time = "2026-03-02T15:46:51.389Z" }, - { url = "https://files.pythonhosted.org/packages/de/03/9752eb2a41afdd8568e41ac3c3128e32a0a73eada5ab80483083604a56d1/sqlalchemy-2.0.48-cp310-cp310-win_amd64.whl", hash = "sha256:841a94c66577661c1f088ac958cd767d7c9bf507698f45afffe7a4017049de76", size = 2140928, upload-time = "2026-03-02T15:46:52.992Z" }, - { url = "https://files.pythonhosted.org/packages/d7/6d/b8b78b5b80f3c3ab3f7fa90faa195ec3401f6d884b60221260fd4d51864c/sqlalchemy-2.0.48-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b4c575df7368b3b13e0cebf01d4679f9a28ed2ae6c1cd0b1d5beffb6b2007dc", size = 2157184, upload-time = "2026-03-02T15:38:28.161Z" }, - { url = "https://files.pythonhosted.org/packages/21/4b/4f3d4a43743ab58b95b9ddf5580a265b593d017693df9e08bd55780af5bb/sqlalchemy-2.0.48-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e83e3f959aaa1c9df95c22c528096d94848a1bc819f5d0ebf7ee3df0ca63db6c", size = 3313555, upload-time = "2026-03-02T15:58:57.21Z" }, - { url = "https://files.pythonhosted.org/packages/21/dd/3b7c53f1dbbf736fd27041aee68f8ac52226b610f914085b1652c2323442/sqlalchemy-2.0.48-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f7b7243850edd0b8b97043f04748f31de50cf426e939def5c16bedb540698f7", size = 3313057, upload-time = "2026-03-02T15:52:29.366Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cc/3e600a90ae64047f33313d7d32e5ad025417f09d2ded487e8284b5e21a15/sqlalchemy-2.0.48-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:82745b03b4043e04600a6b665cb98697c4339b24e34d74b0a2ac0a2488b6f94d", size = 3265431, upload-time = "2026-03-02T15:58:59.096Z" }, - { url = "https://files.pythonhosted.org/packages/8b/19/780138dacfe3f5024f4cf96e4005e91edf6653d53d3673be4844578faf1d/sqlalchemy-2.0.48-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5e088bf43f6ee6fec7dbf1ef7ff7774a616c236b5c0cb3e00662dd71a56b571", size = 3287646, upload-time = "2026-03-02T15:52:31.569Z" }, - { url = "https://files.pythonhosted.org/packages/40/fd/f32ced124f01a23151f4777e4c705f3a470adc7bd241d9f36a7c941a33bf/sqlalchemy-2.0.48-cp311-cp311-win32.whl", hash = "sha256:9c7d0a77e36b5f4b01ca398482230ab792061d243d715299b44a0b55c89fe617", size = 2116956, upload-time = "2026-03-02T15:46:54.535Z" }, - { url = "https://files.pythonhosted.org/packages/58/d5/dd767277f6feef12d05651538f280277e661698f617fa4d086cce6055416/sqlalchemy-2.0.48-cp311-cp311-win_amd64.whl", hash = "sha256:583849c743e0e3c9bb7446f5b5addeacedc168d657a69b418063dfdb2d90081c", size = 2141627, upload-time = "2026-03-02T15:46:55.849Z" }, - { url = "https://files.pythonhosted.org/packages/ef/91/a42ae716f8925e9659df2da21ba941f158686856107a61cc97a95e7647a3/sqlalchemy-2.0.48-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:348174f228b99f33ca1f773e85510e08927620caa59ffe7803b37170df30332b", size = 2155737, upload-time = "2026-03-02T15:49:13.207Z" }, - { url = "https://files.pythonhosted.org/packages/b9/52/f75f516a1f3888f027c1cfb5d22d4376f4b46236f2e8669dcb0cddc60275/sqlalchemy-2.0.48-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53667b5f668991e279d21f94ccfa6e45b4e3f4500e7591ae59a8012d0f010dcb", size = 3337020, upload-time = "2026-03-02T15:50:34.547Z" }, - { url = "https://files.pythonhosted.org/packages/37/9a/0c28b6371e0cdcb14f8f1930778cb3123acfcbd2c95bb9cf6b4a2ba0cce3/sqlalchemy-2.0.48-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34634e196f620c7a61d18d5cf7dc841ca6daa7961aed75d532b7e58b309ac894", size = 3349983, upload-time = "2026-03-02T15:53:25.542Z" }, - { url = "https://files.pythonhosted.org/packages/1c/46/0aee8f3ff20b1dcbceb46ca2d87fcc3d48b407925a383ff668218509d132/sqlalchemy-2.0.48-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:546572a1793cc35857a2ffa1fe0e58571af1779bcc1ffa7c9fb0839885ed69a9", size = 3279690, upload-time = "2026-03-02T15:50:36.277Z" }, - { url = "https://files.pythonhosted.org/packages/ce/8c/a957bc91293b49181350bfd55e6dfc6e30b7f7d83dc6792d72043274a390/sqlalchemy-2.0.48-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:07edba08061bc277bfdc772dd2a1a43978f5a45994dd3ede26391b405c15221e", size = 3314738, upload-time = "2026-03-02T15:53:27.519Z" }, - { url = "https://files.pythonhosted.org/packages/4b/44/1d257d9f9556661e7bdc83667cc414ba210acfc110c82938cb3611eea58f/sqlalchemy-2.0.48-cp312-cp312-win32.whl", hash = "sha256:908a3fa6908716f803b86896a09a2c4dde5f5ce2bb07aacc71ffebb57986ce99", size = 2115546, upload-time = "2026-03-02T15:54:31.591Z" }, - { url = "https://files.pythonhosted.org/packages/f2/af/c3c7e1f3a2b383155a16454df62ae8c62a30dd238e42e68c24cebebbfae6/sqlalchemy-2.0.48-cp312-cp312-win_amd64.whl", hash = "sha256:68549c403f79a8e25984376480959975212a670405e3913830614432b5daa07a", size = 2142484, upload-time = "2026-03-02T15:54:34.072Z" }, - { url = "https://files.pythonhosted.org/packages/d1/c6/569dc8bf3cd375abc5907e82235923e986799f301cd79a903f784b996fca/sqlalchemy-2.0.48-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e3070c03701037aa418b55d36532ecb8f8446ed0135acb71c678dbdf12f5b6e4", size = 2152599, upload-time = "2026-03-02T15:49:14.41Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ff/f4e04a4bd5a24304f38cb0d4aa2ad4c0fb34999f8b884c656535e1b2b74c/sqlalchemy-2.0.48-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2645b7d8a738763b664a12a1542c89c940daa55196e8d73e55b169cc5c99f65f", size = 3278825, upload-time = "2026-03-02T15:50:38.269Z" }, - { url = "https://files.pythonhosted.org/packages/fe/88/cb59509e4668d8001818d7355d9995be90c321313078c912420603a7cb95/sqlalchemy-2.0.48-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b19151e76620a412c2ac1c6f977ab1b9fa7ad43140178345136456d5265b32ed", size = 3295200, upload-time = "2026-03-02T15:53:29.366Z" }, - { url = "https://files.pythonhosted.org/packages/87/dc/1609a4442aefd750ea2f32629559394ec92e89ac1d621a7f462b70f736ff/sqlalchemy-2.0.48-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5b193a7e29fd9fa56e502920dca47dffe60f97c863494946bd698c6058a55658", size = 3226876, upload-time = "2026-03-02T15:50:39.802Z" }, - { url = "https://files.pythonhosted.org/packages/37/c3/6ae2ab5ea2fa989fbac4e674de01224b7a9d744becaf59bb967d62e99bed/sqlalchemy-2.0.48-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:36ac4ddc3d33e852da9cb00ffb08cea62ca05c39711dc67062ca2bb1fae35fd8", size = 3265045, upload-time = "2026-03-02T15:53:31.421Z" }, - { url = "https://files.pythonhosted.org/packages/6f/82/ea4665d1bb98c50c19666e672f21b81356bd6077c4574e3d2bbb84541f53/sqlalchemy-2.0.48-cp313-cp313-win32.whl", hash = "sha256:389b984139278f97757ea9b08993e7b9d1142912e046ab7d82b3fbaeb0209131", size = 2113700, upload-time = "2026-03-02T15:54:35.825Z" }, - { url = "https://files.pythonhosted.org/packages/b7/2b/b9040bec58c58225f073f5b0c1870defe1940835549dafec680cbd58c3c3/sqlalchemy-2.0.48-cp313-cp313-win_amd64.whl", hash = "sha256:d612c976cbc2d17edfcc4c006874b764e85e990c29ce9bd411f926bbfb02b9a2", size = 2139487, upload-time = "2026-03-02T15:54:37.079Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f4/7b17bd50244b78a49d22cc63c969d71dc4de54567dc152a9b46f6fae40ce/sqlalchemy-2.0.48-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69f5bc24904d3bc3640961cddd2523e361257ef68585d6e364166dfbe8c78fae", size = 3558851, upload-time = "2026-03-02T15:57:48.607Z" }, - { url = "https://files.pythonhosted.org/packages/20/0d/213668e9aca61d370f7d2a6449ea4ec699747fac67d4bda1bb3d129025be/sqlalchemy-2.0.48-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd08b90d211c086181caed76931ecfa2bdfc83eea3cfccdb0f82abc6c4b876cb", size = 3525525, upload-time = "2026-03-02T16:04:38.058Z" }, - { url = "https://files.pythonhosted.org/packages/85/d7/a84edf412979e7d59c69b89a5871f90a49228360594680e667cb2c46a828/sqlalchemy-2.0.48-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1ccd42229aaac2df431562117ac7e667d702e8e44afdb6cf0e50fa3f18160f0b", size = 3466611, upload-time = "2026-03-02T15:57:50.759Z" }, - { url = "https://files.pythonhosted.org/packages/86/55/42404ce5770f6be26a2b0607e7866c31b9a4176c819e9a7a5e0a055770be/sqlalchemy-2.0.48-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f0dcbc588cd5b725162c076eb9119342f6579c7f7f55057bb7e3c6ff27e13121", size = 3475812, upload-time = "2026-03-02T16:04:40.092Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ae/29b87775fadc43e627cf582fe3bda4d02e300f6b8f2747c764950d13784c/sqlalchemy-2.0.48-cp313-cp313t-win32.whl", hash = "sha256:9764014ef5e58aab76220c5664abb5d47d5bc858d9debf821e55cfdd0f128485", size = 2141335, upload-time = "2026-03-02T15:52:51.518Z" }, - { url = "https://files.pythonhosted.org/packages/91/44/f39d063c90f2443e5b46ec4819abd3d8de653893aae92df42a5c4f5843de/sqlalchemy-2.0.48-cp313-cp313t-win_amd64.whl", hash = "sha256:e2f35b4cccd9ed286ad62e0a3c3ac21e06c02abc60e20aa51a3e305a30f5fa79", size = 2173095, upload-time = "2026-03-02T15:52:52.79Z" }, - { url = "https://files.pythonhosted.org/packages/f7/b3/f437eaa1cf028bb3c927172c7272366393e73ccd104dcf5b6963f4ab5318/sqlalchemy-2.0.48-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e2d0d88686e3d35a76f3e15a34e8c12d73fc94c1dea1cd55782e695cc14086dd", size = 2154401, upload-time = "2026-03-02T15:49:17.24Z" }, - { url = "https://files.pythonhosted.org/packages/6c/1c/b3abdf0f402aa3f60f0df6ea53d92a162b458fca2321d8f1f00278506402/sqlalchemy-2.0.48-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49b7bddc1eebf011ea5ab722fdbe67a401caa34a350d278cc7733c0e88fecb1f", size = 3274528, upload-time = "2026-03-02T15:50:41.489Z" }, - { url = "https://files.pythonhosted.org/packages/f2/5e/327428a034407651a048f5e624361adf3f9fbac9d0fa98e981e9c6ff2f5e/sqlalchemy-2.0.48-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:426c5ca86415d9b8945c7073597e10de9644802e2ff502b8e1f11a7a2642856b", size = 3279523, upload-time = "2026-03-02T15:53:32.962Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ca/ece73c81a918add0965b76b868b7b5359e068380b90ef1656ee995940c02/sqlalchemy-2.0.48-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:288937433bd44e3990e7da2402fabc44a3c6c25d3704da066b85b89a85474ae0", size = 3224312, upload-time = "2026-03-02T15:50:42.996Z" }, - { url = "https://files.pythonhosted.org/packages/88/11/fbaf1ae91fa4ee43f4fe79661cead6358644824419c26adb004941bdce7c/sqlalchemy-2.0.48-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8183dc57ae7d9edc1346e007e840a9f3d6aa7b7f165203a99e16f447150140d2", size = 3246304, upload-time = "2026-03-02T15:53:34.937Z" }, - { url = "https://files.pythonhosted.org/packages/fa/a8/5fb0deb13930b4f2f698c5541ae076c18981173e27dd00376dbaea7a9c82/sqlalchemy-2.0.48-cp314-cp314-win32.whl", hash = "sha256:1182437cb2d97988cfea04cf6cdc0b0bb9c74f4d56ec3d08b81e23d621a28cc6", size = 2116565, upload-time = "2026-03-02T15:54:38.321Z" }, - { url = "https://files.pythonhosted.org/packages/95/7e/e83615cb63f80047f18e61e31e8e32257d39458426c23006deeaf48f463b/sqlalchemy-2.0.48-cp314-cp314-win_amd64.whl", hash = "sha256:144921da96c08feb9e2b052c5c5c1d0d151a292c6135623c6b2c041f2a45f9e0", size = 2142205, upload-time = "2026-03-02T15:54:39.831Z" }, - { url = "https://files.pythonhosted.org/packages/83/e3/69d8711b3f2c5135e9cde5f063bc1605860f0b2c53086d40c04017eb1f77/sqlalchemy-2.0.48-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aee45fd2c6c0f2b9cdddf48c48535e7471e42d6fb81adfde801da0bd5b93241", size = 3563519, upload-time = "2026-03-02T15:57:52.387Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4f/a7cce98facca73c149ea4578981594aaa5fd841e956834931de503359336/sqlalchemy-2.0.48-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7cddca31edf8b0653090cbb54562ca027c421c58ddde2c0685f49ff56a1690e0", size = 3528611, upload-time = "2026-03-02T16:04:42.097Z" }, - { url = "https://files.pythonhosted.org/packages/cd/7d/5936c7a03a0b0cb0fa0cc425998821c6029756b0855a8f7ee70fba1de955/sqlalchemy-2.0.48-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7a936f1bb23d370b7c8cc079d5fce4c7d18da87a33c6744e51a93b0f9e97e9b3", size = 3472326, upload-time = "2026-03-02T15:57:54.423Z" }, - { url = "https://files.pythonhosted.org/packages/f4/33/cea7dfc31b52904efe3dcdc169eb4514078887dff1f5ae28a7f4c5d54b3c/sqlalchemy-2.0.48-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e004aa9248e8cb0a5f9b96d003ca7c1c0a5da8decd1066e7b53f59eb8ce7c62b", size = 3478453, upload-time = "2026-03-02T16:04:44.584Z" }, - { url = "https://files.pythonhosted.org/packages/c8/95/32107c4d13be077a9cae61e9ae49966a35dc4bf442a8852dd871db31f62e/sqlalchemy-2.0.48-cp314-cp314t-win32.whl", hash = "sha256:b8438ec5594980d405251451c5b7ea9aa58dda38eb7ac35fb7e4c696712ee24f", size = 2147209, upload-time = "2026-03-02T15:52:54.274Z" }, - { url = "https://files.pythonhosted.org/packages/d2/d7/1e073da7a4bc645eb83c76067284a0374e643bc4be57f14cc6414656f92c/sqlalchemy-2.0.48-cp314-cp314t-win_amd64.whl", hash = "sha256:d854b3970067297f3a7fbd7a4683587134aa9b3877ee15aa29eea478dc68f933", size = 2182198, upload-time = "2026-03-02T15:52:55.606Z" }, - { url = "https://files.pythonhosted.org/packages/46/2c/9664130905f03db57961b8980b05cab624afd114bf2be2576628a9f22da4/sqlalchemy-2.0.48-py3-none-any.whl", hash = "sha256:a66fe406437dd65cacd96a72689a3aaaecaebbcd62d81c5ac1c0fdbeac835096", size = 1940202, upload-time = "2026-03-02T15:52:43.285Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/29/9c/271aa905cf2964f841371a97f3e63ab692bf51b4423d0491e67bc7f64037/sqlalchemy-2.0.54.tar.gz", hash = "sha256:baa8521e8ee9f24e75dfc7aaabc08020e551ef0d48d7c3e3536f5cddf277586b", size = 9969559, upload-time = "2026-09-15T21:06:57.337Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/2f/7a427d7e10bc5f285d41fa1cadf46031ffc6c84f35a9d6e19737587a4eb0/sqlalchemy-2.0.54-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:24ae093dec196ba37fc2beb0316de53e7871d3d246a50faecbbb53034e41ded2", size = 2189267, upload-time = "2026-09-15T22:11:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/84/1f/80122de187c0f80cb47addce97609160004a8605010afdd815e608fcf003/sqlalchemy-2.0.54-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8cc6532f930c27974e9239e5ce5abebe7600ba9807cea4fcf42f1b6cab18fe7", size = 3358362, upload-time = "2026-09-15T22:33:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/c8/93/9d26b109f5472017bf6e7a1b9dac12ac5f6165ba37a297834d0690c42cf3/sqlalchemy-2.0.54-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e7a76d5dce712ce50435d0f97181eb955ec27d138c004176f01282e063bac52", size = 3357692, upload-time = "2026-09-15T22:33:14.781Z" }, + { url = "https://files.pythonhosted.org/packages/e3/d4/63e800b45bd4fa59cb4b14c013130e0aa92c7a019a0b8bf68115643bf32b/sqlalchemy-2.0.54-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f5c09090b1a7c4d389d1431f820931e8df318f82caafc53f9a72c872fef467c5", size = 3308258, upload-time = "2026-09-15T22:33:38.469Z" }, + { url = "https://files.pythonhosted.org/packages/f4/fe/6c96fe018b9db15476b46619228dc0749cdd9b4dc8baab89aca59205b605/sqlalchemy-2.0.54-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:762cfe4d340c56368256d936a98b620a9a5650e49c1c84eba51d6edd17ffefb2", size = 3336734, upload-time = "2026-09-15T22:33:16.533Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f1/8d225d6b77863ae83ba51e913fe5954bcf58b49dd54229f0b5ba0bc983f0/sqlalchemy-2.0.54-cp310-cp310-win32.whl", hash = "sha256:6b6d4e601c4f6d85e99bb3416107cc9418c5603ca73d4ee0f5f8d79c2a1ed9e8", size = 2144294, upload-time = "2026-09-15T22:41:20.424Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f6/4247fdd087bf1b39be90930bb2b276a1c55041540628c8736512b4242669/sqlalchemy-2.0.54-cp310-cp310-win_amd64.whl", hash = "sha256:03cbf8d9a67da618bd65500a5eb3ddac89caf4c61e99b2f03fa4a1952a0725a9", size = 2170318, upload-time = "2026-09-15T22:41:21.973Z" }, + { url = "https://files.pythonhosted.org/packages/76/bf/7339b18ef05c03335a32f077e450c8040eab0c952c2964e32cd889eafc22/sqlalchemy-2.0.54-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d03084f3352dd92048cb19c71d90f116d076c9c7937e0ebc7752c4685de6d38", size = 2187776, upload-time = "2026-09-15T22:32:35.535Z" }, + { url = "https://files.pythonhosted.org/packages/9b/bb/3b6f5f3f51582d63af4c88b1e70da7888e936dee6b86a237b7633fb3f216/sqlalchemy-2.0.54-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92622fbbda1b1fe1632f3402a6e516a93c0e41d9158839c6b3dfb12117f26b72", size = 3428952, upload-time = "2026-09-15T22:40:14.441Z" }, + { url = "https://files.pythonhosted.org/packages/77/d5/ffd6b59795ebec81d582ad866f71ec7c73d356768e0442c6a07bba61c40c/sqlalchemy-2.0.54-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5800ddea045c2c860ef1d359a07a3066c7c0c426f45e3abc3874e116cb3c6937", size = 3428566, upload-time = "2026-09-15T22:35:33.739Z" }, + { url = "https://files.pythonhosted.org/packages/63/ff/1f0bffb0653b30711e2445a48968eff9412b4e5bf0870f1e72e8a69595ef/sqlalchemy-2.0.54-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1019abef05a4b5eafc8eae6fb483167fa28a4dbe5f518d577b744f31a5276a37", size = 3377697, upload-time = "2026-09-15T22:40:16.494Z" }, + { url = "https://files.pythonhosted.org/packages/97/76/774906be0d41cc4a14ecfbc015112fb7d0a3f1c3393f2c179d10388f8f0b/sqlalchemy-2.0.54-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b67749f7da3985a529cefbb1474783cb91ef44371cb9713630bade3de908760d", size = 3402966, upload-time = "2026-09-15T22:35:35.726Z" }, + { url = "https://files.pythonhosted.org/packages/cd/bb/1d7fe50ace612bf342d3440d8b09544746949120326c3dc487cfc2592fba/sqlalchemy-2.0.54-cp311-cp311-win32.whl", hash = "sha256:2f61a70b3b82e2ec7ad6a4f2301422b9ca93ff06917983e41317bcae878bddf6", size = 2142297, upload-time = "2026-09-15T21:25:19.917Z" }, + { url = "https://files.pythonhosted.org/packages/d6/49/18fe80f64b1f6bcd422e649e40a78031f8624b2fde22ca54e950dfa6e25a/sqlalchemy-2.0.54-cp311-cp311-win_amd64.whl", hash = "sha256:1d887fbd5d248e250807bd801e697fc73e3b44866ce5f093dbc90512e75bde25", size = 2169222, upload-time = "2026-09-15T21:25:21.114Z" }, + { url = "https://files.pythonhosted.org/packages/c6/30/75504fd1d70458000e85a3e772333dd0fbf80254b0dc4d41c99922e4d112/sqlalchemy-2.0.54-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffba7eb2d67c7505e82a0902aa854d8824b74c28a183820d6a8bd3cfd0f812c2", size = 2187596, upload-time = "2026-09-15T22:32:37.031Z" }, + { url = "https://files.pythonhosted.org/packages/a2/3d/5dfbb9528a391186a99986daecc5cbe003f34408dee91db8f5cdcf917040/sqlalchemy-2.0.54-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63cae7210fea9899e0bf35c1f1ae55d3ddd9c6d47cae8b6b43d945afa79dd65b", size = 3448745, upload-time = "2026-09-15T22:40:18.168Z" }, + { url = "https://files.pythonhosted.org/packages/d9/93/34fdc4a4faced77037a6b3ba1db1acd92e9bd12c21e229b327edd1d3e881/sqlalchemy-2.0.54-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:68d994e9b0d0423a02a20039631fa6fcbb7fa829a992f7605025774940305d19", size = 3466644, upload-time = "2026-09-15T22:35:37.7Z" }, + { url = "https://files.pythonhosted.org/packages/66/68/4beab40ae60ac3d679dbbb46bc0d2bb277013264a0bde37ecaf4b6780e98/sqlalchemy-2.0.54-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3de32cc6721eb42c3aad35bcfb244bb7a18f66c00f3582aae6281d6287a339b5", size = 3401519, upload-time = "2026-09-15T22:40:19.907Z" }, + { url = "https://files.pythonhosted.org/packages/e5/df/a24757e3249b1c7c1c5f0317666d16a8ab901ba239819ecef29eb4ed172e/sqlalchemy-2.0.54-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d31a2bc06a854ee52dd86b455be4df7c750b28817e2d1b884e31fff126c4fd7b", size = 3432590, upload-time = "2026-09-15T22:35:39.434Z" }, + { url = "https://files.pythonhosted.org/packages/d9/0c/69559e3d90d200fbfe9c44aff39a9ff4e506b1a3e9557a203d69df721244/sqlalchemy-2.0.54-cp312-cp312-win32.whl", hash = "sha256:32de6deded25e8b9b11d07428d496ff24dfbc882b8e990c177266948cb5f3d9e", size = 2140825, upload-time = "2026-09-15T21:25:22.444Z" }, + { url = "https://files.pythonhosted.org/packages/d9/10/4a0f7113664c2877906db143709c52ddbd28a88e4d51f0b520bc30048ba7/sqlalchemy-2.0.54-cp312-cp312-win_amd64.whl", hash = "sha256:d65f8ca742ef1e1e14bc417ef59dc2ddf207a7b66b30cfdc6152447314e030cf", size = 2170005, upload-time = "2026-09-15T21:25:23.885Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/021ccd7a838425f52e67c02edc541e8d53be8a27a103e684e14b18aac17b/sqlalchemy-2.0.54-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b374e3bc91e246a942592a98ba6a23be76fff21358b00546ac8c0ebc0fd0e00b", size = 2184259, upload-time = "2026-09-15T22:28:59.153Z" }, + { url = "https://files.pythonhosted.org/packages/74/8f/f95de908a4af7ac3a6925cdcb837a93274f608925a0fbc786df38811bccf/sqlalchemy-2.0.54-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31d5458672a6f72db2c087f4a5098b3c8503ea0254186ff29205d63afa9401a4", size = 3398792, upload-time = "2026-09-15T22:29:35.218Z" }, + { url = "https://files.pythonhosted.org/packages/84/26/bd327a1a6be223e438c98ac8faf299307d83797225a81f3b209607a9fd98/sqlalchemy-2.0.54-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cad78d04254967bdbcccbed5e631d88fe4868530946ab0929aa45e9032849518", size = 3417562, upload-time = "2026-09-15T22:40:32.637Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b1/f13a8fe8e8e167d8e65484b91b827897b8a5ca1efd7ba6e87a5f63412fed/sqlalchemy-2.0.54-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:48611087a75d26d798003645c688c7d3cfc26b89dbe4a2c568d6b378d330deae", size = 3349138, upload-time = "2026-09-15T22:29:36.729Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b7/5fd58f03281a74c57f02f509433e1be58a79d9f5b0e14020e665e0dcb614/sqlalchemy-2.0.54-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d6adf80277372a89910a0f3ccfe960b846d279dc55b366dd5c5ec07f41c84758", size = 3382943, upload-time = "2026-09-15T22:40:34.587Z" }, + { url = "https://files.pythonhosted.org/packages/4b/73/e29fa88dfc809857a55e6523911d345d00cab4adf87032c21ade8874223c/sqlalchemy-2.0.54-cp313-cp313-win32.whl", hash = "sha256:264460333ed0b177cbb1956355d0ee4e0cab83fb415c934ce12a25db2e7be39c", size = 2138931, upload-time = "2026-09-15T22:42:51.801Z" }, + { url = "https://files.pythonhosted.org/packages/89/9d/885e7491836f3dad368c74f0d5b93551d37b0ec66c9e8753240ee988308f/sqlalchemy-2.0.54-cp313-cp313-win_amd64.whl", hash = "sha256:cf89e92bf0d4204a6afcc17af27b9271ed9c7e34e17d6f80c085d431ea4a1747", size = 2166496, upload-time = "2026-09-15T22:42:53.603Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c0/4a6503c9d22d6d00a5631082ab1484222ecf7d573db791e0f53161bf7745/sqlalchemy-2.0.54-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:abd6b21bc58e91c1932eb5d6d7f1bd44a551dfec7b6a7f517c3638ccd67233a0", size = 2185899, upload-time = "2026-09-15T22:29:00.581Z" }, + { url = "https://files.pythonhosted.org/packages/12/28/f4424f618bd1f373761a32a821d53ce2c257350e9894bae9b968cb03d8fd/sqlalchemy-2.0.54-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5417322b3c025dd82918725d3bf09ec105fac95efc195722b8b06e1d9c381139", size = 3394763, upload-time = "2026-09-15T22:29:38.131Z" }, + { url = "https://files.pythonhosted.org/packages/59/d2/7f0c77f8e042cb5f28275fea29c3080b4ac6fd4b3fdd7f59ff1ef3e28c11/sqlalchemy-2.0.54-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f84099e4b04a5c2d44500a2a8302eee5af4bc6fee63e8c6e9cf6786e747280e", size = 3402800, upload-time = "2026-09-15T22:40:36.509Z" }, + { url = "https://files.pythonhosted.org/packages/af/32/3eaa930bcf71d17a72587081d2706a5fb97ab3f11a7e0fb838f581f7cff1/sqlalchemy-2.0.54-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a0956dc754d3884da7fe60097110ec7a8a105d26afa2f0844468f4b1598c6912", size = 3341469, upload-time = "2026-09-15T22:29:39.682Z" }, + { url = "https://files.pythonhosted.org/packages/eb/cc/cddb6cbd4408e5c55b3bf722be26b9d3b54d42509f901b7ecc15debd3d1f/sqlalchemy-2.0.54-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:87ba8834318b0d8dc94fc6f405d071b5c08be32a6c3fd68107fd6952ee949615", size = 3373232, upload-time = "2026-09-15T22:40:39.181Z" }, + { url = "https://files.pythonhosted.org/packages/34/2f/9c2aa5efc642b7f3b985d13565cd1a5e78856e079ef3022796fea5180498/sqlalchemy-2.0.54-cp314-cp314-win32.whl", hash = "sha256:842540e4382472f23c79589995752648d14696a8200d0807ed8c5c59c92ade44", size = 2142018, upload-time = "2026-09-15T22:42:55.118Z" }, + { url = "https://files.pythonhosted.org/packages/e2/0b/3594f1f51769feb3022d686135dc5d8682a12345ed15ae61d0c0ca42cbee/sqlalchemy-2.0.54-cp314-cp314-win_amd64.whl", hash = "sha256:f4e8f955d13af83fb4e35c3472e5377ee22d3445eada1e5e48199588edb69835", size = 2169220, upload-time = "2026-09-15T22:42:56.727Z" }, + { url = "https://files.pythonhosted.org/packages/c3/a5/c211a9a7af83222509519407e16a4db760c6df3d03be69ebc5414d465321/sqlalchemy-2.0.54-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ca05f4e7852cf48083b0cf157e4f9504b7068780422a50fa82f45353b8c5e14a", size = 2208458, upload-time = "2026-09-15T22:30:05.718Z" }, + { url = "https://files.pythonhosted.org/packages/cb/2e/490ad7b3731116cb48ba170f7722eaa99a89707193e54389ca84b7ad55af/sqlalchemy-2.0.54-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18a8b6417cbb7b735cf91c2b59453c2a554cefa0a8d7bd15aa35740739410d77", size = 3660585, upload-time = "2026-09-15T22:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/eb/25/15dfe6814847eeda773bd58ab6cf42a94b0176e5cf25a578fc1165777160/sqlalchemy-2.0.54-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4e55a0b96a1577a1e108c91ccdeeb9cd92768f28ce206597311c3bf6d6423abd", size = 3624442, upload-time = "2026-09-15T22:36:38.377Z" }, + { url = "https://files.pythonhosted.org/packages/aa/19/724d0a6a2fb2a86ff2d6008e581c258f722d9b7d8e52adc7b79085febdd4/sqlalchemy-2.0.54-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:69cab115c40fd02c5a22c68e4ee630fa6ef9a1650f1de944419aab1f7096fc4f", size = 3562972, upload-time = "2026-09-15T22:36:08.581Z" }, + { url = "https://files.pythonhosted.org/packages/49/bb/9df1bd81c2f2d000cf5e7a1b1a9b331468a3aab939ad983355e701fa42b2/sqlalchemy-2.0.54-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e08397c6c42f53b2488acde9108b8bfefd52d7afd1bf2f03d2ffcab7a204aceb", size = 3576479, upload-time = "2026-09-15T22:36:40.272Z" }, + { url = "https://files.pythonhosted.org/packages/df/c0/b5775465d3b89061d7c46057c31c56ff8fb6c509550b2b0c6570ffc248b3/sqlalchemy-2.0.54-cp314-cp314t-win32.whl", hash = "sha256:b9086b8ad48280ef6a7ba68262d5e44f7db1c4cb1973e8cdae8a9f467ae66f51", size = 2174794, upload-time = "2026-09-15T22:31:45.925Z" }, + { url = "https://files.pythonhosted.org/packages/77/f8/296c2e46b4ccd3f29b00b954ef2f195dde32f98e352ed21de1d292cedc0d/sqlalchemy-2.0.54-cp314-cp314t-win_amd64.whl", hash = "sha256:b67c1744e453af833667fc1b84de07adb4a64f3536ef52a8ec5ac2b941d43970", size = 2211942, upload-time = "2026-09-15T22:31:47.368Z" }, + { url = "https://files.pythonhosted.org/packages/24/a1/bd5e3e99bc9c8863b51ac5b9b03008a7f2da8c6b59695992f5c654e1265b/sqlalchemy-2.0.54-py3-none-any.whl", hash = "sha256:7e33a631ab1474f8fe6b910bd1a07b7b8009c4c78cdd3fb18001b03e3bc2e1d2", size = 1958015, upload-time = "2026-09-15T22:24:22.95Z" }, ] [package.optional-dependencies] From 954fb2e9cfe9f517598649ca4397de4cf1090c18 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Fri, 2 Oct 2026 18:18:26 +1000 Subject: [PATCH 04/16] fix(mysql): lock trusted identity provisioning --- openviking/server/account_stores/mysql.py | 6 +++--- tests/server/test_account_services.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index 5db4d57cf4..078845ad63 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -677,10 +677,12 @@ async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> di continue async with self._transaction() as session: account = await session.scalar( - select(Account).where( + select(Account) + .where( Account.resource_id == self._resource_id, Account.account_id == account_id, ) + .with_for_update() ) if account is None: try: @@ -694,8 +696,6 @@ async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> di created_accounts += 1 except IntegrityError: account = await self._lock_account(session, account_id) - else: - account = await self._lock_account(session, account_id) if account.deletion_task_id: continue existing = set( diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index 8b2a4d1497..010a7574dd 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -541,6 +541,9 @@ async def test_mysql_managers_share_authoritative_state(monkeypatch): ) assert account_created assert await second.get_deletion("trusted", "carol") == account_deletion + blocked = await second.ensure_trusted_identities({"trusted": {"late-user"}}) + assert blocked == {"created_accounts": 0, "created_users": 0} + assert await first.get_registered_user_role("trusted", "late-user") is None key = await first.create_account("acme", "alice") assert (await second.resolve_identity(key)).role == Role.ADMIN From 488891e03d491d24ed3f65df1c01e3ff84653b9d Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Fri, 2 Oct 2026 18:18:43 +1000 Subject: [PATCH 05/16] fix(redis): harden account cache consistency --- docs/en/guides/01-configuration.md | 7 + docs/zh/guides/01-configuration.md | 5 + .../server/account_stores/redis_mysql.py | 16 +- .../server/test_redis_mysql_account_store.py | 141 +++++++++++++++++- 4 files changed, 161 insertions(+), 8 deletions(-) diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index e8a242861a..cc15aece6c 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1969,6 +1969,13 @@ index members, allowing account deletion or recreation to clear only that account's current cached users, groups, deletion records, and credential data. API-key bindings are checked against account and user credential fences, so normal key replacement, revocation, and user deletion invalidate old bindings. +Redis automatically removes the index when its last sorted-set member is removed. +Cleanup does not issue a separate index deletion, which could remove a newer +fill's index after lease loss. Provisioning +and key replacement clear negative lookups both with and without a user hint. +Redis socket connect and read timeouts default to 5 seconds each; override them +with `socket_connect_timeout` and `socket_timeout` query parameters in `redis.url`. +These are socket timeouts, not an overall request deadline. ```json { diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index c761abf92c..054426ee0a 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1945,6 +1945,11 @@ schema 变更按顺序存放于 `openviking/server/account_stores/sql/changes/` 级、按过期时间排序的 key 索引；每次写入已索引缓存时会清理过期成员，因此删除或重建 Account 时只会清理该 Account 当前的 User、Group、删除记录和凭据缓存。API Key 绑定会 校验 Account 和 User 的凭据 fence，所以正常的换 Key、吊销和删 User 会使旧绑定失效。 +索引的最后一个有序集合成员被移除后，Redis 会自动删除空索引；清理结束时不再单独删除 +索引，避免失去租约的旧请求误删新回填的索引。创建身份或换 Key 会同时清除携带和未携带 +User 提示的负缓存。 +Redis socket 连接和读取超时默认分别为 5 秒，可通过 `redis.url` 中的 +`socket_connect_timeout`、`socket_timeout` 查询参数覆盖；这不是整个请求的总超时。 ```json { diff --git a/openviking/server/account_stores/redis_mysql.py b/openviking/server/account_stores/redis_mysql.py index ec1380a1b8..56c8a6b7da 100644 --- a/openviking/server/account_stores/redis_mysql.py +++ b/openviking/server/account_stores/redis_mysql.py @@ -88,6 +88,7 @@ """ _ACCOUNT_CACHE_CLEAR_BATCH_SIZE = 500 +_REDIS_SOCKET_TIMEOUT_SECONDS = 5 class _CacheLockUnavailable(Exception): @@ -181,7 +182,12 @@ async def load(self) -> None: raise RuntimeError( "Redis MySQL account storage requires openviking[redis-mysql]" ) from None - self._redis = Redis.from_url(self._redis_url, decode_responses=True) + self._redis = Redis.from_url( + self._redis_url, + decode_responses=True, + socket_connect_timeout=_REDIS_SOCKET_TIMEOUT_SECONDS, + socket_timeout=_REDIS_SOCKET_TIMEOUT_SECONDS, + ) await self._redis.ping() except BaseException: await self.close() @@ -235,12 +241,11 @@ def _negative_binding_key(self, account_id: str, api_key: str, user_id_hint: str digest_input = self._resource_id + "\0" + api_key + "\0" + (user_id_hint or "") return self._key(account_id, f"binding-miss:{_digest(digest_input)}") - def _binding_cache_keys( - self, account_id: str, api_key: str, user_id: str - ) -> tuple[str, str]: + def _binding_cache_keys(self, account_id: str, api_key: str, user_id: str) -> tuple[str, ...]: return ( self._binding_key(account_id, api_key), self._negative_binding_key(account_id, api_key, user_id), + self._negative_binding_key(account_id, api_key, None), ) async def _delete(self, *keys: str) -> None: @@ -415,7 +420,8 @@ async def _clear_account_cache(self, account_id: str) -> None: if not keys: break await self._delete_and_unindex(account_id, *keys) - await self._delete(index_key) + # ZREM removes an emptied sorted set automatically. A separate DEL + # could remove a newer fill's index after this lease is lost. except Exception: logger.exception("Failed to clear Redis account cache for %s", account_id) diff --git a/tests/server/test_redis_mysql_account_store.py b/tests/server/test_redis_mysql_account_store.py index d1cc0c15bd..7ec14e9c0b 100644 --- a/tests/server/test_redis_mysql_account_store.py +++ b/tests/server/test_redis_mysql_account_store.py @@ -9,7 +9,7 @@ from openviking.server.account_stores.redis_mysql import RedisMySQLAccountStore from openviking.server.api_keys.new import APIKeyManager from openviking.server.identity import Role -from openviking_cli.exceptions import InvalidArgumentError +from openviking_cli.exceptions import InvalidArgumentError, UnauthenticatedError class FakeRedis: @@ -49,6 +49,8 @@ def _prune_zset(self, key: str): for member, score in tuple(members.items()): if score <= self.now_ms: del members[member] + if not members: + self.zsets.pop(key, None) async def eval(self, script: str, numkeys: int, *args): keys = args[:numkeys] @@ -77,6 +79,8 @@ async def eval(self, script: str, numkeys: int, *args): members = self.zsets.get(keys[0], {}) for key in keys[1:]: members.pop(key, None) + if not members: + self.zsets.pop(keys[0], None) return len(keys) - 1 if "-- prune-cache-index" in script: self._prune_zset(keys[0]) @@ -210,7 +214,9 @@ async def begin_deletion(self, account_id: str, user_id: str | None, **_kwargs): return self.user_deletion, True -def _store(api_key: str) -> tuple[RedisMySQLAccountStore, FakeMySQLAuthority]: +def _store( + api_key: str, *, redis_client=None, lock_wait_seconds: int = 30 +) -> tuple[RedisMySQLAccountStore, FakeMySQLAuthority]: mysql = FakeMySQLAuthority(api_key) store = RedisMySQLAccountStore( params={ @@ -218,10 +224,11 @@ def _store(api_key: str) -> tuple[RedisMySQLAccountStore, FakeMySQLAuthority]: "url": "redis://unused", "ttl_seconds": 60, "lock_ttl_seconds": 60, + "lock_wait_seconds": lock_wait_seconds, } }, mysql_store=mysql, - redis_client=FakeRedis(), + redis_client=redis_client if redis_client is not None else FakeRedis(), ) return store, mysql @@ -594,3 +601,131 @@ async def test_user_group_and_deletion_reads_are_cached_and_invalidated(): "account_deletion": 1, "user_deletion": 2, } + + +async def test_deleting_recreated_account_rejects_key_after_delayed_cleanup(): + api_key = _api_key() + redis = FakeRedis() + store, _ = _store(api_key, redis_client=redis, lock_wait_seconds=1) + await store.load() + original_zrange = redis.zrange + saw_empty = asyncio.Event() + resume_clear = asyncio.Event() + paused = False + + async def delayed_empty_reply(*args): + nonlocal paused + result = await original_zrange(*args) + if not result and not paused: + paused = True + saw_empty.set() + await resume_clear.wait() + return result + + redis.zrange = delayed_empty_reply + old_clear = asyncio.create_task(store.delete_account("acme")) + try: + await asyncio.wait_for(saw_empty.wait(), 2) + await asyncio.wait_for(store.create_account_with_api_key("acme", "alice", api_key), 5) + manager = APIKeyManager("root", store) + assert (await manager.resolve_identity(api_key)).role == Role.ADMIN + + resume_clear.set() + await asyncio.wait_for(old_clear, 2) + + await store.delete_account("acme") + with pytest.raises(UnauthenticatedError, match="Invalid API key"): + await manager.resolve_identity(api_key) + finally: + resume_clear.set() + await asyncio.gather(old_clear, return_exceptions=True) + await store.close() + + +@pytest.mark.parametrize("operation", ["create_account", "create_user", "replace_key"]) +@pytest.mark.parametrize("user_hint", [None, "alice"]) +async def test_provisioned_key_is_immediately_usable(operation, user_hint): + new_key = _api_key("new") + store, mysql = _store(_api_key("old")) + if operation == "create_account": + mysql.account_exists = False + mysql.user_exists = False + elif operation == "create_user": + mysql.user_exists = False + await store.load() + try: + assert ( + await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint=user_hint) + is None + ) + if operation == "create_account": + await store.create_account_with_api_key("acme", "alice", new_key) + elif operation == "create_user": + await store.create_user_with_api_key("acme", "alice", Role.USER, new_key) + else: + await store.replace_active_user_api_key("acme", "alice", new_key) + assert await store.verify_api_key( + new_key, account_id_hint="acme", user_id_hint=user_hint + ) == ("acme", "alice") + finally: + await store.close() + + +async def test_key_rotation_rejects_old_key_after_inflight_verification(): + old_key, new_key = _api_key("old"), _api_key("new") + store, mysql = _store(old_key, lock_wait_seconds=1) + await store.load() + read_started = asyncio.Event() + resume_read = asyncio.Event() + original_verify = mysql.verify_api_key + + async def delayed_verification(api_key, **hints): + result = await original_verify(api_key, **hints) + if api_key == old_key and not read_started.is_set(): + read_started.set() + await resume_read.wait() + return result + + mysql.verify_api_key = delayed_verification + reader = asyncio.create_task( + store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") + ) + try: + await asyncio.wait_for(read_started.wait(), 2) + await asyncio.wait_for(store.replace_active_user_api_key("acme", "alice", new_key), 5) + assert await store.verify_api_key( + new_key, account_id_hint="acme", user_id_hint="alice" + ) == ("acme", "alice") + resume_read.set() + # After the earlier request finishes, only the new key remains valid. + await asyncio.wait_for(reader, 2) + assert ( + await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") + is None + ) + assert await store.verify_api_key( + new_key, account_id_hint="acme", user_id_hint="alice" + ) == ("acme", "alice") + finally: + resume_read.set() + await asyncio.gather(reader, return_exceptions=True) + await store.close() + + +async def test_redis_timeouts_allow_mysql_reads_and_writes(): + redis = FakeRedis() + store, _ = _store(_api_key(), redis_client=redis) + await store.load() + + async def timeout(*_args, **_kwargs): + raise TimeoutError("Redis did not respond") + + redis.get = timeout + redis.set = timeout + redis.delete = timeout + redis.eval = timeout + try: + await store.set_role("acme", "alice", Role.USER) + assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.USER} + finally: + await store.close() From fe5000bda6d5c74df8134669ce83874f41b58b0a Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Mon, 12 Oct 2026 11:28:41 +0800 Subject: [PATCH 06/16] refactor(account-store): simplify provider contract coverage --- openviking/server/account_stores/mysql.py | 23 +- .../server/account_stores/redis_mysql.py | 52 +-- tests/server/test_account_services.py | 138 ++----- .../server/test_redis_mysql_account_store.py | 358 +++--------------- 4 files changed, 113 insertions(+), 458 deletions(-) diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index 078845ad63..fa3a17920b 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -12,7 +12,6 @@ import secrets import ssl as ssl_module from contextlib import asynccontextmanager -from datetime import datetime from typing import AsyncGenerator from uuid import uuid4 @@ -92,10 +91,6 @@ def _substring_pattern(value: str) -> str: return f"%{escaped}%" -def _timestamp(value: datetime) -> str: - return value.isoformat() - - def _credential_digest(api_key: str) -> str: """Return the lookup digest for a high-entropy API key.""" return hashlib.sha256(api_key.encode()).hexdigest() @@ -188,11 +183,6 @@ def _decrypt_credential(self, account_id: str, user_id: str, material: str) -> s except (InvalidTag, ValueError, UnicodeError): return None - def _display_credential(self, account_id: str, user_id: str, material: str) -> str | None: - if material.startswith(_DIGEST_CREDENTIAL_PREFIX): - return None - return self._decrypt_credential(account_id, user_id, material) - def _matches_credential( self, account_id: str, user_id: str, material: str, api_key: str ) -> bool: @@ -389,7 +379,7 @@ async def _protect_last_admin(self, session: AsyncSession, account, user) -> Non def _account_summary(account, user_count: int) -> AccountSummary: return { "account_id": account.account_id, - "created_at": _timestamp(account.created_at), + "created_at": account.created_at.isoformat(), "user_count": user_count, "status": "deleting" if account.deletion_task_id else "active", **({"task_id": account.deletion_task_id} if account.deletion_task_id else {}), @@ -572,14 +562,7 @@ async def list_users_page( async with self._session_factory() as session: # One HTTP response must derive its rows and summary from one MVCC snapshot. await session.connection(execution_options={"isolation_level": "REPEATABLE READ"}) - account = await session.scalar( - select(Account).where( - Account.resource_id == self._resource_id, - Account.account_id == account_id, - ) - ) - if account is None: - raise NotFoundError(account_id, "account") + account = await self._account_for_read(session, account_id) base = [ User.resource_id == self._resource_id, User.account_pk == account.account_pk, @@ -640,7 +623,7 @@ async def list_users_page( for user in users: row: UserSummary = {"user_id": user.user_id, "role": user.role} if credential := credentials.get(user.user_pk): - api_key = self._display_credential(account_id, user.user_id, credential.material) + api_key = self._decrypt_credential(account_id, user.user_id, credential.material) if api_key is None: row["key_prefix"] = credential.prefix else: diff --git a/openviking/server/account_stores/redis_mysql.py b/openviking/server/account_stores/redis_mysql.py index 56c8a6b7da..2517c729cc 100644 --- a/openviking/server/account_stores/redis_mysql.py +++ b/openviking/server/account_stores/redis_mysql.py @@ -481,17 +481,13 @@ async def _mutate_account( ) return result - def _user_cache_keys( - self, account_id: str, user_id: str, *, credentials: bool = False - ) -> tuple[str, ...]: - keys = ( + def _user_cache_keys(self, account_id: str, user_id: str) -> tuple[str, ...]: + return ( self._user_key(account_id, user_id), self._group_ids_key(account_id, user_id), self._user_deletion_key(account_id, user_id), + self._user_credential_fence_key(account_id, user_id), ) - if credentials: - return keys + (self._user_credential_fence_key(account_id, user_id),) - return keys async def _cached_binding( self, @@ -531,9 +527,7 @@ async def _cached_binding( ): return True, (binding_account_id, user_id) negative = _decode( - await self._redis.get( - self._negative_binding_key(account_id, api_key, user_id_hint) - ) + await self._redis.get(self._negative_binding_key(account_id, api_key, user_id_hint)) ) negative_hit, negative_value = self._read_nullable(negative) return (True, None) if negative_hit and negative_value is None else (False, None) @@ -721,7 +715,7 @@ async def create_user(self, account_id: str, user_id: str, role: str) -> None: await self._mutate_account( account_id, lambda: self._mysql.create_user(account_id, user_id, role), - lambda: self._user_cache_keys(account_id, user_id, credentials=True), + lambda: self._user_cache_keys(account_id, user_id), ) async def create_user_with_api_key( @@ -739,12 +733,10 @@ async def create_user_with_api_key( role, api_key, ), - lambda: self._user_cache_keys( - account_id, - user_id, - credentials=True, - ) - + self._binding_cache_keys(account_id, api_key, user_id), + lambda: ( + self._user_cache_keys(account_id, user_id) + + self._binding_cache_keys(account_id, api_key, user_id) + ), ) async def list_users_page(self, account_id: str, **kwargs) -> UsersPage: @@ -791,9 +783,7 @@ async def delete_group(self, account_id: str, group_id: str) -> None: lambda: (), ) - async def ensure_trusted_identities( - self, identities: dict[str, set[str]] - ) -> dict[str, int]: + async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> dict[str, int]: result = {"created_accounts": 0, "created_users": 0} for account_id, user_ids in identities.items(): created = await self._mutate_account( @@ -838,9 +828,7 @@ async def finish_deletion(self, account_id: str, user_id: str | None, task_id: s return await self._mutate_account( account_id, lambda: self._mysql.finish_deletion(account_id, user_id, task_id), - lambda: () - if user_id is None - else self._user_cache_keys(account_id, user_id, credentials=True), + lambda: () if user_id is None else self._user_cache_keys(account_id, user_id), account_wide=user_id is None, ) @@ -851,20 +839,20 @@ async def replace_active_user_api_key( account_id, lambda: self._mysql.replace_active_user_api_key(account_id, user_id, api_key), lambda: ( - self._user_credential_fence_key(account_id, user_id), - ) - + self._binding_cache_keys(account_id, api_key, user_id), + (self._user_credential_fence_key(account_id, user_id),) + + self._binding_cache_keys(account_id, api_key, user_id) + ), ) - async def revoke_active_api_keys( - self, account_id: str, user_id: str | None = None - ) -> None: + async def revoke_active_api_keys(self, account_id: str, user_id: str | None = None) -> None: await self._mutate_account( account_id, lambda: self._mysql.revoke_active_api_keys(account_id, user_id), - lambda: (self._account_credential_fence_key(account_id),) - if user_id is None - else (self._user_credential_fence_key(account_id, user_id),), + lambda: ( + (self._account_credential_fence_key(account_id),) + if user_id is None + else (self._user_credential_fence_key(account_id, user_id),) + ), ) async def get_user_key_fingerprint(self, account_id: str, user_id: str) -> str | None: diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index 010a7574dd..ff8d3fcefa 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -11,13 +11,12 @@ import pytest from sqlalchemy import delete -from sqlalchemy.dialects import mysql from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import create_async_engine from openviking.server.account_stores import mysql as mysql_store_module -from openviking.server.account_stores.mysql import MySQLAccountStore, _substring_pattern -from openviking.server.account_stores.mysql_schema import Account, Base, Group, GroupMember +from openviking.server.account_stores.mysql import MySQLAccountStore +from openviking.server.account_stores.mysql_schema import Account, Base from openviking.server.api_keys import APIKeyManager from openviking.server.api_keys.legacy import FileStore from openviking.server.identity import Role @@ -183,23 +182,8 @@ async def test_file_trusted_batch_rejects_oversized_account_before_any_write(): await manager.close() -async def test_file_account_creation_is_atomic_when_credential_write_fails(): - class FailingAGFS(InMemoryAGFS): - def write(self, path, content, ctx=None): - if path == "/local/acme/_system/users.json": - raise OSError("credential store unavailable") - return super().write(path, content, ctx) - - store = FileStore(SimpleNamespace(agfs=FailingAGFS())) - manager = APIKeyManager("root", store) - await manager.load() - - with pytest.raises(OSError, match="credential store unavailable"): - await manager.create_account("acme", "alice") - assert await manager.get_account("acme") is None - - -async def test_file_user_creation_is_atomic_when_credential_write_fails(): +@pytest.mark.parametrize("operation", ["account", "user"]) +async def test_file_creation_is_atomic_when_credential_write_fails(operation): class FailingAGFS(InMemoryAGFS): fail_user_write = False @@ -212,13 +196,23 @@ def write(self, path, content, ctx=None): store = FileStore(SimpleNamespace(agfs=agfs)) manager = APIKeyManager("root", store) await manager.load() - admin_key = await manager.create_account("acme", "alice") + if operation == "user": + admin_key = await manager.create_account("acme", "alice") agfs.fail_user_write = True - with pytest.raises(OSError, match="credential store unavailable"): - await manager.register_user("acme", "bob") - assert await manager.get_user("acme", "bob") is None - assert (await manager.resolve_identity(admin_key)).user_id == "alice" + try: + with pytest.raises(OSError, match="credential store unavailable"): + if operation == "account": + await manager.create_account("acme", "alice") + else: + await manager.register_user("acme", "bob") + assert await manager.get_user("acme", "bob") is None + if operation == "account": + assert await manager.get_account("acme") is None + else: + assert (await manager.resolve_identity(admin_key)).user_id == "alice" + finally: + await manager.close() async def test_regenerate_key_preserves_management_not_found_errors(): @@ -232,18 +226,6 @@ async def test_regenerate_key_preserves_management_not_found_errors(): await manager.regenerate_key("acme", "missing") -async def test_file_store_allows_only_one_active_credential_per_user(): - store = _file_store() - await store.load() - await store.create_account_with_api_key("acme", "alice", "initial-key") - - await store.replace_active_user_api_key("acme", "alice", "first-key") - await store.replace_active_user_api_key("acme", "alice", "second-key") - assert await store.verify_api_key("first-key") is None - active = await store.verify_api_key("second-key") - assert active == ("acme", "alice") - - async def test_file_hashing_never_persists_plaintext_credentials(): agfs = InMemoryAGFS() manager = build_api_key_manager( @@ -256,12 +238,13 @@ async def test_file_hashing_never_persists_plaintext_credentials(): user_key = await manager.register_user("acme", "bob") rotated_key = await manager.regenerate_key("acme", "bob") - users = json.loads(agfs._files["/local/acme/_system/users.json"])["users"] - assert users["alice"]["key"].startswith("$argon2") - assert users["bob"]["key"].startswith("$argon2") - assert account_key.encode() not in agfs._files["/local/acme/_system/users.json"] - assert user_key.encode() not in agfs._files["/local/acme/_system/users.json"] - assert rotated_key.encode() not in agfs._files["/local/acme/_system/users.json"] + persisted = agfs.read("/local/acme/_system/users.json") + for key in (account_key, user_key, rotated_key): + assert key.encode() not in persisted + assert (await manager.resolve_identity(account_key)).user_id == "alice" + assert (await manager.resolve_identity(rotated_key)).user_id == "bob" + with pytest.raises(UnauthenticatedError): + await manager.resolve_identity(user_key) async def test_file_load_migrates_plaintext_credential_to_argon2(): @@ -277,9 +260,7 @@ async def test_file_load_migrates_plaintext_credential_to_argon2(): ) await hashed.load() - users = json.loads(agfs._files["/local/acme/_system/users.json"])["users"] - assert users["alice"]["key"].startswith("$argon2") - assert key.encode() not in agfs._files["/local/acme/_system/users.json"] + assert key.encode() not in agfs.read("/local/acme/_system/users.json") assert (await hashed.resolve_identity(key)).user_id == "alice" @@ -355,29 +336,6 @@ def test_mysql_requires_resource_id(monkeypatch): MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) -def test_mysql_substring_filter_escapes_sql_wildcards(): - assert _substring_pattern("alice") == "%alice%" - assert _substring_pattern("a_b%=c") == "%a=_b=%==c%" - - -def test_mysql_schema_uses_fixed_tables_and_scoped_memberships(): - assert set(Base.metadata.tables) == { - "ov_accounts", - "ov_accounts_users", - "ov_accounts_groups", - "ov_accounts_group_members", - "ov_accounts_credentials", - } - membership = GroupMember.__table__ - assert "account_pk" in membership.c - assert all( - {"resource_id", "account_pk"}.issubset( - {element.parent.name for element in constraint.elements} - ) - for constraint in membership.foreign_key_constraints - ) - - def test_mysql_rejects_removed_table_parameter(monkeypatch): monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") with pytest.raises(InvalidArgumentError, match="Unknown MySQL account store parameters: table"): @@ -391,40 +349,6 @@ def test_mysql_rejects_removed_table_parameter(monkeypatch): ) -async def test_mysql_verification_excludes_deleting_identities(monkeypatch): - monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") - store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) - statements = [] - - class Result: - def all(self): - return [] - - class Session: - async def __aenter__(self): - return self - - async def __aexit__(self, *_args): - return None - - async def execute(self, statement): - statements.append( - str( - statement.compile( - dialect=mysql.dialect(), - compile_kwargs={"literal_binds": True}, - ) - ) - ) - return Result() - - store._sessions = lambda: Session() - - assert await store.verify_api_key("key", account_id_hint="acme") is None - statement = statements[0] - assert statement.count("deletion_task_id IS NULL") == 2 - - async def test_mysql_load_rejects_incomplete_schema_and_disposes_engine(monkeypatch): monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) @@ -445,13 +369,7 @@ async def __aexit__(self, *args): pass async def execute(self, statement): - table = statement.get_final_froms()[0].name - if table == Group.__tablename__: - raise OperationalError( - str(statement), - {}, - Exception(1146, f"Table '{table}' doesn't exist"), - ) + raise OperationalError(str(statement), {}, Exception(1146, "Table doesn't exist")) monkeypatch.setattr(mysql_store_module, "create_async_engine", lambda *args, **kwargs: engine) monkeypatch.setattr( diff --git a/tests/server/test_redis_mysql_account_store.py b/tests/server/test_redis_mysql_account_store.py index 7ec14e9c0b..6edfdc5a1f 100644 --- a/tests/server/test_redis_mysql_account_store.py +++ b/tests/server/test_redis_mysql_account_store.py @@ -70,9 +70,7 @@ async def eval(self, script: str, numkeys: int, *args): self.values[keys[1]] = values[1] if str(values[4]) == "1": self._prune_zset(keys[2]) - self.zsets.setdefault(keys[2], {})[keys[1]] = ( - self.now_ms + int(values[2]) * 1000 - ) + self.zsets.setdefault(keys[2], {})[keys[1]] = self.now_ms + int(values[2]) * 1000 return 1 if "-- delete-and-unindex" in script: await self.delete(*keys[1:]) @@ -89,9 +87,8 @@ async def eval(self, script: str, numkeys: int, *args): class FakeMySQLAuthority: - _resource_id = "test-resource" - - def __init__(self, api_key: str): + def __init__(self, api_key: str, resource_id: str = "test-resource"): + self._resource_id = resource_id self.api_key = api_key self.account_exists = True self.user_exists = True @@ -99,13 +96,6 @@ def __init__(self, api_key: str): self.group_ids = ("engineering",) self.account_deletion = None self.user_deletion = None - self.calls = { - "verify": 0, - "user": 0, - "groups": 0, - "account_deletion": 0, - "user_deletion": 0, - } async def load(self): return None @@ -120,7 +110,6 @@ async def verify_api_key( account_id_hint: str | None = None, user_id_hint: str | None = None, ): - self.calls["verify"] += 1 return ( ("acme", "alice") if ( @@ -134,7 +123,6 @@ async def verify_api_key( ) async def get_user(self, account_id: str, user_id: str): - self.calls["user"] += 1 if ( not self.account_exists or not self.user_exists @@ -144,7 +132,6 @@ async def get_user(self, account_id: str, user_id: str): return dict(self.user) async def get_user_group_ids(self, account_id: str, user_id: str): - self.calls["groups"] += 1 return ( self.group_ids if self.account_exists and (account_id, user_id) == ("acme", "alice") @@ -155,9 +142,7 @@ async def get_deletion(self, account_id: str, user_id: str | None = None): if not self.account_exists or account_id != "acme": return None if user_id is None: - self.calls["account_deletion"] += 1 return self.account_deletion - self.calls["user_deletion"] += 1 return self.account_deletion or self.user_deletion async def replace_active_user_api_key(self, account_id: str, user_id: str, api_key: str): @@ -215,9 +200,9 @@ async def begin_deletion(self, account_id: str, user_id: str | None, **_kwargs): def _store( - api_key: str, *, redis_client=None, lock_wait_seconds: int = 30 + api_key: str, *, redis_client=None, lock_wait_seconds: int = 30, resource_id="test-resource" ) -> tuple[RedisMySQLAccountStore, FakeMySQLAuthority]: - mysql = FakeMySQLAuthority(api_key) + mysql = FakeMySQLAuthority(api_key, resource_id) store = RedisMySQLAccountStore( params={ "redis": { @@ -240,14 +225,6 @@ def _api_key(secret: str = "secret") -> str: return f"{encoded_account}.{encoded_user}.{encoded_secret}" -def test_account_keys_include_resource_in_hash_tag(): - store, _ = _store(_api_key()) - - assert "{dGVzdC1yZXNvdXJjZQBhY21l}" in store._user_key("acme", "alice") - assert "{dGVzdC1yZXNvdXJjZQBhY21l}" in store._binding_key("acme", _api_key()) - assert store._lock_wait_seconds == 30 - - def test_redis_timeouts_must_be_positive_integers(): mysql = FakeMySQLAuthority(_api_key()) for name, value in ( @@ -266,26 +243,29 @@ def test_redis_timeouts_must_be_positive_integers(): ) -async def test_verify_api_key_caches_public_binding_read(): - api_key = _api_key() - store, mysql = _store(api_key) - await store.load() - - assert await store.verify_api_key(api_key, account_id_hint="acme", user_id_hint="alice") == ( - "acme", - "alice", - ) - assert await store.verify_api_key(api_key, account_id_hint="acme", user_id_hint="alice") == ( - "acme", - "alice", - ) - - assert mysql.calls["verify"] == 1 +async def test_resources_do_not_share_cached_identities(): + redis = FakeRedis() + first_key, second_key = _api_key("first"), _api_key("second") + first, _ = _store(first_key, redis_client=redis, resource_id="first") + second, _ = _store(second_key, redis_client=redis, resource_id="second") + await first.load() + await second.load() + try: + assert await first.verify_api_key(first_key, account_id_hint="acme") == ("acme", "alice") + assert await second.verify_api_key(first_key, account_id_hint="acme") is None + assert await second.verify_api_key(second_key, account_id_hint="acme") == ("acme", "alice") + await first.get_user("acme", "alice") + await second.set_role("acme", "alice", Role.USER) + assert (await first.get_user("acme", "alice"))["role"] == Role.ADMIN + assert (await second.get_user("acme", "alice"))["role"] == Role.USER + finally: + await first.close() + await second.close() async def test_cached_binding_respects_user_id_hint(): api_key = _api_key() - store, mysql = _store(api_key) + store, _ = _store(api_key) await store.load() assert await store.verify_api_key( @@ -306,284 +286,68 @@ async def test_cached_binding_respects_user_id_hint(): account_id_hint="acme", user_id_hint="alice", ) == ("acme", "alice") - assert mysql.calls["verify"] == 2 - - -async def test_concurrent_user_cache_misses_query_mysql_once(): - store, mysql = _store(_api_key()) - await store.load() - read_started = asyncio.Event() - allow_read = asyncio.Event() - - async def delayed_get_user(account_id: str, user_id: str): - mysql.calls["user"] += 1 - read_started.set() - await allow_read.wait() - return {"user_id": user_id, "role": Role.ADMIN} - - mysql.get_user = delayed_get_user - reads = [ - asyncio.create_task(store.get_user("acme", "alice")) - for _ in range(5) - ] - await read_started.wait() - await asyncio.sleep(0.05) - assert mysql.calls["user"] == 1 - - allow_read.set() - assert await asyncio.gather(*reads) == [ - {"user_id": "alice", "role": Role.ADMIN} - ] * 5 - assert mysql.calls["user"] == 1 - - -async def test_expired_cache_lease_cannot_refill_cache(): - store, _ = _store(_api_key()) - await store.load() - redis = store._redis - key = store._user_key("acme", "alice") - redis.values[store._lock_key("acme")] = "new-owner" - - written = await store._set_if_locked( - "acme", - "expired-owner", - key, - {"found": True, "value": {"user_id": "alice", "role": Role.ADMIN}}, - ) - - assert written is False - assert key not in redis.values - - -async def test_write_lock_timeout_revokes_stale_lease_before_invalidation(): - store, mysql = _store(_api_key()) - store._lock_wait_seconds = 0.05 - await store.load() - assert await store.get_user("acme", "alice") is not None - redis = store._redis - redis.values[store._lock_key("acme")] = "other-owner" - - mutation = asyncio.create_task(store.set_role("acme", "alice", Role.USER)) - await asyncio.sleep(0.01) - assert mysql.user["role"] == Role.USER - assert not mutation.done() - await mutation - - assert store._lock_key("acme") not in redis.values - assert store._user_key("acme", "alice") not in redis.values - assert not await store._set_if_locked( - "acme", - "other-owner", - store._user_key("acme", "alice"), - {"found": True, "value": {"user_id": "alice", "role": Role.ADMIN}}, - ) -async def test_invalidation_failure_does_not_block_mysql_mutation(): - store, mysql = _store(_api_key()) - await store.load() - assert await store.get_user("acme", "alice") is not None - - async def fail_invalidation(*_args): - raise ConnectionError("redis unavailable") - - store._delete_and_unindex = fail_invalidation - - await store.set_role("acme", "alice", Role.USER) - - assert mysql.user["role"] == Role.USER - assert store._user_key("acme", "alice") in store._redis.values - - -async def test_redis_outage_does_not_block_mysql_mutation(): - store, mysql = _store(_api_key()) - await store.load() - - async def unavailable(*_args, **_kwargs): - raise ConnectionError("redis unavailable") - - store._redis.set = unavailable - store._redis.delete = unavailable - store._redis.eval = unavailable - - await store.set_role("acme", "alice", Role.USER) - - assert mysql.user["role"] == Role.USER - - -async def test_api_key_manager_uses_cached_public_store_reads(): +async def test_cached_identity_survives_authority_read_failure(monkeypatch): api_key = _api_key() store, mysql = _store(api_key) await store.load() manager = APIKeyManager("root", store) first = await manager.resolve_identity(api_key) + + async def unavailable(*_args, **_kwargs): + raise ConnectionError("MySQL unavailable") + + for method in ("verify_api_key", "get_user", "get_deletion"): + monkeypatch.setattr(mysql, method, unavailable) second = await manager.resolve_identity(api_key) assert (first.account_id, first.user_id, first.role) == ("acme", "alice", Role.ADMIN) assert second == first - assert mysql.calls == { - "verify": 1, - "user": 1, - "groups": 0, - "account_deletion": 1, - "user_deletion": 1, - } - - -async def test_replacing_key_invalidates_cached_binding_and_prior_negative(): - old_key = _api_key("old") - new_key = _api_key("new") - store, mysql = _store(old_key) - await store.load() - assert await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") - assert await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint="alice") is None - assert await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint="alice") is None - assert mysql.calls["verify"] == 2 - assert store._binding_key("acme", new_key) not in await store._redis.zrange( - store._cache_keys_index_key("acme"), - 0, - -1, - ) - - await store.replace_active_user_api_key("acme", "alice", new_key) - - assert await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") is None - assert await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint="alice") == ( - "acme", - "alice", - ) - assert mysql.calls["verify"] == 4 -async def test_creating_account_invalidates_prior_negative_key_binding(): - api_key = _api_key("new-account") - store, mysql = _store(api_key) - mysql.account_exists = False - mysql.user_exists = False - await store.load() - - assert await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="alice", - ) is None - - await store.create_account_with_api_key("acme", "alice", api_key) - - assert await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="alice", - ) == ("acme", "alice") - - -async def test_creating_user_invalidates_prior_negative_key_binding(): - api_key = _api_key("new-user") - store, mysql = _store(api_key) - mysql.user_exists = False - await store.load() - - assert await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="alice", - ) is None - - await store.create_user_with_api_key("acme", "alice", Role.USER, api_key) - - assert await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="alice", - ) == ("acme", "alice") - - -async def test_account_delete_and_recreate_clear_all_indexed_cache_keys(): +async def test_recreated_account_does_not_inherit_cached_identity(): api_key = _api_key() - store, mysql = _store(api_key) + store, _ = _store(api_key) await store.load() - redis = store._redis + manager = APIKeyManager("root", store) - assert await store.get_user("acme", "alice") + assert (await manager.resolve_identity(api_key)).role == Role.ADMIN assert await store.get_user_group_ids("acme", "alice") == ("engineering",) - assert await store.get_deletion("acme", "alice") is None - assert await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="alice", + await store.begin_deletion( + "acme", "alice", task_id="delete-1", owner_account_id="acme", owner_user_id="alice" ) - index_key = store._cache_keys_index_key("acme") - indexed_keys = set(await redis.zrange(index_key, 0, -1)) - assert store._user_key("acme", "alice") in indexed_keys - assert store._group_ids_key("acme", "alice") in indexed_keys - assert store._account_deletion_key("acme") in indexed_keys - assert store._user_deletion_key("acme", "alice") in indexed_keys - assert store._binding_key("acme", api_key) in indexed_keys - assert store._account_credential_fence_key("acme") in indexed_keys - assert store._user_credential_fence_key("acme", "alice") in indexed_keys + with pytest.raises(UnauthenticatedError): + await manager.resolve_identity(api_key) await store.delete_account("acme") - - assert index_key not in redis.zsets - assert not indexed_keys.intersection(redis.values) assert await store.get_user("acme", "alice") is None + with pytest.raises(UnauthenticatedError): + await manager.resolve_identity(api_key) - await store.create_account_with_api_key("acme", "alice", api_key) - - assert await store.get_user("acme", "alice") == { - "user_id": "alice", - "role": Role.ADMIN, - } - - -async def test_refill_prunes_expired_cache_index_members(): - store, _ = _store(_api_key()) - await store.load() - redis = store._redis - index_key = store._cache_keys_index_key("acme") - user_key = store._user_key("acme", "alice") - group_key = store._group_ids_key("acme", "alice") - - assert await store.get_user("acme", "alice") - redis.now_ms += 60_001 - assert await store.get_user_group_ids("acme", "alice") == ("engineering",) - - assert await redis.zrange(index_key, 0, -1) == [group_key] - assert user_key not in redis.zsets[index_key] - - -async def test_targeted_invalidation_removes_cache_index_member(): - store, _ = _store(_api_key()) - await store.load() - redis = store._redis - index_key = store._cache_keys_index_key("acme") - user_key = store._user_key("acme", "alice") - - assert await store.get_user("acme", "alice") - assert user_key in await redis.zrange(index_key, 0, -1) - - await store.set_role("acme", "alice", Role.USER) - - assert user_key not in await redis.zrange(index_key, 0, -1) + new_key = _api_key("recreated") + await store.create_account_with_api_key("acme", "alice", new_key) + assert (await manager.resolve_identity(new_key)).role == Role.ADMIN + assert await store.get_user_group_ids("acme", "alice") == () + assert await store.get_deletion("acme", "alice") is None + with pytest.raises(UnauthenticatedError): + await manager.resolve_identity(api_key) async def test_user_group_and_deletion_reads_are_cached_and_invalidated(): api_key = _api_key() - store, mysql = _store(api_key) + store, _ = _store(api_key) await store.load() - assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.ADMIN} assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.ADMIN} await store.set_role("acme", "alice", Role.USER) assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.USER} - assert await store.get_user_group_ids("acme", "alice") == ("engineering",) assert await store.get_user_group_ids("acme", "alice") == ("engineering",) await store.add_group_member("acme", "product", "alice") assert await store.get_user_group_ids("acme", "alice") == ("engineering", "product") - assert await store.get_deletion("acme", "alice") is None assert await store.get_deletion("acme", "alice") is None await store.begin_deletion( "acme", @@ -594,14 +358,6 @@ async def test_user_group_and_deletion_reads_are_cached_and_invalidated(): ) assert await store.get_deletion("acme", "alice") == {"task_id": "delete-1"} - assert mysql.calls == { - "verify": 0, - "user": 2, - "groups": 2, - "account_deletion": 1, - "user_deletion": 2, - } - async def test_deleting_recreated_account_rejects_key_after_delayed_cleanup(): api_key = _api_key() @@ -645,8 +401,9 @@ async def delayed_empty_reply(*args): @pytest.mark.parametrize("operation", ["create_account", "create_user", "replace_key"]) @pytest.mark.parametrize("user_hint", [None, "alice"]) async def test_provisioned_key_is_immediately_usable(operation, user_hint): + old_key = _api_key("old") new_key = _api_key("new") - store, mysql = _store(_api_key("old")) + store, mysql = _store(old_key) if operation == "create_account": mysql.account_exists = False mysql.user_exists = False @@ -654,6 +411,10 @@ async def test_provisioned_key_is_immediately_usable(operation, user_hint): mysql.user_exists = False await store.load() try: + if operation == "replace_key": + assert await store.verify_api_key( + old_key, account_id_hint="acme", user_id_hint=user_hint + ) == ("acme", "alice") assert ( await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint=user_hint) is None @@ -667,6 +428,10 @@ async def test_provisioned_key_is_immediately_usable(operation, user_hint): assert await store.verify_api_key( new_key, account_id_hint="acme", user_id_hint=user_hint ) == ("acme", "alice") + assert ( + await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint=user_hint) + is None + ) finally: await store.close() @@ -712,13 +477,14 @@ async def delayed_verification(api_key, **hints): await store.close() -async def test_redis_timeouts_allow_mysql_reads_and_writes(): +@pytest.mark.parametrize("error", [ConnectionError, TimeoutError]) +async def test_redis_failure_allows_mysql_reads_and_writes(error): redis = FakeRedis() store, _ = _store(_api_key(), redis_client=redis) await store.load() async def timeout(*_args, **_kwargs): - raise TimeoutError("Redis did not respond") + raise error("Redis did not respond") redis.get = timeout redis.set = timeout From 70914786537728a245f01eeedfdad9cf5e29c4f0 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Mon, 12 Oct 2026 10:27:07 +0800 Subject: [PATCH 07/16] fix(account-store): preserve fresh user lookup --- openviking/server/account_stores/mysql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index fa3a17920b..93788a0fa2 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -558,7 +558,9 @@ async def list_users_page( role_filter: str | None = None, page: int = 1, query_filter: str | None = None, + fresh: bool = False, ) -> UsersPage: + del fresh async with self._session_factory() as session: # One HTTP response must derive its rows and summary from one MVCC snapshot. await session.connection(execution_options={"isolation_level": "REPEATABLE READ"}) From 5219b48a09c6afa38c9fbd9d51cb13f7e60334af Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Mon, 12 Oct 2026 10:52:36 +0800 Subject: [PATCH 08/16] fix(mysql): validate account identifier lengths --- openviking/server/account_stores/mysql.py | 35 +++++++++--- tests/server/test_account_services.py | 67 +++++++++++++++++++++++ 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index 93788a0fa2..36c3a87779 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -56,10 +56,20 @@ validate_user_id, ) +_MYSQL_IDENTIFIER_MAX_LENGTH = 255 -def _validate(value: str, validator) -> None: + +def _validate_length(value: str, name: str) -> None: + if len(value) > _MYSQL_IDENTIFIER_MAX_LENGTH: + raise InvalidArgumentError( + f"{name} must not exceed {_MYSQL_IDENTIFIER_MAX_LENGTH} characters." + ) + + +def _validate(value: str, validator, name: str) -> None: if error := validator(value): raise InvalidArgumentError(error) + _validate_length(value, name) def _validate_role(role: str) -> Role: @@ -118,6 +128,7 @@ def __init__(self, *, params: dict[str, object] | None = None) -> None: self._resource_id = os.environ.get("OV_RESOURCE_ID", "").strip() if not self._resource_id: raise InvalidArgumentError("OV_RESOURCE_ID is required for MySQL account storage") + _validate_length(self._resource_id, "OV_RESOURCE_ID") credential_storage = values.pop("credential_storage", "hash") encryption_key = values.pop("credential_encryption_key", None) @@ -431,8 +442,8 @@ async def create_account_with_api_key( admin_user_id: str, api_key: str, ) -> None: - _validate(account_id, validate_account_id) - _validate(admin_user_id, validate_user_id) + _validate(account_id, validate_account_id, "account_id") + _validate(admin_user_id, validate_user_id, "admin_user_id") try: async with self._transaction() as session: account = Account(resource_id=self._resource_id, account_id=account_id) @@ -459,7 +470,7 @@ async def create_account_with_api_key( raise AlreadyExistsError(account_id, "account") from exc async def create_user(self, account_id: str, user_id: str, role: str) -> None: - _validate(user_id, validate_user_id) + _validate(user_id, validate_user_id, "user_id") role = _validate_role(role) try: async with self._transaction() as session: @@ -484,7 +495,7 @@ async def create_user_with_api_key( role: str, api_key: str, ) -> None: - _validate(user_id, validate_user_id) + _validate(user_id, validate_user_id, "user_id") role = _validate_role(role) try: async with self._transaction() as session: @@ -652,9 +663,9 @@ async def set_role(self, account_id: str, user_id: str, role: str) -> None: async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> dict[str, int]: for account_id, user_ids in identities.items(): - _validate(account_id, validate_account_id) + _validate(account_id, validate_account_id, "account_id") for user_id in user_ids: - _validate(user_id, validate_user_id) + _validate(user_id, validate_user_id, "user_id") created_accounts = 0 created_users = 0 for account_id, user_ids in identities.items(): @@ -715,6 +726,8 @@ async def begin_deletion( owner_account_id: str, owner_user_id: str, ) -> tuple[DeletionRecord, bool]: + _validate_length(owner_account_id, "owner_account_id") + _validate_length(owner_user_id, "owner_user_id") async with self._transaction() as session: account = await self._lock_account(session, account_id) record = account @@ -743,6 +756,8 @@ async def replace_deletion_task( owner_account_id: str, owner_user_id: str, ) -> DeletionRecord: + _validate_length(owner_account_id, "owner_account_id") + _validate_length(owner_user_id, "owner_user_id") async with self._transaction() as session: account = await self._lock_account(session, account_id) record = account @@ -851,7 +866,11 @@ async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: ] async def create_group(self, account_id: str, group_id: str) -> GroupSummary: - _validate(group_id, lambda value: validate_identifier_part(value, "group_id")) + _validate( + group_id, + lambda value: validate_identifier_part(value, "group_id"), + "group_id", + ) try: async with self._transaction() as session: account = await self._lock_account(session, account_id) diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index ff8d3fcefa..904b7a3fef 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -336,6 +336,73 @@ def test_mysql_requires_resource_id(monkeypatch): MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) +def test_mysql_resource_id_must_fit_schema(monkeypatch): + params = {"user": "unused", "password": "unused", "database": "unused"} + monkeypatch.setenv("OV_RESOURCE_ID", "r" * 255) + MySQLAccountStore(params=params) + + monkeypatch.setenv("OV_RESOURCE_ID", "r" * 256) + with pytest.raises(InvalidArgumentError, match="OV_RESOURCE_ID must not exceed 255"): + MySQLAccountStore(params=params) + + +@pytest.mark.parametrize( + "operation", + [ + pytest.param( + lambda store, value: store.create_account_with_api_key(value, "admin", "key"), + id="account", + ), + pytest.param( + lambda store, value: store.create_account_with_api_key("account", value, "key"), + id="account-admin", + ), + pytest.param( + lambda store, value: store.create_user("account", value, "user"), + id="user", + ), + pytest.param( + lambda store, value: store.ensure_trusted_identities({value: {"user"}}), + id="trusted-account", + ), + pytest.param( + lambda store, value: store.ensure_trusted_identities({"account": {value}}), + id="trusted-user", + ), + pytest.param( + lambda store, value: store.create_group("account", value), + id="group", + ), + pytest.param( + lambda store, value: store.begin_deletion( + "account", + None, + task_id="task", + owner_account_id=value, + owner_user_id="owner", + ), + id="deletion-owner-account", + ), + pytest.param( + lambda store, value: store.begin_deletion( + "account", + None, + task_id="task", + owner_account_id="owner", + owner_user_id=value, + ), + id="deletion-owner-user", + ), + ], +) +async def test_mysql_identifiers_must_fit_schema(monkeypatch, operation): + monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") + store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) + + with pytest.raises(InvalidArgumentError, match="must not exceed 255 characters"): + await operation(store, "a" * 256) + + def test_mysql_rejects_removed_table_parameter(monkeypatch): monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") with pytest.raises(InvalidArgumentError, match="Unknown MySQL account store parameters: table"): From a00dbd1f35c893ae9bfcabdc7a10a7e263601852 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Mon, 12 Oct 2026 11:30:17 +0800 Subject: [PATCH 09/16] test(account-store): keep shared auth coverage in base --- tests/server/test_auth.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index 21d84995c2..dfe92bb2b7 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -1286,18 +1286,13 @@ async def test_trusted_mode_with_root_api_key_accepts_matching_api_key(): assert identity.user_id == "alice" -async def test_trusted_mode_root_key_allows_admin_role_header(auth_app, monkeypatch): +async def test_trusted_mode_root_key_allows_admin_role_header(auth_app): """Trusted upstreams with the root key may assert ADMIN role for tenant routes.""" manager = auth_app.state.api_key_manager account_id = _uid() await manager.create_account(account_id, "owner") await manager.register_user(account_id, "alice", "user") - async def fail_role_lookup(*_args): - raise AssertionError("asserted roles must not read the account store") - - monkeypatch.setattr(manager, "get_registered_user_role", fail_role_lookup) - request = _make_request( "/api/v1/resources", headers={ From 6d61ec7cc21cf05f6c7e54c89f117a386a44bdf6 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Mon, 12 Oct 2026 11:34:15 +0800 Subject: [PATCH 10/16] test(account-store): validate configured provider --- tests/server/test_store_assembly.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/server/test_store_assembly.py b/tests/server/test_store_assembly.py index fbc3925b59..7d8514d9d0 100644 --- a/tests/server/test_store_assembly.py +++ b/tests/server/test_store_assembly.py @@ -27,6 +27,11 @@ def test_unknown_account_store_is_rejected(): ) +def test_unknown_account_store_is_rejected_during_config_validation(): + with pytest.raises(ValueError, match="account_store.provider must be one of"): + AccountStoreConfig(provider="missing") + + @pytest.mark.parametrize("injected", [False, True]) async def test_assembled_manager_supports_account_authentication(injected): account_store = FileStore(SimpleNamespace(agfs=InMemoryAGFS())) From 3d54ab465b370d15ef97f9f75a0bb953f652e8b6 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Mon, 12 Oct 2026 21:53:03 +0800 Subject: [PATCH 11/16] fix(account-store): implement keyless create_account for distributed providers Main's #5933 added the abstract AccountStore.create_account to support trusted keyless identity provisioning. Implement it in the MySQL and Redis/MySQL providers so they remain instantiable and preserve the file provider's keyless-admin creation semantics. --- openviking/server/account_stores/mysql.py | 24 +++++++++++++++++++ .../server/account_stores/redis_mysql.py | 12 ++++++++++ 2 files changed, 36 insertions(+) diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index 36c3a87779..283bcd4e35 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -436,6 +436,30 @@ async def get_user(self, account_id: str, user_id: str) -> UserSummary | None: ) return UserSummary(user_id=user.user_id, role=user.role) if user else None + async def create_account( + self, + account_id: str, + admin_user_id: str, + ) -> None: + _validate(account_id, validate_account_id, "account_id") + _validate(admin_user_id, validate_user_id, "admin_user_id") + try: + async with self._transaction() as session: + account = Account(resource_id=self._resource_id, account_id=account_id) + session.add(account) + await session.flush() + session.add( + User( + resource_id=self._resource_id, + account_pk=account.account_pk, + user_id=admin_user_id, + role=Role.ADMIN, + ) + ) + await session.flush() + except IntegrityError as exc: + raise AlreadyExistsError(account_id, "account") from exc + async def create_account_with_api_key( self, account_id: str, diff --git a/openviking/server/account_stores/redis_mysql.py b/openviking/server/account_stores/redis_mysql.py index 2517c729cc..307aa62690 100644 --- a/openviking/server/account_stores/redis_mysql.py +++ b/openviking/server/account_stores/redis_mysql.py @@ -683,6 +683,18 @@ async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: async def get_account(self, account_id: str) -> AccountSummary | None: return await self._mysql.get_account(account_id) + async def create_account( + self, + account_id: str, + admin_user_id: str, + ) -> None: + await self._mutate_account( + account_id, + lambda: self._mysql.create_account(account_id, admin_user_id), + lambda: self._user_cache_keys(account_id, admin_user_id), + account_wide=True, + ) + async def create_account_with_api_key( self, account_id: str, From 6407d9604854dccca67c8d6186729213fc698f05 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Tue, 13 Oct 2026 20:57:00 +0800 Subject: [PATCH 12/16] fix(account-store): isolate providers by service name --- docs/en/guides/01-configuration.md | 13 +- docs/zh/guides/01-configuration.md | 11 +- openviking/server/account_stores/mysql.py | 138 +++++++++--------- .../server/account_stores/mysql_schema.py | 70 ++++----- .../server/account_stores/redis_mysql.py | 23 ++- openviking/server/account_stores/sql/init.sql | 64 ++++---- openviking/server/app.py | 1 + openviking/server/auth/plugins/api_key.py | 1 + openviking/server/auth/plugins/trusted.py | 1 + openviking/server/store_assembly.py | 17 ++- tests/server/test_account_services.py | 75 +++++++--- tests/server/test_auth.py | 7 + .../server/test_redis_mysql_account_store.py | 100 ++++++++++++- tests/server/test_store_assembly.py | 18 +++ 14 files changed, 359 insertions(+), 180 deletions(-) diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index cc15aece6c..0dec0a7616 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1925,21 +1925,22 @@ encrypted storage to hash storage, retain `credential_encryption_key` until all encrypted credentials have been rotated. Replacing that encryption key directly is not supported because `aesgcm:v1` records do not carry a key identifier. -Install the MySQL extra and set a non-empty `OV_RESOURCE_ID` before running the -server. This value is stored as `resource_id` on every account-store table and -is the deployment boundary when multiple OpenViking resources share a database. -The provider rejects startup when it is absent. +Install the MySQL extra and configure the top-level `service_name`. Its value is +stored in the `service_name` column on every account-store table and is the +deployment boundary when multiple OpenViking clusters share a database. Changing +`service_name` selects a different logical account-store partition. The provider never creates or migrates database tables. Create the normalized tables in `openviking/server/account_stores/sql/init.sql` through your normal DBA or deployment process before starting the server. Future reviewed schema changes are stored as ordered SQL files in `openviking/server/account_stores/sql/changes/`. The five `ov_accounts*` table -names are fixed; use `OV_RESOURCE_ID` rather than separate table prefixes to -isolate OpenViking deployments that share a database. +names are fixed; use distinct top-level `service_name` values rather than +separate table prefixes to isolate OpenViking clusters that share a database. ```json { + "service_name": "production", "server": { "account_store": { "provider": "mysql", diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index 054426ee0a..bd1128326f 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1909,18 +1909,19 @@ prefix；加密模式可为同一列表工作流解密展示 key。`credential_s 迁移到哈希模式时，应保留 `credential_encryption_key`，直到所有旧加密凭证完成 轮换。`aesgcm:v1` 记录不包含 key id，因此不支持直接替换该加密密钥。 -安装 MySQL extra 后，启动服务前必须设置非空的 `OV_RESOURCE_ID`。该值会作为 -每张 Account Store 表上的 `resource_id` 写入，并在多个 OpenViking resource -共用一个数据库时充当部署隔离边界。未设置时 Provider 会拒绝启动。 +安装 MySQL extra 后，通过顶层 `service_name` 配置部署名称。该值会写入每张 +Account Store 表的 `service_name` 列，并在多个 OpenViking 集群共用一个数据库时 +充当隔离边界。修改 `service_name` 会选择另一组逻辑 Account Store 分区。 Provider 不会创建或迁移数据库表。启动服务前，应通过现有 DBA 或部署流程手工创建 `openviking/server/account_stores/sql/init.sql` 中定义的规范化表。后续经评审的 schema 变更按顺序存放于 `openviking/server/account_stores/sql/changes/`。 -五张 `ov_accounts*` 表的名称固定；多个 OpenViking resource 共用数据库时， -通过 `OV_RESOURCE_ID` 隔离部署，不使用不同表前缀。 +五张 `ov_accounts*` 表的名称固定；多个 OpenViking 集群共用数据库时，通过不同的 +顶层 `service_name` 隔离部署，不使用不同表前缀。 ```json { + "service_name": "production", "server": { "account_store": { "provider": "mysql", diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index 283bcd4e35..b86c21deda 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -8,7 +8,6 @@ import binascii import hashlib import hmac -import os import secrets import ssl as ssl_module from contextlib import asynccontextmanager @@ -55,6 +54,7 @@ validate_identifier_part, validate_user_id, ) +from openviking_cli.utils.config.config_utils import normalize_service_name _MYSQL_IDENTIFIER_MAX_LENGTH = 255 @@ -121,14 +121,20 @@ def _deletion(record) -> DeletionRecord | None: class MySQLAccountStore(AccountStore): - """A manually provisioned relational account store scoped by ``resource_id``.""" + """A manually provisioned relational account store scoped by service name.""" - def __init__(self, *, params: dict[str, object] | None = None) -> None: + def __init__( + self, + *, + service_name: str, + params: dict[str, object] | None = None, + ) -> None: values = dict(params or {}) - self._resource_id = os.environ.get("OV_RESOURCE_ID", "").strip() - if not self._resource_id: - raise InvalidArgumentError("OV_RESOURCE_ID is required for MySQL account storage") - _validate_length(self._resource_id, "OV_RESOURCE_ID") + try: + self._service_name = normalize_service_name(service_name) + except ValueError as exc: + raise InvalidArgumentError(str(exc)) from None + _validate_length(self._service_name, "service_name") credential_storage = values.pop("credential_storage", "hash") encryption_key = values.pop("credential_encryption_key", None) @@ -214,7 +220,7 @@ def _new_credential( ): return Credential( credential_id=uuid4().hex, - resource_id=self._resource_id, + service_name=self._service_name, user_pk=user_pk, active_user_pk=user_pk, material=self._encode_credential(account_id, user_id, api_key), @@ -339,7 +345,7 @@ async def _lock_account(self, session: AsyncSession, account_id: str): account = await session.scalar( select(Account) .where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, ) .with_for_update() @@ -352,7 +358,7 @@ async def _lock_user(self, session: AsyncSession, account, user_id: str): user = await session.scalar( select(User) .where( - User.resource_id == self._resource_id, + User.service_name == self._service_name, User.account_pk == account.account_pk, User.user_id == user_id, ) @@ -377,7 +383,7 @@ async def _protect_last_admin(self, session: AsyncSession, account, user) -> Non select(func.count()) .select_from(User) .where( - User.resource_id == self._resource_id, + User.service_name == self._service_name, User.account_pk == account.account_pk, User.role == Role.ADMIN, User.deletion_task_id.is_(None), @@ -404,12 +410,12 @@ async def get_account(self, account_id: str) -> AccountSummary | None: .outerjoin( User, and_( - User.resource_id == Account.resource_id, + User.service_name == Account.service_name, User.account_pk == Account.account_pk, ), ) .where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, ) .group_by(Account.account_pk) @@ -424,12 +430,12 @@ async def get_user(self, account_id: str, user_id: str) -> UserSummary | None: .join( Account, and_( - Account.resource_id == User.resource_id, + Account.service_name == User.service_name, Account.account_pk == User.account_pk, ), ) .where( - User.resource_id == self._resource_id, + User.service_name == self._service_name, Account.account_id == account_id, User.user_id == user_id, ) @@ -445,12 +451,12 @@ async def create_account( _validate(admin_user_id, validate_user_id, "admin_user_id") try: async with self._transaction() as session: - account = Account(resource_id=self._resource_id, account_id=account_id) + account = Account(service_name=self._service_name, account_id=account_id) session.add(account) await session.flush() session.add( User( - resource_id=self._resource_id, + service_name=self._service_name, account_pk=account.account_pk, user_id=admin_user_id, role=Role.ADMIN, @@ -470,11 +476,11 @@ async def create_account_with_api_key( _validate(admin_user_id, validate_user_id, "admin_user_id") try: async with self._transaction() as session: - account = Account(resource_id=self._resource_id, account_id=account_id) + account = Account(service_name=self._service_name, account_id=account_id) session.add(account) await session.flush() user = User( - resource_id=self._resource_id, + service_name=self._service_name, account_pk=account.account_pk, user_id=admin_user_id, role=Role.ADMIN, @@ -502,7 +508,7 @@ async def create_user(self, account_id: str, user_id: str, role: str) -> None: self._require_active(account) session.add( User( - resource_id=self._resource_id, + service_name=self._service_name, account_pk=account.account_pk, user_id=user_id, role=role, @@ -526,7 +532,7 @@ async def create_user_with_api_key( account = await self._lock_account(session, account_id) self._require_active(account) user = User( - resource_id=self._resource_id, + service_name=self._service_name, account_pk=account.account_pk, user_id=user_id, role=role, @@ -556,7 +562,7 @@ async def list_accounts( page: int = 1, query_filter: str | None = None, ) -> list[AccountSummary]: - predicates = [Account.resource_id == self._resource_id] + predicates = [Account.service_name == self._service_name] if name_filter: predicates.append(Account.account_id.like(_like_pattern(name_filter), escape="=")) if query := (query_filter or "").strip(): @@ -571,7 +577,7 @@ async def list_accounts( .outerjoin( User, and_( - User.resource_id == Account.resource_id, + User.service_name == Account.service_name, User.account_pk == Account.account_pk, ), ) @@ -601,7 +607,7 @@ async def list_users_page( await session.connection(execution_options={"isolation_level": "REPEATABLE READ"}) account = await self._account_for_read(session, account_id) base = [ - User.resource_id == self._resource_id, + User.service_name == self._service_name, User.account_pk == account.account_pk, User.deletion_task_id.is_(None), ] @@ -619,7 +625,7 @@ async def list_users_page( .join( User, and_( - User.resource_id == Credential.resource_id, + User.service_name == Credential.service_name, User.user_pk == Credential.user_pk, ), ) @@ -649,7 +655,7 @@ async def list_users_page( for credential in ( await session.scalars( select(Credential).where( - Credential.resource_id == self._resource_id, + Credential.service_name == self._service_name, Credential.user_pk.in_([user.user_pk for user in users]), Credential.revoked.is_(False), ) @@ -699,7 +705,7 @@ async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> di account = await session.scalar( select(Account) .where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, ) .with_for_update() @@ -708,7 +714,7 @@ async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> di try: async with session.begin_nested(): account = Account( - resource_id=self._resource_id, + service_name=self._service_name, account_id=account_id, ) session.add(account) @@ -722,7 +728,7 @@ async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> di ( await session.scalars( select(User.user_id).where( - User.resource_id == self._resource_id, + User.service_name == self._service_name, User.account_pk == account.account_pk, User.user_id.in_(user_ids), ) @@ -732,7 +738,7 @@ async def ensure_trusted_identities(self, identities: dict[str, set[str]]) -> di for user_id in sorted(user_ids - existing): session.add( User( - resource_id=self._resource_id, + service_name=self._service_name, account_pk=account.account_pk, user_id=user_id, role=Role.USER, @@ -829,7 +835,7 @@ async def get_deletion( async with self._session_factory() as session: account = await session.scalar( select(Account).where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, ) ) @@ -842,13 +848,13 @@ async def get_deletion( .outerjoin( User, and_( - User.resource_id == Account.resource_id, + User.service_name == Account.service_name, User.account_pk == Account.account_pk, User.user_id == user_id, ), ) .where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, ) ) @@ -863,7 +869,7 @@ async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: accounts = ( await session.scalars( select(Account).where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.deletion_task_id.is_not(None), ) ) @@ -874,12 +880,12 @@ async def iter_deletions(self) -> list[tuple[str, str | None, DeletionRecord]]: .join( User, and_( - User.resource_id == Account.resource_id, + User.service_name == Account.service_name, User.account_pk == Account.account_pk, ), ) .where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.deletion_task_id.is_(None), User.deletion_task_id.is_not(None), ) @@ -901,7 +907,7 @@ async def create_group(self, account_id: str, group_id: str) -> GroupSummary: self._require_active(account) session.add( Group( - resource_id=self._resource_id, + service_name=self._service_name, account_pk=account.account_pk, group_id=group_id, ) @@ -914,7 +920,7 @@ async def create_group(self, account_id: str, group_id: str) -> GroupSummary: async def _account_for_read(self, session: AsyncSession, account_id: str): account = await session.scalar( select(Account).where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, ) ) @@ -931,13 +937,13 @@ async def get_groups(self, account_id: str) -> list[GroupSummary]: .outerjoin( GroupMember, and_( - GroupMember.resource_id == Group.resource_id, + GroupMember.service_name == Group.service_name, GroupMember.account_pk == Group.account_pk, GroupMember.group_pk == Group.group_pk, ), ) .where( - Group.resource_id == self._resource_id, + Group.service_name == self._service_name, Group.account_pk == account.account_pk, ) .group_by(Group.group_pk) @@ -951,7 +957,7 @@ async def get_group_members(self, account_id: str, group_id: str) -> list[str]: account = await self._account_for_read(session, account_id) group = await session.scalar( select(Group).where( - Group.resource_id == self._resource_id, + Group.service_name == self._service_name, Group.account_pk == account.account_pk, Group.group_id == group_id, ) @@ -965,13 +971,13 @@ async def get_group_members(self, account_id: str, group_id: str) -> list[str]: .join( GroupMember, and_( - GroupMember.resource_id == User.resource_id, + GroupMember.service_name == User.service_name, GroupMember.account_pk == User.account_pk, GroupMember.user_pk == User.user_pk, ), ) .where( - GroupMember.resource_id == self._resource_id, + GroupMember.service_name == self._service_name, GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, ) @@ -989,14 +995,14 @@ async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, .join( User, and_( - User.resource_id == Account.resource_id, + User.service_name == Account.service_name, User.account_pk == Account.account_pk, ), ) .outerjoin( GroupMember, and_( - GroupMember.resource_id == User.resource_id, + GroupMember.service_name == User.service_name, GroupMember.account_pk == User.account_pk, GroupMember.user_pk == User.user_pk, ), @@ -1004,13 +1010,13 @@ async def get_user_group_ids(self, account_id: str, user_id: str) -> tuple[str, .outerjoin( Group, and_( - Group.resource_id == GroupMember.resource_id, + Group.service_name == GroupMember.service_name, Group.account_pk == GroupMember.account_pk, Group.group_pk == GroupMember.group_pk, ), ) .where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, User.user_id == user_id, ) @@ -1026,7 +1032,7 @@ async def add_group_member(self, account_id: str, group_id: str, user_id: str) - group = await session.scalar( select(Group) .where( - Group.resource_id == self._resource_id, + Group.service_name == self._service_name, Group.account_pk == account.account_pk, Group.group_id == group_id, ) @@ -1039,7 +1045,7 @@ async def add_group_member(self, account_id: str, group_id: str, user_id: str) - if ( await session.scalar( select(GroupMember.group_pk).where( - GroupMember.resource_id == self._resource_id, + GroupMember.service_name == self._service_name, GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, GroupMember.user_pk == user.user_pk, @@ -1049,7 +1055,7 @@ async def add_group_member(self, account_id: str, group_id: str, user_id: str) - ): session.add( GroupMember( - resource_id=self._resource_id, + service_name=self._service_name, account_pk=account.account_pk, group_pk=group.group_pk, user_pk=user.user_pk, @@ -1064,7 +1070,7 @@ async def remove_group_member(self, account_id: str, group_id: str, user_id: str group = await session.scalar( select(Group) .where( - Group.resource_id == self._resource_id, + Group.service_name == self._service_name, Group.account_pk == account.account_pk, Group.group_id == group_id, ) @@ -1077,13 +1083,13 @@ async def remove_group_member(self, account_id: str, group_id: str, user_id: str .join( User, and_( - User.resource_id == GroupMember.resource_id, + User.service_name == GroupMember.service_name, User.account_pk == GroupMember.account_pk, User.user_pk == GroupMember.user_pk, ), ) .where( - GroupMember.resource_id == self._resource_id, + GroupMember.service_name == self._service_name, GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, User.account_pk == account.account_pk, @@ -1102,7 +1108,7 @@ async def delete_group(self, account_id: str, group_id: str) -> None: group = await session.scalar( select(Group) .where( - Group.resource_id == self._resource_id, + Group.service_name == self._service_name, Group.account_pk == account.account_pk, Group.group_id == group_id, ) @@ -1114,7 +1120,7 @@ async def delete_group(self, account_id: str, group_id: str) -> None: select(func.count()) .select_from(GroupMember) .where( - GroupMember.resource_id == self._resource_id, + GroupMember.service_name == self._service_name, GroupMember.account_pk == account.account_pk, GroupMember.group_pk == group.group_pk, ) @@ -1132,14 +1138,14 @@ async def replace_active_user_api_key( self._require_active(user) sequence = await session.scalar( select(func.coalesce(func.max(Credential.sequence), 0)).where( - Credential.resource_id == self._resource_id, + Credential.service_name == self._service_name, Credential.user_pk == user.user_pk, ) ) await session.execute( update(Credential) .where( - Credential.resource_id == self._resource_id, + Credential.service_name == self._service_name, Credential.user_pk == user.user_pk, Credential.revoked.is_(False), ) @@ -1168,7 +1174,7 @@ async def verify_api_key( user_id_hint: str | None = None, ) -> tuple[str, str] | None: predicates = [ - Credential.resource_id == self._resource_id, + Credential.service_name == self._service_name, Credential.lookup_digest == _credential_digest(api_key), Credential.revoked.is_(False), Account.deletion_task_id.is_(None), @@ -1185,14 +1191,14 @@ async def verify_api_key( .join( User, and_( - User.resource_id == Credential.resource_id, + User.service_name == Credential.service_name, User.user_pk == Credential.user_pk, ), ) .join( Account, and_( - Account.resource_id == User.resource_id, + Account.service_name == User.service_name, Account.account_pk == User.account_pk, ), ) @@ -1209,7 +1215,7 @@ async def revoke_active_api_keys(self, account_id: str, user_id: str | None = No account = await session.scalar( select(Account) .where( - Account.resource_id == self._resource_id, + Account.service_name == self._service_name, Account.account_id == account_id, ) .with_for_update() @@ -1217,14 +1223,14 @@ async def revoke_active_api_keys(self, account_id: str, user_id: str | None = No if account is None: return user_ids = select(User.user_pk).where( - User.resource_id == self._resource_id, + User.service_name == self._service_name, User.account_pk == account.account_pk, *([User.user_id == user_id] if user_id is not None else []), ) await session.execute( update(Credential) .where( - Credential.resource_id == self._resource_id, + Credential.service_name == self._service_name, Credential.user_pk.in_(user_ids), Credential.revoked.is_(False), ) @@ -1243,19 +1249,19 @@ async def get_user_key_fingerprint(self, account_id: str, user_id: str) -> str | .join( User, and_( - User.resource_id == Credential.resource_id, + User.service_name == Credential.service_name, User.user_pk == Credential.user_pk, ), ) .join( Account, and_( - Account.resource_id == User.resource_id, + Account.service_name == User.service_name, Account.account_pk == User.account_pk, ), ) .where( - Credential.resource_id == self._resource_id, + Credential.service_name == self._service_name, Credential.revoked.is_(False), Account.account_id == account_id, Account.deletion_task_id.is_(None), diff --git a/openviking/server/account_stores/mysql_schema.py b/openviking/server/account_stores/mysql_schema.py index 9968b3496e..9cd2f62450 100644 --- a/openviking/server/account_stores/mysql_schema.py +++ b/openviking/server/account_stores/mysql_schema.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0 """Fixed SQLAlchemy schema for the MySQL account-store provider. -``resource_id`` is the tenant boundary. It is deliberately present in every +``service_name`` is the tenant boundary. It is deliberately present in every primary lookup, unique key, and foreign-key relationship so one database can hold many OpenViking deployments safely. """ @@ -34,25 +34,25 @@ class Account(Base): __tablename__ = "ov_accounts" __table_args__ = ( UniqueConstraint( - "resource_id", + "service_name", "account_id", name="uq_ov_accounts_account_identity", ), UniqueConstraint( - "resource_id", + "service_name", "account_pk", - name="uq_ov_accounts_account_resource_pk", + name="uq_ov_accounts_account_service_pk", ), Index( "ix_ov_accounts_account_deletion", - "resource_id", + "service_name", "deletion_task_id", ), {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, ) account_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + service_name: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) account_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) created_at: Mapped[object] = mapped_column( DATETIME(fsp=6), server_default=text("CURRENT_TIMESTAMP(6)") @@ -70,44 +70,44 @@ class User(Base): __tablename__ = "ov_accounts_users" __table_args__ = ( ForeignKeyConstraint( - ["resource_id", "account_pk"], - ["ov_accounts.resource_id", "ov_accounts.account_pk"], + ["service_name", "account_pk"], + ["ov_accounts.service_name", "ov_accounts.account_pk"], ondelete="CASCADE", name="fk_ov_accounts_user_account", ), UniqueConstraint( - "resource_id", + "service_name", "account_pk", "user_id", name="uq_ov_accounts_user_identity", ), UniqueConstraint( - "resource_id", + "service_name", "user_pk", - name="uq_ov_accounts_user_resource_pk", + name="uq_ov_accounts_user_service_pk", ), UniqueConstraint( - "resource_id", + "service_name", "account_pk", "user_pk", name="uq_ov_accounts_user_account_pk", ), Index( "ix_ov_accounts_user_admin", - "resource_id", + "service_name", "account_pk", "role", ), Index( "ix_ov_accounts_user_deletion", - "resource_id", + "service_name", "deletion_task_id", ), {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, ) user_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + service_name: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) account_pk: Mapped[int] = mapped_column(BigInteger) user_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) role: Mapped[str] = mapped_column(String(16)) @@ -127,24 +127,24 @@ class Group(Base): __tablename__ = "ov_accounts_groups" __table_args__ = ( ForeignKeyConstraint( - ["resource_id", "account_pk"], - ["ov_accounts.resource_id", "ov_accounts.account_pk"], + ["service_name", "account_pk"], + ["ov_accounts.service_name", "ov_accounts.account_pk"], ondelete="CASCADE", name="fk_ov_accounts_group_account", ), UniqueConstraint( - "resource_id", + "service_name", "account_pk", "group_id", name="uq_ov_accounts_group_identity", ), UniqueConstraint( - "resource_id", + "service_name", "group_pk", - name="uq_ov_accounts_group_resource_pk", + name="uq_ov_accounts_group_service_pk", ), UniqueConstraint( - "resource_id", + "service_name", "account_pk", "group_pk", name="uq_ov_accounts_group_account_pk", @@ -153,7 +153,7 @@ class Group(Base): ) group_pk: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + service_name: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) account_pk: Mapped[int] = mapped_column(BigInteger) group_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) created_at: Mapped[object] = mapped_column( @@ -164,11 +164,11 @@ class Group(Base): class GroupMember(Base): __tablename__ = "ov_accounts_group_members" __table_args__ = ( - PrimaryKeyConstraint("resource_id", "account_pk", "group_pk", "user_pk"), + PrimaryKeyConstraint("service_name", "account_pk", "group_pk", "user_pk"), ForeignKeyConstraint( - ["resource_id", "account_pk", "group_pk"], + ["service_name", "account_pk", "group_pk"], [ - "ov_accounts_groups.resource_id", + "ov_accounts_groups.service_name", "ov_accounts_groups.account_pk", "ov_accounts_groups.group_pk", ], @@ -176,9 +176,9 @@ class GroupMember(Base): name="fk_ov_accounts_member_group", ), ForeignKeyConstraint( - ["resource_id", "account_pk", "user_pk"], + ["service_name", "account_pk", "user_pk"], [ - "ov_accounts_users.resource_id", + "ov_accounts_users.service_name", "ov_accounts_users.account_pk", "ov_accounts_users.user_pk", ], @@ -187,14 +187,14 @@ class GroupMember(Base): ), Index( "ix_ov_accounts_member_user", - "resource_id", + "service_name", "account_pk", "user_pk", ), {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}, ) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + service_name: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) account_pk: Mapped[int] = mapped_column(BigInteger) group_pk: Mapped[int] = mapped_column(BigInteger) user_pk: Mapped[int] = mapped_column(BigInteger) @@ -204,26 +204,26 @@ class Credential(Base): __tablename__ = "ov_accounts_credentials" __table_args__ = ( ForeignKeyConstraint( - ["resource_id", "user_pk"], - ["ov_accounts_users.resource_id", "ov_accounts_users.user_pk"], + ["service_name", "user_pk"], + ["ov_accounts_users.service_name", "ov_accounts_users.user_pk"], ondelete="CASCADE", name="fk_ov_accounts_credential_user", ), UniqueConstraint( - "resource_id", + "service_name", "active_user_pk", name="uq_ov_accounts_active_credential", ), Index( "ix_ov_accounts_credential_lookup", - "resource_id", + "service_name", "lookup_digest", "revoked", "user_pk", ), Index( "ix_ov_accounts_credential_history", - "resource_id", + "service_name", "user_pk", "revoked", "sequence", @@ -232,7 +232,7 @@ class Credential(Base): ) credential_id: Mapped[str] = mapped_column(String(32, collation="ascii_bin"), primary_key=True) - resource_id: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) + service_name: Mapped[str] = mapped_column(String(255, collation="utf8mb4_bin")) user_pk: Mapped[int] = mapped_column(BigInteger) material: Mapped[str] = mapped_column(Text(collation="ascii_bin")) lookup_digest: Mapped[str] = mapped_column(String(64, collation="ascii_bin")) diff --git a/openviking/server/account_stores/redis_mysql.py b/openviking/server/account_stores/redis_mysql.py index 307aa62690..b0954023ba 100644 --- a/openviking/server/account_stores/redis_mysql.py +++ b/openviking/server/account_stores/redis_mysql.py @@ -25,6 +25,7 @@ from openviking.server.account_stores.mysql import MySQLAccountStore from openviking_cli.exceptions import InvalidArgumentError from openviking_cli.utils import get_logger +from openviking_cli.utils.config.config_utils import normalize_service_name logger = get_logger(__name__) T = TypeVar("T") @@ -124,10 +125,15 @@ class RedisMySQLAccountStore(AccountStore): def __init__( self, *, + service_name: str, params: dict[str, object] | None = None, mysql_store: MySQLAccountStore | None = None, redis_client: Any | None = None, ) -> None: + try: + normalized_service_name = normalize_service_name(service_name) + except ValueError as exc: + raise InvalidArgumentError(str(exc)) from None values = dict(params or {}) mysql_params = values.pop("mysql", None) redis_params = values.pop("redis", None) @@ -137,7 +143,14 @@ def __init__( if mysql_store is None: if not isinstance(mysql_params, dict): raise InvalidArgumentError("redis_mysql account storage requires mysql parameters") - mysql_store = MySQLAccountStore(params=mysql_params) + mysql_store = MySQLAccountStore( + service_name=normalized_service_name, + params=mysql_params, + ) + elif mysql_store._service_name != normalized_service_name: + raise InvalidArgumentError( + "Redis and MySQL account stores must use the same service_name" + ) if not isinstance(redis_params, dict): raise InvalidArgumentError("redis_mysql account storage requires redis parameters") @@ -167,7 +180,7 @@ def __init__( self._mysql = mysql_store self._index_ttl_seconds = self._ttl_seconds + self._lock_ttl_seconds - self._resource_id = mysql_store._resource_id + self._service_name = normalized_service_name self._redis = redis_client self._redis_url = url self._owns_redis = redis_client is None @@ -204,7 +217,7 @@ async def close(self) -> None: await self._mysql.close() def _account_tag(self, account_id: str) -> str: - return _key_part(f"{self._resource_id}\0{account_id}") + return _key_part(f"{self._service_name}\0{account_id}") def _key(self, account_id: str, suffix: str) -> str: return f"{self._key_prefix}:v1:{{{self._account_tag(account_id)}}}:{suffix}" @@ -234,11 +247,11 @@ def _user_credential_fence_key(self, account_id: str, user_id: str) -> str: return self._key(account_id, f"credential-fence:user:{_key_part(user_id)}") def _binding_key(self, account_id: str, api_key: str) -> str: - digest_input = self._resource_id + "\0" + api_key + digest_input = self._service_name + "\0" + api_key return self._key(account_id, f"binding:{_digest(digest_input)}") def _negative_binding_key(self, account_id: str, api_key: str, user_id_hint: str | None) -> str: - digest_input = self._resource_id + "\0" + api_key + "\0" + (user_id_hint or "") + digest_input = self._service_name + "\0" + api_key + "\0" + (user_id_hint or "") return self._key(account_id, f"binding-miss:{_digest(digest_input)}") def _binding_cache_keys(self, account_id: str, api_key: str, user_id: str) -> tuple[str, ...]: diff --git a/openviking/server/account_stores/sql/init.sql b/openviking/server/account_stores/sql/init.sql index f179ebe0b4..3fae25d302 100644 --- a/openviking/server/account_stores/sql/init.sql +++ b/openviking/server/account_stores/sql/init.sql @@ -1,24 +1,24 @@ -- OpenViking MySQL account store initial schema. -- Execute manually before starting a MySQL account-store deployment. --- Table names are fixed and resource_id isolates OpenViking deployments. +-- Table names are fixed and service_name isolates OpenViking deployments. CREATE TABLE `ov_accounts` ( account_pk BIGINT NOT NULL AUTO_INCREMENT, - resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + service_name VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, account_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), deletion_task_id VARCHAR(64) NULL, deletion_owner_account_id VARCHAR(255) COLLATE utf8mb4_bin NULL, deletion_owner_user_id VARCHAR(255) COLLATE utf8mb4_bin NULL, PRIMARY KEY (account_pk), - CONSTRAINT uq_ov_accounts_account_identity UNIQUE (resource_id, account_id), - CONSTRAINT uq_ov_accounts_account_resource_pk UNIQUE (resource_id, account_pk), - INDEX ix_ov_accounts_account_deletion (resource_id, deletion_task_id) + CONSTRAINT uq_ov_accounts_account_identity UNIQUE (service_name, account_id), + CONSTRAINT uq_ov_accounts_account_service_pk UNIQUE (service_name, account_pk), + INDEX ix_ov_accounts_account_deletion (service_name, deletion_task_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_users` ( user_pk BIGINT NOT NULL AUTO_INCREMENT, - resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + service_name VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, account_pk BIGINT NOT NULL, user_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, `role` VARCHAR(16) NOT NULL, @@ -28,56 +28,56 @@ CREATE TABLE `ov_accounts_users` ( deletion_owner_user_id VARCHAR(255) COLLATE utf8mb4_bin NULL, PRIMARY KEY (user_pk), CONSTRAINT fk_ov_accounts_user_account - FOREIGN KEY (resource_id, account_pk) - REFERENCES `ov_accounts` (resource_id, account_pk) + FOREIGN KEY (service_name, account_pk) + REFERENCES `ov_accounts` (service_name, account_pk) ON DELETE CASCADE, CONSTRAINT uq_ov_accounts_user_identity - UNIQUE (resource_id, account_pk, user_id), - CONSTRAINT uq_ov_accounts_user_resource_pk UNIQUE (resource_id, user_pk), + UNIQUE (service_name, account_pk, user_id), + CONSTRAINT uq_ov_accounts_user_service_pk UNIQUE (service_name, user_pk), CONSTRAINT uq_ov_accounts_user_account_pk - UNIQUE (resource_id, account_pk, user_pk), - INDEX ix_ov_accounts_user_admin (resource_id, account_pk, `role`), - INDEX ix_ov_accounts_user_deletion (resource_id, deletion_task_id) + UNIQUE (service_name, account_pk, user_pk), + INDEX ix_ov_accounts_user_admin (service_name, account_pk, `role`), + INDEX ix_ov_accounts_user_deletion (service_name, deletion_task_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_groups` ( group_pk BIGINT NOT NULL AUTO_INCREMENT, - resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + service_name VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, account_pk BIGINT NOT NULL, group_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), PRIMARY KEY (group_pk), CONSTRAINT fk_ov_accounts_group_account - FOREIGN KEY (resource_id, account_pk) - REFERENCES `ov_accounts` (resource_id, account_pk) + FOREIGN KEY (service_name, account_pk) + REFERENCES `ov_accounts` (service_name, account_pk) ON DELETE CASCADE, CONSTRAINT uq_ov_accounts_group_identity - UNIQUE (resource_id, account_pk, group_id), - CONSTRAINT uq_ov_accounts_group_resource_pk UNIQUE (resource_id, group_pk), + UNIQUE (service_name, account_pk, group_id), + CONSTRAINT uq_ov_accounts_group_service_pk UNIQUE (service_name, group_pk), CONSTRAINT uq_ov_accounts_group_account_pk - UNIQUE (resource_id, account_pk, group_pk) + UNIQUE (service_name, account_pk, group_pk) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_group_members` ( - resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + service_name VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, account_pk BIGINT NOT NULL, group_pk BIGINT NOT NULL, user_pk BIGINT NOT NULL, - PRIMARY KEY (resource_id, account_pk, group_pk, user_pk), + PRIMARY KEY (service_name, account_pk, group_pk, user_pk), CONSTRAINT fk_ov_accounts_member_group - FOREIGN KEY (resource_id, account_pk, group_pk) - REFERENCES `ov_accounts_groups` (resource_id, account_pk, group_pk) + FOREIGN KEY (service_name, account_pk, group_pk) + REFERENCES `ov_accounts_groups` (service_name, account_pk, group_pk) ON DELETE CASCADE, CONSTRAINT fk_ov_accounts_member_user - FOREIGN KEY (resource_id, account_pk, user_pk) - REFERENCES `ov_accounts_users` (resource_id, account_pk, user_pk) + FOREIGN KEY (service_name, account_pk, user_pk) + REFERENCES `ov_accounts_users` (service_name, account_pk, user_pk) ON DELETE CASCADE, - INDEX ix_ov_accounts_member_user (resource_id, account_pk, user_pk) + INDEX ix_ov_accounts_member_user (service_name, account_pk, user_pk) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `ov_accounts_credentials` ( credential_id VARCHAR(32) COLLATE ascii_bin NOT NULL, - resource_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, + service_name VARCHAR(255) COLLATE utf8mb4_bin NOT NULL, user_pk BIGINT NOT NULL, material TEXT COLLATE ascii_bin NOT NULL, lookup_digest VARCHAR(64) COLLATE ascii_bin NOT NULL, @@ -90,13 +90,13 @@ CREATE TABLE `ov_accounts_credentials` ( active_user_pk BIGINT NULL, PRIMARY KEY (credential_id), CONSTRAINT fk_ov_accounts_credential_user - FOREIGN KEY (resource_id, user_pk) - REFERENCES `ov_accounts_users` (resource_id, user_pk) + FOREIGN KEY (service_name, user_pk) + REFERENCES `ov_accounts_users` (service_name, user_pk) ON DELETE CASCADE, CONSTRAINT uq_ov_accounts_active_credential - UNIQUE (resource_id, active_user_pk), + UNIQUE (service_name, active_user_pk), INDEX ix_ov_accounts_credential_lookup - (resource_id, lookup_digest, revoked, user_pk), + (service_name, lookup_digest, revoked, user_pk), INDEX ix_ov_accounts_credential_history - (resource_id, user_pk, revoked, sequence) + (service_name, user_pk, revoked, sequence) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/openviking/server/app.py b/openviking/server/app.py index 5c0a8c6278..f74e3078f4 100644 --- a/openviking/server/app.py +++ b/openviking/server/app.py @@ -452,6 +452,7 @@ async def _oauth_gc_loop(store) -> None: # noqa: ANN001 ) app.state.config = config + app.state.service_name = get_openviking_config().service_name app.state.server_config_overrides = server_overrides or {} from openviking.server.restart import RestartController diff --git a/openviking/server/auth/plugins/api_key.py b/openviking/server/auth/plugins/api_key.py index 75d44bb926..48345216d2 100644 --- a/openviking/server/auth/plugins/api_key.py +++ b/openviking/server/auth/plugins/api_key.py @@ -181,6 +181,7 @@ async def initialize(self, app, service, config) -> None: root_key=config.root_api_key, viking_fs=service.viking_fs, api_key_hashing_enabled=config.api_key_hashing_enabled, + service_name=app.state.service_name, account_store_provider=config.account_store.provider, account_store_params=config.account_store.params, account_store_watch_enabled=config.api_key_watch_enabled, diff --git a/openviking/server/auth/plugins/trusted.py b/openviking/server/auth/plugins/trusted.py index 4b87d08293..08a493f6a6 100644 --- a/openviking/server/auth/plugins/trusted.py +++ b/openviking/server/auth/plugins/trusted.py @@ -204,6 +204,7 @@ async def initialize(self, app, service, config) -> None: root_key=config.root_api_key, viking_fs=service.viking_fs, api_key_hashing_enabled=config.api_key_hashing_enabled, + service_name=app.state.service_name, account_store_provider=config.account_store.provider, account_store_params=config.account_store.params, account_store_watch_enabled=config.api_key_watch_enabled, diff --git a/openviking/server/store_assembly.py b/openviking/server/store_assembly.py index 0e636acd97..4fbbd0dd7a 100644 --- a/openviking/server/store_assembly.py +++ b/openviking/server/store_assembly.py @@ -20,6 +20,7 @@ def build_api_key_manager( viking_fs: VikingFS | None, *, api_key_hashing_enabled: bool = False, + service_name: str | None = None, account_store_provider: str = "file", account_store_params: dict[str, object] | None = None, account_store_watch_enabled: bool = False, @@ -34,6 +35,7 @@ def build_api_key_manager( store = build_account_store( viking_fs=viking_fs, api_key_hashing_enabled=api_key_hashing_enabled, + service_name=service_name, account_store_provider=provider, account_store_params=account_store_params, account_store_watch_enabled=account_store_watch_enabled, @@ -47,6 +49,7 @@ def build_account_store( *, viking_fs: VikingFS | None, api_key_hashing_enabled: bool, + service_name: str | None = None, account_store_provider: str, account_store_params: dict[str, object] | None, account_store_watch_enabled: bool, @@ -58,6 +61,10 @@ def build_account_store( return account_store provider = account_store_provider.strip().lower() + if provider in {"mysql", "redis_mysql"} and service_name is None: + raise InvalidArgumentError( + f"service_name is required for the {provider} account store provider" + ) if provider == "file": file_system = cast(VikingFS, viking_fs) return FileStore( @@ -74,7 +81,10 @@ def build_account_store( raise RuntimeError( "MySQL account storage requires openviking[mysql]" ) from None - return MySQLAccountStore(params=account_store_params) + return MySQLAccountStore( + service_name=service_name, + params=account_store_params, + ) if provider == "redis_mysql": try: from openviking.server.account_stores.redis_mysql import ( @@ -84,7 +94,10 @@ def build_account_store( raise RuntimeError( "Redis MySQL account storage requires openviking[redis-mysql]" ) from None - return RedisMySQLAccountStore(params=account_store_params) + return RedisMySQLAccountStore( + service_name=service_name, + params=account_store_params, + ) raise InvalidArgumentError( f"Unknown account store provider '{account_store_provider}'. " "Available providers: file, mysql, redis_mysql" diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index 904b7a3fef..1ba114dc0d 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -330,20 +330,21 @@ def read(self, path, ctx=None): assert (await store.get_deletion("acme", "bob"))["task_id"] == "delete" -def test_mysql_requires_resource_id(monkeypatch): - monkeypatch.delenv("OV_RESOURCE_ID", raising=False) - with pytest.raises(InvalidArgumentError, match="OV_RESOURCE_ID is required"): - MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) +def test_mysql_uses_service_name_instead_of_resource_id_env(monkeypatch): + monkeypatch.setenv("OV_RESOURCE_ID", "ignored-resource") + store = MySQLAccountStore( + service_name="account-cluster", + params={"user": "unused", "password": "unused", "database": "unused"}, + ) + assert store._service_name == "account-cluster" -def test_mysql_resource_id_must_fit_schema(monkeypatch): +def test_mysql_service_name_must_fit_schema(): params = {"user": "unused", "password": "unused", "database": "unused"} - monkeypatch.setenv("OV_RESOURCE_ID", "r" * 255) - MySQLAccountStore(params=params) + MySQLAccountStore(service_name="r" * 255, params=params) - monkeypatch.setenv("OV_RESOURCE_ID", "r" * 256) - with pytest.raises(InvalidArgumentError, match="OV_RESOURCE_ID must not exceed 255"): - MySQLAccountStore(params=params) + with pytest.raises(InvalidArgumentError, match="service_name must not exceed 255"): + MySQLAccountStore(service_name="r" * 256, params=params) @pytest.mark.parametrize( @@ -395,30 +396,34 @@ def test_mysql_resource_id_must_fit_schema(monkeypatch): ), ], ) -async def test_mysql_identifiers_must_fit_schema(monkeypatch, operation): - monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") - store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) +async def test_mysql_identifiers_must_fit_schema(operation): + store = MySQLAccountStore( + service_name="test-service", + params={"user": "unused", "password": "unused", "database": "unused"}, + ) with pytest.raises(InvalidArgumentError, match="must not exceed 255 characters"): await operation(store, "a" * 256) -def test_mysql_rejects_removed_table_parameter(monkeypatch): - monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") +def test_mysql_rejects_removed_table_parameter(): with pytest.raises(InvalidArgumentError, match="Unknown MySQL account store parameters: table"): MySQLAccountStore( + service_name="test-service", params={ "user": "unused", "password": "unused", "database": "unused", "table": "tenant_accounts", - } + }, ) async def test_mysql_load_rejects_incomplete_schema_and_disposes_engine(monkeypatch): - monkeypatch.setenv("OV_RESOURCE_ID", "test-resource") - store = MySQLAccountStore(params={"user": "unused", "password": "unused", "database": "unused"}) + store = MySQLAccountStore( + service_name="test-service", + params={"user": "unused", "password": "unused", "database": "unused"}, + ) class FakeEngine: disposed = False @@ -463,11 +468,11 @@ async def test_file_store_rejects_deleting_nonempty_group(): assert await store.get_group_members("acme", "engineering") == ["alice"] -async def test_mysql_managers_share_authoritative_state(monkeypatch): +async def test_mysql_managers_share_authoritative_state(): raw = os.environ.get("OV_TEST_MYSQL") if not raw: pytest.skip("Set OV_TEST_MYSQL to connection JSON for the MySQL test") - monkeypatch.setenv("OV_RESOURCE_ID", f"test-{uuid4().hex}") + service_name = f"test-{uuid4().hex}" encryption_key = base64.urlsafe_b64encode(b"k" * 32).decode() params = { **json.loads(raw), @@ -476,12 +481,22 @@ async def test_mysql_managers_share_authoritative_state(monkeypatch): first = build_api_key_manager( "root", None, + service_name=service_name, account_store_provider="mysql", account_store_params={**params, "credential_storage": "encrypted"}, ) second = build_api_key_manager( "root", None, + service_name=service_name, + account_store_provider="mysql", + account_store_params={**params, "credential_storage": "hash"}, + ) + isolated_service_name = f"test-{uuid4().hex}" + isolated = build_api_key_manager( + "root", + None, + service_name=isolated_service_name, account_store_provider="mysql", account_store_params={**params, "credential_storage": "hash"}, ) @@ -489,11 +504,12 @@ async def test_mysql_managers_share_authoritative_state(monkeypatch): async with setup_engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) await connection.execute( - delete(Account).where(Account.resource_id == first._store._resource_id) + delete(Account).where(Account.service_name.in_([service_name, isolated_service_name])) ) await setup_engine.dispose() await first.load() await second.load() + await isolated.load() try: trusted_results = await asyncio.gather( first.ensure_trusted_identities({"trusted": {"alice", "bob"}}), @@ -543,10 +559,25 @@ async def test_mysql_managers_share_authoritative_state(monkeypatch): await first.create_group("acme", "engineering") await first.add_group_member("acme", "engineering", "alice") assert await second.get_user_group_ids("acme", "alice") == ("engineering",) + + assert await isolated.get_account("acme") is None + isolated_key = await isolated.create_account("acme", "alice") + assert (await isolated.resolve_identity(isolated_key)).role == Role.ADMIN + with pytest.raises(UnauthenticatedError): + await isolated.resolve_identity(rotated) + with pytest.raises(UnauthenticatedError): + await first.resolve_identity(isolated_key) + await isolated.register_user("acme", "dave") + await isolated.set_role("acme", "dave", Role.ADMIN) + assert await isolated.get_registered_user_role("acme", "dave") == Role.ADMIN + assert await first.get_registered_user_role("acme", "dave") == Role.USER finally: async with first._store._engine.begin() as connection: await connection.execute( - delete(Account).where(Account.resource_id == first._store._resource_id) + delete(Account).where( + Account.service_name.in_([service_name, isolated_service_name]) + ) ) + await isolated.close() await second.close() await first.close() diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index dfe92bb2b7..a0cb291b29 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -55,6 +55,7 @@ def write(self, path: str, data): return "OK" def read(self, path: str, offset: int = 0, size: int = -1, stream: bool = False): + del stream if path not in self.files: raise FileNotFoundError(path) data = self.files[path] @@ -1006,6 +1007,7 @@ async def test_trusted_identity_registration_batches_data_plane_identity(auth_se for restarted in (False, True): app = FastAPI() app.state.config = config + app.state.service_name = "openviking-server" plugin = TrustedAuthPlugin() await plugin.initialize(app, auth_service, config) try: @@ -1043,6 +1045,7 @@ async def test_invalid_trusted_identity_does_not_block_later_registration( ) app = FastAPI() app.state.config = config + app.state.service_name = "openviking-server" plugin = TrustedAuthPlugin() await plugin.initialize(app, auth_service, config) try: @@ -1095,6 +1098,7 @@ async def test_trusted_identity_registration_skips_identity_loaded_from_registry config = ServerConfig(auth_mode="trusted", root_api_key=ROOT_KEY) app = FastAPI() app.state.config = config + app.state.service_name = "openviking-server" first_plugin = TrustedAuthPlugin() await first_plugin.initialize(app, auth_service, config) try: @@ -1104,6 +1108,7 @@ async def test_trusted_identity_registration_skips_identity_loaded_from_registry restarted_app = FastAPI() restarted_app.state.config = config + restarted_app.state.service_name = "openviking-server" restarted_plugin = TrustedAuthPlugin() await restarted_plugin.initialize(restarted_app, auth_service, config) try: @@ -1142,6 +1147,7 @@ async def test_trusted_registration_respects_configuration_and_route(auth_servic ) app = FastAPI() app.state.config = config + app.state.service_name = "openviking-server" plugin = TrustedAuthPlugin() await plugin.initialize(app, auth_service, config) try: @@ -1182,6 +1188,7 @@ async def test_trusted_identity_registration_retries_failed_batch_without_exceed ) app = FastAPI() app.state.config = config + app.state.service_name = "openviking-server" plugin = TrustedAuthPlugin() await plugin.initialize(app, auth_service, config) try: diff --git a/tests/server/test_redis_mysql_account_store.py b/tests/server/test_redis_mysql_account_store.py index 6edfdc5a1f..e7513dc639 100644 --- a/tests/server/test_redis_mysql_account_store.py +++ b/tests/server/test_redis_mysql_account_store.py @@ -3,12 +3,19 @@ import asyncio import base64 +import json +import os +from uuid import uuid4 import pytest +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import create_async_engine +from openviking.server.account_stores.mysql_schema import Account, Base from openviking.server.account_stores.redis_mysql import RedisMySQLAccountStore from openviking.server.api_keys.new import APIKeyManager from openviking.server.identity import Role +from openviking.server.store_assembly import build_api_key_manager from openviking_cli.exceptions import InvalidArgumentError, UnauthenticatedError @@ -87,8 +94,8 @@ async def eval(self, script: str, numkeys: int, *args): class FakeMySQLAuthority: - def __init__(self, api_key: str, resource_id: str = "test-resource"): - self._resource_id = resource_id + def __init__(self, api_key: str, service_name: str = "test-service"): + self._service_name = service_name self.api_key = api_key self.account_exists = True self.user_exists = True @@ -200,10 +207,11 @@ async def begin_deletion(self, account_id: str, user_id: str | None, **_kwargs): def _store( - api_key: str, *, redis_client=None, lock_wait_seconds: int = 30, resource_id="test-resource" + api_key: str, *, redis_client=None, lock_wait_seconds: int = 30, service_name="test-service" ) -> tuple[RedisMySQLAccountStore, FakeMySQLAuthority]: - mysql = FakeMySQLAuthority(api_key, resource_id) + mysql = FakeMySQLAuthority(api_key, service_name) store = RedisMySQLAccountStore( + service_name=service_name, params={ "redis": { "url": "redis://unused", @@ -237,17 +245,18 @@ def test_redis_timeouts_must_be_positive_integers(): ): with pytest.raises(InvalidArgumentError, match=f"redis.{name} must be a positive integer"): RedisMySQLAccountStore( + service_name="test-service", params={"redis": {"url": "redis://unused", name: value}}, mysql_store=mysql, redis_client=FakeRedis(), ) -async def test_resources_do_not_share_cached_identities(): +async def test_service_names_do_not_share_cached_identities(): redis = FakeRedis() first_key, second_key = _api_key("first"), _api_key("second") - first, _ = _store(first_key, redis_client=redis, resource_id="first") - second, _ = _store(second_key, redis_client=redis, resource_id="second") + first, _ = _store(first_key, redis_client=redis, service_name="first") + second, _ = _store(second_key, redis_client=redis, service_name="second") await first.load() await second.load() try: @@ -263,6 +272,83 @@ async def test_resources_do_not_share_cached_identities(): await second.close() +async def test_real_redis_mysql_service_name_isolation(): + raw_mysql = os.environ.get("OV_TEST_MYSQL") + redis_url = os.environ.get("OV_TEST_REDIS") + if not raw_mysql or not redis_url: + pytest.skip("Set OV_TEST_MYSQL and OV_TEST_REDIS for the Redis MySQL integration test") + + first_service = f"test-{uuid4().hex}" + second_service = f"test-{uuid4().hex}" + key_prefix = f"ov:test:{uuid4().hex}" + params = { + "mysql": json.loads(raw_mysql), + "redis": { + "url": redis_url, + "key_prefix": key_prefix, + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + }, + } + first = build_api_key_manager( + "root", + None, + service_name=first_service, + account_store_provider="redis_mysql", + account_store_params=params, + ) + second = build_api_key_manager( + "root", + None, + service_name=second_service, + account_store_provider="redis_mysql", + account_store_params=params, + ) + setup_engine = create_async_engine(first._store._mysql._url) + async with setup_engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + await connection.execute( + delete(Account).where(Account.service_name.in_([first_service, second_service])) + ) + await setup_engine.dispose() + + await first.load() + await second.load() + try: + first_key = await first.create_account("acme", "alice") + second_key = await second.create_account("acme", "alice") + + assert (await first.resolve_identity(first_key)).role == Role.ADMIN + assert (await second.resolve_identity(second_key)).role == Role.ADMIN + with pytest.raises(UnauthenticatedError): + await first.resolve_identity(second_key) + with pytest.raises(UnauthenticatedError): + await second.resolve_identity(first_key) + + first_user_key = first._store._user_key("acme", "alice") + second_user_key = second._store._user_key("acme", "alice") + assert first_user_key != second_user_key + assert await first._store._redis.exists(first_user_key) + assert await second._store._redis.exists(second_user_key) + + await first.register_user("acme", "dave") + await second.register_user("acme", "dave") + await first.set_role("acme", "dave", Role.ADMIN) + assert await first.get_registered_user_role("acme", "dave") == Role.ADMIN + assert await second.get_registered_user_role("acme", "dave") == Role.USER + finally: + redis_client = first._store._redis + keys = [key async for key in redis_client.scan_iter(match=f"{key_prefix}:*")] + if keys: + await redis_client.delete(*keys) + async with first._store._mysql._engine.begin() as connection: + await connection.execute( + delete(Account).where(Account.service_name.in_([first_service, second_service])) + ) + await second.close() + await first.close() + + async def test_cached_binding_respects_user_id_hint(): api_key = _api_key() store, _ = _store(api_key) diff --git a/tests/server/test_store_assembly.py b/tests/server/test_store_assembly.py index 7d8514d9d0..dd78b48945 100644 --- a/tests/server/test_store_assembly.py +++ b/tests/server/test_store_assembly.py @@ -32,6 +32,24 @@ def test_unknown_account_store_is_rejected_during_config_validation(): AccountStoreConfig(provider="missing") +def test_mysql_account_store_uses_service_name(): + store = build_account_store( + viking_fs=None, + api_key_hashing_enabled=False, + service_name="account-cluster", + account_store_provider="mysql", + account_store_params={ + "user": "unused", + "password": "unused", + "database": "unused", + }, + account_store_watch_enabled=False, + account_store_watch_interval_seconds=30, + ) + + assert store._service_name == "account-cluster" + + @pytest.mark.parametrize("injected", [False, True]) async def test_assembled_manager_supports_account_authentication(injected): account_store = FileStore(SimpleNamespace(agfs=InMemoryAGFS())) From f910e069b9909884764346dda8f8d536365c62f0 Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Wed, 14 Oct 2026 15:22:05 +0800 Subject: [PATCH 13/16] fix(account-store): bound identity and credential history --- docs/en/api/08-admin.md | 2 +- docs/zh/api/08-admin.md | 2 +- openviking/core/identifiers.py | 8 +++ openviking/server/account_stores/mysql.py | 28 +++++++- openviking/server/api_keys/legacy.py | 10 +++ openviking/server/auth/plugins/trusted.py | 8 ++- tests/server/test_account_services.py | 78 +++++++++++++++++++---- tests/server/test_auth.py | 59 +++++++++++++++++ tests/server/test_identity_auth.py | 11 ++-- 9 files changed, 182 insertions(+), 24 deletions(-) diff --git a/docs/en/api/08-admin.md b/docs/en/api/08-admin.md index 94d0109be8..182439a57e 100644 --- a/docs/en/api/08-admin.md +++ b/docs/en/api/08-admin.md @@ -956,7 +956,7 @@ Create a new workspace with its first admin user. | settings | object | No | `null` | Initial Account runtime configuration. ROOT-only; supports `feishu`, `github`, `acl`, `agent_evolution`, `vlm`, `query_planner`, `embedding`, and `vectordb`. `vectordb` and Embedding create-only fields can only be set here | **Notes:** -- File-backed account creation rejects an `account_id` over 255 UTF-8 bytes with `INVALID_ARGUMENT` before writing the account registry. Trusted auto-registration rejects such new accounts on the request, so they cannot block other identities from registering. Existing accounts and users remain supported; user and peer ID rules are unchanged. +- File-backed account creation rejects an `account_id` over 255 UTF-8 bytes with `INVALID_ARGUMENT` before writing the account registry. Newly persisted user IDs are limited to 255 characters across account-store providers. Trusted mode rejects an oversized new user or a new oversized account before enqueueing auto-registration, so invalid identities cannot block other identities from registering. Existing oversized File users and accounts remain readable. - In `trusted` mode, only identity and role are created: no personal key, hash, or key index is generated. The response omits `user_key`, and `seed` is unused. - Omit `seed` for the default random API key. Set `OV_KEY_SEED` to a securely generated secret before running a deterministic example; the Go example also requires `import "os"`. Short or predictable seeds make the key guessable. - `settings` receives structural and effective Embedding/VectorDB pair validation before the Account and its directories are created. A failed validation leaves no Account, user, or configuration file. diff --git a/docs/zh/api/08-admin.md b/docs/zh/api/08-admin.md index b554f78bf6..b0e78ab3a9 100644 --- a/docs/zh/api/08-admin.md +++ b/docs/zh/api/08-admin.md @@ -827,7 +827,7 @@ User override 并重新继承上述默认值，请 PATCH `{"memory_policy": null | settings | object | 否 | `null` | Account 运行时初始配置。仅 ROOT 可提交；支持 `feishu`、`github`、`acl`、`agent_evolution`、`vlm`、`query_planner`、`embedding` 和 `vectordb`。其中 `vectordb` 及 Embedding 创建期字段只能在这里设置 | **说明：** -- 文件存储账户创建时，超过 255 个 UTF-8 字节的 `account_id` 会在写入账户注册表前返回 `INVALID_ARGUMENT`。trusted 自动注册会在请求时拒绝这类新账户，避免阻塞其他身份的注册。既有账户和用户仍可使用，user ID 和 peer ID 规则保持不变。 +- 文件存储账户创建时，超过 255 个 UTF-8 字节的 `account_id` 会在写入账户注册表前返回 `INVALID_ARGUMENT`。新持久化的 user ID 在所有账户存储 Provider 中最多 255 个字符。trusted 模式会在加入自动注册队列前拒绝超长的新 user ID 和新建的超长账户，避免无效身份阻塞其他身份注册；已有的超长 File user 和 account 仍可读取。 - 在 `trusted` 模式下，只创建身份和角色，不生成个人密钥、密钥哈希或密钥索引；响应不包含 `user_key`，`seed` 不参与处理。 - 省略 `seed` 时随机生成 API Key。运行确定性示例前，将 `OV_KEY_SEED` 设为安全生成的秘密值；Go 示例还需导入 `os`。不要使用短字符串或可预测内容作为 seed。 - `settings` 在账号和目录创建前完成结构与有效 Embedding/VectorDB 联合校验。校验失败不会留下 Account、用户或配置文件。 diff --git a/openviking/core/identifiers.py b/openviking/core/identifiers.py index 85d02bf1b9..5d6fb3b0d1 100644 --- a/openviking/core/identifiers.py +++ b/openviking/core/identifiers.py @@ -6,6 +6,7 @@ from typing import Optional _VALIDATION_PATTERN = re.compile(r"^[a-zA-Z0-9_.@-]+$") +MAX_IDENTIFIER_LENGTH = 255 def validate_identifier_part(part: str, part_name: str) -> str | None: @@ -51,3 +52,10 @@ def validate_account_id(account_id: str) -> str | None: def validate_user_id(user_id: str) -> str | None: """Validate a user id.""" return validate_identifier_part(user_id, "user_id") + + +def validate_identifier_length(value: str, name: str) -> str | None: + """Validate the persistence length of a new account-store identifier.""" + if len(value) > MAX_IDENTIFIER_LENGTH: + return f"{name} must not exceed {MAX_IDENTIFIER_LENGTH} characters." + return None diff --git a/openviking/server/account_stores/mysql.py b/openviking/server/account_stores/mysql.py index b86c21deda..d3507e617b 100644 --- a/openviking/server/account_stores/mysql.py +++ b/openviking/server/account_stores/mysql.py @@ -16,7 +16,7 @@ from cryptography.exceptions import InvalidTag from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from sqlalchemy import and_, case, func, select, update +from sqlalchemy import and_, case, delete, func, select, update from sqlalchemy.engine import URL from sqlalchemy.exc import DBAPIError, IntegrityError from sqlalchemy.ext.asyncio import ( @@ -26,6 +26,7 @@ create_async_engine, ) +from openviking.core.identifiers import MAX_IDENTIFIER_LENGTH from openviking.server.account_stores.base import AccountStore from openviking.server.account_stores.models import ( AccountSummary, @@ -56,7 +57,8 @@ ) from openviking_cli.utils.config.config_utils import normalize_service_name -_MYSQL_IDENTIFIER_MAX_LENGTH = 255 +_MYSQL_IDENTIFIER_MAX_LENGTH = MAX_IDENTIFIER_LENGTH +_RETAINED_REVOKED_CREDENTIALS = 10 def _validate_length(value: str, name: str) -> None: @@ -1165,6 +1167,28 @@ async def replace_active_user_api_key( sequence=sequence + 1, ) ) + await session.flush() + old_credentials = list( + ( + await session.scalars( + select(Credential.credential_id) + .where( + Credential.service_name == self._service_name, + Credential.user_pk == user.user_pk, + Credential.revoked.is_(True), + ) + .order_by(Credential.sequence.desc()) + .offset(_RETAINED_REVOKED_CREDENTIALS) + ) + ).all() + ) + if old_credentials: + await session.execute( + delete(Credential).where( + Credential.service_name == self._service_name, + Credential.credential_id.in_(old_credentials), + ) + ) async def verify_api_key( self, diff --git a/openviking/server/api_keys/legacy.py b/openviking/server/api_keys/legacy.py index e8407e6fc7..a18647458c 100644 --- a/openviking/server/api_keys/legacy.py +++ b/openviking/server/api_keys/legacy.py @@ -15,6 +15,7 @@ from argon2.exceptions import VerificationError from typing_extensions import deprecated +from openviking.core.identifiers import validate_identifier_length from openviking.pyagfs import AGFSAlreadyExistsError, AGFSNotFoundError, AsyncAGFSClient from openviking.pyagfs.async_client import fs_ctx_from_agfs_path from openviking.server.account_stores.base import AccountStore @@ -468,6 +469,8 @@ async def _create_account_identity( validate_file_account_id(account_id) if error := validate_user_id(admin_user_id): raise InvalidArgumentError(error) + if error := validate_identifier_length(admin_user_id, "admin_user_id"): + raise InvalidArgumentError(error) if account_id in self._accounts: raise AlreadyExistsError(account_id, "account") @@ -525,6 +528,8 @@ async def _create_user_identity( resolved_role = validate_account_user_role(role) if error := validate_user_id(user_id): raise InvalidArgumentError(error) + if error := validate_identifier_length(user_id, "user_id"): + raise InvalidArgumentError(error) self._ensure_account_active(account_id) account = self._accounts.get(account_id) if account is None: @@ -644,6 +649,11 @@ async def _ensure_trusted_identities_unlocked( error = validate_user_id(user_id) if error: raise InvalidArgumentError(error) + account = self._accounts.get(account_id) + if account is None or user_id not in account.users: + error = validate_identifier_length(user_id, "user_id") + if error: + raise InvalidArgumentError(error) created_accounts = 0 created_users = 0 diff --git a/openviking/server/auth/plugins/trusted.py b/openviking/server/auth/plugins/trusted.py index 08a493f6a6..c2c7f80caf 100644 --- a/openviking/server/auth/plugins/trusted.py +++ b/openviking/server/auth/plugins/trusted.py @@ -12,7 +12,11 @@ from fastapi import Request -from openviking.core.identifiers import validate_account_id, validate_user_id +from openviking.core.identifiers import ( + validate_account_id, + validate_identifier_length, + validate_user_id, +) from openviking.server.api_keys import APIKeyManager from openviking.server.api_keys.legacy import validate_file_account_id from openviking.server.auth.plugin import AuthPlugin @@ -233,6 +237,8 @@ async def _queue_trusted_identity(self, account_id: str, user_id: str) -> None: return if await manager.get_user(account_id, user_id) is not None: return + if length_error := validate_identifier_length(user_id, "user_id"): + raise InvalidArgumentError(length_error) if user_id in self._in_flight.get(account_id, set()): return if user_id in self._pending.get(account_id, set()): diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index 1ba114dc0d..c9dfbbe601 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -10,13 +10,13 @@ from uuid import uuid4 import pytest -from sqlalchemy import delete +from sqlalchemy import and_, delete, select from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import create_async_engine from openviking.server.account_stores import mysql as mysql_store_module from openviking.server.account_stores.mysql import MySQLAccountStore -from openviking.server.account_stores.mysql_schema import Account, Base +from openviking.server.account_stores.mysql_schema import Account, Base, Credential, User from openviking.server.api_keys import APIKeyManager from openviking.server.api_keys.legacy import FileStore from openviking.server.identity import Role @@ -55,6 +55,10 @@ async def test_legacy_user_without_role_defaults_to_user_during_authentication(a manager = build_api_key_manager("root", SimpleNamespace(agfs=agfs)) await manager.load() try: + assert await manager.ensure_trusted_identities({account_id: {user_id}}) == { + "created_accounts": 0, + "created_users": 0, + } identity = await manager.resolve_identity("legacy-key") assert (identity.account_id, identity.user_id, identity.role) == ( account_id, @@ -113,7 +117,7 @@ async def test_file_account_creation_respects_path_component_limit(issue_user_ke @pytest.mark.parametrize("issue_user_keys", [True, False], ids=["api-key", "trusted"]) @pytest.mark.parametrize("length", [255, 256, 1024]) -async def test_file_account_creation_allows_shortened_admin_user_component( +async def test_file_account_creation_respects_admin_user_id_limit( issue_user_keys, length, ): @@ -125,19 +129,23 @@ async def test_file_account_creation_allows_shortened_admin_user_component( account_id = "acme" admin_user_id = "u" * length try: - key = await manager.create_account(account_id, admin_user_id) - assert await manager.get_registered_user_role(account_id, admin_user_id) == Role.ADMIN - if issue_user_keys: - assert (await manager.resolve_identity(key)).user_id == admin_user_id + if length <= 255: + key = await manager.create_account(account_id, admin_user_id) + assert await manager.get_registered_user_role(account_id, admin_user_id) == Role.ADMIN + if issue_user_keys: + assert (await manager.resolve_identity(key)).user_id == admin_user_id else: - assert key is None + with pytest.raises(InvalidArgumentError, match="user_id.*255"): + await manager.create_account(account_id, admin_user_id) + assert await manager.get_account(account_id) is None + assert await manager.get_user(account_id, admin_user_id) is None finally: await manager.close() @pytest.mark.parametrize("issue_user_keys", [True, False], ids=["api-key", "trusted"]) @pytest.mark.parametrize("length", [255, 256, 1024]) -async def test_file_user_creation_allows_shortened_user_component(issue_user_keys, length): +async def test_file_user_creation_respects_user_id_limit(issue_user_keys, length): agfs = InMemoryAGFS() manager = build_api_key_manager( "root", SimpleNamespace(agfs=agfs), issue_user_keys=issue_user_keys @@ -146,12 +154,17 @@ async def test_file_user_creation_allows_shortened_user_component(issue_user_key user_id = "u" * length try: await manager.create_account("acme", "alice") - key = await manager.register_user("acme", user_id) - assert await manager.get_registered_user_role("acme", user_id) == Role.USER - if issue_user_keys: - assert (await manager.resolve_identity(key)).user_id == user_id + if length <= 255: + key = await manager.register_user("acme", user_id) + assert await manager.get_registered_user_role("acme", user_id) == Role.USER + if issue_user_keys: + assert (await manager.resolve_identity(key)).user_id == user_id + else: + assert key is None else: - assert key is None + with pytest.raises(InvalidArgumentError, match="user_id.*255"): + await manager.register_user("acme", user_id) + assert await manager.get_user("acme", user_id) is None finally: await manager.close() @@ -556,6 +569,43 @@ async def test_mysql_managers_share_authoritative_state(): assert (await second.resolve_identity(rotated)).role == Role.ADMIN with pytest.raises(UnauthenticatedError): await second.resolve_identity(key) + rotation_keys = [key, rotated] + for _ in range(11): + rotation_keys.append(await first.regenerate_key("acme", "alice")) + assert (await second.resolve_identity(rotation_keys[-1])).role == Role.ADMIN + async with first._store._session_factory() as session: + credentials = list( + ( + await session.scalars( + select(Credential) + .join( + User, + and_( + User.service_name == Credential.service_name, + User.user_pk == Credential.user_pk, + ), + ) + .join( + Account, + and_( + Account.service_name == User.service_name, + Account.account_pk == User.account_pk, + ), + ) + .where( + Credential.service_name == service_name, + Account.account_id == "acme", + User.user_id == "alice", + ) + .order_by(Credential.sequence) + ) + ).all() + ) + assert len(credentials) == 11 + assert [credential.sequence for credential in credentials] == list(range(3, 14)) + assert sum(not credential.revoked for credential in credentials) == 1 + with pytest.raises(UnauthenticatedError): + await second.resolve_identity(rotation_keys[1]) await first.create_group("acme", "engineering") await first.add_group_member("acme", "engineering", "alice") assert await second.get_user_group_ids("acme", "alice") == ("engineering",) diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index a0cb291b29..76618aa126 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -1091,6 +1091,65 @@ async def test_invalid_trusted_identity_does_not_block_later_registration( await plugin.shutdown() +async def test_oversized_trusted_user_does_not_block_later_registration( + auth_service, +): + """A user id beyond the shared limit is rejected before it reaches the batch.""" + from openviking.server.auth.plugins import TrustedAuthPlugin + + config = ServerConfig( + auth_mode="trusted", + root_api_key=ROOT_KEY, + trusted_identity_flush_interval_seconds=300, + ) + app = FastAPI() + app.state.config = config + app.state.service_name = "openviking-server" + plugin = TrustedAuthPlugin() + await plugin.initialize(app, auth_service, config) + try: + oversized_request = _make_request( + "/api/v1/resources", + headers={ + "X-OpenViking-Account": "long-account", + "X-OpenViking-User": "u" * 256, + }, + auth_enabled=False, + auth_mode="trusted", + ) + oversized_request.app.state.config = config + with pytest.raises(InvalidArgumentError, match="user_id.*255"): + await plugin.resolve_identity( + oversized_request, + x_openviking_account="long-account", + x_openviking_user="u" * 256, + ) + + valid_request = _make_request( + "/api/v1/resources", + headers={ + "X-OpenViking-Account": "good-account", + "X-OpenViking-User": "alice", + }, + auth_enabled=False, + auth_mode="trusted", + ) + valid_request.app.state.config = config + await plugin.resolve_identity( + valid_request, + x_openviking_account="good-account", + x_openviking_user="alice", + ) + await plugin.flush_trusted_identities() + + assert await _stored_users(auth_service, "long-account") == [] + assert await _stored_users(auth_service, "good-account") == [ + {"user_id": "alice", "role": "user"} + ] + finally: + await plugin.shutdown() + + async def test_trusted_identity_registration_skips_identity_loaded_from_registry(auth_service): """A restarted plugin must not queue an identity already loaded from storage.""" from openviking.server.auth.plugins import TrustedAuthPlugin diff --git a/tests/server/test_identity_auth.py b/tests/server/test_identity_auth.py index d2475c0735..140d1d784d 100644 --- a/tests/server/test_identity_auth.py +++ b/tests/server/test_identity_auth.py @@ -140,14 +140,15 @@ async def test_oversized_trusted_account_does_not_block_valid_registration(runti x_openviking_user="bob", ) - long_user_id = "u" * 1024 - await plugin.resolve_identity( - request, api_key="root", x_openviking_account="acme", x_openviking_user=long_user_id - ) + long_user_id = "u" * 256 + with pytest.raises(InvalidArgumentError, match="user_id.*255"): + await plugin.resolve_identity( + request, api_key="root", x_openviking_account="acme", x_openviking_user=long_user_id + ) await plugin.flush_trusted_identities() assert await runtime.get_registered_user_role(account_id, "alice") == Role.USER - assert await runtime.get_registered_user_role("acme", long_user_id) == Role.USER + assert await runtime.get_registered_user_role("acme", long_user_id) is None async def test_trusted_registration_preserves_existing_oversized_account(): From d31f49da1d618ddc7aaa4b8bd8c0db10fa160deb Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Wed, 14 Oct 2026 17:40:15 +0800 Subject: [PATCH 14/16] feat(account-store): support Redis deployment topologies --- docs/en/guides/01-configuration.md | 37 +++-- docs/zh/guides/01-configuration.md | 30 ++-- .../server/account_stores/redis_client.py | 132 ++++++++++++++++++ .../server/account_stores/redis_mysql.py | 117 ++++++++++++---- tests/server/test_redis_client.py | 78 +++++++++++ .../server/test_redis_mysql_account_store.py | 72 ++++++++-- 6 files changed, 406 insertions(+), 60 deletions(-) create mode 100644 openviking/server/account_stores/redis_client.py create mode 100644 tests/server/test_redis_client.py diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index 0dec0a7616..7463545071 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1958,8 +1958,10 @@ separate table prefixes to isolate OpenViking clusters that share a database. ``` `redis_mysql` requires the `redis-mysql` extra and wraps the same MySQL -parameters. It caches API-key bindings, users, per-user group IDs, and deletion -records for a fixed 60 seconds by default. Cache fills and account mutations +parameters. Its `cache` block follows the top-level `cache` `provider/params` +shape. The Python client currently supports `standalone`, `cluster`, and +`sentinel` Redis deployments. It caches API-key bindings, users, per-user group +IDs, and deletion records for a fixed 60 seconds by default. Cache fills and account mutations share an account-scoped Redis lease; normal cache hits do not acquire that lease. Mutations commit to MySQL first, then invalidate affected cache entries. If Redis is unavailable, the MySQL mutation still succeeds and stale cache @@ -1974,9 +1976,10 @@ Redis automatically removes the index when its last sorted-set member is removed Cleanup does not issue a separate index deletion, which could remove a newer fill's index after lease loss. Provisioning and key replacement clear negative lookups both with and without a user hint. -Redis socket connect and read timeouts default to 5 seconds each; override them -with `socket_connect_timeout` and `socket_timeout` query parameters in `redis.url`. -These are socket timeouts, not an overall request deadline. +Redis connection and command timeouts are configured with +`cache.params.connect_timeout_ms` and `cache.params.command_timeout_ms`; both +default to 5000 milliseconds when omitted. These are socket timeouts, not an +overall request deadline. ```json { @@ -1990,12 +1993,24 @@ These are socket timeouts, not an overall request deadline. "password": "secret", "database": "openviking" }, - "redis": { - "url": "rediss://redis.example.com:6379/0", - "key_prefix": "ov:account-auth", - "ttl_seconds": 60, - "lock_ttl_seconds": 60, - "lock_wait_seconds": 30 + "cache": { + "provider": "redis", + "params": { + "mode": "sentinel", + "endpoints": [ + "rediss://sentinel-1.example.com:26379", + "rediss://sentinel-2.example.com:26379", + "rediss://sentinel-3.example.com:26379" + ], + "master_name": "openviking", + "username": "openviking", + "password_env": "OPENVIKING_REDIS_PASSWORD", + "sentinel_password_env": "OPENVIKING_SENTINEL_PASSWORD", + "key_prefix": "ov:account-auth", + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + "lock_wait_seconds": 30 + } } } } diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index bd1128326f..a546685082 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1939,6 +1939,8 @@ schema 变更按顺序存放于 `openviking/server/account_stores/sql/changes/` ``` `redis_mysql` 需要安装 `redis-mysql` extra，并在参数中嵌套同一份 MySQL 配置。 +其 `cache` 配置沿用顶层 `cache` 的 `provider/params` 结构，目前 Python +客户端支持 `standalone`、`cluster` 和 `sentinel` 三种 Redis 部署模式。 它默认以固定 60 秒 TTL 缓存 API Key 绑定、User、每个 User 的 Group ID 和删除记录。 缓存重建与同一 Account 的修改共用 Redis 租约，正常缓存命中不获取该租约。修改会先 提交 MySQL，再失效相关缓存。Redis 不可用时 MySQL 修改仍会成功，遗留缓存最多保持到 @@ -1949,8 +1951,8 @@ Account 时只会清理该 Account 当前的 User、Group、删除记录和凭 索引的最后一个有序集合成员被移除后，Redis 会自动删除空索引；清理结束时不再单独删除 索引，避免失去租约的旧请求误删新回填的索引。创建身份或换 Key 会同时清除携带和未携带 User 提示的负缓存。 -Redis socket 连接和读取超时默认分别为 5 秒，可通过 `redis.url` 中的 -`socket_connect_timeout`、`socket_timeout` 查询参数覆盖；这不是整个请求的总超时。 +Redis 连接和命令超时通过 `cache.params.connect_timeout_ms` 和 +`cache.params.command_timeout_ms` 配置，未配置时均为 5000 毫秒；这不是整个请求的总超时。 ```json { @@ -1964,12 +1966,24 @@ Redis socket 连接和读取超时默认分别为 5 秒，可通过 `redis.url` "password": "secret", "database": "openviking" }, - "redis": { - "url": "rediss://redis.example.com:6379/0", - "key_prefix": "ov:account-auth", - "ttl_seconds": 60, - "lock_ttl_seconds": 60, - "lock_wait_seconds": 30 + "cache": { + "provider": "redis", + "params": { + "mode": "sentinel", + "endpoints": [ + "rediss://sentinel-1.example.com:26379", + "rediss://sentinel-2.example.com:26379", + "rediss://sentinel-3.example.com:26379" + ], + "master_name": "openviking", + "username": "openviking", + "password_env": "OPENVIKING_REDIS_PASSWORD", + "sentinel_password_env": "OPENVIKING_SENTINEL_PASSWORD", + "key_prefix": "ov:account-auth", + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + "lock_wait_seconds": 30 + } } } } diff --git a/openviking/server/account_stores/redis_client.py b/openviking/server/account_stores/redis_client.py new file mode 100644 index 0000000000..e1ad9a9ed8 --- /dev/null +++ b/openviking/server/account_stores/redis_client.py @@ -0,0 +1,132 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Topology-aware asyncio Redis client construction for account stores.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse + +from openviking_cli.exceptions import InvalidArgumentError +from openviking_cli.utils.config.agfs_config import RedisCacheConfig + + +def _secret(value: str, env_name: str, field_name: str) -> str | None: + if value and env_name: + raise InvalidArgumentError(f"{field_name} and {field_name}_env cannot both be configured") + if env_name: + resolved = os.environ.get(env_name) + if not resolved: + raise InvalidArgumentError( + f"{field_name}_env points to an unset or empty environment variable" + ) + return resolved + return value or None + + +def _tls_options(config: RedisCacheConfig, scheme: str) -> dict[str, Any]: + if scheme == "rediss": + return { + "ssl": True, + "ssl_cert_reqs": "none" if config.tls_insecure_skip_verify else "required", + } + return {} + + +def _endpoint_parts(config: RedisCacheConfig) -> list[tuple[str, int]]: + result: list[tuple[str, int]] = [] + for endpoint in config.endpoints: + parsed = urlparse(endpoint) + result.append((parsed.hostname or "", parsed.port or 6379)) + return result + + +@dataclass +class RedisClientHandle: + """A Redis client plus Sentinel discovery clients owned by the factory.""" + + client: Any + sentinel_clients: tuple[Any, ...] = () + + async def close(self) -> None: + try: + close = getattr(self.client, "aclose", None) + if close is not None: + await close() + finally: + for sentinel in self.sentinel_clients: + close = getattr(sentinel, "aclose", None) + if close is not None: + await close() + + +def create_redis_client(raw_params: dict[str, object]) -> RedisClientHandle: + """Create a redis-py client for standalone, cluster, or Sentinel Redis.""" + try: + config = RedisCacheConfig.model_validate(raw_params) + except Exception as exc: + raise InvalidArgumentError(f"Invalid Redis cache configuration: {exc}") from exc + + password = _secret(config.password, config.password_env, "cache.params.password") + sentinel_password = _secret( + config.sentinel_password, + config.sentinel_password_env, + "cache.params.sentinel_password", + ) + scheme = urlparse(config.endpoints[0]).scheme + common: dict[str, Any] = { + "decode_responses": True, + "socket_connect_timeout": config.connect_timeout_ms / 1000, + "socket_timeout": config.command_timeout_ms / 1000, + "username": config.username or None, + "password": password, + "max_connections": config.pool_size, + **_tls_options(config, scheme), + } + + if config.mode == "standalone": + from redis.asyncio import Redis + + return RedisClientHandle( + Redis.from_url( + config.endpoints[0], + db=config.db, + **common, + ) + ) + + if config.mode == "cluster": + from redis.asyncio.cluster import ClusterNode, RedisCluster + + return RedisClientHandle( + RedisCluster( + startup_nodes=[ClusterNode(host, port) for host, port in _endpoint_parts(config)], + db=0, + **common, + ) + ) + + from redis.asyncio.sentinel import Sentinel + + sentinel_common = { + "decode_responses": True, + "socket_connect_timeout": config.connect_timeout_ms / 1000, + "socket_timeout": config.command_timeout_ms / 1000, + **_tls_options(config, scheme), + } + sentinel = Sentinel( + _endpoint_parts(config), + sentinel_kwargs={ + **sentinel_common, + "username": config.sentinel_username or None, + "password": sentinel_password, + }, + **common, + db=config.db, + ) + return RedisClientHandle( + sentinel.master_for(config.master_name or "", **common), + tuple(sentinel.sentinels), + ) diff --git a/openviking/server/account_stores/redis_mysql.py b/openviking/server/account_stores/redis_mysql.py index b0954023ba..fbf2f70ce0 100644 --- a/openviking/server/account_stores/redis_mysql.py +++ b/openviking/server/account_stores/redis_mysql.py @@ -13,6 +13,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager from typing import Any, TypeVar +from urllib.parse import parse_qs, unquote, urlsplit from openviking.server.account_stores.base import AccountStore from openviking.server.account_stores.models import ( @@ -23,6 +24,10 @@ UserSummary, ) from openviking.server.account_stores.mysql import MySQLAccountStore +from openviking.server.account_stores.redis_client import ( + RedisClientHandle, + create_redis_client, +) from openviking_cli.exceptions import InvalidArgumentError from openviking_cli.utils import get_logger from openviking_cli.utils.config.config_utils import normalize_service_name @@ -89,7 +94,6 @@ """ _ACCOUNT_CACHE_CLEAR_BATCH_SIZE = 500 -_REDIS_SOCKET_TIMEOUT_SECONDS = 5 class _CacheLockUnavailable(Exception): @@ -114,6 +118,47 @@ def _decode(raw: str | bytes | None) -> dict[str, Any] | None: return value if isinstance(value, dict) else None +def _legacy_redis_cache_params(values: dict[str, object]) -> dict[str, object]: + url = values.pop("url", None) + if not isinstance(url, str) or not url: + raise InvalidArgumentError("redis.url must be a non-empty string") + parsed = urlsplit(url) + if parsed.scheme not in {"redis", "rediss"} or not parsed.hostname: + raise InvalidArgumentError("redis.url must use a redis:// or rediss:// URL") + try: + port = parsed.port or 6379 + except ValueError as exc: + raise InvalidArgumentError("redis.url port is invalid") from exc + host = parsed.hostname + if ":" in host and not host.startswith("["): + host = f"[{host}]" + params: dict[str, object] = { + "mode": "standalone", + "endpoints": [f"{parsed.scheme}://{host}:{port}"], + **values, + } + if parsed.username and "username" not in params: + params["username"] = unquote(parsed.username) + if parsed.password and "password" not in params: + params["password"] = unquote(parsed.password) + if parsed.path not in {"", "/"} and "db" not in params: + try: + params["db"] = int(parsed.path.lstrip("/")) + except ValueError as exc: + raise InvalidArgumentError("redis.url database path is invalid") from exc + query = parse_qs(parsed.query) + for query_name, config_name in ( + ("socket_connect_timeout", "connect_timeout_ms"), + ("socket_timeout", "command_timeout_ms"), + ): + if query_name in query and config_name not in params: + try: + params[config_name] = max(1, int(float(query[query_name][0]) * 1000)) + except (IndexError, ValueError) as exc: + raise InvalidArgumentError(f"redis.url {query_name} is invalid") from exc + return params + + class RedisMySQLAccountStore(AccountStore): """Cache hot account reads while preserving MySQL as the authority. @@ -136,7 +181,8 @@ def __init__( raise InvalidArgumentError(str(exc)) from None values = dict(params or {}) mysql_params = values.pop("mysql", None) - redis_params = values.pop("redis", None) + cache_spec = values.pop("cache", None) + legacy_redis_params = values.pop("redis", None) if values: names = ", ".join(sorted(values)) raise InvalidArgumentError(f"Unknown Redis MySQL account store parameters: {names}") @@ -151,38 +197,57 @@ def __init__( raise InvalidArgumentError( "Redis and MySQL account stores must use the same service_name" ) - if not isinstance(redis_params, dict): - raise InvalidArgumentError("redis_mysql account storage requires redis parameters") - - redis_values = dict(redis_params) - url = redis_values.pop("url", None) + if cache_spec is None and legacy_redis_params is not None: + if not isinstance(legacy_redis_params, dict): + raise InvalidArgumentError("redis_mysql account storage requires cache parameters") + legacy_values = dict(legacy_redis_params) + cache_spec = { + "provider": "redis", + "params": _legacy_redis_cache_params(legacy_values), + } + if not isinstance(cache_spec, dict): + raise InvalidArgumentError("redis_mysql account storage requires cache parameters") + unknown_cache_fields = set(cache_spec) - {"provider", "params"} + if unknown_cache_fields: + names = ", ".join(sorted(unknown_cache_fields)) + raise InvalidArgumentError(f"Unknown Redis cache fields: {names}") + cache_provider = cache_spec.get("provider") + cache_params = cache_spec.get("params") + if cache_provider != "redis" or not isinstance(cache_params, dict): + raise InvalidArgumentError( + "redis_mysql account storage cache must use provider='redis' with a params object" + ) + redis_values = dict(cache_params) self._key_prefix = redis_values.pop("key_prefix", "ov:account-cache") - self._ttl_seconds = redis_values.pop("ttl_seconds", 60) + self._ttl_seconds = redis_values.pop( + "ttl_seconds", + redis_values.get("default_ttl_seconds", 60), + ) self._lock_ttl_seconds = redis_values.pop("lock_ttl_seconds", 60) self._lock_wait_seconds = redis_values.pop("lock_wait_seconds", 30) - if redis_values: - names = ", ".join(sorted(redis_values)) - raise InvalidArgumentError(f"Unknown Redis cache parameters: {names}") - if not isinstance(url, str) or not url: - raise InvalidArgumentError("redis.url must be a non-empty string") + redis_values.setdefault("connect_timeout_ms", 5000) + redis_values.setdefault("command_timeout_ms", 5000) if not isinstance(self._key_prefix, str) or not self._key_prefix: - raise InvalidArgumentError("redis.key_prefix must be a non-empty string") + raise InvalidArgumentError("cache.params.key_prefix must be a non-empty string") if "{" in self._key_prefix or "}" in self._key_prefix: raise InvalidArgumentError( - "redis.key_prefix must not contain Redis hash-tag characters" + "cache.params.key_prefix must not contain Redis hash-tag characters" ) for name in ("ttl_seconds", "lock_ttl_seconds", "lock_wait_seconds"): value = getattr(self, f"_{name}") if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise InvalidArgumentError(f"redis.{name} must be a positive integer") + raise InvalidArgumentError(f"cache.params.{name} must be a positive integer") if self._lock_ttl_seconds < self._ttl_seconds: - raise InvalidArgumentError("redis.lock_ttl_seconds must be at least redis.ttl_seconds") + raise InvalidArgumentError( + "cache.params.lock_ttl_seconds must be at least cache.params.ttl_seconds" + ) self._mysql = mysql_store self._index_ttl_seconds = self._ttl_seconds + self._lock_ttl_seconds self._service_name = normalized_service_name self._redis = redis_client - self._redis_url = url + self._redis_config = redis_values + self._redis_handle: RedisClientHandle | None = None self._owns_redis = redis_client is None async def load(self) -> None: @@ -190,17 +255,12 @@ async def load(self) -> None: try: if self._redis is None: try: - from redis.asyncio import Redis + self._redis_handle = create_redis_client(self._redis_config) except ImportError: raise RuntimeError( "Redis MySQL account storage requires openviking[redis-mysql]" ) from None - self._redis = Redis.from_url( - self._redis_url, - decode_responses=True, - socket_connect_timeout=_REDIS_SOCKET_TIMEOUT_SECONDS, - socket_timeout=_REDIS_SOCKET_TIMEOUT_SECONDS, - ) + self._redis = self._redis_handle.client await self._redis.ping() except BaseException: await self.close() @@ -208,12 +268,11 @@ async def load(self) -> None: async def close(self) -> None: try: - if self._redis is not None and self._owns_redis: - close = getattr(self._redis, "aclose", None) - if close is not None: - await close() + if self._redis_handle is not None and self._owns_redis: + await self._redis_handle.close() finally: self._redis = None + self._redis_handle = None await self._mysql.close() def _account_tag(self, account_id: str) -> str: diff --git a/tests/server/test_redis_client.py b/tests/server/test_redis_client.py new file mode 100644 index 0000000000..c7620dad6e --- /dev/null +++ b/tests/server/test_redis_client.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 + +import pytest +from redis.asyncio import Redis +from redis.asyncio.cluster import RedisCluster + +from openviking.server.account_stores.redis_client import create_redis_client +from openviking_cli.exceptions import InvalidArgumentError + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("mode", "endpoints", "extra", "expected_type"), + [ + ("standalone", ["redis://127.0.0.1:6379"], {}, Redis), + ( + "cluster", + ["redis://127.0.0.1:7000", "redis://127.0.0.1:7001"], + {}, + RedisCluster, + ), + ], +) +async def test_create_redis_client_supports_direct_topologies( + mode, endpoints, extra, expected_type +): + handle = create_redis_client( + { + "mode": mode, + "endpoints": endpoints, + **extra, + } + ) + try: + assert isinstance(handle.client, expected_type) + finally: + await handle.close() + + +@pytest.mark.asyncio +async def test_create_redis_client_supports_sentinel(): + handle = create_redis_client( + { + "mode": "sentinel", + "endpoints": [ + "redis://127.0.0.1:26379", + "redis://127.0.0.1:26380", + ], + "master_name": "openviking", + } + ) + try: + assert isinstance(handle.client, Redis) + assert len(handle.sentinel_clients) == 2 + finally: + await handle.close() + + +def test_create_redis_client_rejects_invalid_topology_config(): + with pytest.raises(InvalidArgumentError, match="standalone mode requires exactly one"): + create_redis_client( + { + "mode": "standalone", + "endpoints": [ + "redis://127.0.0.1:6379", + "redis://127.0.0.1:6380", + ], + } + ) + + with pytest.raises(InvalidArgumentError, match="master_name"): + create_redis_client( + { + "mode": "sentinel", + "endpoints": ["redis://127.0.0.1:26379"], + } + ) diff --git a/tests/server/test_redis_mysql_account_store.py b/tests/server/test_redis_mysql_account_store.py index e7513dc639..ad5240aec5 100644 --- a/tests/server/test_redis_mysql_account_store.py +++ b/tests/server/test_redis_mysql_account_store.py @@ -213,11 +213,15 @@ def _store( store = RedisMySQLAccountStore( service_name=service_name, params={ - "redis": { - "url": "redis://unused", - "ttl_seconds": 60, - "lock_ttl_seconds": 60, - "lock_wait_seconds": lock_wait_seconds, + "cache": { + "provider": "redis", + "params": { + "mode": "standalone", + "endpoints": ["redis://unused"], + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + "lock_wait_seconds": lock_wait_seconds, + }, } }, mysql_store=mysql, @@ -243,15 +247,55 @@ def test_redis_timeouts_must_be_positive_integers(): ("lock_ttl_seconds", 0), ("lock_wait_seconds", -1), ): - with pytest.raises(InvalidArgumentError, match=f"redis.{name} must be a positive integer"): + with pytest.raises( + InvalidArgumentError, + match=f"cache.params.{name} must be a positive integer", + ): RedisMySQLAccountStore( service_name="test-service", - params={"redis": {"url": "redis://unused", name: value}}, + params={ + "cache": { + "provider": "redis", + "params": { + "mode": "standalone", + "endpoints": ["redis://unused"], + name: value, + }, + } + }, mysql_store=mysql, redis_client=FakeRedis(), ) +def test_legacy_redis_url_is_translated_to_cache_config(): + mysql = FakeMySQLAuthority(_api_key()) + store = RedisMySQLAccountStore( + service_name="test-service", + params={ + "redis": { + "url": ( + "redis://cache-user:cache-password@redis.example.com:6381/2" + "?socket_connect_timeout=0.25&socket_timeout=0.5" + ), + "key_prefix": "legacy-cache", + } + }, + mysql_store=mysql, + redis_client=FakeRedis(), + ) + + assert store._redis_config == { + "mode": "standalone", + "endpoints": ["redis://redis.example.com:6381"], + "username": "cache-user", + "password": "cache-password", + "db": 2, + "connect_timeout_ms": 250, + "command_timeout_ms": 500, + } + + async def test_service_names_do_not_share_cached_identities(): redis = FakeRedis() first_key, second_key = _api_key("first"), _api_key("second") @@ -283,11 +327,15 @@ async def test_real_redis_mysql_service_name_isolation(): key_prefix = f"ov:test:{uuid4().hex}" params = { "mysql": json.loads(raw_mysql), - "redis": { - "url": redis_url, - "key_prefix": key_prefix, - "ttl_seconds": 60, - "lock_ttl_seconds": 60, + "cache": { + "provider": "redis", + "params": { + "mode": "standalone", + "endpoints": [redis_url], + "key_prefix": key_prefix, + "ttl_seconds": 60, + "lock_ttl_seconds": 60, + }, }, } first = build_api_key_manager( From 55186e6ca03451cc5e801fe1c6b49d00da13880c Mon Sep 17 00:00:00 2001 From: runyunzhou Date: Wed, 14 Oct 2026 19:55:36 +0800 Subject: [PATCH 15/16] fix(account-store): handle Redis TLS and test dependencies --- .../server/account_stores/redis_client.py | 25 ++++++++--- pyproject.toml | 1 + tests/server/test_redis_client.py | 41 ++++++++++++++----- uv.lock | 2 + 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/openviking/server/account_stores/redis_client.py b/openviking/server/account_stores/redis_client.py index e1ad9a9ed8..f96f8c2572 100644 --- a/openviking/server/account_stores/redis_client.py +++ b/openviking/server/account_stores/redis_client.py @@ -26,12 +26,19 @@ def _secret(value: str, env_name: str, field_name: str) -> str | None: return value or None -def _tls_options(config: RedisCacheConfig, scheme: str) -> dict[str, Any]: +def _tls_options( + config: RedisCacheConfig, + scheme: str, + *, + include_ssl_flag: bool, +) -> dict[str, Any]: if scheme == "rediss": - return { - "ssl": True, + options = { "ssl_cert_reqs": "none" if config.tls_insecure_skip_verify else "required", } + if include_ssl_flag: + options["ssl"] = True + return options return {} @@ -83,7 +90,6 @@ def create_redis_client(raw_params: dict[str, object]) -> RedisClientHandle: "username": config.username or None, "password": password, "max_connections": config.pool_size, - **_tls_options(config, scheme), } if config.mode == "standalone": @@ -94,6 +100,7 @@ def create_redis_client(raw_params: dict[str, object]) -> RedisClientHandle: config.endpoints[0], db=config.db, **common, + **_tls_options(config, scheme, include_ssl_flag=False), ) ) @@ -105,6 +112,7 @@ def create_redis_client(raw_params: dict[str, object]) -> RedisClientHandle: startup_nodes=[ClusterNode(host, port) for host, port in _endpoint_parts(config)], db=0, **common, + **_tls_options(config, scheme, include_ssl_flag=True), ) ) @@ -114,7 +122,7 @@ def create_redis_client(raw_params: dict[str, object]) -> RedisClientHandle: "decode_responses": True, "socket_connect_timeout": config.connect_timeout_ms / 1000, "socket_timeout": config.command_timeout_ms / 1000, - **_tls_options(config, scheme), + **_tls_options(config, scheme, include_ssl_flag=True), } sentinel = Sentinel( _endpoint_parts(config), @@ -124,9 +132,14 @@ def create_redis_client(raw_params: dict[str, object]) -> RedisClientHandle: "password": sentinel_password, }, **common, + **_tls_options(config, scheme, include_ssl_flag=True), db=config.db, ) return RedisClientHandle( - sentinel.master_for(config.master_name or "", **common), + sentinel.master_for( + config.master_name or "", + **common, + **_tls_options(config, scheme, include_ssl_flag=True), + ), tuple(sentinel.sentinels), ) diff --git a/pyproject.toml b/pyproject.toml index d2c4889782..57f65f8181 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ redis-mysql = [ ] test = [ "SQLAlchemy[asyncio]>=2.0,<2.1", + "redis>=5.0,<7", "pytest>=7.0.0", "pytest-asyncio>=0.21.0", "pytest-xdist>=3.5.0", diff --git a/tests/server/test_redis_client.py b/tests/server/test_redis_client.py index c7620dad6e..d23e592a8a 100644 --- a/tests/server/test_redis_client.py +++ b/tests/server/test_redis_client.py @@ -2,8 +2,6 @@ # SPDX-License-Identifier: AGPL-3.0 import pytest -from redis.asyncio import Redis -from redis.asyncio.cluster import RedisCluster from openviking.server.account_stores.redis_client import create_redis_client from openviking_cli.exceptions import InvalidArgumentError @@ -11,35 +9,40 @@ @pytest.mark.asyncio @pytest.mark.parametrize( - ("mode", "endpoints", "extra", "expected_type"), + ("mode", "endpoints", "expected_type_name"), [ - ("standalone", ["redis://127.0.0.1:6379"], {}, Redis), + ("standalone", ["redis://127.0.0.1:6379"], "Redis"), ( "cluster", ["redis://127.0.0.1:7000", "redis://127.0.0.1:7001"], - {}, - RedisCluster, + "RedisCluster", ), ], ) -async def test_create_redis_client_supports_direct_topologies( - mode, endpoints, extra, expected_type -): +async def test_create_redis_client_supports_direct_topologies(mode, endpoints, expected_type_name): + redis_asyncio = pytest.importorskip("redis.asyncio") + if mode == "cluster": + from redis.asyncio.cluster import RedisCluster + + expected_type = RedisCluster + else: + expected_type = redis_asyncio.Redis handle = create_redis_client( { "mode": mode, "endpoints": endpoints, - **extra, } ) try: assert isinstance(handle.client, expected_type) + assert type(handle.client).__name__ == expected_type_name finally: await handle.close() @pytest.mark.asyncio async def test_create_redis_client_supports_sentinel(): + redis_asyncio = pytest.importorskip("redis.asyncio") handle = create_redis_client( { "mode": "sentinel", @@ -51,12 +54,28 @@ async def test_create_redis_client_supports_sentinel(): } ) try: - assert isinstance(handle.client, Redis) + assert isinstance(handle.client, redis_asyncio.Redis) assert len(handle.sentinel_clients) == 2 finally: await handle.close() +@pytest.mark.asyncio +async def test_create_redis_client_supports_standalone_tls_url(): + pytest.importorskip("redis.asyncio") + handle = create_redis_client( + { + "mode": "standalone", + "endpoints": ["rediss://127.0.0.1:6379"], + "tls_insecure_skip_verify": True, + } + ) + try: + assert handle.client.connection_pool.connection_class.__name__ == "SSLConnection" + finally: + await handle.close() + + def test_create_redis_client_rejects_invalid_topology_config(): with pytest.raises(InvalidArgumentError, match="standalone mode requires exactly one"): create_redis_client( diff --git a/uv.lock b/uv.lock index 04c377af66..d88b650c02 100644 --- a/uv.lock +++ b/uv.lock @@ -3696,6 +3696,7 @@ test = [ { name = "pytest-cov" }, { name = "pytest-xdist" }, { name = "ragas" }, + { name = "redis" }, { name = "sqlalchemy", extra = ["asyncio"] }, ] @@ -3804,6 +3805,7 @@ requires-dist = [ { name = "rapidfuzz", specifier = ">=3.14,<4" }, { name = "readability-lxml", marker = "extra == 'bot'", specifier = ">=0.8.0" }, { name = "redis", marker = "extra == 'redis-mysql'", specifier = ">=5.0,<7" }, + { name = "redis", marker = "extra == 'test'", specifier = ">=5.0,<7" }, { name = "requests", specifier = ">=2.33.0" }, { name = "rich", marker = "extra == 'bot'", specifier = ">=13.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, From 16d3f903fd4f47979e386753e16c300158102b20 Mon Sep 17 00:00:00 2001 From: qin-ctx Date: Wed, 14 Oct 2026 20:22:25 +0800 Subject: [PATCH 16/16] test(account-store): consolidate provider contract coverage --- tests/server/test_account_services.py | 453 ++++--------- tests/server/test_auth.py | 59 -- tests/server/test_identity_auth.py | 2 +- tests/server/test_redis_client.py | 97 --- .../server/test_redis_mysql_account_store.py | 631 ------------------ tests/server/test_store_assembly.py | 37 +- 6 files changed, 136 insertions(+), 1143 deletions(-) delete mode 100644 tests/server/test_redis_client.py delete mode 100644 tests/server/test_redis_mysql_account_store.py diff --git a/tests/server/test_account_services.py b/tests/server/test_account_services.py index c9dfbbe601..447ec7bacf 100644 --- a/tests/server/test_account_services.py +++ b/tests/server/test_account_services.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: AGPL-3.0 import asyncio -import base64 import json import os import threading @@ -10,13 +9,10 @@ from uuid import uuid4 import pytest -from sqlalchemy import and_, delete, select -from sqlalchemy.exc import OperationalError +from sqlalchemy import delete, func, select from sqlalchemy.ext.asyncio import create_async_engine -from openviking.server.account_stores import mysql as mysql_store_module -from openviking.server.account_stores.mysql import MySQLAccountStore -from openviking.server.account_stores.mysql_schema import Account, Base, Credential, User +from openviking.server.account_stores.mysql_schema import Account, Base, Credential from openviking.server.api_keys import APIKeyManager from openviking.server.api_keys.legacy import FileStore from openviking.server.identity import Role @@ -195,8 +191,23 @@ async def test_file_trusted_batch_rejects_oversized_account_before_any_write(): await manager.close() -@pytest.mark.parametrize("operation", ["account", "user"]) -async def test_file_creation_is_atomic_when_credential_write_fails(operation): +async def test_file_account_creation_is_atomic_when_credential_write_fails(): + class FailingAGFS(InMemoryAGFS): + def write(self, path, content, ctx=None): + if path == "/local/acme/_system/users.json": + raise OSError("credential store unavailable") + return super().write(path, content, ctx) + + store = FileStore(SimpleNamespace(agfs=FailingAGFS())) + manager = APIKeyManager("root", store) + await manager.load() + + with pytest.raises(OSError, match="credential store unavailable"): + await manager.create_account("acme", "alice") + assert await manager.get_account("acme") is None + + +async def test_file_user_creation_is_atomic_when_credential_write_fails(): class FailingAGFS(InMemoryAGFS): fail_user_write = False @@ -209,23 +220,13 @@ def write(self, path, content, ctx=None): store = FileStore(SimpleNamespace(agfs=agfs)) manager = APIKeyManager("root", store) await manager.load() - if operation == "user": - admin_key = await manager.create_account("acme", "alice") + admin_key = await manager.create_account("acme", "alice") agfs.fail_user_write = True - try: - with pytest.raises(OSError, match="credential store unavailable"): - if operation == "account": - await manager.create_account("acme", "alice") - else: - await manager.register_user("acme", "bob") - assert await manager.get_user("acme", "bob") is None - if operation == "account": - assert await manager.get_account("acme") is None - else: - assert (await manager.resolve_identity(admin_key)).user_id == "alice" - finally: - await manager.close() + with pytest.raises(OSError, match="credential store unavailable"): + await manager.register_user("acme", "bob") + assert await manager.get_user("acme", "bob") is None + assert (await manager.resolve_identity(admin_key)).user_id == "alice" async def test_regenerate_key_preserves_management_not_found_errors(): @@ -239,42 +240,108 @@ async def test_regenerate_key_preserves_management_not_found_errors(): await manager.regenerate_key("acme", "missing") -async def test_file_hashing_never_persists_plaintext_credentials(): - agfs = InMemoryAGFS() - manager = build_api_key_manager( +async def test_distributed_credentials_are_shared_and_service_scoped(): + """Warm caches must reject rotated keys without crossing service boundaries.""" + raw_mysql = os.environ.get("OV_TEST_MYSQL") + redis_url = os.environ.get("OV_TEST_REDIS") + if not raw_mysql or not redis_url: + pytest.skip("Set OV_TEST_MYSQL and OV_TEST_REDIS for the distributed credential contract") + + service_name = f"test-{uuid4().hex}" + isolated_service_name = f"test-{uuid4().hex}" + key_prefix = f"ov:test:{uuid4().hex}" + params = { + "mysql": json.loads(raw_mysql), + "cache": { + "provider": "redis", + "params": { + "mode": "standalone", + "endpoints": [redis_url], + "key_prefix": key_prefix, + }, + }, + } + writer = build_api_key_manager( "root", - SimpleNamespace(agfs=agfs), - api_key_hashing_enabled=True, + None, + service_name=service_name, + account_store_provider="redis_mysql", + account_store_params=params, ) - await manager.load() - account_key = await manager.create_account("acme", "alice") - user_key = await manager.register_user("acme", "bob") - rotated_key = await manager.regenerate_key("acme", "bob") - - persisted = agfs.read("/local/acme/_system/users.json") - for key in (account_key, user_key, rotated_key): - assert key.encode() not in persisted - assert (await manager.resolve_identity(account_key)).user_id == "alice" - assert (await manager.resolve_identity(rotated_key)).user_id == "bob" - with pytest.raises(UnauthenticatedError): - await manager.resolve_identity(user_key) - - -async def test_file_load_migrates_plaintext_credential_to_argon2(): - agfs = InMemoryAGFS() - plaintext = build_api_key_manager("root", SimpleNamespace(agfs=agfs)) - await plaintext.load() - key = await plaintext.create_account("acme", "alice") - - hashed = build_api_key_manager( + reader = build_api_key_manager( "root", - SimpleNamespace(agfs=agfs), - api_key_hashing_enabled=True, + None, + service_name=service_name, + account_store_provider="redis_mysql", + account_store_params=params, + ) + isolated = build_api_key_manager( + "root", + None, + service_name=isolated_service_name, + account_store_provider="redis_mysql", + account_store_params=params, ) - await hashed.load() + mysql = writer._store._mysql + engine = create_async_engine(mysql._url, **mysql._engine_options) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + await writer.load() + await reader.load() + await isolated.load() + + old_key = await writer.create_account("acme", "alice") + isolated_key = await isolated.create_account("acme", "alice") + assert (await reader.resolve_identity(old_key)).role == Role.ADMIN + assert (await isolated.resolve_identity(isolated_key)).role == Role.ADMIN + with pytest.raises(UnauthenticatedError): + await isolated.resolve_identity(old_key) + with pytest.raises(UnauthenticatedError): + await reader.resolve_identity(isolated_key) - assert key.encode() not in agfs.read("/local/acme/_system/users.json") - assert (await hashed.resolve_identity(key)).user_id == "alice" + # Repeated rotation must bound history and invalidate an already cached key. + for _ in range(12): + active_key = await writer.regenerate_key("acme", "alice") + with pytest.raises(UnauthenticatedError): + await reader.resolve_identity(old_key) + assert (await reader.resolve_identity(active_key)).user_id == "alice" + assert (await isolated.resolve_identity(isolated_key)).role == Role.ADMIN + async with engine.connect() as connection: + assert ( + await connection.scalar( + select(func.count()) + .select_from(Credential) + .where(Credential.service_name == service_name) + ) + == 11 + ) + assert ( + await connection.scalar( + select(func.count()) + .select_from(Credential) + .where(Credential.service_name == service_name, Credential.revoked.is_(False)) + ) + == 1 + ) + finally: + try: + redis_client = writer._store._redis + if redis_client is not None: + keys = [key async for key in redis_client.scan_iter(match=f"{key_prefix}:*")] + if keys: + await redis_client.delete(*keys) + async with engine.begin() as connection: + await connection.execute( + delete(Account).where( + Account.service_name.in_([service_name, isolated_service_name]) + ) + ) + finally: + await isolated.close() + await reader.close() + await writer.close() + await engine.dispose() async def test_file_management_user_read_refreshes_only_explicitly(): @@ -343,132 +410,6 @@ def read(self, path, ctx=None): assert (await store.get_deletion("acme", "bob"))["task_id"] == "delete" -def test_mysql_uses_service_name_instead_of_resource_id_env(monkeypatch): - monkeypatch.setenv("OV_RESOURCE_ID", "ignored-resource") - store = MySQLAccountStore( - service_name="account-cluster", - params={"user": "unused", "password": "unused", "database": "unused"}, - ) - assert store._service_name == "account-cluster" - - -def test_mysql_service_name_must_fit_schema(): - params = {"user": "unused", "password": "unused", "database": "unused"} - MySQLAccountStore(service_name="r" * 255, params=params) - - with pytest.raises(InvalidArgumentError, match="service_name must not exceed 255"): - MySQLAccountStore(service_name="r" * 256, params=params) - - -@pytest.mark.parametrize( - "operation", - [ - pytest.param( - lambda store, value: store.create_account_with_api_key(value, "admin", "key"), - id="account", - ), - pytest.param( - lambda store, value: store.create_account_with_api_key("account", value, "key"), - id="account-admin", - ), - pytest.param( - lambda store, value: store.create_user("account", value, "user"), - id="user", - ), - pytest.param( - lambda store, value: store.ensure_trusted_identities({value: {"user"}}), - id="trusted-account", - ), - pytest.param( - lambda store, value: store.ensure_trusted_identities({"account": {value}}), - id="trusted-user", - ), - pytest.param( - lambda store, value: store.create_group("account", value), - id="group", - ), - pytest.param( - lambda store, value: store.begin_deletion( - "account", - None, - task_id="task", - owner_account_id=value, - owner_user_id="owner", - ), - id="deletion-owner-account", - ), - pytest.param( - lambda store, value: store.begin_deletion( - "account", - None, - task_id="task", - owner_account_id="owner", - owner_user_id=value, - ), - id="deletion-owner-user", - ), - ], -) -async def test_mysql_identifiers_must_fit_schema(operation): - store = MySQLAccountStore( - service_name="test-service", - params={"user": "unused", "password": "unused", "database": "unused"}, - ) - - with pytest.raises(InvalidArgumentError, match="must not exceed 255 characters"): - await operation(store, "a" * 256) - - -def test_mysql_rejects_removed_table_parameter(): - with pytest.raises(InvalidArgumentError, match="Unknown MySQL account store parameters: table"): - MySQLAccountStore( - service_name="test-service", - params={ - "user": "unused", - "password": "unused", - "database": "unused", - "table": "tenant_accounts", - }, - ) - - -async def test_mysql_load_rejects_incomplete_schema_and_disposes_engine(monkeypatch): - store = MySQLAccountStore( - service_name="test-service", - params={"user": "unused", "password": "unused", "database": "unused"}, - ) - - class FakeEngine: - disposed = False - - async def dispose(self): - self.disposed = True - - engine = FakeEngine() - - class FakeSession: - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - pass - - async def execute(self, statement): - raise OperationalError(str(statement), {}, Exception(1146, "Table doesn't exist")) - - monkeypatch.setattr(mysql_store_module, "create_async_engine", lambda *args, **kwargs: engine) - monkeypatch.setattr( - mysql_store_module, - "async_sessionmaker", - lambda *args, **kwargs: FakeSession, - ) - - with pytest.raises(RuntimeError, match="schema is missing or incompatible"): - await store.load() - - assert engine.disposed - - async def test_file_store_rejects_deleting_nonempty_group(): store = _file_store() await store.load() @@ -479,155 +420,3 @@ async def test_file_store_rejects_deleting_nonempty_group(): with pytest.raises(FailedPreconditionError, match="Group must be empty"): await store.delete_group("acme", "engineering") assert await store.get_group_members("acme", "engineering") == ["alice"] - - -async def test_mysql_managers_share_authoritative_state(): - raw = os.environ.get("OV_TEST_MYSQL") - if not raw: - pytest.skip("Set OV_TEST_MYSQL to connection JSON for the MySQL test") - service_name = f"test-{uuid4().hex}" - encryption_key = base64.urlsafe_b64encode(b"k" * 32).decode() - params = { - **json.loads(raw), - "credential_encryption_key": encryption_key, - } - first = build_api_key_manager( - "root", - None, - service_name=service_name, - account_store_provider="mysql", - account_store_params={**params, "credential_storage": "encrypted"}, - ) - second = build_api_key_manager( - "root", - None, - service_name=service_name, - account_store_provider="mysql", - account_store_params={**params, "credential_storage": "hash"}, - ) - isolated_service_name = f"test-{uuid4().hex}" - isolated = build_api_key_manager( - "root", - None, - service_name=isolated_service_name, - account_store_provider="mysql", - account_store_params={**params, "credential_storage": "hash"}, - ) - setup_engine = create_async_engine(first._store._url, **first._store._engine_options) - async with setup_engine.begin() as connection: - await connection.run_sync(Base.metadata.create_all) - await connection.execute( - delete(Account).where(Account.service_name.in_([service_name, isolated_service_name])) - ) - await setup_engine.dispose() - await first.load() - await second.load() - await isolated.load() - try: - trusted_results = await asyncio.gather( - first.ensure_trusted_identities({"trusted": {"alice", "bob"}}), - second.ensure_trusted_identities({"trusted": {"bob", "carol"}}), - ) - assert sum(result["created_accounts"] for result in trusted_results) == 1 - assert sum(result["created_users"] for result in trusted_results) == 3 - assert await first.get_registered_user_role("trusted", "alice") == Role.USER - assert await second.get_registered_user_role("trusted", "bob") == Role.USER - assert await first.get_registered_user_role("trusted", "carol") == Role.USER - assert (await first.get_account("trusted"))["user_count"] == 3 - repeat = await second.ensure_trusted_identities({"trusted": {"alice", "bob", "carol"}}) - assert repeat == {"created_accounts": 0, "created_users": 0} - - deletion, created = await first.begin_deletion( - "trusted", - "carol", - task_id="delete-carol", - owner_account_id="system", - owner_user_id="system", - ) - assert created - assert await second.get_deletion("trusted", "carol") == deletion - account_deletion, account_created = await first.begin_deletion( - "trusted", - None, - task_id="delete-trusted", - owner_account_id="system", - owner_user_id="system", - ) - assert account_created - assert await second.get_deletion("trusted", "carol") == account_deletion - blocked = await second.ensure_trusted_identities({"trusted": {"late-user"}}) - assert blocked == {"created_accounts": 0, "created_users": 0} - assert await first.get_registered_user_role("trusted", "late-user") is None - - key = await first.create_account("acme", "alice") - assert (await second.resolve_identity(key)).role == Role.ADMIN - user_key = await first.register_user("acme", "dave") - assert (await second.resolve_identity(user_key)).role == Role.USER - hashed_key = await second.create_account("hashed", "bob") - assert (await first.resolve_identity(hashed_key)).role == Role.ADMIN - rotated = await first.regenerate_key("acme", "alice") - assert (await second.resolve_identity(rotated)).role == Role.ADMIN - with pytest.raises(UnauthenticatedError): - await second.resolve_identity(key) - rotation_keys = [key, rotated] - for _ in range(11): - rotation_keys.append(await first.regenerate_key("acme", "alice")) - assert (await second.resolve_identity(rotation_keys[-1])).role == Role.ADMIN - async with first._store._session_factory() as session: - credentials = list( - ( - await session.scalars( - select(Credential) - .join( - User, - and_( - User.service_name == Credential.service_name, - User.user_pk == Credential.user_pk, - ), - ) - .join( - Account, - and_( - Account.service_name == User.service_name, - Account.account_pk == User.account_pk, - ), - ) - .where( - Credential.service_name == service_name, - Account.account_id == "acme", - User.user_id == "alice", - ) - .order_by(Credential.sequence) - ) - ).all() - ) - assert len(credentials) == 11 - assert [credential.sequence for credential in credentials] == list(range(3, 14)) - assert sum(not credential.revoked for credential in credentials) == 1 - with pytest.raises(UnauthenticatedError): - await second.resolve_identity(rotation_keys[1]) - await first.create_group("acme", "engineering") - await first.add_group_member("acme", "engineering", "alice") - assert await second.get_user_group_ids("acme", "alice") == ("engineering",) - - assert await isolated.get_account("acme") is None - isolated_key = await isolated.create_account("acme", "alice") - assert (await isolated.resolve_identity(isolated_key)).role == Role.ADMIN - with pytest.raises(UnauthenticatedError): - await isolated.resolve_identity(rotated) - with pytest.raises(UnauthenticatedError): - await first.resolve_identity(isolated_key) - await isolated.register_user("acme", "dave") - await isolated.set_role("acme", "dave", Role.ADMIN) - assert await isolated.get_registered_user_role("acme", "dave") == Role.ADMIN - assert await first.get_registered_user_role("acme", "dave") == Role.USER - finally: - async with first._store._engine.begin() as connection: - await connection.execute( - delete(Account).where( - Account.service_name.in_([service_name, isolated_service_name]) - ) - ) - await isolated.close() - await second.close() - await first.close() diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py index 76618aa126..a0cb291b29 100644 --- a/tests/server/test_auth.py +++ b/tests/server/test_auth.py @@ -1091,65 +1091,6 @@ async def test_invalid_trusted_identity_does_not_block_later_registration( await plugin.shutdown() -async def test_oversized_trusted_user_does_not_block_later_registration( - auth_service, -): - """A user id beyond the shared limit is rejected before it reaches the batch.""" - from openviking.server.auth.plugins import TrustedAuthPlugin - - config = ServerConfig( - auth_mode="trusted", - root_api_key=ROOT_KEY, - trusted_identity_flush_interval_seconds=300, - ) - app = FastAPI() - app.state.config = config - app.state.service_name = "openviking-server" - plugin = TrustedAuthPlugin() - await plugin.initialize(app, auth_service, config) - try: - oversized_request = _make_request( - "/api/v1/resources", - headers={ - "X-OpenViking-Account": "long-account", - "X-OpenViking-User": "u" * 256, - }, - auth_enabled=False, - auth_mode="trusted", - ) - oversized_request.app.state.config = config - with pytest.raises(InvalidArgumentError, match="user_id.*255"): - await plugin.resolve_identity( - oversized_request, - x_openviking_account="long-account", - x_openviking_user="u" * 256, - ) - - valid_request = _make_request( - "/api/v1/resources", - headers={ - "X-OpenViking-Account": "good-account", - "X-OpenViking-User": "alice", - }, - auth_enabled=False, - auth_mode="trusted", - ) - valid_request.app.state.config = config - await plugin.resolve_identity( - valid_request, - x_openviking_account="good-account", - x_openviking_user="alice", - ) - await plugin.flush_trusted_identities() - - assert await _stored_users(auth_service, "long-account") == [] - assert await _stored_users(auth_service, "good-account") == [ - {"user_id": "alice", "role": "user"} - ] - finally: - await plugin.shutdown() - - async def test_trusted_identity_registration_skips_identity_loaded_from_registry(auth_service): """A restarted plugin must not queue an identity already loaded from storage.""" from openviking.server.auth.plugins import TrustedAuthPlugin diff --git a/tests/server/test_identity_auth.py b/tests/server/test_identity_auth.py index 140d1d784d..dc66b7b7b2 100644 --- a/tests/server/test_identity_auth.py +++ b/tests/server/test_identity_auth.py @@ -124,7 +124,7 @@ async def test_trusted_explicit_user_role_with_root_key(runtime): assert identity.role == Role.USER -async def test_oversized_trusted_account_does_not_block_valid_registration(runtime): +async def test_oversized_trusted_identity_does_not_block_valid_registration(runtime): request = await make_request(runtime, "trusted", trusted_flush_interval=300) plugin = request.app.state.auth_plugin account_id = "a" * 255 diff --git a/tests/server/test_redis_client.py b/tests/server/test_redis_client.py deleted file mode 100644 index d23e592a8a..0000000000 --- a/tests/server/test_redis_client.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 - -import pytest - -from openviking.server.account_stores.redis_client import create_redis_client -from openviking_cli.exceptions import InvalidArgumentError - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("mode", "endpoints", "expected_type_name"), - [ - ("standalone", ["redis://127.0.0.1:6379"], "Redis"), - ( - "cluster", - ["redis://127.0.0.1:7000", "redis://127.0.0.1:7001"], - "RedisCluster", - ), - ], -) -async def test_create_redis_client_supports_direct_topologies(mode, endpoints, expected_type_name): - redis_asyncio = pytest.importorskip("redis.asyncio") - if mode == "cluster": - from redis.asyncio.cluster import RedisCluster - - expected_type = RedisCluster - else: - expected_type = redis_asyncio.Redis - handle = create_redis_client( - { - "mode": mode, - "endpoints": endpoints, - } - ) - try: - assert isinstance(handle.client, expected_type) - assert type(handle.client).__name__ == expected_type_name - finally: - await handle.close() - - -@pytest.mark.asyncio -async def test_create_redis_client_supports_sentinel(): - redis_asyncio = pytest.importorskip("redis.asyncio") - handle = create_redis_client( - { - "mode": "sentinel", - "endpoints": [ - "redis://127.0.0.1:26379", - "redis://127.0.0.1:26380", - ], - "master_name": "openviking", - } - ) - try: - assert isinstance(handle.client, redis_asyncio.Redis) - assert len(handle.sentinel_clients) == 2 - finally: - await handle.close() - - -@pytest.mark.asyncio -async def test_create_redis_client_supports_standalone_tls_url(): - pytest.importorskip("redis.asyncio") - handle = create_redis_client( - { - "mode": "standalone", - "endpoints": ["rediss://127.0.0.1:6379"], - "tls_insecure_skip_verify": True, - } - ) - try: - assert handle.client.connection_pool.connection_class.__name__ == "SSLConnection" - finally: - await handle.close() - - -def test_create_redis_client_rejects_invalid_topology_config(): - with pytest.raises(InvalidArgumentError, match="standalone mode requires exactly one"): - create_redis_client( - { - "mode": "standalone", - "endpoints": [ - "redis://127.0.0.1:6379", - "redis://127.0.0.1:6380", - ], - } - ) - - with pytest.raises(InvalidArgumentError, match="master_name"): - create_redis_client( - { - "mode": "sentinel", - "endpoints": ["redis://127.0.0.1:26379"], - } - ) diff --git a/tests/server/test_redis_mysql_account_store.py b/tests/server/test_redis_mysql_account_store.py deleted file mode 100644 index ad5240aec5..0000000000 --- a/tests/server/test_redis_mysql_account_store.py +++ /dev/null @@ -1,631 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 - -import asyncio -import base64 -import json -import os -from uuid import uuid4 - -import pytest -from sqlalchemy import delete -from sqlalchemy.ext.asyncio import create_async_engine - -from openviking.server.account_stores.mysql_schema import Account, Base -from openviking.server.account_stores.redis_mysql import RedisMySQLAccountStore -from openviking.server.api_keys.new import APIKeyManager -from openviking.server.identity import Role -from openviking.server.store_assembly import build_api_key_manager -from openviking_cli.exceptions import InvalidArgumentError, UnauthenticatedError - - -class FakeRedis: - def __init__(self): - self.values: dict[str, str] = {} - self.zsets: dict[str, dict[str, int]] = {} - self.now_ms = 0 - - async def ping(self): - return True - - async def get(self, key: str): - return self.values.get(key) - - async def set(self, key: str, value, *, ex=None, nx=False, px=None): - del ex, px - if nx and key in self.values: - return False - self.values[key] = str(value) - return True - - async def delete(self, *keys: str): - removed = 0 - for key in keys: - removed += self.values.pop(key, None) is not None - removed += self.zsets.pop(key, None) is not None - return removed - - async def zrange(self, key: str, start: int, end: int): - members = sorted(self.zsets.get(key, {}).items(), key=lambda item: (item[1], item[0])) - if end == -1: - return [member for member, _ in members[start:]] - return [member for member, _ in members[start : end + 1]] - - def _prune_zset(self, key: str): - members = self.zsets.get(key, {}) - for member, score in tuple(members.items()): - if score <= self.now_ms: - del members[member] - if not members: - self.zsets.pop(key, None) - - async def eval(self, script: str, numkeys: int, *args): - keys = args[:numkeys] - values = args[numkeys:] - if "-- release-lock" in script: - if self.values.get(keys[0]) == values[0]: - return await self.delete(keys[0]) - return 0 - if "-- get-if-locked" in script: - if self.values.get(keys[0]) != values[0]: - return [0] - value = self.values.get(keys[1]) - return [1] if value is None else [2, value] - if "-- set-if-locked" in script: - if self.values.get(keys[0]) != values[0]: - return 0 - self.values[keys[1]] = values[1] - if str(values[4]) == "1": - self._prune_zset(keys[2]) - self.zsets.setdefault(keys[2], {})[keys[1]] = self.now_ms + int(values[2]) * 1000 - return 1 - if "-- delete-and-unindex" in script: - await self.delete(*keys[1:]) - members = self.zsets.get(keys[0], {}) - for key in keys[1:]: - members.pop(key, None) - if not members: - self.zsets.pop(keys[0], None) - return len(keys) - 1 - if "-- prune-cache-index" in script: - self._prune_zset(keys[0]) - return 0 - raise AssertionError("Unknown Lua script") - - -class FakeMySQLAuthority: - def __init__(self, api_key: str, service_name: str = "test-service"): - self._service_name = service_name - self.api_key = api_key - self.account_exists = True - self.user_exists = True - self.user = {"user_id": "alice", "role": Role.ADMIN} - self.group_ids = ("engineering",) - self.account_deletion = None - self.user_deletion = None - - async def load(self): - return None - - async def close(self): - return None - - async def verify_api_key( - self, - api_key: str, - *, - account_id_hint: str | None = None, - user_id_hint: str | None = None, - ): - return ( - ("acme", "alice") - if ( - self.account_exists - and self.user_exists - and api_key == self.api_key - and account_id_hint in (None, "acme") - and user_id_hint in (None, "alice") - ) - else None - ) - - async def get_user(self, account_id: str, user_id: str): - if ( - not self.account_exists - or not self.user_exists - or (account_id, user_id) != ("acme", "alice") - ): - return None - return dict(self.user) - - async def get_user_group_ids(self, account_id: str, user_id: str): - return ( - self.group_ids - if self.account_exists and (account_id, user_id) == ("acme", "alice") - else () - ) - - async def get_deletion(self, account_id: str, user_id: str | None = None): - if not self.account_exists or account_id != "acme": - return None - if user_id is None: - return self.account_deletion - return self.account_deletion or self.user_deletion - - async def replace_active_user_api_key(self, account_id: str, user_id: str, api_key: str): - assert (account_id, user_id) == ("acme", "alice") - self.api_key = api_key - - async def create_account_with_api_key( - self, - account_id: str, - admin_user_id: str, - api_key: str, - ): - assert (account_id, admin_user_id) == ("acme", "alice") - self.account_exists = True - self.user_exists = True - self.api_key = api_key - self.user = {"user_id": "alice", "role": Role.ADMIN} - self.group_ids = () - self.account_deletion = None - self.user_deletion = None - - async def create_user_with_api_key( - self, - account_id: str, - user_id: str, - role: str, - api_key: str, - ): - assert (account_id, user_id) == ("acme", "alice") - self.user_exists = True - self.api_key = api_key - self.user = {"user_id": user_id, "role": role} - - async def delete_account(self, account_id: str): - assert account_id == "acme" - self.account_exists = False - - async def revoke_active_api_keys(self, account_id: str, user_id: str | None = None): - assert account_id == "acme" - self.api_key = "" - - async def set_role(self, account_id: str, user_id: str, role: str): - assert (account_id, user_id) == ("acme", "alice") - self.user["role"] = role - - async def add_group_member(self, account_id: str, group_id: str, user_id: str): - assert (account_id, user_id) == ("acme", "alice") - self.group_ids = tuple(sorted(set(self.group_ids) | {group_id})) - return True - - async def begin_deletion(self, account_id: str, user_id: str | None, **_kwargs): - assert (account_id, user_id) == ("acme", "alice") - self.user_deletion = {"task_id": "delete-1"} - return self.user_deletion, True - - -def _store( - api_key: str, *, redis_client=None, lock_wait_seconds: int = 30, service_name="test-service" -) -> tuple[RedisMySQLAccountStore, FakeMySQLAuthority]: - mysql = FakeMySQLAuthority(api_key, service_name) - store = RedisMySQLAccountStore( - service_name=service_name, - params={ - "cache": { - "provider": "redis", - "params": { - "mode": "standalone", - "endpoints": ["redis://unused"], - "ttl_seconds": 60, - "lock_ttl_seconds": 60, - "lock_wait_seconds": lock_wait_seconds, - }, - } - }, - mysql_store=mysql, - redis_client=redis_client if redis_client is not None else FakeRedis(), - ) - return store, mysql - - -def _api_key(secret: str = "secret") -> str: - encoded_account = base64.urlsafe_b64encode(b"acme").decode().rstrip("=") - encoded_user = base64.urlsafe_b64encode(b"alice").decode().rstrip("=") - encoded_secret = base64.urlsafe_b64encode(secret.encode()).decode().rstrip("=") - return f"{encoded_account}.{encoded_user}.{encoded_secret}" - - -def test_redis_timeouts_must_be_positive_integers(): - mysql = FakeMySQLAuthority(_api_key()) - for name, value in ( - ("ttl_seconds", 1.5), - ("lock_ttl_seconds", 1.5), - ("lock_wait_seconds", 1.5), - ("ttl_seconds", True), - ("lock_ttl_seconds", 0), - ("lock_wait_seconds", -1), - ): - with pytest.raises( - InvalidArgumentError, - match=f"cache.params.{name} must be a positive integer", - ): - RedisMySQLAccountStore( - service_name="test-service", - params={ - "cache": { - "provider": "redis", - "params": { - "mode": "standalone", - "endpoints": ["redis://unused"], - name: value, - }, - } - }, - mysql_store=mysql, - redis_client=FakeRedis(), - ) - - -def test_legacy_redis_url_is_translated_to_cache_config(): - mysql = FakeMySQLAuthority(_api_key()) - store = RedisMySQLAccountStore( - service_name="test-service", - params={ - "redis": { - "url": ( - "redis://cache-user:cache-password@redis.example.com:6381/2" - "?socket_connect_timeout=0.25&socket_timeout=0.5" - ), - "key_prefix": "legacy-cache", - } - }, - mysql_store=mysql, - redis_client=FakeRedis(), - ) - - assert store._redis_config == { - "mode": "standalone", - "endpoints": ["redis://redis.example.com:6381"], - "username": "cache-user", - "password": "cache-password", - "db": 2, - "connect_timeout_ms": 250, - "command_timeout_ms": 500, - } - - -async def test_service_names_do_not_share_cached_identities(): - redis = FakeRedis() - first_key, second_key = _api_key("first"), _api_key("second") - first, _ = _store(first_key, redis_client=redis, service_name="first") - second, _ = _store(second_key, redis_client=redis, service_name="second") - await first.load() - await second.load() - try: - assert await first.verify_api_key(first_key, account_id_hint="acme") == ("acme", "alice") - assert await second.verify_api_key(first_key, account_id_hint="acme") is None - assert await second.verify_api_key(second_key, account_id_hint="acme") == ("acme", "alice") - await first.get_user("acme", "alice") - await second.set_role("acme", "alice", Role.USER) - assert (await first.get_user("acme", "alice"))["role"] == Role.ADMIN - assert (await second.get_user("acme", "alice"))["role"] == Role.USER - finally: - await first.close() - await second.close() - - -async def test_real_redis_mysql_service_name_isolation(): - raw_mysql = os.environ.get("OV_TEST_MYSQL") - redis_url = os.environ.get("OV_TEST_REDIS") - if not raw_mysql or not redis_url: - pytest.skip("Set OV_TEST_MYSQL and OV_TEST_REDIS for the Redis MySQL integration test") - - first_service = f"test-{uuid4().hex}" - second_service = f"test-{uuid4().hex}" - key_prefix = f"ov:test:{uuid4().hex}" - params = { - "mysql": json.loads(raw_mysql), - "cache": { - "provider": "redis", - "params": { - "mode": "standalone", - "endpoints": [redis_url], - "key_prefix": key_prefix, - "ttl_seconds": 60, - "lock_ttl_seconds": 60, - }, - }, - } - first = build_api_key_manager( - "root", - None, - service_name=first_service, - account_store_provider="redis_mysql", - account_store_params=params, - ) - second = build_api_key_manager( - "root", - None, - service_name=second_service, - account_store_provider="redis_mysql", - account_store_params=params, - ) - setup_engine = create_async_engine(first._store._mysql._url) - async with setup_engine.begin() as connection: - await connection.run_sync(Base.metadata.create_all) - await connection.execute( - delete(Account).where(Account.service_name.in_([first_service, second_service])) - ) - await setup_engine.dispose() - - await first.load() - await second.load() - try: - first_key = await first.create_account("acme", "alice") - second_key = await second.create_account("acme", "alice") - - assert (await first.resolve_identity(first_key)).role == Role.ADMIN - assert (await second.resolve_identity(second_key)).role == Role.ADMIN - with pytest.raises(UnauthenticatedError): - await first.resolve_identity(second_key) - with pytest.raises(UnauthenticatedError): - await second.resolve_identity(first_key) - - first_user_key = first._store._user_key("acme", "alice") - second_user_key = second._store._user_key("acme", "alice") - assert first_user_key != second_user_key - assert await first._store._redis.exists(first_user_key) - assert await second._store._redis.exists(second_user_key) - - await first.register_user("acme", "dave") - await second.register_user("acme", "dave") - await first.set_role("acme", "dave", Role.ADMIN) - assert await first.get_registered_user_role("acme", "dave") == Role.ADMIN - assert await second.get_registered_user_role("acme", "dave") == Role.USER - finally: - redis_client = first._store._redis - keys = [key async for key in redis_client.scan_iter(match=f"{key_prefix}:*")] - if keys: - await redis_client.delete(*keys) - async with first._store._mysql._engine.begin() as connection: - await connection.execute( - delete(Account).where(Account.service_name.in_([first_service, second_service])) - ) - await second.close() - await first.close() - - -async def test_cached_binding_respects_user_id_hint(): - api_key = _api_key() - store, _ = _store(api_key) - await store.load() - - assert await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="alice", - ) == ("acme", "alice") - assert ( - await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="bob", - ) - is None - ) - assert await store.verify_api_key( - api_key, - account_id_hint="acme", - user_id_hint="alice", - ) == ("acme", "alice") - - -async def test_cached_identity_survives_authority_read_failure(monkeypatch): - api_key = _api_key() - store, mysql = _store(api_key) - await store.load() - manager = APIKeyManager("root", store) - - first = await manager.resolve_identity(api_key) - - async def unavailable(*_args, **_kwargs): - raise ConnectionError("MySQL unavailable") - - for method in ("verify_api_key", "get_user", "get_deletion"): - monkeypatch.setattr(mysql, method, unavailable) - second = await manager.resolve_identity(api_key) - - assert (first.account_id, first.user_id, first.role) == ("acme", "alice", Role.ADMIN) - assert second == first - - -async def test_recreated_account_does_not_inherit_cached_identity(): - api_key = _api_key() - store, _ = _store(api_key) - await store.load() - manager = APIKeyManager("root", store) - - assert (await manager.resolve_identity(api_key)).role == Role.ADMIN - assert await store.get_user_group_ids("acme", "alice") == ("engineering",) - await store.begin_deletion( - "acme", "alice", task_id="delete-1", owner_account_id="acme", owner_user_id="alice" - ) - with pytest.raises(UnauthenticatedError): - await manager.resolve_identity(api_key) - - await store.delete_account("acme") - assert await store.get_user("acme", "alice") is None - with pytest.raises(UnauthenticatedError): - await manager.resolve_identity(api_key) - - new_key = _api_key("recreated") - await store.create_account_with_api_key("acme", "alice", new_key) - assert (await manager.resolve_identity(new_key)).role == Role.ADMIN - assert await store.get_user_group_ids("acme", "alice") == () - assert await store.get_deletion("acme", "alice") is None - with pytest.raises(UnauthenticatedError): - await manager.resolve_identity(api_key) - - -async def test_user_group_and_deletion_reads_are_cached_and_invalidated(): - api_key = _api_key() - store, _ = _store(api_key) - await store.load() - - assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.ADMIN} - await store.set_role("acme", "alice", Role.USER) - assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.USER} - - assert await store.get_user_group_ids("acme", "alice") == ("engineering",) - await store.add_group_member("acme", "product", "alice") - assert await store.get_user_group_ids("acme", "alice") == ("engineering", "product") - - assert await store.get_deletion("acme", "alice") is None - await store.begin_deletion( - "acme", - "alice", - task_id="delete-1", - owner_account_id="acme", - owner_user_id="alice", - ) - assert await store.get_deletion("acme", "alice") == {"task_id": "delete-1"} - - -async def test_deleting_recreated_account_rejects_key_after_delayed_cleanup(): - api_key = _api_key() - redis = FakeRedis() - store, _ = _store(api_key, redis_client=redis, lock_wait_seconds=1) - await store.load() - original_zrange = redis.zrange - saw_empty = asyncio.Event() - resume_clear = asyncio.Event() - paused = False - - async def delayed_empty_reply(*args): - nonlocal paused - result = await original_zrange(*args) - if not result and not paused: - paused = True - saw_empty.set() - await resume_clear.wait() - return result - - redis.zrange = delayed_empty_reply - old_clear = asyncio.create_task(store.delete_account("acme")) - try: - await asyncio.wait_for(saw_empty.wait(), 2) - await asyncio.wait_for(store.create_account_with_api_key("acme", "alice", api_key), 5) - manager = APIKeyManager("root", store) - assert (await manager.resolve_identity(api_key)).role == Role.ADMIN - - resume_clear.set() - await asyncio.wait_for(old_clear, 2) - - await store.delete_account("acme") - with pytest.raises(UnauthenticatedError, match="Invalid API key"): - await manager.resolve_identity(api_key) - finally: - resume_clear.set() - await asyncio.gather(old_clear, return_exceptions=True) - await store.close() - - -@pytest.mark.parametrize("operation", ["create_account", "create_user", "replace_key"]) -@pytest.mark.parametrize("user_hint", [None, "alice"]) -async def test_provisioned_key_is_immediately_usable(operation, user_hint): - old_key = _api_key("old") - new_key = _api_key("new") - store, mysql = _store(old_key) - if operation == "create_account": - mysql.account_exists = False - mysql.user_exists = False - elif operation == "create_user": - mysql.user_exists = False - await store.load() - try: - if operation == "replace_key": - assert await store.verify_api_key( - old_key, account_id_hint="acme", user_id_hint=user_hint - ) == ("acme", "alice") - assert ( - await store.verify_api_key(new_key, account_id_hint="acme", user_id_hint=user_hint) - is None - ) - if operation == "create_account": - await store.create_account_with_api_key("acme", "alice", new_key) - elif operation == "create_user": - await store.create_user_with_api_key("acme", "alice", Role.USER, new_key) - else: - await store.replace_active_user_api_key("acme", "alice", new_key) - assert await store.verify_api_key( - new_key, account_id_hint="acme", user_id_hint=user_hint - ) == ("acme", "alice") - assert ( - await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint=user_hint) - is None - ) - finally: - await store.close() - - -async def test_key_rotation_rejects_old_key_after_inflight_verification(): - old_key, new_key = _api_key("old"), _api_key("new") - store, mysql = _store(old_key, lock_wait_seconds=1) - await store.load() - read_started = asyncio.Event() - resume_read = asyncio.Event() - original_verify = mysql.verify_api_key - - async def delayed_verification(api_key, **hints): - result = await original_verify(api_key, **hints) - if api_key == old_key and not read_started.is_set(): - read_started.set() - await resume_read.wait() - return result - - mysql.verify_api_key = delayed_verification - reader = asyncio.create_task( - store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") - ) - try: - await asyncio.wait_for(read_started.wait(), 2) - await asyncio.wait_for(store.replace_active_user_api_key("acme", "alice", new_key), 5) - assert await store.verify_api_key( - new_key, account_id_hint="acme", user_id_hint="alice" - ) == ("acme", "alice") - resume_read.set() - # After the earlier request finishes, only the new key remains valid. - await asyncio.wait_for(reader, 2) - assert ( - await store.verify_api_key(old_key, account_id_hint="acme", user_id_hint="alice") - is None - ) - assert await store.verify_api_key( - new_key, account_id_hint="acme", user_id_hint="alice" - ) == ("acme", "alice") - finally: - resume_read.set() - await asyncio.gather(reader, return_exceptions=True) - await store.close() - - -@pytest.mark.parametrize("error", [ConnectionError, TimeoutError]) -async def test_redis_failure_allows_mysql_reads_and_writes(error): - redis = FakeRedis() - store, _ = _store(_api_key(), redis_client=redis) - await store.load() - - async def timeout(*_args, **_kwargs): - raise error("Redis did not respond") - - redis.get = timeout - redis.set = timeout - redis.delete = timeout - redis.eval = timeout - try: - await store.set_role("acme", "alice", Role.USER) - assert await store.get_user("acme", "alice") == {"user_id": "alice", "role": Role.USER} - finally: - await store.close() diff --git a/tests/server/test_store_assembly.py b/tests/server/test_store_assembly.py index dd78b48945..136707a2c2 100644 --- a/tests/server/test_store_assembly.py +++ b/tests/server/test_store_assembly.py @@ -1,10 +1,12 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 +import ssl from types import SimpleNamespace import pytest +from openviking.server.account_stores.redis_client import create_redis_client from openviking.server.api_keys.legacy import FileStore from openviking.server.config import AccountStoreConfig from openviking.server.store_assembly import ( @@ -15,7 +17,9 @@ from tests.server.account_store_fakes import InMemoryAGFS -def test_unknown_account_store_is_rejected(): +async def test_account_store_connection_configuration(): + with pytest.raises(ValueError, match="account_store.provider must be one of"): + AccountStoreConfig(provider="missing") with pytest.raises(InvalidArgumentError, match="Unknown account store provider"): build_account_store( viking_fs=SimpleNamespace(agfs=InMemoryAGFS()), @@ -26,28 +30,15 @@ def test_unknown_account_store_is_rejected(): account_store_watch_interval_seconds=30, ) - -def test_unknown_account_store_is_rejected_during_config_validation(): - with pytest.raises(ValueError, match="account_store.provider must be one of"): - AccountStoreConfig(provider="missing") - - -def test_mysql_account_store_uses_service_name(): - store = build_account_store( - viking_fs=None, - api_key_hashing_enabled=False, - service_name="account-cluster", - account_store_provider="mysql", - account_store_params={ - "user": "unused", - "password": "unused", - "database": "unused", - }, - account_store_watch_enabled=False, - account_store_watch_interval_seconds=30, - ) - - assert store._service_name == "account-cluster" + redis_connection = pytest.importorskip("redis.asyncio.connection") + handle = create_redis_client({"mode": "standalone", "endpoints": ["rediss://127.0.0.1:6379"]}) + try: + # Constructing only the pool would miss unsupported connection arguments. + connection = handle.client.connection_pool.make_connection() + assert isinstance(connection, redis_connection.SSLConnection) + assert connection.ssl_context.get().verify_mode == ssl.CERT_REQUIRED + finally: + await handle.close() @pytest.mark.parametrize("injected", [False, True])