From b98e8296bb690761a2e43ce0106d8048cd2974a6 Mon Sep 17 00:00:00 2001 From: EterUltimate <1831303476@qq.com> Date: Fri, 18 Sep 2026 10:42:34 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E5=91=BD=E4=BB=A4=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E7=9B=B4=E6=8E=A5=E6=94=BE=E8=A1=8C=E4=B8=8E=20LightR?= =?UTF-8?q?AG=20LLM=20=E5=93=8D=E5=BA=94=E7=BC=93=E5=AD=98=E6=B2=BB?= =?UTF-8?q?=E7=90=86=EF=BC=884.3.0=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #254: 新增 enable_command_pass_through 开关（默认开启），以系统级 命令前缀（/ ! # . 等）开头的消息跳过 LLM Hook 上下文注入直接处理， 避免功能命令响应被上下文拉取拉长；命令识别抽为 CommandFilter.is_command_text。 Issue #253: LightRAG 构造时显式关闭 enable_llm_cache 与 enable_llm_cache_for_entity_extract（可用新增配置 lightrag_enable_llm_cache 开启，默认关），修复 kv_store_llm_response_cache.json 无上限增长导致的 冷加载变慢与 LLM Hook 批量超时；缓存关闭时启动与实例创建自动清理残留 缓存文件；新增管理员命令 /clean_rag_cache 手动清理并报告释放空间。 Fixes #254 Fixes #253 --- CHANGELOG.md | 17 ++ README.md | 2 +- README_EN.md | 2 +- __init__.py | 2 +- _conf_schema.json | 12 + config.py | 8 + core/plugin_lifecycle.py | 1 + docs/README.md | 2 +- docs/configuration.md | 7 + main.py | 10 + metadata.yaml | 2 +- services/commands/command_filter.py | 11 +- services/commands/handlers.py | 33 +++ services/hooks/llm_hook_handler.py | 18 ++ .../integration/lightrag_knowledge_manager.py | 137 ++++++++++ .../test_command_passthrough_rag_cache.py | 239 ++++++++++++++++++ web_src/package.json | 2 +- webui/services/config_service.py | 6 + 18 files changed, 504 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_command_passthrough_rag_cache.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fa0eba7a..911eb766 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ 所有重要更改都将记录在此文件中。 +## [Unreleased] + +### 命令消息直接放行（issue #254） + +- 新增 `enable_command_pass_through` 开关（默认开启）：以系统级命令前缀（`/` `!` `#` `.` 等）开头的命令消息会跳过 LLM Hook 上下文注入直接处理，避免功能命令（如其他插件的管理/工具命令）的响应被上下文拉取拉长。 +- 命令识别逻辑抽为 `CommandFilter.is_command_text`，与消息收集路径的命令过滤保持同一前缀约定。 + +### LightRAG LLM 响应缓存治理（issue #253） + +- 修复 `kv_store_llm_response_cache.json` 无上限增长（长期运行单群可达 500MB+，冷加载 5~7 秒并触发 LLM Hook 批量超时）：插件构造 LightRAG 时显式关闭 `enable_llm_cache` 与 `enable_llm_cache_for_entity_extract`，可通过新增配置 `lightrag_enable_llm_cache`（默认关闭）开启。 +- 缓存关闭时自动清理历史残留：知识管理器启动时与每个群实例创建前会删除旧的 `kv_store_llm_response_cache.json`（该文件为纯缓存，删除后 JsonKVStorage 以空缓存加载，不影响图谱与向量数据）。 +- 新增管理员命令 `/clean_rag_cache`：手动清理全部（或指定）群的 LLM 响应缓存，热实例走 LightRAG `aclear_cache` API，冷群直接移除缓存文件，并报告释放空间。 + +### 版本 + +- 版本号由 4.2.1 提升至 **4.3.0**。 + ## [4.2.1] - 2026-09-12 ### LivingMemory 2.7 适配 diff --git a/README.md b/README.md index e9523143..bfd1b4dd 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ 让 AstrBot 在群聊中持续采集、学习、审查并注入上下文，使 Bot 逐步具备表达风格、群组黑话、社交关系、长期记忆和人格演化能力。 -[![Version](https://img.shields.io/badge/version-4.2.1-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) +[![Version](https://img.shields.io/badge/version-4.3.0-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) [![License](https://img.shields.io/badge/license-AGPL--3.0-green.svg)](LICENSE) [![AstrBot](https://img.shields.io/badge/AstrBot-%3E%3D4.11.4-orange.svg)](https://github.com/Soulter/AstrBot) [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/) diff --git a/README_EN.md b/README_EN.md index dd682a37..6fe1c236 100644 --- a/README_EN.md +++ b/README_EN.md @@ -14,7 +14,7 @@
-[![Version](https://img.shields.io/badge/version-4.2.1-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) [![License](https://img.shields.io/badge/license-AGPL--3.0-green.svg)](LICENSE) [![AstrBot](https://img.shields.io/badge/AstrBot-%3E%3D4.11.4-orange.svg)](https://github.com/Soulter/AstrBot) [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/) +[![Version](https://img.shields.io/badge/version-4.3.0-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) [![License](https://img.shields.io/badge/license-AGPL--3.0-green.svg)](LICENSE) [![AstrBot](https://img.shields.io/badge/AstrBot-%3E%3D4.11.4-orange.svg)](https://github.com/Soulter/AstrBot) [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/) [Features](#what-we-can-do) · [Quick Start](#quick-start) · [Web UI](#visual-management-interface) · [Community](#community) · [Contributing](CONTRIBUTING.md) diff --git a/__init__.py b/__init__.py index 7e4e83c7..9c763cf0 100644 --- a/__init__.py +++ b/__init__.py @@ -1,5 +1,5 @@ # AstrBot 自学习插件 -__version__ = "4.2.1" +__version__ = "4.3.0" # Ensure parent namespace packages ("data", "data.plugins") are # durably registered in sys.modules. AstrBot loads plugins via diff --git a/_conf_schema.json b/_conf_schema.json index d977f13b..3170c3ff 100644 --- a/_conf_schema.json +++ b/_conf_schema.json @@ -850,6 +850,12 @@ "hint": "开启后每次回复前会并行拉取社交、记忆、黑话、few-shot 等上下文；默认关闭以避免高频模型调用", "default": false }, + "enable_command_pass_through": { + "description": "命令消息直接放行", + "type": "bool", + "hint": "开启后，以系统级命令前缀（/ ! # . 等）开头的命令消息会跳过 LLM Hook 上下文注入直接处理，避免功能命令响应被拉长", + "default": true + }, "use_sqlalchemy": { "description": "强制使用 SQLAlchemy ORM", "type": "bool", @@ -977,6 +983,12 @@ "hint": "LightRAG检索模式。local=仅实体邻域检索（低延迟），hybrid=实体邻域+全局社区聚合（高质量但慢约4-5秒），naive=纯向量检索，global=仅全局社区，mix=混合模式。若同时委托 LivingMemory，hybrid/mix 会叠加记忆检索与融合上下文，可能显著增加 LLM 调用与 token 消耗，优先建议 local/naive。", "default": "local" }, + "lightrag_enable_llm_cache": { + "description": "LightRAG 缓存 LLM 响应", + "type": "bool", + "hint": "开启后 LightRAG 会缓存实体抽取与查询的 LLM 响应（kv_store_llm_response_cache.json）。该文件无上限且会拖慢冷加载；自然语言聊天查询重复率低，默认关闭。开启后可在群内用 /clean_rag_cache 清理。", + "default": false + }, "memory_engine": { "description": "记忆引擎", "type": "string", diff --git a/config.py b/config.py index b136d6e9..afcc26e0 100644 --- a/config.py +++ b/config.py @@ -183,6 +183,7 @@ class PluginConfig(BaseModel): # v2 Architecture: Knowledge engine knowledge_engine: str = "legacy" # "lightrag" | "legacy" lightrag_query_mode: str = "local" # "naive" | "local" | "global" | "hybrid" | "mix" + lightrag_enable_llm_cache: bool = False # LightRAG 缓存 LLM 响应；默认关闭避免 kv_store_llm_response_cache.json 无上限增长 # v2 Architecture: Memory engine memory_engine: str = "legacy" # "mem0" | "legacy" @@ -242,6 +243,7 @@ class PluginConfig(BaseModel): service_stop_timeout: int = 5 # 单个服务停止超时 enable_llm_hooks: bool = False # 启用 LLM Hook 上下文注入，默认关闭以避免高频调用 llm_hook_context_timeout: float = 3.0 # LLM Hook 单个上下文源超时（秒） + enable_command_pass_through: bool = True # 命令消息直接放行：跳过 LLM Hook 上下文注入，保证命令响应速度 # PersonaUpdater配置 persona_merge_strategy: str = "smart" # 人格合并策略: "replace", "append", "prepend", "smart" @@ -485,6 +487,9 @@ def create_from_config(cls, config: dict, data_dir: Optional[str] = None) -> 'Pl ), knowledge_engine=v2_settings.get('knowledge_engine', 'legacy'), lightrag_query_mode=v2_settings.get('lightrag_query_mode', 'local'), + lightrag_enable_llm_cache=v2_settings.get( + 'lightrag_enable_llm_cache', False + ), memory_engine=v2_settings.get('memory_engine', 'legacy'), # 功能融合设置 @@ -602,6 +607,9 @@ def create_from_config(cls, config: dict, data_dir: Optional[str] = None) -> 'Pl service_stop_timeout=runtime_internal_settings.get('service_stop_timeout', 5), enable_llm_hooks=runtime_internal_settings.get('enable_llm_hooks', False), llm_hook_context_timeout=float(runtime_internal_settings.get('llm_hook_context_timeout', 3.0)), + enable_command_pass_through=runtime_internal_settings.get( + 'enable_command_pass_through', True + ), llm_hook_injection_target=runtime_internal_settings.get( 'llm_hook_injection_target', CACHE_FRIENDLY_LLM_HOOK_TARGET, diff --git a/core/plugin_lifecycle.py b/core/plugin_lifecycle.py index 1c8f26ed..9a72c283 100644 --- a/core/plugin_lifecycle.py +++ b/core/plugin_lifecycle.py @@ -267,6 +267,7 @@ def bootstrap( db_manager=p.db_manager, llm_adapter=p.llm_adapter, remember_service=p.remember_service, + v2_integration=getattr(p, "v2_integration", None), ) p._command_filter = CommandFilter() diff --git a/docs/README.md b/docs/README.md index 43c55f55..5dbe4684 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ AstrBot 自主学习插件的实现文档和使用文档。 - 插件名: `astrbot_plugin_self_learning` - 展示名: `self-learning` -- 当前元数据版本: `4.2.1` +- 当前元数据版本: `4.3.0` - 最低 AstrBot 版本: `4.11.4` - 主要入口: `main.py` - 配置入口: `_conf_schema.json`, `config.py` diff --git a/docs/configuration.md b/docs/configuration.md index 8aacd4f7..53c96428 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -149,6 +149,7 @@ PostgreSQL 支持 `postgresql_schema`，非 `public` 时会自动创建 schema | `rerank_min_candidates` | `3` | 候选数低于该值跳过 rerank | | `knowledge_engine` | `legacy` | `legacy` 或 `lightrag` | | `lightrag_query_mode` | `local` | LightRAG 查询模式 | +| `lightrag_enable_llm_cache` | `false` | LightRAG 缓存 LLM 响应（实体抽取与查询）。缓存文件无上限且拖慢冷加载，聊天场景重复率低，默认关闭；开启后可用 `/clean_rag_cache` 清理 | | `memory_engine` | `legacy` | `legacy` 或 `mem0` | 成本提示: 当 `knowledge_engine="lightrag"` 且 `lightrag_query_mode` 为 @@ -156,6 +157,12 @@ PostgreSQL 支持 `postgresql_schema`，非 `public` 时会自动创建 schema LivingMemory 已加载后会叠加 LightRAG 全局/混合检索与记忆检索，可能明显增加 LLM 调用和 token 消耗。优先建议使用 `local`/`naive`，或只保留一种记忆/检索策略。 +`lightrag_enable_llm_cache` 关闭时（默认），插件会在启动与各群实例创建时自动 +删除残留的 `kv_store_llm_response_cache.json`（纯缓存数据，不影响图谱/向量）； +也可随时用管理员命令 `/clean_rag_cache` 手动清理。若开启缓存导致该文件增长过 +大引发冷加载变慢与 LLM Hook 超时，可关闭该开关后重启，或适当调大 +`llm_hook_context_timeout` 作为缓解。 + 只有 `knowledge_engine != "legacy"` 或 `memory_engine != "legacy"` 时才创建 `V2LearningIntegration`。 `embedding_provider_id` 只显示 Embedding Provider，`rerank_provider_id` 只显示 Reranker Provider。聊天模型不会混入这两个下拉框。 diff --git a/main.py b/main.py index c07c8d7a..b01334ee 100644 --- a/main.py +++ b/main.py @@ -558,6 +558,16 @@ async def remember_command(self, event: AstrMessageEvent): async for result in self._command_handlers.remember(event): yield result + @filter.command("clean_rag_cache") + @filter.permission_type(filter.PermissionType.ADMIN) + async def clean_rag_cache_command(self, event: AstrMessageEvent): + """清理 LightRAG LLM 响应缓存""" + if not self._command_handlers: + yield event.plain_result("插件服务未就绪，请检查启动日志") + return + async for result in self._command_handlers.clean_rag_cache(event): + yield result + @filter.command("affection_status") @filter.permission_type(filter.PermissionType.ADMIN) async def affection_status_command(self, event: AstrMessageEvent): diff --git a/metadata.yaml b/metadata.yaml index 40d447b3..7355be79 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -2,7 +2,7 @@ name: "astrbot_plugin_self_learning" author: "NickMo, EterUltimate" display_name: "self-learning" description: "SELF LEARNING 自主学习插件 — 让 AI 聊天机器人自主学习对话风格、理解群组黑话、管理社交关系与好感度、自适应人格演化，像真人一样自然对话。(使用前必须手动备份人格数据)" -version: "4.2.1" +version: "4.3.0" repo: "https://github.com/NickCharlie/astrbot_plugin_self_learning" tags: - "自学习" diff --git a/services/commands/command_filter.py b/services/commands/command_filter.py index a8851bb5..6d4c56d9 100644 --- a/services/commands/command_filter.py +++ b/services/commands/command_filter.py @@ -14,6 +14,7 @@ class CommandFilter: "affection_status", "set_mood", "remember", + "clean_rag_cache", ] def is_astrbot_command(self, event: Any) -> bool: @@ -29,8 +30,16 @@ def is_astrbot_command(self, event: Any) -> bool: if self.is_plugin_command(message_text): return True + return self.is_command_text(message_text) + + @staticmethod + def is_command_text(message_text: Any) -> bool: + """判断纯文本是否为命令格式（系统级命令前缀 + 命令词）""" + if not message_text: + return False + command_prefixes = ["/", "!", "#", "."] - stripped_text = message_text.strip() + stripped_text = str(message_text).strip() if stripped_text and stripped_text[0] in command_prefixes: if len(stripped_text) > 1 and stripped_text[1].isalpha(): return True diff --git a/services/commands/handlers.py b/services/commands/handlers.py index 3acd3f32..f6f64c12 100644 --- a/services/commands/handlers.py +++ b/services/commands/handlers.py @@ -23,6 +23,7 @@ def __init__( db_manager: Any, llm_adapter: Any, remember_service: Any = None, + v2_integration: Any = None, ): self._config = plugin_config self._service_factory = service_factory @@ -34,6 +35,7 @@ def __init__( self._db_manager = db_manager self._llm_adapter = llm_adapter self._remember_service = remember_service + self._v2_integration = v2_integration self._force_learning_in_progress: set = set() # learning_status @@ -290,6 +292,37 @@ async def remember(self, event: Any) -> AsyncGenerator: logger.error(f"remember 命令处理失败: {e}", exc_info=True) yield event.plain_result(f"remember 失败：{str(e)}") + # clean_rag_cache + + async def clean_rag_cache(self, event: Any) -> AsyncGenerator: + """清理 LightRAG LLM 响应缓存（issue #253 维护入口）""" + try: + knowledge_manager = getattr( + self._v2_integration, "_knowledge_manager", None + ) + if not hasattr(knowledge_manager, "clear_llm_response_cache"): + yield event.plain_result( + "当前知识引擎不是 lightrag，无需清理 LightRAG 缓存" + ) + return + + yield event.plain_result("正在清理 LightRAG LLM 响应缓存...") + result = await knowledge_manager.clear_llm_response_cache() + + freed_mb = result.get("freed_bytes", 0) / 1024 / 1024 + cleared = result.get("cleared", []) or [] + errors = result.get("errors", []) or [] + lines = [f"清理完成：{len(cleared)} 个群，释放 {freed_mb:.1f} MB"] + if cleared: + lines.append("涉及群组: " + ", ".join(str(g) for g in cleared)) + for error in errors: + lines.append(f"失败: {error}") + yield event.plain_result("\n".join(lines)) + + except Exception as e: + logger.error(f"clean_rag_cache 命令处理失败: {e}", exc_info=True) + yield event.plain_result(f"清理 LightRAG 缓存失败：{str(e)}") + # affection_status async def affection_status(self, event: Any) -> AsyncGenerator: diff --git a/services/hooks/llm_hook_handler.py b/services/hooks/llm_hook_handler.py index c56c76ca..ac447019 100644 --- a/services/hooks/llm_hook_handler.py +++ b/services/hooks/llm_hook_handler.py @@ -28,6 +28,10 @@ from ...utils.persona_selection import get_event_persona_scope except ImportError: from utils.persona_selection import get_event_persona_scope +try: + from ...services.commands.command_filter import CommandFilter +except ImportError: + from services.commands.command_filter import CommandFilter try: from astrbot.core.agent.message import TextPart @@ -80,6 +84,7 @@ def __init__( self._db_manager = db_manager self._feature_delegation = feature_delegation self._shadow_mode_service = shadow_mode_service + self._command_filter = CommandFilter() if self._shadow_mode_service is None and db_manager is not None: try: from ..shadow_mode import ShadowModeService @@ -105,6 +110,19 @@ async def handle(self, event: AstrMessageEvent, req: Any) -> None: logger.debug("[LLM Hook] 总开关未启用，跳过上下文注入") return + # 命令/系统级唤醒词消息直接放行：命令回复追求即时响应， + # 上下文注入对命令处理没有价值，反而会占用最长 3s 的预算。 + if getattr(self._config, "enable_command_pass_through", True): + message_text = ( + getattr(event, "message_str", None) or event.get_message_str() + ) + if self._command_filter.is_command_text(message_text): + logger.debug( + f"[LLM Hook] 命令消息直接放行，跳过上下文注入: " + f"{str(message_text)[:80]}" + ) + return + if not self._diversity_manager: logger.debug("[LLM Hook] diversity_manager未初始化,跳过多样性注入") return diff --git a/services/integration/lightrag_knowledge_manager.py b/services/integration/lightrag_knowledge_manager.py index beb0691a..1b68ef6c 100644 --- a/services/integration/lightrag_knowledge_manager.py +++ b/services/integration/lightrag_knowledge_manager.py @@ -44,6 +44,9 @@ QueryParam = None # type: ignore[assignment,misc] EmbeddingFunc = None # type: ignore[assignment,misc] +# Filename of the LightRAG LLM response cache KV store (JsonKVStorage). +LLM_RESPONSE_CACHE_FILENAME = "kv_store_llm_response_cache.json" + class LightRAGKnowledgeManager: """Knowledge manager backed by the LightRAG library. @@ -105,6 +108,18 @@ def __init__( async def start(self) -> bool: """Start the knowledge manager service.""" self._status = ServiceLifecycle.RUNNING + + # With the LLM response cache disabled, drop stale cache files left + # over from previous runs: JsonKVStorage always loads the file on + # initialization, so a multi-hundred-MB leftover would slow down every + # cold start even when nothing reads from it. + freed = self._sweep_stale_cache_files() + if freed: + logger.info( + f"[LightRAG] LLM response cache disabled; removed stale cache " + f"file(s) totalling {freed / 1024 / 1024:.1f} MB" + ) + logger.info("[LightRAG] Knowledge manager started") return True @@ -331,6 +346,116 @@ async def get_knowledge_graph_statistics( # Internal helpers + def _llm_cache_enabled(self) -> bool: + """Whether LightRAG should cache LLM responses (default: off). + + Natural-language chat queries rarely repeat, so the unbounded + ``llm_response_cache`` grew to hundreds of MB per group and slowed + every cold start; the cache therefore stays opt-in. + """ + return bool(getattr(self._config, "lightrag_enable_llm_cache", False)) + + def _remove_stale_cache_file(self, working_dir: str) -> int: + """Delete a leftover ``kv_store_llm_response_cache.json`` if present. + + Safe only while no live LightRAG instance owns *working_dir* (a warm + instance would re-persist its in-memory copy on finalize). Returns + the size in bytes of the removed file, or 0 when nothing was removed. + """ + cache_file = os.path.join(working_dir, LLM_RESPONSE_CACHE_FILENAME) + try: + if not os.path.isfile(cache_file): + return 0 + size = os.path.getsize(cache_file) + os.remove(cache_file) + return size + except OSError as exc: + logger.warning(f"[LightRAG] Could not remove LLM cache file: {exc}") + return 0 + + def _sweep_stale_cache_files(self) -> int: + """Remove stale LLM cache files across all groups (cache disabled).""" + if self._llm_cache_enabled(): + return 0 + freed_total = 0 + try: + group_dirs = [ + os.path.join(self._base_dir, name) + for name in os.listdir(self._base_dir) + if os.path.isdir(os.path.join(self._base_dir, name)) + ] + except OSError: + return 0 + for working_dir in group_dirs: + freed_total += self._remove_stale_cache_file(working_dir) + return freed_total + + async def clear_llm_response_cache( + self, group_ids: Optional[List[str]] = None + ) -> Dict[str, Any]: + """Clear the LLM response cache, optionally for specific groups. + + Warm instances go through ``LightRAG.aclear_cache`` so the in-memory + KV store stays consistent; cold groups have the cache file removed + directly (JsonKVStorage loads a missing file as empty). Groups with + the cache enabled keep their file untouched unless cleared explicitly + through the LightRAG API. + + Returns: + Dict with ``cleared`` group ids, ``freed_bytes`` and ``errors``. + """ + try: + group_names = [ + name + for name in os.listdir(self._base_dir) + if os.path.isdir(os.path.join(self._base_dir, name)) + ] + except OSError: + group_names = [] + + if group_ids: + wanted = set(group_ids) + group_names = [name for name in group_names if name in wanted] + for group_id in group_ids: + if group_id not in group_names: + group_names.append(group_id) + + cleared: List[str] = [] + errors: List[str] = [] + freed_total = 0 + + for group_id in group_names: + working_dir = os.path.join(self._base_dir, group_id) + cache_file = os.path.join(working_dir, LLM_RESPONSE_CACHE_FILENAME) + try: + size_before = ( + os.path.getsize(cache_file) + if os.path.isfile(cache_file) + else 0 + ) + rag = self._instances.get(group_id) + if rag is not None: + await rag.aclear_cache() + else: + self._remove_stale_cache_file(working_dir) + size_after = ( + os.path.getsize(cache_file) + if os.path.isfile(cache_file) + else 0 + ) + freed = max(size_before - size_after, 0) + if size_before or size_after: + cleared.append(group_id) + freed_total += freed + except Exception as exc: + errors.append(f"{group_id}: {exc}") + + return { + "cleared": cleared, + "freed_bytes": freed_total, + "errors": errors, + } + async def _get_rag(self, group_id: str) -> LightRAG: """Return the LightRAG instance for *group_id*, creating if needed. @@ -361,6 +486,18 @@ async def _get_rag(self, group_id: str) -> LightRAG: "entity_extract_max_gleaning": 1, } + # LLM response cache opt-in (default off): without it, lightrag's + # kv_store_llm_response_cache.json grows without bound and slows + # every cold load (issue #253). + enable_cache = self._llm_cache_enabled() + rag_kwargs["enable_llm_cache"] = enable_cache + rag_kwargs["enable_llm_cache_for_entity_extract"] = enable_cache + if not enable_cache: + # No live instance owns this dir here (guarded by the init + # lock), so removing a leftover cache file is safe and keeps + # cold starts fast. + self._remove_stale_cache_file(working_dir) + # Attach embedding function -- required for vector storage. if not self._embedding: raise RuntimeError( diff --git a/tests/unit/test_command_passthrough_rag_cache.py b/tests/unit/test_command_passthrough_rag_cache.py new file mode 100644 index 00000000..ee5735ad --- /dev/null +++ b/tests/unit/test_command_passthrough_rag_cache.py @@ -0,0 +1,239 @@ +"""Issue #254 / #253 回归测试：命令直接放行与 LightRAG LLM 响应缓存治理""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from self_learning_EterU.services.commands.command_filter import CommandFilter +from self_learning_EterU.services.hooks.llm_hook_handler import LLMHookHandler +from self_learning_EterU.services.integration import lightrag_knowledge_manager as lkm + + +# --------------------------------------------------------------------------- +# Issue #254: 命令/系统级唤醒词直接放行 +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "text,expected", + [ + ("/learning_status", True), + ("#remember 这段要记住", True), + ("!provider", True), + (".help", True), + ("今天天气怎么样", False), + ("/123", False), + ("//_comment", False), + ("", False), + (None, False), + ], +) +def test_command_filter_detects_command_text(text, expected): + assert CommandFilter.is_command_text(text) is expected + + +def _make_event(message_text): + return SimpleNamespace( + message_str=message_text, + get_message_str=lambda: message_text, + get_group_id=lambda: "group-1", + get_sender_id=lambda: "user-1", + unified_msg_origin="qq:group:group-1", + ) + + +def _make_handler(config, diversity_manager): + return LLMHookHandler( + plugin_config=config, + diversity_manager=diversity_manager, + social_context_injector=None, + v2_integration=None, + jargon_query_service=None, + temporary_persona_updater=None, + perf_tracker=SimpleNamespace(record=lambda payload: None), + group_id_to_unified_origin={}, + ) + + +def _hook_config(**overrides): + values = { + "enable_llm_hooks": True, + "enable_command_pass_through": True, + "llm_hook_context_timeout": 0.5, + } + values.update(overrides) + return SimpleNamespace(**values) + + +@pytest.mark.asyncio +async def test_llm_hook_skips_injection_for_command_messages(): + diversity_manager = AsyncMock() + handler = _make_handler(_hook_config(), diversity_manager) + req = SimpleNamespace(prompt="帮我记住一段话", extra_user_content_parts=None) + + await handler.handle(_make_event("/remember 记住这段"), req) + + diversity_manager.build_diversity_prompt_injection.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_llm_hook_still_injects_for_normal_messages(): + diversity_manager = AsyncMock() + diversity_manager.build_diversity_prompt_injection.return_value = "风格注入" + diversity_manager.get_current_style.return_value = "default" + diversity_manager.get_current_pattern.return_value = "chat" + handler = _make_handler(_hook_config(), diversity_manager) + req = SimpleNamespace(prompt="今天天气怎么样", extra_user_content_parts=None) + + await handler.handle(_make_event("今天天气怎么样"), req) + + diversity_manager.build_diversity_prompt_injection.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_llm_hook_pass_through_can_be_disabled(): + diversity_manager = AsyncMock() + diversity_manager.build_diversity_prompt_injection.return_value = None + handler = _make_handler( + _hook_config(enable_command_pass_through=False), diversity_manager + ) + req = SimpleNamespace(prompt="/provider", extra_user_content_parts=None) + + await handler.handle(_make_event("/provider 查看列表"), req) + + diversity_manager.build_diversity_prompt_injection.assert_awaited_once() + + +# --------------------------------------------------------------------------- +# Issue #253: LightRAG llm_response_cache 治理 +# --------------------------------------------------------------------------- + + +@pytest.fixture +def rag_env(tmp_path, monkeypatch): + """Construct a manager without the optional lightrag dependency.""" + monkeypatch.setattr(lkm, "_LIGHTRAG_AVAILABLE", True) + config = SimpleNamespace( + data_dir=str(tmp_path), + lightrag_enable_llm_cache=False, + ) + manager = lkm.LightRAGKnowledgeManager( + config, llm_adapter=object(), embedding_provider=None + ) + return manager, tmp_path + + +def _write_cache_file(base_dir, group_id, content=b"x" * 2048): + group_dir = base_dir / "lightrag" / group_id + group_dir.mkdir(parents=True, exist_ok=True) + cache_file = group_dir / lkm.LLM_RESPONSE_CACHE_FILENAME + cache_file.write_bytes(content) + return cache_file + + +@pytest.mark.asyncio +async def test_start_sweeps_stale_cache_files_when_disabled(rag_env): + manager, tmp_path = rag_env + cache_file = _write_cache_file(tmp_path, "12345") + + await manager.start() + + assert not cache_file.exists() + + +@pytest.mark.asyncio +async def test_start_keeps_cache_files_when_enabled(tmp_path, monkeypatch): + monkeypatch.setattr(lkm, "_LIGHTRAG_AVAILABLE", True) + config = SimpleNamespace( + data_dir=str(tmp_path), + lightrag_enable_llm_cache=True, + ) + manager = lkm.LightRAGKnowledgeManager( + config, llm_adapter=object(), embedding_provider=None + ) + cache_file = _write_cache_file(tmp_path, "12345") + + await manager.start() + + assert cache_file.exists() + + +@pytest.mark.asyncio +async def test_clear_llm_response_cache_removes_cold_files(rag_env): + manager, tmp_path = rag_env + cache_a = _write_cache_file(tmp_path, "111") + cache_b = _write_cache_file(tmp_path, "222") + expected_bytes = cache_a.stat().st_size + cache_b.stat().st_size + + result = await manager.clear_llm_response_cache() + + assert sorted(result["cleared"]) == ["111", "222"] + assert result["freed_bytes"] >= expected_bytes + assert not cache_a.exists() and not cache_b.exists() + + +@pytest.mark.asyncio +async def test_clear_llm_response_cache_group_filter(rag_env): + manager, tmp_path = rag_env + cache_a = _write_cache_file(tmp_path, "111") + cache_b = _write_cache_file(tmp_path, "222") + + result = await manager.clear_llm_response_cache(group_ids=["111"]) + + assert result["cleared"] == ["111"] + assert not cache_a.exists() or cache_a.stat().st_size == 0 + assert cache_b.exists() + + +@pytest.mark.asyncio +async def test_clear_llm_response_cache_uses_api_for_warm_instances( + rag_env, monkeypatch +): + manager, tmp_path = rag_env + cache_file = _write_cache_file(tmp_path, "111") + warm_rag = SimpleNamespace(aclear_cache=AsyncMock()) + manager._instances["111"] = warm_rag + + result = await manager.clear_llm_response_cache(group_ids=["111"]) + + warm_rag.aclear_cache.assert_awaited_once() + assert result["cleared"] == ["111"] + assert result["errors"] == [] + assert cache_file.exists() # 模拟实例不落盘，文件保持原样 + + +@pytest.mark.asyncio +async def test_get_rag_disables_llm_cache_by_default(tmp_path, monkeypatch): + monkeypatch.setattr(lkm, "_LIGHTRAG_AVAILABLE", True) + captured_kwargs = {} + + class _DummyRAG: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def initialize_storages(self): + return None + + async def initialize_pipeline_status(self): + return None + + monkeypatch.setattr(lkm, "LightRAG", _DummyRAG) + monkeypatch.setattr( + lkm, "EmbeddingFunc", lambda **kwargs: SimpleNamespace(**kwargs) + ) + embedding = SimpleNamespace(get_dim=lambda: 1024, get_embeddings=AsyncMock()) + config = SimpleNamespace( + data_dir=str(tmp_path), + lightrag_enable_llm_cache=False, + ) + manager = lkm.LightRAGKnowledgeManager( + config, llm_adapter=object(), embedding_provider=embedding + ) + + stale = _write_cache_file(tmp_path, "12345") + await manager._get_rag("12345") + + assert captured_kwargs["enable_llm_cache"] is False + assert captured_kwargs["enable_llm_cache_for_entity_extract"] is False + assert not stale.exists() diff --git a/web_src/package.json b/web_src/package.json index 116a78ea..65317f59 100644 --- a/web_src/package.json +++ b/web_src/package.json @@ -1,6 +1,6 @@ { "name": "astrbot-self-learning-dashboard", - "version": "4.2.1", + "version": "4.3.0", "description": "Solid.js source for the self-learning plugin dashboard", "type": "module", "scripts": { diff --git a/webui/services/config_service.py b/webui/services/config_service.py index 4d8e2661..c1670183 100644 --- a/webui/services/config_service.py +++ b/webui/services/config_service.py @@ -181,6 +181,12 @@ def _load_schema_definition() -> Dict[str, Any]: "hint": "开启后每次回复前会并行拉取社交、记忆、黑话、few-shot 等上下文；默认关闭以避免高频模型调用", "default": False, }, + "enable_command_pass_through": { + "description": "命令消息直接放行", + "type": "bool", + "hint": "开启后，以系统级命令前缀（/ ! # . 等）开头的命令消息会跳过 LLM Hook 上下文注入直接处理，避免功能命令响应被拉长", + "default": True, + }, "use_sqlalchemy": { "description": "强制使用 SQLAlchemy ORM", "type": "bool", From 419980815dfe329971b355d6fbe954080d557983 Mon Sep 17 00:00:00 2001 From: EterUltimate <1831303476@qq.com> Date: Fri, 18 Sep 2026 10:52:49 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(commands):=20/clean=5Frag=5Fcache=20?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=8C=87=E5=AE=9A=E7=BE=A4=E5=8F=B7=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 Sourcery 审查意见补齐群过滤实现：解析命令 payload 中的空格/逗号 分隔群号并传入 clear_llm_response_cache(group_ids=...)，与文档描述 保持一致；无参数时清理全部群。 --- services/commands/handlers.py | 25 +++++- .../test_command_passthrough_rag_cache.py | 87 +++++++++++++++++++ 2 files changed, 109 insertions(+), 3 deletions(-) diff --git a/services/commands/handlers.py b/services/commands/handlers.py index f6f64c12..d4a394dd 100644 --- a/services/commands/handlers.py +++ b/services/commands/handlers.py @@ -295,7 +295,11 @@ async def remember(self, event: Any) -> AsyncGenerator: # clean_rag_cache async def clean_rag_cache(self, event: Any) -> AsyncGenerator: - """清理 LightRAG LLM 响应缓存（issue #253 维护入口）""" + """清理 LightRAG LLM 响应缓存（issue #253 维护入口） + + 用法：``/clean_rag_cache`` 清理全部群；``/clean_rag_cache 12345`` 或 + 空格/逗号分隔的多个群号只清理指定群。 + """ try: knowledge_manager = getattr( self._v2_integration, "_knowledge_manager", None @@ -306,13 +310,28 @@ async def clean_rag_cache(self, event: Any) -> AsyncGenerator: ) return + payload = self._extract_command_payload(event, "clean_rag_cache") + group_ids = [ + group_id + for group_id in payload.replace(",", " ").split() + if group_id + ] + yield event.plain_result("正在清理 LightRAG LLM 响应缓存...") - result = await knowledge_manager.clear_llm_response_cache() + result = await knowledge_manager.clear_llm_response_cache( + group_ids=group_ids or None + ) freed_mb = result.get("freed_bytes", 0) / 1024 / 1024 cleared = result.get("cleared", []) or [] errors = result.get("errors", []) or [] - lines = [f"清理完成：{len(cleared)} 个群，释放 {freed_mb:.1f} MB"] + if group_ids: + lines = [ + f"清理完成：指定 {len(group_ids)} 个群，" + f"涉及 {len(cleared)} 个群，释放 {freed_mb:.1f} MB" + ] + else: + lines = [f"清理完成：{len(cleared)} 个群，释放 {freed_mb:.1f} MB"] if cleared: lines.append("涉及群组: " + ", ".join(str(g) for g in cleared)) for error in errors: diff --git a/tests/unit/test_command_passthrough_rag_cache.py b/tests/unit/test_command_passthrough_rag_cache.py index ee5735ad..0723538a 100644 --- a/tests/unit/test_command_passthrough_rag_cache.py +++ b/tests/unit/test_command_passthrough_rag_cache.py @@ -6,6 +6,7 @@ import pytest from self_learning_EterU.services.commands.command_filter import CommandFilter +from self_learning_EterU.services.commands.handlers import PluginCommandHandlers from self_learning_EterU.services.hooks.llm_hook_handler import LLMHookHandler from self_learning_EterU.services.integration import lightrag_knowledge_manager as lkm @@ -237,3 +238,89 @@ async def initialize_pipeline_status(self): assert captured_kwargs["enable_llm_cache"] is False assert captured_kwargs["enable_llm_cache_for_entity_extract"] is False assert not stale.exists() + + +# --------------------------------------------------------------------------- +# /clean_rag_cache 命令 +# --------------------------------------------------------------------------- + + +def _make_command_handler(v2_integration): + return PluginCommandHandlers( + plugin_config=SimpleNamespace(), + service_factory=None, + message_collector=None, + persona_manager=None, + progressive_learning=None, + affection_manager=None, + temporary_persona_updater=None, + db_manager=None, + llm_adapter=None, + v2_integration=v2_integration, + ) + + +def _command_event(message_text): + return SimpleNamespace( + get_message_str=lambda: message_text, + plain_result=lambda text: text, + ) + + +async def _collect(async_gen): + return [item async for item in async_gen] + + +@pytest.mark.asyncio +async def test_clean_rag_cache_parses_group_argument(): + clear_mock = AsyncMock( + return_value={"cleared": ["111"], "freed_bytes": 1024, "errors": []} + ) + handler = _make_command_handler( + SimpleNamespace( + _knowledge_manager=SimpleNamespace( + clear_llm_response_cache=clear_mock + ) + ) + ) + + replies = await _collect( + handler.clean_rag_cache(_command_event("/clean_rag_cache 111")) + ) + + clear_mock.assert_awaited_once_with(group_ids=["111"]) + assert any("清理完成" in reply for reply in replies) + + +@pytest.mark.asyncio +async def test_clean_rag_cache_without_argument_clears_all_groups(): + clear_mock = AsyncMock( + return_value={"cleared": ["111", "222"], "freed_bytes": 2048, "errors": []} + ) + handler = _make_command_handler( + SimpleNamespace( + _knowledge_manager=SimpleNamespace( + clear_llm_response_cache=clear_mock + ) + ) + ) + + replies = await _collect( + handler.clean_rag_cache(_command_event("/clean_rag_cache")) + ) + + clear_mock.assert_awaited_once_with(group_ids=None) + assert any("111" in reply and "222" in reply for reply in replies) + + +@pytest.mark.asyncio +async def test_clean_rag_cache_reports_non_lightrag_engine(): + handler = _make_command_handler( + SimpleNamespace(_knowledge_manager=None) + ) + + replies = await _collect( + handler.clean_rag_cache(_command_event("/clean_rag_cache")) + ) + + assert any("不是 lightrag" in reply for reply in replies) From ed35d105b1f203e044f607241117723539a1258c Mon Sep 17 00:00:00 2001 From: EterUltimate <1831303476@qq.com> Date: Fri, 18 Sep 2026 11:11:26 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(security):=20=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E6=B8=85=E7=90=86=E8=B7=AF=E5=BE=84=E6=A0=A1=E9=AA=8C=E4=B8=8E?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=94=81=E7=AB=9E=E6=80=81=EF=BC=9B?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=9F=BA=E5=87=86=E4=BF=AE=E6=AD=A3=E4=B8=BA?= =?UTF-8?q?=204.2.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 Sourcery 复审意见： - /clean_rag_cache 群号参数按安全格式校验，缓存文件删除前用 realpath 校验目标路径位于 LightRAG 数据目录内，阻断路径穿越删除； - clear_llm_response_cache 与 _get_rag 共用 per-group 初始化锁并在 锁内重新判定冷/热，消除并发初始化下缓存复活的竞态。 版本基准按仓库惯例改为最近已发布 tag（4.2.1）+0.0.1 = 4.2.2， 同步 6 处版本号并将 CHANGELOG [Unreleased] 定版为 [4.2.2]。 --- CHANGELOG.md | 9 +- README.md | 2 +- README_EN.md | 2 +- __init__.py | 2 +- docs/README.md | 2 +- metadata.yaml | 2 +- .../integration/lightrag_knowledge_manager.py | 93 +++++++++++++------ .../test_command_passthrough_rag_cache.py | 62 +++++++++++++ web_src/package.json | 2 +- 9 files changed, 138 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 911eb766..4ea656eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ 所有重要更改都将记录在此文件中。 -## [Unreleased] +## [4.2.2] - 2026-09-18 ### 命令消息直接放行（issue #254） @@ -15,9 +15,14 @@ - 缓存关闭时自动清理历史残留：知识管理器启动时与每个群实例创建前会删除旧的 `kv_store_llm_response_cache.json`（该文件为纯缓存，删除后 JsonKVStorage 以空缓存加载，不影响图谱与向量数据）。 - 新增管理员命令 `/clean_rag_cache`：手动清理全部（或指定）群的 LLM 响应缓存，热实例走 LightRAG `aclear_cache` API，冷群直接移除缓存文件，并报告释放空间。 +### 审查修复 + +- `/clean_rag_cache` 的群号参数按安全格式校验（仅接受字母/数字/下划线/连字符/冒号，拒绝路径穿越）；缓存文件删除前校验目标路径始终位于 LightRAG 数据目录内。 +- 缓存清理与群实例初始化共用同一把 per-group 锁并在锁内重新判定冷/热，消除并发初始化时"刚清理的缓存被新实例复活"的竞态。 + ### 版本 -- 版本号由 4.2.1 提升至 **4.3.0**。 +- 版本号由 4.2.1 提升至 **4.2.2**。 ## [4.2.1] - 2026-09-12 diff --git a/README.md b/README.md index bfd1b4dd..b0c22996 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ 让 AstrBot 在群聊中持续采集、学习、审查并注入上下文，使 Bot 逐步具备表达风格、群组黑话、社交关系、长期记忆和人格演化能力。 -[![Version](https://img.shields.io/badge/version-4.3.0-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) +[![Version](https://img.shields.io/badge/version-4.2.2-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) [![License](https://img.shields.io/badge/license-AGPL--3.0-green.svg)](LICENSE) [![AstrBot](https://img.shields.io/badge/AstrBot-%3E%3D4.11.4-orange.svg)](https://github.com/Soulter/AstrBot) [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/) diff --git a/README_EN.md b/README_EN.md index 6fe1c236..016f09cc 100644 --- a/README_EN.md +++ b/README_EN.md @@ -14,7 +14,7 @@
-[![Version](https://img.shields.io/badge/version-4.3.0-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) [![License](https://img.shields.io/badge/license-AGPL--3.0-green.svg)](LICENSE) [![AstrBot](https://img.shields.io/badge/AstrBot-%3E%3D4.11.4-orange.svg)](https://github.com/Soulter/AstrBot) [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/) +[![Version](https://img.shields.io/badge/version-4.2.2-blue.svg)](https://github.com/NickCharlie/astrbot_plugin_self_learning) [![License](https://img.shields.io/badge/license-AGPL--3.0-green.svg)](LICENSE) [![AstrBot](https://img.shields.io/badge/AstrBot-%3E%3D4.11.4-orange.svg)](https://github.com/Soulter/AstrBot) [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/) [Features](#what-we-can-do) · [Quick Start](#quick-start) · [Web UI](#visual-management-interface) · [Community](#community) · [Contributing](CONTRIBUTING.md) diff --git a/__init__.py b/__init__.py index 9c763cf0..cca75ed6 100644 --- a/__init__.py +++ b/__init__.py @@ -1,5 +1,5 @@ # AstrBot 自学习插件 -__version__ = "4.3.0" +__version__ = "4.2.2" # Ensure parent namespace packages ("data", "data.plugins") are # durably registered in sys.modules. AstrBot loads plugins via diff --git a/docs/README.md b/docs/README.md index 5dbe4684..62c8d42b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ AstrBot 自主学习插件的实现文档和使用文档。 - 插件名: `astrbot_plugin_self_learning` - 展示名: `self-learning` -- 当前元数据版本: `4.3.0` +- 当前元数据版本: `4.2.2` - 最低 AstrBot 版本: `4.11.4` - 主要入口: `main.py` - 配置入口: `_conf_schema.json`, `config.py` diff --git a/metadata.yaml b/metadata.yaml index 7355be79..4c5970dc 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -2,7 +2,7 @@ name: "astrbot_plugin_self_learning" author: "NickMo, EterUltimate" display_name: "self-learning" description: "SELF LEARNING 自主学习插件 — 让 AI 聊天机器人自主学习对话风格、理解群组黑话、管理社交关系与好感度、自适应人格演化，像真人一样自然对话。(使用前必须手动备份人格数据)" -version: "4.3.0" +version: "4.2.2" repo: "https://github.com/NickCharlie/astrbot_plugin_self_learning" tags: - "自学习" diff --git a/services/integration/lightrag_knowledge_manager.py b/services/integration/lightrag_knowledge_manager.py index 1b68ef6c..d392b13b 100644 --- a/services/integration/lightrag_knowledge_manager.py +++ b/services/integration/lightrag_knowledge_manager.py @@ -22,6 +22,7 @@ import asyncio import os +import re import time from typing import Any, Dict, List, Optional @@ -47,6 +48,10 @@ # Filename of the LightRAG LLM response cache KV store (JsonKVStorage). LLM_RESPONSE_CACHE_FILENAME = "kv_store_llm_response_cache.json" +# Group ids are numeric platform ids (optionally with provider prefixes); +# anything else must never reach filesystem paths. +SAFE_GROUP_ID_PATTERN = re.compile(r"^[A-Za-z0-9_\-:]{1,64}$") + class LightRAGKnowledgeManager: """Knowledge manager backed by the LightRAG library. @@ -362,7 +367,20 @@ def _remove_stale_cache_file(self, working_dir: str) -> int: instance would re-persist its in-memory copy on finalize). Returns the size in bytes of the removed file, or 0 when nothing was removed. """ - cache_file = os.path.join(working_dir, LLM_RESPONSE_CACHE_FILENAME) + # Defense in depth: refuse to operate outside the LightRAG base dir, + # even if a caller smuggles in ``..`` or absolute path components. + base_real = os.path.realpath(self._base_dir) + dir_real = os.path.realpath(working_dir) + if dir_real != base_real and not dir_real.startswith( + base_real + os.sep + ): + logger.warning( + f"[LightRAG] Refusing cache cleanup outside base dir: " + f"{working_dir}" + ) + return 0 + + cache_file = os.path.join(dir_real, LLM_RESPONSE_CACHE_FILENAME) try: if not os.path.isfile(cache_file): return 0 @@ -397,9 +415,9 @@ async def clear_llm_response_cache( Warm instances go through ``LightRAG.aclear_cache`` so the in-memory KV store stays consistent; cold groups have the cache file removed - directly (JsonKVStorage loads a missing file as empty). Groups with - the cache enabled keep their file untouched unless cleared explicitly - through the LightRAG API. + directly (JsonKVStorage loads a missing file as empty). Each group is + serialised through the same per-group init lock used by ``_get_rag`` + so a concurrent initialisation can never resurrect a cleared cache. Returns: Dict with ``cleared`` group ids, ``freed_bytes`` and ``errors``. @@ -413,42 +431,57 @@ async def clear_llm_response_cache( except OSError: group_names = [] + errors: List[str] = [] if group_ids: - wanted = set(group_ids) - group_names = [name for name in group_names if name in wanted] + wanted: List[str] = [] for group_id in group_ids: + if not SAFE_GROUP_ID_PATTERN.match(str(group_id)): + errors.append(f"{group_id}: 非法群号，已跳过") + continue + if str(group_id) not in wanted: + wanted.append(str(group_id)) + group_names = [name for name in group_names if name in wanted] + for group_id in wanted: if group_id not in group_names: group_names.append(group_id) cleared: List[str] = [] - errors: List[str] = [] freed_total = 0 for group_id in group_names: - working_dir = os.path.join(self._base_dir, group_id) - cache_file = os.path.join(working_dir, LLM_RESPONSE_CACHE_FILENAME) - try: - size_before = ( - os.path.getsize(cache_file) - if os.path.isfile(cache_file) - else 0 - ) - rag = self._instances.get(group_id) - if rag is not None: - await rag.aclear_cache() - else: - self._remove_stale_cache_file(working_dir) - size_after = ( - os.path.getsize(cache_file) - if os.path.isfile(cache_file) - else 0 + if group_id not in self._init_locks: + self._init_locks[group_id] = asyncio.Lock() + + # Serialise with _get_rag: after acquiring the lock the warm/cold + # decision is re-checked, so an instance created concurrently is + # cleared via its API instead of racing a file delete. + async with self._init_locks[group_id]: + working_dir = os.path.join(self._base_dir, group_id) + cache_file = os.path.join( + working_dir, LLM_RESPONSE_CACHE_FILENAME ) - freed = max(size_before - size_after, 0) - if size_before or size_after: - cleared.append(group_id) - freed_total += freed - except Exception as exc: - errors.append(f"{group_id}: {exc}") + try: + size_before = ( + os.path.getsize(cache_file) + if os.path.isfile(cache_file) + else 0 + ) + rag = self._instances.get(group_id) + if rag is not None: + await rag.aclear_cache() + else: + self._remove_stale_cache_file(working_dir) + size_after = ( + os.path.getsize(cache_file) + if os.path.isfile(cache_file) + else 0 + ) + freed = max(size_before - size_after, 0) + if size_before or size_after: + cleared.append(group_id) + freed_total += freed + except Exception as exc: + errors.append(f"{group_id}: {exc}") return { "cleared": cleared, diff --git a/tests/unit/test_command_passthrough_rag_cache.py b/tests/unit/test_command_passthrough_rag_cache.py index 0723538a..7fb43701 100644 --- a/tests/unit/test_command_passthrough_rag_cache.py +++ b/tests/unit/test_command_passthrough_rag_cache.py @@ -1,5 +1,6 @@ """Issue #254 / #253 回归测试：命令直接放行与 LightRAG LLM 响应缓存治理""" +import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock @@ -204,6 +205,67 @@ async def test_clear_llm_response_cache_uses_api_for_warm_instances( assert cache_file.exists() # 模拟实例不落盘，文件保持原样 +@pytest.mark.asyncio +async def test_clear_llm_response_cache_rejects_path_traversal_ids(rag_env): + manager, tmp_path = rag_env + outside_dir = tmp_path / "evil" + outside_dir.mkdir() + outside_cache = outside_dir / lkm.LLM_RESPONSE_CACHE_FILENAME + outside_cache.write_bytes(b"x" * 512) + + result = await manager.clear_llm_response_cache( + group_ids=["../evil", "../../etc", "ok123"] + ) + + assert all("非法群号" in err for err in result["errors"]) + assert any("../evil" in err for err in result["errors"]) + assert outside_cache.exists() # 越界路径绝不能被删除 + assert result["cleared"] == [] + + +def test_remove_stale_cache_file_refuses_outside_base_dir(rag_env): + manager, tmp_path = rag_env + outside_dir = tmp_path / "outside" + outside_dir.mkdir() + outside_cache = outside_dir / lkm.LLM_RESPONSE_CACHE_FILENAME + outside_cache.write_bytes(b"x" * 256) + + freed = manager._remove_stale_cache_file(str(outside_dir)) + + assert freed == 0 + assert outside_cache.exists() + + +@pytest.mark.asyncio +async def test_clear_waits_for_concurrent_init_and_uses_api(rag_env): + """清理与并发初始化同一把锁：初始化完成后按热实例走 API 清理。""" + manager, tmp_path = rag_env + _write_cache_file(tmp_path, "111") + warm_rag = SimpleNamespace(aclear_cache=AsyncMock()) + init_started = asyncio.Event() + + async def _slow_get_rag(group_id): + if group_id not in manager._init_locks: + manager._init_locks[group_id] = asyncio.Lock() + async with manager._init_locks[group_id]: + init_started.set() + await asyncio.sleep(0.05) + manager._instances[group_id] = warm_rag + return warm_rag + + init_task = asyncio.create_task(_slow_get_rag("111")) + await init_started.wait() + clear_task = asyncio.create_task( + manager.clear_llm_response_cache(group_ids=["111"]) + ) + await init_task + result = await clear_task + + warm_rag.aclear_cache.assert_awaited_once() + assert result["cleared"] == ["111"] + assert result["errors"] == [] + + @pytest.mark.asyncio async def test_get_rag_disables_llm_cache_by_default(tmp_path, monkeypatch): monkeypatch.setattr(lkm, "_LIGHTRAG_AVAILABLE", True) diff --git a/web_src/package.json b/web_src/package.json index 65317f59..ba03d0ac 100644 --- a/web_src/package.json +++ b/web_src/package.json @@ -1,6 +1,6 @@ { "name": "astrbot-self-learning-dashboard", - "version": "4.3.0", + "version": "4.2.2", "description": "Solid.js source for the self-learning plugin dashboard", "type": "module", "scripts": {