diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6e94304b06..eaa68de585 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -58,7 +58,9 @@ Yuxi 只交付完整知识能力路径。API 始终注册 `external_kb`、`knowl - AgentRun:拥有 Conversation、Message、LangGraph interrupt、线程 FIFO 和 execution tree 语义,通过专用 Run/Attempt 表维护状态、输出与租约。 - 用户定时 Agent:任务定义和 occurrence 独立持久化;worker 锁定到期任务后复用统一 AgentRun Request/Run 链路,排队与执行状态仍由 AgentRunRequest 和 AgentRun 拥有。 -- Durable Task:用于知识库解析、评估和图谱构建。API 只提交持久 `task_type + handler_version + payload`;`worker` 从 registry 惰性加载领域 Handler,并通过 Task 行的唯一 owner、heartbeat 和 lease 执行。知识文件中间态绑定 Task/attempt owner,失联 failure hook 与 Task 终态同事务收敛文件错误态;PG pending 行由启动与周期 publisher 补发。共享 ARQ worker 的执行槽由 Compose 配置,Durable Task 的 PG claim 上限为 4,不能占满 AgentRun 容量。 +- Durable Task:用于知识库解析、Office 文件预览、评估和图谱构建。API 只提交持久 `task_type + handler_version + payload`;`worker` 从 registry 惰性加载领域 Handler,并通过 Task 行的唯一 owner、heartbeat 和 lease 执行。知识文件中间态绑定 Task/attempt owner,失联 failure hook 与 Task 终态同事务收敛文件错误态;PG pending 行由启动与周期 publisher 补发。共享 ARQ worker 的执行槽由 Compose 配置,Durable Task 的 PG claim 上限为 4,不能占满 AgentRun 容量。 + +`knowledge_preview` 在原件注册后生成知识库 Office 文件的共享 PDF,预览 Task 的终态与文件解析、索引状态独立。转换、缓存与读取链路见[Office 文件预览](docs/mechanisms/knowledge-base.md#office-文件预览)。 测试代码位于 `backend/test`,按 `unit`、`integration`、`e2e` 分层。新增或修改后端行为时,测试应放在最能覆盖真实风险的层级。 diff --git a/backend/package/yuxi/knowledge/preview.py b/backend/package/yuxi/knowledge/preview.py index 6748cdc964..160f444a17 100644 --- a/backend/package/yuxi/knowledge/preview.py +++ b/backend/package/yuxi/knowledge/preview.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Awaitable, Callable + from yuxi.knowledge.utils.kb_utils import is_minio_url, parse_minio_url from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository from yuxi.storage.minio import get_minio_client @@ -15,7 +17,13 @@ ) -async def read_knowledge_file_preview(kb_id: str, file_id: str) -> dict: +async def read_knowledge_file_preview( + kb_id: str, + file_id: str, + *, + prepare_office: bool = True, + before_cache_write: Callable[[], Awaitable[None]] | None = None, +) -> dict: """从 Knowledge metadata 与 MinIO 对象生成只读文件预览。""" file_record = await KnowledgeFileRepository().get_by_file_id(file_id) if file_record is None or file_record.kb_id != kb_id: @@ -48,7 +56,23 @@ async def read_knowledge_file_preview(kb_id: str, file_id: str) -> dict: return {**response, **preview_too_large().payload()} if is_office_pdf_preview_file(filename): - pdf_content = await _read_office_pdf_preview(kb_id, file_id, filename, original_path) + pdf_content = await _read_office_pdf_preview( + kb_id, + file_id, + filename, + original_path, + prepare_office=prepare_office, + before_cache_write=before_cache_write, + ) + if pdf_content is None: + return { + **response, + "content": None, + "preview_type": "office", + "supported": True, + "status": "loading", + "message": "正在生成预览,完成后将自动显示...", + } return { **response, "content": pdf_content, @@ -82,10 +106,15 @@ async def _read_office_pdf_preview( file_id: str, filename: str, original_path: str, -) -> bytes: + *, + prepare_office: bool = True, + before_cache_write: Callable[[], Awaitable[None]] | None = None, +) -> bytes | None: minio_client = get_minio_client() bucket_name = minio_client.KB_BUCKETS["parsed"] object_name = f"{kb_id}/preview/{file_id}.pdf" + if not prepare_office: + return await minio_client.adownload_file(bucket_name, object_name, missing_ok=True) if await minio_client.astat_file(bucket_name, object_name) is not None: return await minio_client.adownload_file(bucket_name, object_name) @@ -94,6 +123,8 @@ async def _read_office_pdf_preview( pdf_content = await convert_office_to_pdf(filename, raw_content) except OfficePreviewConversionError as exc: raise ValueError(str(exc)) from exc + if before_cache_write is not None: + await before_cache_write() await minio_client.aupload_file( bucket_name=bucket_name, object_name=object_name, diff --git a/backend/package/yuxi/services/file_preview.py b/backend/package/yuxi/services/file_preview.py index 4f8f3777ed..4d63e5360d 100644 --- a/backend/package/yuxi/services/file_preview.py +++ b/backend/package/yuxi/services/file_preview.py @@ -2,10 +2,9 @@ from __future__ import annotations -import io from urllib.parse import quote -from fastapi.responses import StreamingResponse +from fastapi.responses import Response from yuxi.utils.filepreview import PreviewResult from yuxi.workspace.preview import preview_workspace_file @@ -15,18 +14,18 @@ async def render_file_preview( raw_content: bytes, *, office_cache_key: str, -) -> dict | StreamingResponse: +) -> dict | Response: """把持久文件预览结果转换为 Workspace/Viewer HTTP 响应。""" result = await preview_workspace_file(path, raw_content, office_cache_key=office_cache_key) return _preview_response(result) -def _preview_response(result: PreviewResult) -> dict | StreamingResponse: +def _preview_response(result: PreviewResult) -> dict | Response: if not isinstance(result.content, bytes): return result.payload() filename = result.filename or "preview" - return StreamingResponse( - io.BytesIO(result.content), + return Response( + result.content, media_type=result.media_type or "application/octet-stream", headers={ "Content-Disposition": f"inline; filename*=UTF-8''{quote(filename)}", diff --git a/backend/package/yuxi/services/knowledge_preview_service.py b/backend/package/yuxi/services/knowledge_preview_service.py new file mode 100644 index 0000000000..ac89dc3184 --- /dev/null +++ b/backend/package/yuxi/services/knowledge_preview_service.py @@ -0,0 +1,77 @@ +"""知识库文件注册、后台预览与只读预览状态用例。""" + +from __future__ import annotations + +from yuxi.knowledge.preview import read_knowledge_file_preview +from yuxi.knowledge.runtime import knowledge_base +from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository +from yuxi.services.task_service import Task, TaskContext, tasker +from yuxi.utils.filepreview import MAX_BINARY_PREVIEW_SIZE_BYTES, is_office_pdf_preview_file +from yuxi.utils.logging_config import logger + + +async def register_knowledge_file(kb_id: str, item: str, *, params: dict, operator_id: str) -> dict: + """注册原件后立即提交独立预览任务,解析与预览各自保留状态。""" + file_meta = await knowledge_base.add_file_record(kb_id, item, params=params, operator_id=operator_id) + size = file_meta.get("size") + if is_office_pdf_preview_file(file_meta["filename"]) and ( + size is None or int(size) <= MAX_BINARY_PREVIEW_SIZE_BYTES + ): + try: + await enqueue_knowledge_preview(kb_id, file_meta["file_id"]) + except Exception: + logger.exception( + "预览任务提交失败,原件注册已完成: kb_id={}, file_id={}", + kb_id, + file_meta["file_id"], + ) + return file_meta + + +async def enqueue_knowledge_preview(kb_id: str, file_id: str) -> Task: + """按知识库和文件去重,复用持久 Task 的提交后投递与恢复。""" + payload = {"kb_id": kb_id, "file_id": file_id} + task, _ = await tasker.enqueue_unique_by_payload( + name="知识库文件预览", + task_type="knowledge_preview", + payload=payload, + payload_match=payload, + ) + return task + + +async def read_workspace_knowledge_preview(kb_id: str, file_id: str) -> dict: + """读取现成缓存,缺失时呈现后台任务状态而不在请求中转换。""" + response = await read_knowledge_file_preview(kb_id, file_id, prepare_office=False) + if response.get("status") != "loading": + return response + + task = await tasker.find_task_by_payload( + task_type="knowledge_preview", payload_match={"kb_id": kb_id, "file_id": file_id} + ) + if task is not None and task.status in {"failed", "cancelled"}: + return {**response, "status": "error", "supported": False, "message": "预览生成失败,请下载原件查看"} + if task is None or task.status == "success": + task = await enqueue_knowledge_preview(kb_id, file_id) + return response + + +async def run_knowledge_preview(context: TaskContext) -> dict: + """由真实 worker 生成共享 PDF,发布前核对租约和原件记录。""" + kb_id, file_id = context.payload["kb_id"], context.payload["file_id"] + await context.raise_if_cancelled() + await context.set_progress(10, "正在生成文件预览") + + async def before_cache_write(): + """转换可以较慢,发布缓存前重新检查执行权与文件存在性。""" + await context.raise_if_cancelled() + record = await KnowledgeFileRepository().get_by_file_id(file_id) + if record is None or record.kb_id != kb_id: + raise ValueError("原件已删除,停止发布预览") + + preview = await read_knowledge_file_preview(kb_id, file_id, before_cache_write=before_cache_write) + if not preview.get("binary") or preview.get("preview_type") != "pdf": + raise ValueError("当前文件无法生成 PDF 预览") + result = {"kb_id": kb_id, "file_id": file_id, "preview_type": "pdf", "size": len(preview["content"])} + await context.set_progress(100, "文件预览已就绪") + return result diff --git a/backend/package/yuxi/services/knowledge_task_service.py b/backend/package/yuxi/services/knowledge_task_service.py index 0e09e6b193..ab6f465b40 100644 --- a/backend/package/yuxi/services/knowledge_task_service.py +++ b/backend/package/yuxi/services/knowledge_task_service.py @@ -7,6 +7,7 @@ from yuxi.knowledge.utils import params_for_uploaded_document from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository from yuxi.services.knowledge_folder_service import knowledge_folder_service +from yuxi.services.knowledge_preview_service import register_knowledge_file from yuxi.services.task_service import TaskContext from yuxi.utils.logging_config import logger @@ -60,7 +61,7 @@ async def run_knowledge_ingest(context: TaskContext) -> dict: await context.raise_if_cancelled() await context.set_progress(5.0 + (index / total) * 25.0, f"[1/3] 添加记录 {index}/{total}") try: - file_meta = await knowledge_base.add_file_record( + file_meta = await register_knowledge_file( kb_id, item, params=params_for_uploaded_document(item, params), diff --git a/backend/package/yuxi/services/task_registry.py b/backend/package/yuxi/services/task_registry.py index 03cf92a32e..715d5d3794 100644 --- a/backend/package/yuxi/services/task_registry.py +++ b/backend/package/yuxi/services/task_registry.py @@ -50,6 +50,11 @@ def _load(self, function: str, label: str): "run_knowledge_ingest", failure_function="fail_knowledge_file_task", ), + TaskDefinition( + "knowledge_preview", + "yuxi.services.knowledge_preview_service", + "run_knowledge_preview", + ), TaskDefinition( "knowledge_parse", "yuxi.services.knowledge_task_service", diff --git a/backend/package/yuxi/services/viewer_filesystem_service.py b/backend/package/yuxi/services/viewer_filesystem_service.py index 00d75f410b..ecdd38f69c 100644 --- a/backend/package/yuxi/services/viewer_filesystem_service.py +++ b/backend/package/yuxi/services/viewer_filesystem_service.py @@ -10,7 +10,7 @@ from urllib.parse import quote from fastapi import HTTPException, UploadFile -from fastapi.responses import FileResponse, StreamingResponse +from fastapi.responses import FileResponse, Response from starlette.background import BackgroundTask from yuxi.agents.backends.paths import is_runtime_path, runtime_path_for_workdir_scope from yuxi.services.file_preview import render_file_preview @@ -123,7 +123,7 @@ async def _download_to_temp(workdir, path: str, max_bytes: int) -> str: return temp_path -async def read_viewer_file_content(*, thread_id: str, path: str, current_user, db) -> dict | StreamingResponse: +async def read_viewer_file_content(*, thread_id: str, path: str, current_user, db) -> dict | Response: """从实时 Workdir 读取预览,不经过 MinIO revision。""" access = await _viewer_state(thread_id=thread_id, current_user=current_user, db=db) _validate_viewer_path(access, path) diff --git a/backend/package/yuxi/services/workspace_service.py b/backend/package/yuxi/services/workspace_service.py index 5818c32c64..53ca8dfadb 100644 --- a/backend/package/yuxi/services/workspace_service.py +++ b/backend/package/yuxi/services/workspace_service.py @@ -8,7 +8,7 @@ from urllib.parse import quote from fastapi import HTTPException, UploadFile -from fastapi.responses import FileResponse, StreamingResponse +from fastapi.responses import FileResponse, Response from starlette.background import BackgroundTask from yuxi.agents.backends.paths import runtime_user_data_path from yuxi.repositories.project_repository import ProjectRepository @@ -141,7 +141,7 @@ async def read_workspace_file_bytes(*, path: str, current_user: User) -> tuple[s return PurePosixPath(workspace_path).name, content -async def read_workspace_file_content(*, path: str, current_user: User) -> dict | StreamingResponse: +async def read_workspace_file_content(*, path: str, current_user: User) -> dict | Response: backend = _workspace_backend(current_user) workspace_path = _workspace_path(path) try: diff --git a/backend/package/yuxi/storage/minio/client.py b/backend/package/yuxi/storage/minio/client.py index 0063a123fd..5bfd551979 100644 --- a/backend/package/yuxi/storage/minio/client.py +++ b/backend/package/yuxi/storage/minio/client.py @@ -231,8 +231,8 @@ async def adownload_response(self, bucket_name: str, object_name: str) -> BaseHT raise StorageError(f"对象 '{object_name}' 在存储桶 '{bucket_name}' 中不存在") raise StorageError(f"下载文件失败: {e}") - async def adownload_file(self, bucket_name: str, object_name: str) -> bytes: - """异步下载文件""" + async def adownload_file(self, bucket_name: str, object_name: str, *, missing_ok: bool = False) -> bytes | None: + """异步下载文件,派生缓存可显式允许对象缺失。""" response = None try: response = await asyncio.to_thread(self.client.get_object, bucket_name=bucket_name, object_name=object_name) @@ -242,6 +242,8 @@ async def adownload_file(self, bucket_name: str, object_name: str) -> bytes: except S3Error as e: if e.code == "NoSuchKey": + if missing_ok: + return None raise StorageError(f"对象 '{object_name}' 在存储桶 '{bucket_name}' 中不存在") raise StorageError(f"下载文件失败: {e}") finally: diff --git a/backend/package/yuxi/utils/filepreview.py b/backend/package/yuxi/utils/filepreview.py index d95d18b46f..2093607d47 100644 --- a/backend/package/yuxi/utils/filepreview.py +++ b/backend/package/yuxi/utils/filepreview.py @@ -17,8 +17,9 @@ _MARKDOWN_EXTENSIONS = frozenset({".md", ".markdown", ".mdx"}) _PDF_EXTENSIONS = frozenset({".pdf"}) _HTML_EXTENSIONS = frozenset({".html", ".htm"}) -_OFFICE_PDF_PREVIEW_EXTENSIONS = frozenset({".docx", ".pptx"}) +_OFFICE_PDF_PREVIEW_EXTENSIONS = frozenset({".docx", ".pptx", ".xlsx", ".xls"}) _OFFICE_MEDIA_TYPES = { + ".xls": "application/vnd.ms-excel", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", diff --git a/backend/server/routers/knowledge_router.py b/backend/server/routers/knowledge_router.py index 68b5090f9e..2550f000c2 100644 --- a/backend/server/routers/knowledge_router.py +++ b/backend/server/routers/knowledge_router.py @@ -9,13 +9,14 @@ from fastapi import APIRouter, Body, Depends, File, HTTPException, Query, UploadFile from fastapi.responses import FileResponse from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession from starlette.responses import StreamingResponse from yuxi.config.options import system_options from yuxi.knowledge.base import KBNameConflictError, KBNotFoundError from yuxi.knowledge.chunking.ragflow_like.presets import get_chunk_preset_options from yuxi.knowledge.graphs.milvus_graph_service import GRAPH_TASK_TYPE, MilvusGraphService -from yuxi.knowledge.read_models import KnowledgeBaseDetail from yuxi.knowledge.parser.capabilities import SUPPORTED_FILE_EXTENSIONS, is_supported_file_extension +from yuxi.knowledge.read_models import KnowledgeBaseDetail from yuxi.knowledge.runtime import knowledge_base from yuxi.knowledge.utils import calculate_content_hash, is_minio_url, params_for_uploaded_document, parse_minio_url from yuxi.knowledge.utils.mindmap_utils import ( @@ -37,6 +38,7 @@ resolve_knowledge_base_permission, ) from yuxi.services.knowledge_folder_service import knowledge_folder_service +from yuxi.services.knowledge_preview_service import register_knowledge_file from yuxi.services.ocr_service import parse_document from yuxi.services.task_service import tasker from yuxi.services.workspace_service import read_workspace_file_bytes @@ -46,13 +48,14 @@ from yuxi.utils.upload_utils import MAX_UPLOAD_SIZE_BYTES, read_upload_with_limit, write_upload_to_path from server.utils.auth_middleware import get_admin_user, get_db, get_required_user -from sqlalchemy.ext.asyncio import AsyncSession -from server.utils.knowledge_response import serialize_knowledge_base, serialize_knowledge_base_list from server.utils.knowledge_permissions import ( ensure_knowledge_base_permission as _ensure_database_permission, +) +from server.utils.knowledge_permissions import ( require_knowledge_base_manage, require_knowledge_base_read, ) +from server.utils.knowledge_response import serialize_knowledge_base, serialize_knowledge_base_list knowledge = APIRouter(prefix="/knowledge", tags=["knowledge"]) @@ -780,7 +783,7 @@ async def add_uploaded_documents( failed_items: list[dict] = [] for index, item in enumerate(payload.items): try: - file_meta = await knowledge_base.add_file_record( + file_meta = await register_knowledge_file( kb_id, item, params=params_for_uploaded_document(item, params), diff --git a/backend/server/routers/workspace_router.py b/backend/server/routers/workspace_router.py index dd28b74471..e99eefe0aa 100644 --- a/backend/server/routers/workspace_router.py +++ b/backend/server/routers/workspace_router.py @@ -4,10 +4,9 @@ from urllib.parse import quote from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile -from fastapi.responses import StreamingResponse +from fastapi.responses import Response, StreamingResponse from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession -from server.utils.auth_middleware import get_db, get_required_user from yuxi.services.workspace_service import ( create_workspace_directory, delete_workspace_path, @@ -20,6 +19,8 @@ ) from yuxi.storage.postgres.models_business import User +from server.utils.auth_middleware import get_db, get_required_user + workspace = APIRouter(prefix="/workspace", tags=["workspace"]) workspace_knowledge = APIRouter(prefix="/workspace", tags=["workspace"]) @@ -129,11 +130,11 @@ async def search_workspace_files_route( return await search_workspace_files(query=query, current_user=current_user) -def _binary_preview_response(data: dict) -> StreamingResponse: +def _binary_preview_response(data: dict) -> Response: filename = data.get("filename") or "preview" preview_type = data.get("preview_type") or "unsupported" - return StreamingResponse( - io.BytesIO(data.get("content") or b""), + return Response( + data.get("content") or b"", media_type=data.get("media_type") or "application/octet-stream", headers={ "Content-Disposition": f"inline; filename*=UTF-8''{quote(filename)}", @@ -204,9 +205,9 @@ async def get_workspace_knowledge_file( await _ensure_knowledge_read_access(current_user, kb_id) await _ensure_knowledge_supports_documents(kb_id) try: - from yuxi.knowledge.preview import read_knowledge_file_preview + from yuxi.services.knowledge_preview_service import read_workspace_knowledge_preview - return _preview_response(await read_knowledge_file_preview(kb_id=kb_id, file_id=file_id)) + return _preview_response(await read_workspace_knowledge_preview(kb_id=kb_id, file_id=file_id)) except ValueError as error: _raise_knowledge_read_error(error) diff --git a/backend/test/integration/api/test_knowledge_preview_http.py b/backend/test/integration/api/test_knowledge_preview_http.py new file mode 100644 index 0000000000..01d5e534b1 --- /dev/null +++ b/backend/test/integration/api/test_knowledge_preview_http.py @@ -0,0 +1,430 @@ +"""在隔离 PostgreSQL、Redis、MinIO 与真实 worker 验证提前预览。""" + +import hashlib +import io +import json +import os +import socket +import subprocess +import sys +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace + +import httpx +import psycopg +import pytest +from minio import Minio +from openpyxl import Workbook +from pypdf import PdfReader +from redis import Redis +from sqlalchemy.engine import make_url +from yuxi.utils.auth_utils import SECURITY_SECRET_NAMES + +pytestmark = [pytest.mark.integration, pytest.mark.e2e] + + +def wait_until(read, accept, *, seconds=90): + """等待外部事实达到预期,超时保留最后一次读回值。""" + deadline = time.monotonic() + seconds + value = None + while time.monotonic() < deadline: + value = read() + if accept(value): + return value + time.sleep(0.1) + pytest.fail(f"外部事实未收敛: {value}") + + +@pytest.fixture(scope="module") +def preview_server(tmp_path_factory): + """独立数据库和进程保留真实认证、任务发布、claim 与转换边界。""" + isolated_redis_url = os.environ.get("PREVIEW_TEST_REDIS_URL") + if not isolated_redis_url: + pytest.skip("需要 PREVIEW_TEST_REDIS_URL 指向独立验收 Redis,不能复用开发队列") + with Redis.from_url(isolated_redis_url) as redis: + if next(redis.scan_iter(match="yuxi:worker:health:*"), None) is not None: + pytest.fail("验收 Redis 上已有 worker,拒绝跨数据库消费任务") + configured_url = os.environ.get("POSTGRES_URL") + if not configured_url: + pytest.fail("POSTGRES_URL 未配置真实 PostgreSQL 测试入口") + temporary = tmp_path_factory.mktemp("knowledge-preview-http") + database = "preview_test_" + uuid.uuid4().hex + base = make_url(configured_url) + admin_url = base.set(drivername="postgresql", database="postgres").render_as_string(hide_password=False) + url = base.set(database=database) + sync_url = url.set(drivername="postgresql").render_as_string(hide_password=False) + with psycopg.connect(admin_url, autocommit=True) as connection: + connection.execute(f'CREATE DATABASE "{database}"') + + env = dict( + os.environ, + POSTGRES_URL=url.render_as_string(hide_password=False), + REDIS_URL=isolated_redis_url, + YUXI_SKILL_DATA_DIR=str(temporary / "skill-sources"), + ) + env.update({name: uuid.uuid4().hex + uuid.uuid4().hex for name in SECURITY_SECRET_NAMES}) + processes = [] + logs = [] + client = None + minio_client = None + kb_id = "kb_preview_" + uuid.uuid4().hex[:16] + private_kb = kb_id + "_private" + + seed = """ +import asyncio +from loguru import logger +logger.remove() +from yuxi.storage.postgres.manager import pg_manager, BUSINESS_SCHEMA_VERSION, KNOWLEDGE_SCHEMA_VERSION +from yuxi.storage.postgres.models_business import User, Department +from yuxi.storage.postgres.models_knowledge import KnowledgeBase +from yuxi.utils.auth_utils import AuthUtils +async def seed(): + pg_manager.initialize() + await pg_manager.create_business_tables() + await pg_manager.create_knowledge_tables() + await pg_manager.create_schema_version_table() + await pg_manager.record_schema_version('business', BUSINESS_SCHEMA_VERSION) + await pg_manager.record_schema_version('knowledge', KNOWLEDGE_SCHEMA_VERSION) + async with pg_manager.get_async_session_context() as session: + department = Department(name='预览测试部门') + session.add(department) + await session.flush() + for uid, role in [('preview_admin', 'superadmin'), ('preview_reader', 'user')]: + session.add(User(uid=uid, username=uid, role=role, department_id=department.id, + password_hash=AuthUtils.hash_password('Synthetic-preview-pass!'))) + session.add(KnowledgeBase(kb_id=KB_ID, name='预览测试库', kb_type='milvus', created_by='preview_admin', + share_config={'version': 2, 'read_scope': {'access_level': 'global'}, 'manage_scope': None})) + session.add(KnowledgeBase(kb_id=PRIVATE_KB, name='预览私有库', kb_type='milvus', created_by='preview_admin', + share_config={'version': 2, 'read_scope': None, 'manage_scope': None})) + await pg_manager.close() +asyncio.run(seed()) +""" + seed = f"KB_ID={kb_id!r}\nPRIVATE_KB={private_kb!r}\n" + seed + + def start_process(command, name): + """使用私有日志启动验收进程,关闭时回收。""" + log = (temporary / (name + ".log")).open("w") + logs.append(log) + process = subprocess.Popen(command, env=env, cwd=temporary, stdout=log, stderr=log) + processes.append(process) + return process + + try: + seeded = subprocess.run([sys.executable, "-c", seed], env=env, cwd=temporary, capture_output=True, text=True) + assert seeded.returncode == 0, seeded.stderr[-4000:] + app_file = temporary / "preview_http_app.py" + app_file.write_text( + "from loguru import logger\nlogger.remove()\n" + "from fastapi import FastAPI\n" + "from server.routers.auth_router import auth\n" + "from server.routers.knowledge_router import knowledge\n" + "from server.routers.workspace_router import workspace, workspace_knowledge\n" + "from server.utils.access_log_middleware import AccessLogMiddleware\n" + "from yuxi.storage.postgres.manager import pg_manager\n" + "pg_manager.initialize()\napp = FastAPI()\n" + "app.add_middleware(AccessLogMiddleware)\n" + "app.include_router(auth, prefix='/api')\n" + "app.include_router(knowledge, prefix='/api')\n" + "app.include_router(workspace, prefix='/api')\n" + "app.include_router(workspace_knowledge, prefix='/api')\n", + encoding="utf-8", + ) + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + api = start_process( + [ + sys.executable, + "-m", + "uvicorn", + "preview_http_app:app", + "--app-dir", + str(temporary), + "--host", + "127.0.0.1", + "--port", + str(port), + "--no-access-log", + ], + "api", + ) + endpoint = f"http://127.0.0.1:{port}" + client = httpx.Client(base_url=endpoint, timeout=30, trust_env=False) + + def live(): + """API 必须真的接受 HTTP 请求。""" + if api.poll() is not None: + pytest.fail((temporary / "api.log").read_text()[-4000:]) + try: + return client.get("/api/auth/check-first-run").status_code + except httpx.ConnectError: + return None + + wait_until(live, lambda status: status == 200) + headers = {} + for role, uid in [("admin", "preview_admin"), ("reader", "preview_reader")]: + login = client.post("/api/auth/token", data={"username": uid, "password": "Synthetic-preview-pass!"}) + assert login.status_code == 200, login.text + headers[role] = {"Authorization": "Bearer " + login.json()["access_token"]} + minio_client = Minio( + os.environ["MINIO_URI"].split("://", 1)[-1], + access_key=os.environ["MINIO_ACCESS_KEY"], + secret_key=os.environ["MINIO_SECRET_KEY"], + secure=False, + ) + yield SimpleNamespace( + temporary=temporary, + client=client, + url=sync_url, + kb_id=kb_id, + private_kb=private_kb, + headers=headers, + start_worker=lambda: start_process([sys.executable, "-m", "server.worker_main"], "worker"), + minio=minio_client, + ) + finally: + for process in reversed(processes): + process.terminate() + try: + process.wait(timeout=20) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + for log in logs: + log.close() + if client is not None: + client.close() + try: + if minio_client is not None and minio_client.bucket_exists("knowledgebases"): + for item in minio_client.list_objects("knowledgebases", prefix=kb_id + "/", recursive=True): + minio_client.remove_object("knowledgebases", item.object_name) + finally: + with psycopg.connect(admin_url, autocommit=True) as connection: + connection.execute(f'DROP DATABASE "{database}" WITH (FORCE)') + + +def task_rows(server, file_id): + """从 PostgreSQL 读取对应文件的真实 Task 终态。""" + with psycopg.connect(server.url) as connection: + return connection.execute( + "SELECT id, status, result, error FROM tasks " + "WHERE type='knowledge_preview' AND payload->>'file_id'=%s ORDER BY created_at", + (file_id,), + ).fetchall() + + +def upload_and_add(server, filename, content): + """执行产品实际上传和只添加入口。""" + uploaded = server.client.post( + "/api/knowledge/files/upload", + params={"kb_id": server.kb_id}, + files={"file": (filename, content)}, + headers=server.headers["admin"], + ) + assert uploaded.status_code == 200, (uploaded.text, (server.temporary / "api.log").read_text()[-3500:]) + original = uploaded.json() + path = original["file_path"] + added = server.client.post( + f"/api/knowledge/databases/{server.kb_id}/documents/add", + json={ + "items": [path], + "params": {"content_hashes": {path: original["content_hash"]}, "file_sizes": {path: len(content)}}, + }, + headers=server.headers["admin"], + ) + assert added.status_code == 200 and added.json()["added"] == 1, added.text + return added.json()["items"][0]["file_id"], original + + +def object_bytes(server, key): + """回读真实 MinIO 字节并关闭连接。""" + response = server.minio.get_object("knowledgebases", key) + try: + return response.read() + finally: + response.close() + response.release_conn() + + +def test_uploaded_office_prepares_cache_before_preview_and_reuses_it(preview_server): + """先回读 worker 产物再点预览,原件、权限、去重与失败均由外部事实证明。""" + server = preview_server + workbook = Workbook() + workbook.active["A1"] = "preview-cache-marker" + for row in range(2, 180): + workbook.active.append([row, row * 3, row * 7]) + output = io.BytesIO() + workbook.save(output) + original = output.getvalue() + file_id, uploaded = upload_and_add(server, "preview.xlsx", original) + rows = task_rows(server, file_id) + assert len(rows) == 1 and rows[0][1] == "pending" + preview_params = {"kb_id": server.kb_id, "file_id": file_id} + reader = server.headers["reader"] + # 存量文件没有提前任务时,并发读取在真实唯一索引上收敛。 + legacy_id = "file_preview_legacy" + with psycopg.connect(server.url) as connection: + connection.execute( + "INSERT INTO knowledge_files (file_id, kb_id, filename, minio_url, path, file_size, status) " + "SELECT %s, kb_id, filename, minio_url, path, file_size, status FROM knowledge_files WHERE file_id=%s", + (legacy_id, file_id), + ) + legacy_params = {**preview_params, "file_id": legacy_id} + with ThreadPoolExecutor(max_workers=8) as pool: + responses = list( + pool.map( + lambda _: server.client.get("/api/workspace/knowledge/file", params=legacy_params, headers=reader), + range(8), + ) + ) + assert all(response.status_code == 200 and response.json().get("status") == "loading" for response in responses), [ + (response.status_code, response.text[:500]) for response in responses + ] + assert len(task_rows(server, legacy_id)) == 1 + assert task_rows(server, file_id) == rows + + cache_key = f"{server.kb_id}/preview/{file_id}.pdf" + assert not list(server.minio.list_objects("knowledgebases", prefix=cache_key)) + worker = server.start_worker() + + def worker_facts(): + """启动失败立即保留原因,不能把未运行 worker 当成慢转换。""" + if worker.poll() is not None: + pytest.fail((server.temporary / "worker.log").read_text()[-4500:]) + return task_rows(server, file_id) + + finished = wait_until(worker_facts, lambda records: records and records[0][1] in {"success", "failed"}) + assert finished[0][1] == "success", finished + pdf = object_bytes(server, cache_key) + assert pdf.startswith(b"%PDF-") + assert "preview-cache-marker" in "".join(page.extract_text() or "" for page in PdfReader(io.BytesIO(pdf)).pages) + original_hash = hashlib.sha256(original).hexdigest() + assert hashlib.sha256(object_bytes(server, uploaded["object_name"])).hexdigest() == original_hash + before = server.minio.stat_object("knowledgebases", cache_key) + measurements = [] + for _ in range(2): + started = time.monotonic() + response = server.client.get("/api/workspace/knowledge/file", params=preview_params, headers=reader) + measurements.append(round(time.monotonic() - started, 4)) + assert response.status_code == 200 and response.content == pdf + assert int(response.headers["content-length"]) == len(pdf) + assert response.headers["content-type"] == "application/pdf" + after = server.minio.stat_object("knowledgebases", cache_key) + assert (before.etag, before.last_modified) == (after.etag, after.last_modified) + assert len(task_rows(server, file_id)) == 1 + print(json.dumps({"cached_http_seconds": measurements, "pdf_bytes": len(pdf)}, ensure_ascii=False)) + + # 生成真正的旧版 XLS,再经产品上传和 worker 回读,避免只改变文件后缀。 + legacy_directory = server.temporary / "legacy-xls" + legacy_directory.mkdir() + source = legacy_directory / "legacy.xlsx" + source.write_bytes(original) + converted = subprocess.run( + [ + "libreoffice", + "-env:UserInstallation=" + (legacy_directory / "profile").as_uri(), + "--headless", + "--convert-to", + "xls", + "--outdir", + str(legacy_directory), + str(source), + ], + capture_output=True, + timeout=60, + ) + legacy_file = legacy_directory / "legacy.xls" + assert converted.returncode == 0 and legacy_file.is_file(), converted.stderr + legacy_original = legacy_file.read_bytes() + assert legacy_original.startswith(bytes.fromhex("d0cf11e0a1b11ae1")) + xls_id, xls_uploaded = upload_and_add(server, "legacy.xls", legacy_original) + xls_tasks = wait_until( + lambda: task_rows(server, xls_id), lambda records: records and records[0][1] in {"success", "failed"} + ) + assert len(xls_tasks) == 1 and xls_tasks[0][1] == "success", xls_tasks + xls_pdf = object_bytes(server, f"{server.kb_id}/preview/{xls_id}.pdf") + assert "preview-cache-marker" in "".join(page.extract_text() or "" for page in PdfReader(io.BytesIO(xls_pdf)).pages) + assert ( + hashlib.sha256(object_bytes(server, xls_uploaded["object_name"])).digest() + == hashlib.sha256(legacy_original).digest() + ) + xls_response = server.client.get( + "/api/workspace/knowledge/file", params={"kb_id": server.kb_id, "file_id": xls_id}, headers=reader + ) + assert xls_response.status_code == 200 and xls_response.content == xls_pdf + assert int(xls_response.headers["content-length"]) == len(xls_pdf) + print(json.dumps({"xls_prepared_before_click": True, "xls_pdf_bytes": len(xls_pdf)})) + + count = len(task_rows(server, file_id)) + assert server.client.get("/api/workspace/knowledge/file", params=preview_params).status_code == 401 + assert ( + server.client.get( + "/api/workspace/knowledge/file", params={**preview_params, "kb_id": server.private_kb}, headers=reader + ).status_code + == 403 + ) + assert ( + server.client.get( + "/api/workspace/knowledge/file", params={**preview_params, "file_id": "missing_file"}, headers=reader + ).status_code + == 400 + ) + assert len(task_rows(server, file_id)) == count + + broken_id, broken = upload_and_add(server, "broken.xlsx", b"PK\x03\x04broken workbook") + failed = wait_until(lambda: task_rows(server, broken_id), lambda records: records and records[0][1] == "failed") + assert len(failed) == 1 + response = server.client.get( + "/api/workspace/knowledge/file", params={"kb_id": server.kb_id, "file_id": broken_id}, headers=reader + ) + assert response.status_code == 200 and response.json()["status"] == "error" + assert object_bytes(server, broken["object_name"]) == b"PK\x03\x04broken workbook" + assert not list(server.minio.list_objects("knowledgebases", prefix=f"{server.kb_id}/preview/{broken_id}.pdf")) + + workbook.active["B1"] = "independent-submission-failure" + recovered_output = io.BytesIO() + workbook.save(recovered_output) + recovered_original = recovered_output.getvalue() + + # 只拒绝预览 Task 的真实 INSERT,确认独立事务失败不抹掉已注册原件。 + with psycopg.connect(server.url) as connection: + connection.execute( + "CREATE FUNCTION reject_preview_submission() RETURNS trigger LANGUAGE plpgsql AS $$ " + "BEGIN IF NEW.type='knowledge_preview' THEN " + "RAISE EXCEPTION 'synthetic preview submission failure'; END IF; RETURN NEW; END $$" + ) + connection.execute( + "CREATE TRIGGER reject_preview_submission BEFORE INSERT ON tasks " + "FOR EACH ROW EXECUTE FUNCTION reject_preview_submission()" + ) + try: + recovered_id, recovered_upload = upload_and_add(server, "submission-failure.xlsx", recovered_original) + with psycopg.connect(server.url) as connection: + registered = connection.execute( + "SELECT filename, path, file_size FROM knowledge_files WHERE file_id=%s AND kb_id=%s", + (recovered_id, server.kb_id), + ).fetchone() + assert registered == ("submission-failure.xlsx", recovered_upload["file_path"], len(recovered_original)) + assert object_bytes(server, recovered_upload["object_name"]) == recovered_original + assert not task_rows(server, recovered_id) + finally: + with psycopg.connect(server.url) as connection: + connection.execute("DROP TRIGGER reject_preview_submission ON tasks") + connection.execute("DROP FUNCTION reject_preview_submission()") + + response = server.client.get( + "/api/workspace/knowledge/file", params={"kb_id": server.kb_id, "file_id": recovered_id}, headers=reader + ) + assert response.status_code == 200 and response.json()["status"] == "loading" + recovered_tasks = wait_until( + lambda: task_rows(server, recovered_id), lambda records: records and records[0][1] in {"success", "failed"} + ) + assert len(recovered_tasks) == 1 and recovered_tasks[0][1] == "success", recovered_tasks + recovered_pdf = object_bytes(server, f"{server.kb_id}/preview/{recovered_id}.pdf") + assert "independent-submission-failure" in "".join( + page.extract_text() or "" for page in PdfReader(io.BytesIO(recovered_pdf)).pages + ) + assert object_bytes(server, recovered_upload["object_name"]) == recovered_original + print(json.dumps({"submission_failure_kept_original": True, "read_recovered_preview": True})) diff --git a/backend/test/unit/knowledge/test_office_pdf_preview.py b/backend/test/unit/knowledge/test_office_pdf_preview.py index 249afd51bb..54aff701f0 100644 --- a/backend/test/unit/knowledge/test_office_pdf_preview.py +++ b/backend/test/unit/knowledge/test_office_pdf_preview.py @@ -110,9 +110,11 @@ def test_office_file_entry_exposes_logical_file_availability(tmp_path) -> None: assert "default_preview_mode" not in entry +@pytest.mark.parametrize("filename", ["demo.docx", "demo.pptx", "表格.xlsx", "表格.xls"]) @pytest.mark.asyncio -async def test_read_office_pdf_preview_converts_and_caches_pdf(monkeypatch: pytest.MonkeyPatch) -> None: - stub_file_record(monkeypatch, make_file_record()) +async def test_read_office_pdf_preview_converts_and_caches_pdf(monkeypatch: pytest.MonkeyPatch, filename: str) -> None: + """各类 Office 文件复用 PDF 缓存并保留原件。""" + stub_file_record(monkeypatch, make_file_record(filename=filename)) minio_client = FakeMinioClient() minio_client.objects[("knowledgebases", "db1/upload/demo.docx")] = b"office" convert_calls = 0 @@ -120,7 +122,7 @@ async def test_read_office_pdf_preview_converts_and_caches_pdf(monkeypatch: pyte async def fake_convert(filename: str, content: bytes) -> bytes: nonlocal convert_calls convert_calls += 1 - assert filename == "demo.docx" + assert filename.endswith((".docx", ".pptx", ".xlsx", ".xls")) assert content == b"office" return b"%PDF-1.4\nconverted" @@ -138,11 +140,12 @@ async def fake_convert(filename: str, content: bytes) -> bytes: assert cached_response["content"] == b"%PDF-1.4\nconverted" assert minio_client.objects[("knowledgebases", "db1/preview/file1.pdf")] == b"%PDF-1.4\nconverted" assert convert_calls == 1 + assert minio_client.objects[("knowledgebases", "db1/upload/demo.docx")] == b"office" @pytest.mark.asyncio -async def test_non_docx_pptx_office_files_do_not_get_pdf_preview(monkeypatch: pytest.MonkeyPatch) -> None: - stub_file_record(monkeypatch, make_file_record(filename="demo.xlsx")) +async def test_macro_enabled_spreadsheet_does_not_get_pdf_preview(monkeypatch: pytest.MonkeyPatch) -> None: + stub_file_record(monkeypatch, make_file_record(filename="demo.xlsm")) minio_client = FakeMinioClient() minio_client.objects[("knowledgebases", "db1/upload/demo.docx")] = b"PK\x03\x04excel" monkeypatch.setattr(preview, "get_minio_client", lambda: minio_client) diff --git a/backend/test/unit/routers/test_knowledge_router_cleanup.py b/backend/test/unit/routers/test_knowledge_router_cleanup.py index 2ead392626..e2f0523342 100644 --- a/backend/test/unit/routers/test_knowledge_router_cleanup.py +++ b/backend/test/unit/routers/test_knowledge_router_cleanup.py @@ -3,12 +3,12 @@ import pytest from fastapi import HTTPException, UploadFile - -from server.routers import knowledge_router from yuxi.knowledge.read_models import KnowledgeBaseDetail from yuxi.services import knowledge_task_service from yuxi.services.task_registry import get_task_definition +from server.routers import knowledge_router + pytestmark = pytest.mark.asyncio @@ -437,7 +437,7 @@ async def fake_get_database_info(kb_id: str) -> KnowledgeBaseDetail: return _database_detail() async def fake_add_file_record(kb_id: str, item_path: str, params: dict, operator_id: str | None = None): - return {"file_id": "file_1", "status": "indexing"} + return {"file_id": "file_1", "status": "indexing", "filename": "demo.txt"} async def fake_parse_file(kb_id: str, file_id: str, operator_id: str | None = None, **_kwargs): return {"file_id": file_id, "status": "parsed", "error": None} @@ -495,7 +495,7 @@ async def fake_get_database_info(kb_id: str) -> KnowledgeBaseDetail: async def fake_add_file_record(kb_id: str, item_path: str, params: dict, operator_id: str | None = None): captured_records.append({"kb_id": kb_id, "item": item_path, "params": params, "operator_id": operator_id}) - return {"file_id": f"file_{len(captured_records)}", "status": "uploaded"} + return {"file_id": f"file_{len(captured_records)}", "status": "uploaded", "filename": "demo.txt"} async def fake_parse_file(kb_id: str, file_id: str, operator_id: str | None = None, **_kwargs): return {"file_id": file_id, "status": "parsed", "error": None} diff --git a/backend/test/unit/routers/test_workspace_knowledge_tree.py b/backend/test/unit/routers/test_workspace_knowledge_tree.py index 999073df42..2128488714 100644 --- a/backend/test/unit/routers/test_workspace_knowledge_tree.py +++ b/backend/test/unit/routers/test_workspace_knowledge_tree.py @@ -2,13 +2,13 @@ from fastapi import FastAPI from fastapi.testclient import TestClient +from yuxi.knowledge.read_models import KnowledgeBaseDetail +from yuxi.services import knowledge_preview_service as preview +from yuxi.storage.postgres.models_business import User from server.routers import workspace_router from server.routers.workspace_router import workspace_knowledge from server.utils.auth_middleware import get_required_user -from yuxi.knowledge import preview -from yuxi.knowledge.read_models import KnowledgeBaseDetail -from yuxi.storage.postgres.models_business import User class FakeKnowledgeBase: @@ -135,7 +135,7 @@ def test_workspace_knowledge_tree_rejects_non_document_kb(monkeypatch): assert "不支持文件浏览" in response.json()["detail"] -def test_workspace_knowledge_file_calls_knowledge_preview_directly(monkeypatch): +def test_workspace_knowledge_file_calls_preview_usecase(monkeypatch): client = _build_client(monkeypatch, FakeKnowledgeBase()) calls = [] @@ -150,7 +150,7 @@ async def read_preview(*, kb_id: str, file_id: str) -> dict: "supported": True, } - monkeypatch.setattr(preview, "read_knowledge_file_preview", read_preview) + monkeypatch.setattr(preview, "read_workspace_knowledge_preview", read_preview) response = client.get( "/api/workspace/knowledge/file", diff --git a/backend/test/unit/services/test_binary_preview_response.py b/backend/test/unit/services/test_binary_preview_response.py new file mode 100644 index 0000000000..f113dec6ac --- /dev/null +++ b/backend/test/unit/services/test_binary_preview_response.py @@ -0,0 +1,35 @@ +"""内存二进制预览完整响应的协议回归。""" + +import pytest +from yuxi.services.file_preview import _preview_response +from yuxi.utils.filepreview import PreviewResult + +from server.routers.workspace_router import _binary_preview_response + + +@pytest.mark.parametrize("owner", ["knowledge", "workspace"]) +async def test_memory_pdf_is_one_body_with_length(owner): + """带大量换行的 PDF 必须逐字完整且只发送一个正文块。""" + content = b"%PDF-1.4\n" + b"binary-data\n" * 7100 + if owner == "knowledge": + response = _binary_preview_response( + {"content": content, "filename": "报表.pdf", "preview_type": "pdf", "media_type": "application/pdf"} + ) + else: + response = _preview_response(PreviewResult(content, "pdf", True, "application/pdf", "报表.pdf")) + messages = [] + + async def send(message): + """观察真实 ASGI 协议输出。""" + messages.append(message) + + async def receive(): + """仅提供标准请求结束信号。""" + return {"type": "http.request", "body": b"", "more_body": False} + + await response({"type": "http", "asgi": {"spec_version": "2.4"}}, receive, send) + body = [message for message in messages if message["type"] == "http.response.body"] + assert len(body) == 1 + assert body[0]["body"] == content and not body[0].get("more_body", False) + assert response.headers["content-length"] == str(len(content)) + assert response.headers["x-yuxi-preview-type"] == "pdf" diff --git a/backend/test/unit/services/test_knowledge_preview_service.py b/backend/test/unit/services/test_knowledge_preview_service.py new file mode 100644 index 0000000000..ecd612dae2 --- /dev/null +++ b/backend/test/unit/services/test_knowledge_preview_service.py @@ -0,0 +1,300 @@ +"""知识库预览的注册入口、任务状态与发布控制点。""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from minio.error import S3Error +from yuxi.knowledge import preview +from yuxi.services import knowledge_preview_service as service +from yuxi.services import knowledge_task_service +from yuxi.services.task_registry import get_task_definition +from yuxi.storage.minio.client import MinIOClient +from yuxi.utils.filepreview import MAX_BINARY_PREVIEW_SIZE_BYTES + +from server.routers import knowledge_router + + +@pytest.fixture +def preview_case(monkeypatch): + """准备元数据与外部边界,不替换被测用例。""" + record = SimpleNamespace( + kb_id="kb_1", + file_id="file_1", + filename="book.xlsx", + original_filename=None, + file_size=10, + is_folder=False, + minio_url="minio://knowledgebases/kb_1/upload/book.xlsx", + path=None, + ) + repo = SimpleNamespace(get_by_file_id=AsyncMock(return_value=record)) + client = SimpleNamespace( + KB_BUCKETS={"parsed": "knowledgebases"}, + astat_file=AsyncMock(return_value=None), + adownload_file=AsyncMock(return_value=None), + ) + tasks = SimpleNamespace( + find_task_by_payload=AsyncMock(return_value=None), + enqueue_unique_by_payload=AsyncMock(return_value=(SimpleNamespace(id="preview_1", status="pending"), True)), + ) + monkeypatch.setattr(service, "KnowledgeFileRepository", lambda: repo) + monkeypatch.setattr(preview, "KnowledgeFileRepository", lambda: repo) + monkeypatch.setattr(preview, "get_minio_client", lambda: client) + monkeypatch.setattr(service, "tasker", tasks) + return record, repo, client, tasks + + +@pytest.mark.parametrize("filename", ["book.xlsx", "book.xls", "report.docx", "slides.pptx"]) +async def test_registration_prepares_preview_without_parsing(monkeypatch, preview_case, filename): + """元数据注册后产生独立预览意图,原件状态仍为 uploaded。""" + _, _, _, tasks = preview_case + metadata = {"file_id": "file_1", "filename": filename, "size": 10, "status": "uploaded"} + register = AsyncMock(return_value=metadata) + monkeypatch.setattr(service.knowledge_base, "add_file_record", register) + result = await service.register_knowledge_file("kb_1", "minio://source", params={}, operator_id="u_1") + assert result == metadata and result["status"] == "uploaded" + assert tasks.enqueue_unique_by_payload.await_args.kwargs["payload"] == {"kb_id": "kb_1", "file_id": "file_1"} + assert tasks.enqueue_unique_by_payload.await_args.kwargs["payload_match"] == {"kb_id": "kb_1", "file_id": "file_1"} + + +@pytest.mark.parametrize(("filename", "size"), [("data.txt", 10), ("book.xlsx", MAX_BINARY_PREVIEW_SIZE_BYTES + 1)]) +async def test_registration_skips_unneeded_conversion(monkeypatch, preview_case, filename, size): + """非 Office 与超限原件不进入转换任务。""" + _, _, _, tasks = preview_case + monkeypatch.setattr( + service.knowledge_base, + "add_file_record", + AsyncMock(return_value={"file_id": "file_1", "filename": filename, "size": size, "status": "uploaded"}), + ) + await service.register_knowledge_file("kb_1", "minio://source", params={}, operator_id="u_1") + tasks.enqueue_unique_by_payload.assert_not_awaited() + + +async def test_upload_only_route_uses_prepared_preview_registration(monkeypatch, preview_case): + """直接添加入口必须在首次点击之前提交预览任务。""" + _, _, _, tasks = preview_case + item = "minio://knowledgebases/kb_1/upload/book.xlsx" + monkeypatch.setattr(knowledge_router, "_ensure_database_supports_documents", AsyncMock()) + monkeypatch.setattr( + service.knowledge_base, + "add_file_record", + AsyncMock(return_value={"file_id": "file_1", "filename": "book.xlsx", "size": 10, "status": "uploaded"}), + ) + result = await knowledge_router.add_uploaded_documents( + "kb_1", + knowledge_router.AddUploadedDocumentsRequest(items=[item], params={"content_hashes": {item: "hash_1"}}), + current_user=SimpleNamespace(uid="u_1"), + ) + assert result["added"] == 1 and result["failed"] == 0 + tasks.enqueue_unique_by_payload.assert_awaited_once() + + +async def test_ingest_route_prepares_preview_before_parse(monkeypatch, preview_case): + """批量处理入口的解析失败不取消独立预览意图。""" + _, _, _, tasks = preview_case + monkeypatch.setattr( + service.knowledge_base, + "add_file_record", + AsyncMock(return_value={"file_id": "file_1", "filename": "book.xlsx", "size": 10, "status": "uploaded"}), + ) + monkeypatch.setattr(service.knowledge_base, "parse_file", AsyncMock(side_effect=ValueError("解析器拒绝测试文件"))) + context = SimpleNamespace( + payload={"kb_id": "kb_1", "items": ["minio://source"], "params": {}, "operator_id": "u_1"}, + task_id="ingest_1", + worker_id="owner_1", + set_progress=AsyncMock(), + set_result=AsyncMock(), + raise_if_cancelled=AsyncMock(), + ) + with pytest.raises(RuntimeError, match="失败 1"): + await knowledge_task_service.run_knowledge_ingest(context) + tasks.enqueue_unique_by_payload.assert_awaited_once() + + +@pytest.mark.parametrize("entrypoint", ["upload", "ingest"]) +async def test_preview_submission_failure_keeps_registered_file_and_ingest(monkeypatch, preview_case, entrypoint): + """独立预览意图提交失败,已提交原件仍返回成功并继续解析入库。""" + _, _, _, tasks = preview_case + metadata = {"file_id": "file_1", "filename": "book.xlsx", "size": 10, "status": "uploaded"} + register = AsyncMock(return_value=metadata) + parse = AsyncMock(return_value={**metadata, "status": "parsed"}) + index = AsyncMock(return_value={**metadata, "status": "indexed"}) + monkeypatch.setattr(service.knowledge_base, "add_file_record", register) + monkeypatch.setattr(service.knowledge_base, "parse_file", parse) + monkeypatch.setattr(service.knowledge_base, "update_file_params", AsyncMock()) + monkeypatch.setattr(service.knowledge_base, "index_file", index) + # 告警是同步 logger,业务结果通过返回值与实际解析参数核对。 + alert = Mock() + monkeypatch.setattr(service, "logger", SimpleNamespace(exception=alert), raising=False) + tasks.enqueue_unique_by_payload.side_effect = RuntimeError("测试预览意图无法提交") + item = "minio://knowledgebases/kb_1/upload/book.xlsx" + + if entrypoint == "upload": + monkeypatch.setattr(knowledge_router, "_ensure_database_supports_documents", AsyncMock()) + result = await knowledge_router.add_uploaded_documents( + "kb_1", + knowledge_router.AddUploadedDocumentsRequest(items=[item], params={"content_hashes": {item: "hash_1"}}), + current_user=SimpleNamespace(uid="u_1"), + ) + assert result["added"] == 1 and result["failed"] == 0 + assert result["items"][0]["file_id"] == "file_1" + parse.assert_not_awaited() + else: + context = SimpleNamespace( + payload={"kb_id": "kb_1", "items": [item], "params": {"auto_index": True}, "operator_id": "u_1"}, + task_id="ingest_1", + worker_id="owner_1", + set_progress=AsyncMock(), + set_result=AsyncMock(), + raise_if_cancelled=AsyncMock(), + ) + result = await knowledge_task_service.run_knowledge_ingest(context) + assert result["failed"] == 0 and result["items"] == [{**metadata, "status": "indexed"}] + assert parse.await_args.args == ("kb_1", "file_1") + assert index.await_args.args == ("kb_1", "file_1") + + register.assert_awaited_once() + alert.assert_called_once() + assert alert.call_args.args[1:] == ("kb_1", "file_1") + + +async def test_missing_cache_returns_loading_without_inline_conversion(monkeypatch, preview_case): + """预览请求只提交任务并立即返回状态,转换不能在请求中执行。""" + _, _, _, tasks = preview_case + render = AsyncMock(side_effect=AssertionError("不能在读取请求中转换")) + monkeypatch.setattr(preview, "convert_office_to_pdf", render) + response = await service.read_workspace_knowledge_preview("kb_1", "file_1") + assert response["status"] == "loading" and response["supported"] + assert "生成预览" in response["message"] + tasks.enqueue_unique_by_payload.assert_awaited_once() + render.assert_not_awaited() + + +@pytest.mark.parametrize("status", ["pending", "running"]) +async def test_repeated_read_reuses_active_preview_task(monkeypatch, preview_case, status): + """非终态 Task 在多次读取中保持唯一意图。""" + _, _, _, tasks = preview_case + tasks.find_task_by_payload.return_value = SimpleNamespace(status=status) + for _ in range(2): + result = await service.read_workspace_knowledge_preview("kb_1", "file_1") + assert result["status"] == "loading" + tasks.enqueue_unique_by_payload.assert_not_awaited() + + +@pytest.mark.parametrize("status", ["failed", "cancelled"]) +async def test_failed_preview_ends_loading(preview_case, status): + """失败与取消必须返回错误终态,不能持续轮询。""" + _, _, _, tasks = preview_case + tasks.find_task_by_payload.return_value = SimpleNamespace(status=status) + result = await service.read_workspace_knowledge_preview("kb_1", "file_1") + assert result["status"] == "error" and not result["supported"] + tasks.enqueue_unique_by_payload.assert_not_awaited() + + +async def test_cache_is_authority_after_success_or_task_failure(monkeypatch, preview_case): + """可读取的缓存字节直接返回,不依据任务报告推断缓存缺失。""" + _, _, client, tasks = preview_case + client.astat_file.return_value = 16 + expected = {"content": b"%PDF-1.4", "binary": True, "preview_type": "pdf"} + monkeypatch.setattr(service, "read_knowledge_file_preview", AsyncMock(return_value=expected)) + for _ in range(2): + assert await service.read_workspace_knowledge_preview("kb_1", "file_1") == expected + tasks.find_task_by_payload.assert_not_awaited() + tasks.enqueue_unique_by_payload.assert_not_awaited() + + +async def test_deleted_cache_is_regenerated_after_success(preview_case): + """成功记录不能代替已经丢失的派生对象。""" + _, _, _, tasks = preview_case + tasks.find_task_by_payload.return_value = SimpleNamespace(status="success") + assert (await service.read_workspace_knowledge_preview("kb_1", "file_1"))["status"] == "loading" + tasks.enqueue_unique_by_payload.assert_awaited_once() + + +@pytest.mark.parametrize("target", ["wrong_kb", "folder"]) +async def test_invalid_target_never_submits_preview(preview_case, target): + """目标文件必须属于路由已经授权的同一知识库。""" + record, _, client, tasks = preview_case + record.is_folder = target == "folder" + with pytest.raises(ValueError, match="not found" if target == "wrong_kb" else "folder"): + await service.read_workspace_knowledge_preview("other_kb" if target == "wrong_kb" else "kb_1", "file_1") + client.astat_file.assert_not_awaited() + tasks.enqueue_unique_by_payload.assert_not_awaited() + + +@pytest.mark.parametrize("reason", ["lease_lost", "source_deleted"]) +async def test_worker_guard_stops_cache_write_after_ownership_or_source_loss(monkeypatch, preview_case, reason): + """转换期间失权或原件删除时,实际缓存发布控制点拒绝写入。""" + record, repo, _, _ = preview_case + objects = {("knowledgebases", "kb_1/upload/book.xlsx"): b"office"} + + class Minio: + """保留对象字节以观察发布结果。""" + + KB_BUCKETS = {"parsed": "knowledgebases"} + + async def astat_file(self, bucket, key): + """读取对象存在性。""" + return len(objects[(bucket, key)]) if (bucket, key) in objects else None + + async def adownload_file(self, bucket, key): + """返回真实对象字节。""" + return objects[(bucket, key)] + + async def aupload_file(self, bucket_name, object_name, data, content_type): + """记录发布后的缓存。""" + objects[(bucket_name, object_name)] = data + + monkeypatch.setattr(preview, "KnowledgeFileRepository", lambda: repo) + monkeypatch.setattr(preview, "get_minio_client", Minio) + monkeypatch.setattr(preview, "convert_office_to_pdf", AsyncMock(return_value=b"%PDF-1.4")) + if reason == "source_deleted": + repo.get_by_file_id.side_effect = [record, None] + context = SimpleNamespace( + payload={"kb_id": "kb_1", "file_id": "file_1"}, + set_progress=AsyncMock(), + raise_if_cancelled=AsyncMock( + side_effect=[None, asyncio.CancelledError("lease_lost")] if reason == "lease_lost" else [None, None] + ), + ) + with pytest.raises( + asyncio.CancelledError if reason == "lease_lost" else ValueError, + match="lease_lost" if reason == "lease_lost" else "原件已删除", + ): + await service.run_knowledge_preview(context) + assert objects == {("knowledgebases", "kb_1/upload/book.xlsx"): b"office"} + + +def test_preview_handler_is_registered_on_shipping_path(): + """registry 必须惰性装配实际 worker Handler。""" + assert get_task_definition("knowledge_preview").load_handler() is service.run_knowledge_preview + + +async def test_cache_disappearing_before_download_submits_task_without_inline_conversion(monkeypatch, preview_case): + """缓存存在性快照失效后,实际对象缺失只能交给后台任务。""" + _, _, client, tasks = preview_case + client.astat_file.side_effect = [14, None] + storage = MinIOClient() + + class MissingObject: + """对象存储在下载时报告缓存已删除。""" + + def get_object(self, bucket_name, object_name): + """使用真实存储错误,覆盖缓存读取和 SDK 异常边界。""" + raise S3Error(None, "NoSuchKey", "cache disappeared", object_name, "test", "test") + + storage._client = MissingObject() + client.adownload_file = storage.adownload_file + converter = AsyncMock(side_effect=AssertionError("HTTP 请求不能转换 Office")) + publisher = AsyncMock(side_effect=AssertionError("HTTP 请求不能发布缓存")) + client.aupload_file = publisher + monkeypatch.setattr(preview, "convert_office_to_pdf", converter) + + result = await service.read_workspace_knowledge_preview("kb_1", "file_1") + + assert result["status"] == "loading" + tasks.enqueue_unique_by_payload.assert_awaited_once() + converter.assert_not_awaited() + publisher.assert_not_awaited() diff --git a/backend/test/unit/services/test_workspace_service.py b/backend/test/unit/services/test_workspace_service.py index e539f5e386..ab9ecce5fa 100644 --- a/backend/test/unit/services/test_workspace_service.py +++ b/backend/test/unit/services/test_workspace_service.py @@ -7,11 +7,10 @@ from types import SimpleNamespace import pytest -from fastapi import HTTPException, UploadFile - import yuxi.workspace.preview as file_preview -from yuxi.workspace import paths as workspace_paths +from fastapi import HTTPException, UploadFile from yuxi.services import workspace_service as svc +from yuxi.workspace import paths as workspace_paths def _user() -> SimpleNamespace: @@ -99,7 +98,7 @@ def test_workspace_root_rejects_symlink_root(tmp_path: Path, monkeypatch) -> Non ("filename", "content"), [ ("bad.txt", b"\xff\xfe\x00"), - ("sheet.xlsx", b"PK\x03\x04excel"), + ("sheet.xlsm", b"PK\x03\x04excel"), ], ) @pytest.mark.asyncio @@ -122,29 +121,31 @@ async def test_read_workspace_file_content_returns_unsupported_for_unreadable_fi assert result["supported"] is False +@pytest.mark.parametrize("filename", ["demo.docx", "slides.pptx", "sheet.xlsx", "sheet.xls"]) @pytest.mark.asyncio async def test_read_workspace_file_content_returns_pdf_preview_for_office_file( tmp_path: Path, monkeypatch, + filename: str, ) -> None: + """Office 与 Excel 文件返回可展示的 PDF 响应。""" monkeypatch.setenv("YUXI_USER_DATA_DIR", str(tmp_path / "threads")) user = _user() root = _workspace_root(user) - target = root / "demo.docx" + target = root / filename target.write_bytes(b"office") - async def fake_convert(filename: str, content: bytes) -> bytes: - assert filename == "demo.docx" + async def fake_convert(name: str, content: bytes) -> bytes: + assert name == filename assert content == b"office" return b"%PDF-1.4\npreview" monkeypatch.setenv("YUXI_RUNTIME_DIR", str(tmp_path / "runtime")) monkeypatch.setattr(file_preview, "convert_office_to_pdf", fake_convert) - result = await svc.read_workspace_file_content(path="/demo.docx", current_user=user) - body = b"" - async for chunk in result.body_iterator: - body += chunk + result = await svc.read_workspace_file_content(path=f"/{filename}", current_user=user) + body = result.body + assert result.headers["content-length"] == str(len(body)) assert result.media_type == "application/pdf" assert result.headers["x-yuxi-preview-type"] == "pdf" @@ -152,7 +153,7 @@ async def fake_convert(filename: str, content: bytes) -> bytes: @pytest.mark.asyncio -async def test_read_workspace_file_content_rejects_xlsx_preview( +async def test_read_workspace_file_content_reports_excel_conversion_failure( tmp_path: Path, monkeypatch, ) -> None: @@ -162,11 +163,19 @@ async def test_read_workspace_file_content_rejects_xlsx_preview( target = root / "sheet.xlsx" target.write_bytes(b"PK\x03\x04excel") - result = await svc.read_workspace_file_content(path="/sheet.xlsx", current_user=user) + async def fail_convert(filename: str, content: bytes) -> bytes: + """模拟转换器明确拒绝损坏表格。""" + from yuxi.utils.filepreview import OfficePreviewConversionError - assert result["content"] is None - assert result["preview_type"] == "unsupported" - assert result["supported"] is False + raise OfficePreviewConversionError("表格已损坏") + + monkeypatch.setenv("YUXI_RUNTIME_DIR", str(tmp_path / "runtime")) + monkeypatch.setattr(file_preview, "convert_office_to_pdf", fail_convert) + with pytest.raises(HTTPException) as exc_info: + await svc.read_workspace_file_content(path="/sheet.xlsx", current_user=user) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == "表格已损坏" @pytest.mark.parametrize( @@ -174,6 +183,8 @@ async def test_read_workspace_file_content_rejects_xlsx_preview( [ ("demo.docx", b"office"), ("slides.pptx", b"presentation"), + ("sheet.xlsx", b"spreadsheet"), + ("sheet.xls", b"legacy spreadsheet"), ], ) @pytest.mark.asyncio @@ -206,10 +217,8 @@ async def read_pdf() -> bytes: response = await svc.read_workspace_file_content(path=f"/{filename}", current_user=user) assert response.media_type == "application/pdf" assert response.headers["x-yuxi-preview-type"] == "pdf" - body = b"" - async for chunk in response.body_iterator: - body += chunk - return body + assert response.headers["content-length"] == str(len(response.body)) + return response.body assert await read_pdf() == b"%PDF-1.4\npreview" assert await read_pdf() == b"%PDF-1.4\npreview" diff --git a/backend/test/unit/storage/test_minio_public_images.py b/backend/test/unit/storage/test_minio_public_images.py index cdc17403b7..f59da22cbe 100644 --- a/backend/test/unit/storage/test_minio_public_images.py +++ b/backend/test/unit/storage/test_minio_public_images.py @@ -4,8 +4,7 @@ import pytest from minio.error import S3Error from urllib3 import HTTPResponse - -from yuxi.storage.minio.client import MinIOClient, normalize_public_minio_url +from yuxi.storage.minio.client import MinIOClient, StorageError, normalize_public_minio_url class FakeMinio: @@ -134,3 +133,26 @@ def list_objects(self, *_args, **_kwargs): client._client = MissingBucketClient() assert await client.adelete_objects_by_prefix("kb-images", "kb-1/") == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("code", ["NoSuchKey", "AccessDenied"]) +async def test_optional_cache_download_only_allows_missing_object(code): + """仅对象缺失可以返回空结果,权限等存储故障保持失败。""" + client = MinIOClient() + + class FailingObject: + """提供真实 SDK 错误码。""" + + def get_object(self, bucket_name, object_name): + """模拟真实存储读取失败。""" + raise S3Error(None, code, "test error", object_name, "test", "test") + + client._client = FailingObject() + if code == "NoSuchKey": + assert await client.adownload_file("bucket", "object", missing_ok=True) is None + else: + with pytest.raises(StorageError): + await client.adownload_file("bucket", "object", missing_ok=True) + with pytest.raises(StorageError): + await client.adownload_file("bucket", "object") diff --git a/backend/test/unit/utils/test_filepreview.py b/backend/test/unit/utils/test_filepreview.py index 4d316f6cf1..d3bd295697 100644 --- a/backend/test/unit/utils/test_filepreview.py +++ b/backend/test/unit/utils/test_filepreview.py @@ -71,9 +71,12 @@ def test_render_preview_keeps_unsupported_binary_content_hidden(): assert result.supported is False -def test_office_pdf_preview_scope_only_includes_docx_and_pptx(): +def test_office_pdf_preview_scope_includes_spreadsheets(): assert is_office_pdf_preview_file("demo.docx") is True assert is_office_pdf_preview_file("demo.pptx") is True - assert is_office_pdf_preview_file("demo.xlsx") is False + assert is_office_pdf_preview_file("demo.xlsx") is True + assert is_office_pdf_preview_file("表格.XLS") is True + assert is_office_pdf_preview_file("表格.XLSX") is True + assert is_office_pdf_preview_file("demo.xlsm") is False assert is_office_pdf_preview_file("demo.doc") is False assert is_office_pdf_preview_file("demo.ppt") is False diff --git a/docs/develop-guides/decisions/implemented/2026-10-10-office-preview-preparation-and-buffered-response.md b/docs/develop-guides/decisions/implemented/2026-10-10-office-preview-preparation-and-buffered-response.md new file mode 100644 index 0000000000..729af2943b --- /dev/null +++ b/docs/develop-guides/decisions/implemented/2026-10-10-office-preview-preparation-and-buffered-response.md @@ -0,0 +1,57 @@ +# 知识库文件预览提前生成与完整响应 + +状态:implemented +类型:feature +Owner:backend/package/yuxi/services/knowledge_preview_service.py + +## 问题 + +知识库 Office 文件首次点击时才转换,已有 PDF 缓存也会反复等待。已完整读取到内存的 PDF 经 BytesIO 同步流式响应按行迭代,约 200 KB 的文件产生数千个正文块。工作区还缺少 `.xlsx` 与 `.xls` 的 Office 预览识别。 + +目标是支持 Excel 原件预览、上传注册后在后台准备共享 PDF、缓存命中后完整返回,以及生成中自动显示和失败结束等待。聊天附件提前转换、存量文件批量补转、浏览器持久缓存、PDF 排版和登录导航不在范围内。 + +## 决策 + +### 实现方案 + +扩展现有 Office 格式识别到 `.xlsx` 与 `.xls`,复用 LibreOffice、30 MB 预览大小限制和转换超时。`.xlsm` 保持不支持。 + +原件注册后提交去重的 `knowledge_preview` Task。直接添加与上传后解析共用注册 service;PostgreSQL Task 拥有意图、lease 与终态,提交后投递现有 worker。worker 在发布 MinIO PDF 前核对 lease 与原件记录,原件与文档解析状态保持独立。预览意图提交异常记录带知识库与文件标识的告警,已提交的原件注册结果仍返回,解析与入库继续;后续缺失缓存读取补交任务。现成 PDF 对象拥有可读取结果。 + +工作区在授权后只读缓存;缺失时展示任务状态并补交存量文件的去重任务,失败或取消明确结束等待。已在内存的二进制通过完整 Response 和 Content-Length 返回。WorkspaceView 与 FileDetailModal 每两秒检查当前文件,切换、关闭和离开页面后隔离旧请求。FileDetailModal 的本地激活状态直接控制可见性,KeepAlive 离页即隐藏弹窗与遮罩,父组件的 open prop 延迟回传不影响隐藏。 + +## 替代方案 + +仅提前转换仍会保留缓存命中后的正文传输等待。同步上传等待转换会增加多文件处理时间。独立队列或浏览器持久缓存会新增执行状态或权限失效规则。复用现有 Task、转换器与共享缓存满足当前需求。 + +## 验证 + +| 验收主张 | 失败面 | 语义 Owner | 直接证据 / 命令 | 负向案例 | 当前结果 | +|---|---|---|---|---|---| +| XLSX 与真实 XLS 上传后在首次点击前生成对应 PDF,原件不变 | 只有格式识别或伪造成功状态 | Office 转换器、注册 service、worker、MinIO | `test/integration/api/test_knowledge_preview_http.py` 回读 Task、PDF 标记、原件哈希 | 损坏工作簿转换失败,缓存不发布 | Passed | +| 已有缓存完整返回且无需同步转换 | HTTP 成功但正文逐行阻塞或缓存重写 | Knowledge 预览模块、HTTP Response | `test/unit/services/test_binary_preview_response.py` 与真实 HTTP 重复读取 | 验收进程恢复旧 BytesIO 响应后正文分块断言失败 | Passed | +| 去重、授权、执行权与原件检查守住发布边界 | 并发重复任务、失权或无权限路径被成功伪装 | Task repository、权限依赖、预览 worker | service unit 与真实 HTTP 的并发、拒绝和最终对象断言 | 私有库、错误库、文件夹、失权、删除原件和存储错误 | Passed | +| 页面自动展示完成结果、错误结束等待、旧请求不覆盖新文件 | 永久 loading 或迟到结果覆盖 | WorkspaceView、file_preview | `web/test/unit/workspace_preview_polling.test.js`、`web/test/unit/file_detail_preview_polling.test.js` 与真实页面合成数据验证 | 失败、切换、deactivated、unmounted 和迟到响应 | Passed | + +### 执行命令与观察边界 + +后端在匹配锁文件的离线运行镜像中只读挂载源码执行;验收服务为独立 PostgreSQL 数据库、专用 Redis、MinIO 与真实 worker。前端与文档使用锁文件对应的已有依赖。实际执行如下: + +- `python -m pytest test/unit -m "not slow" -q -o addopts= --disable-warnings`:2547 passed,7 subtests passed。 +- `python -m pytest test/integration/api/test_knowledge_preview_http.py --confcutdir=test/integration/api -m "not slow" -q -o addopts= -s`:1 passed;真实 HTTP、认证、Task、worker、LibreOffice 和 MinIO 字节回读。临时 trigger 只拒绝预览 Task 插入,上传和添加仍成功;删除 trigger 后首次读取补交并生成 PDF。 +- `node --test --test-concurrency=1 "test/**/*.test.js" "test/**/*.spec.js"`:413 passed;详情轮询的 9 项用例执行提取的真实函数及 watcher/lifecycle 回调,属于 unit。 +- `node node_modules/eslint/bin/eslint.js . --max-warnings=0` 与 `node node_modules/vite/bin/vite.js build`:Passed。 +- Ruff 0.16.4 的 `check package`、`format package --check`,以及新增 service unit 与 HTTP 验收文件的 check/format:Passed。 +- `python scripts/verify_engineering_contracts.py` 与 `python -m unittest scripts.test_verify_engineering_contracts`:Passed,后者 63 tests。 +- 在 `docs/` 执行 `node node_modules/vitepress/bin/vitepress.js build`:Passed。 +- 在仓库根目录执行 `git diff HEAD --check` 与 `git diff --cached --check`:Passed。 + +真实浏览器使用组件原始 Vue 实现和合成网络响应,覆盖生成中提示与完成后自动展示。真实 Vue KeepAlive 页面验证生成中离页关闭弹窗、请求计数停止、弹窗与遮罩的 DOM display 为 none,返回页面重新打开后自动显示 PDF。恢复旧可见性 getter 的独立测试进程保留陈旧 props.open=true,两条关闭断言因 visible=true 失败。浏览器合成数据验收和真实 HTTP/worker 验收是两条独立证据,不构成连接真实后端的浏览器 E2E。真实业务文件、其他浏览器和 PDF 排版视觉回归为 Not run。 + +恢复旧 BytesIO 逐行响应的测试进程产生 7102 个正文块,两条协议断言均因目标原因失败。恢复读取时同步转换后,读取边界测试因“不能在读取请求中转换”失败;缺少专用 Redis 或发现已有 worker 的 fixture 在写入前拒绝。XLSX 与真正 OLE 格式 XLS 均在首次点击前生成对应 PDF。 + +## 后果 + +提前转换占用现有 worker 槽位。文件记录与 Task 分开提交,进程崩溃造成的提交间隙由后续缺失缓存读取补交任务恢复。PostgreSQL 检查与 MinIO 发布不构成跨存储事务,删除竞态仍可留下无可读取元数据的派生对象。API 与 worker 需要部署一致的后端版本。 + +验证使用独立 PostgreSQL 数据库、显式指定的专用 Redis、MinIO、真实 HTTP 和 worker;fixture 在断言失败时也清理自身资源。该贡献不需要新增依赖、配置或 Schema 迁移。 diff --git a/docs/mechanisms/knowledge-base.md b/docs/mechanisms/knowledge-base.md index 4ff0cf66c6..3d5d35058c 100644 --- a/docs/mechanisms/knowledge-base.md +++ b/docs/mechanisms/knowledge-base.md @@ -70,13 +70,25 @@ stateDiagram-v2 | 存储 | 拥有的事实 | 不拥有的事实 | | --- | --- | --- | | PostgreSQL | 知识库配置、权限、文件元数据和状态、chunk 正文、图谱处理状态、Task 执行意图与 lease | 原文件字节、向量索引 | -| MinIO | 上传原件、解析 Markdown、解析图片 | 文件当前状态、用户权限 | +| MinIO | 上传原件、解析 Markdown、解析图片、共享 Office 预览 PDF | 文件当前状态、用户权限 | | Milvus | chunk 向量、BM25/混合检索字段、图实体和关系向量 | 权限、文件状态 | | Neo4j | 可选的实体、关系和 chunk 关联 | 原文件、权限和检索排序 | | Redis / ARQ | Task 投递与 worker 唤醒、知识库最小运行配置缓存 | Task 最终状态、配置最终值、文件状态 | Milvus 索引会把 chunk 写入 PostgreSQL 和 Milvus。它不是跨存储事务:任一侧失败时会尝试补偿并把文件置为 `error_indexing`,排查时需要同时查看两侧。 +## Office 文件预览 + +知识库 `.docx`、`.pptx`、`.xlsx` 和 `.xls` 文件在注册原件记录后提交独立的 `knowledge_preview` 任务,上传后只添加和上传后解析两个入口共同使用这个注册用例。已知大小超过 30 MB 的文件不提交预览转换;大小未知时由现有预览读取边界检查对象大小。转换与解析、索引各自保留状态,预览完成不会把文件标记为已解析或已入库。 + +现有 Durable Task worker 调用 Office 转换器,把 PDF 保存到 MinIO 的 `knowledgebases/{kb_id}/preview/{file_id}.pdf`。同一知识库和文件只保留一个非终态预览任务。Task 记录拥有执行意图、租约和终态;对象存储中的 PDF 字节拥有可读取的预览结果。worker 在发布缓存前重新检查执行权和原件记录,跨 PostgreSQL 与 MinIO 的检查和写入不构成事务,删除竞态中的派生对象不授予文件读取权限。 + +个人空间的共享知识库预览先执行知识库读权限检查。缓存已准备好时,接口返回带 `Content-Length` 的完整 PDF;缓存缺失时,接口返回生成中状态,个人空间与文件详情弹窗每两秒检查当前文件,完成后自动显示。切换、关闭或离开页面后,过期响应不能覆盖当前预览;缓存页面离开时组件本地激活状态直接关闭详情弹窗与遮罩。存量文件第一次读取可以提交同一个去重任务;本链路不批量补转存量文件,也不为个人空间文件或聊天附件提前转换。 + +转换失败或取消时,页面结束等待并提示下载原件;原件仍保存独立字节。成功任务的缓存对象丢失时,后续读取重新提交预览任务。预览意图提交异常记录知识库和文件标识,已经提交的原件注册结果仍返回,解析与入库继续。文件记录和任务之间的提交间隙由后续读取缺失缓存时补交任务闭合,Task 首次投递失败仍按持久任务恢复流程处理。 + +注册与预览状态由 `knowledge_preview_service` 用例装配,转换和缓存字节由 Knowledge 预览模块拥有,页面轮询属于呈现。验证时在首次预览请求之前回读任务和 MinIO PDF,再核对重复请求返回字节、缓存版本、原件哈希和权限拒绝结果。 + ## Durable Task 和恢复 上传原文件是同步对象存储操作;批量添加、解析、索引和图谱构建把 `task_type`、Handler 版本和可序列化 payload 保存到 PostgreSQL。提交后 API 只发布 `task_id`,ARQ worker 从 registry 加载领域 Handler。 diff --git "a/docs/public/\351\242\204\350\247\210\344\274\230\345\214\226/\347\246\273\351\241\265\345\205\263\351\227\255\350\257\246\346\203\205.jpg" "b/docs/public/\351\242\204\350\247\210\344\274\230\345\214\226/\347\246\273\351\241\265\345\205\263\351\227\255\350\257\246\346\203\205.jpg" new file mode 100644 index 0000000000..b5849b15d7 Binary files /dev/null and "b/docs/public/\351\242\204\350\247\210\344\274\230\345\214\226/\347\246\273\351\241\265\345\205\263\351\227\255\350\257\246\346\203\205.jpg" differ diff --git "a/docs/public/\351\242\204\350\247\210\344\274\230\345\214\226/\350\257\246\346\203\205\350\207\252\345\212\250\351\242\204\350\247\210.jpg" "b/docs/public/\351\242\204\350\247\210\344\274\230\345\214\226/\350\257\246\346\203\205\350\207\252\345\212\250\351\242\204\350\247\210.jpg" new file mode 100644 index 0000000000..85668ad0d2 Binary files /dev/null and "b/docs/public/\351\242\204\350\247\210\344\274\230\345\214\226/\350\257\246\346\203\205\350\207\252\345\212\250\351\242\204\350\247\210.jpg" differ diff --git a/web/src/components/FileDetailModal.vue b/web/src/components/FileDetailModal.vue index e4bbd57acc..1d322c7f1c 100644 --- a/web/src/components/FileDetailModal.vue +++ b/web/src/components/FileDetailModal.vue @@ -64,7 +64,6 @@ :file="sourcePreviewFile" :file-path="file?.filename || ''" :status="sourcePreview.loading ? 'loading' : ''" - loading-message="正在加载文件内容..." :show-header="false" :show-download="false" :show-inline-html-controls="true" @@ -118,7 +117,7 @@ diff --git a/web/src/components/TaskCenterDrawer.vue b/web/src/components/TaskCenterDrawer.vue index 84266fec2d..e1e244c363 100644 --- a/web/src/components/TaskCenterDrawer.vue +++ b/web/src/components/TaskCenterDrawer.vue @@ -191,6 +191,7 @@ const STATUS_CONFIG = { } const TASK_TYPE_LABELS = { knowledge_ingest: '知识库导入', + knowledge_preview: '文件预览', knowledge_parse: '文档解析', knowledge_index: '文档入库', knowledge_graph_index: '图谱构建', diff --git a/web/src/utils/file_preview.js b/web/src/utils/file_preview.js index 20fde6a59d..92681f6b41 100644 --- a/web/src/utils/file_preview.js +++ b/web/src/utils/file_preview.js @@ -12,7 +12,7 @@ const IMAGE_EXTENSIONS = new Set([ ]) const PDF_EXTENSIONS = new Set(['.pdf']) const HTML_EXTENSIONS = new Set(['.html', '.htm']) -const OFFICE_EXTENSIONS = new Set(['.docx', '.pptx']) +const OFFICE_EXTENSIONS = new Set(['.docx', '.pptx', '.xlsx', '.xls']) const TEXT_EXTENSIONS = new Set([ '.txt', '.text', @@ -170,7 +170,9 @@ export const normalizePreviewResponse = async (response, baseFile = {}) => { ...baseFile, ...payload, content: payload.content ?? '', - loading: false, + loading: payload.status === 'loading', + loadingMessage: payload.status === 'loading' ? payload.message || '正在生成预览...' : '', + errorMessage: payload.status === 'error' ? payload.message || '文件预览失败' : '', status: payload.status || (payload.supported === false ? 'unsupported' : 'ready'), previewType, supported: payload.supported !== false, diff --git a/web/src/views/WorkspaceView.vue b/web/src/views/WorkspaceView.vue index efa2b6a1d7..53535b2df4 100644 --- a/web/src/views/WorkspaceView.vue +++ b/web/src/views/WorkspaceView.vue @@ -210,7 +210,7 @@ ')) + +/** 执行真实弹窗函数、视图 watcher 与卸载清理,仅替换接口与时钟。 */ +function setup(responses) { + const timers = [] + const calls = [] + const revoked = [] + const hooks = {} + const props = reactive({ open: true, kbId: 'kb_1', fileId: 'file_1' }) + const emitted = [] + const visibilityState = runInNewContext(`${visibility}; ({ componentActive, visible })`, { + computed, ref, props, emit: (event, value) => emitted.push([event, value]) + }) + const state = { file: ref({ file_id: 'file_1', filename: 'book.xlsx' }), + sourcePreview: ref({ loading: false, url: '', content: '', type: '', message: '', supported: true }), + viewMode: ref('source'), hasSourcePreview: ref(true), ...visibilityState, + sourcePreviewCandidateType: ref('office'), contentState: ref({}), basicLoading: ref(false), + detailError: ref(''), downloadingOriginal: ref(false), downloadingMarkdown: ref(false) } + const context = { ...state, props, console: { error() {} }, message: { error() {} }, + window: { URL: { revokeObjectURL: (url) => revoked.push(url) }, + setTimeout: (callback, delay) => { assert.equal(delay, 2000); timers.push(callback) } }, + watch: (_sources, callback) => { hooks.viewChanged = callback }, + onActivated: (callback) => { hooks.activated = callback }, + onDeactivated: (callback) => { hooks.deactivated = callback }, + onBeforeUnmount: (callback) => { hooks.unmounted = callback }, + getWorkspaceKnowledgeFileContent: async (kbId, fileId) => { calls.push([kbId, fileId]); return await responses.shift() }, + normalizePreviewResponse: async (response) => response } + const functions = runInNewContext(`let basicRequestSeq = 0, contentRequestSeq = 0, sourceRequestSeq = 0; + ${resets} +${polling} +${watches} +${lifecycle}; + ({ loadSourcePreview, resetSourcePreview })`, context) + return { ...state, ...functions, props, timers, calls, revoked, hooks, emitted } +} + +async function flush() { + await new Promise((resolve) => setImmediate(resolve)) +} + +test('详情弹窗生成中保持等待,完成后自动显示 PDF', async () => { + const state = setup([{ status: 'loading', message: '正在生成预览' }, + { status: 'ready', previewType: 'pdf', previewUrl: 'blob:ready' }]) + const finished = state.loadSourcePreview() + await flush() + assert.equal(state.sourcePreview.value.loading, true) + assert.equal(state.sourcePreview.value.message, '正在生成预览') + await state.loadSourcePreview() + assert.equal(state.calls.length, 1, '重复 watcher 不启动并行读取') + state.timers.shift()() + await finished + assert.equal(state.sourcePreview.value.url, 'blob:ready') + assert.equal(state.sourcePreview.value.type, 'pdf') + assert.equal(state.sourcePreview.value.loading, false) + assert.equal(state.calls.length, 2) +}) + +test('详情弹窗失败进入终态并停止轮询', async () => { + const state = setup([{ status: 'error', supported: false, message: '预览生成失败' }]) + await state.loadSourcePreview() + assert.equal(state.sourcePreview.value.message, '预览生成失败') + assert.equal(state.sourcePreview.value.supported, false) + assert.equal(state.sourcePreview.value.loading, false) + assert.equal(state.timers.length, 0) +}) + +for (const event of ['close', 'view', 'unmount', 'deactivate', 'file']) { + test(`详情弹窗 ${event} 后旧轮询结束,返回源文件可以重新加载`, async () => { + const state = setup([{ status: 'loading', message: '正在生成预览' }, + { status: 'ready', previewUrl: 'blob:new' }]) + const finished = state.loadSourcePreview() + await flush() + if (event === 'unmount') state.hooks.unmounted() + else if (event === 'deactivate') { + state.hooks.deactivated() + assert.equal(state.visible.value, false) + } else if (event === 'file') { + state.resetSourcePreview() + state.props.fileId = 'file_2' + state.file.value = { file_id: 'file_2', filename: 'new.xlsx' } + } else { + state.viewMode.value = event === 'view' ? 'markdown' : 'source' + state.props.open = event !== 'close' + await state.hooks.viewChanged([state.props.open, state.file.value, state.viewMode.value]) + } + state.timers.shift()() + await finished + assert.equal(state.calls.length, 1) + assert.equal(state.sourcePreview.value.loading, false) + assert.equal(state.sourcePreview.value.message, '') + if (event === 'unmount' || event === 'deactivate') return + state.props.open = true + state.viewMode.value = 'source' + await state.loadSourcePreview() + assert.equal(state.sourcePreview.value.url, 'blob:new') + assert.equal(state.calls.length, 2) + }) +} + +test('关闭后的迟到 PDF 不覆盖弹窗状态并释放对象 URL', async () => { + let complete + const state = setup([new Promise((resolve) => { complete = resolve })]) + const finished = state.loadSourcePreview() + state.props.open = false + await state.hooks.viewChanged([false, state.file.value, 'source']) + complete({ status: 'ready', previewUrl: 'blob:old' }) + await finished + assert.equal(state.sourcePreview.value.url, '') + assert.deepEqual(state.revoked, ['blob:old']) +}) + +test('缓存页面关闭直接隐藏弹窗,父 prop 迟到回传也不残留遮罩', () => { + const state = setup([]) + assert.equal(state.visible.value, true) + state.hooks.deactivated() + assert.equal(state.props.open, true, '保留尚未回传的父组件 open prop') + assert.equal(state.visible.value, false, '本地激活状态必须立即关闭 Teleport 弹窗') + assert.deepEqual(state.emitted, [['update:open', false]]) + state.props.open = false + state.hooks.activated() + assert.equal(state.visible.value, false, '返回页面保持关闭,直到用户重新打开') + state.props.open = true + assert.equal(state.visible.value, true) +}) diff --git a/web/test/unit/file_preview.test.js b/web/test/unit/file_preview.test.js new file mode 100644 index 0000000000..78405267ad --- /dev/null +++ b/web/test/unit/file_preview.test.js @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { getPreviewTypeByPath, normalizePreviewResponse } from '../../src/utils/file_preview.js' + +for (const path of ['/报表.xlsx', '/历史.XLS']) { + test(`${path} 进入 Office 预览,PDF 响应保留原件身份`, async () => { + assert.equal(getPreviewTypeByPath(path), 'office') + const result = await normalizePreviewResponse( + new Response('%PDF-1.4\npreview', { headers: { 'content-type': 'application/pdf' } }), + { path, filename: path.slice(1) } + ) + assert.equal(result.previewType, 'pdf') + assert.equal(result.supported, true) + assert.equal(result.path, path) + assert.equal(result.filename, path.slice(1)) + }) +} + +test('Excel 超限或转换失败时保留服务端拒绝结果', async () => { + const result = await normalizePreviewResponse( + Response.json({ preview_type: 'unsupported', supported: false, message: '文件过大' }), + { path: '/报表.xlsx' } + ) + assert.equal(result.status, 'unsupported') + assert.equal(result.supported, false) + assert.equal(result.message, '文件过大') + assert.equal(getPreviewTypeByPath('/报表.xlsm'), 'unsupported') +}) + +test('后台预览生成状态保留加载提示,错误终态结束加载', async () => { + const loading = await normalizePreviewResponse(Response.json({ + status: 'loading', preview_type: 'office', supported: true, message: '正在生成预览...' + }), { path: '/报表.xlsx' }) + assert.equal(loading.loading, true) + assert.equal(loading.loadingMessage, '正在生成预览...') + const failed = await normalizePreviewResponse(Response.json({ + status: 'error', preview_type: 'office', supported: false, message: '预览生成失败' + }), { path: '/报表.xlsx' }) + assert.equal(failed.loading, false) + assert.equal(failed.status, 'error') + assert.equal(failed.errorMessage, '预览生成失败') +}) diff --git a/web/test/unit/workspace_preview_polling.test.js b/web/test/unit/workspace_preview_polling.test.js new file mode 100644 index 0000000000..2f27d9fdbe --- /dev/null +++ b/web/test/unit/workspace_preview_polling.test.js @@ -0,0 +1,102 @@ +import { setImmediate } from 'node:timers' +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { runInNewContext } from 'node:vm' +import test from 'node:test' +import { ref } from 'vue' + +const source = readFileSync(new URL('../../src/views/WorkspaceView.vue', import.meta.url), 'utf8') +const helpers = source.slice(source.indexOf('const startPreviewRequest ='), source.indexOf('const loadWorkspacePreview =')) +const polling = source.slice(source.indexOf('const loadKnowledgePreview ='), source.indexOf('const syncSelectedPaths =')) +const closing = source.slice(source.indexOf('const closePreview ='), source.indexOf('const handleSavePreviewFile =')) +const lifecycle = source.slice(source.indexOf('onDeactivated(() =>'), source.indexOf('watch(useInlinePreview')) +const entry = { source: 'knowledge', kb_id: 'kb_1', file_id: 'file_1', path: '/knowledge/file_1' } + +/** 执行真实预览函数和请求隔离逻辑,只替换网络与时间边界。 */ +function setup(responses) { + const timers = [] + const calls = [] + const errors = [] + const hooks = {} + const state = { previewRequestId: ref(0), selectedEntry: ref(null), previewFile: ref(null), + inlinePreviewVisible: ref(false), previewModalVisible: ref(false), useInlinePreview: ref(true), + loadingPreview: ref(false), previewObjectUrl: ref('') } + const context = { ...state, workspaceResizeObserver: null, stopPreviewResize() {}, + onDeactivated: (handler) => { hooks.deactivated = handler }, onUnmounted: (handler) => { hooks.unmounted = handler }, buildPreviewLoadingFile: (item) => ({ ...item, status: 'loading' }), + buildPreviewErrorFile: (item, error) => ({ ...item, status: 'error', message: error.message }), + revokePreviewObjectUrl() {}, window: { URL: { revokeObjectURL() {} }, setTimeout: (callback) => timers.push(callback) }, + message: { error: (value) => errors.push(value) }, console: { warn() {} }, + KNOWLEDGE_PREVIEW_LOAD_MESSAGES: { log: '预览失败', resolveUserMessage: (error) => error.message }, + getWorkspaceKnowledgeFileContent: async (kbId, fileId) => { calls.push([kbId, fileId]); return await responses.shift() }, + normalizePreviewFile: async (item, response) => ({ ...item, ...response }) } + const funcs = runInNewContext(`${helpers}\n${polling}\n${closing}\n${lifecycle}\n;({ loadKnowledgePreview, startPreviewRequest })`, context) + return { ...state, ...funcs, timers, calls, errors, hooks } +} + +async function flush() { + await new Promise((resolve) => setImmediate(resolve)) +} + +test('生成中显示状态,准备好后自动显示当前文件并结束等待', async () => { + const state = setup([{ status: 'loading', message: '正在生成预览' }, { status: 'ready', content: 'pdf-byte-marker' }]) + const finished = state.loadKnowledgePreview(entry) + await flush() + assert.equal(state.previewFile.value.status, 'loading') + assert.equal(state.loadingPreview.value, true) + assert.equal(state.calls.length, 1) + state.timers.shift()() + await finished + assert.equal(state.previewFile.value.content, 'pdf-byte-marker') + assert.equal(state.loadingPreview.value, false) + assert.equal(state.calls.length, 2) + assert.equal(state.timers.length, 0) +}) + +test('切换文件后旧轮询退出且不清除新文件加载状态', async () => { + const state = setup([{ status: 'loading' }]) + const finished = state.loadKnowledgePreview(entry) + await flush() + state.startPreviewRequest({ ...entry, file_id: 'file_2', path: '/knowledge/file_2' }) + state.timers.shift()() + await finished + assert.equal(state.selectedEntry.value.file_id, 'file_2') + assert.equal(state.previewFile.value.file_id, 'file_2') + assert.equal(state.loadingPreview.value, true) + assert.equal(state.calls.length, 1) +}) + +test('任务失败立即呈现错误,不继续轮询', async () => { + const state = setup([{ status: 'error', message: '预览生成失败' }]) + await state.loadKnowledgePreview(entry) + assert.equal(state.previewFile.value.status, 'error') + assert.equal(state.loadingPreview.value, false) + assert.equal(state.calls.length, 1) + assert.equal(state.timers.length, 0) +}) + +test('过期网络响应不能覆盖后选择的文件', async () => { + let complete + const pending = new Promise((resolve) => { complete = resolve }) + const state = setup([pending]) + const finished = state.loadKnowledgePreview(entry) + state.startPreviewRequest({ ...entry, file_id: 'file_2', path: '/knowledge/file_2' }) + complete({ status: 'ready', content: 'old-pdf' }) + await finished + assert.equal(state.previewFile.value.file_id, 'file_2') + assert.equal(state.loadingPreview.value, true) +}) + +for (const event of ['deactivated', 'unmounted']) { + test(`页面 ${event} 后结束旧轮询并释放预览`, async () => { + const state = setup([{ status: 'loading' }]) + const finished = state.loadKnowledgePreview(entry) + await flush() + state.hooks[event]() + state.timers.shift()() + await finished + assert.equal(state.calls.length, 1) + assert.equal(state.loadingPreview.value, false) + assert.equal(state.selectedEntry.value, null) + assert.equal(state.previewFile.value, null) + }) +}