From 85c3a62d9ba822baf5ce5c232800dc2289efa712 Mon Sep 17 00:00:00 2001 From: Hxy <781921880@qq.com> Date: Mon, 31 Aug 2026 10:32:12 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(dataset/tag):=20=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E6=A0=87=E7=AD=BE=E8=BF=87=E6=BB=A4=E5=8A=9F=E8=83=BD?= =?UTF-8?q?V2=20(#7495)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [AI-COMMIT][ADD] 知识库标签过滤功能 - 标签类型扩展：新增 tagType 字段 (string/number/datetime/array)，仅创建时确定，编辑不可修改 - Collection 标签值：tags 字段扩展为混合格式 (string ObjectId | {tagId, value})，支持新旧共存 - 标签管理 API (pro): batchUpsert / setCollectionTags / batchSetCollectionTags，事务保护 - 搜索过滤增强：filterCollectionByKeyValueTags + checkValue，支持 4 类型 × 13 操作符 - Collection 创建链路：createOneCollection 支持混合 tags 入参 - Workflow + Agent V2：datasetSearch 节点新增 tagFilterConfig 渲染，Agent 搜索传递 collectionFilterMatch - UTC 时间工具：toUTCSeconds / fromUTCSeconds - 错误码 + i18n：501013~501023 标签错误码 × 三国语言 - 历史数据迁移，完全向后兼容 Co-Authored-By: Claude Code * [AI-COMMIT][FIX] 知识库标签系统评审修复：数组值校验、写路径去重、唯一索引、ReDoS 防护、default_tag 按 fromMigration 定位 - Issue 1: 数组标签详情响应校验 500 —— 提取共享 CollectionTagValueFieldSchema 复用 - Issue 2: 写路径重复 key —— 共享 deduplicateTagValues，同一 tagId 值冲突拒绝整个操作 - Issue 3: 契约不一致 —— 修正设计文档 §4.2 契约层次与 OpenAPI 描述（不改运行时） - Issue 4: $regex ReDoS —— safe-regex + 分支重叠兜底检测 + pattern/值长度上限 - Issue 5: 并发唯一性 —— v2 表 {teamId,datasetId,tag} unique 索引 + E11000 兜底 + 名称 trim 校验 - Issue 6: 测试覆盖 —— checkValue array 操作符、去重冲突、详情数组、$regex 拦截、真实 Mongo 端到端 benchmark - Issue 7: default_tag 改名后旧格式过滤失效 —— fromMigration 布尔字段定位迁移承载记录 - Issue 8: 解决 openapi/tag.ts 与 main 的冲突（相对 main 仅新增 datasetTag 一行） Co-Authored-By: Claude * [AI-COMMIT][FIX] 标签值共享规范化函数并移除 deleteTagIds 设计流程 - 导出 validateAndNormalizeTagValue，Collection 创建与 pro 写路径共用同一校验+规范化实现 - createOrGetCollectionTags 重构改用该函数（行为不变） - 新增 validateAndNormalizeTagValue 单测 - 设计文档 §5.3 删除 deleteTagIds 批量删除流程（不在本次范围） Co-Authored-By: Claude --------- Co-authored-by: Claude Code --- ...76\350\256\241\346\226\207\346\241\243.md" | 569 ++++++++++++++++++ .gitignore | 1 + packages/global/common/error/code/dataset.ts | 66 +- packages/global/common/string/time.ts | 18 + .../global/core/dataset/collection/utils.ts | 7 +- packages/global/core/dataset/type.ts | 57 +- .../openapi/core/dataset/collection/api.ts | 23 +- .../core/dataset/collection/createApi.ts | 68 ++- .../openapi/core/dataset/collection/tagApi.ts | 69 ++- packages/global/openapi/core/dataset/index.ts | 2 + .../global/openapi/core/dataset/tag/index.ts | 140 +++++ packages/global/openapi/tag.ts | 1 + .../core/dataset/collection/controller.ts | 1 + .../service/core/dataset/collection/schema.ts | 6 +- .../service/core/dataset/collection/utils.ts | 264 ++++++-- .../service/core/dataset/delete/processor.ts | 6 + .../search/defaultRecall/collectionFilter.ts | 424 ++++++++++--- packages/service/core/dataset/tag/schemaV2.ts | 47 ++ .../core/workflow/dispatch/ai/agent/index.ts | 1 + .../dispatch/ai/agent/sub/dataset/index.ts | 1 + .../workflow/dispatch/ai/agent/sub/utils.ts | 1 + packages/service/package.json | 1 + .../core/dataset/delete/processor.test.ts | 109 ++++ .../search/collectionFilter.benchmark.ts | 383 ++++++++++++ .../dataset/search/collectionFilter.test.ts | 544 +++++++++++++++++ .../search/collectionTagIndex.benchmark.ts | 485 +++++++++++++++ .../service/test/core/dataset/utils.test.ts | 527 +++++++++++++++- .../test/integrations/vectorDB/README.md | 19 +- ...ecallFilterPerformance.integration.test.ts | 225 +++++++ packages/service/types/safe-regex.d.ts | 8 + packages/web/i18n/en/common.json | 12 + packages/web/i18n/en/dataset.json | 7 + packages/web/i18n/zh-CN/common.json | 12 + packages/web/i18n/zh-CN/dataset.json | 7 + packages/web/i18n/zh-Hant/common.json | 12 + packages/web/i18n/zh-Hant/dataset.json | 7 + pnpm-lock.yaml | 16 + .../detail/CollectionCard/TagManageModal.tsx | 4 +- .../detail/CollectionCard/TagsPopOver.tsx | 16 +- .../app/src/pages/api/admin/migrateTags.ts | 128 ++++ .../pages/api/core/dataset/collection/list.ts | 5 +- .../api/core/dataset/collection/listV2.ts | 6 +- .../api/core/dataset/collection/scrollList.ts | 19 +- 43 files changed, 4157 insertions(+), 167 deletions(-) create mode 100644 ".agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" create mode 100644 packages/global/openapi/core/dataset/tag/index.ts create mode 100644 packages/service/core/dataset/tag/schemaV2.ts create mode 100644 packages/service/test/core/dataset/search/collectionFilter.benchmark.ts create mode 100644 packages/service/test/core/dataset/search/collectionFilter.test.ts create mode 100644 packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts create mode 100644 packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts create mode 100644 packages/service/types/safe-regex.d.ts create mode 100644 projects/app/src/pages/api/admin/migrateTags.ts diff --git "a/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 000000000000..bb20093fb518 --- /dev/null +++ "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,569 @@ +# 知识库标签系统设计文档 + +## 1. 文档概述 + +### 1.1 设计目标 + +本设计为 FastGPT 知识库提供统一的类型化标签模型、Collection 标签值管理、标签过滤、历史数据迁移和生命周期清理能力。设计以当前代码实现为准，标签定义存储在 `dataset_collection_tags_v2`，Collection 继续复用 `dataset_collections.tags` 字段。 + +系统支持以下标签类型： + +- `string`：字符串值，支持相等、包含、前缀、后缀和正则匹配。 +- `number`：数值比较。 +- `datetime`：按时间戳进行比较。 +- `array`：字符串数组集合比较，支持集合相等、子集、包含和空值判断。 + +历史标签表 `dataset_collection_tags` 作为迁移来源和兼容数据保留，不再作为新标签管理 API 的写入目标。 + +### 1.2 设计范围 + +| 范围 | 内容 | +|---|---| +| 标签定义 | v2 标签表、标签类型、名称唯一性和标签查询 | +| Collection 标签 | 标签值的创建、转换、设置、批量更新和展示转换 | +| 检索过滤 | 按标签名称、类型和值过滤 Collection，并与创建时间、Collection ID 条件求交集 | +| 历史迁移 | 将旧字符串标签转换为 `default_tag` 数组标签 | +| 生命周期 | dataset 物理删除时清理对应 v2 标签定义 | +| API | FastGPT Collection API、FastGPT 标签过滤 API、fastgpt-pro 标签管理 API | +| 本次不包含 | 前端标签 UI 组件重构、自动执行迁移、删除旧表、Simple 应用标签过滤适配 | + +### 1.3 术语 + +| 术语 | 定义 | +|---|---| +| v1 标签表 | `dataset_collection_tags`，历史标签定义表，无 `tagType` | +| v2 标签表 | `dataset_collection_tags_v2`，当前标签定义表，带 `tagType` | +| Collection | 知识库中的文件、链接、文本等集合对象，持久化于 `dataset_collections` | +| 新格式标签 | `{ tagId, value }`，`tagId` 引用 v2 标签表 | +| 旧格式标签 | 字符串形式的历史标签 ID，通常引用 v1 标签表 | +| `default_tag` | `DEFAULT_TAG` 常量值，用于承载旧字符串标签，类型固定为 `array` | +| `collectionFilterMatch` | 知识库检索使用的 Collection 过滤 JSON 字符串 | + +## 2. 模块职责与边界 + +### 2.1 模块职责 + +1. 创建、查询、更新和删除知识库标签定义。 +2. 根据标签类型校验 Collection 标签值，并将创建入参统一转换为可持久化的新格式。 +3. 为知识库检索提供标签条件解析、MongoDB 粗筛和应用层精确比较。 +4. 为历史 Collection 提供按 dataset 维度的管理员迁移入口。 +5. 在 dataset 异步物理删除时清理该 dataset 及其子 dataset 的 v2 标签定义。 +6. 在 Collection 列表接口中将存储格式的 `tagId` 转换为对外展示的标签名称。 + +### 2.2 模块边界 + +| 方向 | 依赖或边界 | +|---|---| +| 上游 | 标签管理 API、Collection 创建 API、工作流知识检索节点、Agent V2 搜索 | +| 下游 | MongoDB、MongoDB secondary read、现有 dataset 权限系统 | +| 跨仓库依赖 | fastgpt-pro 提供标签管理 API，FastGPT 提供公共类型、数据模型、Collection 和搜索逻辑 | +| 不负责 | 标签 UI 交互、向量库内部索引实现、历史迁移自动调度、旧表下线时间规划 | + +## 3. 系统结构 + +### 3.1 子模块划分 + +| 子模块 | 主要职责 | 关键实现 | +|---|---|---| +| 标签模型 | 定义 v2 标签集合、字段和索引 | `packages/service/core/dataset/tag/schemaV2.ts` | +| Collection 模型 | 保存 Collection 标签值并提供标签索引 | `packages/service/core/dataset/collection/schema.ts` | +| 标签转换工具 | 创建/校验/展示转换标签值 | `packages/service/core/dataset/collection/utils.ts` | +| 标签 API | 提供标签定义和 Collection 标签值管理 | fastgpt-pro `projects/app/src/pages/api/core/dataset/tag/` | +| Collection API | 创建、列表和滚动读取 Collection | `projects/app/src/pages/api/core/dataset/collection/` | +| 搜索过滤 | 解析标签条件并返回 Collection ID | `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | +| 迁移 API | 读取 v1 标签并写入 `default_tag` 新格式 | `projects/app/src/pages/api/admin/migrateTags.ts` | +| 删除处理器 | 删除 dataset 相关 v1/v2 标签定义 | `packages/service/core/dataset/delete/processor.ts` | +| 全局类型 | Zod Schema、标签类型、常量和错误码 | `packages/global/core/dataset/type.ts` | + +### 3.2 依赖关系 + +```mermaid +flowchart TD + Client[标签管理端或工作流调用方] + ProAPI[fastgpt-pro 标签 API] + CollectionAPI[FastGPT Collection API] + Search[collectionFilter.ts] + Migration[admin/migrateTags] + Delete[datasetDeleteProcessor] + Utils[Collection 标签转换工具] + V2[(dataset_collection_tags_v2)] + V1[(dataset_collection_tags)] + Collections[(dataset_collections)] + Types[Global Zod 类型与常量] + + Client --> ProAPI + Client --> CollectionAPI + ProAPI --> V2 + ProAPI --> Collections + CollectionAPI --> Utils + Utils --> V2 + CollectionAPI --> Collections + Search --> V2 + Search --> Collections + Migration --> V1 + Migration --> V2 + Migration --> Collections + Delete --> V2 + Delete --> V1 + Delete --> Collections + Types -.-> ProAPI + Types -.-> CollectionAPI + Types -.-> Search +``` + +### 3.3 总体架构影响 + +本设计不改变 FastGPT 的分层架构、权限模型和检索主流程，仅在知识库标签子系统内新增 v2 标签模型和过滤逻辑。Collection 的 `tags` 字段不新增字段，通过元素类型兼容旧格式和新格式；标签管理 API 的外部路由保持原有组织方式。 + +## 4. 对外接口设计 + +### 4.1 标签管理接口 + +标签管理接口位于 fastgpt-pro，基础路径为 `/proApi/core/dataset/tag`。接口通过现有 dataset 或 Collection 权限进行鉴权，内部统一操作 v2 标签表。 + +| 路径 | 方法 | 作用 | 关键请求 | +|---|---|---|---| +| `/create` | POST | 创建标签 | `datasetId`、`tag`、可选 `tagType` | +| `/update` | POST | 修改标签名称 | `datasetId`、`tagId`、`tag` | +| `/delete` | DELETE | 删除标签定义 | dataset 和标签标识 | +| `/list` | POST | 分页查询标签 | dataset、分页参数 | +| `/getAllTags` | GET | 查询全部标签 | dataset | +| `/usage` | GET | 查询标签使用情况 | dataset、标签标识 | +| `/batchUpsert` | POST | 批量维护标签定义 | `datasetId`、`tags: [{ tag, tagType }]` | +| `/setCollectionTags` | POST | 设置单个 Collection 标签值 | `datasetId`、`collectionId`、`tags` | +| `/batchSetCollectionTags` | POST | 批量设置 Collection 标签值 | `collectionIds`、`tags` | +| `/addToCollections` | POST | 按 Collection 差集添加或移除标签 | `tag`、`collectionIds`、`originCollectionIds`、`datasetId` | + +`tagType` 的默认值为 `string`。标签值由 `CollectionTagValueSchema` 校验，存储格式为： + +```json +{ + "tagId": "v2-tag-object-id", + "value": "标签值或数组值" +} +``` + +### 4.2 Collection 标签接口 + +所有 Collection 创建入口均可接收混合格式标签： + +```text +(string | { tag: string, value: string | number | string[] })[] +``` + +字符串元素被视为历史名称格式，由 `createOrGetCollectionTags` 聚合到当前 dataset 的 `default_tag` 数组标签；对象元素按 `tag`（标签名）解析到 v2 标签定义并进行存在性和类型校验。最终持久化时统一使用 `{ tagId, value }` 格式。 + +> **契约层次**：Collection 创建/更新入参为 `(string | { tag, value })[]`（按标签名解析，与 listV2 返回格式一致）；持久化存储为 `{ tagId, value }[]`；fastgpt-pro 的标签值设置接口（`setCollectionTags`/`batchSetCollectionTags`）入参为 `{ tagId, value }[]`（按标签 ID 解析）。三者格式不同属有意设计，使用时勿混用。 + +Collection 列表接口 `listV2` 和兼容接口 `scrollList` 使用 `collectionTagsToTagLabel` 将存储格式转换为： + +```text +(string | { tag: string, value: string | number | string[] })[] +``` + +其中无法在 v2 表中解析的悬空引用不返回给调用方。 + +### 4.3 搜索过滤接口 + +工作流和 Agent V2 通过 `collectionFilterMatch` 传入过滤条件。过滤入口为 `filterCollectionByMetadata`，支持： + +- `tags.$and`：所有条件必须满足。 +- `tags.$or`：至少一个条件满足。 +- `createTime.$gte/$lte`：按 Collection 创建时间过滤。 +- `collectionIds`：按指定 Collection 或文件夹递归展开后过滤。 + +旧格式字符串或 `null` 条件在入口改写为 `default_tag` 的 `$contains` 或 `$empty` 条件，然后统一进入 `filterCollectionByKeyValueTags`。 + +### 4.4 管理员迁移接口 + +`POST /api/admin/migrateTags` 仅允许 root 调用。接口不接受清理孤儿标签的参数，也不删除 v1 表数据或 v2 标签定义。返回值为： + +```json +{ + "migratedDatasets": 0, + "migratedCollections": 0 +} +``` + +## 5. 内部设计 + +### 5.1 标签数据模型 + +v2 标签定义由 `teamId + datasetId + tag` 确定作用域，该三元组有唯一索引保证并发写入下不重复；`tagType` 决定值校验与过滤比较方式。一个 dataset 的 `default_tag` 记录最多作为历史字符串标签的承载定义使用，由 `fromMigration` 布尔字段标识（迁移或旧格式输入创建），过滤时按该字段定位而非标签名，因此即使该记录被改名，旧格式过滤仍可命中。 + +```mermaid +classDiagram + class DatasetCollectionTagV2 { + ObjectId _id + ObjectId teamId + ObjectId datasetId + string tag + string tagType + boolean fromMigration + } + class CollectionTagValue { + string tagId + string_or_number_or_array value + } + class DatasetCollection { + ObjectId _id + ObjectId teamId + ObjectId datasetId + CollectionTagValue[] tags + } + DatasetCollection "1" --> "0..*" CollectionTagValue + CollectionTagValue "*" --> "1" DatasetCollectionTagV2 : tagId +``` + +### 5.2 标签创建与值归一化 + +```mermaid +sequenceDiagram + actor Client as 调用方 + participant API as Collection Create API + participant Utils as createOrGetCollectionTags + participant V2 as v2 标签表 + participant Col as dataset_collections + + Client->>API: 提交 tags + API->>API: Zod 参数校验和 dataset 权限校验 + API->>Utils: 归一化 string/object 混合标签 + alt 存在 string 标签 + Utils->>V2: 查询或创建 default_tag(array) + Utils-->>API: {tagId: defaultTagId, value: string[]} + end + alt 存在对象标签 + Utils->>V2: 查询 tagId 和 tagType + alt 标签不存在或值类型错误 + V2-->>API: tagNotExist/tagValueInvalid + else 校验通过 + Utils-->>API: 保留对象标签值 + end + end + API->>Col: 写入统一的新格式 tags + Col-->>Client: 创建成功 +``` + +处理规则： + +1. 未提供 `tags` 时不写入标签值；空数组保留为空数组。 +2. string 标签名全部合并为一条 `default_tag` 记录，数组值去重。 +3. object 标签必须引用当前 team、dataset 下存在的 v2 标签。 +4. `string`、`number`、`datetime`、`array` 的值类型分别校验： + - `string` 只能为字符串，长度不超过 256。 + - `number` 必须为有限数值；支持输入 number 或可转换为有限数值的字符串，空字符串、纯空白字符串及无法转换的字符串均无效；校验通过后统一按 number 存储。 + - `datetime` 必须为有效的 Unix 时间戳（毫秒）；支持输入 number 或可转换为有限数值的字符串，空字符串、纯空白字符串、无法转换的字符串及超出 JavaScript `Date` 有效范围的数值均无效；日期字符串（如 `2024-01-01`）不作为输入格式，校验通过后统一按 UTC 毫秒时间戳存储。 + - `array` 只能包含字符串，数组长度不超过 64，单元素长度不超过 256。 +5. 多个输入对象引用同一 `tagId` 且 value 不一致时拒绝整个批量操作。 + +### 5.3 标签批量设置 + +`batchSetCollectionTags` 的执行顺序如下： + +1. 校验请求参数及每个 Collection 的写权限。 +2. 对输入标签按 `tagId` 去重并检测冲突。 +3. 对待覆盖的 `tagId` 执行 `$pull`。 +4. 使用 `$push` 将去重后的 `{ tagId, value }` 写入目标 Collection。 + +删除与写入操作均限定在请求的 `teamId`、`datasetId` 和 `collectionIds` 范围内。无匹配文档视为成功；数据库异常向上抛出，由 API 返回服务端错误。 + +`addToCollections` 计算 `collectionIds - originCollectionIds` 得到添加集合，计算反向差集得到移除集合。添加时先 `$pull` 同一 `tagId`，再 `$push` 新值，确保一个 Collection 中同一标签只有一条记录。自动值规则为：`string` 使用空字符串，`array` 使用非空字符串数组；`number` 和 `datetime` 必须使用显式值接口。 + +### 5.4 标签检索过滤 + +```mermaid +sequenceDiagram + participant Workflow as 工作流/Agent + participant Entry as filterCollectionByMetadata + participant Filter as filterCollectionByKeyValueTags + participant TagDB as v2 标签表 + participant ColDB as dataset_collections + + Workflow->>Entry: collectionFilterMatch + Entry->>Entry: json5.parse + Entry->>Entry: 识别 object、string、null 条件 + alt 旧格式 + Entry->>Entry: string -> default_tag.$contains；null -> default_tag.$empty + end + Entry->>Filter: 新格式条件 + Filter->>TagDB: 按 dataset 和标签名批量查 tagId/tagType + loop 每个 dataset + Filter->>ColDB: 按 tags.tagId 粗筛并使用 hint + Filter->>Filter: 按 tagType 执行 checkValue + end + Filter-->>Entry: collectionId 列表 + Entry->>Entry: 与 createTime、collectionIds 求交集 + Entry-->>Workflow: 最终 collectionId 列表 +``` + +过滤实现分为两层： + +- MongoDB 粗筛：按标签 ID 使用 `$all` 或 `$in` 查询，并通过 `.hint({ teamId: 1, datasetId: 1, 'tags.tagId': 1 })` 强制使用新格式标签索引。 +- 应用层精筛：读取候选 Collection 的标签对象，根据 v2 标签的 `tagType` 和操作符调用 `checkValue`。 + +标签不存在、Collection 未配置该标签、值类型不匹配或比较值不可转换时，条件均视为不满足。`$and` 要求全部满足，`$or` 要求至少满足一项；不存在的标签不会被当作满足条件。 + +### 5.5 核心比较算法 + +```text +checkValue(op, target, storedValue, tagType): boolean + 若 op 为 $empty： + array 类型判断值不存在或数组长度为 0；其他类型判断 null、undefined 或空字符串 + 若 op 为 $notEmpty： + array 类型判断为非空数组；其他类型判断为非空值 + target 为 null/undefined 时返回 false + + number/datetime：将存储值和目标值转换为数字，转换失败返回 false，执行 eq/ne/gt/lt/gte/lte + array： + $is -> 两个数组去重后集合相等 + $isNot -> 两个数组去重后集合不相等 + $contains -> 存储数组包含目标字符串 + $notContains -> 存储数组不包含目标字符串 + $in -> 存储数组是目标数组的子集 + $notIn -> 存储数组不是目标数组的子集 + string： + $eq/$ne 使用大小写敏感比较 + $contains/$notContains/$startsWith/$endsWith 忽略大小写 + $regex 使用 RegExp，正则构造失败返回 false +``` + +`$regex` 接受用户可控的 pattern，为防灾难性回溯（ReDoS），pattern 长度上限 64、被测值长度上限 256，并拦截不安全 pattern：嵌套量词类（`(a+)+`）由 `safe-regex` 检出，带分支的量词组（`(a|aa)+`）由首字符重叠检测兜底，超限或不安全均视为不匹配。 + +array 比较的时间复杂度为 `O(n+m)`，其他类型为 `O(1)`；单个 array 值受长度限制，应用层比较不会产生无界内存增长。 + +### 5.6 历史数据迁移 + +迁移按 v1 表中存在标签的 dataset 逐个处理： + +1. root 鉴权。 +2. 清理 v2 表按 `{ teamId, datasetId, tag }` 重复的行（保留 `_id` 最早的一条），避免唯一索引构建失败。 +3. 从 v1 表读取 dataset 的标签定义，建立 `旧 tagId -> 标签名称` 映射。 +4. 查询或创建当前 dataset 的 v2 `default_tag`，类型为 `array` 并置 `fromMigration: true`；存量按名称创建的 `default_tag` 记录回填该标记。 +5. 查找包含旧字符串标签的 Collection；已是纯新格式的 Collection 跳过。 +6. 将旧字符串 ID 转换为标签名称，与已有 `default_tag` 数组值合并并去重。 +7. 保留其他新格式标签，移除旧字符串元素，使用 `$set` 更新 `tags`。 +8. 返回迁移 dataset 数和 Collection 数，并记录 dataset、Collection 和迁移前标签信息日志。 + +迁移只修改 `tags` 字段，不修改 Collection 的 `_id`、`createTime` 或其他元数据，因此既有 `collectionIds` 和时间过滤仍然有效。重复执行时已转换 Collection 不会再次转换。v1 表在迁移过程中只读，v2 孤儿标签定义保留。 + +### 5.7 dataset 物理删除 + +异步 `datasetDeleteProcessor` 找到 root dataset 及全部子 dataset 后，删除 v1 标签、v2 标签及其他 dataset 关联资源。v2 清理条件为： + +```text +{ teamId, datasetId: { $in: rootAndChildDatasetIds } } +``` + +该操作不会删除其他 team 或其他 dataset 的标签定义；无匹配是正常成功。v2 标签清理位于本体 Mongo session 事务之外，任务失败时依赖现有队列重试和日志处理，不宣称跨操作原子性。 + +## 6. 数据库设计 + +### 6.1 `dataset_collection_tags_v2` + +```javascript +const DatasetCollectionTagsV2Schema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + tag: { + type: String, + required: true + }, + tagType: { + type: String, + default: 'string', + enum: ['string', 'number', 'datetime', 'array'] + }, + fromMigration: { + type: Boolean, + default: false + } +}); + +DatasetCollectionTagsV2Schema.index({ teamId: 1, datasetId: 1, tag: 1 }, { unique: true }); +``` + +字段说明： + +| 字段 | 类型 | 约束 | 说明 | +|---|---|---|---| +| `_id` | ObjectId | 主键 | 标签定义 ID | +| `teamId` | ObjectId | 必填 | 所属团队 | +| `datasetId` | ObjectId | 必填 | 所属知识库 | +| `tag` | String | 必填 | 标签名称；`default_tag` 为系统保留名称 | +| `tagType` | String | 枚举 | `string`、`number`、`datetime`、`array`，默认 `string` | +| `fromMigration` | Boolean | 默认 `false` | 标识该记录是迁移或旧格式输入创建的 `default_tag` 承载记录；过滤按该字段定位而非标签名 | + +`{ teamId, datasetId, tag }` 唯一索引保证同一作用域下标签名不重复，并发写入由数据库兜底；业务层先查后写，撞索引时捕获 E11000 复用已存在记录。 + +### 6.2 `dataset_collections.tags` + +Collection 现有 `tags` 字段继续使用 Mixed 数组，不增加独立字段： + +```javascript +tags: { + type: [], + default: [] +} + +DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, tags: 1 }); +DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, 'tags.tagId': 1 }); +``` + +推荐的新格式： + +```json +[ + { "tagId": "v2-string-id", "value": "产品" }, + { "tagId": "v2-number-id", "value": 2 }, + { "tagId": "v2-datetime-id", "value": 1704067200000 }, + { "tagId": "v2-array-id", "value": ["安全", "高优"] } +] +``` + +旧字符串格式仅用于迁移前历史数据。旧索引保留以支持历史数据读取，新格式过滤使用 `tags.tagId` 索引。 + +### 6.3 v1 表 + +`dataset_collection_tags` 保持原有结构和数据，仅用于历史迁移的只读映射以及现有删除链路的兼容清理。新 API 不读写该表作为当前标签定义来源。 + +## 7. 类型、校验与错误处理 + +### 7.1 全局类型 + +核心定义位于 `packages/global/core/dataset/type.ts`： + +```typescript +DatasetCollectionTagType = 'string' | 'number' | 'datetime' | 'array' +CollectionTagValueType = { + tagId: string; + value: string | number | string[]; +} +DEFAULT_TAG = 'default_tag' +``` + +### 7.2 校验规则 + +| 场景 | 处理 | +|---|---| +| 标签名为空 | 拒绝请求，返回标签名称错误 | +| 标签名重复 | 拒绝请求，返回重复错误 | +| v2 标签不存在 | 返回 `tagNotExist` | +| string 值不是字符串或超长 | 返回 `tagValueInvalid` | +| number 无法转换或超出安全范围 | 返回 `tagValueInvalid` | +| datetime 无法转换为有效时间 | 返回 `tagValueDatetimeInvalid` | +| array 非数组、元素非字符串、长度超过 64 或元素超过 256 | 返回 `arrayTagValueInvalid` | +| 过滤 JSON 解析失败 | 捕获异常，当前请求退化为不执行该过滤 | +| MongoDB 查询失败 | 保留异常并由上层统一返回服务端错误 | +| 旧标签 ID 在 v1 表中不存在 | 迁移时忽略该标签，其他可解析标签继续处理 | +| 同一 tagId 出现冲突值 | 拒绝批量写入，不执行覆盖操作 | + +所有读写操作必须带 `teamId` 和 `datasetId` 作用域，API 入口使用现有权限函数，避免跨团队读取或修改标签。 + +## 8. 可靠性、性能与可运维性 + +### 8.1 一致性 + +- v2 标签定义与 Collection 标签值通过 `tagId` 关联，应用层校验引用关系。 +- 写入 Collection 标签采用 `$pull` 后 `$push`，确保同一标签只有一条有效值。 +- dataset 物理删除按完整 root/child dataset ID 集合清理，防止子 dataset 标签残留。 +- 迁移按 dataset 维度可重复执行，已转换 Collection 跳过。 + +### 8.2 降级策略 + +- 过滤条件 JSON 不合法时不阻断知识检索。 +- 未知或缺失 `tagType` 按 `string` 处理。 +- 标签定义不存在时条件不匹配，不扩大查询结果。 +- secondary 读取用于列表和过滤等只读路径，写入仍使用主库。 + +### 8.3 性能设计 + +1. v2 标签查询使用 `{ teamId, datasetId, tag }` 复合索引。 +2. 新格式 Collection 过滤使用 `{ teamId, datasetId, 'tags.tagId' }` 复合索引。 +3. `filterCollectionByKeyValueTags` 先按 tagId 粗筛，再在应用层执行类型比较，减少精确比较数据量。 +4. 同时存在旧标签索引和新标签索引时，使用 MongoDB `hint` 强制选择 `tags.tagId` 索引，避免查询规划器误选旧多键索引。 +5. array 值最多 64 项，单项最多 256 字符，控制比较和内存开销。 +6. 标签列表接口使用分页、字段投影和 secondary read；Collection 列表仅返回所需字段。 + +### 8.4 运维要求 + +- 迁移前备份 `dataset_collections`，确认迁移范围和结果。 +- 迁移接口由运维按需调用，不自动执行，不删除 v1 数据，不清理 v2 孤儿定义。 +- 生产环境提前创建并检查 `tags.tagId` 索引，避免索引创建阻塞业务。 +- 关注迁移日志、dataset 删除任务日志及 MongoDB 查询慢日志。 +- v1 表的最终下线需另行设计，不属于本模块。 + +## 9. 测试设计 + +### 9.1 单元测试 + +| 类别 | 必测内容 | +|---|---| +| 标签值校验 | 四种 tagType 的合法值、非法值、边界值 | +| `checkValue` | 各类型操作符、空值、类型转换失败、正则异常 | +| array 比较 | `$is`、`$isNot`、`$contains`、`$notContains`、`$in`、`$notIn`、`$empty`、`$notEmpty` | +| 标签归一化 | string、object、混合输入和 `default_tag` 合并 | +| 冲突检测 | 相同 tagId 相同值去重、不同值拒绝 | +| 旧格式改写 | string、null、AND、OR、混合 null/string | + +### 9.2 集成测试 + +1. 创建 Collection 后 string 标签被保存为 `default_tag` 新格式。 +2. 创建和更新四种类型标签值均能正确读写。 +3. `listV2`、`scrollList` 返回标签名称而非内部 tagId。 +4. 标签过滤与时间、Collection ID、文件夹递归条件正确求交集。 +5. Agent V2 搜索可以透传 `collectionFilterMatch`。 +6. 迁移保留已有新格式标签，旧标签名称正确合并且重复调用幂等。 +7. 迁移不修改 Collection `_id` 和 `createTime`。 +8. dataset root/child 物理删除只清理目标 team 和目标 dataset 集合的 v2 标签。 +9. 其他 team、其他 dataset 的同名或同 ID 标签不受影响。 +10. MongoDB 查询命中新格式 `tags.tagId` 索引。 + +### 9.3 性能验证 + +在真实或等价 MongoDB 环境验证 30,000 个 Collection 的标签查询： + +- 有 `tags.tagId` 索引时应明显减少扫描文档数。 +- 两个标签索引并存时应确认查询使用 `hint` 指定的新格式索引。 +- array 应用层比较耗时随数组长度线性增长，且受最大长度限制。 + +## 10. 发布与回滚 + +### 10.1 发布顺序 + +1. 发布包含全局类型、v2 Schema、Collection 索引和搜索过滤逻辑的 FastGPT 服务。 +2. 发布 fastgpt-pro 标签管理 API，使标签读写切换到 v2 表。 +3. 检查 MongoDB v2 集合和两个标签索引。 +4. 备份历史数据后，由管理员按 dataset 范围调用迁移接口。 +5. 观察标签 API、检索过滤和迁移日志，再逐步扩大迁移范围。 + +### 10.2 回滚策略 + +- 未迁移数据仍保留 v1 标签和旧 `tags` 值，可回退到旧代码读取。 +- v2 标签定义和新格式 Collection 数据需通过数据库备份恢复；不通过迁移接口反向回滚。 +- 迁移只覆盖 `tags` 字段，不改 Collection 主键和创建时间，便于按备份恢复标签字段。 +- v1 表不在本次发布中删除，回滚期间保留历史读取基础。 + +## 11. 实现文件索引 + +| 文件 | 作用 | +|---|---| +| `packages/global/core/dataset/type.ts` | 标签类型、标签值 Schema、`DEFAULT_TAG` | +| `packages/global/openapi/core/dataset/collection/tagApi.ts` | 标签 API 请求 Schema | +| `packages/service/core/dataset/tag/schemaV2.ts` | v2 标签 MongoDB Schema | +| `packages/service/core/dataset/collection/schema.ts` | Collection Schema 和标签索引 | +| `packages/service/core/dataset/collection/utils.ts` | 标签创建、校验、存储和展示转换 | +| `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | 标签过滤和旧格式改写 | +| `projects/app/src/pages/api/core/dataset/collection/listV2.ts` | Collection 标签过滤列表接口 | +| `projects/app/src/pages/api/core/dataset/collection/scrollList.ts` | 兼容 Collection 列表接口 | +| `projects/app/src/pages/api/admin/migrateTags.ts` | 历史标签迁移接口 | +| `packages/service/core/dataset/delete/processor.ts` | dataset 物理删除和 v2 标签清理 | + +本设计描述的是同一套标签能力的统一实现，不按迭代或变更批次拆分章节。 \ No newline at end of file diff --git a/.gitignore b/.gitignore index f1aa0474d63d..155b596288e0 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ pro/admin/worker/ /pro/llm_benchmark/content_benchmark/eval-runs/ /pro/llm_benchmark/content_benchmark/.cache/ .gstack/ +.worktrees/ diff --git a/packages/global/common/error/code/dataset.ts b/packages/global/common/error/code/dataset.ts index 8b554c8a0d52..3761ddb77569 100644 --- a/packages/global/common/error/code/dataset.ts +++ b/packages/global/common/error/code/dataset.ts @@ -15,7 +15,21 @@ export enum DatasetErrEnum { notSupportSync = 'notSupportSync', sameApiCollection = 'sameApiCollection', noApiServer = 'noApiServer', - canNotEditAdminPermission = 'canNotEditAdminPermission' + canNotEditAdminPermission = 'canNotEditAdminPermission', + + // Tag errors (501013+) + tagNameDuplicate = 'tagNameDuplicate', + tagNameEmpty = 'tagNameEmpty', + tagNotExist = 'tagNotExist', + tagValueInvalid = 'tagValueInvalid', + tagValueStringTooLong = 'tagValueStringTooLong', + tagValueNumberOutOfRange = 'tagValueNumberOutOfRange', + tagValueDatetimeInvalid = 'tagValueDatetimeInvalid', + noDatasetForTagFilter = 'noDatasetForTagFilter', + noTagsInDataset = 'noTagsInDataset', + noPermissionForDatasetTags = 'noPermissionForDatasetTags', + tagNotSelectedForRef = 'tagNotSelectedForRef', + arrayTagValueInvalid = 'arrayTagValueInvalid' } const datasetErr = [ { @@ -69,6 +83,56 @@ const datasetErr = [ { statusText: DatasetErrEnum.noApiServer, message: i18nT('common:core.dataset.error.noApiServer') + }, + + // Tag errors + { + statusText: DatasetErrEnum.tagNameDuplicate, + message: i18nT('common:core.dataset.error.tagNameDuplicate') + }, + { + statusText: DatasetErrEnum.tagNameEmpty, + message: i18nT('common:core.dataset.error.tagNameEmpty') + }, + { + statusText: DatasetErrEnum.tagNotExist, + message: i18nT('common:core.dataset.error.tagNotExist') + }, + { + statusText: DatasetErrEnum.tagValueInvalid, + message: i18nT('common:core.dataset.error.tagValueInvalid') + }, + { + statusText: DatasetErrEnum.tagValueStringTooLong, + message: i18nT('common:core.dataset.error.tagValueStringTooLong') + }, + { + statusText: DatasetErrEnum.tagValueNumberOutOfRange, + message: i18nT('common:core.dataset.error.tagValueNumberOutOfRange') + }, + { + statusText: DatasetErrEnum.tagValueDatetimeInvalid, + message: i18nT('common:core.dataset.error.tagValueDatetimeInvalid') + }, + { + statusText: DatasetErrEnum.noDatasetForTagFilter, + message: i18nT('common:core.dataset.error.noDatasetForTagFilter') + }, + { + statusText: DatasetErrEnum.noTagsInDataset, + message: i18nT('common:core.dataset.error.noTagsInDataset') + }, + { + statusText: DatasetErrEnum.noPermissionForDatasetTags, + message: i18nT('common:core.dataset.error.noPermissionForDatasetTags') + }, + { + statusText: DatasetErrEnum.tagNotSelectedForRef, + message: i18nT('common:core.dataset.error.tagNotSelectedForRef') + }, + { + statusText: DatasetErrEnum.arrayTagValueInvalid, + message: i18nT('common:core.dataset.error.arrayTagValueInvalid') } ]; export default datasetErr.reduce((acc, cur, index) => { diff --git a/packages/global/common/string/time.ts b/packages/global/common/string/time.ts index b14573e9140c..37c67b19b4f7 100644 --- a/packages/global/common/string/time.ts +++ b/packages/global/common/string/time.ts @@ -7,6 +7,24 @@ import { i18nT } from '../i18n/utils'; dayjs.extend(utc); dayjs.extend(timezone); +/** + * 本地时间转 UTC 毫秒时间戳 + * @param date 本地 Date 对象 + * @returns UTC 毫秒时间戳（number） + */ +export const toUTCSeconds = (date: Date): number => { + return dayjs(date).utc().valueOf(); +}; + +/** + * UTC 毫秒时间戳转 Date 对象 + * @param ms UTC 毫秒时间戳 + * @returns Date 对象 + */ +export const fromUTCSeconds = (ms: number): Date => { + return dayjs(ms).utc().toDate(); +}; + export const formatTime2YMDHMW = (time?: Date | number) => dayjs(time).format('YYYY-MM-DD HH:mm:ss dddd'); export const formatTime2YMDHMS = (time?: Date | number) => diff --git a/packages/global/core/dataset/collection/utils.ts b/packages/global/core/dataset/collection/utils.ts index e2ef77d91494..c4ea83db2933 100644 --- a/packages/global/core/dataset/collection/utils.ts +++ b/packages/global/core/dataset/collection/utils.ts @@ -1,7 +1,12 @@ import { DatasetCollectionTypeEnum } from '../constants'; import { type DatasetCollectionSchemaType } from '../type'; -export const getCollectionSourceData = (collection?: DatasetCollectionSchemaType) => { +export const getCollectionSourceData = ( + collection?: Pick< + DatasetCollectionSchemaType, + 'fileId' | 'rawLink' | 'externalFileId' | 'externalFileUrl' | 'apiFileId' | 'name' + > +) => { return { sourceId: collection?.fileId || diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index a50d68172946..6cf8fc41ad10 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -27,6 +27,32 @@ import { ObjectIdSchema } from '../../common/type/mongo'; import { PermissionSchema } from '../../support/permission/controller'; import { NumSchema } from '../../common/zod'; +/* ===== Tag Type ===== */ +/** 标签类型枚举 */ +export const DatasetCollectionTagTypeEnum = z.enum([ + 'string', + 'number', + 'datetime', + 'array' +] as const); +export type DatasetCollectionTagType = z.infer; + +/** 迁移常量：新建 array 标签记录的 tag 字段固定值，亦是旧格式过滤改写的条件 key */ +export const DEFAULT_TAG = 'default_tag'; + +/** Collection 标签值字段：string/number 存对应值，datetime 存 UTC 毫秒时间戳，array 存 string 数组 */ +export const CollectionTagValueFieldSchema = z.union([z.string(), z.number(), z.array(z.string())]); + +/** Collection 标签值类型（新格式） */ +export const CollectionTagValueSchema = z.object({ + tagId: z.string().meta({ description: '引用 dataset_collection_tags_v2._id' }), + value: CollectionTagValueFieldSchema.meta({ + description: + '标签值。string/number 类型存对应值，datetime 类型存 UTC 毫秒时间戳，array 类型存 string 数组' + }) +}); +export type CollectionTagValueType = z.infer; + /* ===== Chunk ===== */ export const ChunkSettingsSchema = z.object({ trainingType: z @@ -134,7 +160,10 @@ export const DatasetCollectionSchema = ChunkSettingsSchema.omit({ parentId: ParentIdSchema.meta({ description: '父级 ID' }), name: z.string().meta({ description: '名称' }), type: z.enum(DatasetCollectionTypeEnum).meta({ description: '集合类型' }), - tags: z.array(z.string()).optional().meta({ description: '标签' }), + tags: z + .array(z.union([z.string(), CollectionTagValueSchema])) + .optional() + .meta({ description: '标签。支持混合格式：String(ObjectId) 旧格式 | { tagId, value } 新格式' }), createTime: z.coerce.date().meta({ description: '创建时间' }), updateTime: z.coerce.date().meta({ description: '更新时间' }), @@ -165,7 +194,13 @@ export const DatasetCollectionTagsSchema = z.object({ _id: ObjectIdSchema.meta({ description: '标签 ID' }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }), datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), - tag: z.string().meta({ description: '标签' }) + tag: z.string().meta({ description: '标签' }), + tagType: DatasetCollectionTagTypeEnum.default('string').meta({ + description: '标签类型：string(默认)/number/datetime/array' + }), + fromMigration: z.boolean().optional().meta({ + description: '标识该记录是旧标签迁移/旧格式输入创建的 default_tag 承载记录' + }) }); export type DatasetCollectionTagsSchemaType = z.infer; @@ -353,7 +388,10 @@ export type DatasetItemType = z.infer; /* ================= tag ===================== */ export const DatasetTagSchema = z.object({ _id: ObjectIdSchema.meta({ description: '标签 ID' }), - tag: z.string().meta({ description: '标签' }) + tag: z.string().meta({ description: '标签' }), + tagType: DatasetCollectionTagTypeEnum.default('string').meta({ + description: '标签类型：string(默认)/number/datetime/array' + }) }); export type DatasetTagType = z.infer; @@ -365,6 +403,19 @@ export type TagUsageType = z.infer; /* ================= collection ===================== */ export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({ + // 详情接口的 tags 由 collectionTagsToTagLabel 解析为标签名格式（string | { tag, value }），区别于存储格式（string | { tagId, value }） + tags: z + .array( + z.union([ + z.string(), + z.object({ + tag: z.string(), + value: CollectionTagValueFieldSchema + }) + ]) + ) + .optional() + .meta({ description: '标签。string 为标签名；新格式为 { tag, value }' }), sourceName: z.string().meta({ description: '来源名称' }), sourceId: z.string().optional().meta({ description: '来源 ID' }), file: z diff --git a/packages/global/openapi/core/dataset/collection/api.ts b/packages/global/openapi/core/dataset/collection/api.ts index 7345d3aac959..290c27136db3 100644 --- a/packages/global/openapi/core/dataset/collection/api.ts +++ b/packages/global/openapi/core/dataset/collection/api.ts @@ -42,7 +42,15 @@ export const UpdateDatasetCollectionBodySchema = z.object({ id: ObjectIdSchema.optional().describe('集合ID，与 datasetId+externalFileId 二选一'), parentId: ParentIdSchema.describe('父级目录ID'), name: z.string().optional().describe('集合名称'), - tags: z.array(z.string()).optional().describe('标签列表（标签名称，非ID）'), + tags: z + .array( + z.union([ + z.string(), + z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) + ]) + ) + .optional() + .describe('标签列表（支持 String 旧格式 或 { tag, value } 新格式）'), forbid: z.boolean().optional().describe('是否禁用'), createTime: z.coerce.date().optional().describe('创建时间'), datasetId: z.string().optional().describe('数据集ID，配合 externalFileId 使用'), @@ -172,7 +180,18 @@ export const DatasetCollectionsListItemSchema = z updateTime: DatasetCollectionSchema.shape.updateTime, forbid: DatasetCollectionSchema.shape.forbid, trainingType: DatasetCollectionSchema.shape.trainingType, - tags: z.array(z.string()).optional().meta({ description: '标签' }), + tags: z + .array( + z.union([ + z.string(), + z.object({ + tag: z.string(), + value: z.union([z.string(), z.number(), z.array(z.string())]) + }) + ]) + ) + .optional() + .meta({ description: '标签。string 为标签名；新格式为 { tag, value }' }), externalFileId: z.string().optional().meta({ description: '外部文件 ID' }), diff --git a/packages/global/openapi/core/dataset/collection/createApi.ts b/packages/global/openapi/core/dataset/collection/createApi.ts index a91175d7a4d8..2088100cd20c 100644 --- a/packages/global/openapi/core/dataset/collection/createApi.ts +++ b/packages/global/openapi/core/dataset/collection/createApi.ts @@ -19,7 +19,18 @@ const DatasetCollectionStoreDataSchema = ChunkSettingsSchema.extend({ // API 创建集合通用基础 Schema export const ApiCreateCollectionBaseSchema = DatasetCollectionStoreDataSchema.extend({ datasetId: z.string().meta({ description: '数据集 ID' }), - tags: z.array(z.string()).optional().meta({ description: '标签列表' }) + tags: z + .array( + z.union([ + z.string(), + z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) + ]) + ) + .optional() + .meta({ + description: + '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' + }) }); export type ApiCreateDatasetCollectionParams = z.infer; @@ -50,7 +61,18 @@ export const CreateCollectionBodySchema = z.object({ type: z .enum([DatasetCollectionTypeEnum.folder, DatasetCollectionTypeEnum.virtual]) .meta({ description: '集合类型（folder: 文件夹，virtual: 手动集合）' }), - tags: z.array(z.string()).optional().meta({ description: '标签列表' }) + tags: z + .array( + z.union([ + z.string(), + z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) + ]) + ) + .optional() + .meta({ + description: + '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' + }) }); export type CreateCollectionBodyType = z.infer; @@ -154,7 +176,19 @@ export type ImageCreateDatasetCollectionParams = z.infer; // handler 内 parse 用 @@ -179,7 +213,19 @@ export const CreateImageCollectionMultipartSchema = z.object({ // handler 内 parse 用 export const CreateBackupCollectionFormSchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), - parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }) + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + tags: z + .array( + z.union([ + z.string(), + z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) + ]) + ) + .optional() + .meta({ + description: + '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' + }) }); export type CreateBackupCollectionFormType = z.infer; @@ -201,7 +247,19 @@ export const CreateBackupCollectionMultipartSchema = z.object({ // handler 内 parse 用 export const CreateTemplateCollectionFormSchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), - parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }) + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + tags: z + .array( + z.union([ + z.string(), + z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) + ]) + ) + .optional() + .meta({ + description: + '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' + }) }); export type CreateTemplateCollectionFormType = z.infer; diff --git a/packages/global/openapi/core/dataset/collection/tagApi.ts b/packages/global/openapi/core/dataset/collection/tagApi.ts index d960826f61e1..cd7afa1acf77 100644 --- a/packages/global/openapi/core/dataset/collection/tagApi.ts +++ b/packages/global/openapi/core/dataset/collection/tagApi.ts @@ -1,4 +1,8 @@ import z from 'zod'; +import { + DatasetCollectionTagTypeEnum, + CollectionTagValueSchema +} from '../../../../core/dataset/type'; /* ============================================================================ * API: 创建集合标签 @@ -6,7 +10,10 @@ import z from 'zod'; * ============================================================================ */ export const CreateDatasetCollectionTagBodySchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), - tag: z.string().meta({ description: '标签名称' }) + tag: z.string().trim().min(1).meta({ description: '标签名称' }), + tagType: DatasetCollectionTagTypeEnum.optional().meta({ + description: '标签类型：string(默认)/number/datetime/array' + }) }); export type CreateDatasetCollectionTagParams = z.infer; @@ -20,7 +27,8 @@ export const AddTagsToCollectionsBodySchema = z.object({ .meta({ description: '来源集合 ID 列表（用于复制标签）' }), collectionIds: z.array(z.string()).meta({ description: '目标集合 ID 列表' }), datasetId: z.string().meta({ description: '数据集 ID' }), - tag: z.string().meta({ description: '标签名称' }) + tag: z.string().trim().meta({ description: '标签名称' }), + value: z.string().optional().meta({ description: '标签值（仅 string 类型标签支持）' }) }); export type AddTagsToCollectionsParams = z.infer; @@ -31,6 +39,61 @@ export type AddTagsToCollectionsParams = z.infer; + +/* ============================================================================ + * API: 删除集合标签 + * Route: DELETE /proApi/core/dataset/tag/delete + * ============================================================================ */ +export const DeleteDatasetCollectionTagQuerySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + id: z.string().meta({ description: '标签 ID' }) +}); +export type DeleteDatasetCollectionTagQuery = z.infer; + +/* ============================================================================ + * API: 获取知识库全部标签 + * Route: GET /proApi/core/dataset/tag/getAllTags + * ============================================================================ */ +export const GetAllDatasetTagsQuerySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }) +}); +export type GetAllDatasetTagsQuery = z.infer; + +/* ============================================================================ + * API: 批量 Upsert 标签 + * Route: POST /proApi/core/dataset/tag/batchUpsert + * ============================================================================ */ +export const BatchUpsertTagItemSchema = z.object({ + tag: z.string().trim().min(1).meta({ description: '标签名称' }), + tagType: DatasetCollectionTagTypeEnum.optional().meta({ description: '标签类型' }) +}); +export const BatchUpsertTagsBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + tags: z.array(BatchUpsertTagItemSchema).min(1).meta({ description: '标签列表' }) +}); +export type BatchUpsertTagsParams = z.infer; + +/* ============================================================================ + * API: 设置单个 Collection 标签值 + * Route: POST /proApi/core/dataset/tag/setCollectionTags + * ============================================================================ */ +export const SetCollectionTagsBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + collectionId: z.string().meta({ description: '集合 ID' }), + tags: z.array(CollectionTagValueSchema).meta({ description: '标签值列表' }) +}); +export type SetCollectionTagsParams = z.infer; + +/* ============================================================================ + * API: 批量设置 Collection 标签值 + * Route: POST /proApi/core/dataset/tag/batchSetCollectionTags + * ============================================================================ */ +export const BatchSetCollectionTagsBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + collectionIds: z.array(z.string()).min(1).meta({ description: '集合 ID 列表' }), + tags: z.array(CollectionTagValueSchema).meta({ description: '标签值列表' }) +}); +export type BatchSetCollectionTagsParams = z.infer; diff --git a/packages/global/openapi/core/dataset/index.ts b/packages/global/openapi/core/dataset/index.ts index 97a8811d4e84..4ae453975bf6 100644 --- a/packages/global/openapi/core/dataset/index.ts +++ b/packages/global/openapi/core/dataset/index.ts @@ -4,6 +4,7 @@ import { SystemOpenApiTagMap } from '../../tag'; import { DatasetDataPath } from './data'; import { DatasetCollectionPath } from './collection'; import { ApiDatasetPath } from './apiDataset'; +import { DatasetTagPath } from './tag'; import { DatasetFilePath } from './file'; import { DatasetTrainingPath } from './training'; import { DatasetSynonymPath } from './synonym'; @@ -336,6 +337,7 @@ export const DatasetPath: OpenAPIPath = { ...DatasetCollectionPath, ...DatasetDataPath, ...ApiDatasetPath, + ...DatasetTagPath, ...DatasetFilePath, ...DatasetTrainingPath, ...DatasetSynonymPath diff --git a/packages/global/openapi/core/dataset/tag/index.ts b/packages/global/openapi/core/dataset/tag/index.ts new file mode 100644 index 000000000000..53324fe466cf --- /dev/null +++ b/packages/global/openapi/core/dataset/tag/index.ts @@ -0,0 +1,140 @@ +import type { OpenAPIPath } from '../../../type'; +import { DevApiTagsMap } from '../../../tag'; +import { + BatchSetCollectionTagsBodySchema, + BatchUpsertTagsBodySchema, + CreateDatasetCollectionTagBodySchema, + DeleteDatasetCollectionTagQuerySchema, + GetAllDatasetTagsQuerySchema, + SetCollectionTagsBodySchema, + UpdateDatasetCollectionTagBodySchema +} from '../collection/tagApi'; + +export const DatasetTagPath: OpenAPIPath = { + '/proApi/core/dataset/tag/create': { + post: { + summary: '创建标签', + description: '在指定知识库下创建一个新标签', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: CreateDatasetCollectionTagBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回新创建的标签信息' + } + } + } + }, + '/proApi/core/dataset/tag/update': { + post: { + summary: '更新标签', + description: '更新指定标签的名称', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: UpdateDatasetCollectionTagBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回更新后的标签信息' + } + } + } + }, + '/proApi/core/dataset/tag/delete': { + delete: { + summary: '删除标签', + description: '根据标签 ID 删除指定知识库下的标签', + tags: [DevApiTagsMap.datasetTag], + requestParams: { + query: DeleteDatasetCollectionTagQuerySchema + }, + responses: { + 200: { + description: '成功删除标签' + } + } + } + }, + '/proApi/core/dataset/tag/getAllTags': { + get: { + summary: '获取全部标签', + description: '获取指定知识库下的全部标签', + tags: [DevApiTagsMap.datasetTag], + requestParams: { + query: GetAllDatasetTagsQuerySchema + }, + responses: { + 200: { + description: '成功返回全部标签列表' + } + } + } + }, + '/proApi/core/dataset/tag/batchUpsert': { + post: { + summary: '批量管理标签', + description: + '全量创建标签。已存在的跳过，不存在的创建，缺少的删除；修改类型需要先调用delete接口', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: BatchUpsertTagsBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回批量操作结果' + } + } + } + }, + '/proApi/core/dataset/tag/setCollectionTags': { + post: { + summary: '设置集合标签值', + description: '为单个集合设置标签值', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: SetCollectionTagsBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回操作结果' + } + } + } + }, + '/proApi/core/dataset/tag/batchSetCollectionTags': { + post: { + summary: '批量设置集合标签值', + description: '为多个集合批量设置标签值', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: BatchSetCollectionTagsBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回批量操作结果' + } + } + } + } +}; diff --git a/packages/global/openapi/tag.ts b/packages/global/openapi/tag.ts index 673c09971892..34a84849d6c2 100644 --- a/packages/global/openapi/tag.ts +++ b/packages/global/openapi/tag.ts @@ -53,6 +53,7 @@ export const DevApiTagsMap = { datasetData: '数据管理', datasetTraining: '训练管理', datasetApiDataset: 'API 数据集管理', + datasetTag: '标签管理', datasetFile: '知识库文件管理', datasetSynonym: '同义词管理', diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index 576327238cee..e9788e5a62a0 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -292,6 +292,7 @@ export async function createOneCollection({ session, ...props }: CreateOneCollec apiFileParentId } = props; + // Resolve tags: string names → ObjectId, {tag, value} → {tagId, value} const collectionTags = await createOrGetCollectionTags({ tags, teamId, diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 40228fe7c20c..1383b49374d0 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -43,7 +43,7 @@ const DatasetCollectionSchema = new Schema({ required: true }, tags: { - type: [String], + type: [], default: [] }, @@ -110,6 +110,10 @@ defineIndex(DatasetCollectionSchema, { defineIndex(DatasetCollectionSchema, { key: { teamId: 1, datasetId: 1, tags: 1 } }); +// New format tags.tagId filter +defineIndex(DatasetCollectionSchema, { + key: { teamId: 1, datasetId: 1, 'tags.tagId': 1 } +}); // create time filter defineIndex(DatasetCollectionSchema, { key: { teamId: 1, datasetId: 1, createTime: 1 } diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 323e546245bf..0853192dd336 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -1,8 +1,14 @@ import { MongoDatasetCollection } from './schema'; import type { ClientSession } from '../../../common/mongo'; -import { MongoDatasetCollectionTags } from '../tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '../tag/schemaV2'; import { readFromSecondary } from '../../../common/mongo/utils'; -import type { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type'; +import { + DEFAULT_TAG, + type CollectionTagValueType, + type CollectionWithDatasetType, + type DatasetCollectionTagType +} from '@fastgpt/global/core/dataset/type'; +import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { DatasetCollectionDataProcessModeEnum, DatasetCollectionSyncResultEnum, @@ -10,7 +16,6 @@ import { DatasetSourceReadTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { readDatasetSourceRawText } from '../read'; import { hashStr } from '@fastgpt/global/common/string/tools'; import { mongoSessionRun } from '../../../common/mongo/sessionRun'; @@ -65,58 +70,242 @@ export function getCollectionUpdateTime({ name, time }: { time?: Date; name: str return new Date(); } +const normalizeDatasetTagValue = ({ + tagType, + value +}: { + tagType: DatasetCollectionTagType; + value: string | number | string[]; +}): { value: string | number | string[]; error?: DatasetErrEnum } => { + if (tagType !== 'number' && tagType !== 'datetime') return { value }; + + const numericValue = + typeof value === 'number' ? value : typeof value === 'string' ? Number(value) : NaN; + if (typeof value === 'string' && value.trim() === '') { + return { value, error: DatasetErrEnum.tagValueInvalid }; + } + if (!Number.isFinite(numericValue)) return { value, error: DatasetErrEnum.tagValueInvalid }; + if (tagType === 'datetime' && Number.isNaN(new Date(numericValue).getTime())) { + return { value, error: DatasetErrEnum.tagValueDatetimeInvalid }; + } + + return { value: numericValue }; +}; + +export const validateDatasetTagValue = ({ + tagType, + value +}: { + tagType?: DatasetCollectionTagType; + value: string | number | string[]; +}): DatasetErrEnum | undefined => { + const type = tagType || 'string'; + + if (type === 'string' && (typeof value !== 'string' || value.length > 256)) { + return DatasetErrEnum.tagValueInvalid; + } + if (type === 'array') { + if ( + !Array.isArray(value) || + value.length > 64 || + value.some((item) => typeof item !== 'string' || item.length > 256) + ) { + return DatasetErrEnum.arrayTagValueInvalid; + } + return undefined; + } + + return normalizeDatasetTagValue({ tagType: type, value }).error; +}; + +/** + * 校验并规范化单个标签值，返回可直接持久化的 value。 + * - number/datetime：统一按 number 存储（字符串转 number、datetime 按 UTC 毫秒时间戳校验） + * - string/array：仅校验，值原样返回 + * Collection 创建路径与 fastgpt-pro 标签值写路径共用，保证两条写链路存储格式一致 + */ +export const validateAndNormalizeTagValue = ({ + tagType, + value +}: { + tagType?: DatasetCollectionTagType; + value: string | number | string[]; +}): { value: string | number | string[]; error?: DatasetErrEnum } => { + if (tagType === 'number' || tagType === 'datetime') { + return normalizeDatasetTagValue({ tagType, value }); + } + return { value, error: validateDatasetTagValue({ tagType, value }) }; +}; + +const isSameTagValue = (a: string | number | string[], b: string | number | string[]): boolean => { + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) return false; + const bSet = new Set(b); + return a.every((item) => bSet.has(item)); + } + return a === b; +}; + +/** + * 同一 tagId 去重：值相同去重，值冲突拒绝整个批量操作。 + * 所有标签值写入路径（Collection 创建/更新、Pro setCollectionTags/batchSetCollectionTags）复用此逻辑 + */ +export const deduplicateTagValues = async ( + tags: CollectionTagValueType[] +): Promise => { + const seen = new Map(); + const deduped: CollectionTagValueType[] = []; + for (const t of tags) { + if (seen.has(t.tagId)) { + if (!isSameTagValue(seen.get(t.tagId)!, t.value)) { + return Promise.reject(DatasetErrEnum.tagValueInvalid); + } + } else { + seen.set(t.tagId, t.value); + deduped.push(t); + } + } + return deduped; +}; + +/** + * 查找或创建 default_tag 承载记录：按 fromMigration 定位（不依赖标签名，改名后仍可复用）， + * 兼容存量按 DEFAULT_TAG 名称创建的记录；并发创建撞唯一索引时复用已存在记录 + */ +async function findOrCreateDefaultTag({ + datasetId, + teamId, + session +}: { + datasetId: string; + teamId: string; + session?: ClientSession; +}) { + const findDefaultTag = () => + MongoDatasetCollectionTagsV2.findOne( + { teamId, datasetId, $or: [{ fromMigration: true }, { tag: DEFAULT_TAG }] }, + undefined, + { session } + ).lean(); + + const existing = await findDefaultTag(); + if (existing) return existing; + + try { + const [created] = await MongoDatasetCollectionTagsV2.create( + [{ teamId, datasetId, tag: DEFAULT_TAG, tagType: 'array', fromMigration: true }], + { session } + ); + return created.toObject ? created.toObject() : created; + } catch (error: any) { + if (error?.code !== 11000) throw error; + const raced = await findDefaultTag(); + if (raced) return raced; + throw error; + } +} + +/** + * 统一解析 collection 创建时的 tags 入参： + * - string 元素（旧格式标签名）→ 归并到 v2 表 default_tag array 标签 + * - {tag, value} 元素 → 查找 v2 表标签并校验值类型，返回 {tagId, value} + * + * 返回值可直接写入 collection.tags 字段存储。 + * 同一 tagId 多条输入：值相同去重，值冲突拒绝整个操作。 + */ export const createOrGetCollectionTags = async ({ tags, datasetId, teamId, session }: { - tags?: string[]; + tags?: (string | { tag: string; value: string | number | string[] })[]; datasetId: string; teamId: string; session?: ClientSession; -}) => { +}): Promise => { if (!tags) return undefined; - if (tags.length === 0) return []; - const existingTags = await MongoDatasetCollectionTags.find( - { - teamId, - datasetId, - tag: { $in: tags } - }, - undefined, - { session } - ).lean(); - - const existingTagContents = existingTags.map((tag) => tag.tag); - const newTagContents = tags.filter((tag) => !existingTagContents.includes(tag)); - - const newTags = await MongoDatasetCollectionTags.insertMany( - newTagContents.map((tagContent) => ({ - teamId, - datasetId, - tag: tagContent - })), - { session, ordered: true } + const stringNames = tags.filter((item): item is string => typeof item === 'string'); + const objectInputs = tags.filter( + (item): item is { tag: string; value: string | number | string[] } => typeof item !== 'string' ); - return [...existingTags.map((tag) => tag._id), ...newTags.map((tag) => tag._id)]; + const trimmedStringNames = stringNames.map((name) => name.trim()); + if (trimmedStringNames.some((name) => !name)) return Promise.reject(DatasetErrEnum.tagNameEmpty); + + const defaultObjectInputs = objectInputs.filter((item) => item.tag.trim() === DEFAULT_TAG); + const regularObjectInputs = objectInputs.filter((item) => item.tag.trim() !== DEFAULT_TAG); + + const regularTagNames = regularObjectInputs.map((item) => item.tag.trim()); + if (regularTagNames.some((name) => !name)) return Promise.reject(DatasetErrEnum.tagNameEmpty); + + const regularTags = regularTagNames.length + ? await MongoDatasetCollectionTagsV2.find( + { teamId, datasetId, tag: { $in: regularTagNames } }, + undefined, + { session } + ).lean() + : []; + const regularTagMap = new Map(regularTags.map((tag) => [tag.tag, tag])); + + const normalizedRegularInputs = regularObjectInputs.map((input) => { + const tagDoc = regularTagMap.get(input.tag.trim()); + if (!tagDoc) { + return { input, value: input.value, error: DatasetErrEnum.tagNotExist }; + } + const tagType = tagDoc.tagType || 'string'; + const { value, error } = validateAndNormalizeTagValue({ tagType, value: input.value }); + return { + tagId: String(tagDoc._id), + value, + error + }; + }); + + for (const { error } of normalizedRegularInputs) { + if (error) return Promise.reject(error); + } + + // default_tag 承载记录：string 名与 tag=default_tag 的对象值合并为单条 array 记录 + const defaultValues: string[] = [...new Set(trimmedStringNames)]; + for (const { value } of defaultObjectInputs) { + const error = validateDatasetTagValue({ tagType: 'array', value }); + if (error) return Promise.reject(error); + if (Array.isArray(value)) defaultValues.push(...value); + } + + const result: CollectionTagValueType[] = []; + + if (defaultValues.length > 0) { + const defaultTag = await findOrCreateDefaultTag({ datasetId, teamId, session }); + result.push({ tagId: String(defaultTag._id), value: [...new Set(defaultValues)] }); + } + + result.push(...normalizedRegularInputs.map(({ tagId, value }) => ({ tagId: tagId!, value }))); + + return deduplicateTagValues(result); }; +/** + * 将 collection 的 tags（混合格式）解析为可重入的输入格式 + * - 旧格式 ObjectId → 标签名（如 "safety"） + * - 新格式 {tagId, value} → {tag: 标签名, value}（如 {tag: "safety", value: "A"}） + * + * 输出结果可作为 createOrGetCollectionTags 的 tags 参数，用于同步、重建等场景 + */ export const collectionTagsToTagLabel = async ({ datasetId, tags }: { datasetId: string; - tags?: string[]; -}) => { + tags?: (string | CollectionTagValueType)[]; +}): Promise<(string | { tag: string; value: string | number | string[] })[] | undefined> => { if (!tags) return undefined; - if (tags.length === 0) return; + if (tags.length === 0) return []; - // Get all the tags - const collectionTags = await MongoDatasetCollectionTags.find({ datasetId }, undefined, { + const collectionTags = await MongoDatasetCollectionTagsV2.find({ datasetId }, undefined, { ...readFromSecondary }).lean(); const tagsMap = new Map(); @@ -126,9 +315,16 @@ export const collectionTagsToTagLabel = async ({ return tags .map((tag) => { - return tagsMap.get(tag) || ''; + if (typeof tag === 'string') { + const tagName = tagsMap.get(tag); + return tagName ?? null; + } + const tagName = tagsMap.get(tag.tagId); + return tagName ? { tag: tagName, value: tag.value } : null; }) - .filter(Boolean); + .filter( + (item): item is string | { tag: string; value: string | number | string[] } => item !== null + ); }; export const syncCollection = async (collection: CollectionWithDatasetType) => { diff --git a/packages/service/core/dataset/delete/processor.ts b/packages/service/core/dataset/delete/processor.ts index a4876781aafb..f1f65d1467be 100644 --- a/packages/service/core/dataset/delete/processor.ts +++ b/packages/service/core/dataset/delete/processor.ts @@ -2,6 +2,7 @@ import type { Processor } from '@fastgpt/dal/redis/bullmq'; import { addDatasetDeleteJob, type DatasetDeleteJobData } from './index'; import { delDatasetRelevantData, findDatasetAndAllChildren } from '../controller'; import { MongoDatasetCollectionTags } from '../tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '../tag/schemaV2'; import { removeDatasetSyncJobScheduler } from '../datasetSync'; import { mongoSessionRun } from '../../../common/mongo/sessionRun'; import { MongoDataset } from '../schema'; @@ -92,6 +93,11 @@ const deleteDatasets = async ({ datasetId: { $in: datasetIds } }); + await MongoDatasetCollectionTagsV2.deleteMany({ + teamId, + datasetId: { $in: datasetIds } + }); + // Delete dataset avatar for await (const dataset of datasets) { await removeImageByPath(dataset.avatar); diff --git a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts index 70c9fa4a8e9a..aef7b095e3b6 100644 --- a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts +++ b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts @@ -1,9 +1,313 @@ import json5 from 'json5'; +import safeRegex from 'safe-regex'; import { MongoDatasetCollection } from '../../collection/schema'; -import { MongoDatasetCollectionTags } from '../../tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '../../tag/schemaV2'; +import { DEFAULT_TAG } from '@fastgpt/global/core/dataset/type'; import { readFromSecondary } from '../../../../common/mongo/utils'; import { computeFilterIntersection } from '../utils'; +/* ========== New format key-value tag filtering types ========== */ + +/** A single key-value tag condition: { tagName: { $op: value } } */ +type TagCondition = Record>; + +/* ========== checkValue: pure value comparsion ========== */ + +// safe-regex 漏检带分支的量词组（如 (a|aa)+ 在 V8 下呈指数回溯），补充首字符重叠检测： +// 逐层展平分组，量词作用域内含分支且分支首字符重叠 → 不安全 +const hasAmbiguousAlternation = (pattern: string): boolean => { + let s = pattern + .replace(/\\./g, ' ') + .replace(/\[[^\]]*\]/g, ' ') + .replace(/\(\?:|\(\?=/g, '('); + const innerRe = /\(([^()]*)\)([+*?]|\{\d+(?:,\d*)?\})?/; + let m: RegExpExecArray | null; + while ((m = innerRe.exec(s))) { + if (m[2] && m[1].includes('|')) { + const firstChars = new Set(); + for (const alt of m[1].split('|')) { + const c = alt.replace(/^[\\^]/, '').charAt(0); + if (firstChars.has(c)) return true; + firstChars.add(c); + } + } + s = s.slice(0, m.index) + 'x' + s.slice(m.index + m[0].length); + } + return false; +}; + +type CompareOp = + | '$eq' + | '$ne' + | '$gt' + | '$lt' + | '$gte' + | '$lte' + | '$contains' + | '$notContains' + | '$startsWith' + | '$endsWith' + | '$regex' + | '$is' + | '$isNot' + | '$in' + | '$notIn' + | '$empty' + | '$notEmpty'; + +export function checkValue( + op: CompareOp, + target: unknown, + storedVal: string | number | string[] | null | undefined, + tagType: string +): boolean { + if (op === '$empty') { + return tagType === 'array' + ? !Array.isArray(storedVal) || storedVal.length === 0 + : storedVal === null || storedVal === undefined || storedVal === ''; + } + if (op === '$notEmpty') { + return tagType === 'array' + ? Array.isArray(storedVal) && storedVal.length > 0 + : storedVal !== null && storedVal !== undefined && storedVal !== ''; + } + + if (target === null || target === undefined) return false; + + switch (tagType) { + case 'number': + case 'datetime': { + const stored = Number(storedVal); + const t = Number(target); + if (isNaN(stored) || isNaN(t)) return false; + switch (op) { + case '$eq': + return stored === t; + case '$ne': + return stored !== t; + case '$gt': + return stored > t; + case '$lt': + return stored < t; + case '$gte': + return stored >= t; + case '$lte': + return stored <= t; + default: + return false; + } + } + case 'array': { + if (!Array.isArray(storedVal)) return false; + const stored = storedVal; + const targetArray = Array.isArray(target) + ? target.filter((item): item is string => typeof item === 'string') + : []; + const equal = (left: string[], right: string[]) => { + const rightSet = new Set(right); + return new Set(left).size === rightSet.size && left.every((item) => rightSet.has(item)); + }; + const subset = (left: string[], right: string[]) => + left.every((item) => right.includes(item)); + switch (op) { + case '$is': + return Array.isArray(target) && equal(stored, targetArray); + case '$isNot': + return Array.isArray(target) && !equal(stored, targetArray); + case '$contains': + return typeof target === 'string' && stored.includes(target); + case '$notContains': + return typeof target === 'string' && !stored.includes(target); + case '$in': + return Array.isArray(target) && subset(stored, targetArray); + case '$notIn': + return Array.isArray(target) && !subset(stored, targetArray); + default: + return false; + } + } + case 'string': + default: { + const stored = String(storedVal ?? ''); + const t = String(target); + switch (op) { + case '$eq': + return stored === t; + case '$ne': + return stored !== t; + case '$contains': + return stored.toLowerCase().includes(t.toLowerCase()); + case '$notContains': + return !stored.toLowerCase().includes(t.toLowerCase()); + case '$startsWith': + return stored.toLowerCase().startsWith(t.toLowerCase()); + case '$endsWith': + return stored.toLowerCase().endsWith(t.toLowerCase()); + case '$regex': + // 用户可控 pattern：限制长度并拦截灾难性回溯，防止 ReDoS + try { + if (t.length > 64 || stored.length > 256) return false; + if (!safeRegex(t) || hasAmbiguousAlternation(t)) return false; + return new RegExp(t).test(stored); + } catch { + return false; + } + default: + return false; + } + } + } +} + +/* ========== filterCollectionByKeyValueTags ========== */ + +/** + * Filter collections by key-value tag conditions (new format). + * + * AND conditions must all be satisfied; OR conditions need at least one match. + * A condition whose tag does not exist in a dataset fails that condition: + * AND → no match; OR → that condition does not count as a match. + */ +export async function filterCollectionByKeyValueTags({ + $and, + $or, + teamId, + datasetIds +}: { + $and: TagCondition[]; + $or: TagCondition[]; + teamId: string; + datasetIds: string[]; +}): Promise { + const allConditions = [...$and, ...$or]; + const tagNames = new Set(); + let hasDefaultTag = false; + for (const cond of allConditions) { + const tagName = Object.keys(cond)[0]; + if (!tagName) continue; + if (tagName === DEFAULT_TAG) hasDefaultTag = true; + else tagNames.add(tagName); + } + if (tagNames.size === 0 && !hasDefaultTag) return undefined; + + // 普通标签按名称查询；default_tag 承载记录按 fromMigration 定位，不依赖标签名（改名后旧格式过滤仍命中） + const [regularTagDocs, defaultTagDocs] = await Promise.all([ + tagNames.size + ? MongoDatasetCollectionTagsV2.find( + { + teamId, + datasetId: { $in: datasetIds }, + tag: { $in: Array.from(tagNames) } + }, + '_id datasetId tag tagType', + { ...readFromSecondary } + ).lean() + : [], + hasDefaultTag + ? MongoDatasetCollectionTagsV2.find( + { teamId, datasetId: { $in: datasetIds }, fromMigration: true }, + '_id datasetId tag tagType', + { ...readFromSecondary } + ).lean() + : [] + ]); + + const datasetTagMap = new Map>(); + const addToMap = (dsId: string, tagName: string, id: string, type: string) => { + if (!datasetTagMap.has(dsId)) datasetTagMap.set(dsId, new Map()); + datasetTagMap.get(dsId)!.set(tagName, { id, type }); + }; + for (const doc of regularTagDocs) { + addToMap(String(doc.datasetId), doc.tag, String(doc._id), doc.tagType || 'string'); + } + // default_tag 记录同时挂 DEFAULT_TAG 键与实际标签名键；每 dataset 的 DEFAULT_TAG 键只取一条避免歧义 + for (const doc of defaultTagDocs) { + const dsId = String(doc.datasetId); + const id = String(doc._id); + const type = doc.tagType || 'string'; + addToMap(dsId, doc.tag, id, type); + if (!datasetTagMap.get(dsId)?.has(DEFAULT_TAG)) { + addToMap(dsId, DEFAULT_TAG, id, type); + } + } + if (datasetTagMap.size === 0) return []; + + // 3. Check a single value condition against one collection's tags. + // Tag missing in the dataset → not satisfied; entry missing in the collection + // → not satisfied. + const matchCondition = ( + cond: TagCondition, + tagMap: Map, + tagsArr: Array<{ tagId: string; value?: string | number | string[] }> + ): boolean => { + const tagName = Object.keys(cond)[0]; + const tagInfo = tagMap.get(tagName); + if (!tagInfo) return false; + const entry = tagsArr.find((t) => t.tagId === tagInfo.id); + if (!entry) return false; + const opObj = cond[tagName] as Record; + const op = Object.keys(opObj)[0]; + return checkValue(op as CompareOp, opObj[op], entry.value, tagInfo.type); + }; + + const allCollectionIds: string[] = []; + + // 4. Iterate each dataset (the same tag name may map to different tagIds per dataset) + for (const [dsId, tagMap] of datasetTagMap) { + const andTagIds = ($and || []) + .map((cond) => tagMap.get(Object.keys(cond)[0])?.id) + .filter((id): id is string => Boolean(id)); + const orTagIds = ($or || []) + .map((cond) => tagMap.get(Object.keys(cond)[0])?.id) + .filter((id): id is string => Boolean(id)); + + if (andTagIds.length === 0 && orTagIds.length === 0) continue; + + // Mongo pre-filter by tagId. AND → $all (more precise); pure OR → $in. + // A single 'tags.tagId' predicate keeps the compound index + // { teamId, datasetId, 'tags.tagId' } usable; exact matching happens below. + const tagIdQuery = + andTagIds.length > 0 + ? { 'tags.tagId': { $all: andTagIds } } + : { 'tags.tagId': { $in: orTagIds } }; + + const collections = await MongoDatasetCollection.find( + { teamId, datasetId: dsId, ...tagIdQuery }, + '_id tags', + { ...readFromSecondary } + ) + // 生产环境同时存在 {teamId,datasetId,tags} 与 {teamId,datasetId,'tags.tagId'} 两个多键索引时， + // 查询规划器可能误选前者（在整段 tags 数组上建索引，无法精准定位 tagId），导致全表 FETCH（实测慢约 8x）。 + // 这里用 hint 强制走 tags.tagId 索引，避免依赖规划器的索引选择。 + .hint({ teamId: 1, datasetId: 1, 'tags.tagId': 1 }) + .lean(); + + // 5. Application-layer value comparison + for (const col of collections) { + const tagsArr = ( + (col.tags || []) as Array<{ tagId?: string; value?: string | number | string[] } | string> + ).filter( + (t): t is { tagId: string; value?: string | number | string[] } => + typeof t === 'object' && t !== null && Boolean(t.tagId) + ); + + // AND: all must pass + const andOk = ($and || []).every((cond) => matchCondition(cond, tagMap, tagsArr)); + if (!andOk) continue; + + // OR: at least one must pass + if ($or?.length) { + const orOk = $or.some((cond) => matchCondition(cond, tagMap, tagsArr)); + if (!orOk) continue; + } + + allCollectionIds.push(String(col._id)); + } + } + + return allCollectionIds.length > 0 ? allCollectionIds : []; +} + export const getForbidCollectionIdList = async ({ teamId, datasetIds @@ -104,103 +408,33 @@ export const filterCollectionByMetadata = async ({ try { const jsonMatch = json5.parse(collectionFilterMatch); - const andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined; - const orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined; - - if (andTags && andTags.length > 0) { - const uniqueAndTags = Array.from(new Set(andTags)); - if (uniqueAndTags.includes(null) && uniqueAndTags.some((tag) => typeof tag === 'string')) { - return []; - } - if (uniqueAndTags.every((tag) => typeof tag === 'string')) { - const matchedTags = await MongoDatasetCollectionTags.find( - { - teamId, - datasetId: { $in: datasetIds }, - tag: { $in: uniqueAndTags as string[] } - }, - '_id datasetId tag', - { ...readFromSecondary } - ).lean(); - - // Group tags by dataset - const datasetTagMap = new Map }>(); - - matchedTags.forEach((tag) => { - const datasetId = String(tag.datasetId); - if (!datasetTagMap.has(datasetId)) { - datasetTagMap.set(datasetId, { - tagIds: [], - tagNames: new Set() - }); - } + const andTagsRaw = jsonMatch?.tags?.$and as unknown[] | undefined; + const orTagsRaw = jsonMatch?.tags?.$or as unknown[] | undefined; - const datasetData = datasetTagMap.get(datasetId)!; - datasetData.tagIds.push(String(tag._id)); - datasetData.tagNames.add(tag.tag); - }); - - const validDatasetIds = Array.from(datasetTagMap.entries()) - .filter(([, data]) => uniqueAndTags.every((tag) => data.tagNames.has(tag as string))) - .map(([datasetId]) => datasetId); - - if (validDatasetIds.length === 0) return []; - - const collectionsPromises = validDatasetIds.map((datasetId) => { - const { tagIds } = datasetTagMap.get(datasetId)!; - return MongoDatasetCollection.find( - { - teamId, - datasetId, - tags: { $all: tagIds } - }, - '_id', - { ...readFromSecondary } - ).lean(); - }); - - const collectionsResults = await Promise.all(collectionsPromises); - tagCollectionIdList = collectionsResults.flat().map((item) => String(item._id)); - } else if (uniqueAndTags.every((tag) => tag === null)) { - const collections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - $or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }] - }, - '_id', - { ...readFromSecondary } - ).lean(); - tagCollectionIdList = collections.map((item) => String(item._id)); - } - } else if (orTags && orTags.length > 0) { - // Get tagId by tag string - const orTagArray = await MongoDatasetCollectionTags.find( - { - teamId, - datasetId: { $in: datasetIds }, - tag: { $in: orTags.filter((tag) => tag !== null) } - }, - '_id', - { ...readFromSecondary } - ).lean(); - const orTagIds = orTagArray.map((item) => String(item._id)); + const isConditionObject = (item: unknown): item is TagCondition => + typeof item === 'object' && !Array.isArray(item) && item !== null; + const rewriteLegacyTags = (items: unknown[] | undefined): TagCondition[] => + (items || []).map((item) => { + if (isConditionObject(item)) return item; + if (item === null) return { [DEFAULT_TAG]: { $empty: true } }; + return { [DEFAULT_TAG]: { $contains: String(item) } }; + }); + const hasLegacyMixedNull = (items: unknown[] | undefined) => + Boolean( + items?.some((item) => item === null) && items.some((item) => typeof item === 'string') + ); - // Get collections by tagId - const collections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - $or: [ - { tags: { $in: orTagIds } }, - ...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : []) - ] - }, - '_id', - { ...readFromSecondary } - ).lean(); + if (hasLegacyMixedNull(andTagsRaw) || hasLegacyMixedNull(orTagsRaw)) return []; - tagCollectionIdList = collections.map((item) => String(item._id)); + const rewrittenAnd = rewriteLegacyTags(andTagsRaw); + const rewrittenOr = rewriteLegacyTags(orTagsRaw); + if (rewrittenAnd.length > 0 || rewrittenOr.length > 0) { + tagCollectionIdList = await filterCollectionByKeyValueTags({ + $and: rewrittenAnd, + $or: rewrittenOr, + teamId, + datasetIds + }); } // time diff --git a/packages/service/core/dataset/tag/schemaV2.ts b/packages/service/core/dataset/tag/schemaV2.ts new file mode 100644 index 000000000000..721ee27bccc9 --- /dev/null +++ b/packages/service/core/dataset/tag/schemaV2.ts @@ -0,0 +1,47 @@ +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { defineIndex, connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; +import { DatasetCollectionName } from '../schema'; +import { type DatasetCollectionTagsSchemaType } from '@fastgpt/global/core/dataset/type'; +const { Schema } = connectionMongo; + +export const DatasetCollectionTagsV2Name = 'dataset_collection_tags_v2'; + +const DatasetCollectionTagsV2Schema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + tag: { + type: String, + required: true + // COMMENT: 标签名称。default_tag 承载记录由 fromMigration 标识，名称可改 + }, + tagType: { + type: String, + default: 'string', + enum: ['string', 'number', 'datetime', 'array'] + // COMMENT: 标签类型。string=字符串比较, number=数值比较, datetime=时间戳比较, array=字符串数组集合比较(V2.0) + }, + fromMigration: { + type: Boolean, + default: false + // COMMENT: 迁移/旧格式输入创建的 default_tag 承载记录；过滤按该字段定位而非标签名 + } +}); + +// unique 索引保证并发写入下同一作用域标签名不重复 +defineIndex(DatasetCollectionTagsV2Schema, { + key: { teamId: 1, datasetId: 1, tag: 1 }, + options: { unique: true } +}); + +export const MongoDatasetCollectionTagsV2 = getMongoModel( + DatasetCollectionTagsV2Name, + DatasetCollectionTagsV2Schema +); diff --git a/packages/service/core/workflow/dispatch/ai/agent/index.ts b/packages/service/core/workflow/dispatch/ai/agent/index.ts index 631961f082f4..1a5baeb660e2 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/index.ts @@ -78,6 +78,7 @@ export type DispatchAgentModuleProps = ModuleDispatchProps<{ /** @deprecated */ [NodeInputKeyEnum.datasetSearchExtensionModel]?: string; [NodeInputKeyEnum.datasetSearchExtensionBg]?: string; + [NodeInputKeyEnum.collectionFilterMatch]?: string; [NodeInputKeyEnum.authTmbId]?: boolean; [NodeInputKeyEnum.useAgentSandbox]?: boolean; [NodeInputKeyEnum.sandboxEntrypoint]?: string; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts index a03f215fbeed..7ae8318c5485 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts @@ -263,6 +263,7 @@ export const dispatchAgentDatasetSearch = async ({ datasetSearchUsingExtensionQuery: datasetParams.datasetSearchUsingExtensionQuery ?? false, datasetSearchExtensionModel: extensionModelData, datasetSearchExtensionBg: datasetParams.datasetSearchExtensionBg, + collectionFilterMatch: datasetParams.collectionFilterMatch, userKey }; const { diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts index 5790270a6a2d..918bb5149e26 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts @@ -136,6 +136,7 @@ export const getAgentDatasetParams = ( datasetSearchExtensionModelId: params[NodeInputKeyEnum.datasetSearchExtensionModelId], datasetSearchExtensionModel: params[NodeInputKeyEnum.datasetSearchExtensionModel], datasetSearchExtensionBg: params[NodeInputKeyEnum.datasetSearchExtensionBg], + collectionFilterMatch: params[NodeInputKeyEnum.collectionFilterMatch], [NodeInputKeyEnum.authTmbId]: params[NodeInputKeyEnum.authTmbId] }; }; diff --git a/packages/service/package.json b/packages/service/package.json index 34a4c16f307b..fa9977becd97 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -71,6 +71,7 @@ "proxy-addr": "catalog:", "proxy-agent": "catalog:", "proxy-from-env": "^1.1.0", + "safe-regex": "^2.1.1", "turndown": "^7.1.2", "undici": "catalog:", "winston": "^3.17.0", diff --git a/packages/service/test/core/dataset/delete/processor.test.ts b/packages/service/test/core/dataset/delete/processor.test.ts index 6c42178bbd82..d08a92907123 100644 --- a/packages/service/test/core/dataset/delete/processor.test.ts +++ b/packages/service/test/core/dataset/delete/processor.test.ts @@ -7,6 +7,8 @@ import { } from '@fastgpt/global/support/permission/constant'; import { datasetDeleteProcessor } from '@fastgpt/service/core/dataset/delete/processor'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema'; import { getUser } from '@test/datas/users'; import { @@ -57,6 +59,13 @@ describe('datasetDeleteProcessor', () => { name: 'retained dataset', type: DatasetTypeEnum.dataset }); + const otherTeamDataset = await MongoDataset.create({ + teamId: otherTeamUser.teamId, + tmbId: otherTeamUser.tmbId, + name: 'other team dataset', + type: DatasetTypeEnum.dataset, + deleteTime + }); const synonym = await MongoDatasetSynonym.create({ teamId: user.teamId, datasetId: childDataset._id, @@ -78,6 +87,50 @@ describe('datasetDeleteProcessor', () => { fingerprint: 'refund' }); + await MongoDatasetCollectionTags.insertOne({ + teamId: user.teamId, + datasetId: childDataset._id, + tag: 'legacy-child-tag' + }); + await MongoDatasetCollectionTagsV2.insertMany([ + { + teamId: user.teamId, + datasetId: rootDataset._id, + tag: 'root-v2-tag', + tagType: 'string' + }, + { + teamId: user.teamId, + datasetId: childDataset._id, + tag: 'child-v2-tag', + tagType: 'string' + }, + { + teamId: user.teamId, + datasetId: retainedDataset._id, + tag: 'retained-v2-tag-1', + tagType: 'string' + }, + { + teamId: user.teamId, + datasetId: retainedDataset._id, + tag: 'retained-v2-tag-2', + tagType: 'string' + }, + { + teamId: otherTeamUser.teamId, + datasetId: rootDataset._id, + tag: 'other-team-same-dataset-id', + tagType: 'string' + }, + { + teamId: otherTeamUser.teamId, + datasetId: otherTeamDataset._id, + tag: 'other-team-dataset-tag', + tagType: 'string' + } + ]); + await MongoResourcePermission.insertMany([ { teamId: user.teamId, @@ -156,6 +209,36 @@ describe('datasetDeleteProcessor', () => { await MongoDataset.countDocuments({ _id: { $in: [rootDataset._id, childDataset._id] } }) ).toBe(0); expect(await MongoDataset.countDocuments({ _id: retainedDataset._id })).toBe(1); + expect( + await MongoDatasetCollectionTags.countDocuments({ + teamId: user.teamId, + datasetId: childDataset._id + }) + ).toBe(0); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: user.teamId, + datasetId: { $in: [rootDataset._id, childDataset._id] } + }) + ).toBe(0); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: user.teamId, + datasetId: retainedDataset._id + }) + ).toBe(2); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: otherTeamUser.teamId, + datasetId: rootDataset._id + }) + ).toBe(1); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: otherTeamUser.teamId, + datasetId: otherTeamDataset._id + }) + ).toBe(1); await expect(MongoDatasetSynonym.countDocuments({ datasetId: childDataset._id })).resolves.toBe( 0 ); @@ -163,4 +246,30 @@ describe('datasetDeleteProcessor', () => { MongoDatasetSynonymMapping.countDocuments({ datasetId: childDataset._id }) ).resolves.toBe(0); }); + + it('deletes a dataset without v2 tags', async () => { + const user = await getUser('dataset-delete-no-v2-tags'); + const dataset = await MongoDataset.create({ + teamId: user.teamId, + tmbId: user.tmbId, + name: 'dataset without v2 tags', + type: DatasetTypeEnum.dataset, + deleteTime: new Date() + }); + + await datasetDeleteProcessor({ + data: { + teamId: user.teamId, + datasetId: String(dataset._id) + } + } as never); + + expect(await MongoDataset.countDocuments({ _id: dataset._id })).toBe(0); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: user.teamId, + datasetId: dataset._id + }) + ).toBe(0); + }); }); diff --git a/packages/service/test/core/dataset/search/collectionFilter.benchmark.ts b/packages/service/test/core/dataset/search/collectionFilter.benchmark.ts new file mode 100644 index 000000000000..524c58ab9058 --- /dev/null +++ b/packages/service/test/core/dataset/search/collectionFilter.benchmark.ts @@ -0,0 +1,383 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockMongoDatasetCollectionFind = vi.hoisted(() => vi.fn()); +const mockMongoDatasetCollectionTagsFind = vi.hoisted(() => vi.fn()); + +vi.mock('@fastgpt/service/core/dataset/collection/schema', () => ({ + MongoDatasetCollection: { + find: mockMongoDatasetCollectionFind + } +})); + +vi.mock('@fastgpt/service/core/dataset/tag/schemaV2', () => ({ + MongoDatasetCollectionTagsV2: { + find: mockMongoDatasetCollectionTagsFind + } +})); + +import { + filterCollectionByKeyValueTags, + filterCollectionByMetadata +} from '../../../../core/dataset/search/defaultRecall/collectionFilter'; + +/** + * 标签过滤性能基准测试 + * + * 1. 新tag横向对比（1000 collections）： + * - 1 个 dataset vs 10 个 dataset + * - 1 个 filter vs 10 个 filter + * 2. 新tag vs 旧tag 纵向对比（1000 collections）： + * - 新格式（key-value，JS 端过滤） vs 旧格式（字符串数组，mongo $all 过滤） + * + * 运行方式：pnpm test:benchmark（vitest.benchmark.config.ts 的 include 命中 test 目录下 + * 所有 *.benchmark.ts 文件） + */ + +const COLLECTION_COUNT = 1000; + +type CollectionItem = { _id: string; tags?: unknown }; + +/* ========== mock 辅助：按查询中的 datasetId 返回对应数据集的集合 ========== */ + +function collectionsForQuery( + query: { datasetId?: string | { $in?: string[] } } | undefined, + collectionsByDataset: Record +): CollectionItem[] { + const ds = query?.datasetId; + if (typeof ds === 'string') return collectionsByDataset[ds] ?? []; + if (ds && Array.isArray(ds.$in)) { + return ds.$in.flatMap((id) => collectionsByDataset[id] ?? []); + } + return []; +} + +function setupMock({ + tagDocs, + collectionsByDataset +}: { + tagDocs: unknown[]; + collectionsByDataset: Record; +}) { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue(tagDocs) + }); + mockMongoDatasetCollectionFind.mockImplementation((query: any) => { + const data = collectionsForQuery(query, collectionsByDataset); + return { + hint: () => ({ lean: vi.fn().mockResolvedValue(data) }), + lean: vi.fn().mockResolvedValue(data) + }; + }); +} + +/* ========== 计时辅助 ========== */ + +interface MeasureResult { + avg: number; + min: number; + max: number; + median: number; + times: number[]; + /** 单次调用的真实 CPU 消耗（user+system，毫秒） */ + cpuPerCallMs: number; + cpuTotalMs: number; + /** 单次调用的堆峰值增长（KB） */ + peakHeapKB: number; + /** 单次调用正堆增长的平均（KB） */ + avgPositiveHeapKB: number; +} + +async function measure( + name: string, + fn: () => unknown | Promise, + iterations = 200, + warmups = 20 +): Promise { + // 预热避开 JIT / 首次模块加载抖动 + for (let i = 0; i < warmups; i++) await fn(); + + const cpuBefore = process.cpuUsage(); + const times: number[] = []; + const heapDeltas: number[] = []; + for (let i = 0; i < iterations; i++) { + const heapBefore = process.memoryUsage().heapUsed; + const start = performance.now(); + await fn(); + times.push(performance.now() - start); + heapDeltas.push(process.memoryUsage().heapUsed - heapBefore); + } + const cpuDelta = process.cpuUsage(cpuBefore); // { user, system } 微秒 + + const avg = times.reduce((a, b) => a + b, 0) / times.length; + const min = Math.min(...times); + const max = Math.max(...times); + const sorted = [...times].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; + + const cpuTotalMs = (cpuDelta.user + cpuDelta.system) / 1000; + const cpuPerCallMs = cpuTotalMs / iterations; + const positiveHeaps = heapDeltas.filter((d) => d > 0); + const peakHeapKB = Math.max(...heapDeltas, 0) / 1024; + const avgPositiveHeapKB = + positiveHeaps.length > 0 + ? positiveHeaps.reduce((a, b) => a + b, 0) / positiveHeaps.length / 1024 + : 0; + + console.log(` [${name}]`); + console.log( + ` 平均耗时: ${avg.toFixed(3)}ms 最小: ${min.toFixed(3)}ms 最大: ${max.toFixed(3)}ms 中位数: ${median.toFixed(3)}ms` + ); + console.log( + ` CPU: 平均 ${cpuPerCallMs.toFixed(3)}ms/次 (user+system，${iterations} 次总计 ${cpuTotalMs.toFixed( + 1 + )}ms)` + ); + console.log( + ` 内存: 单次堆增长峰值 ${peakHeapKB.toFixed(1)}KB | 正增长平均 ${avgPositiveHeapKB.toFixed( + 1 + )}KB (heapUsed，受 GC 影响)` + ); + return { + avg, + min, + max, + median, + times, + cpuPerCallMs, + cpuTotalMs, + peakHeapKB, + avgPositiveHeapKB + }; +} + +/* ========== 数据生成 ========== */ + +// 每个 dataset 定义一个 number 类型的 key-value 标签 +const tagDocsForDataset = (count: number) => + Array.from({ length: count }, (_, i) => ({ + _id: `tag-${i + 1}`, + datasetId: `ds-${i + 1}`, + tag: 'version', + tagType: 'number' + })); + +// 共 COLLECTION_COUNT 个 collection，均分到 count 个 dataset，value 覆盖 0..999 +const collectionsByDatasetFor = (count: number): Record => { + const result: Record = {}; + const perDataset = COLLECTION_COUNT / count; + for (let d = 1; d <= count; d++) { + result[`ds-${d}`] = Array.from({ length: perDataset }, (_, i) => ({ + _id: `col-${d}-${i}`, + tags: [{ tagId: `tag-${d}`, value: (d - 1) * perDataset + i }] + })); + } + return result; +}; + +/* ========== 新tag横向对比 ========== */ + +describe('新tag横向对比（1000 collections）', () => { + it('1 个 dataset vs 10 个 dataset', async () => { + const condition = [{ version: { $gte: 500 } }]; + const datasetIds10 = Array.from({ length: 10 }, (_, i) => `ds-${i + 1}`); + + setupMock({ tagDocs: tagDocsForDataset(1), collectionsByDataset: collectionsByDatasetFor(1) }); + const result1 = await filterCollectionByKeyValueTags({ + $and: condition, + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + expect(result1?.length).toBe(500); + + const r1 = await measure('1 dataset', () => + filterCollectionByKeyValueTags({ + $and: condition, + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }) + ); + expect(r1.avg).toBeLessThan(50); + + setupMock({ + tagDocs: tagDocsForDataset(10), + collectionsByDataset: collectionsByDatasetFor(10) + }); + const result10 = await filterCollectionByKeyValueTags({ + $and: condition, + $or: [], + teamId: 'team-1', + datasetIds: datasetIds10 + }); + expect(result10?.length).toBe(500); + + const r10 = await measure('10 datasets', () => + filterCollectionByKeyValueTags({ + $and: condition, + $or: [], + teamId: 'team-1', + datasetIds: datasetIds10 + }) + ); + expect(r10.avg).toBeLessThan(100); + + console.log( + ` [横向对比] 1 dataset 平均: ${r1.avg.toFixed(3)}ms (CPU ${r1.cpuPerCallMs.toFixed( + 3 + )}ms) | 10 datasets 平均: ${r10.avg.toFixed(3)}ms (CPU ${r10.cpuPerCallMs.toFixed( + 3 + )}ms) | 倍率: ${(r10.avg / r1.avg).toFixed(2)}x` + ); + }); + + it('1 个 filter vs 10 个 filter', async () => { + // 10 个条件全部命中（value 0..999），避免短路提前退出，测满全部条件判断 + const tenConditions = [ + { version: { $gte: 0 } }, + { version: { $lte: 999 } }, + { version: { $gt: -1 } }, + { version: { $lt: 1000 } }, + { version: { $ne: -1 } }, + { version: { $gte: 0 } }, + { version: { $lte: 999 } }, + { version: { $gt: -1 } }, + { version: { $lt: 1000 } }, + { version: { $ne: -1 } } + ]; + + setupMock({ tagDocs: tagDocsForDataset(1), collectionsByDataset: collectionsByDatasetFor(1) }); + + const result1 = await filterCollectionByKeyValueTags({ + $and: [{ version: { $gte: 500 } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + expect(result1?.length).toBe(500); + + const r1 = await measure('1 filter', () => + filterCollectionByKeyValueTags({ + $and: [{ version: { $gte: 500 } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }) + ); + expect(r1.avg).toBeLessThan(50); + + const result10 = await filterCollectionByKeyValueTags({ + $and: tenConditions, + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + expect(result10?.length).toBe(1000); + + const r10 = await measure('10 filters', () => + filterCollectionByKeyValueTags({ + $and: tenConditions, + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }) + ); + expect(r10.avg).toBeLessThan(100); + + console.log( + ` [横向对比] 1 filter 平均: ${r1.avg.toFixed(3)}ms (CPU ${r1.cpuPerCallMs.toFixed( + 3 + )}ms) | 10 filters 平均: ${r10.avg.toFixed(3)}ms (CPU ${r10.cpuPerCallMs.toFixed( + 3 + )}ms) | 倍率: ${(r10.avg / r1.avg).toFixed(2)}x` + ); + }); +}); + +/* ========== 新tag vs 旧tag 纵向对比 ========== */ + +describe('新tag vs 旧tag 纵向对比（100 / 1000 collections）', () => { + beforeEach(() => { + (global as any).feConfigs = { isPlus: true }; + }); + afterEach(() => { + (global as any).feConfigs = {}; + }); + + it('过滤时间对比', async () => { + // 与设计文档对齐：100 / 1000 collections，每个 collection 含 1~3 个 tags + for (const count of [100, COLLECTION_COUNT]) { + const threshold = Math.floor(count / 2); + + // 新格式：key-value 标签，JS 端值过滤（第 1 个 tag 为命中的 version，其余为干扰项） + const newCollections: Record = { + 'ds-1': Array.from({ length: count }, (_, i) => ({ + _id: `col-${i}`, + tags: [ + { tagId: 'tag-version', value: i }, + ...Array.from({ length: i % 3 }, (_, k) => ({ tagId: `extra-${k}`, value: 'x' })) + ] + })) + }; + setupMock({ + tagDocs: [{ _id: 'tag-version', datasetId: 'ds-1', tag: 'version', tagType: 'number' }], + collectionsByDataset: newCollections + }); + const newResult = await filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: JSON.stringify({ + tags: { $and: [{ version: { $gte: threshold } }] } + }) + }); + expect(newResult?.length).toBeGreaterThan(0); + + const rNew = await measure(`新标签 key-value (${count} collections)`, () => + filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: JSON.stringify({ + tags: { $and: [{ version: { $gte: threshold } }] } + }) + }) + ); + + // 旧格式：字符串数组标签，mongo $all 过滤 + const oldCollections: Record = { + 'ds-1': Array.from({ length: count }, (_, i) => ({ + _id: `col-${i}`, + tags: ['tag-1'] + })) + }; + setupMock({ + tagDocs: [{ _id: 'tag-1', datasetId: 'ds-1', tag: 'Tag1' }], + collectionsByDataset: oldCollections + }); + const oldResult = await filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) + }); + expect(oldResult?.length).toBeGreaterThan(0); + + const rOld = await measure(`旧标签 string array (${count} collections)`, () => + filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) + }) + ); + + console.log( + ` [纵向对比 ${count} collections] 新标签平均: ${rNew.avg.toFixed( + 3 + )}ms (CPU ${rNew.cpuPerCallMs.toFixed(3)}ms) | 旧标签平均: ${rOld.avg.toFixed( + 3 + )}ms (CPU ${rOld.cpuPerCallMs.toFixed(3)}ms) | 新/旧倍率: ${(rNew.avg / rOld.avg).toFixed( + 2 + )}x | 新增 JS 过滤成本: ${(rNew.avg - rOld.avg).toFixed(3)}ms` + ); + } + }); +}); diff --git a/packages/service/test/core/dataset/search/collectionFilter.test.ts b/packages/service/test/core/dataset/search/collectionFilter.test.ts new file mode 100644 index 000000000000..dd79c3bf588f --- /dev/null +++ b/packages/service/test/core/dataset/search/collectionFilter.test.ts @@ -0,0 +1,544 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockMongoDatasetCollectionFind = vi.hoisted(() => vi.fn()); +const mockMongoDatasetCollectionTagsFind = vi.hoisted(() => vi.fn()); + +vi.mock('@fastgpt/service/core/dataset/collection/schema', () => ({ + MongoDatasetCollection: { + find: mockMongoDatasetCollectionFind + } +})); + +vi.mock('@fastgpt/service/core/dataset/tag/schemaV2', () => ({ + MongoDatasetCollectionTagsV2: { + find: mockMongoDatasetCollectionTagsFind + } +})); + +import { + checkValue, + filterCollectionByKeyValueTags, + filterCollectionByMetadata +} from '../../../../core/dataset/search/defaultRecall/collectionFilter'; +import { DEFAULT_TAG } from '@fastgpt/global/core/dataset/type'; + +/** + * mock MongoDatasetCollection.find 的链式返回：同时支持 `.hint(...).lean()` 与 `.lean()` + * （filterCollectionByKeyValueTags 用 hint 强制走 tags.tagId 索引） + */ +const mockFind = (data: unknown[]) => { + const chain = { + hint: () => chain, + lean: vi.fn().mockResolvedValue(data) + }; + return chain; +}; + +describe('filterCollectionByKeyValueTags', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); + mockMongoDatasetCollectionFind.mockReturnValue(mockFind([])); + }); + + it('returns undefined when no conditions provided', async () => { + const result = await filterCollectionByKeyValueTags({ + $and: [], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + expect(result).toBeUndefined(); + }); + + it('filters string tags by $eq', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([ + { + _id: 'col-1', + tags: [{ tagId: 'tag-1', value: 'Product A' }] + }, + { + _id: 'col-2', + tags: [{ tagId: 'tag-1', value: 'Product B' }] + } + ]) + ); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ product: { $eq: 'Product A' } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + expect(result).toEqual(['col-1']); + }); + + it('filters number tags by $gte', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'version', tagType: 'number' }]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([ + { _id: 'col-1', tags: [{ tagId: 'tag-1', value: 2 }] }, + { _id: 'col-2', tags: [{ tagId: 'tag-1', value: 1 }] } + ]) + ); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ version: { $gte: 2 } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + expect(result).toEqual(['col-1']); + }); + + it('filters datetime tags by $lt', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'date', tagType: 'datetime' }]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([ + { _id: 'col-1', tags: [{ tagId: 'tag-1', value: 1704067200000 }] }, + { _id: 'col-2', tags: [{ tagId: 'tag-1', value: 1704153600000 }] } + ]) + ); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ date: { $lt: 1704153600000 } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + expect(result).toEqual(['col-1']); + }); + + it('handles AND + OR combination', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { _id: 'tag-a', datasetId: 'ds-1', tag: 'product', tagType: 'string' }, + { _id: 'tag-b', datasetId: 'ds-1', tag: 'version', tagType: 'number' }, + { _id: 'tag-c', datasetId: 'ds-1', tag: 'category', tagType: 'string' } + ]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([ + { + _id: 'col-1', + tags: [ + { tagId: 'tag-a', value: 'Product A' }, + { tagId: 'tag-b', value: 2 }, + { tagId: 'tag-c', value: 'warranty' } + ] + }, + { + _id: 'col-2', + tags: [ + { tagId: 'tag-a', value: 'Product A' }, + { tagId: 'tag-b', value: 2 }, + { tagId: 'tag-c', value: 'manual' } + ] + }, + { + _id: 'col-3', + tags: [ + { tagId: 'tag-a', value: 'Product A' }, + { tagId: 'tag-b', value: 1 } + ] + } + ]) + ); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ product: { $eq: 'Product A' } }, { version: { $gte: 2 } }], + $or: [{ category: { $eq: 'warranty' } }], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + expect(result).toEqual(['col-1']); + }); + + it('returns empty array when AND tag does not exist', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ product: { $eq: 'Product A' } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + expect(result).toEqual([]); + }); + + it('fails when OR tag does not exist', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([{ _id: 'col-1', tags: [{ tagId: 'tag-1', value: 'A' }] }]) + ); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ product: { $eq: 'A' } }], + $or: [{ missing: { $eq: 'x' } }], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + // 该 OR 条件标签不存在 → 条件匹配失败，OR 不通过 + expect(result).toEqual([]); + }); + + it('$empty/$notEmpty only match existing tag entries', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([ + { _id: 'col-1', tags: [{ tagId: 'tag-1', value: 'A' }] }, + { _id: 'col-2', tags: ['old-string-tag'] }, + { _id: 'col-3', tags: [{ tagId: 'tag-1', value: '' }] } + ]) + ); + + const emptyResult = await filterCollectionByKeyValueTags({ + $and: [{ product: { $empty: true } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + // 仅「存在标签条目且值为空」的 col-3 命中；无标签条目的 col-2 不命中 + expect(emptyResult).toEqual(['col-3']); + + const notEmptyResult = await filterCollectionByKeyValueTags({ + $and: [{ product: { $notEmpty: true } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + expect(notEmptyResult).toEqual(['col-1']); + }); + + it('resolves default_tag condition via fromMigration marker, even when renamed', async () => { + // 迁移承载记录已改名（tag 不再是 default_tag），fromMigration 标记仍在 → 旧条件仍命中 + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + _id: 'migrated-id', + datasetId: 'ds-1', + tag: '历史标签', + tagType: 'array', + fromMigration: true + } + ]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([{ _id: 'col-1', tags: [{ tagId: 'migrated-id', value: ['Tag1', 'Tag2'] }] }]) + ); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ [DEFAULT_TAG]: { $contains: 'Tag1' } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + expect(result).toEqual(['col-1']); + }); + + it('returns empty when no fromMigration record exists for default_tag condition', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); + mockMongoDatasetCollectionFind.mockReturnValue(mockFind([])); + + const result = await filterCollectionByKeyValueTags({ + $and: [{ [DEFAULT_TAG]: { $contains: 'Tag1' } }], + $or: [], + teamId: 'team-1', + datasetIds: ['ds-1'] + }); + + expect(result).toEqual([]); + }); +}); + +describe('filterCollectionByMetadata', () => { + beforeEach(() => { + vi.clearAllMocks(); + (global as any).feConfigs = { isPlus: true }; + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); + mockMongoDatasetCollectionFind.mockReturnValue(mockFind([])); + }); + + afterEach(() => { + (global as any).feConfigs = {}; + }); + + it('routes new format to filterCollectionByKeyValueTags', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([{ _id: 'col-1', tags: [{ tagId: 'tag-1', value: 'Product A' }] }]) + ); + + const result = await filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: JSON.stringify({ + tags: { + $and: [{ product: { $eq: 'Product A' } }] + } + }) + }); + + expect(result).toEqual(['col-1']); + }); + + it('rewrites old format to default_tag conditions', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([ + { _id: 'default-tag-1', datasetId: 'ds-1', tag: 'default_tag', tagType: 'array' } + ]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([{ _id: 'col-1', tags: [{ tagId: 'default-tag-1', value: ['Tag1'] }] }]) + ); + + const result = await filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) + }); + + expect(result).toEqual(['col-1']); + }); + + it('matches legacy string filter for renamed default_tag via fromMigration', async () => { + // 迁移承载记录被改名后，旧格式字符串过滤仍按 fromMigration 定位命中 + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([ + { + _id: 'migrated-id', + datasetId: 'ds-1', + tag: '历史标签', + tagType: 'array', + fromMigration: true + } + ]) + }); + mockMongoDatasetCollectionFind.mockReturnValue( + mockFind([{ _id: 'col-1', tags: [{ tagId: 'migrated-id', value: ['Tag1'] }] }]) + ); + + const result = await filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) + }); + + expect(result).toEqual(['col-1']); + }); + + it('returns undefined when collectionFilterMatch is invalid JSON', async () => { + const result = await filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['ds-1'], + collectionFilterMatch: 'not-json{ broken' + }); + + expect(result).toBeUndefined(); + }); +}); + +describe('checkValue', () => { + describe('string type', () => { + it('$eq is case-sensitive', () => { + expect(checkValue('$eq', 'Product A', 'product a', 'string')).toBe(false); + expect(checkValue('$eq', 'Product A', 'Product A', 'string')).toBe(true); + }); + + it('$ne is case-sensitive', () => { + expect(checkValue('$ne', 'Product A', 'product a', 'string')).toBe(true); + expect(checkValue('$ne', 'Product A', 'Product B', 'string')).toBe(true); + expect(checkValue('$ne', 'Product A', 'Product A', 'string')).toBe(false); + }); + + it('$contains checks substring ignoring case', () => { + expect(checkValue('$contains', 'Foo', 'foobar', 'string')).toBe(true); + expect(checkValue('$contains', 'xyz', 'foobar', 'string')).toBe(false); + }); + + it('$notContains negates substring check', () => { + expect(checkValue('$notContains', 'xyz', 'foobar', 'string')).toBe(true); + expect(checkValue('$notContains', 'Foo', 'foobar', 'string')).toBe(false); + }); + + it('$startsWith ignores case', () => { + expect(checkValue('$startsWith', 'foo', 'FOOBAR', 'string')).toBe(true); + expect(checkValue('$startsWith', 'bar', 'foobar', 'string')).toBe(false); + }); + + it('$endsWith ignores case', () => { + expect(checkValue('$endsWith', 'bar', 'FOOBAR', 'string')).toBe(true); + expect(checkValue('$endsWith', 'foo', 'foobar', 'string')).toBe(false); + }); + + it('$regex matches valid patterns', () => { + expect(checkValue('$regex', '^foo', 'foobar', 'string')).toBe(true); + expect(checkValue('$regex', '^bar', 'foobar', 'string')).toBe(false); + }); + + it('$regex returns false for invalid patterns', () => { + expect(checkValue('$regex', '[invalid', 'foobar', 'string')).toBe(false); + }); + + it('$regex rejects catastrophic backtracking patterns', () => { + // 嵌套量词族由 safe-regex 检出，带分支量词组（(a|aa)+）由首字符重叠兜底 + expect(checkValue('$regex', '(a+)+$', 'aaaaab', 'string')).toBe(false); + expect(checkValue('$regex', '(a*)*$', 'aaaaab', 'string')).toBe(false); + expect(checkValue('$regex', '(a|aa)+$', 'aaaaab', 'string')).toBe(false); + }); + + it('$regex accepts benign patterns', () => { + expect(checkValue('$regex', 'foo', 'foobar', 'string')).toBe(true); + expect(checkValue('$regex', '^foo', 'foobar', 'string')).toBe(true); + expect(checkValue('$regex', '\\d+', 'abc123', 'string')).toBe(true); + expect(checkValue('$regex', '(ab)+', 'ababab', 'string')).toBe(true); + }); + + it('$regex returns false for overlong pattern or stored value', () => { + expect(checkValue('$regex', 'a'.repeat(65), 'aaaaa', 'string')).toBe(false); + expect(checkValue('$regex', 'a', 'x'.repeat(257), 'string')).toBe(false); + }); + + it('returns false when stored value is null/undefined/empty', () => { + expect(checkValue('$eq', 'x', null, 'string')).toBe(false); + expect(checkValue('$contains', 'x', undefined, 'string')).toBe(false); + expect(checkValue('$startsWith', 'x', '', 'string')).toBe(false); + }); + }); + + describe('number type', () => { + it('compares with $eq/$ne/$gt/$lt/$gte/$lte', () => { + expect(checkValue('$eq', 2, 2, 'number')).toBe(true); + expect(checkValue('$ne', 2, 3, 'number')).toBe(true); + expect(checkValue('$gt', 1, 2, 'number')).toBe(true); + expect(checkValue('$lt', 3, 2, 'number')).toBe(true); + expect(checkValue('$gte', 2, 2, 'number')).toBe(true); + expect(checkValue('$lte', 2, 2, 'number')).toBe(true); + }); + + it('returns false for NaN stored or target', () => { + expect(checkValue('$eq', 2, NaN, 'number')).toBe(false); + expect(checkValue('$eq', NaN, 2, 'number')).toBe(false); + }); + + it('coerces string numbers', () => { + expect(checkValue('$eq', '2', 2, 'number')).toBe(true); + expect(checkValue('$eq', 2, '2', 'number')).toBe(true); + }); + }); + + describe('datetime type', () => { + it('compares unix millisecond timestamps', () => { + expect(checkValue('$eq', 1704067200000, 1704067200000, 'datetime')).toBe(true); + expect(checkValue('$gt', 1704067200000, 1704153600000, 'datetime')).toBe(true); + expect(checkValue('$lt', 1704153600000, 1704067200000, 'datetime')).toBe(true); + }); + + it('returns false for NaN stored or target', () => { + expect(checkValue('$eq', 1704067200000, NaN, 'datetime')).toBe(false); + expect(checkValue('$eq', NaN, 1704067200000, 'datetime')).toBe(false); + }); + }); + + describe('array type', () => { + it('$is/$isNot compare arrays as sets (order-insensitive)', () => { + expect(checkValue('$is', ['a', 'b'], ['b', 'a'], 'array')).toBe(true); + expect(checkValue('$is', ['a', 'b'], ['a', 'c'], 'array')).toBe(false); + expect(checkValue('$isNot', ['a', 'b'], ['a', 'c'], 'array')).toBe(true); + expect(checkValue('$isNot', ['a', 'b'], ['b', 'a'], 'array')).toBe(false); + }); + + it('$contains/$notContains check single-string membership', () => { + expect(checkValue('$contains', 'a', ['a', 'b'], 'array')).toBe(true); + expect(checkValue('$contains', 'c', ['a', 'b'], 'array')).toBe(false); + expect(checkValue('$notContains', 'c', ['a', 'b'], 'array')).toBe(true); + expect(checkValue('$notContains', 'a', ['a', 'b'], 'array')).toBe(false); + }); + + it('$in/$notIn check subset', () => { + expect(checkValue('$in', ['a', 'b', 'c'], ['a', 'b'], 'array')).toBe(true); + expect(checkValue('$in', ['a', 'c'], ['a', 'b'], 'array')).toBe(false); + expect(checkValue('$notIn', ['a', 'c'], ['a', 'b'], 'array')).toBe(true); + expect(checkValue('$notIn', ['a', 'b', 'c'], ['a', 'b'], 'array')).toBe(false); + }); + + it('$empty/$notEmpty for arrays', () => { + expect(checkValue('$empty', true, [], 'array')).toBe(true); + expect(checkValue('$empty', true, undefined, 'array')).toBe(true); + expect(checkValue('$empty', true, ['a'], 'array')).toBe(false); + expect(checkValue('$notEmpty', true, ['a'], 'array')).toBe(true); + expect(checkValue('$notEmpty', true, [], 'array')).toBe(false); + }); + + it('returns false when stored value is not an array or target is not an array', () => { + expect(checkValue('$is', ['a'], 'not-array', 'array')).toBe(false); + expect(checkValue('$contains', 'a', 'not-array', 'array')).toBe(false); + expect(checkValue('$in', 'a', ['a'], 'array')).toBe(false); + }); + }); + + describe('empty operators', () => { + it('$empty treats null/undefined/empty string as empty', () => { + expect(checkValue('$empty', true, null, 'string')).toBe(true); + expect(checkValue('$empty', true, undefined, 'string')).toBe(true); + expect(checkValue('$empty', true, '', 'string')).toBe(true); + expect(checkValue('$empty', true, 'x', 'string')).toBe(false); + expect(checkValue('$empty', true, 0, 'number')).toBe(false); + }); + + it('$notEmpty reverses empty logic', () => { + expect(checkValue('$notEmpty', true, 'x', 'string')).toBe(true); + expect(checkValue('$notEmpty', true, null, 'string')).toBe(false); + expect(checkValue('$notEmpty', true, undefined, 'string')).toBe(false); + expect(checkValue('$notEmpty', true, '', 'string')).toBe(false); + }); + }); + + it('returns false for unsupported operator or null target', () => { + expect(checkValue('$unsupported' as any, 'x', 'x', 'string')).toBe(false); + expect(checkValue('$eq', null, 'x', 'string')).toBe(false); + }); +}); diff --git a/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts b/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts new file mode 100644 index 000000000000..be7b78582e23 --- /dev/null +++ b/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts @@ -0,0 +1,485 @@ +import { describe, expect, it } from 'vitest'; +import { getRootUser } from '@test/datas/users'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { filterCollectionByKeyValueTags } from '@fastgpt/service/core/dataset/search/defaultRecall/collectionFilter'; +import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; + +/** + * dataset_collections 两个标签索引性能对比（真实 MongoDB） + * + * 对比索引： + * 1. { teamId: 1, datasetId: 1, tags: 1 } —— 旧格式 string[] 标签，查询 `tags: { $all }` + * 2. { teamId: 1, datasetId: 1, 'tags.tagId': 1 } —— 新格式 { tagId, value }[] 标签，查询 `tags.tagId: { $all }` + * + * 对每个索引分别测「有索引」与「无索引」两个场景： + * - explain('executionStats')：断言有索引时使用被测索引（IXSCAN）且扫描量远小于全表； + * 索引缺失时扫描量接近全表（优化器退到其他复合索引前缀或 COLLSCAN） + * - 服务端执行时间：多次 explain 采样 executionTimeMillis，取平均与中位（避免单次噪声） + * - 墙钟延迟：预热 + 多次迭代取中位数，对比索引带来的加速 + * + * 运行方式： + * pnpm test:benchmark -- collectionTagIndex.benchmark.ts + * 使用真实 MongoDB：FASTGPT_TEST_MONGODB_URI=mongodb://localhost:27017 \ + * pnpm test:benchmark -- collectionTagIndex.benchmark.ts + * + * 注意：test/setup.ts 会在每个用例结束后清空所有集合文档（保留索引）， + * 因此数据准备必须在单个 it 内完成，不能跨用例复用 beforeAll 数据。 + */ + +const COLLECTION_COUNT = 30_000; +const TAG_COUNT = 10; +/** 命中 5% 的 collection 携带查询目标 tags（index % 20 === 0） */ +const HIT_EVERY = 20; +const INSERT_BATCH = 5_000; +const MEASURE_ITERATIONS = 20; +const MEASURE_WARMUPS = 5; + +/** 目标两个复合索引（用 key spec 控制创建/删除，避免依赖自动生成的索引名） */ +const TAGS_INDEX_KEY: Record = { teamId: 1, datasetId: 1, tags: 1 }; +const TAGS_TAGID_INDEX_KEY: Record = { teamId: 1, datasetId: 1, 'tags.tagId': 1 }; +/** MongoDB 自动生成的索引名（key 字段名_方向 拼接） */ +const TAGS_INDEX_NAME = 'teamId_1_datasetId_1_tags_1'; +const TAGS_TAGID_INDEX_NAME = 'teamId_1_datasetId_1_tags.tagId_1'; + +type RootUser = Awaited>; + +interface MeasureResult { + avg: number; + min: number; + max: number; + median: number; + times: number[]; +} + +/* ========== 计时辅助 ========== */ + +async function measure( + name: string, + fn: () => Promise, + iterations = MEASURE_ITERATIONS, + warmups = MEASURE_WARMUPS +): Promise { + // 预热避开 JIT / 首次查询缓存抖动 + for (let i = 0; i < warmups; i++) await fn(); + + const times: number[] = []; + for (let i = 0; i < iterations; i++) { + const start = performance.now(); + await fn(); + times.push(performance.now() - start); + } + + const avg = times.reduce((a, b) => a + b, 0) / times.length; + const min = Math.min(...times); + const max = Math.max(...times); + const sorted = [...times].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; + + console.log( + ` ${name.padEnd(26)} 平均: ${avg.toFixed(2).padStart(8)}ms 中位: ${median + .toFixed(2) + .padStart(8)}ms 最小: ${min.toFixed(2).padStart(8)}ms 最大: ${max.toFixed(2).padStart(8)}ms` + ); + return { avg, min, max, median, times }; +} + +/* ========== explain 辅助 ========== */ + +interface ExplainStats { + stages: string[]; + /** 若走 IXSCAN，命中的索引名 */ + indexName: string; + totalDocsExamined: number; + totalKeysExamined: number; + executionTimeMillis: number; + nReturned: number; +} + +/** 递归收集 stage 树上的所有 stage 名（不同 MongoDB 版本嵌套深度不同） */ +function collectStages(stage: any): string[] { + if (!stage) return []; + const result = [stage.stage]; + if (stage.inputStage) result.push(...collectStages(stage.inputStage)); + if (Array.isArray(stage.inputStages)) { + for (const s of stage.inputStages) result.push(...collectStages(s)); + } + return result; +} + +/** 在 stage 树中查找指定名称的 stage */ +function findStage(stage: any, name: string): any | undefined { + if (!stage) return undefined; + if (stage.stage === name) return stage; + if (stage.inputStage) { + const found = findStage(stage.inputStage, name); + if (found) return found; + } + if (Array.isArray(stage.inputStages)) { + for (const sub of stage.inputStages) { + const found = findStage(sub, name); + if (found) return found; + } + } + return undefined; +} + +interface ExplainTimeResult { + avg: number; + median: number; +} + +/** 多次 explain 采样服务端 executionTimeMillis，取平均与中位（避免单次测量噪声） */ +async function measureExplainTime( + query: Record, + iterations = 10, + warmups = 3 +): Promise { + for (let i = 0; i < warmups; i++) { + await MongoDatasetCollection.find(query, '_id').explain('executionStats'); + } + + const times: number[] = []; + for (let i = 0; i < iterations; i++) { + const raw = (await MongoDatasetCollection.find(query, '_id') + .explain('executionStats') + .then((res: any) => res[0] ?? res)) as any; + times.push(raw?.executionStats?.executionTimeMillis ?? 0); + } + + const avg = times.reduce((a, b) => a + b, 0) / times.length; + const sorted = [...times].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; + return { avg, median }; +} + +async function explainQuery( + query: Record, + hint?: Record +): Promise { + let findQuery = MongoDatasetCollection.find(query, '_id'); + if (hint) findQuery = findQuery.hint(hint); + const raw = (await findQuery.explain('executionStats').then((res: any) => res[0] ?? res)) as any; + // debug: 打印完整 explain 结果（executionStages 树可能较大） + // console.log( + // `explain raw: query=${JSON.stringify(query)}, executionStats=${JSON.stringify(raw)}` + // ); + + const winningPlan = raw?.queryPlanner?.winningPlan; + const execStats = raw?.executionStats; + const ixscan = findStage(winningPlan, 'IXSCAN'); + + // executionStats 顶层字段优先，回退到 executionStages 树上的同名字段 + const totalDocsExamined = + execStats?.totalDocsExamined ?? execStats?.executionStages?.totalDocsExamined ?? 0; + const totalKeysExamined = + execStats?.totalKeysExamined ?? execStats?.executionStages?.totalKeysExamined ?? 0; + const executionTimeMillis = execStats?.executionTimeMillis ?? 0; + const nReturned = execStats?.nReturned ?? execStats?.executionStages?.nReturned ?? 0; + + return { + stages: collectStages(winningPlan), + indexName: ixscan?.indexName ?? '', + totalDocsExamined, + totalKeysExamined, + executionTimeMillis, + nReturned + }; +} + +/* ========== 数据准备 ========== */ + +function buildOldTags(index: number, isHit: boolean, tagIds: string[]): string[] { + if (isHit) { + // 命中：同时携带 tagIds[0] 与 tagIds[1]，再加一个干扰 tag + return [tagIds[0], tagIds[1], tagIds[(index + 2) % TAG_COUNT]]; + } + // 非命中：只携带一个 tag，单个元素永远不满足 $all [tagIds[0], tagIds[1]] + return [tagIds[(index + 1) % TAG_COUNT]]; +} + +function buildNewTags( + index: number, + isHit: boolean, + tagIds: string[] +): Array<{ tagId: string; value: string }> { + if (isHit) { + return [ + { tagId: tagIds[0], value: 'A' }, + { tagId: tagIds[1], value: 'B' }, + { tagId: tagIds[(index + 2) % TAG_COUNT], value: 'noise' } + ]; + } + return [{ tagId: tagIds[(index + 1) % TAG_COUNT], value: 'noise' }]; +} + +async function seedTagIndexData({ root, format }: { root: RootUser; format: 'old' | 'new' }) { + const suffix = `${format}-${Date.now()}`; + const dataset = await MongoDataset.create({ + teamId: root.teamId, + tmbId: root.tmbId, + name: `tag-index-${suffix}`, + type: DatasetTypeEnum.dataset, + vectorModel: 'text-embedding-3-small', + agentModel: 'gpt-4o-mini' + }); + + const datasetId = String(dataset._id); + const tagIds = Array.from({ length: TAG_COUNT }, () => new Types.ObjectId().toString()); + + // 分批插入，避免堆尖峰 + for (let start = 0; start < COLLECTION_COUNT; start += INSERT_BATCH) { + const batchSize = Math.min(INSERT_BATCH, COLLECTION_COUNT - start); + const docs = Array.from({ length: batchSize }, (_, k) => { + const i = start + k; + const isHit = i % HIT_EVERY === 0; + return { + teamId: root.teamId, + tmbId: root.tmbId, + datasetId, + type: DatasetCollectionTypeEnum.file, + name: `col-${suffix}-${i}`, + tags: format === 'old' ? buildOldTags(i, isHit, tagIds) : buildNewTags(i, isHit, tagIds) + }; + }); + await MongoDatasetCollection.insertMany(docs); + } + + return { datasetId, tagIds }; +} + +/** 确保两个标签索引都存在（createIndex 幂等；显式指定 name 保证与删除/断言一致） */ +async function ensureTagIndexes() { + await MongoDatasetCollection.collection.createIndex(TAGS_INDEX_KEY, { name: TAGS_INDEX_NAME }); + await MongoDatasetCollection.collection.createIndex(TAGS_TAGID_INDEX_KEY, { + name: TAGS_TAGID_INDEX_NAME + }); +} + +/* ========== 断言场景 ========== */ + +async function assertIndexed( + query: Record, + expectedIndexName: string, + label: string +) { + const exp = await explainQuery(query); + // 必须使用被测标签索引，且索引扫描的文档数远小于全表 + expect(exp.stages).toContain('IXSCAN'); + expect(exp.indexName).toBe(expectedIndexName); + expect(exp.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); + // 索引场景的服务端执行时间：多次 explain 取中位，避免单次噪声（内存 mongo / CI 波动大，取宽松值） + const time = await measureExplainTime(query); + expect(time.median).toBeLessThan(300); + console.log( + ` [${label} 有索引 explain] stage=${exp.stages.join('->')} index=${exp.indexName} examined=${exp.totalDocsExamined} keys=${exp.totalKeysExamined} returned=${exp.nReturned} time(avg/median)=${time.avg.toFixed(1)}/${time.median.toFixed(1)}ms` + ); + return { ...exp, time }; +} + +async function assertUnindexed(query: Record, label: string) { + const exp = await explainQuery(query); + // 标签索引被删除后，优化器只能退到其他复合索引的前缀（teamId+datasetId）扫描全部文档， + // 因此扫描量接近全表（此处可能表现为 IXSCAN on createTime/fileId 索引或 COLLSCAN， + // 不强制 COLLSCAN） + expect(exp.totalDocsExamined).toBeGreaterThanOrEqual(COLLECTION_COUNT * 0.9); + expect(exp.indexName).not.toMatch(/tags/i); + const time = await measureExplainTime(query); + console.log( + ` [${label} 无索引 explain] stage=${exp.stages.join('->')} index=${exp.indexName || 'N/A'} examined=${exp.totalDocsExamined} keys=${exp.totalKeysExamined} returned=${exp.nReturned} time(avg/median)=${time.avg.toFixed(1)}/${time.median.toFixed(1)}ms` + ); + return { ...exp, time }; +} + +/* ========== 测试 ========== */ + +describe('dataset_collections 标签索引性能对比（真实 MongoDB）', () => { + it('旧格式：tags 复合索引 vs 无索引', async () => { + const root = await getRootUser(); + const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'old' }); + const query = { + teamId: root.teamId, + datasetId, + tags: { $all: [tagIds[0], tagIds[1]] } + }; + const runQuery = () => MongoDatasetCollection.find(query, '_id').lean(); + + try { + // 有索引：只保留 tags 复合索引，避免优化器误选 tags.tagId 索引（后者会导致 FETCH 全表） + await ensureTagIndexes(); + await MongoDatasetCollection.collection.dropIndex(TAGS_TAGID_INDEX_NAME); + const expIndexed = await assertIndexed(query, TAGS_INDEX_NAME, '旧格式 tags'); + const mIndexed = await measure('旧格式 + tags 索引', runQuery); + + // 无索引：删掉 tags 复合索引 + await MongoDatasetCollection.collection.dropIndex(TAGS_INDEX_NAME); + const expUnindexed = await assertUnindexed(query, '旧格式 tags'); + const mUnindexed = await measure('旧格式 + 无索引', runQuery); + + // 断言：无索引扫描量/耗时明显大于有索引 + expect(expUnindexed.totalDocsExamined).toBeGreaterThan(expIndexed.totalDocsExamined * 5); + expect(expUnindexed.time.median).toBeGreaterThan(expIndexed.time.median); + expect(mUnindexed.median).toBeGreaterThan(mIndexed.median); + + console.log( + ` [旧格式 tags] examined 倍率: ${(expUnindexed.totalDocsExamined / expIndexed.totalDocsExamined).toFixed(1)}x 执行时间倍率(median): ${(expUnindexed.time.median / expIndexed.time.median).toFixed(1)}x 中位延迟倍率: ${(mUnindexed.median / mIndexed.median).toFixed(1)}x` + ); + } finally { + // 恢复全部索引，避免影响下一个用例 + await ensureTagIndexes(); + } + }, 300_000); + + it('新格式：tags.tagId 复合索引 vs 无索引', async () => { + const root = await getRootUser(); + const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'new' }); + const query = { + teamId: root.teamId, + datasetId, + 'tags.tagId': { $all: [tagIds[0], tagIds[1]] } + }; + const runQuery = () => MongoDatasetCollection.find(query, '_id').lean(); + + try { + // 有索引：只保留 tags.tagId 复合索引，避免优化器误选 tags 索引（后者会导致 FETCH 全表） + await ensureTagIndexes(); + await MongoDatasetCollection.collection.dropIndex(TAGS_INDEX_NAME); + const expIndexed = await assertIndexed(query, TAGS_TAGID_INDEX_NAME, '新格式 tags.tagId'); + const mIndexed = await measure('新格式 + tags.tagId 索引', runQuery); + + // 无索引：删掉 tags.tagId 复合索引 + await MongoDatasetCollection.collection.dropIndex(TAGS_TAGID_INDEX_NAME); + const expUnindexed = await assertUnindexed(query, '新格式 tags.tagId'); + const mUnindexed = await measure('新格式 + 无索引', runQuery); + + // 断言：无索引扫描量/耗时明显大于有索引 + expect(expUnindexed.totalDocsExamined).toBeGreaterThan(expIndexed.totalDocsExamined * 5); + expect(expUnindexed.time.median).toBeGreaterThan(expIndexed.time.median); + expect(mUnindexed.median).toBeGreaterThan(mIndexed.median); + + console.log( + ` [新格式 tags.tagId] examined 倍率: ${(expUnindexed.totalDocsExamined / expIndexed.totalDocsExamined).toFixed(1)}x 执行时间倍率(median): ${(expUnindexed.time.median / expIndexed.time.median).toFixed(1)}x 中位延迟倍率: ${(mUnindexed.median / mIndexed.median).toFixed(1)}x` + ); + } finally { + // 恢复全部索引，避免影响下一个用例 + await ensureTagIndexes(); + } + }, 300_000); + + it('新格式：hint 强制走 tags.tagId 索引（生产两索引并存）', async () => { + const root = await getRootUser(); + const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'new' }); + const query = { + teamId: root.teamId, + datasetId, + 'tags.tagId': { $all: [tagIds[0], tagIds[1]] } + }; + + try { + // 生产状态：两个标签索引并存 + await ensureTagIndexes(); + + // 不加 hint：记录规划器自由选择的索引（优化器行为随环境/数据分布可能不同，不强断言） + const noHint = await explainQuery(query); + // 加 hint：必须强制走 tags.tagId 索引（修复的核心保证） + const hinted = await explainQuery(query, TAGS_TAGID_INDEX_KEY); + expect(hinted.indexName).toBe(TAGS_TAGID_INDEX_NAME); + expect(hinted.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); + // hint 后的扫描量不应比规划器自选更大 + expect(hinted.totalDocsExamined).toBeLessThanOrEqual(noHint.totalDocsExamined); + + console.log( + ` [新格式 hint 验证] 无 hint: ${noHint.indexName} (examined=${noHint.totalDocsExamined}) -> 有 hint: ${hinted.indexName} (examined=${hinted.totalDocsExamined})` + ); + } finally { + await ensureTagIndexes(); + } + }, 300_000); + + it('旧格式：两索引并存时查询仍走 tags 索引（不受 tags.tagId 索引影响）', async () => { + const root = await getRootUser(); + const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'old' }); + const query = { + teamId: root.teamId, + datasetId, + tags: { $all: [tagIds[0], tagIds[1]] } + }; + + try { + // 生产状态：两个标签索引并存 + await ensureTagIndexes(); + + // 不加 hint：旧格式查询按 tags 字段匹配，tags.tagId 索引无法精准服务该查询， + // 规划器应仍选择 tags 索引（否则会像新格式一样退到前缀索引导致全表 FETCH） + const noHint = await explainQuery(query); + expect(noHint.indexName).toBe(TAGS_INDEX_NAME); + expect(noHint.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); + + // 加 hint 强制 tags 索引：结果应同样高效 + const hinted = await explainQuery(query, TAGS_INDEX_KEY); + expect(hinted.indexName).toBe(TAGS_INDEX_NAME); + expect(hinted.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); + + console.log( + ` [旧格式两索引并存] 无 hint: ${noHint.indexName} (examined=${noHint.totalDocsExamined}) -> 有 hint: ${hinted.indexName} (examined=${hinted.totalDocsExamined})` + ); + } finally { + await ensureTagIndexes(); + } + }, 300_000); + + it('端到端：filterCollectionByKeyValueTags 应用层比较成本（真实数据）', async () => { + const root = await getRootUser(); + const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'new' }); + + // 同步真实 v2 标签表，供 filter 按标签名解析 tagId（_id 与集合 tags.tagId 一致） + await MongoDatasetCollectionTagsV2.collection.createIndex( + { teamId: 1, datasetId: 1, tag: 1 }, + { unique: true } + ); + await MongoDatasetCollectionTagsV2.insertMany( + tagIds.map((id, i) => ({ + teamId: root.teamId, + datasetId, + tag: `tag-${i}`, + tagType: 'string', + _id: new Types.ObjectId(id) + })) + ); + + // 生产状态：两个标签索引并存，filter 内部 hint 强制走 tags.tagId 索引 + await ensureTagIndexes(); + + const filter = () => + filterCollectionByKeyValueTags({ + $and: [{ 'tag-0': { $eq: 'A' } }, { 'tag-1': { $eq: 'B' } }], + $or: [], + teamId: root.teamId, + datasetIds: [datasetId] + }); + + // 命中 5% 的 collection（index % 20 === 0）同时携带 tagIds[0]('A') 与 tagIds[1]('B') + const hit = await filter(); + expect(hit?.length).toBe(Math.floor(COLLECTION_COUNT / HIT_EVERY)); + console.log(` [端到端] 命中集合数: ${hit?.length}`); + + const full = await measure('端到端 filterCollectionByKeyValueTags', filter); + + // 对照纯 tags.tagId 索引查询，量化应用层 checkValue 比较增量 + const query = { + teamId: root.teamId, + datasetId, + 'tags.tagId': { $all: [tagIds[0], tagIds[1]] } + }; + const raw = await measure('纯 tags.tagId 索引查询', () => + MongoDatasetCollection.find(query, '_id').hint(TAGS_TAGID_INDEX_KEY).lean() + ); + console.log( + ` [端到端] 应用层比较增量: ${(full.median - raw.median).toFixed(2)}ms（filter 中位 ${full.median.toFixed(2)}ms - 索引查询中位 ${raw.median.toFixed(2)}ms）` + ); + }, 300_000); +}); diff --git a/packages/service/test/core/dataset/utils.test.ts b/packages/service/test/core/dataset/utils.test.ts index 1d9e6c054be6..cb20ace09271 100644 --- a/packages/service/test/core/dataset/utils.test.ts +++ b/packages/service/test/core/dataset/utils.test.ts @@ -12,7 +12,13 @@ import { matchDatasetDataMarkdownImageUrls, uniqueDatasetDataMarkdownImageUrls } from '@fastgpt/service/core/dataset/data/utils'; -import { getTrainingModeByCollection } from '@fastgpt/service/core/dataset/collection/utils'; +import { + createOrGetCollectionTags, + deduplicateTagValues, + getTrainingModeByCollection, + validateAndNormalizeTagValue, + validateDatasetTagValue +} from '@fastgpt/service/core/dataset/collection/utils'; import { DatasetCollectionDataProcessModeEnum, TrainingModeEnum @@ -22,6 +28,8 @@ import type { EmbeddingSystemModelDataType, LLMSystemModelDataType } from '@fastgpt/global/core/ai/model.schema'; +import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; +import { DatasetCollectionItemSchema } from '@fastgpt/global/core/dataset/type'; const mockCreateS3DownloadAccessUrls = vi.hoisted(() => vi.fn(async (params: Array<{ objectKey: string }>) => @@ -31,6 +39,20 @@ const mockCreateS3DownloadAccessUrls = vi.hoisted(() => ) ); +const mockMongoDatasetCollectionTagsFind = vi.hoisted(() => vi.fn()); +const mockMongoDatasetCollectionTagsFindOne = vi.hoisted(() => vi.fn()); +const mockMongoDatasetCollectionTagsCreate = vi.hoisted(() => vi.fn()); + +vi.mock('@fastgpt/service/core/dataset/tag/schemaV2', () => ({ + MongoDatasetCollectionTagsV2: { + find: mockMongoDatasetCollectionTagsFind, + findOne: (...args: unknown[]) => ({ + lean: vi.fn().mockImplementation(() => mockMongoDatasetCollectionTagsFindOne(...args)) + }), + create: mockMongoDatasetCollectionTagsCreate + } +})); + vi.mock('@fastgpt/service/common/s3/utils', () => ({ isS3ObjectKey: vi.fn((key: string, source: string) => { if (!key) return false; @@ -52,6 +74,10 @@ vi.mock('@fastgpt/service/common/s3/contracts/type', () => ({ } })); +vi.mock('@fastgpt/service/core/dataset/tag/schema', () => ({ + MongoDatasetCollectionTags: {} +})); + describe('replaceS3KeyToPreviewUrl', () => { const expiredTime = new Date('2025-12-31'); @@ -688,3 +714,502 @@ describe('getDatasetImageIndexCapability', () => { expect(result.availableVlmModel?.model).toBe('dataset-vlm-model'); }); }); + +describe('validateDatasetTagValue', () => { + it.each([ + ['string', 1, DatasetErrEnum.tagValueInvalid], + ['string', 'a'.repeat(257), DatasetErrEnum.tagValueInvalid], + ['number', '1', undefined], + ['array', ['a'.repeat(257)], DatasetErrEnum.arrayTagValueInvalid], + ['datetime', Number.NaN, DatasetErrEnum.tagValueInvalid] + ])('returns the expected error for invalid %s values', (tagType, value, expected) => { + expect(validateDatasetTagValue({ tagType: tagType as any, value: value as any })).toBe( + expected + ); + }); + + it('accepts numeric strings for number and datetime tags', () => { + expect(validateDatasetTagValue({ tagType: 'number', value: '1.25' })).toBeUndefined(); + expect( + validateDatasetTagValue({ tagType: 'datetime', value: '1704067200000' }) + ).toBeUndefined(); + }); + + it.each(['', ' ', 'abc', 'Infinity'])('rejects invalid numeric string %j', (value) => { + expect(validateDatasetTagValue({ tagType: 'number', value })).toBe( + DatasetErrEnum.tagValueInvalid + ); + expect(validateDatasetTagValue({ tagType: 'datetime', value })).toBe( + DatasetErrEnum.tagValueInvalid + ); + }); + + it('rejects datetime date text', () => { + expect(validateDatasetTagValue({ tagType: 'datetime', value: '2024-01-01' })).toBe( + DatasetErrEnum.tagValueInvalid + ); + }); +}); + +describe('validateAndNormalizeTagValue', () => { + it('normalizes number and datetime string values to numbers', () => { + expect(validateAndNormalizeTagValue({ tagType: 'number', value: '1.25' })).toEqual({ + value: 1.25 + }); + expect(validateAndNormalizeTagValue({ tagType: 'datetime', value: '1704067200000' })).toEqual({ + value: 1704067200000 + }); + expect(validateAndNormalizeTagValue({ tagType: 'number', value: 42 })).toEqual({ value: 42 }); + }); + + it('keeps string and array values unchanged', () => { + expect(validateAndNormalizeTagValue({ tagType: 'string', value: '产品' })).toEqual({ + value: '产品' + }); + expect(validateAndNormalizeTagValue({ tagType: 'array', value: ['安全', '高优'] })).toEqual({ + value: ['安全', '高优'] + }); + }); + + it('surfaces errors for invalid number/datetime strings', () => { + expect(validateAndNormalizeTagValue({ tagType: 'number', value: ' ' })).toEqual({ + value: ' ', + error: DatasetErrEnum.tagValueInvalid + }); + expect(validateAndNormalizeTagValue({ tagType: 'number', value: 'abc' })).toEqual({ + value: 'abc', + error: DatasetErrEnum.tagValueInvalid + }); + expect(validateAndNormalizeTagValue({ tagType: 'datetime', value: '2024-01-01' })).toEqual({ + value: '2024-01-01', + error: DatasetErrEnum.tagValueInvalid + }); + }); + + it('surfaces errors for invalid string/array values', () => { + expect(validateAndNormalizeTagValue({ tagType: 'string', value: 1 as any })).toEqual({ + value: 1, + error: DatasetErrEnum.tagValueInvalid + }); + expect(validateAndNormalizeTagValue({ tagType: 'array', value: ['a'.repeat(257)] })).toEqual({ + value: ['a'.repeat(257)], + error: DatasetErrEnum.arrayTagValueInvalid + }); + }); +}); + +describe('createOrGetCollectionTags', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue(null); + mockMongoDatasetCollectionTagsCreate.mockResolvedValue([]); + }); + + it('returns undefined when tags is undefined', async () => { + const result = await createOrGetCollectionTags({ + tags: undefined, + datasetId: 'ds-1', + teamId: 'team-1' + }); + expect(result).toBeUndefined(); + expect(mockMongoDatasetCollectionTagsFind).not.toHaveBeenCalled(); + }); + + it('returns empty array when tags is empty', async () => { + const result = await createOrGetCollectionTags({ + tags: [], + datasetId: 'ds-1', + teamId: 'team-1' + }); + expect(result).toEqual([]); + expect(mockMongoDatasetCollectionTagsFind).not.toHaveBeenCalled(); + }); + + it('resolves existing string tags to default_tag array value', async () => { + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array' + }); + + const result = await createOrGetCollectionTags({ + tags: ['safety'], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'default-tag-id', value: ['safety'] }]); + }); + + it('creates default_tag for unknown string names', async () => { + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue(null); + mockMongoDatasetCollectionTagsCreate.mockResolvedValue([ + { + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array', + toObject: () => ({ _id: 'default-tag-id', tag: 'default_tag', tagType: 'array' }) + } + ]); + + const result = await createOrGetCollectionTags({ + tags: ['safety'], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'default-tag-id', value: ['safety'] }]); + }); + + it('resolves {tag,value} for existing string tag', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) + }); + + const result = await createOrGetCollectionTags({ + tags: [{ tag: 'safety', value: 'A' }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'tag-id-1', value: 'A' }]); + }); + + it('resolves {tag,value} for existing number tag with number value', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'version', tagType: 'number' }]) + }); + + const result = await createOrGetCollectionTags({ + tags: [{ tag: 'version', value: 2 }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'tag-id-1', value: 2 }]); + }); + + it('resolves {tag,value} for existing datetime tag with number value', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'date', tagType: 'datetime' }]) + }); + + const result = await createOrGetCollectionTags({ + tags: [{ tag: 'date', value: 1704067200000 }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'tag-id-1', value: 1704067200000 }]); + }); + + it('rejects when string tag gets non-string value', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) + }); + + await expect( + createOrGetCollectionTags({ + tags: [{ tag: 'safety', value: 123 }], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.tagValueInvalid); + }); + + it('resolves existing number tag with numeric string value as number', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'version', tagType: 'number' }]) + }); + + const result = await createOrGetCollectionTags({ + tags: [{ tag: 'version', value: '2' }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'tag-id-1', value: 2 }]); + }); + + it('resolves existing datetime tag with numeric string value as number', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'date', tagType: 'datetime' }]) + }); + + const result = await createOrGetCollectionTags({ + tags: [{ tag: 'date', value: '1704067200000' }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'tag-id-1', value: 1704067200000 }]); + }); + + it.each([ + ['number', Infinity], + ['number', 'Infinity'], + ['datetime', Infinity], + ['datetime', 'Infinity'], + ['number', ''], + ['number', ' '], + ['datetime', ''], + ['datetime', ' '], + ['number', 'abc'], + ['datetime', 'abc'], + ['datetime', Number.MAX_VALUE] + ])('rejects invalid %s value %j in creation chain', async (tagType, value) => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'tag-1', tagType }]) + }); + + await expect( + createOrGetCollectionTags({ + tags: [{ tag: 'tag-1', value: value as string | number }], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe( + tagType === 'datetime' && value === Number.MAX_VALUE + ? DatasetErrEnum.tagValueDatetimeInvalid + : DatasetErrEnum.tagValueInvalid + ); + }); + + it('rejects when datetime tag gets non-number value', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'date', tagType: 'datetime' }]) + }); + + await expect( + createOrGetCollectionTags({ + tags: [{ tag: 'date', value: '2024-01-01' }], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.tagValueInvalid); + }); + + it('handles mixed string and {tag,value} inputs', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-2', tag: 'version', tagType: 'number' }]) + }); + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array' + }); + + const result = await createOrGetCollectionTags({ + tags: ['safety', { tag: 'version', value: 3 }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([ + { tagId: 'default-tag-id', value: ['safety'] }, + { tagId: 'tag-id-2', value: 3 } + ]); + }); + + it('passes session through to DB calls', async () => { + const session = { sessionId: 'sess-1' } as any; + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array' + }); + + await createOrGetCollectionTags({ + tags: ['safety'], + datasetId: 'ds-1', + teamId: 'team-1', + session + }); + + expect(mockMongoDatasetCollectionTagsFindOne).toHaveBeenCalledWith( + expect.any(Object), + undefined, + { session } + ); + }); + + it('deduplicates repeated string names', async () => { + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array' + }); + + const result = await createOrGetCollectionTags({ + tags: ['safety', 'safety'], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'default-tag-id', value: ['safety'] }]); + }); + + it('trims string names and object tag lookup keys', async () => { + const session = { sessionId: 'sess-1' } as any; + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array' + }); + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) + }); + + const result = await createOrGetCollectionTags({ + tags: [' safety ', { tag: ' safety ', value: 'A' }], + datasetId: 'ds-1', + teamId: 'team-1', + session + }); + + // 对象 tag 名称 trim 后按 'safety' 查询；string 名 ' safety ' trim 后进 default_tag + expect(mockMongoDatasetCollectionTagsFind).toHaveBeenCalledWith( + expect.objectContaining({ tag: { $in: ['safety'] } }), + undefined, + { session } + ); + expect(result).toEqual([ + { tagId: 'default-tag-id', value: ['safety'] }, + { tagId: 'tag-id-1', value: 'A' } + ]); + }); + + it('rejects blank string names', async () => { + await expect( + createOrGetCollectionTags({ + tags: [' '], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.tagNameEmpty); + }); + + it('rejects blank object tag names', async () => { + await expect( + createOrGetCollectionTags({ + tags: [{ tag: ' ', value: 'A' }], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.tagNameEmpty); + }); + + it('deduplicates identical {tag,value} inputs', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) + }); + + const result = await createOrGetCollectionTags({ + tags: [ + { tag: 'safety', value: 'A' }, + { tag: 'safety', value: 'A' } + ], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'tag-id-1', value: 'A' }]); + }); + + it('rejects conflicting values for the same tag', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) + }); + + await expect( + createOrGetCollectionTags({ + tags: [ + { tag: 'safety', value: 'A' }, + { tag: 'safety', value: 'B' } + ], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.tagValueInvalid); + }); + + it('merges string names with default_tag array values', async () => { + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array' + }); + + const result = await createOrGetCollectionTags({ + tags: ['a', { tag: 'default_tag', value: ['b', 'b'] }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'default-tag-id', value: ['a', 'b'] }]); + }); + + it('supports array value via default_tag object input', async () => { + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ + _id: 'default-tag-id', + tag: 'default_tag', + tagType: 'array' + }); + + const result = await createOrGetCollectionTags({ + tags: [{ tag: 'default_tag', value: ['x', 'y'] }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'default-tag-id', value: ['x', 'y'] }]); + }); + + it('rejects non-array value for default_tag object input', async () => { + await expect( + createOrGetCollectionTags({ + tags: [{ tag: 'default_tag', value: 'not-array' }], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.arrayTagValueInvalid); + }); +}); + +describe('deduplicateTagValues', () => { + it('deduplicates identical values and keeps first occurrence', async () => { + const result = await deduplicateTagValues([ + { tagId: 't1', value: ['a', 'b'] }, + { tagId: 't1', value: ['b', 'a'] } + ]); + expect(result).toEqual([{ tagId: 't1', value: ['a', 'b'] }]); + }); + + it('rejects conflicting values for the same tagId', async () => { + await expect( + deduplicateTagValues([ + { tagId: 't1', value: 'A' }, + { tagId: 't1', value: 'B' } + ]) + ).rejects.toBe(DatasetErrEnum.tagValueInvalid); + }); +}); + +describe('DatasetCollectionItemSchema.tags array values', () => { + it('accepts array values in the detail response tags schema', () => { + const tagsSchema = DatasetCollectionItemSchema.shape.tags; + const result = tagsSchema.parse([ + { tag: 'safety', value: ['a', 'b'] }, + { tag: 'version', value: 2 }, + 'legacy' + ]); + expect(result).toEqual([ + { tag: 'safety', value: ['a', 'b'] }, + { tag: 'version', value: 2 }, + 'legacy' + ]); + }); +}); diff --git a/packages/service/test/integrations/vectorDB/README.md b/packages/service/test/integrations/vectorDB/README.md index 44636e1e24bd..1760ecdc3578 100644 --- a/packages/service/test/integrations/vectorDB/README.md +++ b/packages/service/test/integrations/vectorDB/README.md @@ -18,6 +18,8 @@ cp test/.env.example test/.env.test.local | `PG_URL` | PostgreSQL + pgvector 连接串 | PgVectorCtrl | | `OCEANBASE_URL` | Oceanbase 连接串（后续） | ObVectorCtrl | | `MILVUS_ADDRESS` | Milvus 地址（后续） | MilvusCtrl | +| `MILVUS_PERF_COLLECTION_COUNT` | Milvus 召回性能测试的 collection 数，默认 100 | MilvusCtrl | +| `MILVUS_PERF_VECTORS_PER_COLLECTION` | Milvus 召回性能测试每个 collection 的向量数，默认 100 | MilvusCtrl | 未设置对应环境变量时，该驱动的集成测试会**整体跳过**，不会报错。 @@ -33,7 +35,10 @@ pnpm test FASTGPT_TEST_MODE=integration pnpm test # 运行当前这组 vectorDB 集成测试 -FASTGPT_TEST_MODE=integration pnpm test +pnpm test:vector + +# 只运行 Milvus collectionId 过滤召回性能测试 +cd packages/service && pnpm vitest run -c vitest.integration.config.ts test/integrations/vectorDB/milvus/recallFilterPerformance ``` ## 结构说明 @@ -43,3 +48,15 @@ FASTGPT_TEST_MODE=integration pnpm test - `*/index.integration.test.ts`：各向量库入口，按环境变量决定是否跳过。 新增向量库时：新增一个 `*/index.integration.test.ts`，复用 `testData.ts` 和 `testSuites.ts` 即可。 + +### Milvus collectionId 过滤召回性能测试 + +`milvus/recallFilterPerformance.integration.test.ts` 对应设计文档「端到端检索性能测试」， +范围收窄为只测 Milvus 的向量召回：直接构造 `filterCollectionIdList`（不经过标签过滤/mongo 链路）， +对比 `embRecall` 在「不过滤 / 过滤 10 个 collection / 过滤 100 个 collection」下的召回延迟， +断言过滤延迟与不过滤保持同一数量级（< 10x）。 + +测试在 `beforeAll` 里向 1 个 dataset 下的 100 个 collection 插入向量（默认共 1 万条）， +`flushSync` + `loadCollectionSync` 把数据落到已索引的 sealed segment 后再测，保证延迟接近生产行为。 +数据规模可通过 `MILVUS_PERF_COLLECTION_COUNT` / `MILVUS_PERF_VECTORS_PER_COLLECTION` 覆盖 +（如在大集群上跑到设计文档目标量级）。 diff --git a/packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts b/packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts new file mode 100644 index 000000000000..fc8e7a31a4f3 --- /dev/null +++ b/packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts @@ -0,0 +1,225 @@ +import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest'; +import { VECTOR_DIM, QUERY_VECTOR } from '../testData'; + +// 集成测试需用真实 MilvusCtrl，先解除 test/mocks/common/vector.ts 里的全局 mock +vi.unmock('@fastgpt/service/common/vectorDB/milvus'); +vi.unmock('@fastgpt/service/common/vectorDB/constants'); + +import { MilvusCtrl } from '@fastgpt/service/common/vectorDB/milvus'; +import { DatasetVectorTableName } from '@fastgpt/service/common/vectorDB/constants'; + +/** + * Milvus 端到端召回性能测试（collectionId 过滤） + * + * 对应设计文档「端到端检索性能测试」，范围收窄为： + * - 只测 Milvus（不经过标签过滤 / mongo 链路），直接构造 `filterCollectionIdList` + * - 对比 embRecall 在「不过滤 / 过滤 10 个 collection / 过滤 100 个 collection」下的召回延迟 + * - 目标：过滤召回延迟与不过滤保持同一数量级（< 10x） + * + * 运行要求：设置 `MILVUS_ADDRESS`（见 test/.env.example），未设置则整体跳过。 + * 数据规模可通过环境变量覆盖，默认 100 个 collection × 100 条向量 = 1 万条向量： + * MILVUS_PERF_COLLECTION_COUNT 默认 100 + * MILVUS_PERF_VECTORS_PER_COLLECTION 默认 100 + * + * 运行方式：pnpm test:vector（或直接 vitest -c vitest.integration.config.ts 本文件） + */ + +const isEnabled = Boolean(process.env.MILVUS_ADDRESS); + +const COLLECTION_COUNT = Number(process.env.MILVUS_PERF_COLLECTION_COUNT || 100); +const VECTORS_PER_COLLECTION = Number(process.env.MILVUS_PERF_VECTORS_PER_COLLECTION || 100); +const SEARCH_LIMIT = 10; +const MEASURE_ITERATIONS = 20; +const MEASURE_WARMUPS = 3; + +const teamId = `perf_team_${Date.now()}`; +const datasetId = `perf_dataset_${Date.now()}`; +const collectionIds = Array.from({ length: COLLECTION_COUNT }, (_, i) => `perf_col_${i + 1}`); + +// 确定性向量，便于复现；IP 度量下正数向量的打分稳定 +const buildVector = (seed: number) => + Array.from({ length: VECTOR_DIM }, (_, index) => (((index + seed) % 10) + 1) / 100); + +interface MeasureResult { + avg: number; + min: number; + max: number; + median: number; + times: number[]; +} + +// Milvus 查询是网络调用，只统计墙钟延迟；预热 + 多次取平均/中位数 +const measure = async ( + name: string, + fn: () => Promise, + iterations = MEASURE_ITERATIONS, + warmups = MEASURE_WARMUPS +): Promise => { + for (let i = 0; i < warmups; i++) await fn(); + + const times: number[] = []; + for (let i = 0; i < iterations; i++) { + const start = performance.now(); + await fn(); + times.push(performance.now() - start); + } + + const avg = times.reduce((a, b) => a + b, 0) / times.length; + const min = Math.min(...times); + const max = Math.max(...times); + const sorted = [...times].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; + + console.log( + ` ${name.padEnd(22)} 平均: ${avg.toFixed(2).padStart(8)}ms 中位: ${median + .toFixed(2) + .padStart(8)}ms 最小: ${min.toFixed(2).padStart(8)}ms 最大: ${max + .toFixed(2) + .padStart(8)}ms` + ); + return { avg, min, max, median, times }; +}; + +// 并发插入各 collection 的向量。 +// 实测该实例 500 向量/批的 insert RPC 约 5~6s，而 1000 向量/批会超线性变慢（~26s/批）， +// 因此用小批 + 低并发更划算；并发大包曾触发 gRPC RST_STREAM，保持低并发 + 失败退避重试。 +const INSERT_BATCH = 500; +const INSERT_CONCURRENCY = 3; +const INSERT_MAX_RETRY = 5; + +const insertCollections = async (vectorCtrl: MilvusCtrl) => { + // 只存批次元信息，向量在插入前惰性构建，避免一次性物化全部向量导致 OOM + // （1000×1000 向量全量构建约 12GB 堆） + const batches: { collectionIndex: number; collectionId: string; start: number; count: number }[] = + []; + collectionIds.forEach((collectionId, collectionIndex) => { + for (let start = 0; start < VECTORS_PER_COLLECTION; start += INSERT_BATCH) { + batches.push({ + collectionIndex, + collectionId, + start, + count: Math.min(INSERT_BATCH, VECTORS_PER_COLLECTION - start) + }); + } + }); + + let cursor = 0; + const insertWithRetry = async ( + batch: { collectionIndex: number; collectionId: string; start: number; count: number }, + attempt = 0 + ): Promise => { + const vectors = Array.from({ length: batch.count }, (_, i) => + buildVector(batch.collectionIndex * VECTORS_PER_COLLECTION + batch.start + i) + ); + try { + await vectorCtrl.insert({ + teamId, + datasetId, + collectionId: batch.collectionId, + vectors + }); + } catch (error) { + if (attempt < INSERT_MAX_RETRY) { + await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1))); + return insertWithRetry(batch, attempt + 1); + } + throw error; + } + }; + + const workers = Array.from({ length: Math.min(INSERT_CONCURRENCY, batches.length) }, async () => { + while (cursor < batches.length) { + const batch = batches[cursor++]; + await insertWithRetry(batch); + if (cursor % 25 === 0) { + console.log(` [insert] ${cursor}/${batches.length} batches`); + } + } + }); + await Promise.all(workers); +}; + +describe.skipIf(!isEnabled)('Milvus 端到端召回性能（collectionId 过滤）', () => { + const vectorCtrl = new MilvusCtrl(); + + beforeAll(async () => { + await vectorCtrl.init(); + + console.log( + `\n[Milvus 数据准备] ${COLLECTION_COUNT} collections × ${VECTORS_PER_COLLECTION} vectors，开始插入...` + ); + await insertCollections(vectorCtrl); + console.log(' [insert] 插入完成，flush + load...'); + + // flushSync 等待数据落到已索引的 sealed segment，loadCollectionSync 确保可检索， + // 让延迟接近生产行为（不测 growing segment 的全扫描） + const client = await vectorCtrl.getClient(); + await client.flushSync({ collection_names: [DatasetVectorTableName] }); + await client.loadCollectionSync({ collection_name: DatasetVectorTableName }); + console.log(' [flush] 数据已落盘并加载，开始测量\n'); + }, 10_800_000); + + afterAll(async () => { + try { + await vectorCtrl.delete({ teamId, datasetIds: [datasetId] }); + } catch (error) { + // 清理失败不影响结果 + } + }); + + test('不过滤 vs 过滤 10 / 100 个 collection 的召回延迟', async () => { + console.log( + `\n[Milvus collectionId 过滤召回性能] ${COLLECTION_COUNT} collections × ${VECTORS_PER_COLLECTION} vectors/collection，limit=${SEARCH_LIMIT}` + ); + + const base = { + teamId, + datasetIds: [datasetId], + vector: QUERY_VECTOR, + limit: SEARCH_LIMIT, + forbidCollectionIdList: [] as string[] + }; + + // 1. 基线：不过滤 + const baselineResult = await vectorCtrl.embRecall({ ...base }); + expect(baselineResult.results.length).toBeGreaterThan(0); + const baseline = await measure('不过滤 (baseline)', () => vectorCtrl.embRecall({ ...base })); + + // 2. 过滤 10 个 collection（10% 命中） + const filter10Ids = collectionIds.slice(0, 10); + const filter10Result = await vectorCtrl.embRecall({ + ...base, + filterCollectionIdList: filter10Ids + }); + expect(filter10Result.results.length).toBeGreaterThan(0); + expect(filter10Result.results.every((item) => filter10Ids.includes(item.collectionId))).toBe( + true + ); + const filter10 = await measure('过滤 10 个 collection', () => + vectorCtrl.embRecall({ ...base, filterCollectionIdList: filter10Ids }) + ); + + // 3. 过滤 100 个 collection（全部命中） + const filter100Result = await vectorCtrl.embRecall({ + ...base, + filterCollectionIdList: collectionIds + }); + expect(filter100Result.results.length).toBeGreaterThan(0); + expect(filter100Result.results.every((item) => collectionIds.includes(item.collectionId))).toBe( + true + ); + const filter100 = await measure('过滤 100 个 collection', () => + vectorCtrl.embRecall({ ...base, filterCollectionIdList: collectionIds }) + ); + + // 目标：过滤延迟与不过滤保持同一数量级（< 10x），floor 50ms 避免基线过快时的抖动误判 + console.log( + `\n [对比] 过滤10/不过滤: ${(filter10.avg / baseline.avg).toFixed(2)}x | 过滤100/不过滤: ${( + filter100.avg / baseline.avg + ).toFixed(2)}x` + ); + expect(filter10.avg).toBeLessThan(Math.max(baseline.avg * 10, 50)); + expect(filter100.avg).toBeLessThan(Math.max(baseline.avg * 10, 50)); + }, 300_000); +}); diff --git a/packages/service/types/safe-regex.d.ts b/packages/service/types/safe-regex.d.ts new file mode 100644 index 000000000000..2be2e2dbc89c --- /dev/null +++ b/packages/service/types/safe-regex.d.ts @@ -0,0 +1,8 @@ +declare module 'safe-regex' { + interface SafeRegexOptions { + /** 解析的 AST 节点数上限，超过即视为不安全（默认 25） */ + limit?: number; + } + function safeRegex(re: RegExp | string, opts?: SafeRegexOptions): boolean; + export default safeRegex; +} diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index 81bff3450a62..c0a3aed86433 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -459,6 +459,18 @@ "core.dataset.error.unCreateCollection": "Unauthorized to create dataset collection", "core.dataset.error.unExistDataset": "The dataset does not exist", "core.dataset.error.unLinkCollection": "Unauthorized to operate this external dataset collection", + "core.dataset.error.tagNameDuplicate": "Tag name already exists", + "core.dataset.error.tagNameEmpty": "Tag name cannot be empty", + "core.dataset.error.tagNotExist": "Tag does not exist", + "core.dataset.error.tagValueInvalid": "Tag value format does not match type", + "core.dataset.error.tagValueStringTooLong": "String tag value exceeds maximum length", + "core.dataset.error.tagValueNumberOutOfRange": "Number tag value out of safe integer range", + "core.dataset.error.tagValueDatetimeInvalid": "Datetime tag value format is invalid", + "core.dataset.error.noDatasetForTagFilter": "No dataset selected, cannot configure tag filter", + "core.dataset.error.noTagsInDataset": "No available tags in the current dataset", + "core.dataset.error.noPermissionForDatasetTags": "No permission to access dataset tags", + "core.dataset.error.tagNotSelectedForRef": "Please select a tag before referencing a variable", + "core.dataset.error.arrayTagValueInvalid": "Array tag value invalid (elements must be string, array length ≤ 64, element length ≤ 256)", "core.dataset.externalFile": "External File Library", "core.dataset.file": "File", "core.dataset.folder": "Directory", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 35699a22bc49..8f5462d9f1bf 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -32,6 +32,13 @@ "collection_sync": "Sync data", "collection_sync_confirm_tip": "Start syncing data? FastGPT compares the latest data and, when content has changed, creates a new Collection and deletes the old one.", "collection_tags": "Collection Tags", + "core.dataset.tags.batchUpsert": "Batch Manage Tags", + "core.dataset.tags.datetime": "Datetime", + "core.dataset.tags.number": "Number", + "core.dataset.tags.setTags": "Set Tags", + "core.dataset.tags.string": "String", + "core.dataset.tags.tagType": "Tag Type", + "core.dataset.tags.tagValue": "Tag Value", "common.error.unKnow": "Unknown error", "common_dataset": "General Dataset", "common_dataset_desc": "Build a Dataset from files, web pages, or manually entered content", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index a86c5212ec74..44ce2f6e77d2 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -459,6 +459,18 @@ "core.dataset.error.unCreateCollection": "无权创建知识库集合", "core.dataset.error.unExistDataset": "知识库不存在", "core.dataset.error.unLinkCollection": "无权操作该外部知识库集合", + "core.dataset.error.tagNameDuplicate": "标签名称已存在", + "core.dataset.error.tagNameEmpty": "标签名称不能为空", + "core.dataset.error.tagNotExist": "标签不存在", + "core.dataset.error.tagValueInvalid": "标签值格式不符合类型要求", + "core.dataset.error.tagValueStringTooLong": "字符串标签值超出最大长度限制", + "core.dataset.error.tagValueNumberOutOfRange": "数字标签值超出安全整数范围", + "core.dataset.error.tagValueDatetimeInvalid": "日期时间标签值格式无效", + "core.dataset.error.noDatasetForTagFilter": "未选择知识库，无法配置标签过滤", + "core.dataset.error.noTagsInDataset": "当前知识库没有可用标签", + "core.dataset.error.noPermissionForDatasetTags": "无权限获取知识库的标签", + "core.dataset.error.tagNotSelectedForRef": "请先选择标签后再引用变量", + "core.dataset.error.arrayTagValueInvalid": "数组标签值格式无效（元素须为字符串，数组长度不超过 64，单元素不超过 256 字符）", "core.dataset.externalFile": "外部文件库", "core.dataset.file": "文件", "core.dataset.folder": "目录", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index eeed3f2f42d2..451886d71799 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -32,6 +32,13 @@ "collection_sync": "立即同步", "collection_sync_confirm_tip": "确认开始同步数据？系统将会拉取最新数据进行比较，如果内容不相同，则会创建一个新的集合并删除旧的集合，请确认！", "collection_tags": "集合标签", + "core.dataset.tags.batchUpsert": "批量管理标签", + "core.dataset.tags.datetime": "日期时间", + "core.dataset.tags.number": "数字", + "core.dataset.tags.setTags": "设置标签值", + "core.dataset.tags.string": "文本", + "core.dataset.tags.tagType": "标签类型", + "core.dataset.tags.tagValue": "标签值", "common.error.unKnow": "未知错误", "common_dataset": "通用知识库", "common_dataset_desc": "通过导入文件、网页链接或手动录入形式构建知识库", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index a76cb6dacaf3..de7908e31009 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -459,6 +459,18 @@ "core.dataset.error.unCreateCollection": "無權建立知識庫集合", "core.dataset.error.unExistDataset": "知識庫不存在", "core.dataset.error.unLinkCollection": "無權操作該外部知識庫集合", + "core.dataset.error.tagNameDuplicate": "標籤名稱已存在", + "core.dataset.error.tagNameEmpty": "標籤名稱不能為空", + "core.dataset.error.tagNotExist": "標籤不存在", + "core.dataset.error.tagValueInvalid": "標籤值格式不符合類型要求", + "core.dataset.error.tagValueStringTooLong": "字串標籤值超出最大長度限制", + "core.dataset.error.tagValueNumberOutOfRange": "數字標籤值超出安全整數範圍", + "core.dataset.error.tagValueDatetimeInvalid": "日期時間標籤值格式無效", + "core.dataset.error.noDatasetForTagFilter": "未選擇知識庫，無法配置標籤過濾", + "core.dataset.error.noTagsInDataset": "當前知識庫沒有可用標籤", + "core.dataset.error.noPermissionForDatasetTags": "無權限獲取知識庫的標籤", + "core.dataset.error.tagNotSelectedForRef": "請先選擇標籤後再引用變數", + "core.dataset.error.arrayTagValueInvalid": "陣列標籤值格式無效（元素須為字串，陣列長度不超過 64，單元素不超過 256 字元）", "core.dataset.externalFile": "外部檔案庫", "core.dataset.file": "檔案", "core.dataset.folder": "目錄", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index b71899472cc1..329dc6613edd 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -32,6 +32,13 @@ "collection_sync": "立即同步", "collection_sync_confirm_tip": "確定要開始同步資料嗎？系統會比對最新資料；若內容不同，將建立新集合並刪除舊集合。", "collection_tags": "集合標籤", + "core.dataset.tags.batchUpsert": "批量管理標籤", + "core.dataset.tags.datetime": "日期時間", + "core.dataset.tags.number": "數字", + "core.dataset.tags.setTags": "設定標籤值", + "core.dataset.tags.string": "文字", + "core.dataset.tags.tagType": "標籤類型", + "core.dataset.tags.tagValue": "標籤值", "common.error.unKnow": "未知錯誤", "common_dataset": "通用資料集", "common_dataset_desc": "通過導入文件、網頁鏈接或手動錄入形式構建知識庫", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6bd89b67bb2b..56572d910f88 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -704,6 +704,9 @@ importers: proxy-from-env: specifier: ^1.1.0 version: 1.1.0 + safe-regex: + specifier: ^2.1.1 + version: 2.1.1 turndown: specifier: ^7.1.2 version: 7.2.0 @@ -12979,6 +12982,10 @@ packages: regex@6.1.0: resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==} + regexp-tree@0.1.27: + resolution: {integrity: sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==} + hasBin: true + regexp.prototype.flags@1.5.4: resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==} engines: {node: '>= 0.4'} @@ -13256,6 +13263,9 @@ packages: resolution: {integrity: sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==} engines: {node: '>= 0.4'} + safe-regex@2.1.1: + resolution: {integrity: sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A==} + safe-stable-stringify@2.5.0: resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} engines: {node: '>=10'} @@ -29021,6 +29031,8 @@ snapshots: dependencies: regex-utilities: 2.3.0 + regexp-tree@0.1.27: {} + regexp.prototype.flags@1.5.4: dependencies: call-bind: 1.0.8 @@ -29426,6 +29438,10 @@ snapshots: es-errors: 1.3.0 is-regex: 1.2.1 + safe-regex@2.1.1: + dependencies: + regexp-tree: 0.1.27 + safe-stable-stringify@2.5.0: {} safer-buffer@2.1.2: {} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx index bf3a71ea334b..3c77db4ba2bc 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx @@ -279,8 +279,8 @@ const TagManageModal = ({ onClose }: { onClose: () => void }) => { .includes(currentEditTagContent) ) { onUpdateCollectionTag({ - tag: currentEditTagContent, - _id: item._id + ...item, + tag: currentEditTagContent }); } setCurrentEditTag(undefined); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx index be3311698060..30f8128a5d71 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx @@ -32,7 +32,11 @@ const TagsPopOver = ({ isCreateCollectionTagLoading } = useContextSelector(DatasetPageContext, (v) => v); - const [collectionTags, setCollectionTags] = useState(currentCollection.tags ?? []); + const [collectionTags, setCollectionTags] = useState(() => + (currentCollection.tags ?? []) + .map((tag) => (typeof tag === 'string' ? tag : tag.tag)) + .filter(Boolean) + ); const [checkedTags, setCheckedTags] = useState([]); const [showTagManage, setShowTagManage] = useState(false); const [isUpdateLoading, setIsUpdateLoading] = useState(false); @@ -40,14 +44,8 @@ const TagsPopOver = ({ const tagList = useMemo( () => (collectionTags - ?.map((item) => { - const tagObject = allDatasetTags.find((tag) => tag.tag === item); - return tagObject ? { _id: tagObject._id, tag: tagObject.tag } : null; - }) - .filter((tag) => tag !== null) as { - _id: string; - tag: string; - }[]) || [], + ?.map((item) => allDatasetTags.find((tag) => tag.tag === item) ?? null) + .filter((tag) => tag !== null) as DatasetTagType[]) || [], [collectionTags, allDatasetTags] ); diff --git a/projects/app/src/pages/api/admin/migrateTags.ts b/projects/app/src/pages/api/admin/migrateTags.ts new file mode 100644 index 000000000000..235cce46add1 --- /dev/null +++ b/projects/app/src/pages/api/admin/migrateTags.ts @@ -0,0 +1,128 @@ +import { NextAPI } from '@/service/middleware/entry'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { DEFAULT_TAG, type CollectionTagValueType } from '@fastgpt/global/core/dataset/type'; +import { getLogger } from '@fastgpt/service/common/logger'; + +const logger = getLogger(['migrateTags']); +const isLegacyTag = (tag: unknown): tag is string => typeof tag === 'string'; + +export default NextAPI(async function handler(req) { + await authCert({ req, authRoot: true }); + + // 清理 v2 表重复行，避免 unique 索引构建失败；保留 _id 最早的一条（ObjectId 含创建时间，排序即时间序） + const dupGroups = await MongoDatasetCollectionTagsV2.aggregate([ + { + $group: { + _id: { teamId: '$teamId', datasetId: '$datasetId', tag: '$tag' }, + ids: { $push: '$_id' } + } + }, + { $match: { $expr: { $gt: [{ $size: '$ids' }, 1] } } } + ]); + for (const group of dupGroups) { + const [, ...dups] = [...group.ids].sort(); + await MongoDatasetCollectionTagsV2.deleteMany({ _id: { $in: dups } }); + } + + const datasetIds = (await MongoDatasetCollectionTags.distinct('datasetId')) as string[]; + logger.info(`[TagMigration] Starting tag migration for ${datasetIds.length} datasets`); + let migratedDatasets = 0; + let migratedCollections = 0; + + for (const datasetId of datasetIds) { + const legacyTags = await MongoDatasetCollectionTags.find( + { datasetId }, + '_id tag teamId' + ).lean(); + const tagNameMap = new Map(legacyTags.map((tag) => [String(tag._id), tag.tag])); + logger.info( + `[TagMigration] datasetId=${datasetId}, legacyTags=${JSON.stringify(Array.from(tagNameMap.values()))}` + ); + if (tagNameMap.size === 0) continue; + const firstLegacyTag = legacyTags[0]; + if (!firstLegacyTag) continue; + + // default_tag 承载记录按 fromMigration 定位；存量按名称创建的记录回填标记并确保 tagType=array + let defaultTag = await MongoDatasetCollectionTagsV2.findOne({ + datasetId, + fromMigration: true + }).lean(); + if (!defaultTag) { + const legacyDefaultTag = await MongoDatasetCollectionTagsV2.findOne({ + datasetId, + tag: DEFAULT_TAG + }).lean(); + if (legacyDefaultTag) { + await MongoDatasetCollectionTagsV2.updateOne( + { _id: legacyDefaultTag._id }, + { $set: { fromMigration: true, tagType: 'array' } } + ); + defaultTag = { ...legacyDefaultTag, fromMigration: true, tagType: 'array' }; + } else { + try { + const createdTag = await MongoDatasetCollectionTagsV2.create({ + teamId: firstLegacyTag.teamId, + datasetId, + tag: DEFAULT_TAG, + tagType: 'array', + fromMigration: true + }); + defaultTag = createdTag.toObject(); + } catch (error: any) { + // 并发创建撞 unique 索引 → 复用已存在记录 + if (error?.code !== 11000) throw error; + defaultTag = await MongoDatasetCollectionTagsV2.findOne({ + datasetId, + fromMigration: true + }).lean(); + if (!defaultTag) throw error; + } + } + } + const defaultTagId = String(defaultTag._id); + + const collections = await MongoDatasetCollection.find({ datasetId }, '_id tags').lean(); + let migratedInDataset = 0; + for (const collection of collections) { + const tags = (Array.isArray(collection.tags) ? collection.tags : []) as ( + | string + | CollectionTagValueType + )[]; + if (!tags.some(isLegacyTag)) continue; + + const tagNames = tags + .filter(isLegacyTag) + .map((tagId) => tagNameMap.get(tagId)) + .filter((tagName): tagName is string => Boolean(tagName)); + const existingDefaultTag = tags.find( + (tag): tag is CollectionTagValueType => + typeof tag === 'object' && tag !== null && tag.tagId === defaultTagId + ); + const existingDefaultValues = Array.isArray(existingDefaultTag?.value) + ? existingDefaultTag.value.filter((value): value is string => typeof value === 'string') + : []; + const mergedTagNames = [...new Set([...existingDefaultValues, ...tagNames])]; + const migratedTags: (string | CollectionTagValueType)[] = tags.filter( + (tag) => !isLegacyTag(tag) && tag !== existingDefaultTag + ); + migratedTags.push({ tagId: defaultTagId, value: mergedTagNames }); + logger.info( + `[TagMigration] collectionId=${String(collection._id)}, collectionTags=${JSON.stringify(tags)}` + ); + + await MongoDatasetCollection.updateOne( + { _id: collection._id }, + { $set: { tags: migratedTags } } + ); + migratedCollections += 1; + migratedInDataset += 1; + } + + if (migratedInDataset > 0 || defaultTag) migratedDatasets += 1; + } + + return { migratedDatasets, migratedCollections }; +}); diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index f3d23cfe2970..75313884f754 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -2,7 +2,7 @@ import type { NextApiRequest } from 'next'; import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema'; import { Types } from '@fastgpt/service/common/mongo'; -import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; @@ -100,7 +100,8 @@ async function handler(req: NextApiRequest) { }; } - const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([ + // aggregate 返回原始存储数据（tags 为存储格式），由 collectionTagsToTagLabel 解析后返回 + const [collections, total]: [DatasetCollectionSchemaType[], number] = await Promise.all([ MongoDatasetCollection.aggregate([ { $match: match diff --git a/projects/app/src/pages/api/core/dataset/collection/listV2.ts b/projects/app/src/pages/api/core/dataset/collection/listV2.ts index 76024de032f9..b85e903874ba 100644 --- a/projects/app/src/pages/api/core/dataset/collection/listV2.ts +++ b/projects/app/src/pages/api/core/dataset/collection/listV2.ts @@ -108,7 +108,11 @@ async function handler(req: ApiRequestProps): Promise> { +async function handler(req: ApiRequestProps) { const { datasetId, parentId, searchText, selectFolder, filterTags, simple } = parseApiInput({ req, bodySchema: ScrollCollectionsBodySchema @@ -91,6 +89,10 @@ async function handler( list: await Promise.all( collections.map(async (item) => ({ ...item, + tags: await collectionTagsToTagLabel({ + datasetId, + tags: item.tags + }), dataAmount: 0, trainingAmount: 0, ...defaultCollectionTrainingStatus, @@ -102,7 +104,8 @@ async function handler( }; } - const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([ + // aggregate 返回原始存储数据（tags 为存储格式），由 collectionTagsToTagLabel 解析为对外格式后返回 + const [collections, total]: [DatasetCollectionSchemaType[], number] = await Promise.all([ MongoDatasetCollection.aggregate([ { $match: match @@ -178,6 +181,10 @@ async function handler( const data = await Promise.all( collections.map(async (item) => ({ ...item, + tags: await collectionTagsToTagLabel({ + datasetId, + tags: item.tags + }), permission })) ); From fc8277ee81ed1c7d7dc7b72eb694ae2d20240e22 Mon Sep 17 00:00:00 2001 From: DigHuang <114602213+DigHuang@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:50:03 +0800 Subject: [PATCH 2/4] feat(dataset/tag): upgrade knowledge base tag management (#7619) * feat(dataset/tag): upgrade knowledge base tag management * feat(dataset/tag): support collection tag value settings Add typed collection tag value inputs and persistence for string, number, datetime, and array tags. Refine tag chip overflow rendering, table layout, migration handling, and localized copy. * feat(dataset/tag): improve collection tag management --- .agents/design/core/dataset/index.md | 8 +- ...76\350\256\241\346\226\207\346\241\243.md" | 46 +- packages/global/core/dataset/constants.ts | 23 + packages/global/core/dataset/tagUtils.ts | 23 + packages/global/core/dataset/type.ts | 47 +- .../openapi/core/dataset/collection/api.ts | 47 +- .../openapi/core/dataset/collection/index.ts | 27 +- .../openapi/core/dataset/collection/tagApi.ts | 38 +- .../global/openapi/core/dataset/tag/index.ts | 2 +- .../core/dataset/collection/tagFilter.ts | 89 ++ .../search/defaultRecall/collectionFilter.ts | 8 +- packages/service/core/dataset/tag/schemaV2.ts | 9 +- .../DateTimePicker/SingleDateTimePicker.tsx | 404 ++++++++ .../common/DateTimePicker/index.tsx | 11 +- .../web/components/common/Icon/constants.ts | 21 +- .../common/Icon/icons/common/calendar.svg | 8 + .../Icon/icons/common/checkSquareBroken.svg | 5 + .../icons/common/checkSquareBrokenPrimary.svg | 5 + .../common/checkSquareBrokenPrimaryHover.svg | 5 + .../components/common/Tabs/FillRowTabs.tsx | 4 +- .../common/TagFilter/MultiTagFilter.tsx | 369 +++++++ .../web/components/common/TagFilter/index.tsx | 7 + packages/web/i18n/en/common.json | 1 + packages/web/i18n/en/dataset.json | 53 +- packages/web/i18n/ko-KR/common.json | 1 + packages/web/i18n/ko-KR/dataset.json | 57 +- packages/web/i18n/zh-CN/common.json | 1 + packages/web/i18n/zh-CN/dataset.json | 53 +- packages/web/i18n/zh-Hant/common.json | 1 + packages/web/i18n/zh-Hant/dataset.json | 53 +- packages/web/styles/theme.ts | 10 + pro | 2 +- .../app/src/global/core/api/datasetReq.ts | 4 +- .../CollectionTagBatchModal.tsx | 573 +++++++++++ .../CollectionCard/CollectionTagSetModal.tsx | 183 ++++ .../CollectionCard/CollectionTagTable.tsx | 130 +++ .../dataset/detail/CollectionCard/Context.tsx | 21 +- .../CollectionCard/DatasetTagFilter.tsx | 89 ++ .../CollectionCard/EmptyCollectionTip.tsx | 23 + .../dataset/detail/CollectionCard/Header.tsx | 146 ++- .../CollectionCard/HeaderTagPopOver.tsx | 211 ---- .../detail/CollectionCard/TagCommon.tsx | 300 ++++++ .../detail/CollectionCard/TagManageModal.tsx | 927 ++++++++++-------- .../detail/CollectionCard/TagValueInputs.tsx | 679 +++++++++++++ .../detail/CollectionCard/TagsPopOver.tsx | 334 ++----- .../dataset/detail/CollectionCard/index.tsx | 691 ++++++++----- .../dataset/detail/CollectionCard/tagForm.ts | 85 ++ .../useAppendDatasetTagOption.ts | 36 + .../CollectionCard/useCollectionTagRows.ts | 76 ++ .../dataset/detail/DataCard.tsx | 4 +- .../app/src/pages/api/admin/migrateTags.ts | 17 +- .../pages/api/core/dataset/collection/list.ts | 195 ---- .../api/core/dataset/collection/listV2.ts | 9 +- .../api/core/dataset/collection/scrollList.ts | 199 ---- .../dataset/collection/tagFilterOptions.ts | 32 + .../src/web/core/dataset/api/collection.ts | 21 +- .../dataset/context/datasetPageContext.tsx | 87 +- .../collection/tagFilterOptions.test.ts | 151 +++ .../dataset/collection/trainingStatus.test.ts | 51 +- 59 files changed, 4850 insertions(+), 1862 deletions(-) create mode 100644 packages/global/core/dataset/tagUtils.ts create mode 100644 packages/service/core/dataset/collection/tagFilter.ts create mode 100644 packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx create mode 100644 packages/web/components/common/Icon/icons/common/calendar.svg create mode 100644 packages/web/components/common/Icon/icons/common/checkSquareBroken.svg create mode 100644 packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimary.svg create mode 100644 packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimaryHover.svg create mode 100644 packages/web/components/common/TagFilter/MultiTagFilter.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagSetModal.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx delete mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/tagForm.ts create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/useCollectionTagRows.ts delete mode 100644 projects/app/src/pages/api/core/dataset/collection/list.ts delete mode 100644 projects/app/src/pages/api/core/dataset/collection/scrollList.ts create mode 100644 projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts create mode 100644 projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts diff --git a/.agents/design/core/dataset/index.md b/.agents/design/core/dataset/index.md index 1998352d6c96..f2390705c795 100644 --- a/.agents/design/core/dataset/index.md +++ b/.agents/design/core/dataset/index.md @@ -811,7 +811,7 @@ POST /api/core/dataset/collection/create/link # 链接导入 POST /api/core/dataset/collection/create/text # 文本导入 POST /api/core/dataset/collection/create/images # 图片导入 PUT /api/core/dataset/collection/update # 更新集合 -GET /api/core/dataset/collection/list # 集合列表 +POST /api/core/dataset/collection/listV2 # 集合列表 GET /api/core/dataset/collection/detail # 集合详情 POST /api/core/dataset/collection/sync # 同步集合 GET /api/core/dataset/collection/export # 导出集合 @@ -893,9 +893,9 @@ MongoDatasetData.find(query, fields, { ### 3. 分页优化 ```typescript -// 使用 scrollList 而非传统分页 -// 避免深度分页性能问题 -GET /api/core/dataset/collection/scrollList?lastId=xxx&limit=20 +// 使用 offset 分页，避免深度 pageNum 分页性能问题 +POST /api/core/dataset/collection/listV2 +{ datasetId, offset, pageSize } ``` ### 4. 缓存策略 diff --git "a/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" index bb20093fb518..ebbed2b79b7c 100644 --- "a/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -155,7 +155,7 @@ flowchart TD > **契约层次**：Collection 创建/更新入参为 `(string | { tag, value })[]`（按标签名解析，与 listV2 返回格式一致）；持久化存储为 `{ tagId, value }[]`；fastgpt-pro 的标签值设置接口（`setCollectionTags`/`batchSetCollectionTags`）入参为 `{ tagId, value }[]`（按标签 ID 解析）。三者格式不同属有意设计，使用时勿混用。 -Collection 列表接口 `listV2` 和兼容接口 `scrollList` 使用 `collectionTagsToTagLabel` 将存储格式转换为： +Collection 列表接口 `listV2` 使用 `collectionTagsToTagLabel` 将存储格式转换为： ```text (string | { tag: string, value: string | number | string[] })[] @@ -262,12 +262,12 @@ sequenceDiagram 1. 校验请求参数及每个 Collection 的写权限。 2. 对输入标签按 `tagId` 去重并检测冲突。 -3. 对待覆盖的 `tagId` 执行 `$pull`。 -4. 使用 `$push` 将去重后的 `{ tagId, value }` 写入目标 Collection。 +3. 读出目标 Collection 的当前 tags，在内存中计算覆盖、累加或移除后的结果。 +4. 用 `bulkWrite` 一次性写回各 Collection 的 tags。 删除与写入操作均限定在请求的 `teamId`、`datasetId` 和 `collectionIds` 范围内。无匹配文档视为成功；数据库异常向上抛出，由 API 返回服务端错误。 -`addToCollections` 计算 `collectionIds - originCollectionIds` 得到添加集合，计算反向差集得到移除集合。添加时先 `$pull` 同一 `tagId`，再 `$push` 新值，确保一个 Collection 中同一标签只有一条记录。自动值规则为：`string` 使用空字符串，`array` 使用非空字符串数组；`number` 和 `datetime` 必须使用显式值接口。 +`addToCollections` 计算 `collectionIds - originCollectionIds` 得到添加集合，计算反向差集得到移除集合。添加与移除都先读出当前 tags，在内存中覆盖或去掉该 `tagId`，再用 `bulkWrite` 一次性写回，确保一个 Collection 中同一标签只有一条记录。自动值规则为：`string` 使用空字符串；`number`、`datetime`、`array` 无法通过此接口添加，须走显式值接口。 ### 5.4 标签检索过滤 @@ -472,7 +472,7 @@ DEFAULT_TAG = 'default_tag' ### 8.1 一致性 - v2 标签定义与 Collection 标签值通过 `tagId` 关联，应用层校验引用关系。 -- 写入 Collection 标签采用 `$pull` 后 `$push`，确保同一标签只有一条有效值。 +- 写入 Collection 标签先读出当前 tags，在内存中保证同一 `tagId` 只有一条，再用 `bulkWrite` 写回。 - dataset 物理删除按完整 root/child dataset ID 集合清理，防止子 dataset 标签残留。 - 迁移按 dataset 维度可重复执行，已转换 Collection 跳过。 @@ -517,7 +517,7 @@ DEFAULT_TAG = 'default_tag' 1. 创建 Collection 后 string 标签被保存为 `default_tag` 新格式。 2. 创建和更新四种类型标签值均能正确读写。 -3. `listV2`、`scrollList` 返回标签名称而非内部 tagId。 +3. `listV2` 返回标签名称而非内部 tagId。 4. 标签过滤与时间、Collection ID、文件夹递归条件正确求交集。 5. Agent V2 搜索可以透传 `collectionFilterMatch`。 6. 迁移保留已有新格式标签，旧标签名称正确合并且重复调用幂等。 @@ -562,8 +562,38 @@ DEFAULT_TAG = 'default_tag' | `packages/service/core/dataset/collection/utils.ts` | 标签创建、校验、存储和展示转换 | | `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | 标签过滤和旧格式改写 | | `projects/app/src/pages/api/core/dataset/collection/listV2.ts` | Collection 标签过滤列表接口 | -| `projects/app/src/pages/api/core/dataset/collection/scrollList.ts` | 兼容 Collection 列表接口 | | `projects/app/src/pages/api/admin/migrateTags.ts` | 历史标签迁移接口 | | `packages/service/core/dataset/delete/processor.ts` | dataset 物理删除和 v2 标签清理 | +| `packages/service/core/dataset/collection/tagFilter.ts` | 详情页标签值筛选：已用值聚合与列表查询条件 | +| `projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts` | 标签筛选选项接口 | +| `projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx` | 知识库详情页双栏标签筛选弹窗 | -本设计描述的是同一套标签能力的统一实现，不按迭代或变更批次拆分章节。 \ No newline at end of file +本设计描述的是同一套标签能力的统一实现，不按迭代或变更批次拆分章节。 + +## 12. 知识库详情页标签筛选弹窗 + +Figma：`07页面｜标签筛选`（node `2434:38727`），交互组件标注「两条独立滚动，勾选即生效」。 + +### 12.1 交互 + +1. 触发器默认 `标签 | 全部`；有选中值时展示第一项 `标签名：值`，其余以 `+n` 收起。 +2. 弹窗宽 320px，左右两列独立滚动。 +3. 左侧列出当前知识库全部标签定义；当前列高亮，该标签已选数量以圆形角标展示。 +4. 右侧第一行搜索当前标签的值；下方 checkbox 多选可筛值。左右列固定高度并独立滚动。 +5. 数字/日期/文本：展示当前文件上出现过的全部值。日期展示 `YYYY-MM-DD HH:mm`。 +6. 选项类：展示标签管理里的全部预设 options，并与文件上已用但不在预设里的值取并集。 +7. 勾选即写入列表查询条件，无单独确认按钮。底部展示「已选 N 项」和「清空」（无选中时禁用）。 +8. 右侧无值显示「暂无可筛选值」；搜索无匹配显示「未匹配到相关信息」。 +9. 有筛选条件且列表为空时，空态提供「清空筛选项」。 + +### 12.2 过滤语义 + +- 同一标签多个值：OR（命中任一值）。 +- 不同标签：AND（每个被筛选的标签都要命中）。 +- 选项类按数组包含匹配；文本/数字/日期按等值匹配。 +- `listV2` 只接受 `tagFilters`，按 `{ tagId, values }` 精确匹配；不再兼容旧版 `filterTags` 与字符串标签。 + +### 12.3 接口 + +- `GET /api/core/dataset/collection/tagFilterOptions?datasetId=`：只返回 `{ tagId, values }`（已用值）。标签名和类型复用 `getAllTags`。 +- `POST /api/core/dataset/collection/listV2` 新增 `tagFilters: [{ tagId, values }]`。 \ No newline at end of file diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 04a098adde41..37aac4e45933 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -164,6 +164,29 @@ export const DatasetCollectionSyncResultMap = { } }; +/* ------------ collection tags -------------- */ +export enum DatasetCollectionTagTypeEnum { + string = 'string', + number = 'number', + datetime = 'datetime', + array = 'array' +} + +export const DatasetCollectionTagTypeMap = { + [DatasetCollectionTagTypeEnum.string]: { + label: i18nT('dataset:core.dataset.tags.string') + }, + [DatasetCollectionTagTypeEnum.number]: { + label: i18nT('dataset:core.dataset.tags.number') + }, + [DatasetCollectionTagTypeEnum.datetime]: { + label: i18nT('dataset:core.dataset.tags.date') + }, + [DatasetCollectionTagTypeEnum.array]: { + label: i18nT('dataset:core.dataset.tags.array') + } +}; + export enum DatasetCollectionDataProcessModeEnum { chunk = 'chunk', qa = 'qa', diff --git a/packages/global/core/dataset/tagUtils.ts b/packages/global/core/dataset/tagUtils.ts new file mode 100644 index 000000000000..ed2cb9c9d798 --- /dev/null +++ b/packages/global/core/dataset/tagUtils.ts @@ -0,0 +1,23 @@ +import type { CollectionTagValueType } from './type'; + +/** 标签值去重/列表 key：区分 number 2 与 string "2"。 */ +export const collectionTagValueKey = (value: string | number) => + typeof value === 'number' ? `n:${value}` : `s:${value}`; + +/** 数字按大小、其余按 zh-CN 字典序。前后端筛选项展示共用。 */ +export const sortCollectionTagValues = (values: T[]): T[] => + [...values].sort((a, b) => { + if (typeof a === 'number' && typeof b === 'number') return a - b; + return String(a).localeCompare(String(b), 'zh-CN'); + }); + +/** 筛选项只保留非空字符串和有限数字。 */ +export const isUsableCollectionTagFilterValue = (value: unknown): value is string | number => { + if (typeof value === 'string') return value.length > 0; + if (typeof value === 'number') return Number.isFinite(value); + return false; +}; + +/** 新格式 { tagId, value }；旧字符串标签排除。 */ +export const isCollectionTagValue = (item: unknown): item is CollectionTagValueType => + !!item && typeof item === 'object' && !Array.isArray(item) && 'tagId' in item && 'value' in item; diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index 6cf8fc41ad10..b7488029f6c1 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -10,7 +10,9 @@ import { CollectionTrainingStatusEnum, ChunkSettingModeEnum, ChunkTriggerConfigTypeEnum, - ParagraphChunkAIModeEnum + ParagraphChunkAIModeEnum, + DatasetCollectionTagTypeEnum, + DatasetCollectionTagTypeMap } from './constants'; import { ApiDatasetServerSchema, @@ -29,13 +31,17 @@ import { NumSchema } from '../../common/zod'; /* ===== Tag Type ===== */ /** 标签类型枚举 */ -export const DatasetCollectionTagTypeEnum = z.enum([ - 'string', - 'number', - 'datetime', - 'array' -] as const); -export type DatasetCollectionTagType = z.infer; +export { DatasetCollectionTagTypeEnum, DatasetCollectionTagTypeMap }; +export type DatasetCollectionTagType = `${DatasetCollectionTagTypeEnum}`; + +/** 选项类标签预设 options 上限，Zod 与写入合并共用。 */ +export const DATASET_COLLECTION_TAG_OPTIONS_MAX = 64; + +/** 选项类标签的预设选项，空选项由前端草稿过滤后再提交。 */ +export const DatasetCollectionTagOptionsSchema = z + .array(z.string().trim().min(1)) + .max(DATASET_COLLECTION_TAG_OPTIONS_MAX) + .meta({ description: '选项类标签的预设选项' }); /** 迁移常量：新建 array 标签记录的 tag 字段固定值，亦是旧格式过滤改写的条件 key */ export const DEFAULT_TAG = 'default_tag'; @@ -53,6 +59,23 @@ export const CollectionTagValueSchema = z.object({ }); export type CollectionTagValueType = z.infer; +/** 详情页按标签值筛选的单条条件。同一标签多值为 OR，不同标签由调用方做 AND。 */ +export const CollectionTagFilterItemSchema = z.object({ + tagId: z.string().meta({ + example: '68ad85a7463006c963799a05', + description: '标签 ID' + }), + values: z + .array(z.union([z.string(), z.number()])) + .min(1) + .meta({ + example: ['PRD'], + description: + '选中的标签值。文本/选项为字符串，数字为数值，日期为 UTC 毫秒时间戳。同一标签多值为 OR' + }) +}); +export type CollectionTagFilterItem = z.infer; + /* ===== Chunk ===== */ export const ChunkSettingsSchema = z.object({ trainingType: z @@ -195,9 +218,10 @@ export const DatasetCollectionTagsSchema = z.object({ teamId: ObjectIdSchema.meta({ description: '团队 ID' }), datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), tag: z.string().meta({ description: '标签' }), - tagType: DatasetCollectionTagTypeEnum.default('string').meta({ + tagType: z.enum(DatasetCollectionTagTypeEnum).default(DatasetCollectionTagTypeEnum.string).meta({ description: '标签类型：string(默认)/number/datetime/array' }), + options: DatasetCollectionTagOptionsSchema.optional(), fromMigration: z.boolean().optional().meta({ description: '标识该记录是旧标签迁移/旧格式输入创建的 default_tag 承载记录' }) @@ -389,9 +413,10 @@ export type DatasetItemType = z.infer; export const DatasetTagSchema = z.object({ _id: ObjectIdSchema.meta({ description: '标签 ID' }), tag: z.string().meta({ description: '标签' }), - tagType: DatasetCollectionTagTypeEnum.default('string').meta({ + tagType: z.enum(DatasetCollectionTagTypeEnum).default(DatasetCollectionTagTypeEnum.string).meta({ description: '标签类型：string(默认)/number/datetime/array' - }) + }), + options: DatasetCollectionTagOptionsSchema.optional() }); export type DatasetTagType = z.infer; diff --git a/packages/global/openapi/core/dataset/collection/api.ts b/packages/global/openapi/core/dataset/collection/api.ts index 290c27136db3..a1cfdc49b907 100644 --- a/packages/global/openapi/core/dataset/collection/api.ts +++ b/packages/global/openapi/core/dataset/collection/api.ts @@ -10,6 +10,7 @@ import { TrainingModeEnum } from '../../../../core/dataset/constants'; import { + CollectionTagFilterItemSchema, CollectionTrainingStatusSchema, DatasetCollectionItemSchema, DatasetCollectionSchema @@ -23,20 +24,6 @@ import { transformOptionalChatAuthTargetInput } from '../../chat/api'; -// ============= Scroll Collections ============= -/** - * @deprecated Use ListCollectionV2BodySchema and /core/dataset/collection/listV2 instead. - */ -export const ScrollCollectionsBodySchema = z.object({ - datasetId: z.string(), - parentId: z.string().nullable().optional().default(null), - searchText: z.string().optional().default(''), - selectFolder: z.boolean().optional().default(false), - filterTags: z.array(z.string()).optional().default([]), - simple: z.boolean().optional().default(false) -}); -export type ScrollCollectionsBodyType = z.infer; - // ============= Update Collection ============= export const UpdateDatasetCollectionBodySchema = z.object({ id: ObjectIdSchema.optional().describe('集合ID，与 datasetId+externalFileId 二选一'), @@ -163,7 +150,9 @@ export const ListCollectionV2BodySchema = PaginationSchema.extend({ parentId: z.string().nullable().optional().default(null).meta({ description: '父级目录 ID' }), searchText: z.string().max(100).optional().default('').meta({ description: '搜索文本' }), selectFolder: z.boolean().optional().default(false).meta({ description: '只返回文件夹' }), - filterTags: z.array(z.string()).optional().default([]).meta({ description: '过滤标签' }), + tagFilters: z.array(CollectionTagFilterItemSchema).optional().default([]).meta({ + description: '按标签值过滤。同一标签多值为 OR，不同标签为 AND' + }), simple: z.boolean().optional().default(false).meta({ description: '简单模式（不统计数量）' }) }); export type ListCollectionV2BodyType = z.infer; @@ -275,3 +264,31 @@ export const SyncCollectionResponseSchema = z.enum(DatasetCollectionSyncResultEn description: '同步结果' }); export type SyncCollectionResponseType = z.infer; + +/* ============================================================================ + * API: 获取知识库标签筛选项 + * Route: GET /api/core/dataset/collection/tagFilterOptions + * Method: GET + * Description: 获取知识库下当前已被文件使用的标签值，供详情页双栏筛选使用。标签名和类型由 getAllTags 提供。 + * Tags: ['Dataset'] + * ============================================================================ */ +export const GetTagFilterOptionsQuerySchema = z.object({ + datasetId: z.string().meta({ + example: '68ad85a7463006c963799a05', + description: '数据集 ID' + }) +}); +export type GetTagFilterOptionsQueryType = z.infer; + +export const TagFilterOptionItemSchema = z.object({ + tagId: z.string().meta({ example: '68ad85a7463006c963799a05', description: '标签 ID' }), + values: z + .array(z.union([z.string(), z.number()])) + .meta({ example: ['PRD'], description: '当前已被文件使用的标签值' }) +}); +export type TagFilterOptionItemType = z.infer; + +export const GetTagFilterOptionsResponseSchema = z.object({ + list: z.array(TagFilterOptionItemSchema).meta({ description: '已用标签值列表' }) +}); +export type GetTagFilterOptionsResponseType = z.infer; diff --git a/packages/global/openapi/core/dataset/collection/index.ts b/packages/global/openapi/core/dataset/collection/index.ts index 00ab9ecd4051..6aec7dfa9748 100644 --- a/packages/global/openapi/core/dataset/collection/index.ts +++ b/packages/global/openapi/core/dataset/collection/index.ts @@ -10,8 +10,9 @@ import { GetCollectionTrainingDetailQuerySchema, GetCollectionTrainingDetailResponseSchema, ListCollectionV2BodySchema, + GetTagFilterOptionsQuerySchema, + GetTagFilterOptionsResponseSchema, ReadCollectionSourceBodyRawSchema, - ScrollCollectionsBodySchema, SyncCollectionBodySchema, UpdateDatasetCollectionBodySchema } from './api'; @@ -75,22 +76,22 @@ export const DatasetCollectionPath: OpenAPIPath = { } } }, - '/core/dataset/collection/scrollList': { - post: { - summary: '获取数据集集合列表（滚动分页）', - description: '已废弃：获取数据集集合列表（滚动分页）。请改用 /core/dataset/collection/listV2', - deprecated: true, + '/core/dataset/collection/tagFilterOptions': { + get: { + summary: '获取知识库标签筛选项', + description: '获取知识库下当前已被文件使用的标签值', tags: [DevApiTagsMap.datasetCollection], - requestBody: { - content: { - 'application/json': { - schema: ScrollCollectionsBodySchema - } - } + requestParams: { + query: GetTagFilterOptionsQuerySchema }, responses: { 200: { - description: '成功返回集合列表' + description: '成功返回标签及已用值列表', + content: { + 'application/json': { + schema: GetTagFilterOptionsResponseSchema + } + } } } } diff --git a/packages/global/openapi/core/dataset/collection/tagApi.ts b/packages/global/openapi/core/dataset/collection/tagApi.ts index cd7afa1acf77..6e7063120252 100644 --- a/packages/global/openapi/core/dataset/collection/tagApi.ts +++ b/packages/global/openapi/core/dataset/collection/tagApi.ts @@ -1,9 +1,20 @@ import z from 'zod'; import { DatasetCollectionTagTypeEnum, - CollectionTagValueSchema + CollectionTagValueSchema, + CollectionTagValueFieldSchema, + DatasetCollectionTagOptionsSchema } from '../../../../core/dataset/type'; +export const BatchCollectionTagModeEnum = { + add: 'add', + remove: 'remove' +} as const; +export const BatchCollectionTagModeSchema = z.enum([ + BatchCollectionTagModeEnum.add, + BatchCollectionTagModeEnum.remove +]); + /* ============================================================================ * API: 创建集合标签 * Route: POST /proApi/core/dataset/tag/create @@ -11,7 +22,7 @@ import { export const CreateDatasetCollectionTagBodySchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), tag: z.string().trim().min(1).meta({ description: '标签名称' }), - tagType: DatasetCollectionTagTypeEnum.optional().meta({ + tagType: z.enum(DatasetCollectionTagTypeEnum).optional().meta({ description: '标签类型：string(默认)/number/datetime/array' }) }); @@ -39,7 +50,10 @@ export type AddTagsToCollectionsParams = z.infer; @@ -68,7 +82,7 @@ export type GetAllDatasetTagsQuery = z.infer; + export const BatchSetCollectionTagsBodySchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), collectionIds: z.array(z.string()).min(1).meta({ description: '集合 ID 列表' }), - tags: z.array(CollectionTagValueSchema).meta({ description: '标签值列表' }) + mode: BatchCollectionTagModeSchema.default(BatchCollectionTagModeEnum.add).meta({ + description: 'add：批量添加/覆盖标签；remove：批量移除标签或选项值' + }), + tags: z.array(BatchSetCollectionTagItemSchema).meta({ description: '标签操作列表' }) }); export type BatchSetCollectionTagsParams = z.infer; diff --git a/packages/global/openapi/core/dataset/tag/index.ts b/packages/global/openapi/core/dataset/tag/index.ts index 53324fe466cf..ddddda9a522e 100644 --- a/packages/global/openapi/core/dataset/tag/index.ts +++ b/packages/global/openapi/core/dataset/tag/index.ts @@ -121,7 +121,7 @@ export const DatasetTagPath: OpenAPIPath = { '/proApi/core/dataset/tag/batchSetCollectionTags': { post: { summary: '批量设置集合标签值', - description: '为多个集合批量设置标签值', + description: '为多个集合批量添加、覆盖或移除标签值；number 类型支持累加', tags: [DevApiTagsMap.datasetTag], requestBody: { content: { diff --git a/packages/service/core/dataset/collection/tagFilter.ts b/packages/service/core/dataset/collection/tagFilter.ts new file mode 100644 index 000000000000..45e209eef64f --- /dev/null +++ b/packages/service/core/dataset/collection/tagFilter.ts @@ -0,0 +1,89 @@ +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; +import { + collectionTagValueKey, + isCollectionTagValue, + isUsableCollectionTagFilterValue, + sortCollectionTagValues +} from '@fastgpt/global/core/dataset/tagUtils'; +import type { TagFilterOptionItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { Types } from '../../../common/mongo'; +import { MongoDatasetCollection } from './schema'; + +/** + * 从 Collection 上聚合每个 tagId「当前已被使用」的值。 + * 不返回标签名/类型，由调用方用已有标签定义拼接；选项类只保留实际出现过的选项。 + */ +export const collectUsedTagValues = ( + collections: Array<{ tags?: unknown[] }> +): TagFilterOptionItemType[] => { + const valuesByTagId = new Map>(); + + const addValue = (tagId: string, value: unknown) => { + if (!isUsableCollectionTagFilterValue(value)) return; + const current = valuesByTagId.get(tagId) ?? new Map(); + current.set(collectionTagValueKey(value), value); + valuesByTagId.set(tagId, current); + }; + + for (const collection of collections) { + for (const item of collection.tags ?? []) { + if (!isCollectionTagValue(item)) continue; + + const tagId = String(item.tagId); + if (Array.isArray(item.value)) { + for (const value of item.value) addValue(tagId, value); + continue; + } + addValue(tagId, item.value); + } + } + + return [...valuesByTagId.entries()] + .map(([tagId, values]) => ({ + tagId, + values: sortCollectionTagValues(Array.from(values.values())) + })) + .sort((a, b) => a.tagId.localeCompare(b.tagId)); +}; + +/** + * 按 tagFilters 组装 Collection 列表的标签过滤条件。 + * 同一标签多值为 OR（value $in），不同标签为 AND。 + */ +export const buildCollectionListTagMatch = (tagFilters: CollectionTagFilterItem[] = []) => { + if (tagFilters.length === 0) return {}; + + const conditions = tagFilters.map((filter) => ({ + tags: { + $elemMatch: { + tagId: filter.tagId, + value: { $in: filter.values } + } + } + })); + + if (conditions.length === 1) return conditions[0]; + return { $and: conditions }; +}; + +/** + * 聚合当前知识库 Collection 上已被使用的标签值，不含标签定义。 + */ +export const getDatasetTagFilterOptions = async ({ + teamId, + datasetId +}: { + teamId: string; + datasetId: string; +}): Promise => { + // 筛选弹窗打开后立刻读，走主库避免刚写入的标签值被从库滞后挡住 + const collections = await MongoDatasetCollection.find( + { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(datasetId) + }, + 'tags' + ).lean(); + + return collectUsedTagValues(collections); +}; diff --git a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts index aef7b095e3b6..cd8d2b233e61 100644 --- a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts +++ b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts @@ -3,6 +3,7 @@ import safeRegex from 'safe-regex'; import { MongoDatasetCollection } from '../../collection/schema'; import { MongoDatasetCollectionTagsV2 } from '../../tag/schemaV2'; import { DEFAULT_TAG } from '@fastgpt/global/core/dataset/type'; +import { isCollectionTagValue } from '@fastgpt/global/core/dataset/tagUtils'; import { readFromSecondary } from '../../../../common/mongo/utils'; import { computeFilterIntersection } from '../utils'; @@ -284,12 +285,7 @@ export async function filterCollectionByKeyValueTags({ // 5. Application-layer value comparison for (const col of collections) { - const tagsArr = ( - (col.tags || []) as Array<{ tagId?: string; value?: string | number | string[] } | string> - ).filter( - (t): t is { tagId: string; value?: string | number | string[] } => - typeof t === 'object' && t !== null && Boolean(t.tagId) - ); + const tagsArr = (col.tags || []).filter(isCollectionTagValue); // AND: all must pass const andOk = ($and || []).every((cond) => matchCondition(cond, tagMap, tagsArr)); diff --git a/packages/service/core/dataset/tag/schemaV2.ts b/packages/service/core/dataset/tag/schemaV2.ts index 721ee27bccc9..e025626da985 100644 --- a/packages/service/core/dataset/tag/schemaV2.ts +++ b/packages/service/core/dataset/tag/schemaV2.ts @@ -1,7 +1,8 @@ import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; -import { defineIndex, connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; +import { defineIndex, connectionMongo, getMongoModel } from '../../../common/mongo'; import { DatasetCollectionName } from '../schema'; import { type DatasetCollectionTagsSchemaType } from '@fastgpt/global/core/dataset/type'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; const { Schema } = connectionMongo; export const DatasetCollectionTagsV2Name = 'dataset_collection_tags_v2'; @@ -25,9 +26,13 @@ const DatasetCollectionTagsV2Schema = new Schema({ tagType: { type: String, default: 'string', - enum: ['string', 'number', 'datetime', 'array'] + enum: Object.values(DatasetCollectionTagTypeEnum) // COMMENT: 标签类型。string=字符串比较, number=数值比较, datetime=时间戳比较, array=字符串数组集合比较(V2.0) }, + options: { + type: [String] + // COMMENT: array 类型标签的预设选项。集合写入选项值时合并新增项，标签管理可覆盖删除 + }, fromMigration: { type: Boolean, default: false diff --git a/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx new file mode 100644 index 000000000000..b54897427e12 --- /dev/null +++ b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx @@ -0,0 +1,404 @@ +import React, { useState } from 'react'; +import type { FlexProps } from '@chakra-ui/react'; +import { + Box, + Button, + Flex, + HStack, + Input, + Popover, + PopoverContent, + PopoverTrigger, + Portal, + useDisclosure +} from '@chakra-ui/react'; +import { DayPicker } from 'react-day-picker'; +import 'react-day-picker/dist/style.css'; +import { zhCN } from 'date-fns/locale/zh-CN'; +import type { Locale } from 'date-fns'; +import { addMonths, format, isValid, parse } from 'date-fns'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '../Icon'; + +const DATE_INPUT_FORMAT = 'yyyy-MM-dd'; +const TIME_INPUT_FORMAT = 'HH:mm'; + +export type SingleDateTimePickerProps = { + value?: Date | number; + onChange?: (val: number) => void; + placeholder?: string; + isDisabled?: boolean; + locale?: Locale; +} & Omit; + +/** 将时间戳或 Date 转为有效 Date；空值或非法日期返回 undefined。 */ +const toValidDate = (value?: Date | number) => { + if (value == null) return undefined; + const date = typeof value === 'number' ? new Date(value) : value; + return isValid(date) ? date : undefined; +}; + +/** 仅在输入完整且合法的 yyyy-MM-dd 时解析为 Date，避免半成品输入把日历跳走。 */ +const parseYmdInput = (text: string) => { + if (text.length !== 10) return undefined; + const parsed = parse(text, DATE_INPUT_FORMAT, new Date()); + return isValid(parsed) ? parsed : undefined; +}; + +/** + * 单日期时间选择器：基于 react-day-picker 二次封装。 + * 打开弹窗时用当前 value 初始化草稿；确认时把日期输入与时间输入合并成时间戳。 + */ +export const SingleDateTimePicker = ({ + value, + onChange, + placeholder, + isDisabled, + locale = zhCN, + ...triggerProps +}: SingleDateTimePickerProps) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + + const selectedDate = toValidDate(value); + const [tempDate, setTempDate] = useState(() => selectedDate ?? new Date()); + const [tempMonth, setTempMonth] = useState(() => selectedDate ?? new Date()); + const [dateInputText, setDateInputText] = useState(() => + format(selectedDate ?? new Date(), DATE_INPUT_FORMAT) + ); + const [timeInputText, setTimeInputText] = useState(() => + format(selectedDate ?? new Date(), TIME_INPUT_FORMAT) + ); + + const displayText = selectedDate + ? format(selectedDate, `${DATE_INPUT_FORMAT} ${TIME_INPUT_FORMAT}`) + : ''; + + const applyDraftDate = (date: Date) => { + setTempDate(date); + setTempMonth(date); + setDateInputText(format(date, DATE_INPUT_FORMAT)); + }; + + const handleOpen = () => { + if (isDisabled) return; + const date = toValidDate(value) ?? new Date(); + applyDraftDate(date); + setTimeInputText(format(date, TIME_INPUT_FORMAT)); + onOpen(); + }; + + const handleConfirm = () => { + const result = new Date(parseYmdInput(dateInputText) ?? tempDate); + const [hours = '0', minutes = '0'] = timeInputText.split(':'); + result.setHours(Number(hours), Number(minutes), 0, 0); + onChange?.(result.getTime()); + onClose(); + }; + + return ( + + + + + {displayText || placeholder || t('common:datetime_picker.placeholder')} + + + + + + + + + setTempMonth((prev) => addMonths(prev, -1))} + > + + + + + {format(tempMonth, 'LLLL yyyy', { locale })} + + + setTempMonth((prev) => addMonths(prev, 1))} + > + + + + + + { + if (date) applyDraftDate(date); + }} + components={{ + Nav: () => <>, + MonthCaption: () => <> + }} + formatters={{ + formatWeekdayName: (date) => format(date, 'EEEEEE', { locale }) + }} + /> + + + + + { + const text = e.target.value; + setDateInputText(text); + const parsed = parseYmdInput(text); + if (parsed) { + setTempDate(parsed); + setTempMonth(parsed); + } + }} + _focus={{ borderColor: 'primary.600', boxShadow: 'focus' }} + /> + setTimeInputText(e.target.value)} + _focus={{ borderColor: 'primary.600', boxShadow: 'focus' }} + /> + + + + + + + + + + + ); +}; + +export default SingleDateTimePicker; diff --git a/packages/web/components/common/DateTimePicker/index.tsx b/packages/web/components/common/DateTimePicker/index.tsx index d84811030afc..22907e2279a8 100644 --- a/packages/web/components/common/DateTimePicker/index.tsx +++ b/packages/web/components/common/DateTimePicker/index.tsx @@ -7,6 +7,8 @@ import 'react-day-picker/dist/style.css'; import { zhCN } from 'date-fns/locale/zh-CN'; import MyIcon from '../Icon'; +export * from './SingleDateTimePicker'; + const DateTimePicker = ({ onChange, popPosition = 'bottom', @@ -25,16 +27,13 @@ const DateTimePicker = ({ } & Omit) => { const containerRef = useRef(null); const popoverRef = useRef(null); - const [selectedDate, setSelectedDate] = useState( + const [internalSelectedDate, setInternalSelectedDate] = useState( selectedDateTime || defaultDate ); + const selectedDate = selectedDateTime ?? internalSelectedDate; const [showSelected, setShowSelected] = useState(false); const [position, setPosition] = useState({ top: 0, left: 0 }); - useEffect(() => { - setSelectedDate(selectedDateTime); - }, [selectedDateTime]); - useEffect(() => { if (!showSelected) return; const updatePosition = () => { @@ -169,7 +168,7 @@ const DateTimePicker = ({ selected={selectedDate} disabled={disabled} onSelect={(date) => { - setSelectedDate(date); + setInternalSelectedDate(date); onChange?.(date); setShowSelected(false); }} diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 90cae73dd5ab..9ba4f7ab53ee 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -24,7 +24,12 @@ export const iconPaths = { 'common/backFill': () => import('./icons/common/backFill.svg'), 'common/backLight': () => import('./icons/common/backLight.svg'), 'common/billing': () => import('./icons/common/billing.svg'), + 'common/calendar': () => import('./icons/common/calendar.svg'), 'common/check': () => import('./icons/common/check.svg'), + 'common/checkSquareBroken': () => import('./icons/common/checkSquareBroken.svg'), + 'common/checkSquareBrokenPrimary': () => import('./icons/common/checkSquareBrokenPrimary.svg'), + 'common/checkSquareBrokenPrimaryHover': () => + import('./icons/common/checkSquareBrokenPrimaryHover.svg'), 'common/clearLight': () => import('./icons/common/clearLight.svg'), 'common/closeLight': () => import('./icons/common/closeLight.svg'), 'common/confirm/commonTip': () => import('./icons/common/confirm/commonTip.svg'), @@ -39,16 +44,17 @@ export const iconPaths = { 'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'), 'common/download': () => import('./icons/common/download.svg'), 'common/downloadLine': () => import('./icons/common/downloadLine.svg'), + 'common/edit-filled': () => import('./icons/common/edit-filled.svg'), 'common/edit': () => import('./icons/common/edit.svg'), 'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'), 'common/ellipsis': () => import('./icons/common/ellipsis.svg'), 'common/enable': () => import('./icons/common/enable.svg'), - 'common/filter': () => import('./icons/common/filter.svg'), 'common/error': () => import('./icons/common/error.svg'), 'common/errorFill': () => import('./icons/common/errorFill.svg'), 'common/exclamationMark': () => import('./icons/common/exclamationMark.svg'), 'common/file/move': () => import('./icons/common/file/move.svg'), 'common/fileNotFound': () => import('./icons/common/fileNotFound.svg'), + 'common/filter': () => import('./icons/common/filter.svg'), 'common/first_page': () => import('./icons/common/first_page.svg'), 'common/folderFill': () => import('./icons/common/folderFill.svg'), 'common/folderImport': () => import('./icons/common/folderImport.svg'), @@ -133,8 +139,6 @@ export const iconPaths = { 'core/app/inputGuides': () => import('./icons/core/app/inputGuides.svg'), 'core/app/logsLight': () => import('./icons/core/app/logsLight.svg'), 'core/app/markLight': () => import('./icons/core/app/markLight.svg'), - 'core/app/workflowToolbarAdd': () => import('./icons/core/app/workflowToolbarAdd.svg'), - 'core/app/workflowToolbarSearch': () => import('./icons/core/app/workflowToolbarSearch.svg'), 'core/app/publish/lark': () => import('./icons/core/app/publish/lark.svg'), 'core/app/publish/offiaccount': () => import('./icons/core/app/publish/offiaccount.svg'), 'core/app/publish/wechat': () => import('./icons/core/app/publish/wechat.svg'), @@ -189,6 +193,8 @@ export const iconPaths = { 'core/app/variable/input': () => import('./icons/core/app/variable/input.svg'), 'core/app/workflow/checkPendingImprove': () => import('./icons/core/app/workflow/checkPendingImprove.svg'), + 'core/app/workflowToolbarAdd': () => import('./icons/core/app/workflowToolbarAdd.svg'), + 'core/app/workflowToolbarSearch': () => import('./icons/core/app/workflowToolbarSearch.svg'), 'core/chat/QGFill': () => import('./icons/core/chat/QGFill.svg'), 'core/chat/backText': () => import('./icons/core/chat/backText.svg'), 'core/chat/chatFill': () => import('./icons/core/chat/chatFill.svg'), @@ -198,14 +204,14 @@ export const iconPaths = { 'core/chat/chevronRight': () => import('./icons/core/chat/chevronRight.svg'), 'core/chat/chevronSelector': () => import('./icons/core/chat/chevronSelector.svg'), 'core/chat/chevronUp': () => import('./icons/core/chat/chevronUp.svg'), - 'core/chat/feedback/badLight': () => import('./icons/core/chat/feedback/badLight.svg'), - 'core/chat/feedback/goodLight': () => import('./icons/core/chat/feedback/goodLight.svg'), 'core/chat/deepThinking': () => import('./icons/core/chat/deepThinking.svg'), 'core/chat/dotsHorizontal': () => import('./icons/core/chat/dotsHorizontal.svg'), + 'core/chat/feedback/badLight': () => import('./icons/core/chat/feedback/badLight.svg'), + 'core/chat/feedback/goodLight': () => import('./icons/core/chat/feedback/goodLight.svg'), 'core/chat/fileDownload': () => import('./icons/core/chat/fileDownload.svg'), 'core/chat/fileSelect': () => import('./icons/core/chat/fileSelect.svg'), - 'core/chat/monitor': () => import('./icons/core/chat/monitor.svg'), 'core/chat/markdown': () => import('./icons/core/chat/markdown.svg'), + 'core/chat/monitor': () => import('./icons/core/chat/monitor.svg'), 'core/chat/quoteFill': () => import('./icons/core/chat/quoteFill.svg'), 'core/chat/recordFill': () => import('./icons/core/chat/recordFill.svg'), 'core/chat/sendFill': () => import('./icons/core/chat/sendFill.svg'), @@ -266,9 +272,9 @@ export const iconPaths = { 'core/workflow/debugResult': () => import('./icons/core/workflow/debugResult.svg'), 'core/workflow/edgeArrow': () => import('./icons/core/workflow/edgeArrow.svg'), 'core/workflow/edgeArrowBold': () => import('./icons/core/workflow/edgeArrowBold.svg'), - 'core/workflow/inputType/array': () => import('./icons/core/workflow/inputType/array.svg'), 'core/workflow/inputType/agentGenerated': () => import('./icons/core/workflow/inputType/agentGenerated.svg'), + 'core/workflow/inputType/array': () => import('./icons/core/workflow/inputType/array.svg'), 'core/workflow/inputType/conditional': () => import('./icons/core/workflow/inputType/conditional.svg'), 'core/workflow/inputType/customVariable': () => @@ -430,7 +436,6 @@ export const iconPaths = { empty: () => import('./icons/empty.svg'), export: () => import('./icons/export.svg'), feedback: () => import('./icons/feedback.svg'), - 'common/edit-filled': () => import('./icons/common/edit-filled.svg'), 'file/fill/audio': () => import('./icons/file/fill/audio.svg'), 'file/fill/csv': () => import('./icons/file/fill/csv.svg'), 'file/fill/doc': () => import('./icons/file/fill/doc.svg'), diff --git a/packages/web/components/common/Icon/icons/common/calendar.svg b/packages/web/components/common/Icon/icons/common/calendar.svg new file mode 100644 index 000000000000..849d9c703730 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/calendar.svg @@ -0,0 +1,8 @@ + + + diff --git a/packages/web/components/common/Icon/icons/common/checkSquareBroken.svg b/packages/web/components/common/Icon/icons/common/checkSquareBroken.svg new file mode 100644 index 000000000000..c03255c8ccff --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/checkSquareBroken.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimary.svg b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimary.svg new file mode 100644 index 000000000000..befa75a5b5d6 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimary.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimaryHover.svg b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimaryHover.svg new file mode 100644 index 000000000000..d1e620884bad --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimaryHover.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Tabs/FillRowTabs.tsx b/packages/web/components/common/Tabs/FillRowTabs.tsx index ebec2ada9dce..2db032fa6395 100644 --- a/packages/web/components/common/Tabs/FillRowTabs.tsx +++ b/packages/web/components/common/Tabs/FillRowTabs.tsx @@ -141,12 +141,12 @@ const FillRowTabs = ( ? { bg: 'white', boxShadow: '1.5', - color: 'primary.600' + color: 'primary.700' } : { color: 'myGray.500', _hover: { - color: 'primary.600' + color: 'primary.700' }, onClick: () => { if (scrollPositionKey && scrollContainerRef.current) { diff --git a/packages/web/components/common/TagFilter/MultiTagFilter.tsx b/packages/web/components/common/TagFilter/MultiTagFilter.tsx new file mode 100644 index 000000000000..4dbcd28165c2 --- /dev/null +++ b/packages/web/components/common/TagFilter/MultiTagFilter.tsx @@ -0,0 +1,369 @@ +import React, { useMemo, useState, type ReactNode } from 'react'; +import { Box, Button, Checkbox, Flex, Input, type PlacementWithLogical } from '@chakra-ui/react'; +import { collectionTagValueKey } from '@fastgpt/global/core/dataset/tagUtils'; +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; +import MyIcon from '../Icon'; +import MyBox from '../MyBox'; +import MyPopover from '../MyPopover'; +import FilterButton from './FilterButton'; + +export type MultiTagFilterValue = string | number; + +export type MultiTagFilterGroup = { + tagId: string; + label: string; + values: Array<{ + value: MultiTagFilterValue; + label: string; + }>; +}; + +export type MultiTagFilterLabels = { + title: ReactNode; + all: ReactNode; + searchPlaceholder: string; + selected: ReactNode; + item: ReactNode; + clear: ReactNode; + noValues: ReactNode; + noMatch: ReactNode; +}; + +export type MultiTagFilterProps = { + groups: MultiTagFilterGroup[]; + selected: CollectionTagFilterItem[]; + onSelectedChange: (next: CollectionTagFilterItem[]) => void; + isLoading?: boolean; + isLoadingValues?: boolean; + labels: MultiTagFilterLabels; + placement?: PlacementWithLogical; + offset?: [number, number]; + onOpen?: () => void; +}; + +const FILTER_LIST_H = '168px'; +const filterListScrollSx = { + overscrollBehavior: 'contain', + scrollbarWidth: 'thin', + scrollbarColor: 'var(--chakra-colors-myGray-200) transparent', + '&::-webkit-scrollbar': { w: '4px' }, + '&::-webkit-scrollbar-thumb': { + bg: 'myGray.200', + borderRadius: 'full' + } +}; +const stopWheelPropagation = (e: React.WheelEvent) => e.stopPropagation(); + +/** + * 双栏标签值筛选：左侧分组、右侧勾选值即生效。同一标签多值为 OR，不同标签由调用方按 AND 解释。 + */ +export const toggleMultiTagFilterValue = ( + selected: CollectionTagFilterItem[], + tagId: string, + value: MultiTagFilterValue +): CollectionTagFilterItem[] => { + const current = selected.find((item) => item.tagId === tagId); + if (!current) { + return [...selected, { tagId, values: [value] }]; + } + + const exists = current.values.some((item) => item === value); + const nextValues = exists + ? current.values.filter((item) => item !== value) + : [...current.values, value]; + + if (nextValues.length === 0) { + return selected.filter((item) => item.tagId !== tagId); + } + + return selected.map((item) => (item.tagId === tagId ? { ...item, values: nextValues } : item)); +}; + +const MultiTagFilter = ({ + groups, + selected, + onSelectedChange, + isLoading = false, + isLoadingValues = false, + labels, + placement = 'bottom-start', + offset = [0, 4], + onOpen +}: MultiTagFilterProps) => { + const [activeTagId, setActiveTagId] = useState(''); + const [searchValue, setSearchValue] = useState(''); + const resolvedActiveTagId = groups.some((group) => group.tagId === activeTagId) + ? activeTagId + : (groups[0]?.tagId ?? ''); + + const selectedCountByTagId = useMemo(() => { + const countMap = new Map(); + for (const item of selected) { + countMap.set(item.tagId, item.values.length); + } + return countMap; + }, [selected]); + + const selectedCount = useMemo( + () => selected.reduce((sum, item) => sum + item.values.length, 0), + [selected] + ); + + const summary = useMemo(() => { + if (selectedCount === 0) { + return ( + + {labels.all} + + ); + } + + const firstSelected = selected[0]; + const firstGroup = groups.find((group) => group.tagId === firstSelected.tagId); + const firstValue = firstSelected.values[0]; + const firstValueLabel = + firstGroup?.values.find((item) => item.value === firstValue)?.label ?? String(firstValue); + const firstText = firstGroup ? `${firstGroup.label}：${firstValueLabel}` : firstValueLabel; + const extraCount = selectedCount - 1; + + return ( + + + {firstText} + + {extraCount > 0 && ( + + +{extraCount} + + )} + + ); + }, [groups, labels.all, selected, selectedCount]); + + const activeGroup = groups.find((group) => group.tagId === resolvedActiveTagId); + const filteredValues = useMemo(() => { + if (!activeGroup) return []; + const keyword = searchValue.trim().toLowerCase(); + if (!keyword) return activeGroup.values; + return activeGroup.values.filter((item) => item.label.toLowerCase().includes(keyword)); + }, [activeGroup, searchValue]); + + const isValueChecked = (tagId: string, value: MultiTagFilterValue) => + selected.find((item) => item.tagId === tagId)?.values.some((item) => item === value) ?? false; + + const emptyRightText = (() => { + if (!activeGroup) return ''; + if (isLoadingValues) return ''; + if (activeGroup.values.length === 0) return labels.noValues; + if (filteredValues.length === 0) return labels.noMatch; + return ''; + })(); + + return ( + } + > + {() => ( + e.stopPropagation()}> + + + {groups.map((group) => { + const isActive = group.tagId === resolvedActiveTagId; + const selectedGroupCount = selectedCountByTagId.get(group.tagId) ?? 0; + + return ( + { + setActiveTagId(group.tagId); + setSearchValue(''); + }} + > + + {group.label} + + + {selectedGroupCount > 0 && ( + + {selectedGroupCount} + + )} + + + + ); + })} + + + + + + + setSearchValue(e.target.value)} + /> + {searchValue && ( + setSearchValue('')} + > + + + )} + + + {emptyRightText ? ( + + {emptyRightText} + + ) : ( + filteredValues.map((item) => { + const checked = isValueChecked(resolvedActiveTagId, item.value); + return ( + + onSelectedChange( + toggleMultiTagFilterValue(selected, resolvedActiveTagId, item.value) + ) + } + > + } + /> + + {item.label} + + + ); + }) + )} + + + + + + + + + {labels.selected} + {selectedCount} + {labels.item} + + + + + )} + + ); +}; + +export default React.memo(MultiTagFilter); diff --git a/packages/web/components/common/TagFilter/index.tsx b/packages/web/components/common/TagFilter/index.tsx index 7544d8ab8c52..2a5956337b4b 100644 --- a/packages/web/components/common/TagFilter/index.tsx +++ b/packages/web/components/common/TagFilter/index.tsx @@ -17,6 +17,13 @@ export type { MultiSelectFilterSummary, MultiSelectFilterValue } from './multiSelectFilterUtils'; +export { default as MultiTagFilter, toggleMultiTagFilterValue } from './MultiTagFilter'; +export type { + MultiTagFilterGroup, + MultiTagFilterLabels, + MultiTagFilterProps, + MultiTagFilterValue +} from './MultiTagFilter'; export { FILTER_SEARCH_THRESHOLD } from './FilterSearchInput'; export { default as FilterSearchInput } from './FilterSearchInput'; export { diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index c0a3aed86433..01ae6429f841 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -20,6 +20,7 @@ "Delete": "Delete", "Documents": "Documents", "Done": "Done", + "datetime_picker.placeholder": "YYYY-MM-DD --:--", "Download": "Download", "Edit": "Edit", "Error": "Error", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 8f5462d9f1bf..436681e2adcc 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "Import the CSV file created when you exported a Dataset.", "backup_mode": "Backup import", "backup_template_invalid": "Invalid file format. Check the headers and content. Headers must be q, a, index, metadata (index may repeat; metadata is optional). Excel files must contain a single worksheet with no merged cells", - "batch_delete": "Batch delete", + "batch_delete": "Batch delete files", "chunk_max_tokens": "Max chunk tokens", "chunk_process_params": "Chunk processing parameters", "chunk_size": "Chunk size", @@ -24,6 +24,7 @@ "close_auto_sync": "Turn off auto-sync?", "collection.Create update time": "Creation/Update Time", "collection.export_all_chunks": "Export chunks", + "collection.select_all_filtered": "Select all filtered", "collection.sync.submit": "Sync task submitted", "collection.training_type": "Chunk type", "collection_data_count": "Data amount", @@ -34,10 +35,13 @@ "collection_tags": "Collection Tags", "core.dataset.tags.batchUpsert": "Batch Manage Tags", "core.dataset.tags.datetime": "Datetime", + "core.dataset.tags.array": "Options", + "core.dataset.tags.date": "Date", "core.dataset.tags.number": "Number", "core.dataset.tags.setTags": "Set Tags", "core.dataset.tags.string": "String", "core.dataset.tags.tagType": "Tag Type", + "core.dataset.tags.time": "Time", "core.dataset.tags.tagValue": "Tag Value", "common.error.unKnow": "Unknown error", "common_dataset": "General Dataset", @@ -179,12 +183,57 @@ "tag.Edit_tag": "Edit Tag", "tag.add": "Create", "tag.add_new": "add_new", + "tag.add_option": "Add option", + "tag.add_tag": "Add tag", + "tag.append": "Append", + "tag.attribute": "Attribute", + "tag.batch_add": "Batch add tags", + "tag.batch_edit": "Batch edit tags", + "tag.batch_remove": "Batch remove tags", + "tag.batch_selected_files": "({{num}}) files selected", "tag.cancel": "Cancel", - "tag.delete_tag_confirm": "Confirm to delete the tag?", + "tag.create_failed": "Failed to create tag, please try again", + "tag.create_option": "Create option", + "tag.create_success": "Tag created successfully", + "tag.delete_failed": "Failed to delete tag, please try again", + "tag.delete_success": "Tag deleted successfully", + "tag.delete_tag_confirm_content": "Once deleted, this tag and its tag values added to files will be completely deleted and cannot be undone.", + "tag.delete_tag_confirm_title": "Confirm to delete the tag?", + "tag.enter_name": "Enter tag name", + "tag.enter_option": "Enter option", + "tag.fill_integer": "Enter an integer", + "tag.fill_number": "Enter number", + "tag.fill_value": "Enter tag value", + "tag.filter_clear": "Clear", + "tag.filter_clear_items": "Clear filters", + "tag.filter_empty_prefix": "No files match the current filters. ", + "tag.filter_item": "items", + "tag.filter_no_match": "No matching results", + "tag.filter_no_values": "No filterable values", + "tag.filter_search": "Search", + "tag.filter_selected": "Selected", "tag.manage": "Tagging", + "tag.manage_options": "Manage options", + "tag.name": "Tag name", + "tag.overwrite": "Overwrite", + "tag.overwrite_existing": "Overwrite existing values", + "tag.overwrite_tip": "The value is applied to all selected files. Existing values for this tag are replaced; files without this tag receive the new value.", + "tag.save_failed": "Failed to save tag, please try again", + "tag.save_success": "Tag saved successfully", "tag.searchOrAddTag": "Search or Add Tag", + "tag.search_or_create_option": "Search or create option", + "tag.select_attribute": "Select attribute", + "tag.select_options": "Select options", + "tag.select_tag": "Select tag", + "tag.select_tag_first": "Select a tag first", + "tag.select_tag_value": "Select tag value", + "tag.set": "Tag Settings", + "tag.setting_success": "Tags set successfully", + "tag.setting_tip": "Set tags and corresponding attribute values for the file, used as filter conditions during retrieval.", + "tag.setting_title": "Set Tags", "tag.tags": "Tags", "tag.total_tags": "Total {{total}} tags", + "tag.value": "Tag Value", "template_dataset": "Template import", "template_file_invalid": "Invalid file format. Check the headers and content. Headers must be q, a, index, metadata (index may repeat; metadata is optional). Excel files must contain a single worksheet with no merged cells", "template_mode": "Template import", diff --git a/packages/web/i18n/ko-KR/common.json b/packages/web/i18n/ko-KR/common.json index ccc79fa24cec..759a10bafd1e 100644 --- a/packages/web/i18n/ko-KR/common.json +++ b/packages/web/i18n/ko-KR/common.json @@ -20,6 +20,7 @@ "Delete": "삭제", "Documents": "문서", "Done": "완료", + "datetime_picker.placeholder": "연-월-일 --:--", "Download": "다운로드", "Edit": "편집", "Error": "오류", diff --git a/packages/web/i18n/ko-KR/dataset.json b/packages/web/i18n/ko-KR/dataset.json index 8a77009c91bb..a3c289e748c9 100644 --- a/packages/web/i18n/ko-KR/dataset.json +++ b/packages/web/i18n/ko-KR/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "데이터셋을 내보낼 때 다운로드한 CSV 파일을 다시 가져옵니다.", "backup_mode": "백업 가져오기", "backup_template_invalid": "파일 형식이 올바르지 않습니다. 헤더와 콘텐츠가 요구사항에 맞는지 확인하세요. 헤더는 q, a, index, metadata여야 합니다(index는 반복 가능, metadata는 선택 사항). Excel 파일은 워크시트 하나만 지원하며 병합된 셀을 포함할 수 없습니다", - "batch_delete": "일괄 삭제", + "batch_delete": "파일 일괄 삭제", "chunk_max_tokens": "최대 청크 토큰", "chunk_process_params": "청크 처리 파라미터", "chunk_size": "청크 크기", @@ -24,6 +24,7 @@ "close_auto_sync": "자동 동기화를 끄시겠습니까?", "collection.Create update time": "생성/수정 시간", "collection.export_all_chunks": "청크 내보내기", + "collection.select_all_filtered": "필터 항목 전체 선택", "collection.sync.submit": "동기화 작업이 제출되었습니다", "collection.training_type": "청크 유형", "collection_data_count": "데이터 양", @@ -32,6 +33,13 @@ "collection_sync": "데이터 동기화", "collection_sync_confirm_tip": "데이터 동기화를 시작하시겠습니까? 시스템이 최신 데이터를 가져와 비교합니다. 내용이 다르면 새 컬렉션이 생성되고 기존 컬렉션은 삭제됩니다. 확인해 주세요!", "collection_tags": "컬렉션 태그", + "core.dataset.tags.array": "옵션", + "core.dataset.tags.datetime": "날짜 및 시간", + "core.dataset.tags.number": "숫자", + "core.dataset.tags.string": "텍스트", + "core.dataset.tags.date": "날짜", + "core.dataset.tags.tagType": "태그 유형", + "core.dataset.tags.time": "시간", "common.error.unKnow": "알 수 없는 오류", "common_dataset": "일반 데이터셋", "common_dataset_desc": "파일, 웹 링크 또는 수동 입력을 통해 데이터셋을 구축합니다", @@ -172,12 +180,57 @@ "tag.Edit_tag": "태그 편집", "tag.add": "생성", "tag.add_new": "추가", + "tag.add_option": "옵션 추가", + "tag.add_tag": "태그 추가", + "tag.append": "추가", + "tag.attribute": "속성", + "tag.batch_add": "태그 일괄 추가", + "tag.batch_edit": "태그 일괄 수정", + "tag.batch_remove": "태그 일괄 제거", + "tag.batch_selected_files": "선택됨 ({{num}})개 파일", "tag.cancel": "취소", - "tag.delete_tag_confirm": "태그를 삭제하시겠습니까?", + "tag.create_failed": "태그 추가 실패, 다시 시도해 주세요", + "tag.create_option": "옵션 생성", + "tag.create_success": "태그가 추가되었습니다", + "tag.delete_failed": "태그 삭제 실패, 다시 시도해 주세요", + "tag.delete_success": "태그가 삭제되었습니다", + "tag.delete_tag_confirm_content": "삭제 후 파일에 추가된 해당 태그 및 태그 값이 모두 삭제되며 취소할 수 없습니다.", + "tag.delete_tag_confirm_title": "태그 삭제 확인", + "tag.enter_name": "태그 이름을 입력하세요", + "tag.enter_option": "옵션을 입력하세요", + "tag.fill_integer": "정수를 입력하세요", + "tag.fill_number": "숫자를 입력하세요", + "tag.fill_value": "태그 값을 입력하세요", + "tag.filter_clear": "지우기", + "tag.filter_clear_items": "필터 지우기", + "tag.filter_empty_prefix": "조건에 맞는 파일을 찾지 못했습니다. ", + "tag.filter_item": "개", + "tag.filter_no_match": "일치하는 정보가 없습니다", + "tag.filter_no_values": "필터링할 값이 없습니다", + "tag.filter_search": "검색", + "tag.filter_selected": "선택됨", "tag.manage": "태그 관리", + "tag.manage_options": "옵션 관리", + "tag.name": "태그 이름", + "tag.overwrite": "덮어쓰기", + "tag.overwrite_existing": "기존 값 덮어쓰기", + "tag.overwrite_tip": "시간 또는 숫자 태그를 일괄 설정하면 선택한 모든 항목에 값이 적용됩니다. 해당 태그가 이미 있으면 기존 값이 대체되고, 없으면 새 값이 추가됩니다.", + "tag.save_failed": "태그 저장 실패, 다시 시도해 주세요", + "tag.save_success": "태그가 저장되었습니다", "tag.searchOrAddTag": "태그 검색 또는 추가", + "tag.search_or_create_option": "옵션 검색 또는 생성", + "tag.select_attribute": "속성을 선택하세요", + "tag.select_options": "옵션을 선택하세요", + "tag.select_tag": "태그를 선택하세요", + "tag.select_tag_first": "먼저 태그를 선택하세요", + "tag.select_tag_value": "태그 값을 선택하세요", + "tag.set": "태그 설정", + "tag.setting_success": "태그가 설정되었습니다", + "tag.setting_tip": "검색 시 필터 조건으로 사용할 파일의 태그 및 속성 값을 설정합니다.", + "tag.setting_title": "태그 설정", "tag.tags": "태그", "tag.total_tags": "총 {{total}}개 태그", + "tag.value": "태그 값", "template_dataset": "템플릿 가져오기", "template_file_invalid": "파일 형식이 올바르지 않습니다. 헤더와 콘텐츠가 요구사항에 맞는지 확인하세요. 헤더는 q, a, index, metadata여야 합니다(index는 반복 가능, metadata는 선택 사항). Excel 파일은 워크시트 하나만 지원하며 병합된 셀을 포함할 수 없습니다", "template_mode": "템플릿 가져오기", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index 44ce2f6e77d2..c28f0220a869 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -20,6 +20,7 @@ "Delete": "删除", "Documents": "文档", "Done": "完成", + "datetime_picker.placeholder": "年-月-日 --:--", "Download": "下载", "Edit": "编辑", "Error": "错误", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index 451886d71799..b34c83cd6640 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "可以将导出知识库时，下载的 csv 文件重新导入。", "backup_mode": "备份导入", "backup_template_invalid": "文件格式异常，请检查表头和内容是否符合要求。表头应为 q、a、index、metadata（index 可重复，metadata 可选）；Excel 文件仅支持单个工作表，且不能包含合并单元格", - "batch_delete": "批量删除", + "batch_delete": "批量删除文件", "chunk_max_tokens": "分块上限", "chunk_process_params": "分块处理参数", "chunk_size": "分块大小", @@ -24,6 +24,7 @@ "close_auto_sync": "确认关闭自动同步功能？", "collection.Create update time": "创建/更新时间", "collection.export_all_chunks": "导出分块", + "collection.select_all_filtered": "全选筛选项", "collection.sync.submit": "已提交同步任务", "collection.training_type": "处理模式", "collection_data_count": "数据量", @@ -34,10 +35,13 @@ "collection_tags": "集合标签", "core.dataset.tags.batchUpsert": "批量管理标签", "core.dataset.tags.datetime": "日期时间", + "core.dataset.tags.array": "选项", + "core.dataset.tags.date": "日期", "core.dataset.tags.number": "数字", "core.dataset.tags.setTags": "设置标签值", "core.dataset.tags.string": "文本", "core.dataset.tags.tagType": "标签类型", + "core.dataset.tags.time": "时间", "core.dataset.tags.tagValue": "标签值", "common.error.unKnow": "未知错误", "common_dataset": "通用知识库", @@ -179,12 +183,57 @@ "tag.Edit_tag": "编辑标签", "tag.add": "创建", "tag.add_new": "新建", + "tag.add_option": "添加选项", + "tag.add_tag": "添加标签", + "tag.append": "追加", + "tag.attribute": "属性", + "tag.batch_add": "批量添加标签", + "tag.batch_edit": "批量修改标签", + "tag.batch_remove": "批量移除标签", + "tag.batch_selected_files": "已选 ({{num}}) 个文件", "tag.cancel": "取消选择", - "tag.delete_tag_confirm": "确定删除标签？", + "tag.create_failed": "标签新增失败，请重试", + "tag.create_option": "创建选项", + "tag.create_success": "标签新增成功", + "tag.delete_failed": "标签删除失败，请重试", + "tag.delete_success": "标签删除成功", + "tag.delete_tag_confirm_content": "删除后，已添加到文件中的该标签及标签值将全部删除，且无法撤销。", + "tag.delete_tag_confirm_title": "确认删除标签？", + "tag.enter_name": "请输入标签名称", + "tag.enter_option": "请输入选项", + "tag.fill_integer": "请输入整数值", + "tag.fill_number": "请填写数字", + "tag.fill_value": "请填写标签值", + "tag.filter_clear": "清空", + "tag.filter_clear_items": "清空筛选项", + "tag.filter_empty_prefix": "未找到符合条件的文件，", + "tag.filter_item": "项", + "tag.filter_no_match": "未匹配到相关信息", + "tag.filter_no_values": "暂无可筛选值", + "tag.filter_search": "搜索", + "tag.filter_selected": "已选", "tag.manage": "标签管理", + "tag.manage_options": "管理选项", + "tag.name": "标签名称", + "tag.overwrite": "覆盖", + "tag.overwrite_existing": "覆盖原有值", + "tag.overwrite_tip": "批量设置时间或数值标签时，所设置的值会应用到全部选中内容。若部分内容已存在该标签值，原有值将被替换；未设置该标签的内容将直接添加该值。", + "tag.save_failed": "标签保存失败，请重试", + "tag.save_success": "标签保存成功", "tag.searchOrAddTag": "搜索或添加标签", + "tag.search_or_create_option": "搜索或创建选项", + "tag.select_attribute": "请选择属性", + "tag.select_options": "请选择选项", + "tag.select_tag": "请选择标签", + "tag.select_tag_first": "请先选择标签", + "tag.select_tag_value": "请选择标签值", + "tag.set": "标签设置", + "tag.setting_success": "标签设置成功", + "tag.setting_tip": "设置文件的标签和对应属性值，用于检索时的过滤条件。", + "tag.setting_title": "设置标签", "tag.tags": "标签", "tag.total_tags": "共{{total}}个标签", + "tag.value": "标签值", "template_dataset": "模版导入", "template_file_invalid": "文件格式异常，请检查表头和内容是否符合要求。表头应为 q、a、index、metadata（index 可重复，metadata 可选）；Excel 文件仅支持单个工作表，且不能包含合并单元格", "template_mode": "模板导入", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index de7908e31009..2cc3cf128f18 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -20,6 +20,7 @@ "Delete": "刪除", "Documents": "文件", "Done": "完成", + "datetime_picker.placeholder": "年-月-日 --:--", "Download": "下載", "Edit": "編輯", "Error": "錯誤", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index 329dc6613edd..cabdb9e7db19 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "可以將導出知識庫時，下載的 csv 文件重新導入。", "backup_mode": "備份導入", "backup_template_invalid": "文件格式異常，請檢查表頭和內容是否符合要求。表頭應為 q、a、index、metadata（index 可重複，metadata 可選）；Excel 文件僅支持單個工作表，且不能包含合併單元格", - "batch_delete": "批量刪除", + "batch_delete": "批量刪除檔案", "chunk_max_tokens": "分塊上限", "chunk_process_params": "分塊處理參數", "chunk_size": "分塊大小", @@ -24,6 +24,7 @@ "close_auto_sync": "確認關閉自動同步功能？", "collection.Create update time": "建立／更新時間", "collection.export_all_chunks": "導出分塊", + "collection.select_all_filtered": "全選篩選項", "collection.sync.submit": "已提交同步任務", "collection.training_type": "處理模式", "collection_data_count": "資料量", @@ -34,10 +35,13 @@ "collection_tags": "集合標籤", "core.dataset.tags.batchUpsert": "批量管理標籤", "core.dataset.tags.datetime": "日期時間", + "core.dataset.tags.array": "選項", + "core.dataset.tags.date": "日期", "core.dataset.tags.number": "數字", "core.dataset.tags.setTags": "設定標籤值", "core.dataset.tags.string": "文字", "core.dataset.tags.tagType": "標籤類型", + "core.dataset.tags.time": "時間", "core.dataset.tags.tagValue": "標籤值", "common.error.unKnow": "未知錯誤", "common_dataset": "通用資料集", @@ -179,12 +183,57 @@ "tag.Edit_tag": "編輯標籤", "tag.add": "建立", "tag.add_new": "新增", + "tag.add_option": "新增選項", + "tag.add_tag": "新增標籤", + "tag.append": "追加", + "tag.attribute": "屬性", + "tag.batch_add": "批量新增標籤", + "tag.batch_edit": "批量修改標籤", + "tag.batch_remove": "批量移除標籤", + "tag.batch_selected_files": "已選 ({{num}}) 個檔案", "tag.cancel": "取消", - "tag.delete_tag_confirm": "確定要刪除標籤嗎？", + "tag.create_failed": "標籤新增失敗，請重試", + "tag.create_option": "建立選項", + "tag.create_success": "標籤新增成功", + "tag.delete_failed": "標籤刪除失敗，請重試", + "tag.delete_success": "標籤刪除成功", + "tag.delete_tag_confirm_content": "刪除後，已新增至檔案中的該標籤及標籤值將全部刪除，且無法復原。", + "tag.delete_tag_confirm_title": "確認刪除標籤？", + "tag.enter_name": "請輸入標籤名稱", + "tag.enter_option": "請輸入選項", + "tag.fill_integer": "請輸入整數值", + "tag.fill_number": "請填寫數字", + "tag.fill_value": "請填寫標籤值", + "tag.filter_clear": "清空", + "tag.filter_clear_items": "清空篩選項", + "tag.filter_empty_prefix": "未找到符合條件的檔案，", + "tag.filter_item": "項", + "tag.filter_no_match": "未匹配到相關資訊", + "tag.filter_no_values": "暫無可篩選值", + "tag.filter_search": "搜尋", + "tag.filter_selected": "已選", "tag.manage": "標籤管理", + "tag.manage_options": "管理選項", + "tag.name": "標籤名稱", + "tag.overwrite": "覆蓋", + "tag.overwrite_existing": "覆蓋原有值", + "tag.overwrite_tip": "批次設定時間或數值標籤時，所設定的值會套用到全部選取內容。若部分內容已存在該標籤值，原有值將被取代；未設定該標籤的內容將直接新增該值。", + "tag.save_failed": "標籤儲存失敗，請重試", + "tag.save_success": "標籤儲存成功", "tag.searchOrAddTag": "搜尋或新增標籤", + "tag.search_or_create_option": "搜尋或建立選項", + "tag.select_attribute": "請選擇屬性", + "tag.select_options": "請選擇選項", + "tag.select_tag": "請選擇標籤", + "tag.select_tag_first": "請先選擇標籤", + "tag.select_tag_value": "請選擇標籤值", + "tag.set": "標籤設定", + "tag.setting_success": "標籤設定成功", + "tag.setting_tip": "設定檔案的標籤與對應屬性值，用於檢索時的過濾條件。", + "tag.setting_title": "設定標籤", "tag.tags": "標籤", "tag.total_tags": "共 {{total}} 個標籤", + "tag.value": "標籤值", "template_dataset": "模版導入", "template_file_invalid": "文件格式異常，請檢查表頭和內容是否符合要求。表頭應為 q、a、index、metadata（index 可重複，metadata 可選）；Excel 文件僅支持單個工作表，且不能包含合併單元格", "template_mode": "模板導入", diff --git a/packages/web/styles/theme.ts b/packages/web/styles/theme.ts index 7c6be15928dd..34b212a208bc 100644 --- a/packages/web/styles/theme.ts +++ b/packages/web/styles/theme.ts @@ -654,6 +654,16 @@ const Checkbox = checkBoxMultiStyle({ } } }, + _indeterminate: { + bg: 'primary.50', + borderColor: 'primary.600', + borderWidth: '1px', + color: 'primary.600', + boxShadow: `${shadowLight} !important`, + _hover: { + bg: 'primary.50' + } + }, _hover: { borderColor: 'primary.400' }, diff --git a/pro b/pro index fdfc4a57cf98..5f063bf4e3f5 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit fdfc4a57cf9868b3932188990545659a4059616a +Subproject commit 5f063bf4e3f56eb8dd4d5f7f22154f4e8e54b203 diff --git a/projects/app/src/global/core/api/datasetReq.ts b/projects/app/src/global/core/api/datasetReq.ts index 264a3801dbd5..fd727c09490b 100644 --- a/projects/app/src/global/core/api/datasetReq.ts +++ b/projects/app/src/global/core/api/datasetReq.ts @@ -1,6 +1,6 @@ -import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import type { PaginationProps } from '@fastgpt/global/openapi/api'; import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; /* ===== dataset ===== */ @@ -9,7 +9,7 @@ export type GetDatasetCollectionsProps = PaginationProps<{ datasetId: string; parentId?: ParentIdType; searchText?: string; - filterTags?: string[]; + tagFilters?: CollectionTagFilterItem[]; simple?: boolean; selectFolder?: boolean; }>; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx new file mode 100644 index 000000000000..8dc41ac670ad --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx @@ -0,0 +1,573 @@ +import React, { useMemo, useState } from 'react'; +import { Box, Button, Checkbox, Flex, HStack, useDisclosure } from '@chakra-ui/react'; +import MyModal from '@fastgpt/web/components/v2/common/MyModal'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs'; +import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { postBatchSetCollectionTags } from '@/web/core/dataset/api/collection'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { + BatchCollectionTagModeEnum, + type BatchSetCollectionTagItem +} from '@fastgpt/global/openapi/core/dataset/collection/tagApi'; +import { + DatasetCollectionTagTypeEnum, + type CollectionTagValueType, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; +import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; +import { resolveDisplayedCollectionTag, TagTableContainer } from './TagCommon'; +import TagManageModal from './TagManageModal'; +import { useAppendDatasetTagOption } from './useAppendDatasetTagOption'; +import { useCollectionTagRows } from './useCollectionTagRows'; +import { CollectionTagTable } from './CollectionTagTable'; +import { createEmptyTagRow, type CollectionTagRow } from './tagForm'; + +type BatchTagMode = (typeof BatchCollectionTagModeEnum)[keyof typeof BatchCollectionTagModeEnum]; + +type TagRowType = CollectionTagRow & { + append?: boolean; +}; + +type CollectionTagBatchModalProps = { + collections: DatasetCollectionsListItemType[]; + onClose: () => void; + onSuccess?: () => void; +}; + +type RemoveTagGroup = { + tagId: string; + tagName: string; + tagType: DatasetTagType['tagType']; + values: string[]; +}; + +const createEmptyBatchTagRow = (): TagRowType => ({ + ...createEmptyTagRow(), + append: true +}); + +/** 从已选集合聚合可移除的标签：选项类收集去重后的值，其余类型只展示标签名。 */ +const buildRemoveTagGroups = ( + collections: DatasetCollectionsListItemType[], + tags: DatasetTagType[] +): RemoveTagGroup[] => { + const groups = new Map(); + + for (const collection of collections) { + for (const item of collection.tags ?? []) { + const resolved = resolveDisplayedCollectionTag(item, tags); + if (!resolved) continue; + + const tagId = String(resolved.tagDoc._id); + const tagType = resolved.tagDoc.tagType ?? DatasetCollectionTagTypeEnum.string; + const group = groups.get(tagId) ?? { + tagId, + tagName: resolved.tagDoc.tag, + tagType, + values: [] + }; + + if (tagType === DatasetCollectionTagTypeEnum.array) { + const list = Array.isArray(resolved.value) + ? resolved.value + : resolved.value !== '' + ? [String(resolved.value)] + : []; + for (const value of list) { + if (value && !group.values.includes(value)) { + group.values.push(value); + } + } + } + + groups.set(tagId, group); + } + } + + return [...groups.values()]; +}; + +const CollectionTagBatchModal = ({ + collections, + onClose, + onSuccess +}: CollectionTagBatchModalProps) => { + const { t } = useTranslation(); + const { toast } = useToast(); + const { datasetDetail, allDatasetTags, isLoadingAllDatasetTags, loadAllDatasetTags } = + useContextSelector(DatasetPageContext, (v) => v); + const { + isOpen: isTagManageOpen, + onOpen: onOpenTagManage, + onClose: onCloseTagManage + } = useDisclosure(); + + const [mode, setMode] = useState(BatchCollectionTagModeEnum.add); + const [initialRows] = useState(() => [createEmptyBatchTagRow()]); + const [expandedTagIds, setExpandedTagIds] = useState>(new Set()); + const [selectedTagIds, setSelectedTagIds] = useState>(new Set()); + const [selectedValues, setSelectedValues] = useState>>(new Map()); + + const { + rows, + hasIncompleteRow, + isAddDisabled, + updateRows, + handleAddRow, + handleDeleteRow, + handleTagChange, + handleValueChange + } = useCollectionTagRows({ + tags: allDatasetTags, + initialRows, + createRow: createEmptyBatchTagRow, + patchOnTagChange: (tagDoc) => ({ + append: + (tagDoc?.tagType ?? DatasetCollectionTagTypeEnum.string) === + DatasetCollectionTagTypeEnum.number + }), + onRowDeleted: () => { + toast({ + title: t('dataset:tag.delete_success'), + status: 'success' + }); + } + }); + + const removeGroups = useMemo( + () => buildRemoveTagGroups(collections, allDatasetTags), + [collections, allDatasetTags] + ); + + const { runAsync: onSave, loading: isSaving } = useRequest( + (body: { mode: typeof mode; tags: BatchSetCollectionTagItem[] }) => + postBatchSetCollectionTags({ + datasetId: datasetDetail._id, + collectionIds: collections.map((item) => item._id), + mode: body.mode, + tags: body.tags + }), + { + onSuccess() { + void loadAllDatasetTags(); + onSuccess?.(); + onClose(); + }, + successToast: t('dataset:tag.setting_success'), + errorToast: t('dataset:tag.save_failed') + } + ); + + const { runAsync: onAppendTagOption } = useAppendDatasetTagOption(); + + const removePayload = useMemo(() => { + const tags: BatchSetCollectionTagItem[] = []; + for (const group of removeGroups) { + if (group.tagType === DatasetCollectionTagTypeEnum.array) { + const selected = selectedValues.get(group.tagId); + if (!selected || selected.size === 0) continue; + if (selected.size === group.values.length) { + tags.push({ tagId: group.tagId }); + } else { + tags.push({ tagId: group.tagId, value: [...selected] }); + } + continue; + } + if (selectedTagIds.has(group.tagId)) { + tags.push({ tagId: group.tagId }); + } + } + return tags; + }, [removeGroups, selectedTagIds, selectedValues]); + + const isConfirmDisabled = + isSaving || + (mode === BatchCollectionTagModeEnum.add + ? rows.length === 0 || hasIncompleteRow + : removePayload.length === 0); + + const toggleExpand = (tagId: string) => { + setExpandedTagIds((prev) => { + const next = new Set(prev); + if (next.has(tagId)) next.delete(tagId); + else next.add(tagId); + return next; + }); + }; + + const toggleNonArrayTag = (tagId: string) => { + setSelectedTagIds((prev) => { + const next = new Set(prev); + if (next.has(tagId)) next.delete(tagId); + else next.add(tagId); + return next; + }); + }; + + const toggleArrayValue = (tagId: string, value: string) => { + setSelectedValues((prev) => { + const next = new Map(prev); + const current = new Set(next.get(tagId) ?? []); + if (current.has(value)) current.delete(value); + else current.add(value); + if (current.size === 0) next.delete(tagId); + else next.set(tagId, current); + return next; + }); + }; + + const toggleArrayParent = (tagId: string, allValues: string[]) => { + setSelectedValues((prev) => { + const next = new Map(prev); + const current = next.get(tagId); + const isAllSelected = current && current.size === allValues.length && allValues.length > 0; + if (isAllSelected) { + next.delete(tagId); + } else { + next.set(tagId, new Set(allValues)); + } + return next; + }); + }; + + const handleSubmit = async () => { + if (isConfirmDisabled) return; + + if (mode === BatchCollectionTagModeEnum.remove) { + await onSave({ mode, tags: removePayload }); + return; + } + + const tags: BatchSetCollectionTagItem[] = rows.map((row) => { + const tagDoc = allDatasetTags.find((tag) => String(tag._id) === row.tagId); + const tagType = tagDoc?.tagType ?? DatasetCollectionTagTypeEnum.string; + return { + tagId: row.tagId, + value: row.value as CollectionTagValueType['value'], + ...(tagType === DatasetCollectionTagTypeEnum.number ? { append: Boolean(row.append) } : {}) + }; + }); + + await onSave({ mode: BatchCollectionTagModeEnum.add, tags }); + }; + + const renderAddValueExtra = (row: TagRowType, selectedTag?: DatasetTagType) => { + const tagType = selectedTag?.tagType ?? DatasetCollectionTagTypeEnum.string; + if (tagType === DatasetCollectionTagTypeEnum.number) { + return ( + + {( + [ + { label: t('dataset:tag.append'), append: true }, + { label: t('dataset:tag.overwrite'), append: false } + ] as const + ).map((item) => { + const checked = Boolean(row.append) === item.append; + return ( + + updateRows((current) => + current.map((currentRow) => + currentRow.id === row.id ? { ...currentRow, append: item.append } : currentRow + ) + ) + } + > + + + + + {item.label} + + + ); + })} + + ); + } + if ( + tagType === DatasetCollectionTagTypeEnum.array || + tagType === DatasetCollectionTagTypeEnum.datetime + ) { + return ( + + {t('dataset:tag.overwrite_existing')} + + + ); + } + return null; + }; + + return ( + <> + + {t('dataset:tag.batch_edit')} + {t('dataset:tag.batch_selected_files', { num: collections.length })} + + } + closeOnOverlayClick={false} + bodyStyles={{ + flex: 1, + minH: 0, + pb: 0, + overflow: 'hidden' + }} + footerStyles={{ + justifyContent: 'space-between', + px: 8, + pt: 2, + pb: 8 + }} + footer={ + <> + + + + + + + + } + borderRadius={'10px'} + sx={{ + '.chakra-modal__close-btn': { + top: 2, + right: 2, + w: '36px', + h: '36px' + } + }} + > + + + list={[ + { label: t('dataset:tag.batch_add'), value: BatchCollectionTagModeEnum.add }, + { label: t('dataset:tag.batch_remove'), value: BatchCollectionTagModeEnum.remove } + ]} + value={mode} + onChange={setMode} + outerHeight={'40px'} + itemHeight={'32px'} + px={'12px'} + alignSelf={'flex-start'} + flexShrink={0} + /> + + {mode === BatchCollectionTagModeEnum.add ? ( + { + void onAppendTagOption({ tagId, option }); + }} + renderValueExtra={renderAddValueExtra} + valueFieldProps={{ + disabledPlaceholder: t('dataset:tag.select_tag_first'), + numberShowStepper: true, + numberPlaceholder: t('dataset:tag.fill_integer'), + arrayPlaceholder: t('dataset:tag.select_tag_value') + }} + /> + ) : ( + + + {removeGroups.length === 0 ? ( + + ) : ( + + {removeGroups.map((group) => { + const isArray = group.tagType === DatasetCollectionTagTypeEnum.array; + const selected = selectedValues.get(group.tagId) ?? new Set(); + const isAllSelected = + isArray && group.values.length > 0 && selected.size === group.values.length; + const isIndeterminate = + isArray && selected.size > 0 && selected.size < group.values.length; + const isExpanded = expandedTagIds.has(group.tagId); + const count = isArray ? group.values.length : 1; + + return ( + + isArray + ? toggleArrayParent(group.tagId, group.values) + : toggleNonArrayTag(group.tagId) + } + > + + + + + {group.tagName}（{count}） + + + {isArray && group.values.length > 0 && ( + { + e.stopPropagation(); + toggleExpand(group.tagId); + }} + > + + + )} + + {isArray && + isExpanded && + group.values.map((value) => ( + { + e.stopPropagation(); + toggleArrayValue(group.tagId, value); + }} + > + + + {value} + + + ))} + + ); + })} + + )} + + + )} + + + {isTagManageOpen && } + + ); +}; + +export default React.memo(CollectionTagBatchModal); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagSetModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagSetModal.tsx new file mode 100644 index 000000000000..ae799df67bf2 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagSetModal.tsx @@ -0,0 +1,183 @@ +import React, { useMemo } from 'react'; +import { Box, Button, Flex, HStack, useDisclosure } from '@chakra-ui/react'; +import MyModal from '@fastgpt/web/components/v2/common/MyModal'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { postSetCollectionTags } from '@/web/core/dataset/api/collection'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { type CollectionTagValueType } from '@fastgpt/global/core/dataset/type'; +import TagManageModal from './TagManageModal'; +import { useAppendDatasetTagOption } from './useAppendDatasetTagOption'; +import { useCollectionTagRows } from './useCollectionTagRows'; +import { CollectionTagTable } from './CollectionTagTable'; +import { collectionTagsToRows, createEmptyTagRow } from './tagForm'; + +type CollectionTagSetModalProps = { + collection: DatasetCollectionsListItemType; + onClose: () => void; + onSuccess?: () => void; +}; + +const CollectionTagSetModal = ({ collection, onClose, onSuccess }: CollectionTagSetModalProps) => { + const { t } = useTranslation(); + const { toast } = useToast(); + const { datasetDetail, allDatasetTags, isLoadingAllDatasetTags, loadAllDatasetTags } = + useContextSelector(DatasetPageContext, (v) => v); + const { + isOpen: isTagManageOpen, + onOpen: onOpenTagManage, + onClose: onCloseTagManage + } = useDisclosure(); + + const initialRows = useMemo( + () => collectionTagsToRows(collection.tags, allDatasetTags), + [collection.tags, allDatasetTags] + ); + + const { + rows, + hasIncompleteRow, + isAddDisabled, + handleAddRow, + handleDeleteRow, + handleTagChange, + handleValueChange + } = useCollectionTagRows({ + tags: allDatasetTags, + initialRows, + createRow: createEmptyTagRow, + onRowDeleted: () => { + toast({ + title: t('dataset:tag.delete_success'), + status: 'success' + }); + } + }); + + const { runAsync: onSaveTags, loading: isSaving } = useRequest( + (tags: CollectionTagValueType[]) => + postSetCollectionTags({ + datasetId: datasetDetail._id, + collectionId: collection._id, + tags + }), + { + onSuccess() { + void loadAllDatasetTags(); + onSuccess?.(); + onClose(); + }, + successToast: t('dataset:tag.setting_success'), + errorToast: t('dataset:tag.save_failed') + } + ); + + const { runAsync: onAppendTagOption } = useAppendDatasetTagOption(); + + const isConfirmDisabled = hasIncompleteRow || isSaving; + + const handleSubmit = async () => { + if (isConfirmDisabled) return; + + await onSaveTags(rows.map((row) => ({ tagId: row.tagId, value: row.value }))); + }; + + return ( + <> + + {t('dataset:tag.setting_title')} + + + } + closeOnOverlayClick={false} + bodyStyles={{ + flex: 1, + minH: 0, + pb: 0, + overflow: 'hidden' + }} + footerStyles={{ + justifyContent: 'space-between', + px: 8, + pt: 2, + pb: 8 + }} + footer={ + <> + + + + + + + + } + borderRadius={'10px'} + sx={{ + '.chakra-modal__close-btn': { + top: 2, + right: 2, + w: '36px', + h: '36px' + } + }} + > + { + void onAppendTagOption({ tagId, option }); + }} + /> + + {isTagManageOpen && } + + ); +}; + +export default React.memo(CollectionTagSetModal); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx new file mode 100644 index 000000000000..68c4eb422b8c --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx @@ -0,0 +1,130 @@ +import React from 'react'; +import { Box, Flex } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; +import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; +import { TagActionButton, TagTableContainer, TagTableHeader } from './TagCommon'; +import { TagNameSelect, TagValueField } from './TagValueInputs'; +import { SET_TAG_TABLE_COLUMNS, unusedTagSelectOptions, type CollectionTagRow } from './tagForm'; + +type CollectionTagTableProps = { + rows: T[]; + tags: DatasetTagType[]; + onTagChange: (rowId: string, tagId: string) => void; + onValueChange: (rowId: string, value: string | number | string[]) => void; + onDeleteRow: (rowId: string) => void; + onManage: () => void; + onCreateOption?: (tagId: string, option: string) => void; + /** 批量场景在标签值下方预留固定 16px，避免选类型时行高抖动。 */ + renderValueExtra?: (row: T, tag?: DatasetTagType) => React.ReactNode; + valueFieldProps?: { + disabledPlaceholder?: string; + numberShowStepper?: boolean; + numberPlaceholder?: string; + arrayPlaceholder?: string; + }; +}; + +/** 设置标签 / 批量添加共用的标签表，行高只由是否预留 extra 槽决定，不随当前类型变化。 */ +export const CollectionTagTable = ({ + rows, + tags, + onTagChange, + onValueChange, + onDeleteRow, + onManage, + onCreateOption, + renderValueExtra, + valueFieldProps +}: CollectionTagTableProps) => { + const { t } = useTranslation(); + const hasExtraSlot = Boolean(renderValueExtra); + const rowH = hasExtraSlot ? '96px' : '80px'; + + return ( + + + {t('dataset:tag.name')} + {t('dataset:tag.value')} + {t('common:Operation')} + + + + {rows.length === 0 ? ( + + ) : ( + rows.map((row) => { + const selectedTag = tags.find((tag) => String(tag._id) === row.tagId); + + return ( + + + onTagChange(row.id, newTagId)} + onManage={onManage} + /> + + + + + onValueChange(row.id, val)} + onCreateOption={(option) => { + if (!selectedTag) return; + onCreateOption?.(String(selectedTag._id), option); + }} + {...valueFieldProps} + /> + + {hasExtraSlot && ( + + {renderValueExtra?.(row, selectedTag)} + + )} + + + + } + hoverColor={'red.600'} + onClick={() => onDeleteRow(row.id)} + /> + + + ); + }) + )} + + + ); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx index 983641a9ba1a..550c5d25abed 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx @@ -21,6 +21,7 @@ import dynamic from 'next/dynamic'; import { usePagination } from '@fastgpt/web/hooks/usePagination'; import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; import { useRouter } from 'next/router'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { type WebsiteConfigFormType } from './WebsiteConfig'; @@ -40,8 +41,8 @@ type CollectionPageContextType = { scrollContainerRef: RefObject; searchText: string; setSearchText: Dispatch>; - filterTags: string[]; - setFilterTags: Dispatch>; + tagFilters: CollectionTagFilterItem[]; + setTagFilters: Dispatch>; }; export const CollectionPageContext = createContext({ @@ -67,8 +68,8 @@ export const CollectionPageContext = createContext({ setSearchText: function (_value: SetStateAction): void { throw new Error('Function not implemented.'); }, - filterTags: [], - setFilterTags: function (_value: SetStateAction): void { + tagFilters: [], + setTagFilters: function (_value: SetStateAction): void { throw new Error('Function not implemented.'); } }); @@ -85,7 +86,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => // collection list const [searchText, setSearchText] = useState(''); - const [filterTags, setFilterTags] = useState([]); + const [tagFilters, setTagFilters] = useState([]); const scrollContainerRef = useRef(null); const { data: collections, @@ -103,9 +104,9 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => datasetId, parentId, searchText, - filterTags + tagFilters }, - refreshDeps: [parentId, searchText, filterTags], + refreshDeps: [parentId, searchText, tagFilters], scrollContainerRef }); @@ -161,8 +162,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => searchText, setSearchText, - filterTags, - setFilterTags, + tagFilters, + setTagFilters, collections, Pagination, total, @@ -175,7 +176,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => [ Pagination, collections, - filterTags, + tagFilters, getData, isGetting, onOpenWebsiteModal, diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx new file mode 100644 index 000000000000..9cb55ee7a90e --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx @@ -0,0 +1,89 @@ +import { useContextSelector } from 'use-context-selector'; +import { useTranslation } from 'next-i18next'; +import { useEffect, useMemo } from 'react'; +import { MultiTagFilter, type MultiTagFilterGroup } from '@fastgpt/web/components/common/TagFilter'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { CollectionPageContext } from './Context'; +import { getDatasetTagFilterOptions } from '@/web/core/dataset/api/collection'; +import { buildTagFilterValues, formatCollectionTagValueText } from './TagCommon'; + +const DatasetTagFilter = () => { + const { t } = useTranslation(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags); + const isLoadingAllDatasetTags = useContextSelector( + DatasetPageContext, + (v) => v.isLoadingAllDatasetTags + ); + const tagFilters = useContextSelector(CollectionPageContext, (v) => v.tagFilters); + const setTagFilters = useContextSelector(CollectionPageContext, (v) => v.setTagFilters); + + const { + runAsync: loadUsedValues, + data: usedValuesRes, + loading: isLoadingUsedValues + } = useRequest(() => getDatasetTagFilterOptions(datasetDetail._id), { + manual: false, + ready: !!datasetDetail._id, + refreshDeps: [datasetDetail._id] + }); + + const usedValuesByTagId = useMemo(() => { + const valueMap = new Map>(); + for (const item of usedValuesRes?.list ?? []) { + valueMap.set(item.tagId, item.values); + } + return valueMap; + }, [usedValuesRes]); + + const groups = useMemo( + () => + allDatasetTags.map((tag) => { + const tagId = String(tag._id); + return { + tagId, + label: tag.tag, + values: buildTagFilterValues(tag, usedValuesByTagId.get(tagId) ?? []).map((value) => ({ + value, + label: formatCollectionTagValueText(value, tag.tagType) + })) + }; + }), + [allDatasetTags, usedValuesByTagId] + ); + + useEffect(() => { + if (allDatasetTags.length === 0) return; + const validTagIds = new Set(allDatasetTags.map((tag) => String(tag._id))); + setTagFilters((prev) => { + const next = prev.filter((item) => validTagIds.has(item.tagId)); + return next.length === prev.length ? prev : next; + }); + }, [allDatasetTags, setTagFilters]); + + return ( + { + void loadUsedValues(); + }} + labels={{ + title: t('dataset:tag.tags'), + all: t('common:All'), + searchPlaceholder: t('dataset:tag.filter_search'), + selected: t('dataset:tag.filter_selected'), + item: t('dataset:tag.filter_item'), + clear: t('dataset:tag.filter_clear'), + noValues: t('dataset:tag.filter_no_values'), + noMatch: t('dataset:tag.filter_no_match') + }} + /> + ); +}; + +export default DatasetTagFilter; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx index 30c098e71671..ef1d6e1f39a9 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx @@ -10,8 +10,31 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex const EmptyCollectionTip = () => { const { t } = useTranslation(); const onOpenWebsiteModal = useContextSelector(CollectionPageContext, (v) => v.onOpenWebsiteModal); + const tagFilters = useContextSelector(CollectionPageContext, (v) => v.tagFilters); + const setTagFilters = useContextSelector(CollectionPageContext, (v) => v.setTagFilters); const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + if (tagFilters.length > 0) { + return ( + + {t('dataset:tag.filter_empty_prefix')} + { + setTagFilters([]); + }} + > + {t('dataset:tag.filter_clear_items')} + + + } + /> + ); + } + return ( <> {(datasetDetail.type === DatasetTypeEnum.dataset || diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx index 7b38541fc506..d59cb37f41ed 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Box, Flex, MenuButton, Button, Link, useDisclosure, HStack } from '@chakra-ui/react'; +import { Box, Flex, Button, Link, useDisclosure, HStack } from '@chakra-ui/react'; import { getDatasetCollectionPathById, postDatasetCollection, @@ -30,16 +30,33 @@ import { useContextSelector } from 'use-context-selector'; import { CollectionPageContext } from './Context'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { useSystem } from '@fastgpt/web/hooks/useSystem'; -import HeaderTagPopOver from './HeaderTagPopOver'; +import DatasetTagFilter from './DatasetTagFilter'; import MyBox from '@fastgpt/web/components/common/MyBox'; import Icon from '@fastgpt/web/components/common/Icon'; import MyTag from '@fastgpt/web/components/common/Tag/index'; import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import TagManageModal from './TagManageModal'; const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector')); const BackupImportModal = dynamic(() => import('./BackupImportModal')); const TemplateImportModal = dynamic(() => import('./TemplateImportModal')); +const HeaderImportButton = ({ children }: { children: React.ReactNode }) => ( + +); + const Header = ({ hasTrainingData, hasTrainingError, @@ -99,6 +116,11 @@ const Header = ({ onOpen: onOpenTemplateImportModal, onClose: onCloseTemplateImportModal } = useDisclosure(); + const { + isOpen: isTagManageModalOpen, + onOpen: onOpenTagManageModal, + onClose: onCloseTagManageModal + } = useDisclosure(); const { runAsync: onCreateCollection } = useRequest( async ({ @@ -128,9 +150,9 @@ const Header = ({ const isWebSite = datasetDetail?.type === DatasetTypeEnum.websiteDataset; return ( - - - + + + ({ parentId: path.parentId, @@ -180,8 +202,10 @@ const Header = ({ {/* search input */} {isPc && ( } + feConfigs?.isPlus && } + - {hasTrainingError && ( + {hasTrainingError && ( + + )} + {datasetDetail.type !== DatasetTypeEnum.websiteDataset && + datasetDetail.permission.hasWritePer && + feConfigs?.isPlus && ( )} - {/* diff collection button */} {datasetDetail.permission.hasWritePer && ( @@ -230,35 +268,9 @@ const Header = ({ - - - - - - {t('common:dataset.collections.Create And Import')} - - - + + {t('common:dataset.collections.Create And Import')} + } menuList={[ { @@ -421,35 +433,9 @@ const Header = ({ - - - - - - {t('common:dataset.collections.Create And Import')} - - - + + {t('common:dataset.collections.Create And Import')} + } menuList={[ { @@ -618,6 +604,14 @@ const Header = ({ onClose={onCloseTemplateImportModal} /> )} + {isTagManageModalOpen && ( + { + onCloseTagManageModal(); + getData(1); + }} + /> + )} ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx deleted file mode 100644 index c945557dec18..000000000000 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx +++ /dev/null @@ -1,211 +0,0 @@ -import { Box, Button, Checkbox, Flex, Input, useDisclosure } from '@chakra-ui/react'; -import MyPopover from '@fastgpt/web/components/common/MyPopover'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import MyBox from '@fastgpt/web/components/common/MyBox'; -import { useContextSelector } from 'use-context-selector'; -import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { useTranslation } from 'next-i18next'; -import { CollectionPageContext } from './Context'; -import { isEqual } from 'lodash-es'; -import TagManageModal from './TagManageModal'; -import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; - -const HeaderTagPopOver = () => { - const { t } = useTranslation(); - - const { - searchDatasetTagsResult, - searchTagKey, - setSearchTagKey, - checkedDatasetTag, - setCheckedDatasetTag, - onCreateCollectionTag, - isCreateCollectionTagLoading - } = useContextSelector(DatasetPageContext, (v) => v); - - const { filterTags, setFilterTags, getData } = useContextSelector( - CollectionPageContext, - (v) => v - ); - - const checkedTags = filterTags; - - const { - isOpen: isTagManageModalOpen, - onOpen: onOpenTagManageModal, - onClose: onCloseTagManageModal - } = useDisclosure(); - - const checkTags = (tag: DatasetTagType) => { - let currentCheckedTags = []; - if (checkedTags.includes(tag._id)) { - currentCheckedTags = checkedTags.filter((t) => t !== tag._id); - setCheckedDatasetTag(checkedDatasetTag.filter((t) => t._id !== tag._id)); - } else { - currentCheckedTags = [...checkedTags, tag._id]; - setCheckedDatasetTag([...checkedDatasetTag, tag]); - } - if (isEqual(currentCheckedTags, filterTags)) return; - setFilterTags(currentCheckedTags); - }; - - return ( - <> - - - {t('dataset:tag.tags')} - - {checkedTags.length > 0 && ( - - {`(${checkedTags.length})`} - - )} - - - - - } - > - {({ onClose }) => ( - e.stopPropagation()}> - - setSearchTagKey(e.target.value)} - /> - - - - {searchTagKey && - !searchDatasetTagsResult.map((item) => item.tag).includes(searchTagKey) && ( - onCreateCollectionTag(searchTagKey)} - > - - - {t('dataset:tag.add') + ` "${searchTagKey}"`} - - - )} - - {[ - ...new Map( - [...checkedDatasetTag, ...searchDatasetTagsResult].map((item) => [item._id, item]) - ).values() - ].map((item) => { - const checked = checkedTags.includes(item._id); - return ( - { - e.preventDefault(); - checkTags(item); - }} - > - { - checkTags(item); - }} - size={'md'} - icon={} - /> - {item.tag} - - ); - })} - - - - - - - - )} - - {isTagManageModalOpen && ( - { - onCloseTagManageModal(); - getData(1); - }} - /> - )} - - ); -}; - -export default HeaderTagPopOver; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx new file mode 100644 index 000000000000..bc41f24723da --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx @@ -0,0 +1,300 @@ +import React, { useLayoutEffect, useRef, useState } from 'react'; +import { Box, Flex, type FlexProps } from '@chakra-ui/react'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import dayjs from 'dayjs'; +import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; +import { + DatasetCollectionTagTypeEnum, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; +import { + collectionTagValueKey, + isUsableCollectionTagFilterValue, + sortCollectionTagValues +} from '@fastgpt/global/core/dataset/tagUtils'; + +export type CollectionTagDisplayItem = string | { tag: string; value?: string | number | string[] }; + +export const OVERFLOW_CHIP_GAP_PX = 8; + +/** + * 根据测量宽度计算可见 chip 数量。空间不够时至少留 1 个，其余用 +n。 + */ +export const countVisibleOverflowChips = ({ + chipWidths, + overflowWidth, + containerWidth, + gapPx = OVERFLOW_CHIP_GAP_PX +}: { + chipWidths: number[]; + overflowWidth: number; + containerWidth: number; + gapPx?: number; +}): number => { + if (chipWidths.length === 0) return 0; + + let usedWidth = 0; + let visibleCount = chipWidths.length; + + for (let i = 0; i < chipWidths.length; i++) { + const isLast = i === chipWidths.length - 1; + const gap = isLast ? 0 : gapPx; + const reserved = isLast ? 0 : overflowWidth; + + if (usedWidth + chipWidths[i] + gap + reserved <= containerWidth) { + usedWidth += chipWidths[i] + gap; + continue; + } + + visibleCount = i; + break; + } + + return visibleCount === 0 ? 1 : visibleCount; +}; + +/** 列表/选项输入共用的 chip 溢出测量。itemKey 变化时重新量。 */ +export const useOverflowChipCount = ({ + itemKey, + itemCount, + gapPx = OVERFLOW_CHIP_GAP_PX +}: { + itemKey: unknown; + itemCount: number; + gapPx?: number; +}) => { + const containerRef = useRef(null); + const measureRef = useRef(null); + const [visibleCount, setVisibleCount] = useState(itemCount); + + useLayoutEffect(() => { + const container = containerRef.current; + const measure = measureRef.current; + if (!container || !measure) { + setVisibleCount(itemCount); + return; + } + + const calculate = () => { + const chipEls = Array.from(measure.querySelectorAll('[data-tag-chip]')) as HTMLElement[]; + const overflowEl = measure.querySelector('[data-overflow-chip]') as HTMLElement | null; + setVisibleCount( + countVisibleOverflowChips({ + chipWidths: chipEls.map((el) => el.offsetWidth), + overflowWidth: overflowEl?.offsetWidth ?? 0, + containerWidth: container.offsetWidth, + gapPx + }) + ); + }; + + calculate(); + const observer = new ResizeObserver(calculate); + observer.observe(container); + return () => observer.disconnect(); + }, [gapPx, itemCount, itemKey]); + + return { containerRef, measureRef, visibleCount }; +}; + +export const formatCollectionTagValueText = ( + value: string | number | string[] | undefined, + tagType?: DatasetTagType['tagType'] +): string => { + if (value == null || value === '') return ''; + if (Array.isArray(value)) return value.filter(Boolean).join('、'); + if (tagType === 'datetime') { + const d = dayjs(value); + return d.isValid() ? formatTime2YMDHM(d.valueOf()) : String(value); + } + return String(value); +}; + +/** + * 把列表返回的展示格式（名称或 { tag, value }）解析成标签定义和值。 + * 旧字符串按名称或 ID 找回定义；选项类旧字符串当作单个选项。 + */ +export const resolveDisplayedCollectionTag = ( + item: CollectionTagDisplayItem, + tags: DatasetTagType[] +) => { + const tagNameOrId = typeof item === 'string' ? item : item.tag; + const tagDoc = tags.find((tag) => tag.tag === tagNameOrId || String(tag._id) === tagNameOrId); + if (!tagDoc) return; + + const value = (() => { + if (typeof item !== 'string') { + if (item.value != null && item.value !== '') return item.value; + return tagDoc.tagType === DatasetCollectionTagTypeEnum.array ? [] : ''; + } + return tagDoc.tagType === DatasetCollectionTagTypeEnum.array ? [item] : item; + })(); + + return { tagDoc, value }; +}; + +/** + * 组装筛选弹窗右侧的值列表。 + * 选项类用标签管理里的预设 options，并与文件上已用但不在预设里的值取并集； + * 文本/数字/日期没有预设列表，只展示已用值。 + */ +export const buildTagFilterValues = ( + tag: Pick, + usedValues: Array = [] +): Array => { + const values = new Map(); + const addValue = (value: string | number) => { + if (!isUsableCollectionTagFilterValue(value)) return; + values.set(collectionTagValueKey(value), value); + }; + + if (tag.tagType === DatasetCollectionTagTypeEnum.array) { + for (const option of tag.options ?? []) addValue(option); + } + for (const value of usedValues) addValue(value); + + return sortCollectionTagValues(Array.from(values.values())); +}; + +/** 新格式展示「名称：值」；旧字符串标签只展示名称。 */ +export const formatCollectionTagChipText = ( + item: CollectionTagDisplayItem, + tagDefs: DatasetTagType[] = [] +): string => { + if (typeof item === 'string') return item; + + const tagType = tagDefs.find((def) => def.tag === item.tag)?.tagType; + const valueText = formatCollectionTagValueText(item.value, tagType); + return valueText ? `${item.tag}：${valueText}` : item.tag; +}; + +export const TAG_TOOLTIP_PROPS = { + placement: 'bottom' as const, + offset: [0, 10] as [number, number], + hasArrow: true, + arrowSize: 10, + px: 3, + py: 2, + borderRadius: 'sm', + fontSize: 'xs', + lineHeight: '18px', + color: 'myGray.800', + arrowShadowColor: 'rgba(19, 51, 107, 0.1)', + boxShadow: '0px 4px 5px rgba(19, 51, 107, 0.1), 0px 0px 0.5px rgba(19, 51, 107, 0.1)' +}; + +export const SaveActionIcon = ({ isEnabled }: { isEnabled: boolean }) => { + if (!isEnabled) { + return ; + } + + return ( + + + + + ); +}; + +export type TagActionButtonProps = { + label: string; + icon: React.ReactElement; + onClick?: () => void; + isDisabled?: boolean; + color?: string; + hoverColor?: string; + hoverIconClassName?: string; +}; + +export const TagActionButton = ({ + label, + icon, + onClick, + isDisabled = false, + color = 'myGray.500', + hoverColor, + hoverIconClassName +}: TagActionButtonProps) => ( + + + {icon} + + +); + +export const TagTableContainer = ({ children, ...props }: FlexProps) => ( + + {children} + +); + +export const TagTableHeader = ({ + columns, + children +}: { + columns: string; + children: React.ReactNode; +}) => ( + + {children} + +); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx index 3c77db4ba2bc..96e810bd0e59 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx @@ -1,516 +1,575 @@ -import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { Input, Button, Flex, Box, Checkbox } from '@chakra-ui/react'; -import MyModal from '@fastgpt/web/components/common/MyModal'; +import React, { useEffect, useRef, useState } from 'react'; +import { Box, Button, Flex, Input } from '@chakra-ui/react'; +import MyModal from '@fastgpt/web/components/v2/common/MyModal'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; +import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import MySelect from '@fastgpt/web/components/common/MySelect'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { CollectionPageContext } from './Context'; -import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; import { delDatasetCollectionTag, - getDatasetCollectionTags, - getDatasetCollections, - getTagUsage, - postAddTagsToCollections, + postCreateDatasetCollectionTag, updateDatasetCollectionTag } from '@/web/core/dataset/api/collection'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import MyInput from '@/components/MyInput'; -import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; -import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; +import { + DatasetCollectionTagTypeEnum, + DatasetCollectionTagTypeMap, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; -import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm'; -import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; +import MyPopover from '@fastgpt/web/components/common/MyPopover'; +import { SaveActionIcon, TagActionButton, TagTableContainer, TagTableHeader } from './TagCommon'; + +const TAG_TABLE_COLUMNS = 'minmax(0, 1fr) 180px 100px'; + +const TagOptionManagePopover = ({ + options, + isSaving, + onSave +}: { + options: string[]; + isSaving: boolean; + onSave: (options: string[]) => Promise; +}) => { + const { t } = useTranslation(); + const [draftOptions, setDraftOptions] = useState(options); + const savedOptionsRef = useRef(options); + const inputRefs = useRef<(HTMLInputElement | null)[]>([]); + + const persistOptions = async (nextOptions: string[]) => { + const normalizedOptions = [ + ...new Set(nextOptions.map((option) => option.trim()).filter(Boolean)) + ]; + setDraftOptions(nextOptions); + + try { + await onSave(normalizedOptions); + savedOptionsRef.current = normalizedOptions; + } catch { + setDraftOptions(savedOptionsRef.current); + } + }; + + const handleAddOption = () => { + setDraftOptions((prev) => { + const next = [...prev, '']; + setTimeout(() => { + inputRefs.current[next.length - 1]?.focus(); + }, 50); + return next; + }); + }; + + const handleUpdateOption = (index: number, value: string) => { + setDraftOptions((prev) => { + const next = [...prev]; + next[index] = value; + return next; + }); + }; + + const handleRemoveOption = (index: number) => { + void persistOptions(draftOptions.filter((_, i) => i !== index)); + }; + + const handleKeyDown = (index: number, e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + e.preventDefault(); + if (draftOptions[index]?.trim() && !isSaving) { + void persistOptions(draftOptions); + } + if (index === draftOptions.length - 1) { + handleAddOption(); + } else { + inputRefs.current[index + 1]?.focus(); + } + } + }; + + return ( + + + + } + > + {() => ( + + + + + {t('dataset:tag.add_option')} + + + + {draftOptions.length > 0 && ( + + {draftOptions.map((opt, index) => ( + + { + inputRefs.current[index] = el; + }} + value={opt} + flex={1} + minW={0} + h={'32px'} + px={3} + fontSize={'xs'} + lineHeight={'16px'} + borderRadius={'sm'} + border={'1px solid'} + borderColor={'myGray.200'} + placeholder={t('dataset:tag.enter_option')} + isDisabled={isSaving} + _focus={{ + borderColor: 'primary.600', + boxShadow: 'focus' + }} + onChange={(e) => handleUpdateOption(index, e.target.value)} + onKeyDown={(e) => handleKeyDown(index, e)} + /> + !isSaving && handleRemoveOption(index)} + > + + + + ))} + + )} + + )} + + ); +}; const TagManageModal = ({ onClose }: { onClose: () => void }) => { const { t } = useTranslation(); - const { datasetDetail, onCreateCollectionTag, loadAllDatasetTags, setSearchTagKey } = - useContextSelector(DatasetPageContext, (v) => v); - const { getData, pageNum, collections } = useContextSelector(CollectionPageContext, (v) => v); + const { datasetDetail, allDatasetTags, loadAllDatasetTags } = useContextSelector( + DatasetPageContext, + (v) => v + ); const tagInputRef = useRef(null); const editInputRef = useRef(null); - - const [currentAddTag, setCurrentAddTag] = useState< - (DatasetTagType & { collections: string[] }) | undefined - >(undefined); - const [newTag, setNewTag] = useState(undefined); - const [searchText, setSearchText] = useState(''); - + const [newTagType, setNewTagType] = useState(undefined); const [currentEditTagContent, setCurrentEditTagContent] = useState(undefined); const [currentEditTag, setCurrentEditTag] = useState(undefined); useEffect(() => { - if (newTag !== undefined && tagInputRef.current) { + if (newTag !== undefined) { tagInputRef.current?.focus(); } }, [newTag]); useEffect(() => { - if (currentEditTag !== undefined && editInputRef.current) { + if (currentEditTag !== undefined) { editInputRef.current?.focus(); } }, [currentEditTag]); + const { openConfirm: openDeleteConfirm, ConfirmModal: DeleteConfirmModal } = useConfirm({ + type: 'delete', + title: t('dataset:tag.delete_tag_confirm_title'), + content: t('dataset:tag.delete_tag_confirm_content') + }); + const { runAsync: onDeleteCollectionTag } = useRequest( - (tag: string) => + (tagId: string) => delDatasetCollectionTag({ datasetId: datasetDetail._id, - id: tag + id: tagId }), { onSuccess() { - refreshList(); - setSearchTagKey(''); loadAllDatasetTags(); }, - successToast: t('common:delete_success'), - errorToast: t('common:delete_failed') + successToast: t('dataset:tag.delete_success'), + errorToast: t('dataset:tag.delete_failed') + } + ); + + const { runAsync: onCreateCollectionTag } = useRequest( + ({ tag, tagType }: { tag: string; tagType: DatasetCollectionTagTypeEnum }) => + postCreateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tag, + tagType + }), + { + onSuccess() { + loadAllDatasetTags(); + }, + successToast: t('dataset:tag.create_success'), + errorToast: t('dataset:tag.create_failed') } ); const { runAsync: onUpdateCollectionTag } = useRequest( - async (tag: DatasetTagType) => { - return updateDatasetCollectionTag({ + (tag: DatasetTagType) => + updateDatasetCollectionTag({ datasetId: datasetDetail._id, tagId: tag._id, tag: tag.tag - }); - }, + }), { onSuccess() { - refreshList(); - setSearchTagKey(''); loadAllDatasetTags(); - } + }, + successToast: t('dataset:tag.save_success'), + errorToast: t('dataset:tag.save_failed') } ); - const { runAsync: onSaveCollectionTag } = useRequest( - async ({ - tag, - originCollectionIds, - collectionIds - }: { - tag: string; - originCollectionIds: string[]; - collectionIds: string[]; - }) => { - return postAddTagsToCollections({ - tag, - originCollectionIds, - collectionIds, - datasetId: datasetDetail._id - }); - }, + const { runAsync: onSaveTagOptions, loading: isSavingTagOptions } = useRequest( + ({ tag, options }: { tag: DatasetTagType; options: string[] }) => + updateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tagId: tag._id, + tag: tag.tag, + options + }), { - onFinally() { - getData(pageNum); - }, - successToast: t('common:save_success'), - errorToast: t('common:save_failed') + onSuccess: loadAllDatasetTags, + errorToast: t('dataset:tag.save_failed') } ); - // Tags list - const { - data: collectionTags, - ScrollData, - refreshList, - total: tagsTotal - } = useScrollPagination(getDatasetCollectionTags, { - pageSize: 10, - params: { - datasetId: datasetDetail._id, - searchText - }, - refreshDeps: [searchText], - EmptyTip: - }); + const tagTypeOptions = [ + DatasetCollectionTagTypeEnum.array, + DatasetCollectionTagTypeEnum.number, + DatasetCollectionTagTypeEnum.datetime + ].map((tagType) => ({ + label: t(DatasetCollectionTagTypeMap[tagType].label), + value: tagType + })); - // Collections list - const { data: collectionsList, ScrollData: ScrollDataCollections } = useScrollPagination( - getDatasetCollections, - { - pageSize: 30, - params: { - datasetId: datasetDetail._id, - simple: true, - searchText - }, - refreshDeps: [searchText], - EmptyTip: + const submitNewTag = async () => { + const tag = newTag?.trim(); + if (!tag || !newTagType || allDatasetTags.some((item) => item.tag === tag)) return; + + await onCreateCollectionTag({ tag, tagType: newTagType }); + setNewTag(undefined); + setNewTagType(undefined); + }; + + const submitUpdatedTag = async (tag: DatasetTagType) => { + const content = currentEditTagContent?.trim(); + if ( + content && + content !== tag.tag && + !allDatasetTags.some((item) => item._id !== tag._id && item.tag === content) + ) { + await onUpdateCollectionTag({ ...tag, tag: content }); } - ); + setCurrentEditTag(undefined); + setCurrentEditTagContent(undefined); + }; - const { data: tagUsages } = useRequest(() => getTagUsage(datasetDetail._id), { - manual: false, - refreshDeps: [collections] - }); + const cancelNewTag = () => { + setNewTag(undefined); + setNewTagType(undefined); + }; + + const canSaveNewTag = Boolean( + newTag?.trim() && newTagType && !allDatasetTags.some((item) => item.tag === newTag.trim()) + ); return ( + {t('dataset:tag.manage')} + + + } closeOnOverlayClick={false} + bodyStyles={{ + flex: 1, + minH: 0, + pb: 0, + overflow: 'hidden' + }} + footerStyles={{ + justifyContent: 'flex-start', + px: 8, + pt: 2, + pb: 8 + }} + footer={ + + } + borderRadius={'10px'} + sx={{ + '.chakra-modal__close-btn': { + top: 2, + right: 2, + w: '36px', + h: '36px' + } + }} > - {currentAddTag === undefined ? ( - <> - - - - {t('dataset:tag.total_tags', { - total: tagsTotal - })} - - { - setSearchText(e.target.value); - }} - /> - - - - {newTag !== undefined && ( - - setNewTag(e.target.value)} - ref={tagInputRef} - w={'200px'} - onBlur={async () => { - if (newTag && !collectionTags.map((item) => item.tag).includes(newTag)) { - await onCreateCollectionTag(newTag); - refreshList(); - } - setNewTag(undefined); - }} - /> - - )} - - - {collectionTags.map((item) => { - const tagUsage = tagUsages?.find((tagUsage) => tagUsage.tagId === item._id); - const collections = tagUsage?.collections || []; - const usage = collections.length; + + + {t('dataset:tag.name')} + {t('dataset:tag.attribute')} + {t('common:Operation')} + + + + {allDatasetTags.length === 0 ? ( + + ) : ( + allDatasetTags.map((tag) => { + const isEditing = currentEditTag?._id === tag._id; + const tagType = tag.tagType ?? DatasetCollectionTagTypeEnum.string; + const editedTagContent = currentEditTagContent?.trim(); + const canSaveEditedTag = Boolean( + editedTagContent && + editedTagContent !== tag.tag && + !allDatasetTags.some( + (item) => item._id !== tag._id && item.tag === editedTagContent + ) + ); return ( - - - { - setCurrentAddTag({ ...item, collections }); - }} - cursor={'pointer'} - > - {currentEditTag?._id !== item._id ? ( - - {item.tag} - - ) : ( - setCurrentEditTagContent(e.target.value)} - ref={editInputRef} - w={'200px'} - onBlur={() => { - if ( - currentEditTagContent && - !collectionTags - .map((item) => item.tag) - .includes(currentEditTagContent) - ) { - onUpdateCollectionTag({ - ...item, - tag: currentEditTagContent - }); + + {isEditing ? ( + setCurrentEditTagContent(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + if (canSaveEditedTag) { + void submitUpdatedTag(tag); } + } + }} + /> + ) : ( + + {tag.tag} + + )} + + + {t(DatasetCollectionTagTypeMap[tagType].label)} + {tagType === DatasetCollectionTagTypeEnum.array && ( + onSaveTagOptions({ tag, options })} + /> + )} + + + {isEditing ? ( + <> + } + isDisabled={!canSaveEditedTag} + hoverIconClassName={canSaveEditedTag ? 'tag-save-icon-hover' : undefined} + onClick={() => void submitUpdatedTag(tag)} + /> + } + hoverColor={'myGray.700'} + onClick={() => { setCurrentEditTag(undefined); setCurrentEditTagContent(undefined); }} /> - )} - {`(${usage})`} - - { - setCurrentAddTag({ ...item, collections }); - setSearchText(''); - }} - cursor={'pointer'} - > - - - { - setCurrentEditTag(item); - editInputRef.current?.focus(); - }} - > - - - - - - } - onConfirm={() => onDeleteCollectionTag(item._id)} - /> + + ) : ( + <> + } + hoverColor={'primary.700'} + onClick={() => { + setCurrentEditTag(tag); + setCurrentEditTagContent(tag.tag); + }} + /> + } + hoverColor={'red.600'} + onClick={() => { + openDeleteConfirm({ + onConfirm: () => onDeleteCollectionTag(tag._id) + })(); + }} + /> + + )} - + ); - })} - - - ) : ( - - )} - - ); -}; + }) + )} -export default TagManageModal; - -const AddTagToCollections = ({ - currentAddTag, - setCurrentAddTag, - onSaveCollectionTag, - setSearchText, - collectionsList, - ScrollDataCollections -}: { - currentAddTag: DatasetTagType & { collections: string[] }; - setCurrentAddTag: (tag: (DatasetTagType & { collections: string[] }) | undefined) => void; - onSaveCollectionTag: ({ - tag, - originCollectionIds, - collectionIds - }: { - tag: string; - originCollectionIds: string[]; - collectionIds: string[]; - }) => void; - setSearchText: (text: string) => void; - collectionsList: DatasetCollectionsListItemType[]; - ScrollDataCollections: ReturnType['ScrollData']; -}) => { - const { t } = useTranslation(); - - const [selectedCollections, setSelectedCollections] = useState( - currentAddTag.collections - ); - const [originCollections, setOriginCollections] = useState(currentAddTag.collections); - - const formatCollections = useMemo( - () => - collectionsList.map((collection) => { - const icon = getCollectionIcon({ type: collection.type, name: collection.name }); - return { - id: collection._id, - tags: collection.tags, - name: collection.name, - icon - }; - }), - [collectionsList] - ); - - return ( - <> - - { - setCurrentAddTag(undefined); - setSearchText(''); - }} - /> - { - - - {currentAddTag.tag} - + {newTag !== undefined && ( {`(${selectedCollections.length})`} - - } - - { - setSearchText(e.target.value); - }} - /> - - - - {formatCollections.map((collection) => { - return ( - { - setSelectedCollections((prev) => { - if (prev.includes(collection.id)) { - return prev.filter((id) => id !== collection.id); - } else { - return [...prev, collection.id]; - } - }); - }} + h={'80px'} + flexShrink={0} + borderBottom={'1px solid'} + borderColor={'myGray.150'} > - } - onChange={() => { - setSelectedCollections((prev) => { - if (prev.includes(collection.id)) { - return prev.filter((id) => id !== collection.id); - } else { - return [...prev, collection.id]; - } - }); - }} - isChecked={selectedCollections.includes(collection.id)} - /> - - - {collection.name} + + setNewTag(e.target.value)} + /> - - ); - })} - - + + + value={newTagType} + placeholder={t('dataset:tag.select_attribute')} + list={tagTypeOptions} + width={'100%'} + h={'36px'} + fontSize={'sm'} + lineHeight={'20px'} + letterSpacing={'0.25px'} + onChange={(val) => setNewTagType(val)} + menuPlacement={'bottom-start'} + /> + + + } + isDisabled={!canSaveNewTag} + hoverIconClassName={canSaveNewTag ? 'tag-save-icon-hover' : undefined} + onClick={() => void submitNewTag()} + /> + } + hoverColor={'myGray.700'} + onClick={cancelNewTag} + /> + + + )} + + + + ); }; + +export default React.memo(TagManageModal); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx new file mode 100644 index 000000000000..44be11df78a1 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx @@ -0,0 +1,679 @@ +import React, { useMemo, useState } from 'react'; +import { + Box, + Checkbox, + Flex, + Input, + type InputProps, + Popover, + PopoverContent, + PopoverTrigger, + Portal, + useDisclosure +} from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { SingleDateTimePicker } from '@fastgpt/web/components/common/DateTimePicker'; +import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput'; +import { + DatasetCollectionTagTypeEnum, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; +import { OVERFLOW_CHIP_GAP_PX, useOverflowChipCount } from './TagCommon'; + +export const tagInputBaseStyles: InputProps = { + h: '36px', + borderRadius: 'sm', + border: '1px solid', + borderColor: 'myGray.200', + bg: 'white', + fontSize: 'sm', + color: 'myGray.900', + _hover: { + borderColor: 'primary.300' + }, + _focus: { + borderColor: 'primary.600', + boxShadow: 'focus' + }, + _placeholder: { + color: 'myGray.500' + } +}; +export const StringTagInput = ({ + value, + onChange, + placeholder +}: { + value?: string; + onChange: (val: string) => void; + placeholder?: string; +}) => { + const { t } = useTranslation(); + + return ( + onChange(e.target.value)} + /> + ); +}; + +export const NumberTagInput = ({ + value, + onChange, + placeholder, + showStepper +}: { + value?: number | string; + onChange: (val: number | '') => void; + placeholder?: string; + showStepper?: boolean; +}) => { + const { t } = useTranslation(); + const placeholderText = placeholder || t('dataset:tag.fill_number'); + + if (showStepper) { + return ( + onChange(val === undefined ? '' : val)} + /> + ); + } + + return ( + { + const val = e.target.value; + if (val === '') { + onChange(''); + } else { + const num = Number(val); + if (!Number.isNaN(num)) { + onChange(num); + } + } + }} + /> + ); +}; + +export const DateTimeTagInput = ({ + value, + onChange, + placeholder +}: { + value?: number | string; + onChange: (val: number) => void; + placeholder?: string; +}) => ( + +); + +/** 设置标签弹窗里的标签名称下拉：搜索已有标签并单选，底部「标签管理」打开标签管理弹窗。 */ +export const TagNameSelect = ({ + value, + options, + onChange, + onManage +}: { + value?: string; + options: { label: string; value: string }[]; + onChange: (val: string) => void; + onManage: () => void; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const [search, setSearch] = useState(''); + + const selected = options.find((opt) => opt.value === value); + const filteredOptions = useMemo(() => { + const trimmed = search.trim().toLowerCase(); + if (!trimmed) return options; + return options.filter((opt) => opt.label.toLowerCase().includes(trimmed)); + }, [options, search]); + + const handleClose = () => { + setSearch(''); + onClose(); + }; + + return ( + + + + + {selected?.label || t('dataset:tag.select_tag')} + + + + + + + + setSearch(e.target.value)} + /> + + {filteredOptions.length === 0 ? ( + + {t('common:no_select_data')} + + ) : ( + filteredOptions.map((opt) => { + const isSelected = opt.value === value; + return ( + { + if (opt.value !== value) onChange(opt.value); + handleClose(); + }} + > + {opt.label} + + ); + }) + )} + + + { + handleClose(); + onManage(); + }} + > + + + {t('dataset:tag.manage')} + + + + + + + ); +}; + +const ARRAY_OPTION_HOVER_BG = 'rgba(17, 24, 36, 0.05)'; + +const ArraySelectedChip = ({ + opt, + onRemove +}: { + opt: string; + onRemove?: (opt: string, e: React.MouseEvent) => void; +}) => ( + + + {opt} + + {onRemove && ( + onRemove(opt, e)} + > + + + )} + +); + +const ArrayOverflowChip = ({ count }: { count: number }) => ( + + {`+${count}`} + +); + +export const ArrayTagSelect = ({ + options, + value = [], + onChange, + onCreateOption, + placeholder +}: { + options: string[]; + value?: string[]; + onChange: (val: string[]) => void; + onCreateOption?: (option: string) => void; + placeholder?: string; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const [search, setSearch] = useState(''); + const { containerRef, measureRef, visibleCount } = useOverflowChipCount({ + itemKey: value, + itemCount: value.length + }); + + const allOptions = useMemo(() => { + const list = [...options]; + for (const v of value) { + if (v && !list.includes(v)) { + list.push(v); + } + } + return list.filter(Boolean); + }, [options, value]); + + const filteredOptions = useMemo(() => { + const trimmed = search.trim().toLowerCase(); + if (!trimmed) return allOptions; + return allOptions.filter((opt) => opt.toLowerCase().includes(trimmed)); + }, [search, allOptions]); + + const canCreate = Boolean( + search.trim() && !allOptions.some((opt) => opt.toLowerCase() === search.trim().toLowerCase()) + ); + + const handleCreateOption = () => { + const trimmed = search.trim(); + if (!trimmed) return; + if (!value.includes(trimmed)) { + onChange([...value, trimmed]); + } + onCreateOption?.(trimmed); + setSearch(''); + }; + + const handleToggleOption = (opt: string) => { + if (value.includes(opt)) { + onChange(value.filter((v) => v !== opt)); + } else { + onChange([...value, opt]); + } + }; + + const handleToggleAll = () => { + if (allOptions.length === 0) return; + const isAllSelected = allOptions.every((opt) => value.includes(opt)); + if (isAllSelected) { + onChange([]); + } else { + onChange(allOptions); + } + }; + + const handleRemoveOption = (opt: string, e: React.MouseEvent) => { + e.stopPropagation(); + onChange(value.filter((v) => v !== opt)); + }; + + const visibleValues = value.slice(0, visibleCount); + const overflowCount = Math.max(value.length - visibleCount, 0); + + return ( + { + setSearch(''); + onClose(); + }} + placement={'bottom-start'} + closeOnBlur + matchWidth + > + + + + {value.length === 0 ? ( + + {placeholder || t('dataset:tag.select_options')} + + ) : ( + <> + + {value.map((opt) => ( + + ))} + + + + {visibleValues.map((opt) => ( + + ))} + {overflowCount > 0 && } + + + )} + + + + + + + + setSearch(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + handleCreateOption(); + } + }} + /> + + {canCreate && ( + + + {t('dataset:tag.create_option')} + + + {search.trim()} + + + )} + + {!search.trim() && allOptions.length > 0 && ( + <> + + + {t('common:All')} + + + + + )} + + + {filteredOptions.length === 0 && !canCreate ? ( + + {t('common:no_select_data')} + + ) : ( + filteredOptions.map((opt) => { + const isChecked = value.includes(opt); + return ( + handleToggleOption(opt)} + > + handleToggleOption(opt)} + onClick={(e) => e.stopPropagation()} + size={'sm'} + icon={} + /> + + {opt} + + + ); + }) + )} + + + + + + ); +}; + +/** 按标签类型渲染对应输入。未选标签时展示禁用占位。 */ +export const TagValueField = ({ + tag, + value, + onChange, + onCreateOption, + disabledPlaceholder, + numberShowStepper, + numberPlaceholder, + arrayPlaceholder +}: { + tag?: DatasetTagType; + value: string | number | string[]; + onChange: (val: string | number | string[]) => void; + onCreateOption?: (option: string) => void; + disabledPlaceholder?: string; + numberShowStepper?: boolean; + numberPlaceholder?: string; + arrayPlaceholder?: string; +}) => { + const { t } = useTranslation(); + + if (!tag) { + return ( + + ); + } + + if (tag.tagType === DatasetCollectionTagTypeEnum.number) { + return ( + onChange(val)} + /> + ); + } + + if (tag.tagType === DatasetCollectionTagTypeEnum.datetime) { + return onChange(val)} />; + } + + if (tag.tagType === DatasetCollectionTagTypeEnum.array) { + return ( + onChange(val)} + onCreateOption={onCreateOption} + /> + ); + } + + return ( + onChange(val)} + /> + ); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx index 30f8128a5d71..1628be7e2b0b 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx @@ -1,253 +1,119 @@ -import { Box, Checkbox, Flex, Input } from '@chakra-ui/react'; -import MyPopover from '@fastgpt/web/components/common/MyPopover'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import MyBox from '@fastgpt/web/components/common/MyBox'; -import { putDatasetCollectionById } from '@/web/core/dataset/api/collection'; +import { Flex } from '@chakra-ui/react'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { useTranslation } from 'next-i18next'; -import { useMemo, useRef, useState } from 'react'; -import { useDeepCompareEffect } from 'ahooks'; -import { - type DatasetCollectionItemType, - type DatasetTagType -} from '@fastgpt/global/core/dataset/type'; -import { isEqual } from 'lodash-es'; +import { useMemo } from 'react'; +import { type DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type'; import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import MyPopover from '@fastgpt/web/components/common/MyPopover'; +import MyTag from '@fastgpt/web/components/common/Tag/index'; +import { formatCollectionTagChipText, useOverflowChipCount } from './TagCommon'; + +const TAG_CHIP_PROPS = { + colorSchema: 'cyan' as const, + type: 'fill' as const, + h: '20px', + px: 2, + flexShrink: 0, + fontSize: 'mini', + fontWeight: 'medium', + borderRadius: 'xs' as const +}; + +const TagChip = ({ text }: { text: string }) => ( + + {text} + +); const TagsPopOver = ({ - currentCollection, - hoverBg = 'myGray.50' + currentCollection }: { currentCollection: DatasetCollectionItemType | DatasetCollectionsListItemType; - hoverBg?: string; }) => { - const { t } = useTranslation(); - const { - searchTagKey, - setSearchTagKey, - searchDatasetTagsResult, - allDatasetTags, - onCreateCollectionTag, - isCreateCollectionTagLoading - } = useContextSelector(DatasetPageContext, (v) => v); - - const [collectionTags, setCollectionTags] = useState(() => - (currentCollection.tags ?? []) - .map((tag) => (typeof tag === 'string' ? tag : tag.tag)) - .filter(Boolean) - ); - const [checkedTags, setCheckedTags] = useState([]); - const [showTagManage, setShowTagManage] = useState(false); - const [isUpdateLoading, setIsUpdateLoading] = useState(false); + const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags); - const tagList = useMemo( + const chipItems = useMemo( () => - (collectionTags - ?.map((item) => allDatasetTags.find((tag) => tag.tag === item) ?? null) - .filter((tag) => tag !== null) as DatasetTagType[]) || [], - [collectionTags, allDatasetTags] + (currentCollection.tags ?? []) + .map((item, index) => ({ + id: typeof item === 'string' ? item : `${item.tag}-${index}`, + text: formatCollectionTagChipText(item, allDatasetTags) + })) + .filter((item) => item.text), + [allDatasetTags, currentCollection.tags] ); - const [visibleTags, setVisibleTags] = useState(tagList); - const [overflowTags, setOverflowTags] = useState([]); - const containerRef = useRef(null); + const { containerRef, measureRef, visibleCount } = useOverflowChipCount({ + itemKey: chipItems, + itemCount: chipItems.length + }); - useDeepCompareEffect(() => { - const calculateTags = () => { - if (!containerRef.current || !tagList) return; + if (chipItems.length === 0) return null; - const containerWidth = containerRef.current.offsetWidth; - const tagWidth = 11; - let totalWidth = 30; - let visibleCount = 0; - - for (let i = 0; i < tagList.length; i++) { - const tag = tagList[i]; - const estimatedWidth = tag.tag.length * tagWidth + 16; // 加上左右 padding 的宽度 - if (totalWidth + estimatedWidth <= containerWidth) { - totalWidth += estimatedWidth; - visibleCount++; - } else { - break; - } - } - - setVisibleTags(tagList.slice(0, visibleCount)); - setOverflowTags(tagList.slice(visibleCount)); - }; - - setTimeout(calculateTags, 100); - setCheckedTags(tagList); - - window.addEventListener('resize', calculateTags); - - return () => { - window.removeEventListener('resize', calculateTags); - }; - }, [tagList]); + const visibleTags = chipItems.slice(0, visibleCount); + const overflowTags = chipItems.slice(visibleCount); return ( - { - e.stopPropagation(); - setShowTagManage(true); - }} - cursor={'pointer'} - > - - {visibleTags.map((item, index) => ( - - {item.tag} - - ))} - - {overflowTags.length > 0 && ( - - {`+${overflowTags.length}`} - - )} - - } - onCloseFunc={async () => { - setSearchTagKey(''); - - setShowTagManage(false); - if (isEqual(checkedTags, tagList) || !showTagManage) return; - setIsUpdateLoading(true); - await putDatasetCollectionById({ - id: currentCollection._id, - tags: checkedTags.map((tag) => tag.tag) - }); - setCollectionTags(checkedTags.map((tag) => tag.tag)); - setIsUpdateLoading(false); - }} - display={showTagManage || overflowTags.length > 0 ? 'block' : 'none'} - > - {({}) => ( - <> - {showTagManage ? ( - e.stopPropagation()}> - - setSearchTagKey(e.target.value)} - /> - - - {searchTagKey && - !searchDatasetTagsResult.map((item) => item.tag).includes(searchTagKey) && ( - onCreateCollectionTag(searchTagKey)} - > - - - {t('dataset:tag.add') + ` "${searchTagKey}"`} - - - )} - {searchDatasetTagsResult?.map((item) => { - const tagsList = checkedTags.map((tag) => tag.tag); - return ( - { - e.preventDefault(); - if (tagsList.includes(item.tag)) { - setCheckedTags(checkedTags.filter((t) => t.tag !== item.tag)); - } else { - setCheckedTags([...checkedTags, item]); - } - }} - > - { - if (e.target.checked) { - setCheckedTags([...checkedTags, item]); - } else { - setCheckedTags(checkedTags.filter((t) => t._id !== item._id)); - } - }} - icon={} - /> - {item.tag} - - ); - })} - - - ) : ( - - {overflowTags.map((tag, index) => ( - - {tag.tag} - - ))} - - )} - - )} - + + + {chipItems.map((item) => ( + + ))} + + {`+${chipItems.length}`} + + + + {visibleTags.map((item) => ( + + ))} + {overflowTags.length > 0 && ( + e.stopPropagation()}> + + {`+${overflowTags.length}`} + + + } + > + {() => ( + e.stopPropagation()}> + {overflowTags.map((item) => ( + + ))} + + )} + + )} + + ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx index 0c9aaa8450bf..0afc00317485 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx @@ -29,6 +29,7 @@ import MyMenu from '@fastgpt/web/components/common/MyMenu'; import { useEditTitle } from '@/web/common/hooks/useEditTitle'; import { DatasetCollectionTypeEnum, + DatasetTypeEnum, DatasetStatusEnum, DatasetCollectionSyncResultMap, DatasetCollectionDataProcessModeMap @@ -62,6 +63,8 @@ import { hasDatasetTrainingError as checkDatasetTrainingError } from '@/web/core const Header = dynamic(() => import('./Header')); const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip')); +const CollectionTagSetModal = dynamic(() => import('./CollectionTagSetModal')); +const CollectionTagBatchModal = dynamic(() => import('./CollectionTagBatchModal')); const CollectionCard = () => { const BoxRef = useRef(null); @@ -77,6 +80,8 @@ const CollectionCard = () => { }>(); const [isTrainingErrorModalOpen, setIsTrainingErrorModalOpen] = useState(false); const [hasDatasetTrainingError, setHasDatasetTrainingError] = useState(false); + const [tagSetCollection, setTagSetCollection] = useState(); + const [isBatchTagModalOpen, setIsBatchTagModalOpen] = useState(false); const { collections, @@ -113,9 +118,9 @@ const CollectionCard = () => { isSelected, getRowSelectionProps, setSelectedItems, - FloatingActionBar, isSelecteAll, - selectAllTrigger + selectAllTrigger, + hasSelections } = useTableMultipleSelect({ list: formatCollections, getItemId: (e) => e._id @@ -190,6 +195,10 @@ const CollectionCard = () => { () => !!formatCollections.find((item) => item.trainingAmount > 0), [formatCollections] ); + const enabledCount = useMemo( + () => formatCollections.filter((item) => !item.forbid).length, + [formatCollections] + ); useRequest( async () => { @@ -229,10 +238,16 @@ const CollectionCard = () => { } }); - const isLoading = isUpdating || isSyncing || isGetting || isDropping; + const isPageLoading = isUpdating || isSyncing || isDropping; return ( - + {/* header */}
{ /> {/* collection table */} - - - - - - - - - - - - - - - {formatCollections.map((collection) => ( - { - if (collection.type === DatasetCollectionTypeEnum.folder) { - router.push({ - query: { - datasetId: datasetDetail._id, - parentId: collection._id - } - }); - } else { - router.push({ - query: { - datasetId: datasetDetail._id, - collectionId: collection._id, - currentTab: TabEnum.dataCard - } - }); - } + + +
- - - {t('common:Name')} - - {t('dataset:collection.training_type')}{t('dataset:collection_data_count')}{t('dataset:collection.Create update time')}{t('common:Status')}{t('dataset:Enable')} -
- + + - - - + + + + + + + + + {formatCollections.map((collection) => ( + { + if (collection.type === DatasetCollectionTypeEnum.folder) { + router.push({ + query: { + datasetId: datasetDetail._id, + parentId: collection._id + } + }); + } else { + router.push({ + query: { + datasetId: datasetDetail._id, + collectionId: collection._id, + currentTab: TabEnum.dataCard + } }); - }} - > - - {t(collection.statusText as any)} - - - - - - - + + + + + - - ))} - -
- - e.stopPropagation()}> - toggleSelect(collection)} - /> - - - - - - - {collection.name} - - - - {feConfigs?.isPlus && !!collection.tags?.length && ( - - )} - + } + }, + tbody: { + tr: { + h: '80px', + td: { + h: '80px', + py: 0, + px: 6, + borderBottom: 'sm', + borderLeftRadius: 0, + borderRightRadius: 0 + } + } + } + }} + > +
+ + + {t('common:Name')} - - - {collection.trainingType - ? t( - (DatasetCollectionDataProcessModeMap[collection.trainingType]?.label || - '-') as any - ) - : '-'} - {collection.dataAmount || '-'} - {formatTime2YMDHM(collection.createTime)} - {formatTime2YMDHM(collection.updateTime)} - - - { - e.stopPropagation(); - setTrainingStatesCollection({ - collectionId: collection._id, - permission: collection.permission + + {t('dataset:collection.training_type')}{t('dataset:collection_data_count')}{t('dataset:collection.Create update time')}{t('common:Status')} + {t('dataset:Enable')}({enabledCount}) + {t('common:Operation')}
e.stopPropagation()}> - - onUpdateCollection({ - id: collection._id, - forbid: !e.target.checked - }) + } } - /> - e.stopPropagation()}> - {collection.permission.hasWritePer && ( - + })} + > + + + e.stopPropagation()}> + toggleSelect(collection)} + /> + + + - + + + {collection.name} + + + + {feConfigs?.isPlus && !!collection.tags?.length && ( + + + + )} + + + + {collection.trainingType + ? t( + (DatasetCollectionDataProcessModeMap[collection.trainingType]?.label || + '-') as any + ) + : '-'} + {collection.dataAmount || '-'} + {formatTime2YMDHM(collection.createTime)} + {formatTime2YMDHM(collection.updateTime)} + + + { + e.stopPropagation(); + setTrainingStatesCollection({ + collectionId: collection._id, + permission: collection.permission + }); + }} + > + + {t(collection.statusText as any)} + + + + + e.stopPropagation()}> + + onUpdateCollection({ + id: collection._id, + forbid: !e.target.checked + }) } - menuList={[ - { - children: [ - ...(collectionCanSync(collection.type) - ? [ - { - label: ( - - - {t('dataset:collection_sync')} - - ), - onClick: () => - openSyncConfirm({ - onConfirm: () => { - onclickStartSync(collection._id); - } - })() - } - ] - : []), - { - label: ( - - - {t('common:Move')} - - ), - onClick: () => - setMoveCollectionData({ collectionId: collection._id }) - }, - { - label: ( - - - {t('common:Rename')} - - ), - onClick: () => - onOpenEditTitleModal({ - defaultVal: collection.name, - onSuccess: (newName) => - onUpdateCollection({ - id: collection._id, - name: newName - }) - }) - } - ] - }, - { - children: [ - { - label: ( - - - {t('common:Delete')} - - ), - type: 'danger', - onClick: () => - openDeleteConfirm({ - onConfirm: () => onDelCollection([collection._id]), - customContent: - collection.type === DatasetCollectionTypeEnum.folder - ? t( - 'common:dataset.collections.Confirm to delete the folder' - ) - : t('common:dataset.Confirm to delete the file') - })() - } - ] - } - ]} /> - )} -
+ + e.stopPropagation()}> + {collection.permission.hasWritePer && ( + + + + } + menuList={[ + { + children: [ + ...(collectionCanSync(collection.type) + ? [ + { + label: ( + + + {t('dataset:collection_sync')} + + ), + onClick: () => + openSyncConfirm({ + onConfirm: () => { + onclickStartSync(collection._id); + } + })() + } + ] + : []), + { + label: ( + + + {t('common:Move')} + + ), + onClick: () => + setMoveCollectionData({ collectionId: collection._id }) + }, + { + label: ( + + + {t('common:Rename')} + + ), + onClick: () => + onOpenEditTitleModal({ + defaultVal: collection.name, + onSuccess: (newName) => + onUpdateCollection({ + id: collection._id, + name: newName + }) + }) + }, + ...(feConfigs?.isPlus && + datasetDetail.type !== DatasetTypeEnum.websiteDataset + ? [ + { + label: ( + + + {t('dataset:tag.set')} + + ), + onClick: () => setTagSetCollection(collection) + } + ] + : []) + ] + }, + { + children: [ + { + label: ( + + + {t('common:Delete')} + + ), + type: 'danger', + onClick: () => + openDeleteConfirm({ + onConfirm: () => onDelCollection([collection._id]), + customContent: + collection.type === DatasetCollectionTypeEnum.folder + ? t( + 'common:dataset.collections.Confirm to delete the folder' + ) + : t('common:dataset.Confirm to delete the file') + })() + } + ] + } + ]} + /> + )} + + + ))} + + - {total === 0 && } - + {total === 0 && } + + - + {total > pageSize && !hasSelections && ( + + + + )} + + {/* 勾选后浮在卡片底部中央，距底边 12px */} + {hasSelections && ( + + + + + {t('dataset:collection.select_all_filtered')} + + + {t('dataset:tag.filter_selected')} + + {selectedItems.length} + + {t('dataset:tag.filter_item')} + + + + {datasetDetail.permission.hasWritePer && + datasetDetail.type !== DatasetTypeEnum.websiteDataset && + feConfigs?.isPlus && ( + + )} - } - > - {total > pageSize && ( - - - - )} - + + )} + {!!tagSetCollection && ( + setTagSetCollection(undefined)} + onSuccess={() => { + getData(pageNum); + setTagSetCollection(undefined); + }} + /> + )} + + {isBatchTagModalOpen && ( + setIsBatchTagModalOpen(false)} + onSuccess={() => { + getData(pageNum); + setSelectedItems([]); + setIsBatchTagModalOpen(false); + }} + /> + )} + {!!trainingStatesCollection && ( + tagType === DatasetCollectionTagTypeEnum.array ? [] : ''; + +export const createEmptyTagRow = (): CollectionTagRow => ({ + id: getNanoid(), + tagId: '', + value: '' +}); + +/** 把集合上的展示格式标签灌进设置弹窗表格；没有可解析项时给一行空行。 */ +export const collectionTagsToRows = ( + items: CollectionTagDisplayItem[] | undefined, + tags: DatasetTagType[] +): CollectionTagRow[] => { + const rows: CollectionTagRow[] = []; + for (const item of items ?? []) { + const resolved = resolveDisplayedCollectionTag(item, tags); + if (!resolved) continue; + rows.push({ + id: getNanoid(), + tagId: String(resolved.tagDoc._id), + value: resolved.value + }); + } + return rows.length > 0 ? rows : [createEmptyTagRow()]; +}; + +/** 行数据是否已选标签且填了对应类型的值，用于禁用「添加」和底部确定。 */ +export const isTagRowComplete = ( + row: Pick, + tags: DatasetTagType[] +) => { + if (!row.tagId) return false; + const tagDoc = tags.find((tag) => String(tag._id) === row.tagId); + if (!tagDoc) return false; + + const tagType = tagDoc.tagType ?? DatasetCollectionTagTypeEnum.string; + if (tagType === DatasetCollectionTagTypeEnum.string) { + return typeof row.value === 'string' && row.value.trim().length > 0; + } + if ( + tagType === DatasetCollectionTagTypeEnum.number || + tagType === DatasetCollectionTagTypeEnum.datetime + ) { + return typeof row.value === 'number' && Number.isFinite(row.value); + } + if (tagType === DatasetCollectionTagTypeEnum.array) { + return Array.isArray(row.value) && row.value.length > 0; + } + return false; +}; + +/** 当前行可选的标签：排除其它行已选中的 tagId。 */ +export const unusedTagSelectOptions = ( + tags: DatasetTagType[], + rows: Array<{ id: string; tagId: string }>, + currentRowId: string +) => { + const otherSelectedTagIds = new Set( + rows.filter((row) => row.id !== currentRowId && Boolean(row.tagId)).map((row) => row.tagId) + ); + + return tags + .filter((tag) => !otherSelectedTagIds.has(String(tag._id))) + .map((tag) => ({ + label: tag.tag, + value: String(tag._id) + })); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts new file mode 100644 index 000000000000..4f79aac6ff22 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts @@ -0,0 +1,36 @@ +import { useContextSelector } from 'use-context-selector'; +import { useTranslation } from 'next-i18next'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { updateDatasetCollectionTag } from '@/web/core/dataset/api/collection'; + +/** 给选项类标签追加一个预设 option，并刷新当前知识库标签定义。 */ +export const useAppendDatasetTagOption = () => { + const { t } = useTranslation(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags); + const loadAllDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadAllDatasetTags); + + return useRequest( + ({ tagId, option }: { tagId: string; option: string }) => { + const tag = allDatasetTags.find((item) => String(item._id) === tagId); + if (!tag) return Promise.resolve(); + + const nextOptions = [...new Set([...(tag.options ?? []), option.trim()].filter(Boolean))]; + if (nextOptions.length === (tag.options ?? []).length) { + return Promise.resolve(); + } + return updateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tagId: tag._id, + tag: tag.tag, + options: nextOptions + }); + }, + { + refreshDeps: [datasetDetail._id, allDatasetTags], + onSuccess: loadAllDatasetTags, + errorToast: t('dataset:tag.save_failed') + } + ); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/useCollectionTagRows.ts b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useCollectionTagRows.ts new file mode 100644 index 000000000000..d44ccc6fc1c9 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useCollectionTagRows.ts @@ -0,0 +1,76 @@ +import { useMemo, useState } from 'react'; +import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; +import { emptyTagRowValue, isTagRowComplete, type CollectionTagRow } from './tagForm'; + +/** + * 设置标签 / 批量添加共用的表格行状态。 + * 未编辑时跟随 initialRows（设置弹窗等标签定义加载完再灌入已有值）；编辑后锁在 draft 上。 + */ +export const useCollectionTagRows = ({ + tags, + initialRows, + createRow, + patchOnTagChange, + onRowDeleted +}: { + tags: DatasetTagType[]; + initialRows: T[]; + createRow: () => T; + patchOnTagChange?: (tagDoc?: DatasetTagType) => Partial; + onRowDeleted?: () => void; +}) => { + const [draftRows, setDraftRows] = useState(); + const rows = draftRows ?? initialRows; + + const hasIncompleteRow = useMemo( + () => rows.some((row) => !isTagRowComplete(row, tags)), + [rows, tags] + ); + + const isAddDisabled = hasIncompleteRow || tags.length === 0 || rows.length >= tags.length; + + const updateRows = (updater: (current: T[]) => T[]) => { + setDraftRows((prev) => updater(prev ?? rows)); + }; + + const handleAddRow = () => { + if (isAddDisabled) return; + updateRows((current) => [...current, createRow()]); + }; + + const handleDeleteRow = (id: string) => { + updateRows((current) => current.filter((row) => row.id !== id)); + onRowDeleted?.(); + }; + + const handleTagChange = (rowId: string, newTagId: string) => { + const tagDoc = tags.find((tag) => String(tag._id) === newTagId); + updateRows((current) => + current.map((row) => + row.id === rowId + ? { + ...row, + tagId: newTagId, + value: emptyTagRowValue(tagDoc?.tagType), + ...patchOnTagChange?.(tagDoc) + } + : row + ) + ); + }; + + const handleValueChange = (rowId: string, value: string | number | string[]) => { + updateRows((current) => current.map((row) => (row.id === rowId ? { ...row, value } : row))); + }; + + return { + rows, + hasIncompleteRow, + isAddDisabled, + updateRows, + handleAddRow, + handleDeleteRow, + handleTagChange, + handleValueChange + }; +}; diff --git a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx index 7c8e37124d42..ebd3c22b931a 100644 --- a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx +++ b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx @@ -177,7 +177,9 @@ const DataCard = () => { )} {feConfigs?.isPlus && !!collection?.tags?.length && ( - + + + )} diff --git a/projects/app/src/pages/api/admin/migrateTags.ts b/projects/app/src/pages/api/admin/migrateTags.ts index 235cce46add1..6a6f4dc8e985 100644 --- a/projects/app/src/pages/api/admin/migrateTags.ts +++ b/projects/app/src/pages/api/admin/migrateTags.ts @@ -3,7 +3,11 @@ import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { DEFAULT_TAG, type CollectionTagValueType } from '@fastgpt/global/core/dataset/type'; +import { + DEFAULT_TAG, + DatasetCollectionTagTypeEnum, + type CollectionTagValueType +} from '@fastgpt/global/core/dataset/type'; import { getLogger } from '@fastgpt/service/common/logger'; const logger = getLogger(['migrateTags']); @@ -58,16 +62,20 @@ export default NextAPI(async function handler(req) { if (legacyDefaultTag) { await MongoDatasetCollectionTagsV2.updateOne( { _id: legacyDefaultTag._id }, - { $set: { fromMigration: true, tagType: 'array' } } + { $set: { fromMigration: true, tagType: DatasetCollectionTagTypeEnum.array } } ); - defaultTag = { ...legacyDefaultTag, fromMigration: true, tagType: 'array' }; + defaultTag = { + ...legacyDefaultTag, + fromMigration: true, + tagType: DatasetCollectionTagTypeEnum.array + }; } else { try { const createdTag = await MongoDatasetCollectionTagsV2.create({ teamId: firstLegacyTag.teamId, datasetId, tag: DEFAULT_TAG, - tagType: 'array', + tagType: DatasetCollectionTagTypeEnum.array, fromMigration: true }); defaultTag = createdTag.toObject(); @@ -82,6 +90,7 @@ export default NextAPI(async function handler(req) { } } } + if (!defaultTag) continue; const defaultTagId = String(defaultTag._id); const collections = await MongoDatasetCollection.find({ datasetId }, '_id tags').lean(); diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts deleted file mode 100644 index 75313884f754..000000000000 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ /dev/null @@ -1,195 +0,0 @@ -/** @deprecated */ -import type { NextApiRequest } from 'next'; -import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema'; -import { Types } from '@fastgpt/service/common/mongo'; -import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; -import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; -import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; -import { NextAPI } from '@/service/middleware/entry'; -import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; -import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; -import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; -import { replaceRegChars } from '@fastgpt/global/common/string/tools'; -import { z } from 'zod'; -import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; - -const BodySchema = z.object({ - pageNum: z.number().int().min(1).default(1), - pageSize: z.number().int().min(1).max(100).default(10), - datasetId: z.string(), - parentId: z.string().nullable().optional().default(null), - searchText: z.string().optional().default(''), - selectFolder: z.boolean().optional().default(false), - filterTags: z.array(z.string()).optional().default([]), - simple: z.boolean().optional().default(false) -}); - -async function handler(req: NextApiRequest) { - const { pageNum, pageSize, datasetId, parentId, searchText, selectFolder, filterTags, simple } = - parseApiInput({ req, bodySchema: BodySchema }).body; - const regexText = searchText ? replaceRegChars(searchText) : ''; - - // auth dataset and get my role - const { teamId, permission } = await authDataset({ - req, - authToken: true, - authApiKey: true, - datasetId, - per: ReadPermissionVal - }); - - const match = { - teamId: new Types.ObjectId(teamId), - datasetId: new Types.ObjectId(datasetId), - ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}), - ...(regexText - ? { - name: new RegExp(regexText, 'i') - } - : { - parentId: parentId ? new Types.ObjectId(parentId) : null - }), - ...(filterTags.length ? { tags: { $in: filterTags } } : {}) - }; - - const selectField = { - _id: 1, - parentId: 1, - tmbId: 1, - name: 1, - type: 1, - forbid: 1, - createTime: 1, - updateTime: 1, - trainingType: 1, - fileId: 1, - rawLink: 1, - tags: 1, - externalFileId: 1 - }; - - // not count data amount - if (simple) { - const collections = await MongoDatasetCollection.find(match, undefined, { - ...readFromSecondary - }) - .select(selectField) - .sort({ - updateTime: -1 - }) - .lean(); - - return { - pageNum, - pageSize, - data: await Promise.all( - collections.map(async (item) => ({ - ...item, - tags: await collectionTagsToTagLabel({ - datasetId, - tags: item.tags - }), - dataAmount: 0, - trainingAmount: 0, - permission - })) - ), - total: await MongoDatasetCollection.countDocuments(match) - }; - } - - // aggregate 返回原始存储数据（tags 为存储格式），由 collectionTagsToTagLabel 解析后返回 - const [collections, total]: [DatasetCollectionSchemaType[], number] = await Promise.all([ - MongoDatasetCollection.aggregate([ - { - $match: match - }, - { - $sort: { updateTime: -1 } - }, - { - $skip: (pageNum - 1) * pageSize - }, - { - $limit: pageSize - }, - // count training data - { - $lookup: { - from: DatasetTrainingCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }] - } - } - }, - { $count: 'count' } - ], - as: 'trainingCount' - } - }, - // count collection total data - { - $lookup: { - from: DatasetDataCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [ - { $eq: ['$teamId', '$$team_id'] }, - { $eq: ['$datasetId', '$$dataset_id'] }, - { $eq: ['$collectionId', '$$id'] } - ] - } - } - }, - { $count: 'count' } - ], - as: 'dataCount' - } - }, - { - $project: { - ...selectField, - dataAmount: { - $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] - }, - trainingAmount: { - $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] - } - } - } - ]), - MongoDatasetCollection.countDocuments(match, { - ...readFromSecondary - }) - ]); - - const data = await Promise.all( - collections.map(async (item) => ({ - ...item, - tags: await collectionTagsToTagLabel({ - datasetId, - tags: item.tags - }), - permission - })) - ); - - // count collections - return { - pageNum, - pageSize, - data, - total - }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/listV2.ts b/projects/app/src/pages/api/core/dataset/collection/listV2.ts index b85e903874ba..1992221a9210 100644 --- a/projects/app/src/pages/api/core/dataset/collection/listV2.ts +++ b/projects/app/src/pages/api/core/dataset/collection/listV2.ts @@ -6,6 +6,7 @@ import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; +import { buildCollectionListTagMatch } from '@fastgpt/service/core/dataset/collection/tagFilter'; import { type DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; @@ -77,7 +78,7 @@ async function handler(req: ApiRequestProps): Promise ({ - ...item, - tags: await collectionTagsToTagLabel({ - datasetId, - tags: item.tags - }), - dataAmount: 0, - trainingAmount: 0, - ...defaultCollectionTrainingStatus, - indexAmount: 0, - permission - })) - ), - total: await MongoDatasetCollection.countDocuments(match) - }; - } - - // aggregate 返回原始存储数据（tags 为存储格式），由 collectionTagsToTagLabel 解析为对外格式后返回 - const [collections, total]: [DatasetCollectionSchemaType[], number] = await Promise.all([ - MongoDatasetCollection.aggregate([ - { - $match: match - }, - { - $sort: { updateTime: -1 } - }, - { - $skip: offset - }, - { - $limit: pageSize - }, - // count training data - { - $lookup: { - from: DatasetTrainingCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }] - } - } - }, - { $count: 'count' } - ], - as: 'trainingCount' - } - }, - // count collection total data - { - $lookup: { - from: DatasetDataCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [ - { $eq: ['$teamId', '$$team_id'] }, - { $eq: ['$datasetId', '$$dataset_id'] }, - { $eq: ['$collectionId', '$$id'] } - ] - } - } - }, - { $count: 'count' } - ], - as: 'dataCount' - } - }, - { - $project: { - ...selectField, - dataAmount: { - $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] - }, - trainingAmount: { - $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] - }, - activeTrainingAmount: { $literal: 0 }, - finalErrorAmount: { $literal: 0 }, - hasError: { $literal: false }, - slowestTrainingStatus: { $literal: CollectionTrainingStatusEnum.ready } - } - } - ]), - MongoDatasetCollection.countDocuments(match) - ]); - - const data = await Promise.all( - collections.map(async (item) => ({ - ...item, - tags: await collectionTagsToTagLabel({ - datasetId, - tags: item.tags - }), - permission - })) - ); - - // count collections - return { - list: data, - total - }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts b/projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts new file mode 100644 index 000000000000..1f68b903fdb1 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts @@ -0,0 +1,32 @@ +import { NextAPI } from '@/service/middleware/entry'; +import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; +import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; +import { getDatasetTagFilterOptions } from '@fastgpt/service/core/dataset/collection/tagFilter'; +import type { ApiRequestProps } from '@fastgpt/next/type'; +import { + GetTagFilterOptionsQuerySchema, + GetTagFilterOptionsResponseSchema, + type GetTagFilterOptionsResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; + +async function handler(req: ApiRequestProps): Promise { + const { datasetId } = parseApiInput({ + req, + querySchema: GetTagFilterOptionsQuerySchema + }).query; + + const { teamId } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: ReadPermissionVal + }); + + const list = await getDatasetTagFilterOptions({ teamId, datasetId }); + + return GetTagFilterOptionsResponseSchema.parse({ list }); +} + +export default NextAPI(handler); diff --git a/projects/app/src/web/core/dataset/api/collection.ts b/projects/app/src/web/core/dataset/api/collection.ts index c8a550280347..c7ff360d0213 100644 --- a/projects/app/src/web/core/dataset/api/collection.ts +++ b/projects/app/src/web/core/dataset/api/collection.ts @@ -1,4 +1,4 @@ -import { GET, POST, PUT, DELETE } from '@/web/common/api/request'; +import { GET, POST, DELETE } from '@/web/common/api/request'; import type { ParentTreePathItemType, ParentIdType @@ -20,18 +20,21 @@ import type { } from '@fastgpt/global/openapi/core/dataset/collection/createApi'; import type { AddTagsToCollectionsParams, + BatchSetCollectionTagsParams, CreateDatasetCollectionTagParams, + SetCollectionTagsParams, UpdateDatasetCollectionTagParams } from '@fastgpt/global/openapi/core/dataset/collection/tagApi'; import type { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants'; import type { DatasetCollectionsListItemType, DeleteCollectionBodyType, + GetTagFilterOptionsResponseType, ReadCollectionSourceBodyType, ReadCollectionSourceResponseType, UpdateDatasetCollectionBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api'; -import type { PaginationProps, PaginationResponse } from '@fastgpt/global/openapi/api'; +import type { PaginationResponse } from '@fastgpt/global/openapi/api'; import type { GetCollectionTrainingDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api'; /* ============================= collections ==================================== */ @@ -138,16 +141,18 @@ export const delDatasetCollectionTag = (data: { id: string; datasetId: string }) DELETE(`/proApi/core/dataset/tag/delete`, data); export const updateDatasetCollectionTag = (data: UpdateDatasetCollectionTagParams) => POST(`/proApi/core/dataset/tag/update`, data); -export const getDatasetCollectionTags = ( - data: PaginationProps<{ - datasetId: string; - searchText?: string; - }> -) => POST>(`/proApi/core/dataset/tag/list`, data); export const getTagUsage = (datasetId: string) => GET(`/proApi/core/dataset/tag/tagUsage?datasetId=${datasetId}`); export const getAllTags = (datasetId: string) => GET<{ list: DatasetTagType[] }>(`/proApi/core/dataset/tag/getAllTags?datasetId=${datasetId}`); +export const getDatasetTagFilterOptions = (datasetId: string) => + GET( + `/core/dataset/collection/tagFilterOptions?datasetId=${datasetId}` + ); +export const postSetCollectionTags = (data: SetCollectionTagsParams) => + POST(`/proApi/core/dataset/tag/setCollectionTags`, data); +export const postBatchSetCollectionTags = (data: BatchSetCollectionTagsParams) => + POST(`/proApi/core/dataset/tag/batchSetCollectionTags`, data); /* ================== read source ======================== */ export const getCollectionSource = (data: ReadCollectionSourceBodyType) => diff --git a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx index 2275aa433b57..e5e9039cef40 100644 --- a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx +++ b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx @@ -1,13 +1,8 @@ import { useQuery } from '@tanstack/react-query'; -import { type Dispatch, type ReactNode, type SetStateAction, useState } from 'react'; -import { useTranslation } from 'next-i18next'; +import { type ReactNode, useState } from 'react'; import { createContext } from 'use-context-selector'; import { getDatasetById, getDatasetPaths, putDatasetById } from '../api'; -import { - getAllTags, - getDatasetCollectionTags, - postCreateDatasetCollectionTag -} from '../api/collection'; +import { getAllTags } from '../api/collection'; import { getDatasetTrainingQueue } from '../api/training'; import { defaultDatasetDetail } from '../constants'; import { type UpdateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api'; @@ -23,15 +18,9 @@ type DatasetPageContextType = { loadDatasetDetail: (id: string) => Promise; updateDataset: (data: UpdateDatasetBody) => Promise; - searchDatasetTagsResult: DatasetTagType[]; allDatasetTags: DatasetTagType[]; + isLoadingAllDatasetTags: boolean; loadAllDatasetTags: () => Promise; - checkedDatasetTag: DatasetTagType[]; - setCheckedDatasetTag: React.Dispatch>; - onCreateCollectionTag: (tag: string) => Promise; - isCreateCollectionTagLoading: boolean; - searchTagKey: string; - setSearchTagKey: Dispatch>; paths: ParentTreePathItemType[]; refetchPaths: () => void; @@ -48,29 +37,17 @@ export const DatasetPageContext = createContext({ }, datasetId: '', datasetDetail: defaultDatasetDetail, - loadDatasetDetail: function (id: string): Promise { + loadDatasetDetail: function (_id: string): Promise { throw new Error('Function not implemented.'); }, - updateDataset: function (data: UpdateDatasetBody): Promise { + updateDataset: function (_data: UpdateDatasetBody): Promise { throw new Error('Function not implemented.'); }, - searchDatasetTagsResult: [], allDatasetTags: [], - checkedDatasetTag: [], - setCheckedDatasetTag: function (): void { - throw new Error('Function not implemented.'); - }, + isLoadingAllDatasetTags: false, loadAllDatasetTags: function (): Promise { throw new Error('Function not implemented.'); }, - onCreateCollectionTag: function (tag: string): Promise { - throw new Error('Function not implemented.'); - }, - isCreateCollectionTagLoading: false, - searchTagKey: '', - setSearchTagKey: function (value: SetStateAction): void { - throw new Error('Function not implemented.'); - }, paths: [], refetchPaths: () => {} }); @@ -82,7 +59,6 @@ export const DatasetPageContextProvider = ({ children: ReactNode; datasetId: string; }) => { - const { t } = useTranslation(); const { feConfigs } = useSystemStore(); // dataset detail @@ -105,10 +81,11 @@ export const DatasetPageContextProvider = ({ }; // dataset tags - const [checkedDatasetTag, setCheckedDatasetTag] = useState([]); - const [searchTagKey, setSearchTagKey] = useState(''); - - const { runAsync: loadAllDatasetTags, data: allDatasetTags = [] } = useRequest( + const { + runAsync: loadAllDatasetTags, + data: allDatasetTags = [], + loading: isLoadingAllDatasetTags + } = useRequest( async () => { if (!feConfigs?.isPlus || !datasetDetail._id) return []; @@ -120,38 +97,6 @@ export const DatasetPageContextProvider = ({ refreshDeps: [datasetDetail._id] } ); - const { data: searchDatasetTagsResult = [] } = useRequest( - async () => { - if (!searchTagKey) return allDatasetTags; - const { list } = await getDatasetCollectionTags({ - datasetId: datasetDetail._id, - searchText: searchTagKey, - offset: 0, - pageSize: 15 - }); - return list; - }, - { - manual: false, - throttleWait: 300, - refreshDeps: [datasetDetail._id, searchTagKey, allDatasetTags] - } - ); - const { runAsync: onCreateCollectionTag, loading: isCreateCollectionTagLoading } = useRequest( - (tag: string) => - postCreateDatasetCollectionTag({ - datasetId: datasetDetail._id, - tag - }), - { - refreshDeps: [datasetDetail._id], - onSuccess() { - loadAllDatasetTags(); - }, - successToast: t('common:create_success'), - errorToast: t('common:create_failed') - } - ); // training and rebuild queue const { data: { rebuildingCount = 0, trainingCount = 0 } = {}, refetch: refetchDatasetTraining } = @@ -189,15 +134,9 @@ export const DatasetPageContextProvider = ({ trainingCount, refetchDatasetTraining, - searchDatasetTagsResult, - checkedDatasetTag, - setCheckedDatasetTag, allDatasetTags, - loadAllDatasetTags, - onCreateCollectionTag, - isCreateCollectionTagLoading, - searchTagKey, - setSearchTagKey + isLoadingAllDatasetTags, + loadAllDatasetTags }; return {children}; diff --git a/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts b/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts new file mode 100644 index 000000000000..be016c90ba96 --- /dev/null +++ b/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts @@ -0,0 +1,151 @@ +import listHandler from '@/pages/api/core/dataset/collection/listV2'; +import tagFilterOptionsHandler from '@/pages/api/core/dataset/collection/tagFilterOptions'; +import { + DatasetCollectionTagTypeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { getRootUser } from '@test/datas/users'; +import { Call } from '@test/utils/request'; +import { describe, expect, it } from 'vitest'; + +describe('collection tag filter options and listV2 tagFilters', () => { + it('returns used values and filters collections by tag values', async () => { + const root = await getRootUser(); + const dataset = await MongoDataset.create({ + name: 'tag-filter-dataset', + teamId: root.teamId, + tmbId: root.tmbId, + vectorModel: 'test', + agentModel: 'test' + }); + + const [docType, version] = await MongoDatasetCollectionTagsV2.create([ + { + teamId: root.teamId, + datasetId: dataset._id, + tag: '文档类型', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['PRD', 'unused'] + }, + { + teamId: root.teamId, + datasetId: dataset._id, + tag: '版本', + tagType: DatasetCollectionTagTypeEnum.number + } + ]); + + await MongoDatasetCollection.create([ + { + name: 'prd-file', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [ + { tagId: String(docType._id), value: ['PRD'] }, + { tagId: String(version._id), value: 2 } + ] + }, + { + name: 'spec-file', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [{ tagId: String(docType._id), value: ['spec'] }] + } + ]); + + const optionsRes = await Call(tagFilterOptionsHandler, { + auth: root, + query: { datasetId: String(dataset._id) } + }); + + expect(optionsRes.code).toBe(200); + const docTypeOption = optionsRes.data.list.find((item) => item.tagId === String(docType._id)); + const versionOption = optionsRes.data.list.find((item) => item.tagId === String(version._id)); + expect(docTypeOption?.values).toEqual(['PRD', 'spec']); + expect(versionOption?.values).toEqual([2]); + + const listRes = await Call(listHandler, { + auth: root, + body: { + datasetId: String(dataset._id), + pageSize: 10, + offset: 0, + tagFilters: [{ tagId: String(docType._id), values: ['PRD'] }] + } + }); + + expect(listRes.code).toBe(200); + expect(listRes.data.list.map((item) => item.name)).toEqual(['prd-file']); + }); + + it('requires every selected tag to match when combining filters', async () => { + const root = await getRootUser(); + const dataset = await MongoDataset.create({ + name: 'tag-filter-and-dataset', + teamId: root.teamId, + tmbId: root.tmbId, + vectorModel: 'test', + agentModel: 'test' + }); + + const [docType, version] = await MongoDatasetCollectionTagsV2.create([ + { + teamId: root.teamId, + datasetId: dataset._id, + tag: '文档类型', + tagType: DatasetCollectionTagTypeEnum.array + }, + { + teamId: root.teamId, + datasetId: dataset._id, + tag: '版本', + tagType: DatasetCollectionTagTypeEnum.number + } + ]); + + await MongoDatasetCollection.create([ + { + name: 'both', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [ + { tagId: String(docType._id), value: ['PRD'] }, + { tagId: String(version._id), value: 2 } + ] + }, + { + name: 'only-type', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [{ tagId: String(docType._id), value: ['PRD'] }] + } + ]); + + const listRes = await Call(listHandler, { + auth: root, + body: { + datasetId: String(dataset._id), + pageSize: 10, + offset: 0, + tagFilters: [ + { tagId: String(docType._id), values: ['PRD'] }, + { tagId: String(version._id), values: [2] } + ] + } + }); + + expect(listRes.code).toBe(200); + expect(listRes.data.list.map((item) => item.name)).toEqual(['both']); + }); +}); diff --git a/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts b/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts index b71fd531e800..bbc80c80d90a 100644 --- a/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts +++ b/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts @@ -1,5 +1,4 @@ import listHandler from '@/pages/api/core/dataset/collection/listV2'; -import scrollListHandler from '@/pages/api/core/dataset/collection/scrollList'; import detailHandler from '@/pages/api/core/dataset/collection/detail'; import trainingDetailHandler from '@/pages/api/core/dataset/collection/trainingDetail'; import { @@ -7,7 +6,6 @@ import { DatasetCollectionTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { DatasetCollectionsListItemSchema } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; @@ -60,8 +58,7 @@ describe('collection training status api', () => { body: { datasetId: dataset._id, pageSize: 10, - offset: 0, - filterTags: [] + offset: 0 } }); @@ -230,50 +227,4 @@ describe('collection training status api', () => { expect(res.data.queuedCounts.parse).toBe(1); expect(res.data.queuedCounts.chunk).toBe(0); }); - - it('should keep deprecated scrollList compatible with the collection list item schema', async () => { - const root = await getRootUser(); - const dataset = await MongoDataset.create({ - name: 'test', - teamId: root.teamId, - tmbId: root.tmbId, - vectorModel: 'test', - agentModel: 'test' - }); - const collection = await MongoDatasetCollection.create({ - name: 'test', - type: DatasetCollectionTypeEnum.file, - teamId: root.teamId, - tmbId: root.tmbId, - datasetId: dataset._id - }); - - await MongoDatasetTraining.create({ - teamId: root.teamId, - tmbId: root.tmbId, - datasetId: dataset._id, - collectionId: collection._id, - billId: 'test', - mode: TrainingModeEnum.chunk, - retryCount: 0, - errorMsg: 'final failed' - }); - - const res = await Call(scrollListHandler, { - auth: root, - body: { - datasetId: dataset._id, - pageSize: 10, - offset: 0, - filterTags: [] - } - }); - - expect(res.code).toBe(200); - expect(() => DatasetCollectionsListItemSchema.parse(res.data.list[0])).not.toThrow(); - expect(res.data.list[0]).toMatchObject({ - trainingAmount: 1, - slowestTrainingStatus: CollectionTrainingStatusEnum.ready - }); - }); }); From 1cb762f5e46f88657a9f2e8e475cf61aa82139a8 Mon Sep 17 00:00:00 2001 From: DigHuang <114602213+DigHuang@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:15:25 +0800 Subject: [PATCH 3/4] feat(dataset/tag): add structured workflow tag filters (#7658) * fix(dataset/tag): refine tag filter behavior * feat(dataset/tag): add structured workflow tag filters --- ...76\350\256\241\346\226\207\346\241\243.md" | 141 ++-- packages/global/common/error/code/dataset.ts | 30 - packages/global/common/string/time.ts | 20 +- packages/global/core/app/formEdit/utils.ts | 3 +- packages/global/core/app/type.ts | 7 +- packages/global/core/app/utils.ts | 2 + packages/global/core/dataset/tagUtils.ts | 8 +- packages/global/core/dataset/type.ts | 35 +- .../global/core/dataset/workflowTagFilter.ts | 430 +++++++++++++ packages/global/core/workflow/constants.ts | 1 + .../global/core/workflow/node/constant.ts | 5 + .../global/core/workflow/template/input.ts | 9 + .../workflow/template/system/agent/index.ts | 12 +- .../workflow/template/system/datasetSearch.ts | 12 +- .../openapi/core/dataset/collection/api.ts | 26 +- .../core/dataset/collection/createApi.ts | 72 +-- .../openapi/core/dataset/collection/tagApi.ts | 10 +- .../test/core/app/formEdit/utils.test.ts | 3 +- packages/global/test/core/app/utils.test.ts | 2 + .../global/test/core/dataset/tagUtils.test.ts | 17 + .../core/dataset/workflowTagFilter.test.ts | 172 +++++ .../core/dataset/collection/tagFilter.ts | 92 +-- .../service/core/dataset/collection/utils.ts | 131 ++-- .../search/defaultRecall/collectionFilter.ts | 252 ++------ .../defaultRecall/collectionFilterShared.ts | 113 ++++ .../dataset/search/defaultRecall/index.ts | 4 +- .../defaultRecall/legacy/collectionFilter.ts | 117 ++++ .../search/defaultRecall/legacy/type.ts | 8 + .../search/defaultRecall/multiQueryRecall.ts | 13 +- packages/service/core/dataset/search/type.ts | 5 + packages/service/core/dataset/tag/schemaV2.ts | 4 +- .../dispatch/ai/agent/sub/dataset/index.ts | 7 +- .../core/workflow/dispatch/dataset/search.ts | 22 +- .../core/workflow/dispatch/utils/runtime.ts | 42 ++ .../core/dataset/collection/tagFilter.test.ts | 46 ++ .../core/dataset/delete/processor.test.ts | 26 - .../search/collectionFilter.benchmark.ts | 383 ----------- .../dataset/search/collectionFilter.test.ts | 601 ++++-------------- .../search/collectionTagIndex.benchmark.ts | 513 ++------------- .../search/legacy/collectionFilter.test.ts | 139 ++++ .../dataset/search/multiQueryRecall.test.ts | 71 +++ .../service/test/core/dataset/utils.test.ts | 477 ++------------ .../dispatch/ai/agent/sub/dataset.test.ts | 13 +- .../workflow/dispatch/dataset/search.test.ts | 59 ++ .../test/core/workflow/dispatch/index.test.ts | 75 +++ .../test/integrations/vectorDB/README.md | 19 +- ...ecallFilterPerformance.integration.test.ts | 225 ------- .../DateTimePicker/SingleDateTimePicker.tsx | 45 +- .../web/components/common/Icon/constants.ts | 3 + .../common/Icon/icons/math/greater.svg | 3 + .../common/Icon/icons/math/greaterEqual.svg | 3 + .../common/Icon/icons/math/notEqual.svg | 3 + .../common/TagFilter/FilterButton.tsx | 2 +- .../common/TagFilter/MultiTagFilter.tsx | 13 +- .../web/components/common/TagFilter/index.tsx | 11 +- .../core/workflow/NodeInputSelect.tsx | 6 + packages/web/i18n/en/common.json | 6 - packages/web/i18n/en/dataset.json | 5 - packages/web/i18n/en/system_migration.json | 8 +- packages/web/i18n/en/workflow.json | 27 + packages/web/i18n/ko-KR/dataset.json | 2 - packages/web/i18n/ko-KR/system_migration.json | 8 +- packages/web/i18n/ko-KR/workflow.json | 27 + packages/web/i18n/zh-CN/common.json | 6 - packages/web/i18n/zh-CN/dataset.json | 5 - packages/web/i18n/zh-CN/system_migration.json | 8 +- packages/web/i18n/zh-CN/workflow.json | 27 + packages/web/i18n/zh-Hant/common.json | 6 - packages/web/i18n/zh-Hant/dataset.json | 5 - .../web/i18n/zh-Hant/system_migration.json | 8 +- packages/web/i18n/zh-Hant/workflow.json | 27 + pro | 2 +- .../core/dataset/DatasetTagFilterRows.tsx | 487 ++++++++++++++ .../core/dataset/TagFilterSelects.tsx | 379 +++++++++++ projects/app/src/migration/registry.ts | 27 + .../20260907_migrate_dataset_tags_v2/index.ts | 198 ++++++ .../service.ts | 271 ++++++++ .../app/detail/Edit/ChatAgent/EditForm.tsx | 68 ++ .../app/detail/Edit/ChatAgent/utils.ts | 1 + .../app/detail/Edit/SimpleApp/EditForm.tsx | 124 +++- .../app/detail/Edit/SimpleApp/utils.ts | 29 + .../Flow/nodes/render/RenderInput/Label.tsx | 18 +- .../Flow/nodes/render/RenderInput/index.tsx | 11 + .../templates/DatasetTagFilter.tsx | 186 ++++++ .../CollectionTagBatchModal.tsx | 9 +- .../dataset/detail/CollectionCard/Header.tsx | 6 +- .../detail/CollectionCard/TagCommon.tsx | 11 +- .../detail/CollectionCard/TagManageModal.tsx | 103 ++- .../detail/CollectionCard/TagValueInputs.tsx | 53 +- .../detail/CollectionCard/TagsPopOver.tsx | 136 +++- .../useAppendDatasetTagOption.ts | 2 +- .../app/src/pages/api/admin/migrateTags.ts | 137 ---- .../core/dataset/collection/create/backup.ts | 5 +- .../core/dataset/collection/create/images.ts | 3 +- .../dataset/collection/create/template.ts | 5 +- projects/app/src/pages/app/detail/index.tsx | 3 +- .../src/web/core/dataset/api/collection.ts | 11 +- .../core/workflow/datasetSearchNodeUpgrade.ts | 73 +++ projects/app/src/web/core/workflow/utils.ts | 44 +- .../create/fileSourceLifecycle.test.ts | 9 +- .../dataset/collection/create/images.test.ts | 4 +- .../collection/tagFilterOptions.test.ts | 78 +-- projects/app/test/migration/registry.test.ts | 13 +- .../index.test.ts | 172 +++++ .../service.test.ts | 143 +++++ .../app/detail/Edit/ChatAgent/utils.test.ts | 19 + .../app/detail/Edit/SimpleApp/utils.test.ts | 80 ++- .../test/web/core/app/workflow/utils.test.ts | 88 ++- .../workflow/datasetSearchNodeUpgrade.test.ts | 84 +++ 109 files changed, 4845 insertions(+), 2985 deletions(-) create mode 100644 packages/global/core/dataset/workflowTagFilter.ts create mode 100644 packages/global/test/core/dataset/tagUtils.test.ts create mode 100644 packages/global/test/core/dataset/workflowTagFilter.test.ts create mode 100644 packages/service/core/dataset/search/defaultRecall/collectionFilterShared.ts create mode 100644 packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts create mode 100644 packages/service/core/dataset/search/defaultRecall/legacy/type.ts create mode 100644 packages/service/test/core/dataset/collection/tagFilter.test.ts delete mode 100644 packages/service/test/core/dataset/search/collectionFilter.benchmark.ts create mode 100644 packages/service/test/core/dataset/search/legacy/collectionFilter.test.ts create mode 100644 packages/service/test/core/dataset/search/multiQueryRecall.test.ts delete mode 100644 packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts create mode 100644 packages/web/components/common/Icon/icons/math/greater.svg create mode 100644 packages/web/components/common/Icon/icons/math/greaterEqual.svg create mode 100644 packages/web/components/common/Icon/icons/math/notEqual.svg create mode 100644 projects/app/src/components/core/dataset/DatasetTagFilterRows.tsx create mode 100644 projects/app/src/components/core/dataset/TagFilterSelects.tsx create mode 100644 projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/index.ts create mode 100644 projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/service.ts create mode 100644 projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/DatasetTagFilter.tsx delete mode 100644 projects/app/src/pages/api/admin/migrateTags.ts create mode 100644 projects/app/src/web/core/workflow/datasetSearchNodeUpgrade.ts create mode 100644 projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/index.test.ts create mode 100644 projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/service.test.ts create mode 100644 projects/app/test/web/core/workflow/datasetSearchNodeUpgrade.test.ts diff --git "a/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" index ebbed2b79b7c..04e2e4f0eb67 100644 --- "a/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -25,7 +25,7 @@ | 历史迁移 | 将旧字符串标签转换为 `default_tag` 数组标签 | | 生命周期 | dataset 物理删除时清理对应 v2 标签定义 | | API | FastGPT Collection API、FastGPT 标签过滤 API、fastgpt-pro 标签管理 API | -| 本次不包含 | 前端标签 UI 组件重构、自动执行迁移、删除旧表、Simple 应用标签过滤适配 | +| 本次不包含 | 删除 v1 标签表、反向迁移 v2 数据 | ### 1.3 术语 @@ -36,7 +36,7 @@ | Collection | 知识库中的文件、链接、文本等集合对象，持久化于 `dataset_collections` | | 新格式标签 | `{ tagId, value }`，`tagId` 引用 v2 标签表 | | 旧格式标签 | 字符串形式的历史标签 ID，通常引用 v1 标签表 | -| `default_tag` | `DEFAULT_TAG` 常量值，用于承载旧字符串标签，类型固定为 `array` | +| `default_tag` | 承载记录首次创建时使用的默认名称；作为 API 标签名时按普通字符串处理 | | `collectionFilterMatch` | 知识库检索使用的 Collection 过滤 JSON 字符串 | ## 2. 模块职责与边界 @@ -46,7 +46,7 @@ 1. 创建、查询、更新和删除知识库标签定义。 2. 根据标签类型校验 Collection 标签值，并将创建入参统一转换为可持久化的新格式。 3. 为知识库检索提供标签条件解析、MongoDB 粗筛和应用层精确比较。 -4. 为历史 Collection 提供按 dataset 维度的管理员迁移入口。 +4. 在 App 启动阶段按 dataset 维度自动迁移历史 Collection。 5. 在 dataset 异步物理删除时清理该 dataset 及其子 dataset 的 v2 标签定义。 6. 在 Collection 列表接口中将存储格式的 `tagId` 转换为对外展示的标签名称。 @@ -57,7 +57,7 @@ | 上游 | 标签管理 API、Collection 创建 API、工作流知识检索节点、Agent V2 搜索 | | 下游 | MongoDB、MongoDB secondary read、现有 dataset 权限系统 | | 跨仓库依赖 | fastgpt-pro 提供标签管理 API，FastGPT 提供公共类型、数据模型、Collection 和搜索逻辑 | -| 不负责 | 标签 UI 交互、向量库内部索引实现、历史迁移自动调度、旧表下线时间规划 | +| 不负责 | 向量库内部索引实现、旧表下线时间规划 | ## 3. 系统结构 @@ -71,7 +71,7 @@ | 标签 API | 提供标签定义和 Collection 标签值管理 | fastgpt-pro `projects/app/src/pages/api/core/dataset/tag/` | | Collection API | 创建、列表和滚动读取 Collection | `projects/app/src/pages/api/core/dataset/collection/` | | 搜索过滤 | 解析标签条件并返回 Collection ID | `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | -| 迁移 API | 读取 v1 标签并写入 `default_tag` 新格式 | `projects/app/src/pages/api/admin/migrateTags.ts` | +| 系统迁移 | 分批读取 v1 标签并写入 `default_tag` 新格式 | `projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/` | | 删除处理器 | 删除 dataset 相关 v1/v2 标签定义 | `packages/service/core/dataset/delete/processor.ts` | | 全局类型 | Zod Schema、标签类型、常量和错误码 | `packages/global/core/dataset/type.ts` | @@ -83,7 +83,7 @@ flowchart TD ProAPI[fastgpt-pro 标签 API] CollectionAPI[FastGPT Collection API] Search[collectionFilter.ts] - Migration[admin/migrateTags] + Migration[system migration runner] Delete[datasetDeleteProcessor] Utils[Collection 标签转换工具] V2[(dataset_collection_tags_v2)] @@ -165,31 +165,24 @@ Collection 列表接口 `listV2` 使用 `collectionTagsToTagLabel` 将存储格 ### 4.3 搜索过滤接口 -工作流和 Agent V2 通过 `collectionFilterMatch` 传入过滤条件。过滤入口为 `filterCollectionByMetadata`，支持： +工作流通过 `datasetSearchNode` 的隐藏输入 `collectionFilterVersion` 选择过滤器：显式 `legacy` 调用 `filterLegacyCollectionByMetadata`，显式 `structured` 调用 `filterCollectionByMetadata`；存量节点缺少版本时，仅在存在实际元数据过滤配置时使用 legacy，未配置过滤则直接使用 structured。兼容判断只检查配置是否存在，不根据 `collectionFilterMatch` 的字符串或对象形状推断版本。Agent V2 固定使用结构化过滤。共享能力支持： - `tags.$and`：所有条件必须满足。 - `tags.$or`：至少一个条件满足。 - `createTime.$gte/$lte`：按 Collection 创建时间过滤。 - `collectionIds`：按指定 Collection 或文件夹递归展开后过滤。 -旧格式字符串或 `null` 条件在入口改写为 `default_tag` 的 `$contains` 或 `$empty` 条件，然后统一进入 `filterCollectionByKeyValueTags`。 +新版入口只接受结构化标签条件；字符串或 `null` 条件直接拒绝。Legacy 入口仅按 `fromMigration=true` 查找承载标签，不调用 `filterCollectionByKeyValueTags` 或 `checkValue`。承载标签可以像普通标签一样改名或删除；删除后依赖它的 legacy 字符串条件无法满足，过滤结果为 `[]`，工作流继续运行但不产生知识库召回结果。 -### 4.4 管理员迁移接口 +### 4.4 自动系统迁移 -`POST /api/admin/migrateTags` 仅允许 root 调用。接口不接受清理孤儿标签的参数，也不删除 v1 表数据或 v2 标签定义。返回值为： - -```json -{ - "migratedDatasets": 0, - "migratedCollections": 0 -} -``` +标签迁移不再暴露手工管理员接口。`20260907_migrate_dataset_tags_v2` 注册为阻塞启动任务，由统一 Runner 负责 lease、runId fencing、checkpoint、进度和失败状态。任务失败时 App 不进入 ready。 ## 5. 内部设计 ### 5.1 标签数据模型 -v2 标签定义由 `teamId + datasetId + tag` 确定作用域，该三元组有唯一索引保证并发写入下不重复；`tagType` 决定值校验与过滤比较方式。一个 dataset 的 `default_tag` 记录最多作为历史字符串标签的承载定义使用，由 `fromMigration` 布尔字段标识（迁移或旧格式输入创建），过滤时按该字段定位而非标签名，因此即使该记录被改名，旧格式过滤仍可命中。 +v2 标签定义由 `teamId + datasetId + tag` 确定作用域，该三元组有唯一索引保证并发写入下不重复；`tagType` 决定值校验与过滤比较方式。只有存在旧字符串标签或仍通过兼容 API 写入字符串标签的 dataset 才按需创建一条 `tagType=array`、`fromMigration=true` 的承载定义，首次创建时默认命名为 `default_tag`。`default_tag` 本身是普通标签名称，不承担身份、权限或过滤分流语义；legacy filter 只按 `fromMigration` 定位承载记录。 ```mermaid classDiagram @@ -274,26 +267,25 @@ sequenceDiagram ```mermaid sequenceDiagram participant Workflow as 工作流/Agent - participant Entry as filterCollectionByMetadata - participant Filter as filterCollectionByKeyValueTags + participant Dispatcher as Dataset Search Dispatcher + participant Legacy as legacy filter + participant Structured as structured filter participant TagDB as v2 标签表 participant ColDB as dataset_collections - Workflow->>Entry: collectionFilterMatch - Entry->>Entry: json5.parse - Entry->>Entry: 识别 object、string、null 条件 - alt 旧格式 - Entry->>Entry: string -> default_tag.$contains；null -> default_tag.$empty - end - Entry->>Filter: 新格式条件 - Filter->>TagDB: 按 dataset 和标签名批量查 tagId/tagType - loop 每个 dataset - Filter->>ColDB: 按 tags.tagId 粗筛并使用 hint - Filter->>Filter: 按 tagType 执行 checkValue + Workflow->>Dispatcher: collectionFilterVersion + collectionFilterMatch + alt 显式 legacy，或缺少版本且存在历史过滤配置 + Dispatcher->>Legacy: 旧 JSON 过滤 + Legacy->>TagDB: 按 dataset 查 fromMigration=true + Legacy->>ColDB: 保持旧 AND/OR/null 语义 + else structured + Dispatcher->>Structured: 结构化条件 + Structured->>TagDB: 按 dataset 和标签名查 tagId/tagType + Structured->>ColDB: 按 tags.tagId 粗筛并使用 hint + Structured->>Structured: 按 tagType 执行 checkValue end - Filter-->>Entry: collectionId 列表 - Entry->>Entry: 与 createTime、collectionIds 求交集 - Entry-->>Workflow: 最终 collectionId 列表 + Dispatcher->>Dispatcher: 与 createTime、collectionIds 求交集 + Dispatcher-->>Workflow: 最终 collectionId 列表 ``` 过滤实现分为两层： @@ -333,18 +325,14 @@ array 比较的时间复杂度为 `O(n+m)`，其他类型为 `O(1)`；单个 arr ### 5.6 历史数据迁移 -迁移按 v1 表中存在标签的 dataset 逐个处理： +迁移由统一 System Migration Runner 按稳定 ObjectId 游标分批执行： -1. root 鉴权。 -2. 清理 v2 表按 `{ teamId, datasetId, tag }` 重复的行（保留 `_id` 最早的一条），避免唯一索引构建失败。 -3. 从 v1 表读取 dataset 的标签定义，建立 `旧 tagId -> 标签名称` 映射。 -4. 查询或创建当前 dataset 的 v2 `default_tag`，类型为 `array` 并置 `fromMigration: true`；存量按名称创建的 `default_tag` 记录回填该标记。 -5. 查找包含旧字符串标签的 Collection；已是纯新格式的 Collection 跳过。 -6. 将旧字符串 ID 转换为标签名称，与已有 `default_tag` 数组值合并并去重。 -7. 保留其他新格式标签，移除旧字符串元素，使用 `$set` 更新 `tags`。 -8. 返回迁移 dataset 数和 Collection 数，并记录 dataset、Collection 和迁移前标签信息日志。 +1. 按 `{ teamId, datasetId, tag }` 整理重复 v2 定义；保留 `_id` 最早项，对其余 ID 先从 Collection 删除引用并校验，再删除定义。 +2. 从 v1 表建立 `旧 tagId -> 标签名称` 映射；遇到旧字符串标签时，按需创建或复用当前 dataset 的系统承载标签并合并名称数组。 +3. 保留其他新格式标签，移除旧字符串元素，使用原始 BSON 快照条件更新。 +4. 建立 v2 唯一索引和 `tags.tagId` 索引，校验无旧字符串标签、无重复定义、无重复或非法承载标签。 -迁移只修改 `tags` 字段，不修改 Collection 的 `_id`、`createTime` 或其他元数据，因此既有 `collectionIds` 和时间过滤仍然有效。重复执行时已转换 Collection 不会再次转换。v1 表在迁移过程中只读，v2 孤儿标签定义保留。 +Collection 子阶段只修改 `tags` 字段，不修改 `_id`、`createTime` 或其他元数据，因此既有 `collectionIds` 和时间过滤仍然有效。工作流记录不参与数据迁移：运行时仅将“缺少版本且实际配置了过滤”的存量节点解释为 legacy，空配置直接使用 structured。重复执行时已转换记录不会再次转换；v1 表在迁移过程中只读。 ### 5.7 dataset 物理删除 @@ -397,9 +385,9 @@ DatasetCollectionTagsV2Schema.index({ teamId: 1, datasetId: 1, tag: 1 }, { uniqu | `_id` | ObjectId | 主键 | 标签定义 ID | | `teamId` | ObjectId | 必填 | 所属团队 | | `datasetId` | ObjectId | 必填 | 所属知识库 | -| `tag` | String | 必填 | 标签名称；`default_tag` 为系统保留名称 | +| `tag` | String | 必填 | 标签名称；`default_tag` 按普通字符串处理 | | `tagType` | String | 枚举 | `string`、`number`、`datetime`、`array`，默认 `string` | -| `fromMigration` | Boolean | 默认 `false` | 标识该记录是迁移或旧格式输入创建的 `default_tag` 承载记录；过滤按该字段定位而非标签名 | +| `fromMigration` | Boolean | 默认 `false` | 标识旧标签数据的系统承载记录；仅 legacy filter 用于定位 | `{ teamId, datasetId, tag }` 唯一索引保证同一作用域下标签名不重复，并发写入由数据库兜底；业务层先查后写，撞索引时捕获 E11000 复用已存在记录。 @@ -460,7 +448,8 @@ DEFAULT_TAG = 'default_tag' | number 无法转换或超出安全范围 | 返回 `tagValueInvalid` | | datetime 无法转换为有效时间 | 返回 `tagValueDatetimeInvalid` | | array 非数组、元素非字符串、长度超过 64 或元素超过 256 | 返回 `arrayTagValueInvalid` | -| 过滤 JSON 解析失败 | 捕获异常，当前请求退化为不执行该过滤 | +| Legacy 过滤 JSON 解析失败 | 保持历史降级行为，当前请求不执行该过滤 | +| Structured 过滤不合法 | 拒绝运行，不回退 legacy 或扩大召回范围 | | MongoDB 查询失败 | 保留异常并由上层统一返回服务端错误 | | 旧标签 ID 在 v1 表中不存在 | 迁移时忽略该标签，其他可解析标签继续处理 | | 同一 tagId 出现冲突值 | 拒绝批量写入，不执行覆盖操作 | @@ -495,9 +484,9 @@ DEFAULT_TAG = 'default_tag' ### 8.4 运维要求 - 迁移前备份 `dataset_collections`，确认迁移范围和结果。 -- 迁移接口由运维按需调用，不自动执行，不删除 v1 数据，不清理 v2 孤儿定义。 -- 生产环境提前创建并检查 `tags.tagId` 索引，避免索引创建阻塞业务。 -- 关注迁移日志、dataset 删除任务日志及 MongoDB 查询慢日志。 +- 迁移由启动 Runner 自动执行；运维通迁移页面查看阶段进度，失败时 App 不进入 ready。 +- 唯一索引和 `tags.tagId` 索引在完成校验前创建，运行期可安全使用强制 hint。 +- 关注系统迁移阶段错误、dataset 删除任务日志及 MongoDB 查询慢日志。 - v1 表的最终下线需另行设计，不属于本模块。 ## 9. 测试设计 @@ -509,9 +498,10 @@ DEFAULT_TAG = 'default_tag' | 标签值校验 | 四种 tagType 的合法值、非法值、边界值 | | `checkValue` | 各类型操作符、空值、类型转换失败、正则异常 | | array 比较 | `$is`、`$isNot`、`$contains`、`$notContains`、`$in`、`$notIn`、`$empty`、`$notEmpty` | -| 标签归一化 | string、object、混合输入和 `default_tag` 合并 | +| 标签归一化 | string、object 和混合输入；对象中的 `default_tag` 按普通标签名解析 | | 冲突检测 | 相同 tagId 相同值去重、不同值拒绝 | -| 旧格式改写 | string、null、AND、OR、混合 null/string | +| Legacy 过滤 | string、null、AND 优先、OR、混合 null/string、多 dataset 承载标签 | +| Structured 过滤 | 拒绝 string/null，按类型执行新版条件 | ### 9.2 集成测试 @@ -520,11 +510,13 @@ DEFAULT_TAG = 'default_tag' 3. `listV2` 返回标签名称而非内部 tagId。 4. 标签过滤与时间、Collection ID、文件夹递归条件正确求交集。 5. Agent V2 搜索可以透传 `collectionFilterMatch`。 -6. 迁移保留已有新格式标签，旧标签名称正确合并且重复调用幂等。 +6. 迁移保留已有新格式标签，旧标签名称正确合并，重复执行和 checkpoint 恢复幂等。 7. 迁移不修改 Collection `_id` 和 `createTime`。 8. dataset root/child 物理删除只清理目标 team 和目标 dataset 集合的 v2 标签。 9. 其他 team、其他 dataset 的同名或同 ID 标签不受影响。 10. MongoDB 查询命中新格式 `tags.tagId` 索引。 +11. 缺少版本标记且存在历史过滤配置的旧节点调用 legacy filter；历史空配置和显式 structured 节点调用 structured filter。 +12. 节点升级保留 ID、输入输出与连线，且持久化失败时不切换本地节点。 ### 9.3 性能验证 @@ -538,17 +530,21 @@ DEFAULT_TAG = 'default_tag' ### 10.1 发布顺序 -1. 发布包含全局类型、v2 Schema、Collection 索引和搜索过滤逻辑的 FastGPT 服务。 -2. 发布 fastgpt-pro 标签管理 API，使标签读写切换到 v2 表。 -3. 检查 MongoDB v2 集合和两个标签索引。 -4. 备份历史数据后，由管理员按 dataset 范围调用迁移接口。 -5. 观察标签 API、检索过滤和迁移日志，再逐步扩大迁移范围。 +> 当前数据格式不支持旧 Pod 与迁移后数据并存。发布时必须先停止旧版本业务流量再启动带阻塞迁移的新版本；若要求无停机滚动升级，需要先单独发布能双读对象标签的过渡版本。 + +1. App 启动时执行阻塞式 `20260907_migrate_dataset_tags_v2` 系统迁移，失败时不进入 ready。 +2. 迁移按稳定 `_id` 游标分批处理标签定义和 Collection，批次提交后才保存 checkpoint。 +3. 迁移只为存在旧字符串标签的 dataset 按需创建唯一 `fromMigration=true` 承载标签，并把旧标签名称转换为数组值。 +4. 重复 v2 标签定义按 `_id` 保留最早项；删除定义前先从 Collection 中删除指向重复 ID 的标签值，不向保留项合并。 +5. 全部阶段完成后校验无旧字符串标签、无重复定义、无重复 ID 残留引用及非法承载标签。 +6. 存量 `datasetSearchNode` 不改写；缺少 `collectionFilterVersion` 时按是否存在历史过滤配置选择 legacy 或 structured，新建节点显式保存 structured。 +7. 阻塞迁移成功后才开放业务流量，不在运行期同时读取 v1/v2 标签。 ### 10.2 回滚策略 - 未迁移数据仍保留 v1 标签和旧 `tags` 值，可回退到旧代码读取。 - v2 标签定义和新格式 Collection 数据需通过数据库备份恢复；不通过迁移接口反向回滚。 -- 迁移只覆盖 `tags` 字段，不改 Collection 主键和创建时间，便于按备份恢复标签字段。 +- 迁移不改 Collection 主键、创建时间或工作流节点类型；回滚旧版代码前需从备份恢复 `tags` 字段。 - v1 表不在本次发布中删除，回滚期间保留历史读取基础。 ## 11. 实现文件索引 @@ -560,9 +556,10 @@ DEFAULT_TAG = 'default_tag' | `packages/service/core/dataset/tag/schemaV2.ts` | v2 标签 MongoDB Schema | | `packages/service/core/dataset/collection/schema.ts` | Collection Schema 和标签索引 | | `packages/service/core/dataset/collection/utils.ts` | 标签创建、校验、存储和展示转换 | -| `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | 标签过滤和旧格式改写 | +| `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | 新版结构化标签过滤 | +| `packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts` | 存量节点的 legacy 过滤语义 | | `projects/app/src/pages/api/core/dataset/collection/listV2.ts` | Collection 标签过滤列表接口 | -| `projects/app/src/pages/api/admin/migrateTags.ts` | 历史标签迁移接口 | +| `projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/` | 历史标签阻塞迁移 | | `packages/service/core/dataset/delete/processor.ts` | dataset 物理删除和 v2 标签清理 | | `packages/service/core/dataset/collection/tagFilter.ts` | 详情页标签值筛选：已用值聚合与列表查询条件 | | `projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts` | 标签筛选选项接口 | @@ -596,4 +593,24 @@ Figma：`07页面｜标签筛选`（node `2434:38727`），交互组件标注「 ### 12.3 接口 - `GET /api/core/dataset/collection/tagFilterOptions?datasetId=`：只返回 `{ tagId, values }`（已用值）。标签名和类型复用 `getAllTags`。 -- `POST /api/core/dataset/collection/listV2` 新增 `tagFilters: [{ tagId, values }]`。 \ No newline at end of file +- `POST /api/core/dataset/collection/listV2` 新增 `tagFilters: [{ tagId, values }]`。 + +## 13. 旧新检索节点分流 + +1. `datasetSearchNode` 显式 `legacy` 时调用 legacy filter；显式 `structured` 时调用 structured filter；缺少版本时，有历史过滤配置走 legacy，空配置走 structured。 +2. 节点版本只由隐藏输入决定，不根据 `collectionFilterMatch` 的字符串或对象形状推断。 +3. legacy filter 使用每个 dataset 自己的 `fromMigration=true` 承载标签，保持旧 `$and` 优先于 `$or` 及 `null` 匹配真正空标签 Collection 的语义。 +4. structured filter 只接受结构化条件，字符串或 `null` 标签条件是非法配置，不隐式转换或回退 legacy。 +5. 用户点击升级时保留节点类型、ID、公共参数、输入输出和连线，清空无法安全转换的旧过滤配置，并在保存成功后把版本输入切换为 `structured`。 +6. Agent V2 模板显式保存 `structured`；Agent 搜索服务也显式指定 structured filter，不复用 Dataset Search 的缺省兼容规则。 + +## TODO + +- [x] 新增并注册阻塞式分批标签迁移，包含进度、checkpoint、lease 检查和完成校验。 +- [x] 迁移重复标签定义时先删除 Collection 引用，再删除定义。 +- [x] 使用隐藏版本输入在单一节点类型内分流 legacy/structured 过滤器，删除新链路的 legacy rewrite。 +- [x] 完成节点升级、Agent 配置入口、multipart tags 透传和标签删除引用清理。 +- [x] 承载标签只按 `fromMigration` 定位，`default_tag` 按普通标签名称处理。 +- [x] 补充迁移恢复、旧新语义、API 透传、删除清理和升级行为测试。 +- [x] 运行局部测试、App typecheck、相关 lint 和 `git diff --check`。 +- [ ] 发布前确认采用停机切换，或先发布兼容对象标签的过渡版本；当前实现不支持旧 Pod 与迁移后数据并存。 diff --git a/packages/global/common/error/code/dataset.ts b/packages/global/common/error/code/dataset.ts index 3761ddb77569..de72f57fdaac 100644 --- a/packages/global/common/error/code/dataset.ts +++ b/packages/global/common/error/code/dataset.ts @@ -22,13 +22,7 @@ export enum DatasetErrEnum { tagNameEmpty = 'tagNameEmpty', tagNotExist = 'tagNotExist', tagValueInvalid = 'tagValueInvalid', - tagValueStringTooLong = 'tagValueStringTooLong', - tagValueNumberOutOfRange = 'tagValueNumberOutOfRange', tagValueDatetimeInvalid = 'tagValueDatetimeInvalid', - noDatasetForTagFilter = 'noDatasetForTagFilter', - noTagsInDataset = 'noTagsInDataset', - noPermissionForDatasetTags = 'noPermissionForDatasetTags', - tagNotSelectedForRef = 'tagNotSelectedForRef', arrayTagValueInvalid = 'arrayTagValueInvalid' } const datasetErr = [ @@ -102,34 +96,10 @@ const datasetErr = [ statusText: DatasetErrEnum.tagValueInvalid, message: i18nT('common:core.dataset.error.tagValueInvalid') }, - { - statusText: DatasetErrEnum.tagValueStringTooLong, - message: i18nT('common:core.dataset.error.tagValueStringTooLong') - }, - { - statusText: DatasetErrEnum.tagValueNumberOutOfRange, - message: i18nT('common:core.dataset.error.tagValueNumberOutOfRange') - }, { statusText: DatasetErrEnum.tagValueDatetimeInvalid, message: i18nT('common:core.dataset.error.tagValueDatetimeInvalid') }, - { - statusText: DatasetErrEnum.noDatasetForTagFilter, - message: i18nT('common:core.dataset.error.noDatasetForTagFilter') - }, - { - statusText: DatasetErrEnum.noTagsInDataset, - message: i18nT('common:core.dataset.error.noTagsInDataset') - }, - { - statusText: DatasetErrEnum.noPermissionForDatasetTags, - message: i18nT('common:core.dataset.error.noPermissionForDatasetTags') - }, - { - statusText: DatasetErrEnum.tagNotSelectedForRef, - message: i18nT('common:core.dataset.error.tagNotSelectedForRef') - }, { statusText: DatasetErrEnum.arrayTagValueInvalid, message: i18nT('common:core.dataset.error.arrayTagValueInvalid') diff --git a/packages/global/common/string/time.ts b/packages/global/common/string/time.ts index 37c67b19b4f7..d19063ceaf56 100644 --- a/packages/global/common/string/time.ts +++ b/packages/global/common/string/time.ts @@ -7,24 +7,6 @@ import { i18nT } from '../i18n/utils'; dayjs.extend(utc); dayjs.extend(timezone); -/** - * 本地时间转 UTC 毫秒时间戳 - * @param date 本地 Date 对象 - * @returns UTC 毫秒时间戳（number） - */ -export const toUTCSeconds = (date: Date): number => { - return dayjs(date).utc().valueOf(); -}; - -/** - * UTC 毫秒时间戳转 Date 对象 - * @param ms UTC 毫秒时间戳 - * @returns Date 对象 - */ -export const fromUTCSeconds = (ms: number): Date => { - return dayjs(ms).utc().toDate(); -}; - export const formatTime2YMDHMW = (time?: Date | number) => dayjs(time).format('YYYY-MM-DD HH:mm:ss dddd'); export const formatTime2YMDHMS = (time?: Date | number) => @@ -101,7 +83,7 @@ export const cronParser2Fields = (cronString: string) => { try { const cronField = cronParser.parseExpression(cronString).fields; return cronField; - } catch (error) { + } catch { return null; } }; diff --git a/packages/global/core/app/formEdit/utils.ts b/packages/global/core/app/formEdit/utils.ts index 67ff5c7debfc..3abdea267543 100644 --- a/packages/global/core/app/formEdit/utils.ts +++ b/packages/global/core/app/formEdit/utils.ts @@ -36,7 +36,8 @@ const agentGeneratedDenyRenderTypes = new Set([ FlowNodeInputTypeEnum.selectTool, FlowNodeInputTypeEnum.selectDataset, FlowNodeInputTypeEnum.selectDatasetParamsModal, - FlowNodeInputTypeEnum.settingDatasetQuotePrompt + FlowNodeInputTypeEnum.settingDatasetQuotePrompt, + FlowNodeInputTypeEnum.datasetTagFilter ]); // 工具配置不能处理依赖文件、知识库、模型或外部动态上下文的输入。 diff --git a/packages/global/core/app/type.ts b/packages/global/core/app/type.ts index b8210555fbaf..4b1172794979 100644 --- a/packages/global/core/app/type.ts +++ b/packages/global/core/app/type.ts @@ -2,6 +2,10 @@ import { StoreNodeItemTypeSchema } from '../workflow/type/node'; import { AppTypeEnum } from './constants'; import { NodeInputKeyEnum } from '../workflow/constants'; import { DatasetSearchModeEnum } from '../dataset/constants'; +import { + DatasetTagFilterValueSchema, + DatasetTagFilterVersionSchema +} from '../dataset/workflowTagFilter'; import type { ReasoningEffort } from '../ai/llm/type'; import { StoreEdgeItemTypeSchema } from '../workflow/type/edge'; import type { AppPermission } from '../../support/permission/app/controller'; @@ -271,7 +275,8 @@ export const AppDatasetSearchParamsTypeSchema = z.object({ datasetSearchExtensionBg: z.string().optional(), [NodeInputKeyEnum.authTmbId]: BoolSchema.optional(), - collectionFilterMatch: z.string().optional() + collectionFilterMatch: z.union([z.string(), DatasetTagFilterValueSchema]).optional(), + [NodeInputKeyEnum.collectionFilterVersion]: DatasetTagFilterVersionSchema.optional() }); export type AppDatasetSearchParamsType = z.infer; diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts index 97cefe73e37d..008eea3326d8 100644 --- a/packages/global/core/app/utils.ts +++ b/packages/global/core/app/utils.ts @@ -6,6 +6,7 @@ import { AppTypeEnum } from './constants'; import appErrList from '../../common/error/code/app'; import pluginErrList from '../../common/error/code/plugin'; import { i18nT } from '../../common/i18n/utils'; +import { DatasetTagFilterVersionEnum } from '../dataset/workflowTagFilter'; const deletedPluginErrorList = new Set([ 'plugin.team_not_installed', @@ -34,6 +35,7 @@ export const getDefaultAppForm = (): AppFormEditFormType => { rerankWeight: 0.5, datasetSearchUsingExtensionQuery: true, datasetSearchExtensionBg: '', + [NodeInputKeyEnum.collectionFilterVersion]: DatasetTagFilterVersionEnum.structured, [NodeInputKeyEnum.authTmbId]: false }, selectedTools: [], diff --git a/packages/global/core/dataset/tagUtils.ts b/packages/global/core/dataset/tagUtils.ts index ed2cb9c9d798..d4721c7fc4a3 100644 --- a/packages/global/core/dataset/tagUtils.ts +++ b/packages/global/core/dataset/tagUtils.ts @@ -1,14 +1,10 @@ import type { CollectionTagValueType } from './type'; -/** 标签值去重/列表 key：区分 number 2 与 string "2"。 */ -export const collectionTagValueKey = (value: string | number) => - typeof value === 'number' ? `n:${value}` : `s:${value}`; - -/** 数字按大小、其余按 zh-CN 字典序。前后端筛选项展示共用。 */ +/** 同一标签类型固定：数字按大小，其余按字典序。前后端筛选项展示共用。 */ export const sortCollectionTagValues = (values: T[]): T[] => [...values].sort((a, b) => { if (typeof a === 'number' && typeof b === 'number') return a - b; - return String(a).localeCompare(String(b), 'zh-CN'); + return String(a).localeCompare(String(b)); }); /** 筛选项只保留非空字符串和有限数字。 */ diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index b7488029f6c1..cd0818afef5e 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -34,21 +34,27 @@ import { NumSchema } from '../../common/zod'; export { DatasetCollectionTagTypeEnum, DatasetCollectionTagTypeMap }; export type DatasetCollectionTagType = `${DatasetCollectionTagTypeEnum}`; -/** 选项类标签预设 options 上限，Zod 与写入合并共用。 */ -export const DATASET_COLLECTION_TAG_OPTIONS_MAX = 64; - /** 选项类标签的预设选项，空选项由前端草稿过滤后再提交。 */ export const DatasetCollectionTagOptionsSchema = z .array(z.string().trim().min(1)) - .max(DATASET_COLLECTION_TAG_OPTIONS_MAX) .meta({ description: '选项类标签的预设选项' }); -/** 迁移常量：新建 array 标签记录的 tag 字段固定值，亦是旧格式过滤改写的条件 key */ +/** 旧字符串标签承载记录首次创建时使用的默认名称；身份只由 fromMigration 标识。 */ export const DEFAULT_TAG = 'default_tag'; /** Collection 标签值字段：string/number 存对应值，datetime 存 UTC 毫秒时间戳，array 存 string 数组 */ export const CollectionTagValueFieldSchema = z.union([z.string(), z.number(), z.array(z.string())]); +/** Collection API 的兼容标签输入/展示格式：旧标签名或带名称和值的新格式。 */ +export const CollectionTagLabelSchema = z.union([ + z.string(), + z.object({ + tag: z.string(), + value: CollectionTagValueFieldSchema + }) +]); +export type CollectionTagLabelType = z.infer; + /** Collection 标签值类型（新格式） */ export const CollectionTagValueSchema = z.object({ tagId: z.string().meta({ description: '引用 dataset_collection_tags_v2._id' }), @@ -61,12 +67,12 @@ export type CollectionTagValueType = z.infer; /** 详情页按标签值筛选的单条条件。同一标签多值为 OR，不同标签由调用方做 AND。 */ export const CollectionTagFilterItemSchema = z.object({ - tagId: z.string().meta({ + tagId: ObjectIdSchema.meta({ example: '68ad85a7463006c963799a05', description: '标签 ID' }), values: z - .array(z.union([z.string(), z.number()])) + .array(z.union([z.string().min(1), z.number().finite()])) .min(1) .meta({ example: ['PRD'], @@ -416,7 +422,10 @@ export const DatasetTagSchema = z.object({ tagType: z.enum(DatasetCollectionTagTypeEnum).default(DatasetCollectionTagTypeEnum.string).meta({ description: '标签类型：string(默认)/number/datetime/array' }), - options: DatasetCollectionTagOptionsSchema.optional() + options: DatasetCollectionTagOptionsSchema.optional(), + fromMigration: z.boolean().optional().meta({ + description: '是否为旧标签数据的系统承载定义' + }) }); export type DatasetTagType = z.infer; @@ -430,15 +439,7 @@ export type TagUsageType = z.infer; export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({ // 详情接口的 tags 由 collectionTagsToTagLabel 解析为标签名格式（string | { tag, value }），区别于存储格式（string | { tagId, value }） tags: z - .array( - z.union([ - z.string(), - z.object({ - tag: z.string(), - value: CollectionTagValueFieldSchema - }) - ]) - ) + .array(CollectionTagLabelSchema) .optional() .meta({ description: '标签。string 为标签名；新格式为 { tag, value }' }), sourceName: z.string().meta({ description: '来源名称' }), diff --git a/packages/global/core/dataset/workflowTagFilter.ts b/packages/global/core/dataset/workflowTagFilter.ts new file mode 100644 index 000000000000..69a224b37b6f --- /dev/null +++ b/packages/global/core/dataset/workflowTagFilter.ts @@ -0,0 +1,430 @@ +import { z } from 'zod'; +import { formatTime2YMDHM } from '../../common/string/time'; +import { DatasetCollectionTagTypeEnum } from './constants'; +import type { DatasetCollectionTagType, DatasetTagType } from './type'; + +export const DatasetTagFilterLogicEnum = { + AND: 'AND', + OR: 'OR' +} as const; + +export const DatasetTagFilterValueModeEnum = { + input: 'input', + reference: 'reference' +} as const; +export type DatasetTagFilterValueMode = + (typeof DatasetTagFilterValueModeEnum)[keyof typeof DatasetTagFilterValueModeEnum]; + +export const DatasetTagFilterVersionEnum = { + legacy: 'legacy', + structured: 'structured' +} as const; +export const DatasetTagFilterVersionSchema = z.enum(DatasetTagFilterVersionEnum); +export type DatasetTagFilterVersion = z.infer; + +const hasDatasetTagFilterConfiguration = (value: unknown) => { + if (typeof value === 'string') return value.trim().length > 0; + if (Array.isArray(value)) return value.length > 0; + return value !== undefined && value !== null; +}; + +/** + * 显式版本优先；存量节点缺少版本时，有过滤配置走 legacy，无配置直接使用 structured。 + * 这里只判断是否配置，不根据过滤值的字符串或对象形状推断版本。 + */ +export const resolveDatasetTagFilterVersion = ({ + version, + filterValue +}: { + version: unknown; + filterValue: unknown; +}): DatasetTagFilterVersion => { + if (version === DatasetTagFilterVersionEnum.structured) { + return DatasetTagFilterVersionEnum.structured; + } + if (version === DatasetTagFilterVersionEnum.legacy) return DatasetTagFilterVersionEnum.legacy; + if (version !== undefined && version !== null && version !== '') { + return DatasetTagFilterVersionEnum.legacy; + } + return hasDatasetTagFilterConfiguration(filterValue) + ? DatasetTagFilterVersionEnum.legacy + : DatasetTagFilterVersionEnum.structured; +}; + +/** 旧编辑器只接收字符串；异常存量值显示为空，不用于推断节点版本。 */ +export const normalizeLegacyDatasetTagFilterValue = (value: unknown) => + typeof value === 'string' ? value : ''; + +/** 条件行字段来源：知识库标签，或固定文件属性。 */ +export const DatasetTagFilterFieldEnum = { + tag: 'tag', + createTime: 'createTime', + collectionId: 'collectionId' +} as const; +export type DatasetTagFilterField = + (typeof DatasetTagFilterFieldEnum)[keyof typeof DatasetTagFilterFieldEnum]; + +/** 工作流标签过滤支持的标签类型。string 不进入条件行下拉。 */ +const WorkflowTagFilterTagTypeSchema = z.enum([ + DatasetCollectionTagTypeEnum.number, + DatasetCollectionTagTypeEnum.datetime, + DatasetCollectionTagTypeEnum.array +] as const); +export type WorkflowTagFilterTagType = z.infer; + +export const DatasetTagFilterConditionSchema = z.object({ + field: z.enum(DatasetTagFilterFieldEnum).optional(), + tag: z.string().optional(), + tagType: WorkflowTagFilterTagTypeSchema.optional(), + op: z.string().optional(), + valueMode: z.enum(DatasetTagFilterValueModeEnum).optional(), + value: z.unknown().optional() +}); +export type DatasetTagFilterCondition = z.infer; + +export const DatasetTagFilterValueSchema = z.object({ + logic: z.enum(DatasetTagFilterLogicEnum), + conditions: z.array(DatasetTagFilterConditionSchema) +}); +export type DatasetTagFilterValue = z.infer; + +export type WorkflowTagFilterOption = { + tag: string; + tagType: WorkflowTagFilterTagType; + options: string[]; +}; + +type TagFilterOperator = { + labelKey: string; + value: string; + icon?: string; + iconFlip?: boolean; +}; + +const emptyValueOperators: TagFilterOperator[] = [ + { labelKey: 'workflow:tag_filter_op_empty', value: '$empty' }, + { labelKey: 'workflow:tag_filter_op_not_empty', value: '$notEmpty' } +]; +const emptyOps = new Set(emptyValueOperators.map((item) => item.value)); + +const tagFilterOperators: Record = { + [DatasetCollectionTagTypeEnum.number]: [ + { labelKey: 'workflow:tag_filter_op_eq', value: '$eq', icon: 'math/equal' }, + { labelKey: 'workflow:tag_filter_op_ne', value: '$ne', icon: 'math/notEqual' }, + { labelKey: 'workflow:tag_filter_op_gt', value: '$gt', icon: 'math/greater' }, + { labelKey: 'workflow:tag_filter_op_lt', value: '$lt', icon: 'math/greater', iconFlip: true }, + { labelKey: 'workflow:tag_filter_op_gte', value: '$gte', icon: 'math/greaterEqual' }, + { + labelKey: 'workflow:tag_filter_op_lte', + value: '$lte', + icon: 'math/greaterEqual', + iconFlip: true + }, + ...emptyValueOperators + ], + [DatasetCollectionTagTypeEnum.datetime]: [ + { labelKey: 'workflow:tag_filter_op_is', value: '$eq' }, + { labelKey: 'workflow:tag_filter_op_is_not', value: '$ne' }, + { labelKey: 'workflow:tag_filter_op_after', value: '$gt' }, + { labelKey: 'workflow:tag_filter_op_before', value: '$lt' }, + ...emptyValueOperators + ], + [DatasetCollectionTagTypeEnum.array]: [ + { labelKey: 'workflow:tag_filter_op_is', value: '$is' }, + { labelKey: 'workflow:tag_filter_op_is_not', value: '$isNot' }, + { labelKey: 'workflow:tag_filter_op_in', value: '$in' }, + { labelKey: 'workflow:tag_filter_op_not_in', value: '$notIn' }, + ...emptyValueOperators + ] +}; + +const createTimeOperators = tagFilterOperators[DatasetCollectionTagTypeEnum.number].filter( + (item) => item.value === '$gte' || item.value === '$lte' +); +const collectionIdOperators = tagFilterOperators[DatasetCollectionTagTypeEnum.array].filter( + (item) => item.value === '$in' +); + +export const createEmptyTagFilterCondition = (): DatasetTagFilterCondition => ({ + tag: '', + op: '', + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined +}); + +export const createEmptyTagFilterValue = (): DatasetTagFilterValue => ({ + logic: DatasetTagFilterLogicEnum.AND, + conditions: [createEmptyTagFilterCondition()] +}); + +export const isWorkflowTagFilterTagType = ( + tagType?: DatasetCollectionTagType +): tagType is WorkflowTagFilterTagType => WorkflowTagFilterTagTypeSchema.safeParse(tagType).success; + +/** + * 判断节点/表单 value 是否为新版条件行结构。 + * 旧版 collectionFilterMatch 是 JSON 字符串（tags/createTime/collectionIds）。 + */ +export const isDatasetTagFilterValue = (value: unknown): value is DatasetTagFilterValue => { + return DatasetTagFilterValueSchema.safeParse(value).success; +}; + +/** 条件不需要右侧值输入（为空 / 不为空）。 */ +export const isTagFilterOpWithoutValue = (op?: string) => !!op && emptyOps.has(op); + +export const isTagFilterAttributeField = (field?: DatasetTagFilterField | string) => + field === DatasetTagFilterFieldEnum.createTime || + field === DatasetTagFilterFieldEnum.collectionId; + +export const getTagFilterOpsByType = (tagType?: WorkflowTagFilterTagType) => { + return tagType ? tagFilterOperators[tagType] : []; +}; + +/** 文件属性只暴露检索载荷能准确表达的操作符，其余字段按标签类型选择。 */ +export const getTagFilterOpsByCondition = (condition: DatasetTagFilterCondition) => { + if (condition.field === DatasetTagFilterFieldEnum.createTime) { + return createTimeOperators; + } + if (condition.field === DatasetTagFilterFieldEnum.collectionId) { + return collectionIdOperators; + } + return getTagFilterOpsByType(condition.tagType); +}; + +const parseMaybeJson = (value: unknown): unknown => { + if (typeof value !== 'string') return value; + const trimmed = value.trim(); + if (!trimmed) return value; + if ( + !( + (trimmed.startsWith('{') && trimmed.endsWith('}')) || + (trimmed.startsWith('[') && trimmed.endsWith(']')) + ) + ) { + return value; + } + try { + return JSON.parse(trimmed); + } catch { + return value; + } +}; + +type TagConditionObject = Record>; + +export const formatTagOptionKey = (tag: string, tagType: string) => `${tag}\0${tagType}`; + +export const parseTagOptionKey = (value: string) => { + const splitIndex = value.indexOf('\0'); + if (splitIndex < 0) return; + return { + tag: value.slice(0, splitIndex), + tagType: value.slice(splitIndex + 1) as WorkflowTagFilterOption['tagType'] + }; +}; + +/** + * 多知识库标签下拉：各库 number/datetime/array 标签按「名称 + 类型」取交集。 + * 只在部分库出现、或同名不同类型的项不进入下拉。array 的 options 取并集去重。 + */ +export const intersectWorkflowTagOptions = ( + tagLists: Pick[][] +): WorkflowTagFilterOption[] => { + if (tagLists.length === 0) return []; + + const maps = tagLists.map((list) => { + const map = new Map(); + for (const item of list) { + if (!isWorkflowTagFilterTagType(item.tagType)) continue; + const key = formatTagOptionKey(item.tag, item.tagType); + const prev = map.get(key); + const options = Array.from( + new Set([...(prev?.options ?? []), ...(item.options ?? []).filter(Boolean)]) + ); + map.set(key, { tag: item.tag, tagType: item.tagType, options }); + } + return map; + }); + + const [first, ...rest] = maps; + if (!first) return []; + + const result: WorkflowTagFilterOption[] = []; + for (const [key, option] of first) { + if (!rest.every((item) => item.has(key))) continue; + const mergedOptions = new Set(option.options); + for (const item of rest) { + const other = item.get(key); + other?.options.forEach((value) => mergedOptions.add(value)); + } + result.push({ + tag: option.tag, + tagType: option.tagType, + options: Array.from(mergedOptions) + }); + } + return result; +}; + +const isTagCondition = (condition: DatasetTagFilterCondition) => + !condition.field || condition.field === DatasetTagFilterFieldEnum.tag; + +const buildTagConditionObject = ( + condition: DatasetTagFilterCondition +): TagConditionObject | undefined => { + if (!isTagCondition(condition)) return; + const tag = condition.tag?.trim(); + const op = condition.op; + if (!tag || !op) return; + if (isTagFilterOpWithoutValue(op)) { + return { [tag]: { [op]: true } }; + } + if (condition.value === undefined || condition.value === null || condition.value === '') { + return; + } + return { [tag]: { [op]: condition.value } }; +}; + +const toCreateTimeString = (value: unknown): string | undefined => { + if (typeof value === 'number' && Number.isFinite(value)) { + return formatTime2YMDHM(value) || undefined; + } + if (typeof value !== 'string') return; + const trimmed = value.trim(); + if (!trimmed) return; + const parsed = Date.parse(trimmed); + return Number.isNaN(parsed) ? undefined : formatTime2YMDHM(parsed) || undefined; +}; + +const toIdList = (value: unknown): string[] => { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof value === 'string') { + return value + .split(/[\s,，]+/) + .map((item) => item.trim()) + .filter(Boolean); + } + if (typeof value === 'number' && Number.isFinite(value)) { + return [String(value)]; + } + return []; +}; + +/** + * 把条件行编成检索入口 JSON：tags + 可选 createTime / collectionIds。 + * logic 只作用于 tags；文件属性在检索协议中是顶层约束，始终与标签结果求交集。 + * 创建时间多行按更严的 $gte/$lte 合并，Collection ID 多行合并为一个去重白名单。 + * 未填完的行会被丢掉；没有任何有效字段时返回 undefined。 + */ +export const serializeDatasetTagFilterValue = ( + value: DatasetTagFilterValue +): string | undefined => { + const tagConditions = value.conditions + .map(buildTagConditionObject) + .filter((item): item is TagConditionObject => Boolean(item)); + + const createTime: { $gte?: string; $lte?: string } = {}; + for (const condition of value.conditions) { + if (condition.field !== DatasetTagFilterFieldEnum.createTime || !condition.op) continue; + const time = toCreateTimeString(condition.value); + if (!time) continue; + if (condition.op === '$gte') { + if (!createTime.$gte || time > createTime.$gte) createTime.$gte = time; + continue; + } + if (condition.op === '$lte') { + if (!createTime.$lte || time < createTime.$lte) createTime.$lte = time; + } + } + + const idLists = value.conditions + .filter( + (condition) => + condition.field === DatasetTagFilterFieldEnum.collectionId && condition.op === '$in' + ) + .map((condition) => toIdList(condition.value)) + .filter((list) => list.length > 0); + const collectionIds = [...new Set(idLists.flat())]; + + const payload: Record = {}; + if (tagConditions.length > 0) { + const key = value.logic === DatasetTagFilterLogicEnum.OR ? '$or' : '$and'; + payload.tags = { [key]: tagConditions }; + } + if (createTime.$gte || createTime.$lte) { + payload.createTime = createTime; + } + if (collectionIds.length > 0) { + payload.collectionIds = collectionIds; + } + if (Object.keys(payload).length === 0) return undefined; + return JSON.stringify(payload); +}; + +/** 条件行引用值是 `[nodeId, outputKey]`，不复用工作流 utils 以免 dataset ↔ workflow 循环依赖。 */ +const isReferenceTuple = (value: unknown): value is [string, string?] => + Array.isArray(value) && + value.length === 2 && + typeof value[0] === 'string' && + (value[1] === undefined || typeof value[1] === 'string'); + +const resolveConditionValue = ( + condition: DatasetTagFilterCondition, + resolveReference: (value: unknown) => unknown +): DatasetTagFilterCondition => { + if (condition.valueMode !== DatasetTagFilterValueModeEnum.reference) return condition; + if (!isReferenceTuple(condition.value)) return { ...condition, value: undefined }; + return { ...condition, value: resolveReference(condition.value) }; +}; + +/** + * 运行时把 collectionFilterMatch 统一成检索 JSON 字符串。 + * 整段引用、旧 JSON 字符串原样（或解析后若是条件行再序列化）；条件行会先解析行内引用。 + */ +export const formatCollectionFilterMatchParam = ({ + value, + resolveReference = () => undefined +}: { + value: unknown; + resolveReference?: (value: unknown) => unknown; +}): string | undefined => { + if (value === undefined || value === null || value === '') return undefined; + + const parsed = parseMaybeJson(value); + const structured = isDatasetTagFilterValue(parsed) ? parsed : undefined; + + if (structured) { + const resolved: DatasetTagFilterValue = { + logic: structured.logic, + conditions: structured.conditions.map((condition) => + resolveConditionValue(condition, resolveReference) + ) + }; + return serializeDatasetTagFilterValue(resolved); + } + + if (typeof value === 'string') return value; + if (typeof value === 'object') return JSON.stringify(value); + return undefined; +}; + +/** + * 已选库变化后，丢掉不在新交集里的标签行。 + * 文件属性和尚未选择字段的空行都保留，否则「添加过滤条件」会被立刻清掉。 + */ +export const pruneTagFilterConditions = ( + value: DatasetTagFilterValue, + options: WorkflowTagFilterOption[] +): DatasetTagFilterValue => { + const valid = new Set(options.map((item) => formatTagOptionKey(item.tag, item.tagType))); + const conditions = value.conditions.filter((condition) => { + if (isTagFilterAttributeField(condition.field) || !condition.tag) return true; + return !!condition.tagType && valid.has(formatTagOptionKey(condition.tag, condition.tagType)); + }); + return { + ...value, + conditions: conditions.length > 0 ? conditions : [createEmptyTagFilterCondition()] + }; +}; diff --git a/packages/global/core/workflow/constants.ts b/packages/global/core/workflow/constants.ts index 7686320e04e2..1c54c52789c3 100644 --- a/packages/global/core/workflow/constants.ts +++ b/packages/global/core/workflow/constants.ts @@ -202,6 +202,7 @@ export enum NodeInputKeyEnum { datasetSearchExtensionModel = 'datasetSearchExtensionModel', datasetSearchExtensionBg = 'datasetSearchExtensionBg', datasetSearchInput = 'datasetSearchInput', + collectionFilterVersion = 'collectionFilterVersion', collectionFilterMatch = 'collectionFilterMatch', authTmbId = 'authTmbId', datasetDeepSearch = 'datasetDeepSearch', diff --git a/packages/global/core/workflow/node/constant.ts b/packages/global/core/workflow/node/constant.ts index d10f6284cf87..a50311d74236 100644 --- a/packages/global/core/workflow/node/constant.ts +++ b/packages/global/core/workflow/node/constant.ts @@ -37,6 +37,8 @@ export enum FlowNodeInputTypeEnum { timeRangeSelect = 'timeRangeSelect', password = 'password', + datasetTagFilter = 'datasetTagFilter', + agentGenerated = 'agentGenerated' // for compatibility for >= v4.16.0 } export const FlowNodeInputMap: Record< @@ -117,6 +119,9 @@ export const FlowNodeInputMap: Record< [FlowNodeInputTypeEnum.password]: { icon: 'core/workflow/inputType/password' }, + [FlowNodeInputTypeEnum.datasetTagFilter]: { + icon: 'core/workflow/inputType/input' + }, [FlowNodeInputTypeEnum.agentGenerated]: { icon: 'core/workflow/inputType/agentGenerated' } diff --git a/packages/global/core/workflow/template/input.ts b/packages/global/core/workflow/template/input.ts index 54024f44e795..86334b4d52db 100644 --- a/packages/global/core/workflow/template/input.ts +++ b/packages/global/core/workflow/template/input.ts @@ -4,6 +4,15 @@ import { WorkflowIOValueTypeEnum } from '../constants'; import { chatNodeSystemPromptTip, systemPromptTip } from './tip'; import { type FlowNodeInputItemType } from '../type/io'; import { i18nT } from '../../../common/i18n/utils'; +import { DatasetTagFilterVersionEnum } from '../../dataset/workflowTagFilter'; + +export const Input_Template_Dataset_Tag_Filter_Version: FlowNodeInputItemType = { + key: NodeInputKeyEnum.collectionFilterVersion, + renderTypeList: [FlowNodeInputTypeEnum.hidden], + label: '', + valueType: WorkflowIOValueTypeEnum.string, + value: DatasetTagFilterVersionEnum.structured +}; export const Input_Template_History: FlowNodeInputItemType = { key: NodeInputKeyEnum.history, diff --git a/packages/global/core/workflow/template/system/agent/index.ts b/packages/global/core/workflow/template/system/agent/index.ts index 6e2c7e183fac..57af857f7995 100644 --- a/packages/global/core/workflow/template/system/agent/index.ts +++ b/packages/global/core/workflow/template/system/agent/index.ts @@ -12,13 +12,14 @@ import { NodeInputKeyEnum } from '../../../constants'; import { + Input_Template_Dataset_Tag_Filter_Version, + Input_Template_File_Link, Input_Template_SettingAiModel, Input_Template_System_Prompt, Input_Template_UserChatInput } from '../../input'; import { chatNodeSystemPromptTip, systemPromptTip } from '../../tip'; import { i18nT } from '../../../../../common/i18n/utils'; -import { Input_Template_File_Link } from '../../input'; import { Output_Template_Error_Message } from '../../output'; import { DatasetSearchModeEnum } from '../../../../dataset/constants'; @@ -246,6 +247,15 @@ export const AgentNode: FlowNodeTemplateType = { label: '', valueType: WorkflowIOValueTypeEnum.boolean, value: false + }, + Input_Template_Dataset_Tag_Filter_Version, + { + key: NodeInputKeyEnum.collectionFilterMatch, + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + label: i18nT('workflow:tag_filter'), + valueType: WorkflowIOValueTypeEnum.string, + isPro: true, + description: i18nT('workflow:tag_filter_description') } ], outputs: [ diff --git a/packages/global/core/workflow/template/system/datasetSearch.ts b/packages/global/core/workflow/template/system/datasetSearch.ts index be6ca3a01c7c..65c14abc6e85 100644 --- a/packages/global/core/workflow/template/system/datasetSearch.ts +++ b/packages/global/core/workflow/template/system/datasetSearch.ts @@ -12,7 +12,7 @@ import { NodeOutputKeyEnum, FlowNodeTemplateTypeEnum } from '../../constants'; -import { Input_Template_UserChatInput } from '../input'; +import { Input_Template_Dataset_Tag_Filter_Version, Input_Template_UserChatInput } from '../input'; import { DatasetSearchModeEnum } from '../../../dataset/constants'; import { i18nT } from '../../../../common/i18n/utils'; import { Output_Template_Error_Message } from '../output'; @@ -34,7 +34,7 @@ export const DatasetSearchModule: FlowNodeTemplateType = { isTool: true, catchError: false, courseUrl: '/guide/build/workflow/nodes/dataset_search', - version: '4.9.2', + version: '4.17.0', inputs: [ { key: NodeInputKeyEnum.datasetSelectList, @@ -131,14 +131,14 @@ export const DatasetSearchModule: FlowNodeTemplateType = { valueType: WorkflowIOValueTypeEnum.arrayString, toolDescription: i18nT('workflow:content_to_search') }, + Input_Template_Dataset_Tag_Filter_Version, { key: NodeInputKeyEnum.collectionFilterMatch, - renderTypeList: [FlowNodeInputTypeEnum.textarea, FlowNodeInputTypeEnum.reference], - label: i18nT('workflow:collection_metadata_filter'), - + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + label: i18nT('workflow:tag_filter'), valueType: WorkflowIOValueTypeEnum.string, isPro: true, - description: i18nT('workflow:filter_description') + description: i18nT('workflow:tag_filter_description') } ], outputs: [ diff --git a/packages/global/openapi/core/dataset/collection/api.ts b/packages/global/openapi/core/dataset/collection/api.ts index a1cfdc49b907..e05e008a2e86 100644 --- a/packages/global/openapi/core/dataset/collection/api.ts +++ b/packages/global/openapi/core/dataset/collection/api.ts @@ -11,6 +11,7 @@ import { } from '../../../../core/dataset/constants'; import { CollectionTagFilterItemSchema, + CollectionTagLabelSchema, CollectionTrainingStatusSchema, DatasetCollectionItemSchema, DatasetCollectionSchema @@ -30,12 +31,7 @@ export const UpdateDatasetCollectionBodySchema = z.object({ parentId: ParentIdSchema.describe('父级目录ID'), name: z.string().optional().describe('集合名称'), tags: z - .array( - z.union([ - z.string(), - z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) - ]) - ) + .array(CollectionTagLabelSchema) .optional() .describe('标签列表（支持 String 旧格式 或 { tag, value } 新格式）'), forbid: z.boolean().optional().describe('是否禁用'), @@ -169,18 +165,7 @@ export const DatasetCollectionsListItemSchema = z updateTime: DatasetCollectionSchema.shape.updateTime, forbid: DatasetCollectionSchema.shape.forbid, trainingType: DatasetCollectionSchema.shape.trainingType, - tags: z - .array( - z.union([ - z.string(), - z.object({ - tag: z.string(), - value: z.union([z.string(), z.number(), z.array(z.string())]) - }) - ]) - ) - .optional() - .meta({ description: '标签。string 为标签名；新格式为 { tag, value }' }), + tags: DatasetCollectionItemSchema.shape.tags, externalFileId: z.string().optional().meta({ description: '外部文件 ID' }), @@ -273,14 +258,13 @@ export type SyncCollectionResponseType = z.infer; -export const TagFilterOptionItemSchema = z.object({ +const TagFilterOptionItemSchema = z.object({ tagId: z.string().meta({ example: '68ad85a7463006c963799a05', description: '标签 ID' }), values: z .array(z.union([z.string(), z.number()])) diff --git a/packages/global/openapi/core/dataset/collection/createApi.ts b/packages/global/openapi/core/dataset/collection/createApi.ts index 2088100cd20c..74f8b8317971 100644 --- a/packages/global/openapi/core/dataset/collection/createApi.ts +++ b/packages/global/openapi/core/dataset/collection/createApi.ts @@ -1,5 +1,5 @@ import z from 'zod'; -import { ChunkSettingsSchema } from '../../../../core/dataset/type'; +import { ChunkSettingsSchema, CollectionTagLabelSchema } from '../../../../core/dataset/type'; import { DatasetCollectionTypeEnum } from '../../../../core/dataset/constants'; import { ParentIdSchema } from '../../../../common/parentFolder/type'; import { ObjectIdSchema } from '../../../../common/type/mongo'; @@ -16,21 +16,15 @@ const DatasetCollectionStoreDataSchema = ChunkSettingsSchema.extend({ customPdfParse: z.boolean().optional().meta({ description: '自定义 PDF 解析' }) }); +const CollectionTagsInputSchema = z.array(CollectionTagLabelSchema).optional().meta({ + description: + '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' +}); + // API 创建集合通用基础 Schema export const ApiCreateCollectionBaseSchema = DatasetCollectionStoreDataSchema.extend({ datasetId: z.string().meta({ description: '数据集 ID' }), - tags: z - .array( - z.union([ - z.string(), - z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) - ]) - ) - .optional() - .meta({ - description: - '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' - }) + tags: CollectionTagsInputSchema }); export type ApiCreateDatasetCollectionParams = z.infer; @@ -61,18 +55,7 @@ export const CreateCollectionBodySchema = z.object({ type: z .enum([DatasetCollectionTypeEnum.folder, DatasetCollectionTypeEnum.virtual]) .meta({ description: '集合类型（folder: 文件夹，virtual: 手动集合）' }), - tags: z - .array( - z.union([ - z.string(), - z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) - ]) - ) - .optional() - .meta({ - description: - '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' - }) + tags: CollectionTagsInputSchema }); export type CreateCollectionBodyType = z.infer; @@ -177,18 +160,7 @@ export const CreateImageCollectionDataSchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), collectionName: z.string().meta({ description: '集合名称' }), - tags: z - .array( - z.union([ - z.string(), - z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) - ]) - ) - .optional() - .meta({ - description: - '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' - }) + tags: CollectionTagsInputSchema }); export type CreateImageCollectionDataType = z.infer; // handler 内 parse 用 @@ -214,18 +186,7 @@ export const CreateImageCollectionMultipartSchema = z.object({ export const CreateBackupCollectionFormSchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), - tags: z - .array( - z.union([ - z.string(), - z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) - ]) - ) - .optional() - .meta({ - description: - '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' - }) + tags: CollectionTagsInputSchema }); export type CreateBackupCollectionFormType = z.infer; @@ -248,18 +209,7 @@ export const CreateBackupCollectionMultipartSchema = z.object({ export const CreateTemplateCollectionFormSchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), - tags: z - .array( - z.union([ - z.string(), - z.object({ tag: z.string(), value: z.union([z.string(), z.number(), z.array(z.string())]) }) - ]) - ) - .optional() - .meta({ - description: - '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' - }) + tags: CollectionTagsInputSchema }); export type CreateTemplateCollectionFormType = z.infer; diff --git a/packages/global/openapi/core/dataset/collection/tagApi.ts b/packages/global/openapi/core/dataset/collection/tagApi.ts index 6e7063120252..98342dd13bf1 100644 --- a/packages/global/openapi/core/dataset/collection/tagApi.ts +++ b/packages/global/openapi/core/dataset/collection/tagApi.ts @@ -10,7 +10,7 @@ export const BatchCollectionTagModeEnum = { add: 'add', remove: 'remove' } as const; -export const BatchCollectionTagModeSchema = z.enum([ +const BatchCollectionTagModeSchema = z.enum([ BatchCollectionTagModeEnum.add, BatchCollectionTagModeEnum.remove ]); @@ -41,7 +41,6 @@ export const AddTagsToCollectionsBodySchema = z.object({ tag: z.string().trim().meta({ description: '标签名称' }), value: z.string().optional().meta({ description: '标签值（仅 string 类型标签支持）' }) }); -export type AddTagsToCollectionsParams = z.infer; /* ============================================================================ * API: 更新集合标签 @@ -65,7 +64,6 @@ export const DeleteDatasetCollectionTagQuerySchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), id: z.string().meta({ description: '标签 ID' }) }); -export type DeleteDatasetCollectionTagQuery = z.infer; /* ============================================================================ * API: 获取知识库全部标签 @@ -74,13 +72,12 @@ export type DeleteDatasetCollectionTagQuery = z.infer; /* ============================================================================ * API: 批量 Upsert 标签 * Route: POST /proApi/core/dataset/tag/batchUpsert * ============================================================================ */ -export const BatchUpsertTagItemSchema = z.object({ +const BatchUpsertTagItemSchema = z.object({ tag: z.string().trim().min(1).meta({ description: '标签名称' }), tagType: z.enum(DatasetCollectionTagTypeEnum).optional().meta({ description: '标签类型' }) }); @@ -88,7 +85,6 @@ export const BatchUpsertTagsBodySchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), tags: z.array(BatchUpsertTagItemSchema).min(1).meta({ description: '标签列表' }) }); -export type BatchUpsertTagsParams = z.infer; /* ============================================================================ * API: 设置单个 Collection 标签值 @@ -105,7 +101,7 @@ export type SetCollectionTagsParams = z.infer { FlowNodeInputTypeEnum.hidden, FlowNodeInputTypeEnum.selectDataset, FlowNodeInputTypeEnum.selectLLMModel, - FlowNodeInputTypeEnum.customVariable + FlowNodeInputTypeEnum.customVariable, + FlowNodeInputTypeEnum.datasetTagFilter ])('should hide unsupported parent tool configuration type %s', (renderType) => { expect(canInputBeConfiguredAsToolParam(createMockInput({ renderTypeList: [renderType] }))).toBe( false diff --git a/packages/global/test/core/app/utils.test.ts b/packages/global/test/core/app/utils.test.ts index fbdd8c909dc3..557cf1b9357c 100644 --- a/packages/global/test/core/app/utils.test.ts +++ b/packages/global/test/core/app/utils.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { getDefaultAppForm, getAppType, formatToolError } from '@fastgpt/global/core/app/utils'; import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; describe('getDefaultAppForm', () => { it('should return default app form with correct structure', () => { @@ -34,6 +35,7 @@ describe('getDefaultAppForm', () => { rerankWeight: 0.5, datasetSearchUsingExtensionQuery: true, datasetSearchExtensionBg: '', + [NodeInputKeyEnum.collectionFilterVersion]: 'structured', authTmbId: false }); }); diff --git a/packages/global/test/core/dataset/tagUtils.test.ts b/packages/global/test/core/dataset/tagUtils.test.ts new file mode 100644 index 000000000000..a3f473875c1c --- /dev/null +++ b/packages/global/test/core/dataset/tagUtils.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { + isCollectionTagValue, + isUsableCollectionTagFilterValue, + sortCollectionTagValues +} from '@fastgpt/global/core/dataset/tagUtils'; + +describe('dataset tag utilities', () => { + it('validates stored/filter values and sorts display values', () => { + expect(['PRD', 0].every(isUsableCollectionTagFilterValue)).toBe(true); + expect(['', Number.NaN, [], null].some(isUsableCollectionTagFilterValue)).toBe(false); + expect(isCollectionTagValue({ tagId: 'tag-1', value: ['A'] })).toBe(true); + expect(isCollectionTagValue('legacy-id')).toBe(false); + expect(sortCollectionTagValues([2, 10, 0])).toEqual([0, 2, 10]); + expect(sortCollectionTagValues(['spec', 'PRD'])).toEqual(['PRD', 'spec']); + }); +}); diff --git a/packages/global/test/core/dataset/workflowTagFilter.test.ts b/packages/global/test/core/dataset/workflowTagFilter.test.ts new file mode 100644 index 000000000000..4dfaa6fca552 --- /dev/null +++ b/packages/global/test/core/dataset/workflowTagFilter.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from 'vitest'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { + createEmptyTagFilterValue, + DatasetTagFilterFieldEnum, + DatasetTagFilterLogicEnum, + DatasetTagFilterValueModeEnum, + DatasetTagFilterVersionEnum, + formatCollectionFilterMatchParam, + getTagFilterOpsByCondition, + intersectWorkflowTagOptions, + isDatasetTagFilterValue, + normalizeLegacyDatasetTagFilterValue, + pruneTagFilterConditions, + resolveDatasetTagFilterVersion, + serializeDatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; + +describe('dataset tag filter version', () => { + it('uses structured only when explicitly marked and does not infer from the filter value', () => { + expect( + resolveDatasetTagFilterVersion({ + version: DatasetTagFilterVersionEnum.structured, + filterValue: 'legacy config' + }) + ).toBe(DatasetTagFilterVersionEnum.structured); + expect( + resolveDatasetTagFilterVersion({ version: undefined, filterValue: 'legacy config' }) + ).toBe(DatasetTagFilterVersionEnum.legacy); + expect(resolveDatasetTagFilterVersion({ version: undefined, filterValue: undefined })).toBe( + DatasetTagFilterVersionEnum.structured + ); + expect(resolveDatasetTagFilterVersion({ version: 'invalid', filterValue: undefined })).toBe( + DatasetTagFilterVersionEnum.legacy + ); + expect(normalizeLegacyDatasetTagFilterValue('{"tags":{}}')).toBe('{"tags":{}}'); + expect(normalizeLegacyDatasetTagFilterValue(createEmptyTagFilterValue())).toBe(''); + }); +}); + +describe('dataset tag filter options', () => { + it('keeps supported tags shared by every dataset regardless of migration metadata', () => { + expect( + intersectWorkflowTagOptions([ + [ + { + tag: 'default_tag', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['legacy'], + fromMigration: true + }, + { tag: 'status', tagType: DatasetCollectionTagTypeEnum.array, options: ['open'] }, + { tag: 'title', tagType: DatasetCollectionTagTypeEnum.string, options: [] } + ], + [ + { + tag: 'default_tag', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['archived'], + fromMigration: true + }, + { + tag: 'status', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['closed', 'open'] + }, + { tag: 'only-second', tagType: DatasetCollectionTagTypeEnum.number, options: [] } + ] + ]) + ).toEqual([ + { + tag: 'default_tag', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['legacy', 'archived'] + }, + { tag: 'status', tagType: DatasetCollectionTagTypeEnum.array, options: ['open', 'closed'] } + ]); + }); + + it('limits file attributes to operations supported by the search payload', () => { + expect( + getTagFilterOpsByCondition({ field: DatasetTagFilterFieldEnum.createTime }).map( + (item) => item.value + ) + ).toEqual(['$gte', '$lte']); + expect( + getTagFilterOpsByCondition({ field: DatasetTagFilterFieldEnum.collectionId }).map( + (item) => item.value + ) + ).toEqual(['$in']); + }); +}); + +describe('serializeDatasetTagFilterValue', () => { + it('serializes tag logic and top-level file constraints while dropping incomplete rows', () => { + const result = serializeDatasetTagFilterValue({ + logic: DatasetTagFilterLogicEnum.OR, + conditions: [ + { + tag: 'status', + tagType: DatasetCollectionTagTypeEnum.array, + op: '$in', + value: ['open'] + }, + { + field: DatasetTagFilterFieldEnum.createTime, + op: '$gte', + value: '2026-03-01T08:00:00' + }, + { + field: DatasetTagFilterFieldEnum.collectionId, + op: '$in', + value: 'id-1, id-2, id-1' + }, + { tag: 'incomplete' } + ] + }); + + expect(JSON.parse(result ?? '')).toEqual({ + tags: { $or: [{ status: { $in: ['open'] } }] }, + createTime: { $gte: expect.any(String) }, + collectionIds: ['id-1', 'id-2'] + }); + expect(serializeDatasetTagFilterValue(createEmptyTagFilterValue())).toBeUndefined(); + }); +}); + +describe('formatCollectionFilterMatchParam', () => { + it('resolves structured row references and preserves legacy strings without converting them', () => { + expect( + formatCollectionFilterMatchParam({ + value: { + logic: DatasetTagFilterLogicEnum.AND, + conditions: [ + { + tag: 'price', + tagType: DatasetCollectionTagTypeEnum.number, + op: '$gte', + valueMode: DatasetTagFilterValueModeEnum.reference, + value: ['node', 'price'] + } + ] + }, + resolveReference: () => 10 + }) + ).toBe(JSON.stringify({ tags: { $and: [{ price: { $gte: 10 } }] } })); + + const legacy = '{"tags":{"$and":["legacy"]}}'; + expect(formatCollectionFilterMatchParam({ value: legacy })).toBe(legacy); + expect(formatCollectionFilterMatchParam({ value: undefined })).toBeUndefined(); + }); +}); + +describe('pruneTagFilterConditions', () => { + it('removes unavailable tags but keeps attributes and a usable empty row', () => { + const result = pruneTagFilterConditions( + { + logic: DatasetTagFilterLogicEnum.AND, + conditions: [ + { tag: 'gone', tagType: DatasetCollectionTagTypeEnum.number, op: '$eq', value: 1 }, + { field: DatasetTagFilterFieldEnum.createTime, op: '$gte', value: 1 } + ] + }, + [] + ); + + expect(result.conditions).toEqual([ + { field: DatasetTagFilterFieldEnum.createTime, op: '$gte', value: 1 } + ]); + expect(isDatasetTagFilterValue(result)).toBe(true); + }); +}); diff --git a/packages/service/core/dataset/collection/tagFilter.ts b/packages/service/core/dataset/collection/tagFilter.ts index 45e209eef64f..70595ed5d0db 100644 --- a/packages/service/core/dataset/collection/tagFilter.ts +++ b/packages/service/core/dataset/collection/tagFilter.ts @@ -1,7 +1,5 @@ import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; import { - collectionTagValueKey, - isCollectionTagValue, isUsableCollectionTagFilterValue, sortCollectionTagValues } from '@fastgpt/global/core/dataset/tagUtils'; @@ -9,43 +7,6 @@ import type { TagFilterOptionItemType } from '@fastgpt/global/openapi/core/datas import { Types } from '../../../common/mongo'; import { MongoDatasetCollection } from './schema'; -/** - * 从 Collection 上聚合每个 tagId「当前已被使用」的值。 - * 不返回标签名/类型，由调用方用已有标签定义拼接；选项类只保留实际出现过的选项。 - */ -export const collectUsedTagValues = ( - collections: Array<{ tags?: unknown[] }> -): TagFilterOptionItemType[] => { - const valuesByTagId = new Map>(); - - const addValue = (tagId: string, value: unknown) => { - if (!isUsableCollectionTagFilterValue(value)) return; - const current = valuesByTagId.get(tagId) ?? new Map(); - current.set(collectionTagValueKey(value), value); - valuesByTagId.set(tagId, current); - }; - - for (const collection of collections) { - for (const item of collection.tags ?? []) { - if (!isCollectionTagValue(item)) continue; - - const tagId = String(item.tagId); - if (Array.isArray(item.value)) { - for (const value of item.value) addValue(tagId, value); - continue; - } - addValue(tagId, item.value); - } - } - - return [...valuesByTagId.entries()] - .map(([tagId, values]) => ({ - tagId, - values: sortCollectionTagValues(Array.from(values.values())) - })) - .sort((a, b) => a.tagId.localeCompare(b.tagId)); -}; - /** * 按 tagFilters 组装 Collection 列表的标签过滤条件。 * 同一标签多值为 OR（value $in），不同标签为 AND。 @@ -66,8 +27,14 @@ export const buildCollectionListTagMatch = (tagFilters: CollectionTagFilterItem[ return { $and: conditions }; }; +type UsedTagValueGroup = { + _id: string; + values: unknown[]; +}; + /** * 聚合当前知识库 Collection 上已被使用的标签值，不含标签定义。 + * Mongo unwind + $addToSet 按 tagId 去重后再拉回 Node，空串和非有限数字在本地丢掉。 */ export const getDatasetTagFilterOptions = async ({ teamId, @@ -76,14 +43,47 @@ export const getDatasetTagFilterOptions = async ({ teamId: string; datasetId: string; }): Promise => { - // 筛选弹窗打开后立刻读，走主库避免刚写入的标签值被从库滞后挡住 - const collections = await MongoDatasetCollection.find( + const grouped = await MongoDatasetCollection.aggregate([ { - teamId: new Types.ObjectId(teamId), - datasetId: new Types.ObjectId(datasetId) + $match: { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(datasetId) + } }, - 'tags' - ).lean(); + { $project: { tags: 1 } }, + { $unwind: '$tags' }, + { + $match: { + 'tags.tagId': { $exists: true, $nin: [null, ''] }, + 'tags.value': { $exists: true } + } + }, + { + $project: { + tagId: { $toString: '$tags.tagId' }, + values: { + $cond: { + if: { $isArray: '$tags.value' }, + then: '$tags.value', + else: ['$tags.value'] + } + } + } + }, + { $unwind: '$values' }, + { + $group: { + _id: '$tagId', + values: { $addToSet: '$values' } + } + } + ]); - return collectUsedTagValues(collections); + return grouped + .map((item) => ({ + tagId: String(item._id), + values: sortCollectionTagValues(item.values.filter(isUsableCollectionTagFilterValue)) + })) + .filter((item) => item.values.length > 0) + .sort((a, b) => a.tagId.localeCompare(b.tagId)); }; diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 0853192dd336..a1bbd622cf72 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -4,6 +4,7 @@ import { MongoDatasetCollectionTagsV2 } from '../tag/schemaV2'; import { readFromSecondary } from '../../../common/mongo/utils'; import { DEFAULT_TAG, + type CollectionTagLabelType, type CollectionTagValueType, type CollectionWithDatasetType, type DatasetCollectionTagType @@ -70,28 +71,6 @@ export function getCollectionUpdateTime({ name, time }: { time?: Date; name: str return new Date(); } -const normalizeDatasetTagValue = ({ - tagType, - value -}: { - tagType: DatasetCollectionTagType; - value: string | number | string[]; -}): { value: string | number | string[]; error?: DatasetErrEnum } => { - if (tagType !== 'number' && tagType !== 'datetime') return { value }; - - const numericValue = - typeof value === 'number' ? value : typeof value === 'string' ? Number(value) : NaN; - if (typeof value === 'string' && value.trim() === '') { - return { value, error: DatasetErrEnum.tagValueInvalid }; - } - if (!Number.isFinite(numericValue)) return { value, error: DatasetErrEnum.tagValueInvalid }; - if (tagType === 'datetime' && Number.isNaN(new Date(numericValue).getTime())) { - return { value, error: DatasetErrEnum.tagValueDatetimeInvalid }; - } - - return { value: numericValue }; -}; - export const validateDatasetTagValue = ({ tagType, value @@ -99,23 +78,7 @@ export const validateDatasetTagValue = ({ tagType?: DatasetCollectionTagType; value: string | number | string[]; }): DatasetErrEnum | undefined => { - const type = tagType || 'string'; - - if (type === 'string' && (typeof value !== 'string' || value.length > 256)) { - return DatasetErrEnum.tagValueInvalid; - } - if (type === 'array') { - if ( - !Array.isArray(value) || - value.length > 64 || - value.some((item) => typeof item !== 'string' || item.length > 256) - ) { - return DatasetErrEnum.arrayTagValueInvalid; - } - return undefined; - } - - return normalizeDatasetTagValue({ tagType: type, value }).error; + return validateAndNormalizeTagValue({ tagType, value }).error; }; /** @@ -131,10 +94,33 @@ export const validateAndNormalizeTagValue = ({ tagType?: DatasetCollectionTagType; value: string | number | string[]; }): { value: string | number | string[]; error?: DatasetErrEnum } => { - if (tagType === 'number' || tagType === 'datetime') { - return normalizeDatasetTagValue({ tagType, value }); + const type = tagType ?? 'string'; + + if (type === 'string') { + const error = typeof value !== 'string' || value.length > 256; + return error ? { value, error: DatasetErrEnum.tagValueInvalid } : { value }; + } + if (type === 'array') { + const error = + !Array.isArray(value) || + value.length > 64 || + value.some((item) => typeof item !== 'string' || item.length > 256); + return error ? { value, error: DatasetErrEnum.arrayTagValueInvalid } : { value }; } - return { value, error: validateDatasetTagValue({ tagType, value }) }; + + if (typeof value === 'string' && value.trim() === '') { + return { value, error: DatasetErrEnum.tagValueInvalid }; + } + const numericValue = + typeof value === 'number' ? value : typeof value === 'string' ? Number(value) : NaN; + if (!Number.isFinite(numericValue)) { + return { value, error: DatasetErrEnum.tagValueInvalid }; + } + if (type === 'datetime' && Number.isNaN(new Date(numericValue).getTime())) { + return { value, error: DatasetErrEnum.tagValueDatetimeInvalid }; + } + + return { value: numericValue }; }; const isSameTagValue = (a: string | number | string[], b: string | number | string[]): boolean => { @@ -158,7 +144,7 @@ export const deduplicateTagValues = async ( for (const t of tags) { if (seen.has(t.tagId)) { if (!isSameTagValue(seen.get(t.tagId)!, t.value)) { - return Promise.reject(DatasetErrEnum.tagValueInvalid); + throw DatasetErrEnum.tagValueInvalid; } } else { seen.set(t.tagId, t.value); @@ -169,10 +155,10 @@ export const deduplicateTagValues = async ( }; /** - * 查找或创建 default_tag 承载记录：按 fromMigration 定位（不依赖标签名，改名后仍可复用）， - * 兼容存量按 DEFAULT_TAG 名称创建的记录；并发创建撞唯一索引时复用已存在记录 + * 查找或创建旧字符串标签的承载记录，只按 fromMigration 定位。 + * default_tag 只是首次创建时使用的普通名称，不参与后续身份判断。 */ -async function findOrCreateDefaultTag({ +export async function ensureDatasetTagMigrationCarrier({ datasetId, teamId, session @@ -182,11 +168,9 @@ async function findOrCreateDefaultTag({ session?: ClientSession; }) { const findDefaultTag = () => - MongoDatasetCollectionTagsV2.findOne( - { teamId, datasetId, $or: [{ fromMigration: true }, { tag: DEFAULT_TAG }] }, - undefined, - { session } - ).lean(); + MongoDatasetCollectionTagsV2.findOne({ teamId, datasetId, fromMigration: true }, undefined, { + session + }).lean(); const existing = await findDefaultTag(); if (existing) return existing; @@ -219,7 +203,7 @@ export const createOrGetCollectionTags = async ({ teamId, session }: { - tags?: (string | { tag: string; value: string | number | string[] })[]; + tags?: CollectionTagLabelType[]; datasetId: string; teamId: string; session?: ClientSession; @@ -233,29 +217,26 @@ export const createOrGetCollectionTags = async ({ ); const trimmedStringNames = stringNames.map((name) => name.trim()); - if (trimmedStringNames.some((name) => !name)) return Promise.reject(DatasetErrEnum.tagNameEmpty); - - const defaultObjectInputs = objectInputs.filter((item) => item.tag.trim() === DEFAULT_TAG); - const regularObjectInputs = objectInputs.filter((item) => item.tag.trim() !== DEFAULT_TAG); + if (trimmedStringNames.some((name) => !name)) throw DatasetErrEnum.tagNameEmpty; - const regularTagNames = regularObjectInputs.map((item) => item.tag.trim()); - if (regularTagNames.some((name) => !name)) return Promise.reject(DatasetErrEnum.tagNameEmpty); + const tagNames = objectInputs.map((item) => item.tag.trim()); + if (tagNames.some((name) => !name)) throw DatasetErrEnum.tagNameEmpty; - const regularTags = regularTagNames.length + const tagDefinitions = tagNames.length ? await MongoDatasetCollectionTagsV2.find( - { teamId, datasetId, tag: { $in: regularTagNames } }, + { teamId, datasetId, tag: { $in: tagNames } }, undefined, { session } ).lean() : []; - const regularTagMap = new Map(regularTags.map((tag) => [tag.tag, tag])); + const tagDefinitionMap = new Map(tagDefinitions.map((tag) => [tag.tag, tag])); - const normalizedRegularInputs = regularObjectInputs.map((input) => { - const tagDoc = regularTagMap.get(input.tag.trim()); + const normalizedObjectInputs = objectInputs.map((input) => { + const tagDoc = tagDefinitionMap.get(input.tag.trim()); if (!tagDoc) { - return { input, value: input.value, error: DatasetErrEnum.tagNotExist }; + return { value: input.value, error: DatasetErrEnum.tagNotExist }; } - const tagType = tagDoc.tagType || 'string'; + const tagType = tagDoc.tagType ?? 'string'; const { value, error } = validateAndNormalizeTagValue({ tagType, value: input.value }); return { tagId: String(tagDoc._id), @@ -264,26 +245,20 @@ export const createOrGetCollectionTags = async ({ }; }); - for (const { error } of normalizedRegularInputs) { - if (error) return Promise.reject(error); + for (const { error } of normalizedObjectInputs) { + if (error) throw error; } - // default_tag 承载记录：string 名与 tag=default_tag 的对象值合并为单条 array 记录 const defaultValues: string[] = [...new Set(trimmedStringNames)]; - for (const { value } of defaultObjectInputs) { - const error = validateDatasetTagValue({ tagType: 'array', value }); - if (error) return Promise.reject(error); - if (Array.isArray(value)) defaultValues.push(...value); - } const result: CollectionTagValueType[] = []; if (defaultValues.length > 0) { - const defaultTag = await findOrCreateDefaultTag({ datasetId, teamId, session }); + const defaultTag = await ensureDatasetTagMigrationCarrier({ datasetId, teamId, session }); result.push({ tagId: String(defaultTag._id), value: [...new Set(defaultValues)] }); } - result.push(...normalizedRegularInputs.map(({ tagId, value }) => ({ tagId: tagId!, value }))); + result.push(...normalizedObjectInputs.map(({ tagId, value }) => ({ tagId: tagId!, value }))); return deduplicateTagValues(result); }; @@ -301,7 +276,7 @@ export const collectionTagsToTagLabel = async ({ }: { datasetId: string; tags?: (string | CollectionTagValueType)[]; -}): Promise<(string | { tag: string; value: string | number | string[] })[] | undefined> => { +}): Promise => { if (!tags) return undefined; if (tags.length === 0) return []; @@ -322,9 +297,7 @@ export const collectionTagsToTagLabel = async ({ const tagName = tagsMap.get(tag.tagId); return tagName ? { tag: tagName, value: tag.value } : null; }) - .filter( - (item): item is string | { tag: string; value: string | number | string[] } => item !== null - ); + .filter((item): item is CollectionTagLabelType => item !== null); }; export const syncCollection = async (collection: CollectionWithDatasetType) => { diff --git a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts index cd8d2b233e61..3ddbcea603a9 100644 --- a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts +++ b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts @@ -2,10 +2,10 @@ import json5 from 'json5'; import safeRegex from 'safe-regex'; import { MongoDatasetCollection } from '../../collection/schema'; import { MongoDatasetCollectionTagsV2 } from '../../tag/schemaV2'; -import { DEFAULT_TAG } from '@fastgpt/global/core/dataset/type'; import { isCollectionTagValue } from '@fastgpt/global/core/dataset/tagUtils'; import { readFromSecondary } from '../../../../common/mongo/utils'; -import { computeFilterIntersection } from '../utils'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { applySharedCollectionMetadataFilters } from './collectionFilterShared'; /* ========== New format key-value tag filtering types ========== */ @@ -182,54 +182,34 @@ export async function filterCollectionByKeyValueTags({ }): Promise { const allConditions = [...$and, ...$or]; const tagNames = new Set(); - let hasDefaultTag = false; for (const cond of allConditions) { const tagName = Object.keys(cond)[0]; if (!tagName) continue; - if (tagName === DEFAULT_TAG) hasDefaultTag = true; - else tagNames.add(tagName); + tagNames.add(tagName); } - if (tagNames.size === 0 && !hasDefaultTag) return undefined; + if (tagNames.size === 0) return undefined; - // 普通标签按名称查询；default_tag 承载记录按 fromMigration 定位，不依赖标签名（改名后旧格式过滤仍命中） - const [regularTagDocs, defaultTagDocs] = await Promise.all([ - tagNames.size - ? MongoDatasetCollectionTagsV2.find( - { - teamId, - datasetId: { $in: datasetIds }, - tag: { $in: Array.from(tagNames) } - }, - '_id datasetId tag tagType', - { ...readFromSecondary } - ).lean() - : [], - hasDefaultTag - ? MongoDatasetCollectionTagsV2.find( - { teamId, datasetId: { $in: datasetIds }, fromMigration: true }, - '_id datasetId tag tagType', - { ...readFromSecondary } - ).lean() - : [] - ]); + const tagDocs = await MongoDatasetCollectionTagsV2.find( + { + teamId, + datasetId: { $in: datasetIds }, + tag: { $in: [...tagNames] } + }, + '_id datasetId tag tagType', + { ...readFromSecondary } + ).lean(); const datasetTagMap = new Map>(); const addToMap = (dsId: string, tagName: string, id: string, type: string) => { - if (!datasetTagMap.has(dsId)) datasetTagMap.set(dsId, new Map()); - datasetTagMap.get(dsId)!.set(tagName, { id, type }); - }; - for (const doc of regularTagDocs) { - addToMap(String(doc.datasetId), doc.tag, String(doc._id), doc.tagType || 'string'); - } - // default_tag 记录同时挂 DEFAULT_TAG 键与实际标签名键；每 dataset 的 DEFAULT_TAG 键只取一条避免歧义 - for (const doc of defaultTagDocs) { - const dsId = String(doc.datasetId); - const id = String(doc._id); - const type = doc.tagType || 'string'; - addToMap(dsId, doc.tag, id, type); - if (!datasetTagMap.get(dsId)?.has(DEFAULT_TAG)) { - addToMap(dsId, DEFAULT_TAG, id, type); + const tagMap = datasetTagMap.get(dsId); + if (tagMap) { + tagMap.set(tagName, { id, type }); + return; } + datasetTagMap.set(dsId, new Map([[tagName, { id, type }]])); + }; + for (const doc of tagDocs) { + addToMap(String(doc.datasetId), doc.tag, String(doc._id), doc.tagType ?? 'string'); } if (datasetTagMap.size === 0) return []; @@ -255,10 +235,10 @@ export async function filterCollectionByKeyValueTags({ // 4. Iterate each dataset (the same tag name may map to different tagIds per dataset) for (const [dsId, tagMap] of datasetTagMap) { - const andTagIds = ($and || []) + const andTagIds = $and .map((cond) => tagMap.get(Object.keys(cond)[0])?.id) .filter((id): id is string => Boolean(id)); - const orTagIds = ($or || []) + const orTagIds = $or .map((cond) => tagMap.get(Object.keys(cond)[0])?.id) .filter((id): id is string => Boolean(id)); @@ -285,14 +265,14 @@ export async function filterCollectionByKeyValueTags({ // 5. Application-layer value comparison for (const col of collections) { - const tagsArr = (col.tags || []).filter(isCollectionTagValue); + const tagsArr = (col.tags ?? []).filter(isCollectionTagValue); // AND: all must pass - const andOk = ($and || []).every((cond) => matchCondition(cond, tagMap, tagsArr)); + const andOk = $and.every((cond) => matchCondition(cond, tagMap, tagsArr)); if (!andOk) continue; // OR: at least one must pass - if ($or?.length) { + if ($or.length > 0) { const orOk = $or.some((cond) => matchCondition(cond, tagMap, tagsArr)); if (!orOk) continue; } @@ -323,12 +303,7 @@ export const getForbidCollectionIdList = async ({ return collections.map((item) => String(item._id)); }; -/** - * 按知识库集合元数据过滤 collectionId。 - * - * 标签过滤保持原有语义：`$and` 优先生效，且 `$and` 中字符串标签和 null 不能共存。 - * 输入 collectionIds 可以是文件夹，会递归展开为实际文件集合。 - */ +/** 新版知识库检索节点元数据过滤，只接受结构化标签条件。 */ export const filterCollectionByMetadata = async ({ teamId, datasetIds, @@ -338,141 +313,46 @@ export const filterCollectionByMetadata = async ({ datasetIds: string[]; collectionFilterMatch?: string; }): Promise => { - const getAllCollectionIds = async ({ - parentCollectionIds - }: { - parentCollectionIds?: string[]; - }): Promise => { - if (!parentCollectionIds) return; - if (parentCollectionIds.length === 0) { - return []; - } - - const collections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - _id: { $in: parentCollectionIds } - }, - '_id type', - { - ...readFromSecondary - } - ).lean(); - - const resultIds = new Set(); - collections.forEach((item) => { - if (item.type !== 'folder') { - resultIds.add(String(item._id)); - } - }); - - const folderIds = collections - .filter((item) => item.type === 'folder') - .map((item) => String(item._id)); - - // Get all child collection ids - if (folderIds.length) { - const childCollections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - parentId: { $in: folderIds } - }, - '_id type', - { - ...readFromSecondary - } - ).lean(); - - const childIds = await getAllCollectionIds({ - parentCollectionIds: childCollections.map((item) => String(item._id)) - }); - - childIds?.forEach((id) => resultIds.add(id)); - } - - return Array.from(resultIds); - }; - if (!collectionFilterMatch || !global.feConfigs.isPlus) return; - let tagCollectionIdList: string[] | undefined = undefined; - let createTimeCollectionIdList: string[] | undefined = undefined; - let inputCollectionIdList: string[] | undefined = undefined; - - try { - const jsonMatch = json5.parse(collectionFilterMatch); - - const andTagsRaw = jsonMatch?.tags?.$and as unknown[] | undefined; - const orTagsRaw = jsonMatch?.tags?.$or as unknown[] | undefined; - - const isConditionObject = (item: unknown): item is TagCondition => - typeof item === 'object' && !Array.isArray(item) && item !== null; - const rewriteLegacyTags = (items: unknown[] | undefined): TagCondition[] => - (items || []).map((item) => { - if (isConditionObject(item)) return item; - if (item === null) return { [DEFAULT_TAG]: { $empty: true } }; - return { [DEFAULT_TAG]: { $contains: String(item) } }; - }); - const hasLegacyMixedNull = (items: unknown[] | undefined) => - Boolean( - items?.some((item) => item === null) && items.some((item) => typeof item === 'string') - ); - - if (hasLegacyMixedNull(andTagsRaw) || hasLegacyMixedNull(orTagsRaw)) return []; - - const rewrittenAnd = rewriteLegacyTags(andTagsRaw); - const rewrittenOr = rewriteLegacyTags(orTagsRaw); - if (rewrittenAnd.length > 0 || rewrittenOr.length > 0) { - tagCollectionIdList = await filterCollectionByKeyValueTags({ - $and: rewrittenAnd, - $or: rewrittenOr, - teamId, - datasetIds - }); - } - - // time - const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined; - const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined; - if (getCreateTime || lteCreateTime) { - const collections = await MongoDatasetCollection.find( - { + const metadataMatch = json5.parse(collectionFilterMatch) as { + tags?: { $and?: unknown[]; $or?: unknown[] }; + createTime?: { $gte?: string; $lte?: string }; + collectionIds?: string[]; + }; + const isConditionObject = (item: unknown): item is TagCondition => { + if (typeof item !== 'object' || Array.isArray(item) || item === null) return false; + const tagNames = Object.keys(item); + if (tagNames.length !== 1 || !tagNames[0]) return false; + const operation = Reflect.get(item, tagNames[0]); + return ( + typeof operation === 'object' && + !Array.isArray(operation) && + operation !== null && + Object.keys(operation).length === 1 + ); + }; + const parseConditions = (items?: unknown[]): TagCondition[] => { + if (!items) return []; + if (!items.every(isConditionObject)) throw CommonErrEnum.invalidParams; + return items; + }; + const andTags = parseConditions(metadataMatch.tags?.$and); + const orTags = parseConditions(metadataMatch.tags?.$or); + const tagCollectionIds = + andTags.length > 0 || orTags.length > 0 + ? await filterCollectionByKeyValueTags({ + $and: andTags, + $or: orTags, teamId, - datasetId: { $in: datasetIds }, - createTime: { - ...(getCreateTime && { $gte: new Date(getCreateTime) }), - ...(lteCreateTime && { - $lte: new Date(lteCreateTime) - }) - } - }, - '_id' - ); - createTimeCollectionIdList = collections.map((item) => String(item._id)); - } - - // collectionIds - const inputCollectionIds = jsonMatch?.collectionIds as string[] | undefined; - if (Array.isArray(inputCollectionIds) && inputCollectionIds.length > 0) { - inputCollectionIdList = await getAllCollectionIds({ - parentCollectionIds: inputCollectionIds - }); - if (inputCollectionIdList && inputCollectionIdList.length === 0) { - return []; - } - } - - // Concat tag, time and collectionIds - const collectionIds = computeFilterIntersection([ - tagCollectionIdList, - createTimeCollectionIdList, - inputCollectionIdList - ]); - - return await getAllCollectionIds({ - parentCollectionIds: collectionIds - }); - } catch {} + datasetIds + }) + : undefined; + + return applySharedCollectionMetadataFilters({ + teamId, + datasetIds, + metadataMatch, + tagCollectionIds + }); }; diff --git a/packages/service/core/dataset/search/defaultRecall/collectionFilterShared.ts b/packages/service/core/dataset/search/defaultRecall/collectionFilterShared.ts new file mode 100644 index 000000000000..a2b972713cb7 --- /dev/null +++ b/packages/service/core/dataset/search/defaultRecall/collectionFilterShared.ts @@ -0,0 +1,113 @@ +import { MongoDatasetCollection } from '../../collection/schema'; +import { readFromSecondary } from '../../../../common/mongo/utils'; +import { computeFilterIntersection } from '../utils'; + +type CollectionMetadataMatch = { + createTime?: { $gte?: string; $lte?: string }; + collectionIds?: string[]; +}; + +/** + * 将文件或文件夹 ID 展开为真正可召回的非文件夹 Collection ID。 + * 查询始终受 teamId + datasetIds 限制，防止跨知识库展开。 + */ +export const expandCollectionIds = async ({ + teamId, + datasetIds, + parentCollectionIds +}: { + teamId: string; + datasetIds: string[]; + parentCollectionIds?: string[]; +}): Promise => { + if (!parentCollectionIds) return; + if (parentCollectionIds.length === 0) return []; + + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + _id: { $in: parentCollectionIds } + }, + '_id type', + { ...readFromSecondary } + ).lean(); + + const resultIds = new Set( + collections.filter((item) => item.type !== 'folder').map((item) => String(item._id)) + ); + const folderIds = collections + .filter((item) => item.type === 'folder') + .map((item) => String(item._id)); + + if (folderIds.length > 0) { + const children = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + parentId: { $in: folderIds } + }, + '_id', + { ...readFromSecondary } + ).lean(); + const childIds = await expandCollectionIds({ + teamId, + datasetIds, + parentCollectionIds: children.map((item) => String(item._id)) + }); + childIds?.forEach((id) => resultIds.add(id)); + } + + return [...resultIds]; +}; + +/** 将标签结果与时间、Collection ID 条件求交，新旧标签链路共用。 */ +export const applySharedCollectionMetadataFilters = async ({ + teamId, + datasetIds, + metadataMatch, + tagCollectionIds +}: { + teamId: string; + datasetIds: string[]; + metadataMatch: CollectionMetadataMatch; + tagCollectionIds?: string[]; +}): Promise => { + const getCreateTime = metadataMatch.createTime?.$gte; + const lteCreateTime = metadataMatch.createTime?.$lte; + const createTimeCollectionIds = + getCreateTime || lteCreateTime + ? ( + await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + createTime: { + ...(getCreateTime ? { $gte: new Date(getCreateTime) } : {}), + ...(lteCreateTime ? { $lte: new Date(lteCreateTime) } : {}) + } + }, + '_id', + { ...readFromSecondary } + ) + ).map((item) => String(item._id)) + : undefined; + + const inputCollectionIds = + Array.isArray(metadataMatch.collectionIds) && metadataMatch.collectionIds.length > 0 + ? await expandCollectionIds({ + teamId, + datasetIds, + parentCollectionIds: metadataMatch.collectionIds + }) + : undefined; + if (inputCollectionIds?.length === 0) return []; + + const collectionIds = computeFilterIntersection([ + tagCollectionIds, + createTimeCollectionIds, + inputCollectionIds + ]); + + return expandCollectionIds({ teamId, datasetIds, parentCollectionIds: collectionIds }); +}; diff --git a/packages/service/core/dataset/search/defaultRecall/index.ts b/packages/service/core/dataset/search/defaultRecall/index.ts index a7b666e9b7b1..67216b040535 100644 --- a/packages/service/core/dataset/search/defaultRecall/index.ts +++ b/packages/service/core/dataset/search/defaultRecall/index.ts @@ -49,7 +49,8 @@ export async function searchDatasetData( rerankModel, rerankWeight = 0.5, datasetIds = [], - collectionFilterMatch + collectionFilterMatch, + collectionFilterMode } = props; const searchMode = DatasetSearchModeMap[inputSearchMode] @@ -98,6 +99,7 @@ export async function searchDatasetData( model, imageQueries, collectionFilterMatch, + collectionFilterMode, embeddingLimit, fullTextLimit, textQueries, diff --git a/packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts b/packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts new file mode 100644 index 000000000000..62d971936a03 --- /dev/null +++ b/packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts @@ -0,0 +1,117 @@ +import json5 from 'json5'; +import { MongoDatasetCollection } from '../../../collection/schema'; +import { MongoDatasetCollectionTagsV2 } from '../../../tag/schemaV2'; +import { readFromSecondary } from '../../../../../common/mongo/utils'; +import { applySharedCollectionMetadataFilters } from '../collectionFilterShared'; +import type { LegacyCollectionFilterMatch } from './type'; + +/** + * 按 dataset 读取迁移承载标签。每个知识库独立解析 tagId，不依赖可修改的标签名。 + */ +const getMigrationTagIds = async ({ + teamId, + datasetIds +}: { + teamId: string; + datasetIds: string[]; +}) => + MongoDatasetCollectionTagsV2.find( + { teamId, datasetId: { $in: datasetIds }, fromMigration: true }, + '_id datasetId', + { ...readFromSecondary } + ).lean(); + +/** + * 存量知识库检索节点的标签过滤。 + * `$and` 存在时完全忽略 `$or`；null 只匹配 tags 真正为空或不存在的 Collection。 + */ +const filterLegacyCollectionByTags = async ({ + teamId, + datasetIds, + tags +}: { + teamId: string; + datasetIds: string[]; + tags?: LegacyCollectionFilterMatch['tags']; +}): Promise => { + const andTags = tags?.$and; + const orTags = tags?.$or; + const activeTags = andTags?.length ? andTags : orTags; + if (!activeTags?.length) return; + + const hasNull = activeTags.includes(null); + const stringTags = [ + ...new Set(activeTags.filter((tag): tag is string => typeof tag === 'string')) + ]; + if (andTags?.length && hasNull && stringTags.length > 0) return []; + + if (andTags?.length && hasNull) { + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + $or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }] + }, + '_id', + { ...readFromSecondary } + ).lean(); + return collections.map((item) => String(item._id)); + } + + const migrationTags = stringTags.length ? await getMigrationTagIds({ teamId, datasetIds }) : []; + const queries = migrationTags.map((migrationTag) => ({ + teamId, + datasetId: migrationTag.datasetId, + tags: { + $elemMatch: { + tagId: String(migrationTag._id), + value: andTags?.length ? { $all: stringTags } : { $in: stringTags } + } + } + })); + + const match = { + teamId, + datasetId: { $in: datasetIds }, + $or: [ + ...queries, + ...(!andTags?.length && hasNull ? [{ tags: { $size: 0 } }, { tags: { $exists: false } }] : []) + ] + }; + if (match.$or.length === 0) return []; + + const collections = await MongoDatasetCollection.find(match, '_id', { + ...readFromSecondary + }).lean(); + return collections.map((item) => String(item._id)); +}; + +/** 旧节点完整元数据过滤入口；旧配置解析失败时保持历史降级行为。 */ +export const filterLegacyCollectionByMetadata = async ({ + teamId, + datasetIds, + collectionFilterMatch +}: { + teamId: string; + datasetIds: string[]; + collectionFilterMatch?: string; +}): Promise => { + if (!collectionFilterMatch || !global.feConfigs.isPlus) return; + + try { + const metadataMatch = json5.parse(collectionFilterMatch) as LegacyCollectionFilterMatch; + const tagCollectionIds = await filterLegacyCollectionByTags({ + teamId, + datasetIds, + tags: metadataMatch.tags + }); + return applySharedCollectionMetadataFilters({ + teamId, + datasetIds, + metadataMatch, + tagCollectionIds + }); + } catch { + return; + } +}; diff --git a/packages/service/core/dataset/search/defaultRecall/legacy/type.ts b/packages/service/core/dataset/search/defaultRecall/legacy/type.ts new file mode 100644 index 000000000000..c000a0c48265 --- /dev/null +++ b/packages/service/core/dataset/search/defaultRecall/legacy/type.ts @@ -0,0 +1,8 @@ +export type LegacyCollectionFilterMatch = { + tags?: { + $and?: Array; + $or?: Array; + }; + createTime?: { $gte?: string; $lte?: string }; + collectionIds?: string[]; +}; diff --git a/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts b/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts index a930ff753a2d..3392bca9bd0d 100644 --- a/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts +++ b/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts @@ -1,7 +1,10 @@ import { getForbidCollectionIdList, filterCollectionByMetadata } from './collectionFilter'; +import { filterLegacyCollectionByMetadata } from './legacy/collectionFilter'; import { embeddingRecall } from './embeddingRecall'; import { fullTextRecall } from './fullTextRecall'; import type { EmbeddingSystemModelDataType } from '@fastgpt/global/core/ai/model.schema'; +import type { CollectionFilterMode } from '../type'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; /** * 默认召回的并行调度层。 @@ -14,6 +17,7 @@ export const multiQueryRecall = async ({ model, imageQueries, collectionFilterMatch, + collectionFilterMode = DatasetTagFilterVersionEnum.structured, embeddingLimit, fullTextLimit, textQueries, @@ -24,6 +28,7 @@ export const multiQueryRecall = async ({ model: EmbeddingSystemModelDataType; imageQueries: string[]; collectionFilterMatch?: string; + collectionFilterMode?: CollectionFilterMode; embeddingLimit: number; fullTextLimit: number; textQueries: string[]; @@ -34,11 +39,9 @@ export const multiQueryRecall = async ({ teamId, datasetIds }), - filterCollectionByMetadata({ - teamId, - datasetIds, - collectionFilterMatch - }) + collectionFilterMode === DatasetTagFilterVersionEnum.legacy + ? filterLegacyCollectionByMetadata({ teamId, datasetIds, collectionFilterMatch }) + : filterCollectionByMetadata({ teamId, datasetIds, collectionFilterMatch }) ]); const [ diff --git a/packages/service/core/dataset/search/type.ts b/packages/service/core/dataset/search/type.ts index 727ef1c7296a..00941a8afbf3 100644 --- a/packages/service/core/dataset/search/type.ts +++ b/packages/service/core/dataset/search/type.ts @@ -8,6 +8,9 @@ import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/ty import type { ChatItemMiniType } from '@fastgpt/global/core/chat/type'; import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type'; +import type { DatasetTagFilterVersion } from '@fastgpt/global/core/dataset/workflowTagFilter'; + +export type CollectionFilterMode = DatasetTagFilterVersion; export type SearchDatasetDataProps = { histories: ChatItemMiniType[]; @@ -47,6 +50,8 @@ export type SearchDatasetDataProps = { } */ collectionFilterMatch?: string; + /** 由节点 Dispatcher 明确指定，不根据过滤值形状推断。 */ + collectionFilterMode?: CollectionFilterMode; }; export type SearchDatasetDataResponse = { diff --git a/packages/service/core/dataset/tag/schemaV2.ts b/packages/service/core/dataset/tag/schemaV2.ts index e025626da985..e60c6c8c9670 100644 --- a/packages/service/core/dataset/tag/schemaV2.ts +++ b/packages/service/core/dataset/tag/schemaV2.ts @@ -5,7 +5,7 @@ import { type DatasetCollectionTagsSchemaType } from '@fastgpt/global/core/datas import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; const { Schema } = connectionMongo; -export const DatasetCollectionTagsV2Name = 'dataset_collection_tags_v2'; +const DatasetCollectionTagsV2Name = 'dataset_collection_tags_v2'; const DatasetCollectionTagsV2Schema = new Schema({ teamId: { @@ -21,7 +21,7 @@ const DatasetCollectionTagsV2Schema = new Schema({ tag: { type: String, required: true - // COMMENT: 标签名称。default_tag 承载记录由 fromMigration 标识，名称可改 + // COMMENT: 标签名称。迁移承载记录的身份只由 fromMigration 标识 }, tagType: { type: String, diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts index 7ae8318c5485..e21fd71b9da4 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts @@ -23,6 +23,7 @@ import { getLogger, LogCategories } from '../../../../../../../common/logger'; import type { DispatchSubAppResponse } from '../../type'; import type { AppFormEditFormType } from '@fastgpt/global/core/app/formEdit/type'; import { DatasetSearchToolSchema } from './utils'; +import { formatCollectionFilterMatchParam } from '@fastgpt/global/core/dataset/workflowTagFilter'; import { parseJsonArgs } from '../../../../../../ai/utils'; import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; @@ -34,6 +35,7 @@ import { import { filterDatasetsByTmbId } from '../../../../../../dataset/utils'; import { normalizeDatasetSearchInput } from '../../../../dataset/utils'; import type { LLMSystemModelDataType } from '@fastgpt/global/core/ai/model.schema'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; const logger = getLogger(LogCategories.MODULE.AI.AGENT); type DatasetSearchParams = { @@ -263,7 +265,10 @@ export const dispatchAgentDatasetSearch = async ({ datasetSearchUsingExtensionQuery: datasetParams.datasetSearchUsingExtensionQuery ?? false, datasetSearchExtensionModel: extensionModelData, datasetSearchExtensionBg: datasetParams.datasetSearchExtensionBg, - collectionFilterMatch: datasetParams.collectionFilterMatch, + collectionFilterMatch: formatCollectionFilterMatchParam({ + value: datasetParams.collectionFilterMatch + }), + collectionFilterMode: DatasetTagFilterVersionEnum.structured, userKey }; const { diff --git a/packages/service/core/workflow/dispatch/dataset/search.ts b/packages/service/core/workflow/dispatch/dataset/search.ts index b63212599ce7..ffa01def1555 100644 --- a/packages/service/core/workflow/dispatch/dataset/search.ts +++ b/packages/service/core/workflow/dispatch/dataset/search.ts @@ -25,6 +25,11 @@ import { createQueryExtensionChildNodeResponse } from './nodeResponse'; import { normalizeDatasetSearchInput } from './utils'; +import type { CollectionFilterMode } from '../../../dataset/search/type'; +import { + resolveDatasetTagFilterVersion, + type DatasetTagFilterVersion +} from '@fastgpt/global/core/dataset/workflowTagFilter'; const logger = getLogger(LogCategories.MODULE.WORKFLOW.DATASET); @@ -43,6 +48,7 @@ type DatasetSearchProps = ModuleDispatchProps<{ [NodeInputKeyEnum.datasetSearchRerankWeight]?: number; [NodeInputKeyEnum.collectionFilterMatch]: string; + [NodeInputKeyEnum.collectionFilterVersion]?: DatasetTagFilterVersion; [NodeInputKeyEnum.authTmbId]?: boolean; [NodeInputKeyEnum.datasetSearchUsingExtensionQuery]: boolean; @@ -60,9 +66,10 @@ export type DatasetSearchResponse = DispatchNodeResultType<{ [NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[]; }>; -export async function dispatchDatasetSearch( +/** 根据节点保存的显式版本选择标签过滤语义；缺少版本的存量节点固定走 legacy。 */ +export const dispatchDatasetSearch = async ( props: DatasetSearchProps -): Promise { +): Promise => { const { runningAppInfo: { teamId }, runningUserInfo: { tmbId }, @@ -77,6 +84,7 @@ export async function dispatchDatasetSearch( datasetSearchInput = [], authTmbId = false, collectionFilterMatch, + collectionFilterVersion, searchMode, embeddingWeight, usingReRank, @@ -101,6 +109,11 @@ export async function dispatchDatasetSearch( return Promise.reject(i18nT('chat:dataset_quote_type error')); } + const collectionFilterMode: CollectionFilterMode = resolveDatasetTagFilterVersion({ + version: collectionFilterVersion, + filterValue: collectionFilterMatch + }); + if (datasets.length === 0) { return getNodeErrResponse({ error: i18nT('common:core.chat.error.Select dataset empty') }); } @@ -180,7 +193,8 @@ export async function dispatchDatasetSearch( usingReRank, rerankModel: rerankModelData, rerankWeight, - collectionFilterMatch + collectionFilterMatch, + collectionFilterMode }; const useDeepSearch = datasetDeepSearch && textQueries.length > 0; const { @@ -361,4 +375,4 @@ export async function dispatchDatasetSearch( logger.error('Dataset search dispatch failed', { error }); return getNodeErrResponse({ error }); } -} +}; diff --git a/packages/service/core/workflow/dispatch/utils/runtime.ts b/packages/service/core/workflow/dispatch/utils/runtime.ts index 63a583e0efaa..bf6958436307 100644 --- a/packages/service/core/workflow/dispatch/utils/runtime.ts +++ b/packages/service/core/workflow/dispatch/utils/runtime.ts @@ -4,12 +4,14 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; +import type { ReferenceValueType } from '@fastgpt/global/core/workflow/type/io'; import type { WorkflowVariableStateLike } from '../../types/runtime'; import { getReferenceVariableValue, valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils'; import { nodeInputIsReference } from '@fastgpt/global/core/workflow/utils'; +import { formatCollectionFilterMatchParam } from '@fastgpt/global/core/dataset/workflowTagFilter'; import { replaceEditorVariable } from './replaceEditorVariable'; /** @@ -97,6 +99,46 @@ export const getWorkflowNodeRunParams = ({ } } + if ( + input.key === NodeInputKeyEnum.datasetParams && + value && + typeof value === 'object' && + !Array.isArray(value) + ) { + const datasetParams = value as Record; + value = { + ...datasetParams, + collectionFilterMatch: formatCollectionFilterMatchParam({ + value: datasetParams.collectionFilterMatch, + resolveReference: (refValue) => + getReferenceVariableValue({ + value: refValue as ReferenceValueType, + nodesMap: runtimeNodesMap, + variables: getRuntimeVariables(), + isReferenceVal: true + }) + }) + }; + } + + if (input.key === NodeInputKeyEnum.collectionFilterMatch) { + const formatted = formatCollectionFilterMatchParam({ + value, + resolveReference: (refValue) => + getReferenceVariableValue({ + value: refValue as ReferenceValueType, + nodesMap: runtimeNodesMap, + variables: getRuntimeVariables(), + isReferenceVal: true + }) + }); + if (input.canEdit && dynamicInput && params[dynamicInput.key]) { + params[dynamicInput.key][input.key] = formatted; + } + params[input.key] = formatted; + return; + } + // Dynamic input is stored in the dynamic key if (input.canEdit && dynamicInput && params[dynamicInput.key]) { params[dynamicInput.key][input.key] = valueTypeFormat(value, input.valueType); diff --git a/packages/service/test/core/dataset/collection/tagFilter.test.ts b/packages/service/test/core/dataset/collection/tagFilter.test.ts new file mode 100644 index 000000000000..89d830ba7e25 --- /dev/null +++ b/packages/service/test/core/dataset/collection/tagFilter.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { buildCollectionListTagMatch } from '@fastgpt/service/core/dataset/collection/tagFilter'; + +describe('buildCollectionListTagMatch', () => { + it('returns empty object when no filters are selected', () => { + expect(buildCollectionListTagMatch()).toEqual({}); + expect(buildCollectionListTagMatch([])).toEqual({}); + }); + + it('uses $elemMatch for one tag and $and across tags', () => { + expect(buildCollectionListTagMatch([{ tagId: 'type', values: ['PRD'] }])).toEqual({ + tags: { + $elemMatch: { + tagId: 'type', + value: { $in: ['PRD'] } + } + } + }); + + expect( + buildCollectionListTagMatch([ + { tagId: 'type', values: ['PRD'] }, + { tagId: 'version', values: [2] } + ]) + ).toEqual({ + $and: [ + { + tags: { + $elemMatch: { + tagId: 'type', + value: { $in: ['PRD'] } + } + } + }, + { + tags: { + $elemMatch: { + tagId: 'version', + value: { $in: [2] } + } + } + } + ] + }); + }); +}); diff --git a/packages/service/test/core/dataset/delete/processor.test.ts b/packages/service/test/core/dataset/delete/processor.test.ts index d08a92907123..e1b7ea85bb84 100644 --- a/packages/service/test/core/dataset/delete/processor.test.ts +++ b/packages/service/test/core/dataset/delete/processor.test.ts @@ -246,30 +246,4 @@ describe('datasetDeleteProcessor', () => { MongoDatasetSynonymMapping.countDocuments({ datasetId: childDataset._id }) ).resolves.toBe(0); }); - - it('deletes a dataset without v2 tags', async () => { - const user = await getUser('dataset-delete-no-v2-tags'); - const dataset = await MongoDataset.create({ - teamId: user.teamId, - tmbId: user.tmbId, - name: 'dataset without v2 tags', - type: DatasetTypeEnum.dataset, - deleteTime: new Date() - }); - - await datasetDeleteProcessor({ - data: { - teamId: user.teamId, - datasetId: String(dataset._id) - } - } as never); - - expect(await MongoDataset.countDocuments({ _id: dataset._id })).toBe(0); - expect( - await MongoDatasetCollectionTagsV2.countDocuments({ - teamId: user.teamId, - datasetId: dataset._id - }) - ).toBe(0); - }); }); diff --git a/packages/service/test/core/dataset/search/collectionFilter.benchmark.ts b/packages/service/test/core/dataset/search/collectionFilter.benchmark.ts deleted file mode 100644 index 524c58ab9058..000000000000 --- a/packages/service/test/core/dataset/search/collectionFilter.benchmark.ts +++ /dev/null @@ -1,383 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -const mockMongoDatasetCollectionFind = vi.hoisted(() => vi.fn()); -const mockMongoDatasetCollectionTagsFind = vi.hoisted(() => vi.fn()); - -vi.mock('@fastgpt/service/core/dataset/collection/schema', () => ({ - MongoDatasetCollection: { - find: mockMongoDatasetCollectionFind - } -})); - -vi.mock('@fastgpt/service/core/dataset/tag/schemaV2', () => ({ - MongoDatasetCollectionTagsV2: { - find: mockMongoDatasetCollectionTagsFind - } -})); - -import { - filterCollectionByKeyValueTags, - filterCollectionByMetadata -} from '../../../../core/dataset/search/defaultRecall/collectionFilter'; - -/** - * 标签过滤性能基准测试 - * - * 1. 新tag横向对比（1000 collections）： - * - 1 个 dataset vs 10 个 dataset - * - 1 个 filter vs 10 个 filter - * 2. 新tag vs 旧tag 纵向对比（1000 collections）： - * - 新格式（key-value，JS 端过滤） vs 旧格式（字符串数组，mongo $all 过滤） - * - * 运行方式：pnpm test:benchmark（vitest.benchmark.config.ts 的 include 命中 test 目录下 - * 所有 *.benchmark.ts 文件） - */ - -const COLLECTION_COUNT = 1000; - -type CollectionItem = { _id: string; tags?: unknown }; - -/* ========== mock 辅助：按查询中的 datasetId 返回对应数据集的集合 ========== */ - -function collectionsForQuery( - query: { datasetId?: string | { $in?: string[] } } | undefined, - collectionsByDataset: Record -): CollectionItem[] { - const ds = query?.datasetId; - if (typeof ds === 'string') return collectionsByDataset[ds] ?? []; - if (ds && Array.isArray(ds.$in)) { - return ds.$in.flatMap((id) => collectionsByDataset[id] ?? []); - } - return []; -} - -function setupMock({ - tagDocs, - collectionsByDataset -}: { - tagDocs: unknown[]; - collectionsByDataset: Record; -}) { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue(tagDocs) - }); - mockMongoDatasetCollectionFind.mockImplementation((query: any) => { - const data = collectionsForQuery(query, collectionsByDataset); - return { - hint: () => ({ lean: vi.fn().mockResolvedValue(data) }), - lean: vi.fn().mockResolvedValue(data) - }; - }); -} - -/* ========== 计时辅助 ========== */ - -interface MeasureResult { - avg: number; - min: number; - max: number; - median: number; - times: number[]; - /** 单次调用的真实 CPU 消耗（user+system，毫秒） */ - cpuPerCallMs: number; - cpuTotalMs: number; - /** 单次调用的堆峰值增长（KB） */ - peakHeapKB: number; - /** 单次调用正堆增长的平均（KB） */ - avgPositiveHeapKB: number; -} - -async function measure( - name: string, - fn: () => unknown | Promise, - iterations = 200, - warmups = 20 -): Promise { - // 预热避开 JIT / 首次模块加载抖动 - for (let i = 0; i < warmups; i++) await fn(); - - const cpuBefore = process.cpuUsage(); - const times: number[] = []; - const heapDeltas: number[] = []; - for (let i = 0; i < iterations; i++) { - const heapBefore = process.memoryUsage().heapUsed; - const start = performance.now(); - await fn(); - times.push(performance.now() - start); - heapDeltas.push(process.memoryUsage().heapUsed - heapBefore); - } - const cpuDelta = process.cpuUsage(cpuBefore); // { user, system } 微秒 - - const avg = times.reduce((a, b) => a + b, 0) / times.length; - const min = Math.min(...times); - const max = Math.max(...times); - const sorted = [...times].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; - - const cpuTotalMs = (cpuDelta.user + cpuDelta.system) / 1000; - const cpuPerCallMs = cpuTotalMs / iterations; - const positiveHeaps = heapDeltas.filter((d) => d > 0); - const peakHeapKB = Math.max(...heapDeltas, 0) / 1024; - const avgPositiveHeapKB = - positiveHeaps.length > 0 - ? positiveHeaps.reduce((a, b) => a + b, 0) / positiveHeaps.length / 1024 - : 0; - - console.log(` [${name}]`); - console.log( - ` 平均耗时: ${avg.toFixed(3)}ms 最小: ${min.toFixed(3)}ms 最大: ${max.toFixed(3)}ms 中位数: ${median.toFixed(3)}ms` - ); - console.log( - ` CPU: 平均 ${cpuPerCallMs.toFixed(3)}ms/次 (user+system，${iterations} 次总计 ${cpuTotalMs.toFixed( - 1 - )}ms)` - ); - console.log( - ` 内存: 单次堆增长峰值 ${peakHeapKB.toFixed(1)}KB | 正增长平均 ${avgPositiveHeapKB.toFixed( - 1 - )}KB (heapUsed，受 GC 影响)` - ); - return { - avg, - min, - max, - median, - times, - cpuPerCallMs, - cpuTotalMs, - peakHeapKB, - avgPositiveHeapKB - }; -} - -/* ========== 数据生成 ========== */ - -// 每个 dataset 定义一个 number 类型的 key-value 标签 -const tagDocsForDataset = (count: number) => - Array.from({ length: count }, (_, i) => ({ - _id: `tag-${i + 1}`, - datasetId: `ds-${i + 1}`, - tag: 'version', - tagType: 'number' - })); - -// 共 COLLECTION_COUNT 个 collection，均分到 count 个 dataset，value 覆盖 0..999 -const collectionsByDatasetFor = (count: number): Record => { - const result: Record = {}; - const perDataset = COLLECTION_COUNT / count; - for (let d = 1; d <= count; d++) { - result[`ds-${d}`] = Array.from({ length: perDataset }, (_, i) => ({ - _id: `col-${d}-${i}`, - tags: [{ tagId: `tag-${d}`, value: (d - 1) * perDataset + i }] - })); - } - return result; -}; - -/* ========== 新tag横向对比 ========== */ - -describe('新tag横向对比（1000 collections）', () => { - it('1 个 dataset vs 10 个 dataset', async () => { - const condition = [{ version: { $gte: 500 } }]; - const datasetIds10 = Array.from({ length: 10 }, (_, i) => `ds-${i + 1}`); - - setupMock({ tagDocs: tagDocsForDataset(1), collectionsByDataset: collectionsByDatasetFor(1) }); - const result1 = await filterCollectionByKeyValueTags({ - $and: condition, - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - expect(result1?.length).toBe(500); - - const r1 = await measure('1 dataset', () => - filterCollectionByKeyValueTags({ - $and: condition, - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }) - ); - expect(r1.avg).toBeLessThan(50); - - setupMock({ - tagDocs: tagDocsForDataset(10), - collectionsByDataset: collectionsByDatasetFor(10) - }); - const result10 = await filterCollectionByKeyValueTags({ - $and: condition, - $or: [], - teamId: 'team-1', - datasetIds: datasetIds10 - }); - expect(result10?.length).toBe(500); - - const r10 = await measure('10 datasets', () => - filterCollectionByKeyValueTags({ - $and: condition, - $or: [], - teamId: 'team-1', - datasetIds: datasetIds10 - }) - ); - expect(r10.avg).toBeLessThan(100); - - console.log( - ` [横向对比] 1 dataset 平均: ${r1.avg.toFixed(3)}ms (CPU ${r1.cpuPerCallMs.toFixed( - 3 - )}ms) | 10 datasets 平均: ${r10.avg.toFixed(3)}ms (CPU ${r10.cpuPerCallMs.toFixed( - 3 - )}ms) | 倍率: ${(r10.avg / r1.avg).toFixed(2)}x` - ); - }); - - it('1 个 filter vs 10 个 filter', async () => { - // 10 个条件全部命中（value 0..999），避免短路提前退出，测满全部条件判断 - const tenConditions = [ - { version: { $gte: 0 } }, - { version: { $lte: 999 } }, - { version: { $gt: -1 } }, - { version: { $lt: 1000 } }, - { version: { $ne: -1 } }, - { version: { $gte: 0 } }, - { version: { $lte: 999 } }, - { version: { $gt: -1 } }, - { version: { $lt: 1000 } }, - { version: { $ne: -1 } } - ]; - - setupMock({ tagDocs: tagDocsForDataset(1), collectionsByDataset: collectionsByDatasetFor(1) }); - - const result1 = await filterCollectionByKeyValueTags({ - $and: [{ version: { $gte: 500 } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - expect(result1?.length).toBe(500); - - const r1 = await measure('1 filter', () => - filterCollectionByKeyValueTags({ - $and: [{ version: { $gte: 500 } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }) - ); - expect(r1.avg).toBeLessThan(50); - - const result10 = await filterCollectionByKeyValueTags({ - $and: tenConditions, - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - expect(result10?.length).toBe(1000); - - const r10 = await measure('10 filters', () => - filterCollectionByKeyValueTags({ - $and: tenConditions, - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }) - ); - expect(r10.avg).toBeLessThan(100); - - console.log( - ` [横向对比] 1 filter 平均: ${r1.avg.toFixed(3)}ms (CPU ${r1.cpuPerCallMs.toFixed( - 3 - )}ms) | 10 filters 平均: ${r10.avg.toFixed(3)}ms (CPU ${r10.cpuPerCallMs.toFixed( - 3 - )}ms) | 倍率: ${(r10.avg / r1.avg).toFixed(2)}x` - ); - }); -}); - -/* ========== 新tag vs 旧tag 纵向对比 ========== */ - -describe('新tag vs 旧tag 纵向对比（100 / 1000 collections）', () => { - beforeEach(() => { - (global as any).feConfigs = { isPlus: true }; - }); - afterEach(() => { - (global as any).feConfigs = {}; - }); - - it('过滤时间对比', async () => { - // 与设计文档对齐：100 / 1000 collections，每个 collection 含 1~3 个 tags - for (const count of [100, COLLECTION_COUNT]) { - const threshold = Math.floor(count / 2); - - // 新格式：key-value 标签，JS 端值过滤（第 1 个 tag 为命中的 version，其余为干扰项） - const newCollections: Record = { - 'ds-1': Array.from({ length: count }, (_, i) => ({ - _id: `col-${i}`, - tags: [ - { tagId: 'tag-version', value: i }, - ...Array.from({ length: i % 3 }, (_, k) => ({ tagId: `extra-${k}`, value: 'x' })) - ] - })) - }; - setupMock({ - tagDocs: [{ _id: 'tag-version', datasetId: 'ds-1', tag: 'version', tagType: 'number' }], - collectionsByDataset: newCollections - }); - const newResult = await filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: JSON.stringify({ - tags: { $and: [{ version: { $gte: threshold } }] } - }) - }); - expect(newResult?.length).toBeGreaterThan(0); - - const rNew = await measure(`新标签 key-value (${count} collections)`, () => - filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: JSON.stringify({ - tags: { $and: [{ version: { $gte: threshold } }] } - }) - }) - ); - - // 旧格式：字符串数组标签，mongo $all 过滤 - const oldCollections: Record = { - 'ds-1': Array.from({ length: count }, (_, i) => ({ - _id: `col-${i}`, - tags: ['tag-1'] - })) - }; - setupMock({ - tagDocs: [{ _id: 'tag-1', datasetId: 'ds-1', tag: 'Tag1' }], - collectionsByDataset: oldCollections - }); - const oldResult = await filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) - }); - expect(oldResult?.length).toBeGreaterThan(0); - - const rOld = await measure(`旧标签 string array (${count} collections)`, () => - filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) - }) - ); - - console.log( - ` [纵向对比 ${count} collections] 新标签平均: ${rNew.avg.toFixed( - 3 - )}ms (CPU ${rNew.cpuPerCallMs.toFixed(3)}ms) | 旧标签平均: ${rOld.avg.toFixed( - 3 - )}ms (CPU ${rOld.cpuPerCallMs.toFixed(3)}ms) | 新/旧倍率: ${(rNew.avg / rOld.avg).toFixed( - 2 - )}x | 新增 JS 过滤成本: ${(rNew.avg - rOld.avg).toFixed(3)}ms` - ); - } - }); -}); diff --git a/packages/service/test/core/dataset/search/collectionFilter.test.ts b/packages/service/test/core/dataset/search/collectionFilter.test.ts index dd79c3bf588f..1f9a8fe34c12 100644 --- a/packages/service/test/core/dataset/search/collectionFilter.test.ts +++ b/packages/service/test/core/dataset/search/collectionFilter.test.ts @@ -1,18 +1,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -const mockMongoDatasetCollectionFind = vi.hoisted(() => vi.fn()); -const mockMongoDatasetCollectionTagsFind = vi.hoisted(() => vi.fn()); +const collectionFindMock = vi.hoisted(() => vi.fn()); +const tagFindMock = vi.hoisted(() => vi.fn()); vi.mock('@fastgpt/service/core/dataset/collection/schema', () => ({ - MongoDatasetCollection: { - find: mockMongoDatasetCollectionFind - } + MongoDatasetCollection: { find: collectionFindMock } })); - vi.mock('@fastgpt/service/core/dataset/tag/schemaV2', () => ({ - MongoDatasetCollectionTagsV2: { - find: mockMongoDatasetCollectionTagsFind - } + MongoDatasetCollectionTagsV2: { find: tagFindMock } })); import { @@ -20,269 +15,121 @@ import { filterCollectionByKeyValueTags, filterCollectionByMetadata } from '../../../../core/dataset/search/defaultRecall/collectionFilter'; -import { DEFAULT_TAG } from '@fastgpt/global/core/dataset/type'; -/** - * mock MongoDatasetCollection.find 的链式返回：同时支持 `.hint(...).lean()` 与 `.lean()` - * （filterCollectionByKeyValueTags 用 hint 强制走 tags.tagId 索引） - */ -const mockFind = (data: unknown[]) => { - const chain = { - hint: () => chain, - lean: vi.fn().mockResolvedValue(data) - }; +const findResult = (data: unknown[]) => { + const chain = { hint: () => chain, lean: vi.fn().mockResolvedValue(data) }; return chain; }; -describe('filterCollectionByKeyValueTags', () => { - beforeEach(() => { - vi.clearAllMocks(); - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([]) - }); - mockMongoDatasetCollectionFind.mockReturnValue(mockFind([])); - }); +const mockTagsAndCollections = ({ + tags = [], + collections = [] +}: { + tags?: any[]; + collections?: any[]; +}) => { + tagFindMock.mockReturnValue({ lean: vi.fn().mockResolvedValue(tags) }); + collectionFindMock.mockReturnValue(findResult(collections)); +}; - it('returns undefined when no conditions provided', async () => { - const result = await filterCollectionByKeyValueTags({ - $and: [], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - expect(result).toBeUndefined(); +const filterTags = (params: { $and?: any[]; $or?: any[] }) => + filterCollectionByKeyValueTags({ + $and: params.$and ?? [], + $or: params.$or ?? [], + teamId: 'team-1', + datasetIds: ['dataset-1'] }); - it('filters string tags by $eq', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi - .fn() - .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([ - { - _id: 'col-1', - tags: [{ tagId: 'tag-1', value: 'Product A' }] - }, - { - _id: 'col-2', - tags: [{ tagId: 'tag-1', value: 'Product B' }] - } - ]) - ); - - const result = await filterCollectionByKeyValueTags({ - $and: [{ product: { $eq: 'Product A' } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - - expect(result).toEqual(['col-1']); +describe('filterCollectionByKeyValueTags', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockTagsAndCollections({}); }); - it('filters number tags by $gte', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi - .fn() - .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'version', tagType: 'number' }]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([ - { _id: 'col-1', tags: [{ tagId: 'tag-1', value: 2 }] }, - { _id: 'col-2', tags: [{ tagId: 'tag-1', value: 1 }] } - ]) - ); - - const result = await filterCollectionByKeyValueTags({ - $and: [{ version: { $gte: 2 } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - - expect(result).toEqual(['col-1']); + it('returns undefined without conditions', async () => { + await expect(filterTags({})).resolves.toBeUndefined(); }); - it('filters datetime tags by $lt', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi - .fn() - .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'date', tagType: 'datetime' }]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([ - { _id: 'col-1', tags: [{ tagId: 'tag-1', value: 1704067200000 }] }, - { _id: 'col-2', tags: [{ tagId: 'tag-1', value: 1704153600000 }] } - ]) - ); - - const result = await filterCollectionByKeyValueTags({ - $and: [{ date: { $lt: 1704153600000 } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] + it.each([ + ['string', '$eq', 'Product A', 'Product A'], + ['number', '$gte', 2, 2], + ['datetime', '$lt', 1704153600000, 1704067200000] + ])('filters %s tag values', async (tagType, op, target, stored) => { + mockTagsAndCollections({ + tags: [{ _id: 'tag-1', datasetId: 'dataset-1', tag: 'field', tagType }], + collections: [ + { _id: 'match', tags: [{ tagId: 'tag-1', value: stored }] }, + { _id: 'missing', tags: [] } + ] }); - expect(result).toEqual(['col-1']); + await expect(filterTags({ $and: [{ field: { [op]: target } }] })).resolves.toEqual(['match']); }); - it('handles AND + OR combination', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([ - { _id: 'tag-a', datasetId: 'ds-1', tag: 'product', tagType: 'string' }, - { _id: 'tag-b', datasetId: 'ds-1', tag: 'version', tagType: 'number' }, - { _id: 'tag-c', datasetId: 'ds-1', tag: 'category', tagType: 'string' } - ]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([ + it('requires every AND condition and one OR condition', async () => { + mockTagsAndCollections({ + tags: [ + { _id: 'product', datasetId: 'dataset-1', tag: 'product', tagType: 'string' }, + { _id: 'version', datasetId: 'dataset-1', tag: 'version', tagType: 'number' }, + { _id: 'category', datasetId: 'dataset-1', tag: 'category', tagType: 'string' } + ], + collections: [ { - _id: 'col-1', + _id: 'match', tags: [ - { tagId: 'tag-a', value: 'Product A' }, - { tagId: 'tag-b', value: 2 }, - { tagId: 'tag-c', value: 'warranty' } + { tagId: 'product', value: 'A' }, + { tagId: 'version', value: 2 }, + { tagId: 'category', value: 'manual' } ] }, { - _id: 'col-2', + _id: 'or-miss', tags: [ - { tagId: 'tag-a', value: 'Product A' }, - { tagId: 'tag-b', value: 2 }, - { tagId: 'tag-c', value: 'manual' } - ] - }, - { - _id: 'col-3', - tags: [ - { tagId: 'tag-a', value: 'Product A' }, - { tagId: 'tag-b', value: 1 } + { tagId: 'product', value: 'A' }, + { tagId: 'version', value: 2 } ] } - ]) - ); - - const result = await filterCollectionByKeyValueTags({ - $and: [{ product: { $eq: 'Product A' } }, { version: { $gte: 2 } }], - $or: [{ category: { $eq: 'warranty' } }], - teamId: 'team-1', - datasetIds: ['ds-1'] + ] }); - expect(result).toEqual(['col-1']); - }); - - it('returns empty array when AND tag does not exist', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([]) - }); - - const result = await filterCollectionByKeyValueTags({ - $and: [{ product: { $eq: 'Product A' } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - - expect(result).toEqual([]); + await expect( + filterTags({ + $and: [{ product: { $eq: 'A' } }, { version: { $gte: 2 } }], + $or: [{ category: { $eq: 'manual' } }] + }) + ).resolves.toEqual(['match']); }); - it('fails when OR tag does not exist', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi - .fn() - .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) + it('matches empty values only when the collection contains that tag', async () => { + mockTagsAndCollections({ + tags: [{ _id: 'tag-1', datasetId: 'dataset-1', tag: 'field', tagType: 'string' }], + collections: [ + { _id: 'empty', tags: [{ tagId: 'tag-1', value: '' }] }, + { _id: 'absent', tags: [] } + ] }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([{ _id: 'col-1', tags: [{ tagId: 'tag-1', value: 'A' }] }]) - ); - const result = await filterCollectionByKeyValueTags({ - $and: [{ product: { $eq: 'A' } }], - $or: [{ missing: { $eq: 'x' } }], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - - // 该 OR 条件标签不存在 → 条件匹配失败，OR 不通过 - expect(result).toEqual([]); + await expect(filterTags({ $and: [{ field: { $empty: true } }] })).resolves.toEqual(['empty']); }); - it('$empty/$notEmpty only match existing tag entries', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi - .fn() - .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([ - { _id: 'col-1', tags: [{ tagId: 'tag-1', value: 'A' }] }, - { _id: 'col-2', tags: ['old-string-tag'] }, - { _id: 'col-3', tags: [{ tagId: 'tag-1', value: '' }] } - ]) - ); - - const emptyResult = await filterCollectionByKeyValueTags({ - $and: [{ product: { $empty: true } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - // 仅「存在标签条目且值为空」的 col-3 命中；无标签条目的 col-2 不命中 - expect(emptyResult).toEqual(['col-3']); + it('rejects missing tags and treats migration metadata as unrelated to structured filtering', async () => { + await expect(filterTags({ $and: [{ missing: { $eq: 'A' } }] })).resolves.toEqual([]); - const notEmptyResult = await filterCollectionByKeyValueTags({ - $and: [{ product: { $notEmpty: true } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - expect(notEmptyResult).toEqual(['col-1']); - }); - - it('resolves default_tag condition via fromMigration marker, even when renamed', async () => { - // 迁移承载记录已改名（tag 不再是 default_tag），fromMigration 标记仍在 → 旧条件仍命中 - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([ + mockTagsAndCollections({ + tags: [ { - _id: 'migrated-id', - datasetId: 'ds-1', - tag: '历史标签', + _id: 'carrier', + datasetId: 'dataset-1', + tag: 'default_tag', tagType: 'array', fromMigration: true } - ]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([{ _id: 'col-1', tags: [{ tagId: 'migrated-id', value: ['Tag1', 'Tag2'] }] }]) - ); - - const result = await filterCollectionByKeyValueTags({ - $and: [{ [DEFAULT_TAG]: { $contains: 'Tag1' } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] + ], + collections: [{ _id: 'legacy', tags: [{ tagId: 'carrier', value: ['A'] }] }] }); - - expect(result).toEqual(['col-1']); - }); - - it('returns empty when no fromMigration record exists for default_tag condition', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([]) - }); - mockMongoDatasetCollectionFind.mockReturnValue(mockFind([])); - - const result = await filterCollectionByKeyValueTags({ - $and: [{ [DEFAULT_TAG]: { $contains: 'Tag1' } }], - $or: [], - teamId: 'team-1', - datasetIds: ['ds-1'] - }); - - expect(result).toEqual([]); + await expect(filterTags({ $and: [{ default_tag: { $contains: 'A' } }] })).resolves.toEqual([ + 'legacy' + ]); }); }); @@ -290,255 +137,63 @@ describe('filterCollectionByMetadata', () => { beforeEach(() => { vi.clearAllMocks(); (global as any).feConfigs = { isPlus: true }; - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([]) + mockTagsAndCollections({ + tags: [{ _id: 'tag-1', datasetId: 'dataset-1', tag: 'field', tagType: 'string' }], + collections: [{ _id: 'match', tags: [{ tagId: 'tag-1', value: 'A' }] }] }); - mockMongoDatasetCollectionFind.mockReturnValue(mockFind([])); }); - afterEach(() => { (global as any).feConfigs = {}; }); - it('routes new format to filterCollectionByKeyValueTags', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi - .fn() - .mockResolvedValue([{ _id: 'tag-1', datasetId: 'ds-1', tag: 'product', tagType: 'string' }]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([{ _id: 'col-1', tags: [{ tagId: 'tag-1', value: 'Product A' }] }]) - ); - - const result = await filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: JSON.stringify({ - tags: { - $and: [{ product: { $eq: 'Product A' } }] - } + it('accepts structured conditions and rejects legacy or malformed configurations', async () => { + await expect( + filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['dataset-1'], + collectionFilterMatch: JSON.stringify({ tags: { $and: [{ field: { $eq: 'A' } }] } }) }) - }); - - expect(result).toEqual(['col-1']); - }); - - it('rewrites old format to default_tag conditions', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi - .fn() - .mockResolvedValue([ - { _id: 'default-tag-1', datasetId: 'ds-1', tag: 'default_tag', tagType: 'array' } - ]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([{ _id: 'col-1', tags: [{ tagId: 'default-tag-1', value: ['Tag1'] }] }]) - ); - - const result = await filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) - }); - - expect(result).toEqual(['col-1']); - }); - - it('matches legacy string filter for renamed default_tag via fromMigration', async () => { - // 迁移承载记录被改名后，旧格式字符串过滤仍按 fromMigration 定位命中 - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([ - { - _id: 'migrated-id', - datasetId: 'ds-1', - tag: '历史标签', - tagType: 'array', - fromMigration: true - } - ]) - }); - mockMongoDatasetCollectionFind.mockReturnValue( - mockFind([{ _id: 'col-1', tags: [{ tagId: 'migrated-id', value: ['Tag1'] }] }]) - ); - - const result = await filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: JSON.stringify({ tags: { $and: ['Tag1'] } }) - }); - - expect(result).toEqual(['col-1']); - }); - - it('returns undefined when collectionFilterMatch is invalid JSON', async () => { - const result = await filterCollectionByMetadata({ - teamId: 'team-1', - datasetIds: ['ds-1'], - collectionFilterMatch: 'not-json{ broken' - }); - - expect(result).toBeUndefined(); + ).resolves.toEqual(['match']); + + for (const value of [ + JSON.stringify({ tags: { $and: ['legacy'] } }), + JSON.stringify({ tags: { $or: [null] } }), + 'not-json{' + ]) { + await expect( + filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['dataset-1'], + collectionFilterMatch: value + }) + ).rejects.toBeTruthy(); + } }); }); describe('checkValue', () => { - describe('string type', () => { - it('$eq is case-sensitive', () => { - expect(checkValue('$eq', 'Product A', 'product a', 'string')).toBe(false); - expect(checkValue('$eq', 'Product A', 'Product A', 'string')).toBe(true); - }); - - it('$ne is case-sensitive', () => { - expect(checkValue('$ne', 'Product A', 'product a', 'string')).toBe(true); - expect(checkValue('$ne', 'Product A', 'Product B', 'string')).toBe(true); - expect(checkValue('$ne', 'Product A', 'Product A', 'string')).toBe(false); - }); - - it('$contains checks substring ignoring case', () => { - expect(checkValue('$contains', 'Foo', 'foobar', 'string')).toBe(true); - expect(checkValue('$contains', 'xyz', 'foobar', 'string')).toBe(false); - }); - - it('$notContains negates substring check', () => { - expect(checkValue('$notContains', 'xyz', 'foobar', 'string')).toBe(true); - expect(checkValue('$notContains', 'Foo', 'foobar', 'string')).toBe(false); - }); - - it('$startsWith ignores case', () => { - expect(checkValue('$startsWith', 'foo', 'FOOBAR', 'string')).toBe(true); - expect(checkValue('$startsWith', 'bar', 'foobar', 'string')).toBe(false); - }); - - it('$endsWith ignores case', () => { - expect(checkValue('$endsWith', 'bar', 'FOOBAR', 'string')).toBe(true); - expect(checkValue('$endsWith', 'foo', 'foobar', 'string')).toBe(false); - }); - - it('$regex matches valid patterns', () => { - expect(checkValue('$regex', '^foo', 'foobar', 'string')).toBe(true); - expect(checkValue('$regex', '^bar', 'foobar', 'string')).toBe(false); - }); - - it('$regex returns false for invalid patterns', () => { - expect(checkValue('$regex', '[invalid', 'foobar', 'string')).toBe(false); - }); - - it('$regex rejects catastrophic backtracking patterns', () => { - // 嵌套量词族由 safe-regex 检出，带分支量词组（(a|aa)+）由首字符重叠兜底 - expect(checkValue('$regex', '(a+)+$', 'aaaaab', 'string')).toBe(false); - expect(checkValue('$regex', '(a*)*$', 'aaaaab', 'string')).toBe(false); - expect(checkValue('$regex', '(a|aa)+$', 'aaaaab', 'string')).toBe(false); - }); - - it('$regex accepts benign patterns', () => { - expect(checkValue('$regex', 'foo', 'foobar', 'string')).toBe(true); - expect(checkValue('$regex', '^foo', 'foobar', 'string')).toBe(true); - expect(checkValue('$regex', '\\d+', 'abc123', 'string')).toBe(true); - expect(checkValue('$regex', '(ab)+', 'ababab', 'string')).toBe(true); - }); - - it('$regex returns false for overlong pattern or stored value', () => { - expect(checkValue('$regex', 'a'.repeat(65), 'aaaaa', 'string')).toBe(false); - expect(checkValue('$regex', 'a', 'x'.repeat(257), 'string')).toBe(false); - }); - - it('returns false when stored value is null/undefined/empty', () => { - expect(checkValue('$eq', 'x', null, 'string')).toBe(false); - expect(checkValue('$contains', 'x', undefined, 'string')).toBe(false); - expect(checkValue('$startsWith', 'x', '', 'string')).toBe(false); - }); - }); - - describe('number type', () => { - it('compares with $eq/$ne/$gt/$lt/$gte/$lte', () => { - expect(checkValue('$eq', 2, 2, 'number')).toBe(true); - expect(checkValue('$ne', 2, 3, 'number')).toBe(true); - expect(checkValue('$gt', 1, 2, 'number')).toBe(true); - expect(checkValue('$lt', 3, 2, 'number')).toBe(true); - expect(checkValue('$gte', 2, 2, 'number')).toBe(true); - expect(checkValue('$lte', 2, 2, 'number')).toBe(true); - }); - - it('returns false for NaN stored or target', () => { - expect(checkValue('$eq', 2, NaN, 'number')).toBe(false); - expect(checkValue('$eq', NaN, 2, 'number')).toBe(false); - }); - - it('coerces string numbers', () => { - expect(checkValue('$eq', '2', 2, 'number')).toBe(true); - expect(checkValue('$eq', 2, '2', 'number')).toBe(true); - }); - }); - - describe('datetime type', () => { - it('compares unix millisecond timestamps', () => { - expect(checkValue('$eq', 1704067200000, 1704067200000, 'datetime')).toBe(true); - expect(checkValue('$gt', 1704067200000, 1704153600000, 'datetime')).toBe(true); - expect(checkValue('$lt', 1704153600000, 1704067200000, 'datetime')).toBe(true); - }); - - it('returns false for NaN stored or target', () => { - expect(checkValue('$eq', 1704067200000, NaN, 'datetime')).toBe(false); - expect(checkValue('$eq', NaN, 1704067200000, 'datetime')).toBe(false); - }); - }); - - describe('array type', () => { - it('$is/$isNot compare arrays as sets (order-insensitive)', () => { - expect(checkValue('$is', ['a', 'b'], ['b', 'a'], 'array')).toBe(true); - expect(checkValue('$is', ['a', 'b'], ['a', 'c'], 'array')).toBe(false); - expect(checkValue('$isNot', ['a', 'b'], ['a', 'c'], 'array')).toBe(true); - expect(checkValue('$isNot', ['a', 'b'], ['b', 'a'], 'array')).toBe(false); - }); - - it('$contains/$notContains check single-string membership', () => { - expect(checkValue('$contains', 'a', ['a', 'b'], 'array')).toBe(true); - expect(checkValue('$contains', 'c', ['a', 'b'], 'array')).toBe(false); - expect(checkValue('$notContains', 'c', ['a', 'b'], 'array')).toBe(true); - expect(checkValue('$notContains', 'a', ['a', 'b'], 'array')).toBe(false); - }); - - it('$in/$notIn check subset', () => { - expect(checkValue('$in', ['a', 'b', 'c'], ['a', 'b'], 'array')).toBe(true); - expect(checkValue('$in', ['a', 'c'], ['a', 'b'], 'array')).toBe(false); - expect(checkValue('$notIn', ['a', 'c'], ['a', 'b'], 'array')).toBe(true); - expect(checkValue('$notIn', ['a', 'b', 'c'], ['a', 'b'], 'array')).toBe(false); - }); - - it('$empty/$notEmpty for arrays', () => { - expect(checkValue('$empty', true, [], 'array')).toBe(true); - expect(checkValue('$empty', true, undefined, 'array')).toBe(true); - expect(checkValue('$empty', true, ['a'], 'array')).toBe(false); - expect(checkValue('$notEmpty', true, ['a'], 'array')).toBe(true); - expect(checkValue('$notEmpty', true, [], 'array')).toBe(false); - }); - - it('returns false when stored value is not an array or target is not an array', () => { - expect(checkValue('$is', ['a'], 'not-array', 'array')).toBe(false); - expect(checkValue('$contains', 'a', 'not-array', 'array')).toBe(false); - expect(checkValue('$in', 'a', ['a'], 'array')).toBe(false); - }); - }); - - describe('empty operators', () => { - it('$empty treats null/undefined/empty string as empty', () => { - expect(checkValue('$empty', true, null, 'string')).toBe(true); - expect(checkValue('$empty', true, undefined, 'string')).toBe(true); - expect(checkValue('$empty', true, '', 'string')).toBe(true); - expect(checkValue('$empty', true, 'x', 'string')).toBe(false); - expect(checkValue('$empty', true, 0, 'number')).toBe(false); - }); - - it('$notEmpty reverses empty logic', () => { - expect(checkValue('$notEmpty', true, 'x', 'string')).toBe(true); - expect(checkValue('$notEmpty', true, null, 'string')).toBe(false); - expect(checkValue('$notEmpty', true, undefined, 'string')).toBe(false); - expect(checkValue('$notEmpty', true, '', 'string')).toBe(false); - }); - }); - - it('returns false for unsupported operator or null target', () => { - expect(checkValue('$unsupported' as any, 'x', 'x', 'string')).toBe(false); + it.each([ + ['$eq', 'Product A', 'Product A', 'string', true], + ['$contains', 'foo', 'FOOBAR', 'string', true], + ['$gte', 2, '2', 'number', true], + ['$lt', 2, 1, 'datetime', true], + ['$is', ['a', 'b'], ['b', 'a'], 'array', true], + ['$in', ['a', 'b', 'c'], ['a', 'b'], 'array', true], + ['$empty', true, undefined, 'array', true], + ['$unsupported', 'x', 'x', 'string', false] + ])('compares %s for %s values', (op, target, stored, tagType, expected) => { + expect(checkValue(op as any, target, stored as any, tagType)).toBe(expected); + }); + + it('rejects invalid value shapes and unsafe regex patterns', () => { expect(checkValue('$eq', null, 'x', 'string')).toBe(false); + expect(checkValue('$eq', 2, Number.NaN, 'number')).toBe(false); + expect(checkValue('$is', ['a'], 'not-array', 'array')).toBe(false); + expect(checkValue('$regex', 'foo', 'foobar', 'string')).toBe(true); + expect(checkValue('$regex', '[invalid', 'foobar', 'string')).toBe(false); + expect(checkValue('$regex', '(a+)+$', 'aaaaab', 'string')).toBe(false); + expect(checkValue('$regex', '(a|aa)+$', 'aaaaab', 'string')).toBe(false); + expect(checkValue('$regex', 'a'.repeat(65), 'aaaaa', 'string')).toBe(false); + expect(checkValue('$regex', 'a', 'x'.repeat(257), 'string')).toBe(false); }); }); diff --git a/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts b/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts index be7b78582e23..2e533b899b90 100644 --- a/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts +++ b/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts @@ -1,485 +1,62 @@ import { describe, expect, it } from 'vitest'; -import { getRootUser } from '@test/datas/users'; import { Types } from '@fastgpt/service/common/mongo'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; -import { filterCollectionByKeyValueTags } from '@fastgpt/service/core/dataset/search/defaultRecall/collectionFilter'; -import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; - -/** - * dataset_collections 两个标签索引性能对比（真实 MongoDB） - * - * 对比索引： - * 1. { teamId: 1, datasetId: 1, tags: 1 } —— 旧格式 string[] 标签，查询 `tags: { $all }` - * 2. { teamId: 1, datasetId: 1, 'tags.tagId': 1 } —— 新格式 { tagId, value }[] 标签，查询 `tags.tagId: { $all }` - * - * 对每个索引分别测「有索引」与「无索引」两个场景： - * - explain('executionStats')：断言有索引时使用被测索引（IXSCAN）且扫描量远小于全表； - * 索引缺失时扫描量接近全表（优化器退到其他复合索引前缀或 COLLSCAN） - * - 服务端执行时间：多次 explain 采样 executionTimeMillis，取平均与中位（避免单次噪声） - * - 墙钟延迟：预热 + 多次迭代取中位数，对比索引带来的加速 - * - * 运行方式： - * pnpm test:benchmark -- collectionTagIndex.benchmark.ts - * 使用真实 MongoDB：FASTGPT_TEST_MONGODB_URI=mongodb://localhost:27017 \ - * pnpm test:benchmark -- collectionTagIndex.benchmark.ts - * - * 注意：test/setup.ts 会在每个用例结束后清空所有集合文档（保留索引）， - * 因此数据准备必须在单个 it 内完成，不能跨用例复用 beforeAll 数据。 - */ +import { getRootUser } from '@test/datas/users'; -const COLLECTION_COUNT = 30_000; -const TAG_COUNT = 10; -/** 命中 5% 的 collection 携带查询目标 tags（index % 20 === 0） */ +const COLLECTION_COUNT = 5_000; const HIT_EVERY = 20; -const INSERT_BATCH = 5_000; -const MEASURE_ITERATIONS = 20; -const MEASURE_WARMUPS = 5; - -/** 目标两个复合索引（用 key spec 控制创建/删除，避免依赖自动生成的索引名） */ -const TAGS_INDEX_KEY: Record = { teamId: 1, datasetId: 1, tags: 1 }; -const TAGS_TAGID_INDEX_KEY: Record = { teamId: 1, datasetId: 1, 'tags.tagId': 1 }; -/** MongoDB 自动生成的索引名（key 字段名_方向 拼接） */ -const TAGS_INDEX_NAME = 'teamId_1_datasetId_1_tags_1'; -const TAGS_TAGID_INDEX_NAME = 'teamId_1_datasetId_1_tags.tagId_1'; - -type RootUser = Awaited>; - -interface MeasureResult { - avg: number; - min: number; - max: number; - median: number; - times: number[]; -} - -/* ========== 计时辅助 ========== */ - -async function measure( - name: string, - fn: () => Promise, - iterations = MEASURE_ITERATIONS, - warmups = MEASURE_WARMUPS -): Promise { - // 预热避开 JIT / 首次查询缓存抖动 - for (let i = 0; i < warmups; i++) await fn(); - - const times: number[] = []; - for (let i = 0; i < iterations; i++) { - const start = performance.now(); - await fn(); - times.push(performance.now() - start); - } - - const avg = times.reduce((a, b) => a + b, 0) / times.length; - const min = Math.min(...times); - const max = Math.max(...times); - const sorted = [...times].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; - - console.log( - ` ${name.padEnd(26)} 平均: ${avg.toFixed(2).padStart(8)}ms 中位: ${median - .toFixed(2) - .padStart(8)}ms 最小: ${min.toFixed(2).padStart(8)}ms 最大: ${max.toFixed(2).padStart(8)}ms` - ); - return { avg, min, max, median, times }; -} - -/* ========== explain 辅助 ========== */ - -interface ExplainStats { - stages: string[]; - /** 若走 IXSCAN，命中的索引名 */ - indexName: string; - totalDocsExamined: number; - totalKeysExamined: number; - executionTimeMillis: number; - nReturned: number; -} - -/** 递归收集 stage 树上的所有 stage 名（不同 MongoDB 版本嵌套深度不同） */ -function collectStages(stage: any): string[] { - if (!stage) return []; - const result = [stage.stage]; - if (stage.inputStage) result.push(...collectStages(stage.inputStage)); - if (Array.isArray(stage.inputStages)) { - for (const s of stage.inputStages) result.push(...collectStages(s)); - } - return result; -} - -/** 在 stage 树中查找指定名称的 stage */ -function findStage(stage: any, name: string): any | undefined { - if (!stage) return undefined; - if (stage.stage === name) return stage; - if (stage.inputStage) { - const found = findStage(stage.inputStage, name); - if (found) return found; - } - if (Array.isArray(stage.inputStages)) { - for (const sub of stage.inputStages) { - const found = findStage(sub, name); - if (found) return found; - } - } - return undefined; -} - -interface ExplainTimeResult { - avg: number; - median: number; -} - -/** 多次 explain 采样服务端 executionTimeMillis，取平均与中位（避免单次测量噪声） */ -async function measureExplainTime( - query: Record, - iterations = 10, - warmups = 3 -): Promise { - for (let i = 0; i < warmups; i++) { - await MongoDatasetCollection.find(query, '_id').explain('executionStats'); - } - - const times: number[] = []; - for (let i = 0; i < iterations; i++) { - const raw = (await MongoDatasetCollection.find(query, '_id') - .explain('executionStats') - .then((res: any) => res[0] ?? res)) as any; - times.push(raw?.executionStats?.executionTimeMillis ?? 0); - } - - const avg = times.reduce((a, b) => a + b, 0) / times.length; - const sorted = [...times].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; - return { avg, median }; -} - -async function explainQuery( - query: Record, - hint?: Record -): Promise { - let findQuery = MongoDatasetCollection.find(query, '_id'); - if (hint) findQuery = findQuery.hint(hint); - const raw = (await findQuery.explain('executionStats').then((res: any) => res[0] ?? res)) as any; - // debug: 打印完整 explain 结果（executionStages 树可能较大） - // console.log( - // `explain raw: query=${JSON.stringify(query)}, executionStats=${JSON.stringify(raw)}` - // ); - - const winningPlan = raw?.queryPlanner?.winningPlan; - const execStats = raw?.executionStats; - const ixscan = findStage(winningPlan, 'IXSCAN'); - - // executionStats 顶层字段优先，回退到 executionStages 树上的同名字段 - const totalDocsExamined = - execStats?.totalDocsExamined ?? execStats?.executionStages?.totalDocsExamined ?? 0; - const totalKeysExamined = - execStats?.totalKeysExamined ?? execStats?.executionStages?.totalKeysExamined ?? 0; - const executionTimeMillis = execStats?.executionTimeMillis ?? 0; - const nReturned = execStats?.nReturned ?? execStats?.executionStages?.nReturned ?? 0; - - return { - stages: collectStages(winningPlan), - indexName: ixscan?.indexName ?? '', - totalDocsExamined, - totalKeysExamined, - executionTimeMillis, - nReturned - }; -} - -/* ========== 数据准备 ========== */ - -function buildOldTags(index: number, isHit: boolean, tagIds: string[]): string[] { - if (isHit) { - // 命中：同时携带 tagIds[0] 与 tagIds[1]，再加一个干扰 tag - return [tagIds[0], tagIds[1], tagIds[(index + 2) % TAG_COUNT]]; - } - // 非命中：只携带一个 tag，单个元素永远不满足 $all [tagIds[0], tagIds[1]] - return [tagIds[(index + 1) % TAG_COUNT]]; -} - -function buildNewTags( - index: number, - isHit: boolean, - tagIds: string[] -): Array<{ tagId: string; value: string }> { - if (isHit) { - return [ - { tagId: tagIds[0], value: 'A' }, - { tagId: tagIds[1], value: 'B' }, - { tagId: tagIds[(index + 2) % TAG_COUNT], value: 'noise' } - ]; - } - return [{ tagId: tagIds[(index + 1) % TAG_COUNT], value: 'noise' }]; -} - -async function seedTagIndexData({ root, format }: { root: RootUser; format: 'old' | 'new' }) { - const suffix = `${format}-${Date.now()}`; - const dataset = await MongoDataset.create({ - teamId: root.teamId, - tmbId: root.tmbId, - name: `tag-index-${suffix}`, - type: DatasetTypeEnum.dataset, - vectorModel: 'text-embedding-3-small', - agentModel: 'gpt-4o-mini' - }); +const TAG_INDEX_KEY = { teamId: 1, datasetId: 1, 'tags.tagId': 1 } as const; +const TAG_INDEX_NAME = 'teamId_1_datasetId_1_tags.tagId_1'; - const datasetId = String(dataset._id); - const tagIds = Array.from({ length: TAG_COUNT }, () => new Types.ObjectId().toString()); +const findIndexName = (stage: unknown): string | undefined => { + if (!stage || typeof stage !== 'object') return; + if (Reflect.get(stage, 'stage') === 'IXSCAN') return Reflect.get(stage, 'indexName'); - // 分批插入，避免堆尖峰 - for (let start = 0; start < COLLECTION_COUNT; start += INSERT_BATCH) { - const batchSize = Math.min(INSERT_BATCH, COLLECTION_COUNT - start); - const docs = Array.from({ length: batchSize }, (_, k) => { - const i = start + k; - const isHit = i % HIT_EVERY === 0; - return { - teamId: root.teamId, - tmbId: root.tmbId, - datasetId, - type: DatasetCollectionTypeEnum.file, - name: `col-${suffix}-${i}`, - tags: format === 'old' ? buildOldTags(i, isHit, tagIds) : buildNewTags(i, isHit, tagIds) - }; - }); - await MongoDatasetCollection.insertMany(docs); - } + const inputStage = findIndexName(Reflect.get(stage, 'inputStage')); + if (inputStage) return inputStage; - return { datasetId, tagIds }; -} + const inputStages = Reflect.get(stage, 'inputStages'); + if (!Array.isArray(inputStages)) return; + return inputStages.map(findIndexName).find(Boolean); +}; -/** 确保两个标签索引都存在（createIndex 幂等；显式指定 name 保证与删除/断言一致） */ -async function ensureTagIndexes() { - await MongoDatasetCollection.collection.createIndex(TAGS_INDEX_KEY, { name: TAGS_INDEX_NAME }); - await MongoDatasetCollection.collection.createIndex(TAGS_TAGID_INDEX_KEY, { - name: TAGS_TAGID_INDEX_NAME - }); -} - -/* ========== 断言场景 ========== */ - -async function assertIndexed( - query: Record, - expectedIndexName: string, - label: string -) { - const exp = await explainQuery(query); - // 必须使用被测标签索引，且索引扫描的文档数远小于全表 - expect(exp.stages).toContain('IXSCAN'); - expect(exp.indexName).toBe(expectedIndexName); - expect(exp.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); - // 索引场景的服务端执行时间：多次 explain 取中位，避免单次噪声（内存 mongo / CI 波动大，取宽松值） - const time = await measureExplainTime(query); - expect(time.median).toBeLessThan(300); - console.log( - ` [${label} 有索引 explain] stage=${exp.stages.join('->')} index=${exp.indexName} examined=${exp.totalDocsExamined} keys=${exp.totalKeysExamined} returned=${exp.nReturned} time(avg/median)=${time.avg.toFixed(1)}/${time.median.toFixed(1)}ms` - ); - return { ...exp, time }; -} - -async function assertUnindexed(query: Record, label: string) { - const exp = await explainQuery(query); - // 标签索引被删除后，优化器只能退到其他复合索引的前缀（teamId+datasetId）扫描全部文档， - // 因此扫描量接近全表（此处可能表现为 IXSCAN on createTime/fileId 索引或 COLLSCAN， - // 不强制 COLLSCAN） - expect(exp.totalDocsExamined).toBeGreaterThanOrEqual(COLLECTION_COUNT * 0.9); - expect(exp.indexName).not.toMatch(/tags/i); - const time = await measureExplainTime(query); - console.log( - ` [${label} 无索引 explain] stage=${exp.stages.join('->')} index=${exp.indexName || 'N/A'} examined=${exp.totalDocsExamined} keys=${exp.totalKeysExamined} returned=${exp.nReturned} time(avg/median)=${time.avg.toFixed(1)}/${time.median.toFixed(1)}ms` - ); - return { ...exp, time }; -} - -/* ========== 测试 ========== */ - -describe('dataset_collections 标签索引性能对比（真实 MongoDB）', () => { - it('旧格式：tags 复合索引 vs 无索引', async () => { +/** 验证新标签查询使用当前 Schema 声明的复合索引，不比较易受环境影响的墙钟时间。 */ +describe('dataset collection tag index', () => { + it('uses tags.tagId index and scans only matching collections', async () => { const root = await getRootUser(); - const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'old' }); - const query = { - teamId: root.teamId, - datasetId, - tags: { $all: [tagIds[0], tagIds[1]] } - }; - const runQuery = () => MongoDatasetCollection.find(query, '_id').lean(); - - try { - // 有索引：只保留 tags 复合索引，避免优化器误选 tags.tagId 索引（后者会导致 FETCH 全表） - await ensureTagIndexes(); - await MongoDatasetCollection.collection.dropIndex(TAGS_TAGID_INDEX_NAME); - const expIndexed = await assertIndexed(query, TAGS_INDEX_NAME, '旧格式 tags'); - const mIndexed = await measure('旧格式 + tags 索引', runQuery); - - // 无索引：删掉 tags 复合索引 - await MongoDatasetCollection.collection.dropIndex(TAGS_INDEX_NAME); - const expUnindexed = await assertUnindexed(query, '旧格式 tags'); - const mUnindexed = await measure('旧格式 + 无索引', runQuery); - - // 断言：无索引扫描量/耗时明显大于有索引 - expect(expUnindexed.totalDocsExamined).toBeGreaterThan(expIndexed.totalDocsExamined * 5); - expect(expUnindexed.time.median).toBeGreaterThan(expIndexed.time.median); - expect(mUnindexed.median).toBeGreaterThan(mIndexed.median); - - console.log( - ` [旧格式 tags] examined 倍率: ${(expUnindexed.totalDocsExamined / expIndexed.totalDocsExamined).toFixed(1)}x 执行时间倍率(median): ${(expUnindexed.time.median / expIndexed.time.median).toFixed(1)}x 中位延迟倍率: ${(mUnindexed.median / mIndexed.median).toFixed(1)}x` + const datasetId = new Types.ObjectId(); + const targetTagId = String(new Types.ObjectId()); + const otherTagId = String(new Types.ObjectId()); + + await MongoDatasetCollection.createIndexes({ background: true }); + for (let start = 0; start < COLLECTION_COUNT; start += 1_000) { + await MongoDatasetCollection.insertMany( + Array.from({ length: Math.min(1_000, COLLECTION_COUNT - start) }, (_, offset) => { + const index = start + offset; + return { + teamId: root.teamId, + tmbId: root.tmbId, + datasetId, + type: 'file', + name: `collection-${index}`, + tags: [{ tagId: index % HIT_EVERY === 0 ? targetTagId : otherTagId, value: index }] + }; + }) ); - } finally { - // 恢复全部索引，避免影响下一个用例 - await ensureTagIndexes(); } - }, 300_000); - - it('新格式：tags.tagId 复合索引 vs 无索引', async () => { - const root = await getRootUser(); - const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'new' }); - const query = { - teamId: root.teamId, - datasetId, - 'tags.tagId': { $all: [tagIds[0], tagIds[1]] } - }; - const runQuery = () => MongoDatasetCollection.find(query, '_id').lean(); - - try { - // 有索引：只保留 tags.tagId 复合索引，避免优化器误选 tags 索引（后者会导致 FETCH 全表） - await ensureTagIndexes(); - await MongoDatasetCollection.collection.dropIndex(TAGS_INDEX_NAME); - const expIndexed = await assertIndexed(query, TAGS_TAGID_INDEX_NAME, '新格式 tags.tagId'); - const mIndexed = await measure('新格式 + tags.tagId 索引', runQuery); - - // 无索引：删掉 tags.tagId 复合索引 - await MongoDatasetCollection.collection.dropIndex(TAGS_TAGID_INDEX_NAME); - const expUnindexed = await assertUnindexed(query, '新格式 tags.tagId'); - const mUnindexed = await measure('新格式 + 无索引', runQuery); - - // 断言：无索引扫描量/耗时明显大于有索引 - expect(expUnindexed.totalDocsExamined).toBeGreaterThan(expIndexed.totalDocsExamined * 5); - expect(expUnindexed.time.median).toBeGreaterThan(expIndexed.time.median); - expect(mUnindexed.median).toBeGreaterThan(mIndexed.median); - - console.log( - ` [新格式 tags.tagId] examined 倍率: ${(expUnindexed.totalDocsExamined / expIndexed.totalDocsExamined).toFixed(1)}x 执行时间倍率(median): ${(expUnindexed.time.median / expIndexed.time.median).toFixed(1)}x 中位延迟倍率: ${(mUnindexed.median / mIndexed.median).toFixed(1)}x` - ); - } finally { - // 恢复全部索引，避免影响下一个用例 - await ensureTagIndexes(); - } - }, 300_000); - it('新格式：hint 强制走 tags.tagId 索引（生产两索引并存）', async () => { - const root = await getRootUser(); - const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'new' }); - const query = { - teamId: root.teamId, - datasetId, - 'tags.tagId': { $all: [tagIds[0], tagIds[1]] } - }; - - try { - // 生产状态：两个标签索引并存 - await ensureTagIndexes(); - - // 不加 hint：记录规划器自由选择的索引（优化器行为随环境/数据分布可能不同，不强断言） - const noHint = await explainQuery(query); - // 加 hint：必须强制走 tags.tagId 索引（修复的核心保证） - const hinted = await explainQuery(query, TAGS_TAGID_INDEX_KEY); - expect(hinted.indexName).toBe(TAGS_TAGID_INDEX_NAME); - expect(hinted.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); - // hint 后的扫描量不应比规划器自选更大 - expect(hinted.totalDocsExamined).toBeLessThanOrEqual(noHint.totalDocsExamined); - - console.log( - ` [新格式 hint 验证] 无 hint: ${noHint.indexName} (examined=${noHint.totalDocsExamined}) -> 有 hint: ${hinted.indexName} (examined=${hinted.totalDocsExamined})` - ); - } finally { - await ensureTagIndexes(); - } - }, 300_000); - - it('旧格式：两索引并存时查询仍走 tags 索引（不受 tags.tagId 索引影响）', async () => { - const root = await getRootUser(); - const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'old' }); - const query = { - teamId: root.teamId, - datasetId, - tags: { $all: [tagIds[0], tagIds[1]] } - }; - - try { - // 生产状态：两个标签索引并存 - await ensureTagIndexes(); - - // 不加 hint：旧格式查询按 tags 字段匹配，tags.tagId 索引无法精准服务该查询， - // 规划器应仍选择 tags 索引（否则会像新格式一样退到前缀索引导致全表 FETCH） - const noHint = await explainQuery(query); - expect(noHint.indexName).toBe(TAGS_INDEX_NAME); - expect(noHint.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); - - // 加 hint 强制 tags 索引：结果应同样高效 - const hinted = await explainQuery(query, TAGS_INDEX_KEY); - expect(hinted.indexName).toBe(TAGS_INDEX_NAME); - expect(hinted.totalDocsExamined).toBeLessThan(COLLECTION_COUNT * 0.2); - - console.log( - ` [旧格式两索引并存] 无 hint: ${noHint.indexName} (examined=${noHint.totalDocsExamined}) -> 有 hint: ${hinted.indexName} (examined=${hinted.totalDocsExamined})` - ); - } finally { - await ensureTagIndexes(); - } - }, 300_000); - - it('端到端：filterCollectionByKeyValueTags 应用层比较成本（真实数据）', async () => { - const root = await getRootUser(); - const { datasetId, tagIds } = await seedTagIndexData({ root, format: 'new' }); - - // 同步真实 v2 标签表，供 filter 按标签名解析 tagId（_id 与集合 tags.tagId 一致） - await MongoDatasetCollectionTagsV2.collection.createIndex( - { teamId: 1, datasetId: 1, tag: 1 }, - { unique: true } - ); - await MongoDatasetCollectionTagsV2.insertMany( - tagIds.map((id, i) => ({ - teamId: root.teamId, - datasetId, - tag: `tag-${i}`, - tagType: 'string', - _id: new Types.ObjectId(id) - })) - ); - - // 生产状态：两个标签索引并存，filter 内部 hint 强制走 tags.tagId 索引 - await ensureTagIndexes(); - - const filter = () => - filterCollectionByKeyValueTags({ - $and: [{ 'tag-0': { $eq: 'A' } }, { 'tag-1': { $eq: 'B' } }], - $or: [], - teamId: root.teamId, - datasetIds: [datasetId] - }); - - // 命中 5% 的 collection（index % 20 === 0）同时携带 tagIds[0]('A') 与 tagIds[1]('B') - const hit = await filter(); - expect(hit?.length).toBe(Math.floor(COLLECTION_COUNT / HIT_EVERY)); - console.log(` [端到端] 命中集合数: ${hit?.length}`); - - const full = await measure('端到端 filterCollectionByKeyValueTags', filter); + const explain = await MongoDatasetCollection.find( + { teamId: root.teamId, datasetId, 'tags.tagId': targetTagId }, + '_id' + ) + .hint(TAG_INDEX_KEY) + .explain('executionStats') + .then((result: any) => result[0] ?? result); - // 对照纯 tags.tagId 索引查询，量化应用层 checkValue 比较增量 - const query = { - teamId: root.teamId, - datasetId, - 'tags.tagId': { $all: [tagIds[0], tagIds[1]] } - }; - const raw = await measure('纯 tags.tagId 索引查询', () => - MongoDatasetCollection.find(query, '_id').hint(TAGS_TAGID_INDEX_KEY).lean() - ); - console.log( - ` [端到端] 应用层比较增量: ${(full.median - raw.median).toFixed(2)}ms（filter 中位 ${full.median.toFixed(2)}ms - 索引查询中位 ${raw.median.toFixed(2)}ms）` + expect(findIndexName(explain.queryPlanner?.winningPlan)).toBe(TAG_INDEX_NAME); + expect(explain.executionStats?.nReturned).toBe(COLLECTION_COUNT / HIT_EVERY); + expect(explain.executionStats?.totalDocsExamined).toBeLessThanOrEqual( + COLLECTION_COUNT / HIT_EVERY ); }, 300_000); }); diff --git a/packages/service/test/core/dataset/search/legacy/collectionFilter.test.ts b/packages/service/test/core/dataset/search/legacy/collectionFilter.test.ts new file mode 100644 index 000000000000..91f129713d7c --- /dev/null +++ b/packages/service/test/core/dataset/search/legacy/collectionFilter.test.ts @@ -0,0 +1,139 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { filterLegacyCollectionByMetadata } from '@fastgpt/service/core/dataset/search/defaultRecall/legacy/collectionFilter'; + +const teamId = new Types.ObjectId(); +const tmbId = new Types.ObjectId(); +const datasetA = new Types.ObjectId(); +const datasetB = new Types.ObjectId(); + +const createCollection = async ({ + datasetId, + tags +}: { + datasetId: Types.ObjectId; + tags: unknown[]; +}) => + MongoDatasetCollection.collection.insertOne({ + teamId, + tmbId, + datasetId, + name: 'test', + type: 'file', + tags + }); + +describe('filterLegacyCollectionByMetadata', () => { + beforeEach(async () => { + global.feConfigs = { ...global.feConfigs, isPlus: true }; + await Promise.all([ + MongoDatasetCollection.collection.deleteMany({ teamId }), + MongoDatasetCollectionTagsV2.collection.deleteMany({ teamId }) + ]); + }); + + afterEach(() => { + global.feConfigs = { ...global.feConfigs, isPlus: false }; + }); + + it('uses AND exclusively when AND and OR are both present', async () => { + const carrier = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId: datasetA, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + }); + const [andCollection, orCollection] = await Promise.all([ + createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrier._id), value: ['A'] }] + }), + createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrier._id), value: ['B'] }] + }) + ]); + + const result = await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetA)], + collectionFilterMatch: JSON.stringify({ tags: { $and: ['A'], $or: ['B'] } }) + }); + + expect(result).toEqual([String(andCollection.insertedId)]); + expect(result).not.toContain(String(orCollection.insertedId)); + }); + + it('matches null only against truly empty tags', async () => { + const carrier = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId: datasetA, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + }); + const empty = await createCollection({ datasetId: datasetA, tags: [] }); + await createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrier._id), value: [] }] + }); + + const result = await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetA)], + collectionFilterMatch: JSON.stringify({ tags: { $and: [null] } }) + }); + expect(result).toEqual([String(empty.insertedId)]); + }); + + it('uses each dataset carrier independently and does not widen when one is missing', async () => { + const [carrierA, carrierB] = await MongoDatasetCollectionTagsV2.create([ + { + teamId, + datasetId: datasetA, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + }, + { + teamId, + datasetId: datasetB, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + } + ]); + const [collectionA, collectionB] = await Promise.all([ + createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrierA._id), value: ['A'] }] + }), + createCollection({ + datasetId: datasetB, + tags: [{ tagId: String(carrierB._id), value: ['A'] }] + }) + ]); + + expect( + await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetA), String(datasetB)], + collectionFilterMatch: JSON.stringify({ tags: { $or: ['A'] } }) + }) + ).toEqual( + expect.arrayContaining([String(collectionA.insertedId), String(collectionB.insertedId)]) + ); + + await MongoDatasetCollectionTagsV2.deleteOne({ _id: carrierB._id }); + expect( + await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetB)], + collectionFilterMatch: JSON.stringify({ tags: { $or: ['A'] } }) + }) + ).toEqual([]); + }); +}); diff --git a/packages/service/test/core/dataset/search/multiQueryRecall.test.ts b/packages/service/test/core/dataset/search/multiQueryRecall.test.ts new file mode 100644 index 000000000000..101b2e795a75 --- /dev/null +++ b/packages/service/test/core/dataset/search/multiQueryRecall.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + getForbidCollectionIdListMock, + filterCollectionByMetadataMock, + filterLegacyCollectionByMetadataMock, + embeddingRecallMock, + fullTextRecallMock +} = vi.hoisted(() => ({ + getForbidCollectionIdListMock: vi.fn(), + filterCollectionByMetadataMock: vi.fn(), + filterLegacyCollectionByMetadataMock: vi.fn(), + embeddingRecallMock: vi.fn(), + fullTextRecallMock: vi.fn() +})); + +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/collectionFilter', () => ({ + getForbidCollectionIdList: getForbidCollectionIdListMock, + filterCollectionByMetadata: filterCollectionByMetadataMock +})); +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/legacy/collectionFilter', () => ({ + filterLegacyCollectionByMetadata: filterLegacyCollectionByMetadataMock +})); +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/embeddingRecall', () => ({ + embeddingRecall: embeddingRecallMock +})); +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/fullTextRecall', () => ({ + fullTextRecall: fullTextRecallMock +})); + +import { multiQueryRecall } from '@fastgpt/service/core/dataset/search/defaultRecall/multiQueryRecall'; + +const baseParams = { + teamId: 'team', + datasetIds: ['dataset'], + model: { model: 'embedding' } as any, + imageQueries: [], + collectionFilterMatch: '{}', + embeddingLimit: 10, + fullTextLimit: 10, + textQueries: ['query'], + imageCaptionQueries: [] +}; + +describe('multiQueryRecall collection filter routing', () => { + beforeEach(() => { + vi.clearAllMocks(); + getForbidCollectionIdListMock.mockResolvedValue([]); + filterCollectionByMetadataMock.mockResolvedValue([]); + filterLegacyCollectionByMetadataMock.mockResolvedValue([]); + embeddingRecallMock.mockResolvedValue({ + tokens: 0, + textEmbeddingRecallResults: [], + imageCaptionEmbeddingRecallResults: [], + imageVectorRecallResults: [] + }); + fullTextRecallMock.mockResolvedValue({ + textFullTextRecallResults: [], + imageCaptionFullTextRecallResults: [] + }); + }); + + it.each([ + ['legacy', filterLegacyCollectionByMetadataMock, filterCollectionByMetadataMock], + ['structured', filterCollectionByMetadataMock, filterLegacyCollectionByMetadataMock] + ] as const)('calls only the %s filter', async (mode, expected, unexpected) => { + await multiQueryRecall({ ...baseParams, collectionFilterMode: mode }); + expect(expected).toHaveBeenCalledOnce(); + expect(unexpected).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/service/test/core/dataset/utils.test.ts b/packages/service/test/core/dataset/utils.test.ts index cb20ace09271..1198873241a2 100644 --- a/packages/service/test/core/dataset/utils.test.ts +++ b/packages/service/test/core/dataset/utils.test.ts @@ -14,7 +14,6 @@ import { } from '@fastgpt/service/core/dataset/data/utils'; import { createOrGetCollectionTags, - deduplicateTagValues, getTrainingModeByCollection, validateAndNormalizeTagValue, validateDatasetTagValue @@ -29,7 +28,6 @@ import type { LLMSystemModelDataType } from '@fastgpt/global/core/ai/model.schema'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; -import { DatasetCollectionItemSchema } from '@fastgpt/global/core/dataset/type'; const mockCreateS3DownloadAccessUrls = vi.hoisted(() => vi.fn(async (params: Array<{ objectKey: string }>) => @@ -717,84 +715,20 @@ describe('getDatasetImageIndexCapability', () => { describe('validateDatasetTagValue', () => { it.each([ - ['string', 1, DatasetErrEnum.tagValueInvalid], - ['string', 'a'.repeat(257), DatasetErrEnum.tagValueInvalid], - ['number', '1', undefined], - ['array', ['a'.repeat(257)], DatasetErrEnum.arrayTagValueInvalid], - ['datetime', Number.NaN, DatasetErrEnum.tagValueInvalid] - ])('returns the expected error for invalid %s values', (tagType, value, expected) => { - expect(validateDatasetTagValue({ tagType: tagType as any, value: value as any })).toBe( - expected - ); - }); - - it('accepts numeric strings for number and datetime tags', () => { - expect(validateDatasetTagValue({ tagType: 'number', value: '1.25' })).toBeUndefined(); - expect( - validateDatasetTagValue({ tagType: 'datetime', value: '1704067200000' }) - ).toBeUndefined(); - }); - - it.each(['', ' ', 'abc', 'Infinity'])('rejects invalid numeric string %j', (value) => { - expect(validateDatasetTagValue({ tagType: 'number', value })).toBe( - DatasetErrEnum.tagValueInvalid - ); - expect(validateDatasetTagValue({ tagType: 'datetime', value })).toBe( - DatasetErrEnum.tagValueInvalid - ); - }); - - it('rejects datetime date text', () => { - expect(validateDatasetTagValue({ tagType: 'datetime', value: '2024-01-01' })).toBe( - DatasetErrEnum.tagValueInvalid - ); - }); -}); - -describe('validateAndNormalizeTagValue', () => { - it('normalizes number and datetime string values to numbers', () => { - expect(validateAndNormalizeTagValue({ tagType: 'number', value: '1.25' })).toEqual({ - value: 1.25 - }); - expect(validateAndNormalizeTagValue({ tagType: 'datetime', value: '1704067200000' })).toEqual({ - value: 1704067200000 - }); - expect(validateAndNormalizeTagValue({ tagType: 'number', value: 42 })).toEqual({ value: 42 }); - }); - - it('keeps string and array values unchanged', () => { - expect(validateAndNormalizeTagValue({ tagType: 'string', value: '产品' })).toEqual({ - value: '产品' - }); - expect(validateAndNormalizeTagValue({ tagType: 'array', value: ['安全', '高优'] })).toEqual({ - value: ['安全', '高优'] - }); - }); - - it('surfaces errors for invalid number/datetime strings', () => { - expect(validateAndNormalizeTagValue({ tagType: 'number', value: ' ' })).toEqual({ - value: ' ', - error: DatasetErrEnum.tagValueInvalid - }); - expect(validateAndNormalizeTagValue({ tagType: 'number', value: 'abc' })).toEqual({ - value: 'abc', - error: DatasetErrEnum.tagValueInvalid - }); - expect(validateAndNormalizeTagValue({ tagType: 'datetime', value: '2024-01-01' })).toEqual({ - value: '2024-01-01', - error: DatasetErrEnum.tagValueInvalid - }); - }); - - it('surfaces errors for invalid string/array values', () => { - expect(validateAndNormalizeTagValue({ tagType: 'string', value: 1 as any })).toEqual({ - value: 1, - error: DatasetErrEnum.tagValueInvalid - }); - expect(validateAndNormalizeTagValue({ tagType: 'array', value: ['a'.repeat(257)] })).toEqual({ - value: ['a'.repeat(257)], - error: DatasetErrEnum.arrayTagValueInvalid - }); + ['string', 'value', 'value', undefined], + ['array', ['a'], ['a'], undefined], + ['number', '1.25', 1.25, undefined], + ['datetime', '1704067200000', 1704067200000, undefined], + ['string', 1, 1, DatasetErrEnum.tagValueInvalid], + ['array', ['a'.repeat(257)], ['a'.repeat(257)], DatasetErrEnum.arrayTagValueInvalid], + ['number', 'abc', 'abc', DatasetErrEnum.tagValueInvalid], + ['datetime', Number.MAX_VALUE, Number.MAX_VALUE, DatasetErrEnum.tagValueDatetimeInvalid] + ])('validates and normalizes %s values', (tagType, value, normalized, error) => { + expect(validateAndNormalizeTagValue({ tagType: tagType as any, value: value as any })).toEqual({ + value: normalized, + ...(error ? { error } : {}) + }); + expect(validateDatasetTagValue({ tagType: tagType as any, value: value as any })).toBe(error); }); }); @@ -808,50 +742,21 @@ describe('createOrGetCollectionTags', () => { mockMongoDatasetCollectionTagsCreate.mockResolvedValue([]); }); - it('returns undefined when tags is undefined', async () => { - const result = await createOrGetCollectionTags({ - tags: undefined, - datasetId: 'ds-1', - teamId: 'team-1' - }); - expect(result).toBeUndefined(); - expect(mockMongoDatasetCollectionTagsFind).not.toHaveBeenCalled(); - }); - - it('returns empty array when tags is empty', async () => { - const result = await createOrGetCollectionTags({ - tags: [], - datasetId: 'ds-1', - teamId: 'team-1' - }); - expect(result).toEqual([]); + it('returns without database work when tags are absent or empty', async () => { + await expect( + createOrGetCollectionTags({ tags: undefined, datasetId: 'ds-1', teamId: 'team-1' }) + ).resolves.toBeUndefined(); + await expect( + createOrGetCollectionTags({ tags: [], datasetId: 'ds-1', teamId: 'team-1' }) + ).resolves.toEqual([]); expect(mockMongoDatasetCollectionTagsFind).not.toHaveBeenCalled(); }); - it('resolves existing string tags to default_tag array value', async () => { - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ - _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array' - }); - - const result = await createOrGetCollectionTags({ - tags: ['safety'], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'default-tag-id', value: ['safety'] }]); - }); - - it('creates default_tag for unknown string names', async () => { - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue(null); + it('creates the migration carrier on demand for legacy string names', async () => { mockMongoDatasetCollectionTagsCreate.mockResolvedValue([ { _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array', - toObject: () => ({ _id: 'default-tag-id', tag: 'default_tag', tagType: 'array' }) + toObject: () => ({ _id: 'default-tag-id' }) } ]); @@ -862,354 +767,74 @@ describe('createOrGetCollectionTags', () => { }); expect(result).toEqual([{ tagId: 'default-tag-id', value: ['safety'] }]); - }); - - it('resolves {tag,value} for existing string tag', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) - }); - - const result = await createOrGetCollectionTags({ - tags: [{ tag: 'safety', value: 'A' }], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'tag-id-1', value: 'A' }]); - }); - - it('resolves {tag,value} for existing number tag with number value', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'version', tagType: 'number' }]) - }); - - const result = await createOrGetCollectionTags({ - tags: [{ tag: 'version', value: 2 }], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'tag-id-1', value: 2 }]); - }); - - it('resolves {tag,value} for existing datetime tag with number value', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'date', tagType: 'datetime' }]) - }); - - const result = await createOrGetCollectionTags({ - tags: [{ tag: 'date', value: 1704067200000 }], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'tag-id-1', value: 1704067200000 }]); - }); - - it('rejects when string tag gets non-string value', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) - }); - - await expect( - createOrGetCollectionTags({ - tags: [{ tag: 'safety', value: 123 }], - datasetId: 'ds-1', - teamId: 'team-1' - }) - ).rejects.toBe(DatasetErrEnum.tagValueInvalid); - }); - - it('resolves existing number tag with numeric string value as number', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'version', tagType: 'number' }]) - }); - - const result = await createOrGetCollectionTags({ - tags: [{ tag: 'version', value: '2' }], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'tag-id-1', value: 2 }]); - }); - - it('resolves existing datetime tag with numeric string value as number', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'date', tagType: 'datetime' }]) - }); - - const result = await createOrGetCollectionTags({ - tags: [{ tag: 'date', value: '1704067200000' }], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'tag-id-1', value: 1704067200000 }]); - }); - - it.each([ - ['number', Infinity], - ['number', 'Infinity'], - ['datetime', Infinity], - ['datetime', 'Infinity'], - ['number', ''], - ['number', ' '], - ['datetime', ''], - ['datetime', ' '], - ['number', 'abc'], - ['datetime', 'abc'], - ['datetime', Number.MAX_VALUE] - ])('rejects invalid %s value %j in creation chain', async (tagType, value) => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'tag-1', tagType }]) - }); - - await expect( - createOrGetCollectionTags({ - tags: [{ tag: 'tag-1', value: value as string | number }], - datasetId: 'ds-1', - teamId: 'team-1' - }) - ).rejects.toBe( - tagType === 'datetime' && value === Number.MAX_VALUE - ? DatasetErrEnum.tagValueDatetimeInvalid - : DatasetErrEnum.tagValueInvalid + expect(mockMongoDatasetCollectionTagsCreate).toHaveBeenCalledWith( + [expect.objectContaining({ tag: 'default_tag', fromMigration: true })], + expect.any(Object) ); }); - it('rejects when datetime tag gets non-number value', async () => { + it('handles mixed legacy and typed inputs with normalization and deduplication', async () => { mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'date', tagType: 'datetime' }]) - }); - - await expect( - createOrGetCollectionTags({ - tags: [{ tag: 'date', value: '2024-01-01' }], - datasetId: 'ds-1', - teamId: 'team-1' - }) - ).rejects.toBe(DatasetErrEnum.tagValueInvalid); - }); - - it('handles mixed string and {tag,value} inputs', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-2', tag: 'version', tagType: 'number' }]) - }); - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ - _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array' + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id', tag: 'score', tagType: 'number' }]) }); + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ _id: 'default-tag-id' }); const result = await createOrGetCollectionTags({ - tags: ['safety', { tag: 'version', value: 3 }], + tags: [' legacy ', 'legacy', { tag: ' score ', value: '2' }], datasetId: 'ds-1', teamId: 'team-1' }); expect(result).toEqual([ - { tagId: 'default-tag-id', value: ['safety'] }, - { tagId: 'tag-id-2', value: 3 } + { tagId: 'default-tag-id', value: ['legacy'] }, + { tagId: 'tag-id', value: 2 } ]); }); - it('passes session through to DB calls', async () => { - const session = { sessionId: 'sess-1' } as any; - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ - _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array' - }); - - await createOrGetCollectionTags({ - tags: ['safety'], - datasetId: 'ds-1', - teamId: 'team-1', - session - }); - - expect(mockMongoDatasetCollectionTagsFindOne).toHaveBeenCalledWith( - expect.any(Object), - undefined, - { session } - ); - }); - - it('deduplicates repeated string names', async () => { - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ - _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array' - }); - - const result = await createOrGetCollectionTags({ - tags: ['safety', 'safety'], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'default-tag-id', value: ['safety'] }]); - }); - - it('trims string names and object tag lookup keys', async () => { - const session = { sessionId: 'sess-1' } as any; - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ - _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array' - }); + it('handles an object input named default_tag as a normal typed tag', async () => { mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) - }); - - const result = await createOrGetCollectionTags({ - tags: [' safety ', { tag: ' safety ', value: 'A' }], - datasetId: 'ds-1', - teamId: 'team-1', - session + lean: vi + .fn() + .mockResolvedValue([ + { _id: 'ordinary-default-tag-id', tag: 'default_tag', tagType: 'string' } + ]) }); - // 对象 tag 名称 trim 后按 'safety' 查询；string 名 ' safety ' trim 后进 default_tag - expect(mockMongoDatasetCollectionTagsFind).toHaveBeenCalledWith( - expect.objectContaining({ tag: { $in: ['safety'] } }), - undefined, - { session } - ); - expect(result).toEqual([ - { tagId: 'default-tag-id', value: ['safety'] }, - { tagId: 'tag-id-1', value: 'A' } - ]); - }); - - it('rejects blank string names', async () => { await expect( createOrGetCollectionTags({ - tags: [' '], + tags: [{ tag: 'default_tag', value: 'ordinary value' }], datasetId: 'ds-1', teamId: 'team-1' }) - ).rejects.toBe(DatasetErrEnum.tagNameEmpty); + ).resolves.toEqual([{ tagId: 'ordinary-default-tag-id', value: 'ordinary value' }]); + expect(mockMongoDatasetCollectionTagsFindOne).not.toHaveBeenCalled(); + expect(mockMongoDatasetCollectionTagsCreate).not.toHaveBeenCalled(); }); - it('rejects blank object tag names', async () => { + it('rejects missing tags and conflicting duplicate values', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); await expect( createOrGetCollectionTags({ - tags: [{ tag: ' ', value: 'A' }], + tags: [{ tag: 'missing', value: 'A' }], datasetId: 'ds-1', teamId: 'team-1' }) - ).rejects.toBe(DatasetErrEnum.tagNameEmpty); - }); - - it('deduplicates identical {tag,value} inputs', async () => { - mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) - }); - - const result = await createOrGetCollectionTags({ - tags: [ - { tag: 'safety', value: 'A' }, - { tag: 'safety', value: 'A' } - ], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'tag-id-1', value: 'A' }]); - }); + ).rejects.toBe(DatasetErrEnum.tagNotExist); - it('rejects conflicting values for the same tag', async () => { mockMongoDatasetCollectionTagsFind.mockReturnValue({ - lean: vi.fn().mockResolvedValue([{ _id: 'tag-id-1', tag: 'safety', tagType: 'string' }]) + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id', tag: 'tag', tagType: 'string' }]) }); - await expect( createOrGetCollectionTags({ tags: [ - { tag: 'safety', value: 'A' }, - { tag: 'safety', value: 'B' } + { tag: 'tag', value: 'A' }, + { tag: 'tag', value: 'B' } ], datasetId: 'ds-1', teamId: 'team-1' }) ).rejects.toBe(DatasetErrEnum.tagValueInvalid); }); - - it('merges string names with default_tag array values', async () => { - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ - _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array' - }); - - const result = await createOrGetCollectionTags({ - tags: ['a', { tag: 'default_tag', value: ['b', 'b'] }], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'default-tag-id', value: ['a', 'b'] }]); - }); - - it('supports array value via default_tag object input', async () => { - mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ - _id: 'default-tag-id', - tag: 'default_tag', - tagType: 'array' - }); - - const result = await createOrGetCollectionTags({ - tags: [{ tag: 'default_tag', value: ['x', 'y'] }], - datasetId: 'ds-1', - teamId: 'team-1' - }); - - expect(result).toEqual([{ tagId: 'default-tag-id', value: ['x', 'y'] }]); - }); - - it('rejects non-array value for default_tag object input', async () => { - await expect( - createOrGetCollectionTags({ - tags: [{ tag: 'default_tag', value: 'not-array' }], - datasetId: 'ds-1', - teamId: 'team-1' - }) - ).rejects.toBe(DatasetErrEnum.arrayTagValueInvalid); - }); -}); - -describe('deduplicateTagValues', () => { - it('deduplicates identical values and keeps first occurrence', async () => { - const result = await deduplicateTagValues([ - { tagId: 't1', value: ['a', 'b'] }, - { tagId: 't1', value: ['b', 'a'] } - ]); - expect(result).toEqual([{ tagId: 't1', value: ['a', 'b'] }]); - }); - - it('rejects conflicting values for the same tagId', async () => { - await expect( - deduplicateTagValues([ - { tagId: 't1', value: 'A' }, - { tagId: 't1', value: 'B' } - ]) - ).rejects.toBe(DatasetErrEnum.tagValueInvalid); - }); -}); - -describe('DatasetCollectionItemSchema.tags array values', () => { - it('accepts array values in the detail response tags schema', () => { - const tagsSchema = DatasetCollectionItemSchema.shape.tags; - const result = tagsSchema.parse([ - { tag: 'safety', value: ['a', 'b'] }, - { tag: 'version', value: 2 }, - 'legacy' - ]); - expect(result).toEqual([ - { tag: 'safety', value: ['a', 'b'] }, - { tag: 'version', value: 2 }, - 'legacy' - ]); - }); }); diff --git a/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts b/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts index cd159e260cc9..5f52a9773d79 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DatasetSearchModeEnum, SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; const { countPromptTokensMock, @@ -175,7 +176,11 @@ describe('dispatchAgentDatasetSearch', () => { embeddingWeight: 0.5, usingReRank: false, rerankWeight: 0.5, - datasetSearchUsingExtensionQuery: true + datasetSearchUsingExtensionQuery: true, + collectionFilterMatch: { + logic: 'AND', + conditions: [{ tag: 'price', tagType: 'number', op: '$gte', value: 10 }] + } } as any }); @@ -185,7 +190,11 @@ describe('dispatchAgentDatasetSearch', () => { expect(result.nodeResponse?.datasetQueries).toEqual(['origin']); expect(defaultSearchDatasetDataMock).toHaveBeenCalledWith( expect.objectContaining({ - textQueries: ['origin'] + textQueries: ['origin'], + collectionFilterMatch: JSON.stringify({ + tags: { $and: [{ price: { $gte: 10 } }] } + }), + collectionFilterMode: DatasetTagFilterVersionEnum.structured }) ); expect(result.nodeResponse?.childrenResponses).toEqual([ diff --git a/packages/service/test/core/workflow/dispatch/dataset/search.test.ts b/packages/service/test/core/workflow/dispatch/dataset/search.test.ts index 55b8c2e115f0..727b4914e2dd 100644 --- a/packages/service/test/core/workflow/dispatch/dataset/search.test.ts +++ b/packages/service/test/core/workflow/dispatch/dataset/search.test.ts @@ -100,6 +100,65 @@ describe('dispatchDatasetSearch', () => { ); }); + it('uses explicit versions and sends unconfigured historical nodes to structured filtering', async () => { + defaultSearchDatasetDataMock.mockResolvedValue({ + searchRes: [], + embeddingTokens: 0, + reRankInputTokens: 0, + usingSimilarityFilter: false, + usingReRank: false + }); + const props = { + runningAppInfo: { teamId: 'team_1' }, + runningUserInfo: { tmbId: 'tmb_1' }, + externalProvider: {}, + histories: [], + node: { name: 'Dataset Search' }, + params: { + datasets: [{ datasetId: 'dataset_1' }], + similarity: 0.4, + limit: 5000, + userChatInput: 'query', + searchMode: DatasetSearchModeEnum.embedding, + usingReRank: false, + datasetSearchUsingExtensionQuery: false, + collectionFilterMatch: '{"tags":{"$and":["legacy"]}}' + }, + usagePush: usagePushMock + } as any; + + await dispatchDatasetSearch(props); + await dispatchDatasetSearch({ + ...props, + params: { ...props.params, collectionFilterVersion: 'structured' } + }); + await dispatchDatasetSearch({ + ...props, + params: { ...props.params, collectionFilterVersion: 'unknown' } + }); + await dispatchDatasetSearch({ + ...props, + params: { + ...props.params, + collectionFilterVersion: undefined, + collectionFilterMatch: undefined + } + }); + + expect(defaultSearchDatasetDataMock.mock.calls[0][0]).toMatchObject({ + collectionFilterMode: 'legacy' + }); + expect(defaultSearchDatasetDataMock.mock.calls[1][0]).toMatchObject({ + collectionFilterMode: 'structured' + }); + expect(defaultSearchDatasetDataMock.mock.calls[2][0]).toMatchObject({ + collectionFilterMode: 'legacy' + }); + expect(defaultSearchDatasetDataMock.mock.calls[3][0]).toMatchObject({ + collectionFilterMode: 'structured' + }); + }); + it('adds query extension as a child node response of dataset search', async () => { defaultSearchDatasetDataMock.mockResolvedValue({ searchRes: [ diff --git a/packages/service/test/core/workflow/dispatch/index.test.ts b/packages/service/test/core/workflow/dispatch/index.test.ts index e57a6fba504e..6fdaf9d7e415 100644 --- a/packages/service/test/core/workflow/dispatch/index.test.ts +++ b/packages/service/test/core/workflow/dispatch/index.test.ts @@ -715,6 +715,81 @@ describe('getWorkflowNodeRunParams', () => { expect(params[NodeInputKeyEnum.addInputParam]).toEqual({ dynamicName: 'Ada' }); expect(params.dynamicName).toBe('Ada'); }); + + it('标签过滤条件行会解析行内引用并编成 tags JSON', () => { + const variableState = createVariableState({ price: 18 }); + const node = createNode('search', FlowNodeTypeEnum.datasetSearchNode); + node.inputs = [ + { + key: NodeInputKeyEnum.collectionFilterMatch, + label: '', + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + valueType: WorkflowIOValueTypeEnum.string, + value: { + logic: 'AND', + conditions: [ + { + tag: 'price', + tagType: 'number', + op: '$gte', + valueMode: 'reference', + value: [VARIABLE_NODE_ID, 'price'] + } + ] + } + } + ]; + + const params = getWorkflowNodeRunParams({ + node, + runtimeNodesMap: new Map(), + variableState: variableState.state + }); + + expect(params[NodeInputKeyEnum.collectionFilterMatch]).toBe( + JSON.stringify({ tags: { $and: [{ price: { $gte: 18 } }] } }) + ); + expect(variableState.getToRuntimeRecordCount()).toBe(1); + }); + + it('Agent 嵌套 datasetParams 会解析标签过滤行内引用', () => { + const variableState = createVariableState({ price: 18 }); + const node = createNode('agent', FlowNodeTypeEnum.agent); + node.inputs = [ + { + key: NodeInputKeyEnum.datasetParams, + label: '', + renderTypeList: [FlowNodeInputTypeEnum.hidden], + valueType: WorkflowIOValueTypeEnum.object, + value: { + datasets: [{ datasetId: 'dataset-1' }], + collectionFilterMatch: { + logic: 'AND', + conditions: [ + { + tag: 'price', + tagType: 'number', + op: '$gte', + valueMode: 'reference', + value: [VARIABLE_NODE_ID, 'price'] + } + ] + } + } + } + ]; + + const params = getWorkflowNodeRunParams({ + node, + runtimeNodesMap: new Map(), + variableState: variableState.state + }); + + expect(params[NodeInputKeyEnum.datasetParams]).toMatchObject({ + datasets: [{ datasetId: 'dataset-1' }], + collectionFilterMatch: JSON.stringify({ tags: { $and: [{ price: { $gte: 18 } }] } }) + }); + }); }); describe('runWorkflow catchError', () => { diff --git a/packages/service/test/integrations/vectorDB/README.md b/packages/service/test/integrations/vectorDB/README.md index 1760ecdc3578..44636e1e24bd 100644 --- a/packages/service/test/integrations/vectorDB/README.md +++ b/packages/service/test/integrations/vectorDB/README.md @@ -18,8 +18,6 @@ cp test/.env.example test/.env.test.local | `PG_URL` | PostgreSQL + pgvector 连接串 | PgVectorCtrl | | `OCEANBASE_URL` | Oceanbase 连接串（后续） | ObVectorCtrl | | `MILVUS_ADDRESS` | Milvus 地址（后续） | MilvusCtrl | -| `MILVUS_PERF_COLLECTION_COUNT` | Milvus 召回性能测试的 collection 数，默认 100 | MilvusCtrl | -| `MILVUS_PERF_VECTORS_PER_COLLECTION` | Milvus 召回性能测试每个 collection 的向量数，默认 100 | MilvusCtrl | 未设置对应环境变量时，该驱动的集成测试会**整体跳过**，不会报错。 @@ -35,10 +33,7 @@ pnpm test FASTGPT_TEST_MODE=integration pnpm test # 运行当前这组 vectorDB 集成测试 -pnpm test:vector - -# 只运行 Milvus collectionId 过滤召回性能测试 -cd packages/service && pnpm vitest run -c vitest.integration.config.ts test/integrations/vectorDB/milvus/recallFilterPerformance +FASTGPT_TEST_MODE=integration pnpm test ``` ## 结构说明 @@ -48,15 +43,3 @@ cd packages/service && pnpm vitest run -c vitest.integration.config.ts test/inte - `*/index.integration.test.ts`：各向量库入口，按环境变量决定是否跳过。 新增向量库时：新增一个 `*/index.integration.test.ts`，复用 `testData.ts` 和 `testSuites.ts` 即可。 - -### Milvus collectionId 过滤召回性能测试 - -`milvus/recallFilterPerformance.integration.test.ts` 对应设计文档「端到端检索性能测试」， -范围收窄为只测 Milvus 的向量召回：直接构造 `filterCollectionIdList`（不经过标签过滤/mongo 链路）， -对比 `embRecall` 在「不过滤 / 过滤 10 个 collection / 过滤 100 个 collection」下的召回延迟， -断言过滤延迟与不过滤保持同一数量级（< 10x）。 - -测试在 `beforeAll` 里向 1 个 dataset 下的 100 个 collection 插入向量（默认共 1 万条）， -`flushSync` + `loadCollectionSync` 把数据落到已索引的 sealed segment 后再测，保证延迟接近生产行为。 -数据规模可通过 `MILVUS_PERF_COLLECTION_COUNT` / `MILVUS_PERF_VECTORS_PER_COLLECTION` 覆盖 -（如在大集群上跑到设计文档目标量级）。 diff --git a/packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts b/packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts deleted file mode 100644 index fc8e7a31a4f3..000000000000 --- a/packages/service/test/integrations/vectorDB/milvus/recallFilterPerformance.integration.test.ts +++ /dev/null @@ -1,225 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest'; -import { VECTOR_DIM, QUERY_VECTOR } from '../testData'; - -// 集成测试需用真实 MilvusCtrl，先解除 test/mocks/common/vector.ts 里的全局 mock -vi.unmock('@fastgpt/service/common/vectorDB/milvus'); -vi.unmock('@fastgpt/service/common/vectorDB/constants'); - -import { MilvusCtrl } from '@fastgpt/service/common/vectorDB/milvus'; -import { DatasetVectorTableName } from '@fastgpt/service/common/vectorDB/constants'; - -/** - * Milvus 端到端召回性能测试（collectionId 过滤） - * - * 对应设计文档「端到端检索性能测试」，范围收窄为： - * - 只测 Milvus（不经过标签过滤 / mongo 链路），直接构造 `filterCollectionIdList` - * - 对比 embRecall 在「不过滤 / 过滤 10 个 collection / 过滤 100 个 collection」下的召回延迟 - * - 目标：过滤召回延迟与不过滤保持同一数量级（< 10x） - * - * 运行要求：设置 `MILVUS_ADDRESS`（见 test/.env.example），未设置则整体跳过。 - * 数据规模可通过环境变量覆盖，默认 100 个 collection × 100 条向量 = 1 万条向量： - * MILVUS_PERF_COLLECTION_COUNT 默认 100 - * MILVUS_PERF_VECTORS_PER_COLLECTION 默认 100 - * - * 运行方式：pnpm test:vector（或直接 vitest -c vitest.integration.config.ts 本文件） - */ - -const isEnabled = Boolean(process.env.MILVUS_ADDRESS); - -const COLLECTION_COUNT = Number(process.env.MILVUS_PERF_COLLECTION_COUNT || 100); -const VECTORS_PER_COLLECTION = Number(process.env.MILVUS_PERF_VECTORS_PER_COLLECTION || 100); -const SEARCH_LIMIT = 10; -const MEASURE_ITERATIONS = 20; -const MEASURE_WARMUPS = 3; - -const teamId = `perf_team_${Date.now()}`; -const datasetId = `perf_dataset_${Date.now()}`; -const collectionIds = Array.from({ length: COLLECTION_COUNT }, (_, i) => `perf_col_${i + 1}`); - -// 确定性向量，便于复现；IP 度量下正数向量的打分稳定 -const buildVector = (seed: number) => - Array.from({ length: VECTOR_DIM }, (_, index) => (((index + seed) % 10) + 1) / 100); - -interface MeasureResult { - avg: number; - min: number; - max: number; - median: number; - times: number[]; -} - -// Milvus 查询是网络调用，只统计墙钟延迟；预热 + 多次取平均/中位数 -const measure = async ( - name: string, - fn: () => Promise, - iterations = MEASURE_ITERATIONS, - warmups = MEASURE_WARMUPS -): Promise => { - for (let i = 0; i < warmups; i++) await fn(); - - const times: number[] = []; - for (let i = 0; i < iterations; i++) { - const start = performance.now(); - await fn(); - times.push(performance.now() - start); - } - - const avg = times.reduce((a, b) => a + b, 0) / times.length; - const min = Math.min(...times); - const max = Math.max(...times); - const sorted = [...times].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; - - console.log( - ` ${name.padEnd(22)} 平均: ${avg.toFixed(2).padStart(8)}ms 中位: ${median - .toFixed(2) - .padStart(8)}ms 最小: ${min.toFixed(2).padStart(8)}ms 最大: ${max - .toFixed(2) - .padStart(8)}ms` - ); - return { avg, min, max, median, times }; -}; - -// 并发插入各 collection 的向量。 -// 实测该实例 500 向量/批的 insert RPC 约 5~6s，而 1000 向量/批会超线性变慢（~26s/批）， -// 因此用小批 + 低并发更划算；并发大包曾触发 gRPC RST_STREAM，保持低并发 + 失败退避重试。 -const INSERT_BATCH = 500; -const INSERT_CONCURRENCY = 3; -const INSERT_MAX_RETRY = 5; - -const insertCollections = async (vectorCtrl: MilvusCtrl) => { - // 只存批次元信息，向量在插入前惰性构建，避免一次性物化全部向量导致 OOM - // （1000×1000 向量全量构建约 12GB 堆） - const batches: { collectionIndex: number; collectionId: string; start: number; count: number }[] = - []; - collectionIds.forEach((collectionId, collectionIndex) => { - for (let start = 0; start < VECTORS_PER_COLLECTION; start += INSERT_BATCH) { - batches.push({ - collectionIndex, - collectionId, - start, - count: Math.min(INSERT_BATCH, VECTORS_PER_COLLECTION - start) - }); - } - }); - - let cursor = 0; - const insertWithRetry = async ( - batch: { collectionIndex: number; collectionId: string; start: number; count: number }, - attempt = 0 - ): Promise => { - const vectors = Array.from({ length: batch.count }, (_, i) => - buildVector(batch.collectionIndex * VECTORS_PER_COLLECTION + batch.start + i) - ); - try { - await vectorCtrl.insert({ - teamId, - datasetId, - collectionId: batch.collectionId, - vectors - }); - } catch (error) { - if (attempt < INSERT_MAX_RETRY) { - await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1))); - return insertWithRetry(batch, attempt + 1); - } - throw error; - } - }; - - const workers = Array.from({ length: Math.min(INSERT_CONCURRENCY, batches.length) }, async () => { - while (cursor < batches.length) { - const batch = batches[cursor++]; - await insertWithRetry(batch); - if (cursor % 25 === 0) { - console.log(` [insert] ${cursor}/${batches.length} batches`); - } - } - }); - await Promise.all(workers); -}; - -describe.skipIf(!isEnabled)('Milvus 端到端召回性能（collectionId 过滤）', () => { - const vectorCtrl = new MilvusCtrl(); - - beforeAll(async () => { - await vectorCtrl.init(); - - console.log( - `\n[Milvus 数据准备] ${COLLECTION_COUNT} collections × ${VECTORS_PER_COLLECTION} vectors，开始插入...` - ); - await insertCollections(vectorCtrl); - console.log(' [insert] 插入完成，flush + load...'); - - // flushSync 等待数据落到已索引的 sealed segment，loadCollectionSync 确保可检索， - // 让延迟接近生产行为（不测 growing segment 的全扫描） - const client = await vectorCtrl.getClient(); - await client.flushSync({ collection_names: [DatasetVectorTableName] }); - await client.loadCollectionSync({ collection_name: DatasetVectorTableName }); - console.log(' [flush] 数据已落盘并加载，开始测量\n'); - }, 10_800_000); - - afterAll(async () => { - try { - await vectorCtrl.delete({ teamId, datasetIds: [datasetId] }); - } catch (error) { - // 清理失败不影响结果 - } - }); - - test('不过滤 vs 过滤 10 / 100 个 collection 的召回延迟', async () => { - console.log( - `\n[Milvus collectionId 过滤召回性能] ${COLLECTION_COUNT} collections × ${VECTORS_PER_COLLECTION} vectors/collection，limit=${SEARCH_LIMIT}` - ); - - const base = { - teamId, - datasetIds: [datasetId], - vector: QUERY_VECTOR, - limit: SEARCH_LIMIT, - forbidCollectionIdList: [] as string[] - }; - - // 1. 基线：不过滤 - const baselineResult = await vectorCtrl.embRecall({ ...base }); - expect(baselineResult.results.length).toBeGreaterThan(0); - const baseline = await measure('不过滤 (baseline)', () => vectorCtrl.embRecall({ ...base })); - - // 2. 过滤 10 个 collection（10% 命中） - const filter10Ids = collectionIds.slice(0, 10); - const filter10Result = await vectorCtrl.embRecall({ - ...base, - filterCollectionIdList: filter10Ids - }); - expect(filter10Result.results.length).toBeGreaterThan(0); - expect(filter10Result.results.every((item) => filter10Ids.includes(item.collectionId))).toBe( - true - ); - const filter10 = await measure('过滤 10 个 collection', () => - vectorCtrl.embRecall({ ...base, filterCollectionIdList: filter10Ids }) - ); - - // 3. 过滤 100 个 collection（全部命中） - const filter100Result = await vectorCtrl.embRecall({ - ...base, - filterCollectionIdList: collectionIds - }); - expect(filter100Result.results.length).toBeGreaterThan(0); - expect(filter100Result.results.every((item) => collectionIds.includes(item.collectionId))).toBe( - true - ); - const filter100 = await measure('过滤 100 个 collection', () => - vectorCtrl.embRecall({ ...base, filterCollectionIdList: collectionIds }) - ); - - // 目标：过滤延迟与不过滤保持同一数量级（< 10x），floor 50ms 避免基线过快时的抖动误判 - console.log( - `\n [对比] 过滤10/不过滤: ${(filter10.avg / baseline.avg).toFixed(2)}x | 过滤100/不过滤: ${( - filter100.avg / baseline.avg - ).toFixed(2)}x` - ); - expect(filter10.avg).toBeLessThan(Math.max(baseline.avg * 10, 50)); - expect(filter100.avg).toBeLessThan(Math.max(baseline.avg * 10, 50)); - }, 300_000); -}); diff --git a/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx index b54897427e12..75301123eaf4 100644 --- a/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx +++ b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx @@ -14,21 +14,33 @@ import { } from '@chakra-ui/react'; import { DayPicker } from 'react-day-picker'; import 'react-day-picker/dist/style.css'; +import { enUS } from 'date-fns/locale/en-US'; +import { ko } from 'date-fns/locale/ko'; import { zhCN } from 'date-fns/locale/zh-CN'; +import { zhTW } from 'date-fns/locale/zh-TW'; import type { Locale } from 'date-fns'; import { addMonths, format, isValid, parse } from 'date-fns'; import { useTranslation } from 'next-i18next'; +import { LangEnum } from '@fastgpt/global/common/i18n/type'; import MyIcon from '../Icon'; const DATE_INPUT_FORMAT = 'yyyy-MM-dd'; const TIME_INPUT_FORMAT = 'HH:mm'; -export type SingleDateTimePickerProps = { +const DATE_FNS_LOCALE_BY_LANG: Record = { + [LangEnum.zh_CN]: zhCN, + [LangEnum.zh_Hant]: zhTW, + [LangEnum.en]: enUS, + [LangEnum.ko_KR]: ko +}; + +type SingleDateTimePickerProps = { value?: Date | number; onChange?: (val: number) => void; placeholder?: string; isDisabled?: boolean; locale?: Locale; + hideCalendarIcon?: boolean; } & Omit; /** 将时间戳或 Date 转为有效 Date；空值或非法日期返回 undefined。 */ @@ -54,10 +66,13 @@ export const SingleDateTimePicker = ({ onChange, placeholder, isDisabled, - locale = zhCN, + locale, + hideCalendarIcon, ...triggerProps }: SingleDateTimePickerProps) => { - const { t } = useTranslation(); + const { t, i18n } = useTranslation(); + const resolvedLocale = locale ?? DATE_FNS_LOCALE_BY_LANG[i18n.language] ?? zhCN; + const weekStartsOn = resolvedLocale.options?.weekStartsOn ?? 0; const { isOpen, onOpen, onClose } = useDisclosure(); const selectedDate = toValidDate(value); @@ -126,13 +141,15 @@ export const SingleDateTimePicker = ({ {displayText || placeholder || t('common:datetime_picker.placeholder')} - + {!hideCalendarIcon && ( + + )} @@ -182,7 +199,7 @@ export const SingleDateTimePicker = ({ - {format(tempMonth, 'LLLL yyyy', { locale })} + {format(tempMonth, 'LLLL yyyy', { locale: resolvedLocale })} <> }} formatters={{ - formatWeekdayName: (date) => format(date, 'EEEEEE', { locale }) + formatWeekdayName: (date) => format(date, 'EEEEEE', { locale: resolvedLocale }) }} /> diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 9ba4f7ab53ee..1d4fb6dc127c 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -463,7 +463,10 @@ export const iconPaths = { loading: () => import('./icons/loading.svg'), 'math/divide': () => import('./icons/math/divide.svg'), 'math/equal': () => import('./icons/math/equal.svg'), + 'math/greater': () => import('./icons/math/greater.svg'), + 'math/greaterEqual': () => import('./icons/math/greaterEqual.svg'), 'math/minus': () => import('./icons/math/minus.svg'), + 'math/notEqual': () => import('./icons/math/notEqual.svg'), 'math/plus': () => import('./icons/math/plus.svg'), 'math/times': () => import('./icons/math/times.svg'), mcp: () => import('./icons/mcp.svg'), diff --git a/packages/web/components/common/Icon/icons/math/greater.svg b/packages/web/components/common/Icon/icons/math/greater.svg new file mode 100644 index 000000000000..9e176a8e7e00 --- /dev/null +++ b/packages/web/components/common/Icon/icons/math/greater.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/web/components/common/Icon/icons/math/greaterEqual.svg b/packages/web/components/common/Icon/icons/math/greaterEqual.svg new file mode 100644 index 000000000000..5526d2a1a441 --- /dev/null +++ b/packages/web/components/common/Icon/icons/math/greaterEqual.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/web/components/common/Icon/icons/math/notEqual.svg b/packages/web/components/common/Icon/icons/math/notEqual.svg new file mode 100644 index 000000000000..3e83cba5c6e9 --- /dev/null +++ b/packages/web/components/common/Icon/icons/math/notEqual.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/web/components/common/TagFilter/FilterButton.tsx b/packages/web/components/common/TagFilter/FilterButton.tsx index 2b07db5e2c01..ac382ad5c970 100644 --- a/packages/web/components/common/TagFilter/FilterButton.tsx +++ b/packages/web/components/common/TagFilter/FilterButton.tsx @@ -44,7 +44,7 @@ export const FilterSummaryValue = ({ ); -export type FilterButtonProps = Omit & { +type FilterButtonProps = Omit & { title: ReactNode; value: ReactNode; }; diff --git a/packages/web/components/common/TagFilter/MultiTagFilter.tsx b/packages/web/components/common/TagFilter/MultiTagFilter.tsx index 4dbcd28165c2..23001fe5e649 100644 --- a/packages/web/components/common/TagFilter/MultiTagFilter.tsx +++ b/packages/web/components/common/TagFilter/MultiTagFilter.tsx @@ -1,13 +1,12 @@ import React, { useMemo, useState, type ReactNode } from 'react'; import { Box, Button, Checkbox, Flex, Input, type PlacementWithLogical } from '@chakra-ui/react'; -import { collectionTagValueKey } from '@fastgpt/global/core/dataset/tagUtils'; import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; import MyIcon from '../Icon'; import MyBox from '../MyBox'; import MyPopover from '../MyPopover'; import FilterButton from './FilterButton'; -export type MultiTagFilterValue = string | number; +type MultiTagFilterValue = string | number; export type MultiTagFilterGroup = { tagId: string; @@ -18,7 +17,7 @@ export type MultiTagFilterGroup = { }>; }; -export type MultiTagFilterLabels = { +type MultiTagFilterLabels = { title: ReactNode; all: ReactNode; searchPlaceholder: string; @@ -29,7 +28,7 @@ export type MultiTagFilterLabels = { noMatch: ReactNode; }; -export type MultiTagFilterProps = { +type MultiTagFilterProps = { groups: MultiTagFilterGroup[]; selected: CollectionTagFilterItem[]; onSelectedChange: (next: CollectionTagFilterItem[]) => void; @@ -57,7 +56,7 @@ const stopWheelPropagation = (e: React.WheelEvent) => e.stopPropagation(); /** * 双栏标签值筛选：左侧分组、右侧勾选值即生效。同一标签多值为 OR，不同标签由调用方按 AND 解释。 */ -export const toggleMultiTagFilterValue = ( +const toggleMultiTagFilterValue = ( selected: CollectionTagFilterItem[], tagId: string, value: MultiTagFilterValue @@ -123,7 +122,7 @@ const MultiTagFilter = ({ const firstValue = firstSelected.values[0]; const firstValueLabel = firstGroup?.values.find((item) => item.value === firstValue)?.label ?? String(firstValue); - const firstText = firstGroup ? `${firstGroup.label}：${firstValueLabel}` : firstValueLabel; + const firstText = firstGroup ? `${firstGroup.label}: ${firstValueLabel}` : firstValueLabel; const extraCount = selectedCount - 1; return ( @@ -300,7 +299,7 @@ const MultiTagFilter = ({ const checked = isValueChecked(resolvedActiveTagId, item.value); return ( {renderTypeData.title} diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index 01ae6429f841..1b06699b68d2 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -464,13 +464,7 @@ "core.dataset.error.tagNameEmpty": "Tag name cannot be empty", "core.dataset.error.tagNotExist": "Tag does not exist", "core.dataset.error.tagValueInvalid": "Tag value format does not match type", - "core.dataset.error.tagValueStringTooLong": "String tag value exceeds maximum length", - "core.dataset.error.tagValueNumberOutOfRange": "Number tag value out of safe integer range", "core.dataset.error.tagValueDatetimeInvalid": "Datetime tag value format is invalid", - "core.dataset.error.noDatasetForTagFilter": "No dataset selected, cannot configure tag filter", - "core.dataset.error.noTagsInDataset": "No available tags in the current dataset", - "core.dataset.error.noPermissionForDatasetTags": "No permission to access dataset tags", - "core.dataset.error.tagNotSelectedForRef": "Please select a tag before referencing a variable", "core.dataset.error.arrayTagValueInvalid": "Array tag value invalid (elements must be string, array length ≤ 64, element length ≤ 256)", "core.dataset.externalFile": "External File Library", "core.dataset.file": "File", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 436681e2adcc..432b750ae775 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -33,16 +33,11 @@ "collection_sync": "Sync data", "collection_sync_confirm_tip": "Start syncing data? FastGPT compares the latest data and, when content has changed, creates a new Collection and deletes the old one.", "collection_tags": "Collection Tags", - "core.dataset.tags.batchUpsert": "Batch Manage Tags", - "core.dataset.tags.datetime": "Datetime", "core.dataset.tags.array": "Options", "core.dataset.tags.date": "Date", "core.dataset.tags.number": "Number", - "core.dataset.tags.setTags": "Set Tags", "core.dataset.tags.string": "String", "core.dataset.tags.tagType": "Tag Type", - "core.dataset.tags.time": "Time", - "core.dataset.tags.tagValue": "Tag Value", "common.error.unKnow": "Unknown error", "common_dataset": "General Dataset", "common_dataset_desc": "Build a Dataset from files, web pages, or manually entered content", diff --git a/packages/web/i18n/en/system_migration.json b/packages/web/i18n/en/system_migration.json index 72d2483a1bec..78ae6f1d1666 100644 --- a/packages/web/i18n/en/system_migration.json +++ b/packages/web/i18n/en/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "Checking dataset owner permissions", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "Checking agent skill owner permissions", "migrations.20260905_backfill_resource_owner_acl.validation": "Validating resource owner permissions", - "migrations.20260905_backfill_resource_owner_acl.result": "Checked {{appsProcessedCount}} apps, {{datasetsProcessedCount}} datasets, and {{agentSkillsProcessedCount}} skills. Backfilled owner permissions for {{appsUpdatedCount}} apps, {{datasetsUpdatedCount}} datasets, and {{agentSkillsUpdatedCount}} skills." + "migrations.20260905_backfill_resource_owner_acl.result": "Checked {{appsProcessedCount}} apps, {{datasetsProcessedCount}} datasets, and {{agentSkillsProcessedCount}} skills. Backfilled owner permissions for {{appsUpdatedCount}} apps, {{datasetsUpdatedCount}} datasets, and {{agentSkillsUpdatedCount}} skills.", + "migrations.20260907_migrate_dataset_tags_v2.name": "Upgrade dataset tag data", + "migrations.20260907_migrate_dataset_tags_v2.description": "Migrates legacy collection tags to typed tags and cleans duplicate tag definitions.", + "migrations.20260907_migrate_dataset_tags_v2.result": "Checked {{datasetsProcessedCount}} datasets, migrated {{collectionsMigratedCount}} collections, and removed {{duplicateDefinitionsDeletedCount}} duplicate tag definitions.", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "Normalize dataset tag definitions", + "migrations.20260907_migrate_dataset_tags_v2.collections": "Migrate collection tag values", + "migrations.20260907_migrate_dataset_tags_v2.validation": "Validate tag migration" } diff --git a/packages/web/i18n/en/workflow.json b/packages/web/i18n/en/workflow.json index 0dc240393ad2..ef9fb58d379d 100644 --- a/packages/web/i18n/en/workflow.json +++ b/packages/web/i18n/en/workflow.json @@ -39,6 +39,33 @@ "code_execution": "Code run", "code_sandbox_intro": "Run a script in the Sandbox for complex data processing. Available syntax and dependencies are limited.", "collection_metadata_filter": "Collection Metadata Filter", + "tag_filter": "Tag filter", + "tag_filter_add_condition": "Add filter", + "tag_filter_description": "Currently supports filtering by tags, creation time, and collection IDs to precisely locate matching files and improve retrieval accuracy and efficiency. For example, if a user asks “How long is the validity period?” and the question is known to be about product A on an order, metadata filtering can lock onto files related to product A and avoid searching other files that also mention “validity period.”", + "tag_filter_file_attrs": "File attributes", + "tag_filter_file_tags": "File tags", + "tag_filter_input_value": "Enter a value", + "tag_filter_op_after": "After", + "tag_filter_op_before": "Before", + "tag_filter_op_empty": "Is empty", + "tag_filter_op_eq": "Equals", + "tag_filter_op_gt": "Greater than", + "tag_filter_op_gte": "Greater than or equal to", + "tag_filter_op_in": "Is in", + "tag_filter_op_is": "Is", + "tag_filter_op_is_not": "Is not", + "tag_filter_op_lt": "Less than", + "tag_filter_op_lte": "Less than or equal to", + "tag_filter_op_ne": "Not equal to", + "tag_filter_op_not_empty": "Is not empty", + "tag_filter_op_not_in": "Is not in", + "tag_filter_select_condition": "Select condition", + "tag_filter_select_option": "Select options", + "tag_filter_select_tag": "Select tag", + "tag_filter_select_time": "Select time", + "tag_filter_upgrade_content": "You can’t switch back after upgrading, and the current filter settings will be cleared. (Copy this node first if you want a backup.)", + "tag_filter_upgrade_cta": "Deprecated. Upgrade to the latest version", + "tag_filter_upgrade_title": "Upgrade to the new version?", "complete_extraction_result": "Complete Extraction Result", "complete_extraction_result_description": "A JSON string, e.g., {\"name\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "Concatenation Result", diff --git a/packages/web/i18n/ko-KR/dataset.json b/packages/web/i18n/ko-KR/dataset.json index a3c289e748c9..85b9ea41283e 100644 --- a/packages/web/i18n/ko-KR/dataset.json +++ b/packages/web/i18n/ko-KR/dataset.json @@ -34,12 +34,10 @@ "collection_sync_confirm_tip": "데이터 동기화를 시작하시겠습니까? 시스템이 최신 데이터를 가져와 비교합니다. 내용이 다르면 새 컬렉션이 생성되고 기존 컬렉션은 삭제됩니다. 확인해 주세요!", "collection_tags": "컬렉션 태그", "core.dataset.tags.array": "옵션", - "core.dataset.tags.datetime": "날짜 및 시간", "core.dataset.tags.number": "숫자", "core.dataset.tags.string": "텍스트", "core.dataset.tags.date": "날짜", "core.dataset.tags.tagType": "태그 유형", - "core.dataset.tags.time": "시간", "common.error.unKnow": "알 수 없는 오류", "common_dataset": "일반 데이터셋", "common_dataset_desc": "파일, 웹 링크 또는 수동 입력을 통해 데이터셋을 구축합니다", diff --git a/packages/web/i18n/ko-KR/system_migration.json b/packages/web/i18n/ko-KR/system_migration.json index c6431a94c98e..bd2bf1cfa583 100644 --- a/packages/web/i18n/ko-KR/system_migration.json +++ b/packages/web/i18n/ko-KR/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "지식 베이스 소유자 권한 확인 중", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "Agent Skill 소유자 권한 확인 중", "migrations.20260905_backfill_resource_owner_acl.validation": "리소스 소유자 권한 검증 중", - "migrations.20260905_backfill_resource_owner_acl.result": "앱 {{appsProcessedCount}}개, 지식 베이스 {{datasetsProcessedCount}}개, 스킬 {{agentSkillsProcessedCount}}개를 확인했습니다. 소유자 권한 보완: 앱 {{appsUpdatedCount}}개, 지식 베이스 {{datasetsUpdatedCount}}개, 스킬 {{agentSkillsUpdatedCount}}개." + "migrations.20260905_backfill_resource_owner_acl.result": "앱 {{appsProcessedCount}}개, 지식 베이스 {{datasetsProcessedCount}}개, 스킬 {{agentSkillsProcessedCount}}개를 확인했습니다. 소유자 권한 보완: 앱 {{appsUpdatedCount}}개, 지식 베이스 {{datasetsUpdatedCount}}개, 스킬 {{agentSkillsUpdatedCount}}개.", + "migrations.20260907_migrate_dataset_tags_v2.name": "지식 베이스 태그 데이터 업그레이드", + "migrations.20260907_migrate_dataset_tags_v2.description": "기존 Collection 태그를 유형화된 태그로 마이그레이션하고 중복 태그 정의를 정리합니다.", + "migrations.20260907_migrate_dataset_tags_v2.result": "지식 베이스 {{datasetsProcessedCount}}개를 확인하고 Collection {{collectionsMigratedCount}}개를 마이그레이션했으며, 중복 태그 정의 {{duplicateDefinitionsDeletedCount}}개를 삭제했습니다.", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "지식 베이스 태그 정의 정리", + "migrations.20260907_migrate_dataset_tags_v2.collections": "Collection 태그 값 마이그레이션", + "migrations.20260907_migrate_dataset_tags_v2.validation": "태그 마이그레이션 검증" } diff --git a/packages/web/i18n/ko-KR/workflow.json b/packages/web/i18n/ko-KR/workflow.json index f84c69ef5dde..555828363c36 100644 --- a/packages/web/i18n/ko-KR/workflow.json +++ b/packages/web/i18n/ko-KR/workflow.json @@ -39,6 +39,33 @@ "code_execution": "코드 샌드박스", "code_sandbox_intro": "샌드박스에서 스크립트 코드를 실행하면 복잡한 데이터 처리를 수행할 수 있지만, 문법과 사용 가능한 의존성에는 제한이 있습니다.", "collection_metadata_filter": "컬렉션 메타데이터 필터", + "tag_filter": "태그 필터", + "tag_filter_add_condition": "필터 조건 추가", + "tag_filter_description": "현재 태그, 생성 시간, 컬렉션 ID 필터를 지원하며, 해당 파일을 정확하게 찾아 검색 정확도와 효율을 높일 수 있습니다. 예: 사용자가 “유효기간은 얼마인가요?”라고 묻고, 해당 질문이 주문의 제품 A에 대한 것임이 확인되면 메타데이터 필터로 제품 A 관련 파일만 잠가, 다른 “유효기간”이 포함된 파일에서 검색하는 것을 피할 수 있습니다.", + "tag_filter_file_attrs": "파일 속성", + "tag_filter_file_tags": "파일 태그", + "tag_filter_input_value": "값 입력", + "tag_filter_op_after": "이후", + "tag_filter_op_before": "이전", + "tag_filter_op_empty": "비어 있음", + "tag_filter_op_eq": "같음", + "tag_filter_op_gt": "보다 큼", + "tag_filter_op_gte": "크거나 같음", + "tag_filter_op_in": "포함됨", + "tag_filter_op_is": "일치", + "tag_filter_op_is_not": "불일치", + "tag_filter_op_lt": "보다 작음", + "tag_filter_op_lte": "작거나 같음", + "tag_filter_op_ne": "같지 않음", + "tag_filter_op_not_empty": "비어 있지 않음", + "tag_filter_op_not_in": "포함되지 않음", + "tag_filter_select_condition": "조건 선택", + "tag_filter_select_option": "옵션을 선택하세요", + "tag_filter_select_tag": "태그 선택", + "tag_filter_select_time": "시간 선택", + "tag_filter_upgrade_content": "전환 후에는 이전 버전으로 돌아갈 수 없으며 현재 필터 설정이 삭제됩니다. (백업이 필요하면 현재 노드를 복사하세요.)", + "tag_filter_upgrade_cta": "더 이상 사용되지 않습니다. 최신 버전으로 업그레이드", + "tag_filter_upgrade_title": "새 버전으로 업그레이드할까요?", "complete_extraction_result": "전체 추출 결과", "complete_extraction_result_description": "JSON 문자열, 예: {\"name\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "연결 결과", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index c28f0220a869..6e4c1b441314 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -464,13 +464,7 @@ "core.dataset.error.tagNameEmpty": "标签名称不能为空", "core.dataset.error.tagNotExist": "标签不存在", "core.dataset.error.tagValueInvalid": "标签值格式不符合类型要求", - "core.dataset.error.tagValueStringTooLong": "字符串标签值超出最大长度限制", - "core.dataset.error.tagValueNumberOutOfRange": "数字标签值超出安全整数范围", "core.dataset.error.tagValueDatetimeInvalid": "日期时间标签值格式无效", - "core.dataset.error.noDatasetForTagFilter": "未选择知识库，无法配置标签过滤", - "core.dataset.error.noTagsInDataset": "当前知识库没有可用标签", - "core.dataset.error.noPermissionForDatasetTags": "无权限获取知识库的标签", - "core.dataset.error.tagNotSelectedForRef": "请先选择标签后再引用变量", "core.dataset.error.arrayTagValueInvalid": "数组标签值格式无效（元素须为字符串，数组长度不超过 64，单元素不超过 256 字符）", "core.dataset.externalFile": "外部文件库", "core.dataset.file": "文件", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index b34c83cd6640..11ad0889df51 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -33,16 +33,11 @@ "collection_sync": "立即同步", "collection_sync_confirm_tip": "确认开始同步数据？系统将会拉取最新数据进行比较，如果内容不相同，则会创建一个新的集合并删除旧的集合，请确认！", "collection_tags": "集合标签", - "core.dataset.tags.batchUpsert": "批量管理标签", - "core.dataset.tags.datetime": "日期时间", "core.dataset.tags.array": "选项", "core.dataset.tags.date": "日期", "core.dataset.tags.number": "数字", - "core.dataset.tags.setTags": "设置标签值", "core.dataset.tags.string": "文本", "core.dataset.tags.tagType": "标签类型", - "core.dataset.tags.time": "时间", - "core.dataset.tags.tagValue": "标签值", "common.error.unKnow": "未知错误", "common_dataset": "通用知识库", "common_dataset_desc": "通过导入文件、网页链接或手动录入形式构建知识库", diff --git a/packages/web/i18n/zh-CN/system_migration.json b/packages/web/i18n/zh-CN/system_migration.json index 381d3ef399e6..f618174124b4 100644 --- a/packages/web/i18n/zh-CN/system_migration.json +++ b/packages/web/i18n/zh-CN/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "检查知识库所有者权限", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "检查 Agent Skill 所有者权限", "migrations.20260905_backfill_resource_owner_acl.validation": "校验资源所有者权限", - "migrations.20260905_backfill_resource_owner_acl.result": "已检查 {{appsProcessedCount}} 个应用、{{datasetsProcessedCount}} 个知识库和 {{agentSkillsProcessedCount}} 个技能。补齐所有者权限：{{appsUpdatedCount}} 个应用、{{datasetsUpdatedCount}} 个知识库和 {{agentSkillsUpdatedCount}} 个技能。" + "migrations.20260905_backfill_resource_owner_acl.result": "已检查 {{appsProcessedCount}} 个应用、{{datasetsProcessedCount}} 个知识库和 {{agentSkillsProcessedCount}} 个技能。补齐所有者权限：{{appsUpdatedCount}} 个应用、{{datasetsUpdatedCount}} 个知识库和 {{agentSkillsUpdatedCount}} 个技能。", + "migrations.20260907_migrate_dataset_tags_v2.name": "升级知识库标签数据", + "migrations.20260907_migrate_dataset_tags_v2.description": "将历史 Collection 标签迁移为类型化标签，并清理重复标签定义。", + "migrations.20260907_migrate_dataset_tags_v2.result": "已检查 {{datasetsProcessedCount}} 个知识库，迁移 {{collectionsMigratedCount}} 个 Collection，删除 {{duplicateDefinitionsDeletedCount}} 个重复标签定义。", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "整理知识库标签定义", + "migrations.20260907_migrate_dataset_tags_v2.collections": "迁移 Collection 标签值", + "migrations.20260907_migrate_dataset_tags_v2.validation": "校验标签迁移结果" } diff --git a/packages/web/i18n/zh-CN/workflow.json b/packages/web/i18n/zh-CN/workflow.json index fa5a06de458d..fb2c22773dd2 100644 --- a/packages/web/i18n/zh-CN/workflow.json +++ b/packages/web/i18n/zh-CN/workflow.json @@ -39,6 +39,33 @@ "code_execution": "代码运行", "code_sandbox_intro": "在沙盒里执行一段脚本代码，可用于进行复杂的数据处理，语法和可用依赖会受到限制。", "collection_metadata_filter": "集合元数据过滤", + "tag_filter": "标签过滤", + "tag_filter_add_condition": "添加过滤条件", + "tag_filter_description": "目前支持标签、创建时间和集合ID过滤，可精确查找对应文件，进而提升检索准确率和效率。如:用户提问“有效期是多久?”，已确定用户询问订单对应产品A，通过元数据过滤，可锁定产品A相关文件，避免在其他含有“有效期”的文件中检索。", + "tag_filter_file_attrs": "文件属性", + "tag_filter_file_tags": "文件标签", + "tag_filter_input_value": "输入值", + "tag_filter_op_after": "晚于", + "tag_filter_op_before": "早于", + "tag_filter_op_empty": "为空", + "tag_filter_op_eq": "等于", + "tag_filter_op_gt": "大于", + "tag_filter_op_gte": "大于等于", + "tag_filter_op_in": "属于", + "tag_filter_op_is": "是", + "tag_filter_op_is_not": "不是", + "tag_filter_op_lt": "小于", + "tag_filter_op_lte": "小于等于", + "tag_filter_op_ne": "不等于", + "tag_filter_op_not_empty": "不为空", + "tag_filter_op_not_in": "不属于", + "tag_filter_select_condition": "选择条件", + "tag_filter_select_option": "请选择选项", + "tag_filter_select_tag": "选择标签", + "tag_filter_select_time": "请选择时间", + "tag_filter_upgrade_content": "切换后将无法返回旧版，且当前过滤设置将被清除。（建议复制当前节点作为备份）", + "tag_filter_upgrade_cta": "已弃用，升级到最新版本", + "tag_filter_upgrade_title": "确认升级到新版？", "complete_extraction_result": "完整提取结果", "complete_extraction_result_description": "一个 JSON 字符串，例如：{\"name:\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "拼接结果", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index 2cc3cf128f18..45e94f5196e4 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -464,13 +464,7 @@ "core.dataset.error.tagNameEmpty": "標籤名稱不能為空", "core.dataset.error.tagNotExist": "標籤不存在", "core.dataset.error.tagValueInvalid": "標籤值格式不符合類型要求", - "core.dataset.error.tagValueStringTooLong": "字串標籤值超出最大長度限制", - "core.dataset.error.tagValueNumberOutOfRange": "數字標籤值超出安全整數範圍", "core.dataset.error.tagValueDatetimeInvalid": "日期時間標籤值格式無效", - "core.dataset.error.noDatasetForTagFilter": "未選擇知識庫，無法配置標籤過濾", - "core.dataset.error.noTagsInDataset": "當前知識庫沒有可用標籤", - "core.dataset.error.noPermissionForDatasetTags": "無權限獲取知識庫的標籤", - "core.dataset.error.tagNotSelectedForRef": "請先選擇標籤後再引用變數", "core.dataset.error.arrayTagValueInvalid": "陣列標籤值格式無效（元素須為字串，陣列長度不超過 64，單元素不超過 256 字元）", "core.dataset.externalFile": "外部檔案庫", "core.dataset.file": "檔案", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index cabdb9e7db19..4638fbbe8284 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -33,16 +33,11 @@ "collection_sync": "立即同步", "collection_sync_confirm_tip": "確定要開始同步資料嗎？系統會比對最新資料；若內容不同，將建立新集合並刪除舊集合。", "collection_tags": "集合標籤", - "core.dataset.tags.batchUpsert": "批量管理標籤", - "core.dataset.tags.datetime": "日期時間", "core.dataset.tags.array": "選項", "core.dataset.tags.date": "日期", "core.dataset.tags.number": "數字", - "core.dataset.tags.setTags": "設定標籤值", "core.dataset.tags.string": "文字", "core.dataset.tags.tagType": "標籤類型", - "core.dataset.tags.time": "時間", - "core.dataset.tags.tagValue": "標籤值", "common.error.unKnow": "未知錯誤", "common_dataset": "通用資料集", "common_dataset_desc": "通過導入文件、網頁鏈接或手動錄入形式構建知識庫", diff --git a/packages/web/i18n/zh-Hant/system_migration.json b/packages/web/i18n/zh-Hant/system_migration.json index d50a9449fafd..69ce49ec7f87 100644 --- a/packages/web/i18n/zh-Hant/system_migration.json +++ b/packages/web/i18n/zh-Hant/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "檢查知識庫擁有者權限", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "檢查 Agent Skill 擁有者權限", "migrations.20260905_backfill_resource_owner_acl.validation": "驗證資源擁有者權限", - "migrations.20260905_backfill_resource_owner_acl.result": "已檢查 {{appsProcessedCount}} 個應用、{{datasetsProcessedCount}} 個知識庫和 {{agentSkillsProcessedCount}} 個技能。補齊擁有者權限：{{appsUpdatedCount}} 個應用、{{datasetsUpdatedCount}} 個知識庫和 {{agentSkillsUpdatedCount}} 個技能。" + "migrations.20260905_backfill_resource_owner_acl.result": "已檢查 {{appsProcessedCount}} 個應用、{{datasetsProcessedCount}} 個知識庫和 {{agentSkillsProcessedCount}} 個技能。補齊擁有者權限：{{appsUpdatedCount}} 個應用、{{datasetsUpdatedCount}} 個知識庫和 {{agentSkillsUpdatedCount}} 個技能。", + "migrations.20260907_migrate_dataset_tags_v2.name": "升級知識庫標籤資料", + "migrations.20260907_migrate_dataset_tags_v2.description": "將歷史 Collection 標籤遷移為類型化標籤，並清理重複標籤定義。", + "migrations.20260907_migrate_dataset_tags_v2.result": "已檢查 {{datasetsProcessedCount}} 個知識庫，遷移 {{collectionsMigratedCount}} 個 Collection，刪除 {{duplicateDefinitionsDeletedCount}} 個重複標籤定義。", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "整理知識庫標籤定義", + "migrations.20260907_migrate_dataset_tags_v2.collections": "遷移 Collection 標籤值", + "migrations.20260907_migrate_dataset_tags_v2.validation": "驗證標籤遷移結果" } diff --git a/packages/web/i18n/zh-Hant/workflow.json b/packages/web/i18n/zh-Hant/workflow.json index 3fd1792641be..9a8231130f12 100644 --- a/packages/web/i18n/zh-Hant/workflow.json +++ b/packages/web/i18n/zh-Hant/workflow.json @@ -39,6 +39,33 @@ "code_execution": "程式碼執行", "code_sandbox_intro": "在沙盒裡執行一段腳本程式碼，可用於進行複雜的資料處理，語法和可用依賴會受到限制。", "collection_metadata_filter": "資料集詮釋資料篩選器", + "tag_filter": "標籤過濾", + "tag_filter_add_condition": "新增過濾條件", + "tag_filter_description": "目前支援標籤、建立時間和集合ID過濾，可精確查找對應檔案，進而提升檢索準確率和效率。如：使用者提問「有效期是多久？」，已確定使用者詢問訂單對應產品A，透過詮釋資料過濾，可鎖定產品A相關檔案，避免在其他含有「有效期」的檔案中檢索。", + "tag_filter_file_attrs": "檔案屬性", + "tag_filter_file_tags": "檔案標籤", + "tag_filter_input_value": "輸入值", + "tag_filter_op_after": "晚於", + "tag_filter_op_before": "早於", + "tag_filter_op_empty": "為空", + "tag_filter_op_eq": "等於", + "tag_filter_op_gt": "大於", + "tag_filter_op_gte": "大於等於", + "tag_filter_op_in": "屬於", + "tag_filter_op_is": "是", + "tag_filter_op_is_not": "不是", + "tag_filter_op_lt": "小於", + "tag_filter_op_lte": "小於等於", + "tag_filter_op_ne": "不等於", + "tag_filter_op_not_empty": "不為空", + "tag_filter_op_not_in": "不屬於", + "tag_filter_select_condition": "選擇條件", + "tag_filter_select_option": "請選擇選項", + "tag_filter_select_tag": "選擇標籤", + "tag_filter_select_time": "請選擇時間", + "tag_filter_upgrade_content": "切換後將無法返回舊版，且目前過濾設定將被清除。（建議複製目前節點作為備份）", + "tag_filter_upgrade_cta": "已棄用，升級到最新版本", + "tag_filter_upgrade_title": "確認升級到新版？", "complete_extraction_result": "完整擷取結果", "complete_extraction_result_description": "一個 JSON 字串，例如：{\"name\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "串接結果", diff --git a/pro b/pro index 5f063bf4e3f5..92078141bdd2 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit 5f063bf4e3f56eb8dd4d5f7f22154f4e8e54b203 +Subproject commit 92078141bdd27042e5fa99a0e37fc0c902a40e06 diff --git a/projects/app/src/components/core/dataset/DatasetTagFilterRows.tsx b/projects/app/src/components/core/dataset/DatasetTagFilterRows.tsx new file mode 100644 index 000000000000..3b7466b02185 --- /dev/null +++ b/projects/app/src/components/core/dataset/DatasetTagFilterRows.tsx @@ -0,0 +1,487 @@ +import React, { useEffect, useMemo } from 'react'; +import { Box, Button, Flex, HStack, Input } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyIconButton from '@fastgpt/web/components/common/Icon/button'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { + createEmptyTagFilterCondition, + createEmptyTagFilterValue, + DatasetTagFilterLogicEnum, + DatasetTagFilterValueModeEnum, + DatasetTagFilterFieldEnum, + getTagFilterOpsByCondition, + intersectWorkflowTagOptions, + isDatasetTagFilterValue, + isTagFilterOpWithoutValue, + isTagFilterAttributeField, + pruneTagFilterConditions, + type DatasetTagFilterCondition, + type DatasetTagFilterValue, + type WorkflowTagFilterOption +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import type { ReferenceItemValueType } from '@fastgpt/global/core/workflow/type/io'; +import type { WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants'; +import { getAllTags } from '@/web/core/dataset/api/collection'; +import { ReferSelector } from '@/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference'; +import { + ArrayTagSelect, + DateTimeTagInput, + NumberTagInput, + tagInputBaseStyles +} from '@/pageComponents/dataset/detail/CollectionCard/TagValueInputs'; +import PromptEditor from '@fastgpt/web/components/common/Textarea/PromptEditor'; +import type { + EditorVariableLabelPickerType, + EditorVariablePickerType +} from '@fastgpt/web/components/common/Textarea/PromptEditor/type'; +import { TagFilterFieldSelect, TagFilterOpSelect } from './TagFilterSelects'; + +export type TagFilterReferenceList = { + label: string | React.ReactNode; + value: string; + children: { + label: string; + value: string; + valueType?: WorkflowIOValueTypeEnum; + }[]; +}[]; + +/** AND/OR 切换，贴在标题右侧，对齐判断器条件组。 */ +export const TagFilterLogicToggle = ({ + value, + onChange +}: { + value?: DatasetTagFilterValue; + onChange: (value: DatasetTagFilterValue) => void; +}) => { + const logic = value?.logic ?? DatasetTagFilterLogicEnum.AND; + + return ( + { + onChange({ + ...(value ?? createEmptyTagFilterValue()), + logic: + logic === DatasetTagFilterLogicEnum.AND + ? DatasetTagFilterLogicEnum.OR + : DatasetTagFilterLogicEnum.AND + }); + }} + > + {logic} + + + ); +}; + +/** 旧版本升级按钮，贴在标题栏右侧。 */ +export const DatasetTagFilterUpgradeButton = ({ + onUpgrade +}: { + onUpgrade: () => Promise; +}) => { + const { t } = useTranslation(); + const { openConfirm, ConfirmModal } = useConfirm({ + type: 'delete', + title: t('workflow:tag_filter_upgrade_title'), + content: t('workflow:tag_filter_upgrade_content') + }); + const { runAsync: runUpgrade, loading } = useRequest(onUpgrade); + + return ( + <> + openConfirm({ onConfirm: runUpgrade })()} + > + {t('workflow:tag_filter_upgrade_cta')} + + + + ); +}; + +/** 旧 JSON 字符串过滤：保留 PromptEditor 编辑，支持变量标签渲染。 */ +export const DatasetTagFilterDeprecated = ({ + value, + onChange, + variables, + variableLabels +}: { + value: string; + onChange: (value: string) => void; + variables?: EditorVariablePickerType[]; + variableLabels?: EditorVariableLabelPickerType[]; +}) => { + const { t } = useTranslation(); + + return ( + + ); +}; + +const valueCellEmbeddedStyles = { + border: 'none', + boxShadow: 'none', + h: '36px', + minH: '36px', + maxH: '36px', + borderRadius: 0, + _hover: { border: 'none' }, + _focus: { border: 'none', boxShadow: 'none' } +}; + +const TagFilterValueCell = ({ + condition, + option, + referenceList, + onChange +}: { + condition: DatasetTagFilterCondition; + option?: WorkflowTagFilterOption; + referenceList: TagFilterReferenceList; + onChange: (patch: Partial) => void; +}) => { + const { t } = useTranslation(); + const isCollectionId = condition.field === DatasetTagFilterFieldEnum.collectionId; + const isReference = + isCollectionId || condition.valueMode === DatasetTagFilterValueModeEnum.reference; + + const literalInput = (() => { + if (!condition.tagType) { + return ( + + ); + } + if (condition.tagType === DatasetCollectionTagTypeEnum.number) { + return ( + onChange({ value: val === '' ? undefined : val })} + /> + ); + } + if (condition.tagType === DatasetCollectionTagTypeEnum.datetime) { + return ( + onChange({ value: val })} + /> + ); + } + return ( + onChange({ value: val })} + /> + ); + })(); + + return ( + + {isCollectionId ? ( + + + + ) : ( + + { + if (isReference) { + onChange({ + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined + }); + return; + } + onChange({ + valueMode: DatasetTagFilterValueModeEnum.reference, + value: ['', undefined] + }); + }} + > + + + + + )} + + {isReference ? ( + onChange({ value: e as ReferenceItemValueType })} + isArray={false} + ButtonProps={{ + ...valueCellEmbeddedStyles, + size: 'sm', + w: '100%', + px: 3, + borderWidth: 0 + }} + /> + ) : ( + literalInput + )} + + + ); +}; + +const TagFilterConditionRow = ({ + condition, + options, + referenceList, + onChange, + onRemove +}: { + condition: DatasetTagFilterCondition; + options: WorkflowTagFilterOption[]; + referenceList: TagFilterReferenceList; + onChange: (patch: Partial) => void; + onRemove: () => void; +}) => { + const option = options.find( + (item) => + !isTagFilterAttributeField(condition.field) && + item.tag === condition.tag && + item.tagType === condition.tagType + ); + const ops = getTagFilterOpsByCondition(condition); + const hideValue = isTagFilterOpWithoutValue(condition.op); + + return ( + + + { + onChange({ + field: next.field, + tag: next.tag ?? '', + tagType: next.tagType, + op: '', + valueMode: next.valueMode ?? DatasetTagFilterValueModeEnum.input, + value: next.value + }); + }} + /> + onChange({ op, value: undefined })} + /> + {!hideValue && ( + + )} + + + + ); +}; + +/** + * 知识库搜索的标签过滤条件行。工作流节点和简易模式共用。 + * 旧 JSON 字符串请走 DatasetTagFilterDeprecated，不要传入本组件。 + */ +const DatasetTagFilterRows = ({ + value, + onChange, + datasetIds, + referenceList +}: { + value: unknown; + onChange: (value: DatasetTagFilterValue) => void; + datasetIds: string[]; + referenceList: TagFilterReferenceList; +}) => { + const { t } = useTranslation(); + const datasetIdsKey = datasetIds.join(','); + const filterValue = isDatasetTagFilterValue(value) ? value : createEmptyTagFilterValue(); + + const { + data: tagLists = [], + loading, + error + } = useRequest( + async () => { + if (datasetIds.length === 0) return []; + const results = await Promise.all(datasetIds.map(getAllTags)); + return results.map(({ list }) => list); + }, + { + manual: false, + refreshDeps: [datasetIdsKey], + errorToast: '' + } + ); + + const options = useMemo(() => intersectWorkflowTagOptions(tagLists), [tagLists]); + + useEffect(() => { + if (loading || error || datasetIds.length === 0 || !isDatasetTagFilterValue(value)) return; + const next = pruneTagFilterConditions(value, options); + if (JSON.stringify(next) === JSON.stringify(value)) return; + onChange(next); + }, [datasetIds.length, error, loading, onChange, options, value]); + + return ( + + + {filterValue.conditions.map((condition, index) => ( + { + onChange({ + ...filterValue, + conditions: filterValue.conditions.map((item, itemIndex) => + itemIndex === index ? { ...item, ...patch } : item + ) + }); + }} + onRemove={() => { + if (filterValue.conditions.length <= 1) { + onChange({ + ...filterValue, + conditions: [createEmptyTagFilterCondition()] + }); + return; + } + onChange({ + ...filterValue, + conditions: filterValue.conditions.filter((_, itemIndex) => itemIndex !== index) + }); + }} + /> + ))} + + + + ); +}; + +export default React.memo(DatasetTagFilterRows); diff --git a/projects/app/src/components/core/dataset/TagFilterSelects.tsx b/projects/app/src/components/core/dataset/TagFilterSelects.tsx new file mode 100644 index 000000000000..25c3816081ff --- /dev/null +++ b/projects/app/src/components/core/dataset/TagFilterSelects.tsx @@ -0,0 +1,379 @@ +import React, { useMemo, useState } from 'react'; +import { + Box, + Button, + Flex, + Input, + Popover, + PopoverContent, + PopoverTrigger, + useDisclosure +} from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import type { IconNameType } from '@fastgpt/web/components/common/Icon/type'; +import { + DatasetTagFilterFieldEnum, + DatasetTagFilterValueModeEnum, + formatTagOptionKey, + parseTagOptionKey, + type DatasetTagFilterCondition, + type DatasetTagFilterField, + type DatasetTagFilterValueMode, + type WorkflowTagFilterOption +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; + +const FILE_TAGS = 'fileTags'; +const FILE_ATTRS = 'fileAttrs'; + +type FieldSelectValue = { + field: DatasetTagFilterField; + tag?: string; + tagType?: WorkflowTagFilterOption['tagType']; + valueMode?: DatasetTagFilterValueMode; + value?: unknown; +}; + +/** 选择标签：左侧文件标签 / 文件属性，右侧可搜索。选中后触发器只展示叶子名称。 */ +export const TagFilterFieldSelect = ({ + condition, + options, + onChange +}: { + condition: DatasetTagFilterCondition; + options: WorkflowTagFilterOption[]; + onChange: (value: FieldSelectValue) => void; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const [search, setSearch] = useState(''); + const selectedLabel = (() => { + if (condition.field === DatasetTagFilterFieldEnum.createTime) { + return t('common:core.dataset.collection.metadata.Createtime' as any); + } + if (condition.field === DatasetTagFilterFieldEnum.collectionId) { + return t('common:core.dataset.collection.id' as any); + } + return condition.tag ?? ''; + })(); + const selectedValue = + condition.field === DatasetTagFilterFieldEnum.createTime + ? DatasetTagFilterFieldEnum.createTime + : condition.field === DatasetTagFilterFieldEnum.collectionId + ? DatasetTagFilterFieldEnum.collectionId + : condition.tag && condition.tagType + ? formatTagOptionKey(condition.tag, condition.tagType) + : ''; + const activeGroup = + condition.field === DatasetTagFilterFieldEnum.createTime || + condition.field === DatasetTagFilterFieldEnum.collectionId + ? FILE_ATTRS + : FILE_TAGS; + + const [group, setGroup] = useState(activeGroup); + + const rightItems = useMemo(() => { + const keyword = search.trim().toLowerCase(); + if (group === FILE_ATTRS) { + return [ + { + label: t('common:core.dataset.collection.metadata.Createtime'), + value: DatasetTagFilterFieldEnum.createTime + }, + { + label: t('common:core.dataset.collection.id'), + value: DatasetTagFilterFieldEnum.collectionId + } + ].filter((item) => !keyword || String(item.label).toLowerCase().includes(keyword)); + } + return options + .filter((item) => !keyword || item.tag.toLowerCase().includes(keyword)) + .map((item) => ({ + label: item.tag, + value: formatTagOptionKey(item.tag, item.tagType) + })); + }, [group, options, search, t]); + + const handleOpen = () => { + setGroup(activeGroup); + setSearch(''); + onOpen(); + }; + + const handleSelect = (value: string) => { + if (value === DatasetTagFilterFieldEnum.createTime) { + onChange({ + field: DatasetTagFilterFieldEnum.createTime, + tagType: DatasetCollectionTagTypeEnum.datetime, + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined + }); + } else if (value === DatasetTagFilterFieldEnum.collectionId) { + onChange({ + field: DatasetTagFilterFieldEnum.collectionId, + tagType: DatasetCollectionTagTypeEnum.array, + valueMode: DatasetTagFilterValueModeEnum.reference, + value: ['', undefined] + }); + } else { + const parsed = parseTagOptionKey(value); + onChange({ + field: DatasetTagFilterFieldEnum.tag, + tag: parsed?.tag, + tagType: parsed?.tagType, + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined + }); + } + onClose(); + }; + + return ( + + + + + + + + + {[ + { value: FILE_TAGS, label: t('workflow:tag_filter_file_tags') }, + { value: FILE_ATTRS, label: t('workflow:tag_filter_file_attrs') } + ].map((item) => { + const selected = group === item.value; + return ( + { + setGroup(item.value); + setSearch(''); + }} + > + {item.label} + + + ); + })} + + + + setSearch(e.target.value)} + /> + + {rightItems.length === 0 ? ( + + {t('common:no_select_data')} + + ) : ( + rightItems.map((item) => { + const selected = group === activeGroup && item.value === selectedValue; + return ( + handleSelect(item.value)} + > + {item.label} + + ); + }) + )} + + + + + + + ); +}; + +type OpItem = { + labelKey: string; + value: string; + icon?: string; + iconFlip?: boolean; +}; + +/** 条件操作符。数字类用稿上的 16px 符号图标 + 12px 文案。 */ +export const TagFilterOpSelect = ({ + value, + list, + onChange +}: { + value?: string; + list: OpItem[]; + onChange: (value: string) => void; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const selected = list.find((item) => item.value === value); + + return ( + + + + + + + {list.map((item) => ( + { + onChange(item.value); + onClose(); + }} + > + {item.icon && ( + + )} + {t(item.labelKey)} + + ))} + + + + ); +}; diff --git a/projects/app/src/migration/registry.ts b/projects/app/src/migration/registry.ts index 9b6affe2501e..d73e3af725a9 100644 --- a/projects/app/src/migration/registry.ts +++ b/projects/app/src/migration/registry.ts @@ -16,6 +16,7 @@ import { backfillResourceCreateTime } from './tasks/4170/20260903_backfill_resou import { backfillBillMetadata } from './tasks/4170/20260905_backfill_bill_metadata'; import { backfillResourceOwnerAcl } from './tasks/4170/20260905_backfill_resource_owner_acl'; import { cleanupTeamMemberRoles } from './tasks/4170/20260907_cleanup_team_member_roles'; +import { migrateDatasetTagsV2 } from './tasks/20260907_migrate_dataset_tags_v2'; export type SystemMigrationLogger = { info: (message: string, metadata?: Record) => void; @@ -309,6 +310,32 @@ export const systemMigrations = [ blockStartup: false, onFailure: SystemMigrationFailurePolicyEnum.continue, run: backfillResourceOwnerAcl + }, + { + id: '20260907_migrate_dataset_tags_v2', + version: '4.17.0', + nameKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.name'), + descriptionKey: i18nT( + 'system_migration:migrations.20260907_migrate_dataset_tags_v2.description' + ), + resultKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.result'), + progressSteps: [ + { + key: 'datasets', + labelKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.datasets') + }, + { + key: 'collections', + labelKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.collections') + }, + { + key: 'validation', + labelKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.validation') + } + ], + blockStartup: true, + onFailure: SystemMigrationFailurePolicyEnum.stop, + run: migrateDatasetTagsV2 } ] as const satisfies readonly SystemMigration[]; diff --git a/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/index.ts b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/index.ts new file mode 100644 index 000000000000..481032480c73 --- /dev/null +++ b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/index.ts @@ -0,0 +1,198 @@ +import { z } from 'zod'; +import { SystemMigrationStatusEnum } from '@fastgpt/global/migration/constants'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { systemMigrationBatchSize } from '@/migration/constants'; +import type { SystemMigrationContext } from '@/migration/registry'; +import { Types } from '@fastgpt/service/common/mongo'; +import { + migrateCollectionTagValues, + migrateDatasetTagDefinitions, + migrationCollections, + validateDatasetTagMigration, + type DatasetTagMigrationRecord, + type MigrationRecordResult +} from './service'; + +const StageKeySchema = z.enum(['datasets', 'collections']); +type StageKey = z.infer; + +const StageStateSchema = z.object({ + initialized: z.boolean(), + completed: z.boolean(), + endId: z.string().nullable(), + lastId: z.string().nullable(), + processedCount: z.number().int().nonnegative(), + migratedCount: z.number().int().nonnegative(), + total: z.number().int().nonnegative(), + deletedDefinitionCount: z.number().int().nonnegative(), + deletedReferenceCollectionCount: z.number().int().nonnegative() +}); +type StageState = z.infer; + +const CheckpointSchema = z.object({ + version: z.literal(1), + stages: z.record(StageKeySchema, StageStateSchema) +}); +type Checkpoint = z.infer; + +const createStageState = (): StageState => ({ + initialized: false, + completed: false, + endId: null, + lastId: null, + processedCount: 0, + migratedCount: 0, + total: 0, + deletedDefinitionCount: 0, + deletedReferenceCollectionCount: 0 +}); + +const stageKeys = StageKeySchema.options; +const createCheckpoint = (): Checkpoint => ({ + version: 1, + stages: Object.fromEntries( + stageKeys.map((key) => [key, createStageState()]) + ) as Checkpoint['stages'] +}); + +const stageQueries: Record> = { + datasets: { type: { $ne: DatasetTypeEnum.folder } }, + collections: {} +}; + +/** + * 4.17.0 阻塞升级：标签定义和 Collection 标签值是新版运行时的前置条件。 + * 任务按两个原始 Mongo 集合的 ObjectId 固定上界分批扫描；单记录写入幂等，业务提交后才推进 checkpoint。 + * 失败直接交给 Runner 阻止 ready，不使用阻塞任务禁止的 failedRecords。 + */ +export const migrateDatasetTagsV2 = async (context: SystemMigrationContext) => { + let checkpoint = (await context.getCheckpoint(CheckpointSchema)) ?? createCheckpoint(); + + const saveStage = async (key: StageKey, state: StageState) => { + checkpoint = { ...checkpoint, stages: { ...checkpoint.stages, [key]: state } }; + await context.saveCheckpoint(checkpoint); + }; + + const runStage = async ( + key: StageKey, + processRecord: (record: DatasetTagMigrationRecord) => Promise + ) => { + const collection = migrationCollections[key]; + const query = stageQueries[key]; + let state = checkpoint.stages[key]; + + if (!state.initialized) { + const lastRecord = await collection + .find({ ...query, _id: { $type: 'objectId' } }, { projection: { _id: 1 } }) + .sort({ _id: -1 }) + .limit(1) + .next(); + const endId = lastRecord ? String(lastRecord._id) : null; + const total = endId + ? await collection.countDocuments({ + ...query, + _id: { $type: 'objectId', $lte: lastRecord!._id } + }) + : 0; + state = { ...state, initialized: true, endId, total }; + await saveStage(key, state); + } + + await context.reportProgress({ + key, + status: state.completed + ? SystemMigrationStatusEnum.succeeded + : SystemMigrationStatusEnum.running, + current: state.processedCount, + total: state.total + }); + if (state.completed) return state; + + while (state.endId && state.lastId !== state.endId) { + context.signal.throwIfAborted(); + await context.assertActive(); + const records = (await collection + .find( + { + ...query, + _id: { + $type: 'objectId', + ...(state.lastId ? { $gt: new Types.ObjectId(state.lastId) } : {}), + $lte: new Types.ObjectId(state.endId) + } + }, + { projection: { _id: 1, teamId: 1 } } + ) + .sort({ _id: 1 }) + .limit(systemMigrationBatchSize) + .toArray()) as DatasetTagMigrationRecord[]; + if (records.length === 0) break; + + let batchResult: MigrationRecordResult = {}; + for (const record of records) { + const result = await processRecord(record); + batchResult = { + migratedCount: (batchResult.migratedCount ?? 0) + (result.migratedCount ?? 0), + deletedDefinitionCount: + (batchResult.deletedDefinitionCount ?? 0) + (result.deletedDefinitionCount ?? 0), + deletedReferenceCollectionCount: + (batchResult.deletedReferenceCollectionCount ?? 0) + + (result.deletedReferenceCollectionCount ?? 0) + }; + } + await context.assertActive(); + state = { + ...state, + lastId: String(records.at(-1)!._id), + processedCount: state.processedCount + records.length, + migratedCount: state.migratedCount + (batchResult.migratedCount ?? 0), + deletedDefinitionCount: + state.deletedDefinitionCount + (batchResult.deletedDefinitionCount ?? 0), + deletedReferenceCollectionCount: + state.deletedReferenceCollectionCount + (batchResult.deletedReferenceCollectionCount ?? 0) + }; + await saveStage(key, state); + await context.reportProgress({ + key, + status: SystemMigrationStatusEnum.running, + current: state.processedCount, + total: state.total + }); + } + + state = { ...state, completed: true }; + await saveStage(key, state); + await context.reportProgress({ + key, + status: SystemMigrationStatusEnum.succeeded, + current: state.processedCount, + total: state.total + }); + return state; + }; + + const datasets = await runStage('datasets', async (record) => { + if (!record.teamId) throw new Error(`Dataset ${String(record._id)} has no teamId`); + return migrateDatasetTagDefinitions({ datasetId: record._id, teamId: record.teamId }); + }); + const collections = await runStage('collections', (record) => + migrateCollectionTagValues({ collectionId: record._id }) + ); + + await context.reportProgress({ key: 'validation', status: SystemMigrationStatusEnum.running }); + await context.assertActive(); + const validation = await validateDatasetTagMigration().catch(async (error) => { + checkpoint = createCheckpoint(); + await context.saveCheckpoint(checkpoint); + throw error; + }); + await context.reportProgress({ key: 'validation', status: SystemMigrationStatusEnum.succeeded }); + + return { + datasetsProcessedCount: datasets.processedCount, + collectionsMigratedCount: collections.migratedCount, + duplicateDefinitionsDeletedCount: datasets.deletedDefinitionCount, + duplicateReferenceCollectionsCleanedCount: datasets.deletedReferenceCollectionCount, + legacyCollectionCount: validation.legacyCollectionCount + }; +}; diff --git a/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/service.ts b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/service.ts new file mode 100644 index 000000000000..1db4608b8a91 --- /dev/null +++ b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/service.ts @@ -0,0 +1,271 @@ +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/type'; +import { Types, type ClientSession } from '@fastgpt/service/common/mongo'; +import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { ensureDatasetTagMigrationCarrier } from '@fastgpt/service/core/dataset/collection/utils'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; + +export type DatasetTagMigrationRecord = { + _id: Types.ObjectId; + teamId?: Types.ObjectId; +}; + +export type MigrationRecordResult = { + migratedCount?: number; + deletedDefinitionCount?: number; + deletedReferenceCollectionCount?: number; +}; + +const asObjectId = (value: unknown) => { + const stringValue = String(value ?? ''); + return Types.ObjectId.isValid(stringValue) ? new Types.ObjectId(stringValue) : undefined; +}; + +/** + * 整理单个知识库的 v2 标签定义：保留最早的同名定义，重复项先删引用再删定义， + * 并整理已有的 fromMigration 承载记录。没有旧字符串标签的知识库不预建承载记录。 + */ +export const migrateDatasetTagDefinitions = async ({ + datasetId, + teamId, + session +}: { + datasetId: Types.ObjectId; + teamId: Types.ObjectId; + session?: ClientSession; +}): Promise => { + const migrate = async (activeSession: ClientSession): Promise => { + const definitions = await MongoDatasetCollectionTagsV2.collection + .find({ teamId, datasetId }, { session: activeSession }) + .sort({ _id: 1 }) + .toArray(); + const deleteIds = new Map(); + + const definitionsByName = new Map(); + for (const definition of definitions) { + if (deleteIds.has(String(definition._id))) continue; + const group = definitionsByName.get(String(definition.tag)) ?? []; + group.push(definition); + definitionsByName.set(String(definition.tag), group); + } + for (const group of definitionsByName.values()) { + group.slice(1).forEach((definition) => { + deleteIds.set(String(definition._id), definition._id); + }); + } + + const migrationDefinitions = definitions.filter( + (definition) => definition.fromMigration === true && !deleteIds.has(String(definition._id)) + ); + const carrier = migrationDefinitions[0]; + migrationDefinitions.slice(1).forEach((definition) => { + deleteIds.set(String(definition._id), definition._id); + }); + + const duplicateIds = [...deleteIds.values()]; + let deletedReferenceCollectionCount = 0; + if (duplicateIds.length > 0) { + const duplicateTagIds: Array = [ + ...duplicateIds, + ...duplicateIds.map(String) + ]; + const duplicateTagIdStrings = duplicateIds.map(String); + const pullResult = await MongoDatasetCollection.collection.updateMany( + { teamId, datasetId, 'tags.tagId': { $in: duplicateTagIds } }, + [ + { + $set: { + tags: { + $filter: { + input: '$tags', + as: 'tag', + cond: { + $not: { + $in: [ + { + $convert: { + input: '$$tag.tagId', + to: 'string', + onError: null, + onNull: null + } + }, + duplicateTagIdStrings + ] + } + } + } + } + } + } + ], + { session: activeSession } + ); + deletedReferenceCollectionCount = pullResult.modifiedCount; + const remainingReferences = await MongoDatasetCollection.collection.countDocuments( + { teamId, datasetId, 'tags.tagId': { $in: duplicateTagIds } }, + { session: activeSession } + ); + if (remainingReferences > 0) { + throw new Error('Duplicate dataset tag references remain after cleanup'); + } + await MongoDatasetCollectionTagsV2.collection.deleteMany( + { _id: { $in: duplicateIds }, teamId, datasetId }, + { session: activeSession } + ); + } + + if (carrier) { + await MongoDatasetCollectionTagsV2.collection.updateOne( + { _id: carrier._id, teamId, datasetId }, + { + $set: { + tagType: DatasetCollectionTagTypeEnum.array, + fromMigration: true + } + }, + { session: activeSession } + ); + } + + return { + migratedCount: 1, + deletedDefinitionCount: duplicateIds.length, + deletedReferenceCollectionCount + }; + }; + + return session ? migrate(session) : mongoSessionRun(migrate); +}; + +/** 将单个 Collection 的旧标签 ID 转换为当前 dataset 承载标签的名称数组。 */ +export const migrateCollectionTagValues = async ({ + collectionId, + session +}: { + collectionId: Types.ObjectId; + session?: ClientSession; +}): Promise => { + const migrate = async (activeSession: ClientSession): Promise => { + const collection = await MongoDatasetCollection.collection.findOne( + { _id: collectionId }, + { projection: { teamId: 1, datasetId: 1, tags: 1 }, session: activeSession } + ); + if (!collection) return {}; + + const currentTags = Array.isArray(collection.tags) ? collection.tags : []; + const legacyTagIds = currentTags.filter((tag): tag is string => typeof tag === 'string'); + if (legacyTagIds.length === 0) return {}; + + const objectIds = legacyTagIds + .map(asObjectId) + .filter((id): id is Types.ObjectId => Boolean(id)); + const legacyDefinitions = objectIds.length + ? await MongoDatasetCollectionTags.collection + .find( + { _id: { $in: objectIds }, teamId: collection.teamId, datasetId: collection.datasetId }, + { projection: { tag: 1 }, session: activeSession } + ) + .toArray() + : []; + const carrier = await ensureDatasetTagMigrationCarrier({ + teamId: String(collection.teamId), + datasetId: String(collection.datasetId), + session: activeSession + }); + + const carrierId = String(carrier._id); + const existingCarrier = currentTags.find( + (tag) => tag && typeof tag === 'object' && String(tag.tagId) === carrierId + ); + const existingValues = Array.isArray(existingCarrier?.value) + ? existingCarrier.value.filter((value: unknown): value is string => typeof value === 'string') + : []; + const migratedValues = legacyDefinitions + .map((definition) => definition.tag) + .filter((tag): tag is string => typeof tag === 'string' && tag.length > 0); + const mergedValues = [...new Set([...existingValues, ...migratedValues])]; + const nextTags = currentTags.filter( + (tag) => typeof tag !== 'string' && String(tag?.tagId ?? '') !== carrierId + ); + if (mergedValues.length > 0) nextTags.push({ tagId: carrierId, value: mergedValues }); + + const result = await MongoDatasetCollection.collection.updateOne( + { _id: collectionId, tags: currentTags }, + { $set: { tags: nextTags } }, + { session: activeSession } + ); + if (result.matchedCount !== 1) { + throw new Error('Collection tags changed concurrently during migration'); + } + return { migratedCount: 1 }; + }; + + return session ? migrate(session) : mongoSessionRun(migrate); +}; + +export const migrationCollections = { + datasets: MongoDataset.collection, + collections: MongoDatasetCollection.collection +}; + +/** 创建标签唯一索引并在迁移成功前校验所有必要不变量。 */ +export const validateDatasetTagMigration = async () => { + await Promise.all([ + MongoDatasetCollectionTagsV2.createIndexes({ background: true }), + MongoDatasetCollection.createIndexes({ background: true }) + ]); + + const [legacyCollectionCount, duplicateGroups, duplicateCarrierGroups, invalidCarrierCount] = + await Promise.all([ + MongoDatasetCollection.collection.countDocuments({ + tags: { $elemMatch: { $type: 'string' } } + }), + MongoDatasetCollectionTagsV2.collection + .aggregate([ + { + $group: { + _id: { teamId: '$teamId', datasetId: '$datasetId', tag: '$tag' }, + n: { $sum: 1 } + } + }, + { $match: { n: { $gt: 1 } } }, + { $limit: 1 } + ]) + .toArray(), + MongoDatasetCollectionTagsV2.collection + .aggregate([ + { + $match: { fromMigration: true } + }, + { + $group: { + _id: { teamId: '$teamId', datasetId: '$datasetId' }, + n: { $sum: 1 } + } + }, + { $match: { n: { $gt: 1 } } }, + { $limit: 1 } + ]) + .toArray(), + MongoDatasetCollectionTagsV2.collection.countDocuments({ + fromMigration: true, + tagType: { $ne: DatasetCollectionTagTypeEnum.array } + }) + ]); + + if ( + legacyCollectionCount > 0 || + duplicateGroups.length > 0 || + duplicateCarrierGroups.length > 0 || + invalidCarrierCount > 0 + ) { + throw new Error( + `Dataset tag migration validation failed: legacyCollections=${legacyCollectionCount}, duplicateTagGroups=${duplicateGroups.length}, duplicateCarrierGroups=${duplicateCarrierGroups.length}, invalidCarriers=${invalidCarrierCount}` + ); + } + + return { legacyCollectionCount }; +}; diff --git a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx index 72ecbac44eaa..85638ebd049a 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx +++ b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx @@ -44,6 +44,16 @@ import { findClientModelByReference, resolveClientModelReferenceId } from '@/web/core/ai/model/modelReference'; +import DatasetTagFilterRows, { + TagFilterLogicToggle +} from '@/components/core/dataset/DatasetTagFilterRows'; +import { formatEditorVariablePickerIcon } from '@fastgpt/global/core/workflow/utils'; +import { workflowSystemVariables } from '@/web/core/app/utils'; +import { VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants'; +import { + isDatasetTagFilterValue, + type DatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; const DatasetSelectModal = dynamic(() => import('@/components/core/app/DatasetSelectModal')); const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal')); @@ -78,6 +88,40 @@ const EditForm = ({ const appId = useContextSelector(AppContext, (v) => v.appId); const selectDatasets = useMemo(() => appForm?.dataset?.datasets, [appForm]); + const formatVariables = useMemo( + () => + formatEditorVariablePickerIcon([ + ...workflowSystemVariables.filter( + (variable) => + !['appId', 'chatId', 'responseChatItemId', 'histories'].includes(variable.key) + ), + ...(appForm.chatConfig.variables ?? []) + ]).map((item) => ({ ...item, label: t(item.label as any) })), + [appForm.chatConfig.variables, t] + ); + const tagFilterReferenceList = useMemo( + () => [ + { + label: t('common:core.module.Variable'), + value: VARIABLE_NODE_ID, + children: formatVariables.map((item) => ({ + label: item.label, + value: item.key, + valueType: item.valueType + })) + } + ], + [formatVariables, t] + ); + const onCollectionFilterMatchChange = useCallback( + (value: DatasetTagFilterValue) => { + setAppForm((state) => ({ + ...state, + dataset: { ...state.dataset, collectionFilterMatch: value } + })); + }, + [setAppForm] + ); const { isWelcomeTextFolded, toggleWelcomeTextFold } = useWelcomeTextFoldState(appId); const { @@ -628,6 +672,30 @@ const EditForm = ({ /> ))} + {appForm.dataset.datasets.length > 0 && feConfigs?.isPlus && ( + + + {t('workflow:tag_filter')} + + + + + + item.datasetId)} + referenceList={tagFilterReferenceList} + /> + + )} {/* File select */} diff --git a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts index 7b750e468e70..c6948f074e6b 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts +++ b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts @@ -351,6 +351,7 @@ export function agentForm2AppWorkflow( datasetSearchExtensionModelId: data.dataset.datasetSearchExtensionModelId, datasetSearchExtensionModel: data.dataset.datasetSearchExtensionModel, datasetSearchExtensionBg: data.dataset.datasetSearchExtensionBg, + collectionFilterMatch: data.dataset.collectionFilterMatch, [NodeInputKeyEnum.authTmbId]: data.dataset.authTmbId }) }, diff --git a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx index a939e094d6a8..356f4bdcf836 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx +++ b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx @@ -37,11 +37,26 @@ import { SANDBOX_ICON } from '@fastgpt/global/core/ai/sandbox/tools'; import SandboxConfigButton from '../../components/SandboxConfigButton'; import { useUserStore } from '@/web/support/user/useUserStore'; import DatasetCard from '@/components/core/app/DatasetCard'; +import DatasetTagFilterRows, { + DatasetTagFilterDeprecated, + DatasetTagFilterUpgradeButton, + TagFilterLogicToggle +} from '@/components/core/dataset/DatasetTagFilterRows'; import { useWelcomeTextFoldState } from '@/components/core/app/useAppEditorUIState'; import { findClientModelByReference, resolveClientModelReferenceId } from '@/web/core/ai/model/modelReference'; +import { NodeInputKeyEnum, VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants'; +import { + createEmptyTagFilterValue, + DatasetTagFilterVersionEnum, + isDatasetTagFilterValue, + normalizeLegacyDatasetTagFilterValue, + resolveDatasetTagFilterVersion, + type DatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { form2AppWorkflow } from './utils'; const DatasetSelectModal = dynamic(() => import('@/components/core/app/DatasetSelectModal')); const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal')); @@ -82,7 +97,7 @@ const EditForm = ({ const showSandbox = feConfigs.show_agent_sandbox; const { teamPlanStatus } = useUserStore(); const enableSandbox = !teamPlanStatus?.standard || !!teamPlanStatus?.standard?.enableSandbox; - const { appDetail } = useContextSelector(AppContext, (v) => v); + const { appDetail, onSaveApp } = useContextSelector(AppContext, (v) => v); const selectDatasets = useMemo(() => appForm?.dataset?.datasets, [appForm]); const [, startTst] = useTransition(); const isAgentSandboxEnabled = !!appForm.aiSettings.useAgentSandbox; @@ -111,7 +126,7 @@ const EditForm = ({ ...item, label: t(item.label as any), parent: { - id: 'VARIABLE_NODE_ID', + id: VARIABLE_NODE_ID, label: t('common:core.module.Variable'), avatar: 'core/workflow/template/variable' } @@ -119,6 +134,39 @@ const EditForm = ({ [appForm.chatConfig.variables, t] ); + const tagFilterReferenceList = useMemo( + () => [ + { + label: t('common:core.module.Variable'), + value: VARIABLE_NODE_ID, + children: formatVariables.map((item) => ({ + label: item.label, + value: item.key, + valueType: item.valueType + })) + } + ], + [formatVariables, t] + ); + + const onCollectionFilterMatchChange = useCallback( + (value: DatasetTagFilterValue | string) => { + setAppForm((state) => ({ + ...state, + dataset: { + ...state.dataset, + collectionFilterMatch: value + } + })); + }, + [setAppForm] + ); + const isLegacyCollectionFilter = + resolveDatasetTagFilterVersion({ + version: appForm.dataset[NodeInputKeyEnum.collectionFilterVersion], + filterValue: appForm.dataset.collectionFilterMatch + }) === DatasetTagFilterVersionEnum.legacy; + const { llmModelList, reRankModelList } = useUserModelLists(); const selectedModel = findClientModelByReference({ @@ -486,6 +534,78 @@ const EditForm = ({ /> ))} + {appForm.dataset.datasets?.length > 0 && feConfigs?.isPlus && ( + + + + {isLegacyCollectionFilter + ? t('workflow:collection_metadata_filter') + : t('workflow:tag_filter')} + + + {isLegacyCollectionFilter ? ( + <> + + { + const nextForm = { + ...appForm, + dataset: { + ...appForm.dataset, + [NodeInputKeyEnum.collectionFilterVersion]: + DatasetTagFilterVersionEnum.structured, + collectionFilterMatch: createEmptyTagFilterValue() + } + }; + const workflow = form2AppWorkflow(nextForm, t); + await onSaveApp({ + ...workflow, + isPublish: false, + chatConfig: nextForm.chatConfig + }); + setAppForm(nextForm); + }} + /> + + ) : ( + + + + )} + + {isLegacyCollectionFilter ? ( + + ) : ( + item.datasetId)} + referenceList={tagFilterReferenceList} + /> + )} + + )} {/* File select */} diff --git a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts index ba7cc684d699..d18c76d39b64 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts +++ b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts @@ -29,6 +29,7 @@ import { AiChatQuoteTemplate } from '@fastgpt/global/core/workflow/template/system/aiChat/index'; import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch'; +import { resolveDatasetTagFilterVersion } from '@fastgpt/global/core/dataset/workflowTagFilter'; import { i18nT } from '@fastgpt/global/common/i18n/utils'; import { Input_Template_File_Link, @@ -121,6 +122,11 @@ export const appWorkflow2Form = ({ NodeInputKeyEnum.sandboxEntrypoint ); } else if (node.flowNodeType === FlowNodeTypeEnum.datasetSearchNode) { + defaultAppForm.dataset[NodeInputKeyEnum.collectionFilterVersion] = + resolveDatasetTagFilterVersion({ + version: findInputValueByKey(node.inputs, NodeInputKeyEnum.collectionFilterVersion), + filterValue: findInputValueByKey(node.inputs, NodeInputKeyEnum.collectionFilterMatch) + }); defaultAppForm.dataset.datasets = findInputValueByKey( node.inputs, NodeInputKeyEnum.datasetSelectList @@ -178,6 +184,10 @@ export const appWorkflow2Form = ({ node.inputs, NodeInputKeyEnum.authTmbId ); + defaultAppForm.dataset.collectionFilterMatch = findInputValueByKey( + node.inputs, + NodeInputKeyEnum.collectionFilterMatch + ); } else if ( node.flowNodeType === FlowNodeTypeEnum.pluginModule || node.flowNodeType === FlowNodeTypeEnum.appModule || @@ -579,6 +589,25 @@ export function form2AppWorkflow( valueType: WorkflowIOValueTypeEnum.boolean, value: formData.dataset.authTmbId }, + { + key: NodeInputKeyEnum.collectionFilterVersion, + renderTypeList: [FlowNodeInputTypeEnum.hidden], + label: '', + valueType: WorkflowIOValueTypeEnum.string, + value: resolveDatasetTagFilterVersion({ + version: formData.dataset[NodeInputKeyEnum.collectionFilterVersion], + filterValue: formData.dataset.collectionFilterMatch + }) + }, + { + key: NodeInputKeyEnum.collectionFilterMatch, + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + label: i18nT('workflow:tag_filter'), + valueType: WorkflowIOValueTypeEnum.string, + isPro: true, + description: i18nT('workflow:tag_filter_description'), + value: formData.dataset.collectionFilterMatch + }, { ...Input_Template_UserChatInput, key: NodeInputKeyEnum.datasetSearchInput, diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx index 92c7d83aa99b..67b70eef488f 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx @@ -21,17 +21,23 @@ type Props = { nodeId: string; input: FlowNodeInputItemType; RightComponent?: React.JSX.Element; + rightInline?: boolean; isTool?: boolean; }; -const InputLabel = ({ nodeId, input, RightComponent, isTool }: Props) => { +const InputLabel = ({ nodeId, input, RightComponent, rightInline, isTool }: Props) => { const { t } = useSafeTranslation(); + const labelText = t(input.label as any); + const descriptionText = input.description ? t(input.description as any) : undefined; + const onChangeNode = useContextSelector(WorkflowActionsContext, (v) => v.onChangeNode); - const { description, required, label, renderTypeList, valueType, valueDesc } = input; + const { required, renderTypeList, valueType, valueDesc } = input; const renderType = getSelectedInputRenderType(input) ?? renderTypeList?.[0] ?? FlowNodeInputTypeEnum.input; + const shouldRenderRightInline = + rightInline ?? renderType === FlowNodeInputTypeEnum.datasetTagFilter; const displayRenderTypeList = useMemo( () => getToolInputDisplayRenderTypeList({ @@ -65,9 +71,9 @@ const InputLabel = ({ nodeId, input, RightComponent, isTool }: Props) => { - {t(label as any)} + {labelText} - {description && } + {descriptionText && } {/* value type */} {[FlowNodeInputTypeEnum.reference, FlowNodeInputTypeEnum.fileSelect].includes(renderType) && ( @@ -122,8 +128,8 @@ const InputLabel = ({ nodeId, input, RightComponent, isTool }: Props) => { {/* Right Component */} {!input.deprecated && RightComponent && ( <> - - {RightComponent} + {!shouldRenderRightInline && } + {shouldRenderRightInline ? {RightComponent} : RightComponent} )} diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx index 9d79c206af14..fcea88faae70 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx @@ -16,6 +16,7 @@ import MyTag from '@fastgpt/web/components/common/Tag/index'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { getSelectedInputRenderType } from '@fastgpt/global/core/workflow/utils'; +import { datasetSearchUsesLegacyFilter } from '@/web/core/workflow/datasetSearchNodeUpgrade'; const RenderList: Record< FlowNodeInputTypeEnum, @@ -91,6 +92,12 @@ const RenderList: Record< [FlowNodeInputTypeEnum.password]: { Component: CommonInputForm }, + [FlowNodeInputTypeEnum.datasetTagFilter]: { + Component: dynamic(() => import('./templates/DatasetTagFilter')), + LableRightComponent: dynamic(() => + import('./templates/DatasetTagFilter').then((mod) => mod.DatasetTagFilterLogic) + ) + }, [FlowNodeInputTypeEnum.agentGenerated]: undefined, [FlowNodeInputTypeEnum.customVariable]: undefined, @@ -261,6 +268,10 @@ const RenderInput = ({ nodeId={nodeId} input={input} RightComponent={RenderComponent?.LableRightComponent} + rightInline={ + renderType === FlowNodeInputTypeEnum.datasetTagFilter && + !datasetSearchUsesLegacyFilter(filterProInputs) + } isTool={isTool} /> )} diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/DatasetTagFilter.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/DatasetTagFilter.tsx new file mode 100644 index 000000000000..5df94c0734f3 --- /dev/null +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/DatasetTagFilter.tsx @@ -0,0 +1,186 @@ +import React, { useCallback, useMemo } from 'react'; +import type { RenderInputProps } from '../type'; +import { useContextSelector } from 'use-context-selector'; +import { NodeInputKeyEnum, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants'; +import { + createEmptyTagFilterValue, + isDatasetTagFilterValue, + normalizeLegacyDatasetTagFilterValue, + type DatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { WorkflowActionsContext } from '@/pageComponents/app/detail/WorkflowComponents/context/workflowActionsContext'; +import { useReference } from './Reference'; +import DatasetTagFilterRows, { + DatasetTagFilterDeprecated, + DatasetTagFilterUpgradeButton, + TagFilterLogicToggle +} from '@/components/core/dataset/DatasetTagFilterRows'; +import { WorkflowBufferDataContext } from '../../../../../context/workflowInitContext'; +import { AppContext } from '@/pageComponents/app/detail/context'; +import { getEditorVariables } from '@/pageComponents/app/detail/WorkflowComponents/utils'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { useMemoEnhance } from '@fastgpt/web/hooks/useMemoEnhance'; +import { useTranslation } from 'next-i18next'; +import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; +import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch'; +import { WorkflowUtilsContext } from '@/pageComponents/app/detail/WorkflowComponents/context/workflowUtilsContext'; +import { + datasetSearchUsesLegacyFilter, + persistLegacyDatasetSearchNodeUpgrade +} from '@/web/core/workflow/datasetSearchNodeUpgrade'; + +const DatasetTagFilterRender = ({ inputs = [], item, nodeId }: RenderInputProps) => { + const { t } = useTranslation(); + const onChangeNode = useContextSelector(WorkflowActionsContext, (v) => v.onChangeNode); + const { getNodeById, edges } = useContextSelector(WorkflowBufferDataContext, (v) => v); + const { appDetail } = useContextSelector(AppContext, (v) => v); + const { feConfigs } = useSystemStore(); + const isLegacyNode = datasetSearchUsesLegacyFilter(inputs); + + const { referenceList } = useReference({ + nodeId, + valueType: WorkflowIOValueTypeEnum.any + }); + const datasetIds = useMemo(() => { + const datasetValue = inputs.find( + (input) => input.key === NodeInputKeyEnum.datasetSelectList + )?.value; + if (!Array.isArray(datasetValue)) return []; + return datasetValue + .map((dataset) => + dataset && typeof dataset === 'object' && 'datasetId' in dataset + ? String(dataset.datasetId ?? '') + : '' + ) + .filter(Boolean); + }, [inputs]); + + const editorVariables = useMemoEnhance(() => { + return getEditorVariables({ + nodeId, + getNodeById, + edges, + appDetail, + t + }); + }, [nodeId, getNodeById, edges, appDetail, t]); + + const externalVariables = useMemo(() => { + return ( + feConfigs?.externalProviderWorkflowVariables?.map((item) => ({ + key: item.key, + label: item.name + })) ?? [] + ); + }, [feConfigs?.externalProviderWorkflowVariables]); + + const allVariables = useMemo( + () => [...(editorVariables ?? []), ...externalVariables], + [editorVariables, externalVariables] + ); + + const onChange = useCallback( + (value: DatasetTagFilterValue | string) => { + onChangeNode({ + nodeId, + type: 'updateInput', + key: item.key, + value: { ...item, value } + }); + }, + [item, nodeId, onChangeNode] + ); + + if (isLegacyNode) { + return ( + + ); + } + + return ( + + ); +}; + +/** 标题右侧组件：新版显示 AND/OR 切换；旧版显示「已弃用，升级到最新版本」 */ +export const DatasetTagFilterLogic = React.memo(function DatasetTagFilterLogic({ + inputs = [], + item, + nodeId +}: RenderInputProps) { + const onChangeNode = useContextSelector(WorkflowActionsContext, (v) => v.onChangeNode); + const flowData2StoreData = useContextSelector(WorkflowUtilsContext, (v) => v.flowData2StoreData); + const { appDetail, onSaveApp } = useContextSelector(AppContext, (v) => v); + const isLegacyNode = datasetSearchUsesLegacyFilter(inputs); + + if (isLegacyNode) { + return ( + { + const upgradedInput = { + ...item, + renderTypeList: [ + FlowNodeInputTypeEnum.datasetTagFilter, + FlowNodeInputTypeEnum.reference + ], + selectedType: FlowNodeInputTypeEnum.datasetTagFilter, + label: + DatasetSearchModule.inputs.find((input) => input.key === item.key)?.label ?? + item.label, + description: + DatasetSearchModule.inputs.find((input) => input.key === item.key)?.description ?? + item.description, + value: createEmptyTagFilterValue() + }; + const workflow = flowData2StoreData(); + if (!workflow) throw new Error('Workflow data is unavailable'); + await persistLegacyDatasetSearchNodeUpgrade({ + nodes: workflow.nodes, + nodeId, + filterInput: upgradedInput, + persist: (nodes) => + onSaveApp({ + ...workflow, + nodes, + isPublish: false, + chatConfig: appDetail.chatConfig + }), + commit: (upgradedNode) => + onChangeNode({ + nodeId, + type: 'attr', + key: 'inputs', + value: upgradedNode.inputs + }) + }); + }} + /> + ); + } + + return ( + { + onChangeNode({ + nodeId, + type: 'updateInput', + key: item.key, + value: { ...item, value } + }); + }} + /> + ); +}); + +export default React.memo(DatasetTagFilterRender); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx index 8dc41ac670ad..2a19106bd478 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx @@ -414,7 +414,12 @@ const CollectionTagBatchModal = ({ { label: t('dataset:tag.batch_remove'), value: BatchCollectionTagModeEnum.remove } ]} value={mode} - onChange={setMode} + onChange={(next) => { + setMode(next); + setExpandedTagIds(new Set()); + setSelectedTagIds(new Set()); + setSelectedValues(new Map()); + }} outerHeight={'40px'} itemHeight={'32px'} px={'12px'} @@ -498,7 +503,7 @@ const CollectionTagBatchModal = ({ color={'myGray.900'} noOfLines={1} > - {group.tagName}（{count}） + {`${group.tagName} (${count})`} {isArray && group.values.length > 0 && ( diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx index d59cb37f41ed..09f71a0e1a74 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx @@ -225,9 +225,9 @@ const Header = ({ )} {/* Tag */} - {datasetDetail.type !== DatasetTypeEnum.websiteDataset && - datasetDetail.permission.hasWritePer && - feConfigs?.isPlus && } + {datasetDetail.type !== DatasetTypeEnum.websiteDataset && feConfigs?.isPlus && ( + + )} {hasTrainingError && ( diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx index bc41f24723da..25e6d800ac07 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx @@ -9,7 +9,6 @@ import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; import { - collectionTagValueKey, isUsableCollectionTagFilterValue, sortCollectionTagValues } from '@fastgpt/global/core/dataset/tagUtils'; @@ -21,7 +20,7 @@ export const OVERFLOW_CHIP_GAP_PX = 8; /** * 根据测量宽度计算可见 chip 数量。空间不够时至少留 1 个，其余用 +n。 */ -export const countVisibleOverflowChips = ({ +const countVisibleOverflowChips = ({ chipWidths, overflowWidth, containerWidth, @@ -103,7 +102,7 @@ export const formatCollectionTagValueText = ( tagType?: DatasetTagType['tagType'] ): string => { if (value == null || value === '') return ''; - if (Array.isArray(value)) return value.filter(Boolean).join('、'); + if (Array.isArray(value)) return value.filter(Boolean).join(', '); if (tagType === 'datetime') { const d = dayjs(value); return d.isValid() ? formatTime2YMDHM(d.valueOf()) : String(value); @@ -146,7 +145,7 @@ export const buildTagFilterValues = ( const values = new Map(); const addValue = (value: string | number) => { if (!isUsableCollectionTagFilterValue(value)) return; - values.set(collectionTagValueKey(value), value); + values.set(String(value), value); }; if (tag.tagType === DatasetCollectionTagTypeEnum.array) { @@ -166,7 +165,7 @@ export const formatCollectionTagChipText = ( const tagType = tagDefs.find((def) => def.tag === item.tag)?.tagType; const valueText = formatCollectionTagValueText(item.value, tagType); - return valueText ? `${item.tag}：${valueText}` : item.tag; + return valueText ? `${item.tag}: ${valueText}` : item.tag; }; export const TAG_TOOLTIP_PROPS = { @@ -211,7 +210,7 @@ export const SaveActionIcon = ({ isEnabled }: { isEnabled: boolean }) => { ); }; -export type TagActionButtonProps = { +type TagActionButtonProps = { label: string; icon: React.ReactElement; onClick?: () => void; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx index 96e810bd0e59..13cbf7e0dfaa 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx @@ -24,26 +24,49 @@ import MyPopover from '@fastgpt/web/components/common/MyPopover'; import { SaveActionIcon, TagActionButton, TagTableContainer, TagTableHeader } from './TagCommon'; const TAG_TABLE_COLUMNS = 'minmax(0, 1fr) 180px 100px'; +const OPTION_ROW_HEIGHT = 32; +const OPTION_ROW_GAP = 2; +const OPTION_LIST_MAX_ROWS = 4; +const OPTION_LIST_MAX_H = + OPTION_ROW_HEIGHT * OPTION_LIST_MAX_ROWS + OPTION_ROW_GAP * (OPTION_LIST_MAX_ROWS - 1); +const normalizeTagOptions = (nextOptions: string[]) => [ + ...new Set(nextOptions.map((option) => option.trim()).filter(Boolean)) +]; + +const isSameTagOptions = (left: string[], right: string[]) => + left.length === right.length && left.every((option, index) => option === right[index]); + +/** + * 选项类标签的预设管理。 + * 输入框失焦、回车和关闭弹层时静默保存；Enter 跳到下一行，最后一行 Enter 追加空输入；列表固定 4 行，超出滚动。 + * 保存成功后不要用 options 内容做 React key，否则弹层会卸载，换行新增会丢。 + */ const TagOptionManagePopover = ({ options, - isSaving, onSave }: { options: string[]; - isSaving: boolean; onSave: (options: string[]) => Promise; }) => { const { t } = useTranslation(); const [draftOptions, setDraftOptions] = useState(options); const savedOptionsRef = useRef(options); + const draftOptionsRef = useRef(draftOptions); const inputRefs = useRef<(HTMLInputElement | null)[]>([]); - const persistOptions = async (nextOptions: string[]) => { - const normalizedOptions = [ - ...new Set(nextOptions.map((option) => option.trim()).filter(Boolean)) - ]; + useEffect(() => { + draftOptionsRef.current = draftOptions; + }, [draftOptions]); + + const resetDraft = (nextOptions: string[]) => { setDraftOptions(nextOptions); + savedOptionsRef.current = nextOptions; + }; + + const persistOptions = async (nextOptions: string[]) => { + const normalizedOptions = normalizeTagOptions(nextOptions); + if (isSameTagOptions(normalizedOptions, savedOptionsRef.current)) return; try { await onSave(normalizedOptions); @@ -53,12 +76,20 @@ const TagOptionManagePopover = ({ } }; + const focusOption = (index: number) => { + setTimeout(() => { + inputRefs.current[index]?.focus(); + }, 50); + }; + const handleAddOption = () => { setDraftOptions((prev) => { + if (prev.length > 0 && !prev[prev.length - 1]?.trim()) { + focusOption(prev.length - 1); + return prev; + } const next = [...prev, '']; - setTimeout(() => { - inputRefs.current[next.length - 1]?.focus(); - }, 50); + focusOption(next.length - 1); return next; }); }; @@ -72,21 +103,21 @@ const TagOptionManagePopover = ({ }; const handleRemoveOption = (index: number) => { - void persistOptions(draftOptions.filter((_, i) => i !== index)); + const next = draftOptionsRef.current.filter((_, i) => i !== index); + setDraftOptions(next); + void persistOptions(next); }; const handleKeyDown = (index: number, e: React.KeyboardEvent) => { - if (e.key === 'Enter') { - e.preventDefault(); - if (draftOptions[index]?.trim() && !isSaving) { - void persistOptions(draftOptions); - } - if (index === draftOptions.length - 1) { - handleAddOption(); - } else { - inputRefs.current[index + 1]?.focus(); - } + if (e.key !== 'Enter') return; + e.preventDefault(); + const currentDraft = draftOptionsRef.current; + void persistOptions(currentDraft); + if (index === currentDraft.length - 1) { + handleAddOption(); + return; } + inputRefs.current[index + 1]?.focus(); }; return ( @@ -103,6 +134,10 @@ const TagOptionManagePopover = ({ bg={'white'} border={'1px solid'} borderColor={'myGray.200'} + onOpenFunc={() => resetDraft(options)} + onCloseFunc={() => { + void persistOptions(draftOptionsRef.current); + }} Trigger={ {draftOptions.length > 0 && ( - + {draftOptions.map((opt, index) => ( handleUpdateOption(index, e.target.value)} + onBlur={() => { + void persistOptions(draftOptionsRef.current); + }} onKeyDown={(e) => handleKeyDown(index, e)} /> !isSaving && handleRemoveOption(index)} + cursor={'pointer'} + _hover={{ bg: 'myGray.05' }} + onMouseDown={(e) => e.preventDefault()} + onClick={() => handleRemoveOption(index)} > @@ -273,7 +316,7 @@ const TagManageModal = ({ onClose }: { onClose: () => void }) => { } ); - const { runAsync: onSaveTagOptions, loading: isSavingTagOptions } = useRequest( + const { runAsync: onSaveTagOptions } = useRequest( ({ tag, options }: { tag: DatasetTagType; options: string[] }) => updateDatasetCollectionTag({ datasetId: datasetDetail._id, @@ -458,10 +501,8 @@ const TagManageModal = ({ onClose }: { onClose: () => void }) => { {t(DatasetCollectionTagTypeMap[tagType].label)} {tagType === DatasetCollectionTagTypeEnum.array && ( onSaveTagOptions({ tag, options })} + onSave={(nextOptions) => onSaveTagOptions({ tag, options: nextOptions })} /> )} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx index 44be11df78a1..196b39b6af16 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx @@ -40,7 +40,7 @@ export const tagInputBaseStyles: InputProps = { color: 'myGray.500' } }; -export const StringTagInput = ({ +const StringTagInput = ({ value, onChange, placeholder @@ -55,25 +55,34 @@ export const StringTagInput = ({ onChange(e.target.value)} /> ); }; +const embeddedFieldStyles = { + border: 'none', + boxShadow: 'none', + _hover: { border: 'none' }, + _focus: { border: 'none', boxShadow: 'none' } +}; + export const NumberTagInput = ({ value, onChange, placeholder, - showStepper + showStepper, + embedded }: { value?: number | string; onChange: (val: number | '') => void; placeholder?: string; showStepper?: boolean; + embedded?: boolean; }) => { const { t } = useTranslation(); - const placeholderText = placeholder || t('dataset:tag.fill_number'); + const placeholderText = placeholder ?? t('dataset:tag.fill_number'); if (showStepper) { return ( @@ -86,7 +95,8 @@ export const NumberTagInput = ({ h: '36px', bg: 'white', fontSize: 'sm', - color: 'myGray.900' + color: 'myGray.900', + ...(embedded ? embeddedFieldStyles : {}) }} value={value === undefined || value === '' ? '' : Number(value)} placeholder={placeholderText} @@ -98,6 +108,7 @@ export const NumberTagInput = ({ return ( void; placeholder?: string; + embedded?: boolean; }) => ( ); @@ -350,13 +366,17 @@ export const ArrayTagSelect = ({ value = [], onChange, onCreateOption, - placeholder + allowCreate = true, + placeholder, + embedded }: { options: string[]; value?: string[]; onChange: (val: string[]) => void; onCreateOption?: (option: string) => void; + allowCreate?: boolean; placeholder?: string; + embedded?: boolean; }) => { const { t } = useTranslation(); const { isOpen, onOpen, onClose } = useDisclosure(); @@ -383,7 +403,9 @@ export const ArrayTagSelect = ({ }, [search, allOptions]); const canCreate = Boolean( - search.trim() && !allOptions.some((opt) => opt.toLowerCase() === search.trim().toLowerCase()) + allowCreate && + search.trim() && + !allOptions.some((opt) => opt.toLowerCase() === search.trim().toLowerCase()) ); const handleCreateOption = () => { @@ -444,11 +466,12 @@ export const ArrayTagSelect = ({ overflow={'hidden'} borderColor={isOpen ? 'primary.600' : 'myGray.200'} boxShadow={isOpen ? 'focus' : 'none'} + {...(embedded ? embeddedFieldStyles : {})} > {value.length === 0 ? ( - {placeholder || t('dataset:tag.select_options')} + {placeholder ?? t('dataset:tag.select_options')} ) : ( <> @@ -497,7 +520,7 @@ export const ArrayTagSelect = ({ { if (e.key === 'Enter') { e.preventDefault(); - handleCreateOption(); + if (allowCreate) handleCreateOption(); } }} /> @@ -638,7 +663,7 @@ export const TagValueField = ({ ); } diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx index 1628be7e2b0b..550d5d7fb77f 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx @@ -1,12 +1,17 @@ -import { Flex } from '@chakra-ui/react'; +import { Box, Flex } from '@chakra-ui/react'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { useMemo } from 'react'; +import { useLayoutEffect, useMemo, useState } from 'react'; import { type DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type'; import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; -import MyPopover from '@fastgpt/web/components/common/MyPopover'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import MyTag from '@fastgpt/web/components/common/Tag/index'; -import { formatCollectionTagChipText, useOverflowChipCount } from './TagCommon'; +import { + formatCollectionTagChipText, + OVERFLOW_CHIP_GAP_PX, + TAG_TOOLTIP_PROPS, + useOverflowChipCount +} from './TagCommon'; const TAG_CHIP_PROPS = { colorSchema: 'cyan' as const, @@ -19,11 +24,50 @@ const TAG_CHIP_PROPS = { borderRadius: 'xs' as const }; -const TagChip = ({ text }: { text: string }) => ( - - {text} - -); +/** 渲染知识库列表中的标签；单个标签成为唯一可见项时允许收缩并展示完整文本。 */ +const TagChip = ({ + text, + isFlexible = false, + withTooltip = false +}: { + text: string; + isFlexible?: boolean; + withTooltip?: boolean; +}) => { + const tagText = ( + + {text} + + ); + + const tagContent = withTooltip ? ( + + {tagText} + + ) : ( + tagText + ); + + const chip = ( + + {tagContent} + + ); + + return chip; +}; const TagsPopOver = ({ currentCollection @@ -48,6 +92,33 @@ const TagsPopOver = ({ itemCount: chipItems.length }); + const [shouldShrinkFirstTag, setShouldShrinkFirstTag] = useState(false); + useLayoutEffect(() => { + const container = containerRef.current; + const measure = measureRef.current; + if (!container || !measure) return; + + const calculate = () => { + const firstTag = measure.querySelector('[data-tag-chip]') as HTMLElement | null; + const overflowChip = container.querySelector('[data-overflow-chip]') as HTMLElement | null; + const hasOnlyOneVisibleTag = visibleCount === 1 && chipItems.length > 1; + + if (!firstTag || !overflowChip || !hasOnlyOneVisibleTag) { + setShouldShrinkFirstTag(false); + return; + } + + const availableWidth = + container.offsetWidth - overflowChip.offsetWidth - OVERFLOW_CHIP_GAP_PX; + setShouldShrinkFirstTag(firstTag.offsetWidth > availableWidth); + }; + + calculate(); + const observer = new ResizeObserver(calculate); + observer.observe(container); + return () => observer.disconnect(); + }, [chipItems.length, containerRef, measureRef, visibleCount]); + if (chipItems.length === 0) return null; const visibleTags = chipItems.slice(0, visibleCount); @@ -83,34 +154,31 @@ const TagsPopOver = ({ h={'20px'} overflow={'hidden'} > - {visibleTags.map((item) => ( - + {visibleTags.map((item, index) => ( + ))} {overflowTags.length > 0 && ( - e.stopPropagation()}> - - {`+${overflowTags.length}`} - - - } + item.text).join('\n')} + shouldWrapChildren={false} + {...TAG_TOOLTIP_PROPS} > - {() => ( - e.stopPropagation()}> - {overflowTags.map((item) => ( - - ))} - - )} - + e.stopPropagation()} + _hover={{ bg: '#DBF3FF' }} + > + + {`+${overflowTags.length}`} + + + )} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts index 4f79aac6ff22..5425cb620436 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts @@ -28,7 +28,7 @@ export const useAppendDatasetTagOption = () => { }); }, { - refreshDeps: [datasetDetail._id, allDatasetTags], + refreshDeps: [datasetDetail._id], onSuccess: loadAllDatasetTags, errorToast: t('dataset:tag.save_failed') } diff --git a/projects/app/src/pages/api/admin/migrateTags.ts b/projects/app/src/pages/api/admin/migrateTags.ts deleted file mode 100644 index 6a6f4dc8e985..000000000000 --- a/projects/app/src/pages/api/admin/migrateTags.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { NextAPI } from '@/service/middleware/entry'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; -import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { - DEFAULT_TAG, - DatasetCollectionTagTypeEnum, - type CollectionTagValueType -} from '@fastgpt/global/core/dataset/type'; -import { getLogger } from '@fastgpt/service/common/logger'; - -const logger = getLogger(['migrateTags']); -const isLegacyTag = (tag: unknown): tag is string => typeof tag === 'string'; - -export default NextAPI(async function handler(req) { - await authCert({ req, authRoot: true }); - - // 清理 v2 表重复行，避免 unique 索引构建失败；保留 _id 最早的一条（ObjectId 含创建时间，排序即时间序） - const dupGroups = await MongoDatasetCollectionTagsV2.aggregate([ - { - $group: { - _id: { teamId: '$teamId', datasetId: '$datasetId', tag: '$tag' }, - ids: { $push: '$_id' } - } - }, - { $match: { $expr: { $gt: [{ $size: '$ids' }, 1] } } } - ]); - for (const group of dupGroups) { - const [, ...dups] = [...group.ids].sort(); - await MongoDatasetCollectionTagsV2.deleteMany({ _id: { $in: dups } }); - } - - const datasetIds = (await MongoDatasetCollectionTags.distinct('datasetId')) as string[]; - logger.info(`[TagMigration] Starting tag migration for ${datasetIds.length} datasets`); - let migratedDatasets = 0; - let migratedCollections = 0; - - for (const datasetId of datasetIds) { - const legacyTags = await MongoDatasetCollectionTags.find( - { datasetId }, - '_id tag teamId' - ).lean(); - const tagNameMap = new Map(legacyTags.map((tag) => [String(tag._id), tag.tag])); - logger.info( - `[TagMigration] datasetId=${datasetId}, legacyTags=${JSON.stringify(Array.from(tagNameMap.values()))}` - ); - if (tagNameMap.size === 0) continue; - const firstLegacyTag = legacyTags[0]; - if (!firstLegacyTag) continue; - - // default_tag 承载记录按 fromMigration 定位；存量按名称创建的记录回填标记并确保 tagType=array - let defaultTag = await MongoDatasetCollectionTagsV2.findOne({ - datasetId, - fromMigration: true - }).lean(); - if (!defaultTag) { - const legacyDefaultTag = await MongoDatasetCollectionTagsV2.findOne({ - datasetId, - tag: DEFAULT_TAG - }).lean(); - if (legacyDefaultTag) { - await MongoDatasetCollectionTagsV2.updateOne( - { _id: legacyDefaultTag._id }, - { $set: { fromMigration: true, tagType: DatasetCollectionTagTypeEnum.array } } - ); - defaultTag = { - ...legacyDefaultTag, - fromMigration: true, - tagType: DatasetCollectionTagTypeEnum.array - }; - } else { - try { - const createdTag = await MongoDatasetCollectionTagsV2.create({ - teamId: firstLegacyTag.teamId, - datasetId, - tag: DEFAULT_TAG, - tagType: DatasetCollectionTagTypeEnum.array, - fromMigration: true - }); - defaultTag = createdTag.toObject(); - } catch (error: any) { - // 并发创建撞 unique 索引 → 复用已存在记录 - if (error?.code !== 11000) throw error; - defaultTag = await MongoDatasetCollectionTagsV2.findOne({ - datasetId, - fromMigration: true - }).lean(); - if (!defaultTag) throw error; - } - } - } - if (!defaultTag) continue; - const defaultTagId = String(defaultTag._id); - - const collections = await MongoDatasetCollection.find({ datasetId }, '_id tags').lean(); - let migratedInDataset = 0; - for (const collection of collections) { - const tags = (Array.isArray(collection.tags) ? collection.tags : []) as ( - | string - | CollectionTagValueType - )[]; - if (!tags.some(isLegacyTag)) continue; - - const tagNames = tags - .filter(isLegacyTag) - .map((tagId) => tagNameMap.get(tagId)) - .filter((tagName): tagName is string => Boolean(tagName)); - const existingDefaultTag = tags.find( - (tag): tag is CollectionTagValueType => - typeof tag === 'object' && tag !== null && tag.tagId === defaultTagId - ); - const existingDefaultValues = Array.isArray(existingDefaultTag?.value) - ? existingDefaultTag.value.filter((value): value is string => typeof value === 'string') - : []; - const mergedTagNames = [...new Set([...existingDefaultValues, ...tagNames])]; - const migratedTags: (string | CollectionTagValueType)[] = tags.filter( - (tag) => !isLegacyTag(tag) && tag !== existingDefaultTag - ); - migratedTags.push({ tagId: defaultTagId, value: mergedTagNames }); - logger.info( - `[TagMigration] collectionId=${String(collection._id)}, collectionTags=${JSON.stringify(tags)}` - ); - - await MongoDatasetCollection.updateOne( - { _id: collection._id }, - { $set: { tags: migratedTags } } - ); - migratedCollections += 1; - migratedInDataset += 1; - } - - if (migratedInDataset > 0 || defaultTag) migratedDatasets += 1; - } - - return { migratedDatasets, migratedCollections }; -}); diff --git a/projects/app/src/pages/api/core/dataset/collection/create/backup.ts b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts index bb9cb81c04aa..88c1fae72ce7 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/backup.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts @@ -29,7 +29,7 @@ async function handler(req: ApiRequestProps) { }); filepaths.push(result.fileMetadata.path); const filename = decodeMultipartFilename(result.fileMetadata.originalname); - const { datasetId, parentId } = CreateBackupCollectionFormSchema.parse(result.data); + const { datasetId, parentId, tags } = CreateBackupCollectionFormSchema.parse(result.data); const { teamId, tmbId, dataset } = await authDataset({ req, @@ -82,7 +82,8 @@ async function handler(req: ApiRequestProps) { name: filename, type: DatasetCollectionTypeEnum.file, fileId, - trainingType: DatasetCollectionDataProcessModeEnum.backup + trainingType: DatasetCollectionDataProcessModeEnum.backup, + tags } }); promoted = true; diff --git a/projects/app/src/pages/api/core/dataset/collection/create/images.ts b/projects/app/src/pages/api/core/dataset/collection/create/images.ts index 7e0ba396f2dc..6405cfaa343c 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/images.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/images.ts @@ -35,7 +35,7 @@ async function handler(req: ApiRequestProps): Promise item.path)); - const { parentId, datasetId, collectionName } = CreateImageCollectionFormSchema.parse( + const { parentId, datasetId, collectionName, tags } = CreateImageCollectionFormSchema.parse( result.data ); @@ -93,6 +93,7 @@ async function handler(req: ApiRequestProps): Promise POST(`/proApi/core/dataset/tag/create`, data); -export const postAddTagsToCollections = (data: AddTagsToCollectionsParams) => - POST(`/proApi/core/dataset/tag/addToCollections`, data); export const delDatasetCollectionTag = (data: { id: string; datasetId: string }) => DELETE(`/proApi/core/dataset/tag/delete`, data); export const updateDatasetCollectionTag = (data: UpdateDatasetCollectionTagParams) => POST(`/proApi/core/dataset/tag/update`, data); -export const getTagUsage = (datasetId: string) => - GET(`/proApi/core/dataset/tag/tagUsage?datasetId=${datasetId}`); export const getAllTags = (datasetId: string) => GET<{ list: DatasetTagType[] }>(`/proApi/core/dataset/tag/getAllTags?datasetId=${datasetId}`); export const getDatasetTagFilterOptions = (datasetId: string) => diff --git a/projects/app/src/web/core/workflow/datasetSearchNodeUpgrade.ts b/projects/app/src/web/core/workflow/datasetSearchNodeUpgrade.ts new file mode 100644 index 000000000000..1221101b4a7a --- /dev/null +++ b/projects/app/src/web/core/workflow/datasetSearchNodeUpgrade.ts @@ -0,0 +1,73 @@ +import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io'; +import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node'; +import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; +import { + DatasetTagFilterVersionEnum, + resolveDatasetTagFilterVersion +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { Input_Template_Dataset_Tag_Filter_Version } from '@fastgpt/global/core/workflow/template/input'; + +/** 从节点输入读取唯一的过滤版本来源，不检查 collectionFilterMatch 的值形状。 */ +export const getDatasetSearchFilterVersion = (inputs: FlowNodeInputItemType[]) => + resolveDatasetTagFilterVersion({ + version: inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion)?.value, + filterValue: inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterMatch)?.value + }); + +export const datasetSearchUsesLegacyFilter = (inputs: FlowNodeInputItemType[]) => + getDatasetSearchFilterVersion(inputs) === DatasetTagFilterVersionEnum.legacy; + +/** 只更新过滤版本和过滤输入，nodeId、节点类型、其他参数及连线引用保持不变。 */ +export const upgradeLegacyDatasetSearchNode = ({ + node, + filterInput +}: { + node: StoreNodeItemType; + filterInput: FlowNodeInputItemType; +}): StoreNodeItemType => { + if (filterInput.key !== NodeInputKeyEnum.collectionFilterMatch) { + throw new Error('Dataset search filter upgrade received an invalid filter input'); + } + const structuredVersionInput = { + ...Input_Template_Dataset_Tag_Filter_Version, + value: DatasetTagFilterVersionEnum.structured + }; + const inputs = node.inputs + .filter( + (input) => + input.key !== NodeInputKeyEnum.collectionFilterVersion && + input.key !== NodeInputKeyEnum.collectionFilterMatch + ) + .concat(structuredVersionInput, filterInput); + + return { + ...node, + inputs + }; +}; + +/** 先持久化再提交本地升级；持久化失败时 commit 不会执行。 */ +export const persistLegacyDatasetSearchNodeUpgrade = async ({ + nodes, + nodeId, + filterInput, + persist, + commit +}: { + nodes: StoreNodeItemType[]; + nodeId: string; + filterInput: FlowNodeInputItemType; + persist: (nodes: StoreNodeItemType[]) => Promise; + commit: (node: StoreNodeItemType) => void; +}) => { + let upgradedNode: StoreNodeItemType | undefined; + const upgradedNodes = nodes.map((node) => { + if (node.nodeId !== nodeId) return node; + upgradedNode = upgradeLegacyDatasetSearchNode({ node, filterInput }); + return upgradedNode; + }); + if (!upgradedNode) throw new Error(`Dataset search node not found: ${nodeId}`); + + await persist(upgradedNodes); + commit(upgradedNode); +}; diff --git a/projects/app/src/web/core/workflow/utils.ts b/projects/app/src/web/core/workflow/utils.ts index 253065c3c81c..d580d32d332e 100644 --- a/projects/app/src/web/core/workflow/utils.ts +++ b/projects/app/src/web/core/workflow/utils.ts @@ -38,6 +38,10 @@ import { workflowSystemVariables } from '../app/utils'; import type { WorkflowDataContextType } from '@/pageComponents/app/detail/WorkflowComponents/context/workflowInitContext'; import type { MyLLMModelItemType } from '@fastgpt/global/openapi/core/ai/model/api'; import { normalizeFlowNodeInputType } from '@fastgpt/global/core/app/formEdit/utils'; +import { + DatasetTagFilterVersionEnum, + resolveDatasetTagFilterVersion +} from '@fastgpt/global/core/dataset/workflowTagFilter'; /** * 将节点模板转换为画布节点，并按创建时语言初始化可编辑文本。 @@ -198,6 +202,20 @@ export const storeNode2FlowNode = ({ isWorkflowSystemModelInput({ node: storeNode, input: templateInput }) ); }; + const collectionFilterVersion = + storeNode.flowNodeType === FlowNodeTypeEnum.datasetSearchNode + ? resolveDatasetTagFilterVersion({ + version: adaptedStoreInputs.find( + (input) => input.key === NodeInputKeyEnum.collectionFilterVersion + )?.value, + filterValue: adaptedStoreInputs.find( + (input) => input.key === NodeInputKeyEnum.collectionFilterMatch + )?.value + }) + : DatasetTagFilterVersionEnum.structured; + const usesLegacyDatasetSearchFilter = + storeNode.flowNodeType === FlowNodeTypeEnum.datasetSearchNode && + collectionFilterVersion === DatasetTagFilterVersionEnum.legacy; // replace item data const nodeItem: FlowNodeItemType = { @@ -223,8 +241,30 @@ export const storeNode2FlowNode = ({ debugLabel: t(inputTemplate.debugLabel ?? (storeInput.debugLabel as any)), toolDescription: t(inputTemplate.toolDescription ?? (storeInput.toolDescription as any)), key: storeInput.key, - selectedType: storeInput.selectedType ?? inputTemplate.selectedType, - value: storeInput.value + label: + usesLegacyDatasetSearchFilter && + inputTemplate.key === NodeInputKeyEnum.collectionFilterMatch + ? 'workflow:collection_metadata_filter' + : inputTemplate.label, + description: + usesLegacyDatasetSearchFilter && + inputTemplate.key === NodeInputKeyEnum.collectionFilterMatch + ? 'workflow:filter_description' + : inputTemplate.description, + selectedType: (() => { + // 旧节点用 textarea 手写 JSON；切到条件行渲染类型，但保留字符串 value 以便展示升级 UI。 + if ( + inputTemplate.key === NodeInputKeyEnum.collectionFilterMatch && + storeInput.selectedType === FlowNodeInputTypeEnum.textarea + ) { + return FlowNodeInputTypeEnum.datasetTagFilter; + } + return storeInput.selectedType ?? inputTemplate.selectedType; + })(), + value: + inputTemplate.key === NodeInputKeyEnum.collectionFilterVersion + ? collectionFilterVersion + : storeInput.value }; }) .concat( diff --git a/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts b/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts index 0ef688b0f18c..230148decbe1 100644 --- a/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts +++ b/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts @@ -91,7 +91,7 @@ describe('dataset multipart file source lifecycle', () => { vi.clearAllMocks(); source.materialize.mockReset(); mockResolveFormData.mockResolvedValue({ - data: { datasetId }, + data: { datasetId, tags: [{ tag: 'score', value: 10 }] }, fileMetadata: { path: '/tmp/source.csv', originalname: 'source.csv', @@ -140,6 +140,13 @@ describe('dataset multipart file source lifecycle', () => { mockParseDatasetImportFile.mock.invocationCallOrder[0] ); expect(mockCleanupPendingDatasetFile).not.toHaveBeenCalled(); + expect(mockCreateCollectionAndInsertData).toHaveBeenCalledWith( + expect.objectContaining({ + createCollectionParams: expect.objectContaining({ + tags: [{ tag: 'score', value: 10 }] + }) + }) + ); } ); diff --git a/projects/app/test/api/core/dataset/collection/create/images.test.ts b/projects/app/test/api/core/dataset/collection/create/images.test.ts index dd4d7b895b77..ca6a8070fff2 100644 --- a/projects/app/test/api/core/dataset/collection/create/images.test.ts +++ b/projects/app/test/api/core/dataset/collection/create/images.test.ts @@ -108,7 +108,8 @@ describe('POST /api/core/dataset/collection/create/images', () => { data: { parentId, datasetId, - collectionName: 'Native image embedding collection' + collectionName: 'Native image embedding collection', + tags: [{ tag: 'score', value: 10 }] }, fileMetadata: [ { @@ -174,6 +175,7 @@ describe('POST /api/core/dataset/collection/create/images', () => { datasetId, type: DatasetCollectionTypeEnum.images, name: 'Native image embedding collection', + tags: [{ tag: 'score', value: 10 }], trainingType: DatasetCollectionDataProcessModeEnum.chunk } }); diff --git a/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts b/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts index be016c90ba96..d20490e6f0e4 100644 --- a/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts +++ b/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts @@ -57,6 +57,18 @@ describe('collection tag filter options and listV2 tagFilters', () => { tmbId: root.tmbId, datasetId: dataset._id, tags: [{ tagId: String(docType._id), value: ['spec'] }] + }, + { + name: 'noise-file', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [ + { tagId: String(docType._id), value: [''] }, + { tagId: String(version._id), value: 0 }, + 'legacy-id' + ] } ]); @@ -69,7 +81,7 @@ describe('collection tag filter options and listV2 tagFilters', () => { const docTypeOption = optionsRes.data.list.find((item) => item.tagId === String(docType._id)); const versionOption = optionsRes.data.list.find((item) => item.tagId === String(version._id)); expect(docTypeOption?.values).toEqual(['PRD', 'spec']); - expect(versionOption?.values).toEqual([2]); + expect(versionOption?.values).toEqual([0, 2]); const listRes = await Call(listHandler, { auth: root, @@ -84,68 +96,4 @@ describe('collection tag filter options and listV2 tagFilters', () => { expect(listRes.code).toBe(200); expect(listRes.data.list.map((item) => item.name)).toEqual(['prd-file']); }); - - it('requires every selected tag to match when combining filters', async () => { - const root = await getRootUser(); - const dataset = await MongoDataset.create({ - name: 'tag-filter-and-dataset', - teamId: root.teamId, - tmbId: root.tmbId, - vectorModel: 'test', - agentModel: 'test' - }); - - const [docType, version] = await MongoDatasetCollectionTagsV2.create([ - { - teamId: root.teamId, - datasetId: dataset._id, - tag: '文档类型', - tagType: DatasetCollectionTagTypeEnum.array - }, - { - teamId: root.teamId, - datasetId: dataset._id, - tag: '版本', - tagType: DatasetCollectionTagTypeEnum.number - } - ]); - - await MongoDatasetCollection.create([ - { - name: 'both', - type: DatasetCollectionTypeEnum.file, - teamId: root.teamId, - tmbId: root.tmbId, - datasetId: dataset._id, - tags: [ - { tagId: String(docType._id), value: ['PRD'] }, - { tagId: String(version._id), value: 2 } - ] - }, - { - name: 'only-type', - type: DatasetCollectionTypeEnum.file, - teamId: root.teamId, - tmbId: root.tmbId, - datasetId: dataset._id, - tags: [{ tagId: String(docType._id), value: ['PRD'] }] - } - ]); - - const listRes = await Call(listHandler, { - auth: root, - body: { - datasetId: String(dataset._id), - pageSize: 10, - offset: 0, - tagFilters: [ - { tagId: String(docType._id), values: ['PRD'] }, - { tagId: String(version._id), values: [2] } - ] - } - }); - - expect(listRes.code).toBe(200); - expect(listRes.data.list.map((item) => item.name)).toEqual(['both']); - }); }); diff --git a/projects/app/test/migration/registry.test.ts b/projects/app/test/migration/registry.test.ts index 63b253b5c6e3..f256a55472cc 100644 --- a/projects/app/test/migration/registry.test.ts +++ b/projects/app/test/migration/registry.test.ts @@ -24,7 +24,7 @@ describe('validateSystemMigrationRegistry', () => { onFailure: SystemMigrationFailurePolicyEnum.stop, progressSteps: [{ key: 'members' }, { key: 'validation' }] }); - expect(systemMigrations.slice(2).map((migration) => migration.id)).toEqual([ + expect(systemMigrations.slice(2, -1).map((migration) => migration.id)).toEqual([ '20260903_backfill_model_permissions', '20260903_backfill_dataset_model_references', '20260903_backfill_evaluation_model_references', @@ -33,10 +33,10 @@ describe('validateSystemMigrationRegistry', () => { '20260905_backfill_bill_metadata', '20260905_backfill_resource_owner_acl' ]); - expect(systemMigrations.slice(2).every((migration) => !migration.blockStartup)).toBe(true); + expect(systemMigrations.slice(2, -1).every((migration) => !migration.blockStartup)).toBe(true); expect( systemMigrations - .slice(2) + .slice(2, -1) .every((migration) => migration.onFailure === SystemMigrationFailurePolicyEnum.continue) ).toBe(true); expect(systemMigrations[1]).toMatchObject({ @@ -45,6 +45,13 @@ describe('validateSystemMigrationRegistry', () => { blockStartup: true, onFailure: SystemMigrationFailurePolicyEnum.stop }); + expect(systemMigrations.at(-1)).toMatchObject({ + id: '20260907_migrate_dataset_tags_v2', + version: '4.17.0', + blockStartup: true, + onFailure: SystemMigrationFailurePolicyEnum.stop, + progressSteps: [{ key: 'datasets' }, { key: 'collections' }, { key: 'validation' }] + }); }); it('accepts an ordered registry with stable IDs', () => { diff --git a/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/index.test.ts b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/index.test.ts new file mode 100644 index 000000000000..9e2c3eb03935 --- /dev/null +++ b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/index.test.ts @@ -0,0 +1,172 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { SystemMigrationContext } from '@/migration/registry'; +import type { SystemMigrationProgressInput } from '@fastgpt/global/migration/schema'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { migrateDatasetTagsV2 } from '@/migration/tasks/20260907_migrate_dataset_tags_v2'; + +vi.mock('@/migration/constants', () => ({ systemMigrationBatchSize: 1 })); + +const createContext = () => { + let checkpoint: Record | undefined; + const context = { + migrationId: '20260907_migrate_dataset_tags_v2', + runId: 'test-run', + signal: new AbortController().signal, + getCheckpoint: async (schema) => + checkpoint === undefined ? undefined : schema.parse(checkpoint), + saveCheckpoint: vi.fn(async (value: Record) => { + checkpoint = structuredClone(value); + }), + assertActive: vi.fn(async () => undefined), + reportProgress: vi.fn(async (_value: SystemMigrationProgressInput) => undefined), + getFailedRecords: vi.fn(async () => []), + reportFailedRecords: vi.fn(async () => undefined), + fail: vi.fn(async () => { + throw new Error('Blocking tasks must throw'); + }), + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } + } satisfies SystemMigrationContext; + return { context, getCheckpoint: () => checkpoint }; +}; + +const seedMigrationData = async () => { + const teamId = new Types.ObjectId(); + const tmbId = new Types.ObjectId(); + const datasetId = new Types.ObjectId(); + await MongoDataset.collection.insertOne({ + _id: datasetId, + teamId, + tmbId, + name: 'dataset', + type: 'dataset' + }); + const legacyTagId = new Types.ObjectId(); + await MongoDatasetCollectionTags.collection.insertOne({ + _id: legacyTagId, + teamId, + datasetId, + tag: 'legacy' + }); + const collectionId = new Types.ObjectId(); + await MongoDatasetCollection.collection.insertOne({ + _id: collectionId, + teamId, + tmbId, + datasetId, + name: 'file', + type: 'file', + tags: [String(legacyTagId)] + }); + return { datasetId, collectionId }; +}; + +describe('migrateDatasetTagsV2', () => { + beforeEach(async () => { + await Promise.all([ + MongoDataset.collection.deleteMany({}), + MongoDatasetCollection.collection.deleteMany({}), + MongoDatasetCollectionTags.collection.deleteMany({}), + MongoDatasetCollectionTagsV2.collection.deleteMany({}) + ]); + }); + + it('migrates legacy collection tags before validation succeeds', async () => { + const ids = await seedMigrationData(); + const { context } = createContext(); + + await expect(migrateDatasetTagsV2(context)).resolves.toMatchObject({ + datasetsProcessedCount: 1, + collectionsMigratedCount: 1, + legacyCollectionCount: 0 + }); + const carrier = await MongoDatasetCollectionTagsV2.collection.findOne({ + datasetId: ids.datasetId, + fromMigration: true + }); + expect(carrier).toMatchObject({ tag: 'default_tag', tagType: 'array' }); + expect( + (await MongoDatasetCollection.collection.findOne({ _id: ids.collectionId }))?.tags + ).toEqual([{ tagId: String(carrier?._id), value: ['legacy'] }]); + expect(context.getFailedRecords).not.toHaveBeenCalled(); + expect(context.reportFailedRecords).not.toHaveBeenCalled(); + }); + + it('can rerun without duplicating carriers or changing migrated data', async () => { + const ids = await seedMigrationData(); + await migrateDatasetTagsV2(createContext().context); + const firstCollection = await MongoDatasetCollection.collection.findOne({ + _id: ids.collectionId + }); + + await expect(migrateDatasetTagsV2(createContext().context)).resolves.toMatchObject({ + collectionsMigratedCount: 0, + duplicateDefinitionsDeletedCount: 0, + legacyCollectionCount: 0 + }); + expect( + await MongoDatasetCollectionTagsV2.collection.countDocuments({ + datasetId: ids.datasetId, + fromMigration: true + }) + ).toBe(1); + expect(await MongoDatasetCollection.collection.findOne({ _id: ids.collectionId })).toEqual( + firstCollection + ); + }); + + it('resumes from the saved checkpoint and safely replays after interruption', async () => { + await seedMigrationData(); + const state = createContext(); + state.context.reportProgress.mockImplementation(async (progress) => { + if (progress.key === 'datasets' && progress.status === 'running' && progress.current === 1) { + throw new Error('interrupted'); + } + }); + + await expect(migrateDatasetTagsV2(state.context)).rejects.toThrow('interrupted'); + expect(state.getCheckpoint()).toMatchObject({ + version: 1, + stages: { datasets: { lastId: expect.any(String), processedCount: 1 } } + }); + state.context.reportProgress.mockResolvedValue(undefined); + await expect(migrateDatasetTagsV2(state.context)).resolves.toMatchObject({ + datasetsProcessedCount: 1, + collectionsMigratedCount: 1 + }); + }); + + it('does not complete when final validation observes a late legacy write', async () => { + const ids = await seedMigrationData(); + const state = createContext(); + state.context.reportProgress.mockImplementation(async (progress) => { + if (progress.key !== 'validation' || progress.status !== 'running') return; + const dataset = await MongoDataset.collection.findOne({ _id: ids.datasetId }); + await MongoDatasetCollection.collection.insertOne({ + teamId: dataset?.teamId, + tmbId: new Types.ObjectId(), + datasetId: ids.datasetId, + name: 'late legacy write', + type: 'file', + tags: [String(new Types.ObjectId())] + }); + }); + + await expect(migrateDatasetTagsV2(state.context)).rejects.toThrow( + 'Dataset tag migration validation failed' + ); + expect(state.getCheckpoint()).toMatchObject({ + stages: { + datasets: { initialized: false, completed: false }, + collections: { initialized: false, completed: false } + } + }); + expect(state.context.reportProgress).not.toHaveBeenCalledWith({ + key: 'validation', + status: 'succeeded' + }); + }); +}); diff --git a/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/service.test.ts b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/service.test.ts new file mode 100644 index 000000000000..5b21ee4a6971 --- /dev/null +++ b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/service.test.ts @@ -0,0 +1,143 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { + migrateCollectionTagValues, + migrateDatasetTagDefinitions +} from '@/migration/tasks/20260907_migrate_dataset_tags_v2/service'; + +const tagUniqueIndexName = 'teamId_1_datasetId_1_tag_1'; +const teamId = new Types.ObjectId(); +const datasetId = new Types.ObjectId(); +const tmbId = new Types.ObjectId(); + +describe('dataset tag v2 migration service', () => { + beforeEach(async () => { + await Promise.all([ + MongoDatasetCollection.collection.deleteMany({ teamId }), + MongoDatasetCollectionTags.collection.deleteMany({ teamId }), + MongoDatasetCollectionTagsV2.collection.deleteMany({ teamId }) + ]); + }); + + afterEach(async () => { + await Promise.all([ + MongoDatasetCollection.collection.deleteMany({ teamId }), + MongoDatasetCollectionTags.collection.deleteMany({ teamId }), + MongoDatasetCollectionTagsV2.collection.deleteMany({ teamId }) + ]); + await MongoDatasetCollectionTagsV2.createIndexes({ background: true }); + }); + + it('removes duplicate references before definitions and remains idempotent', async () => { + await MongoDatasetCollectionTagsV2.collection.dropIndex(tagUniqueIndexName).catch(() => {}); + const keptId = new Types.ObjectId(); + const duplicateId = new Types.ObjectId(); + const unrelatedId = new Types.ObjectId(); + await MongoDatasetCollectionTagsV2.collection.insertMany([ + { _id: keptId, teamId, datasetId, tag: 'product', tagType: 'string' }, + { _id: duplicateId, teamId, datasetId, tag: 'product', tagType: 'string' }, + { _id: unrelatedId, teamId, datasetId, tag: 'score', tagType: 'number' } + ]); + const collection = await MongoDatasetCollection.collection.insertOne({ + teamId, + tmbId, + datasetId, + name: 'file', + type: 'file', + tags: [ + { tagId: String(duplicateId), value: 'FastGPT' }, + { tagId: String(unrelatedId), value: 10 } + ] + }); + + await expect(migrateDatasetTagDefinitions({ datasetId, teamId })).resolves.toMatchObject({ + migratedCount: 1, + deletedDefinitionCount: 1, + deletedReferenceCollectionCount: 1 + }); + expect(await MongoDatasetCollectionTagsV2.collection.findOne({ _id: keptId })).toBeTruthy(); + expect(await MongoDatasetCollectionTagsV2.collection.findOne({ _id: duplicateId })).toBeNull(); + expect( + (await MongoDatasetCollection.collection.findOne({ _id: collection.insertedId }))?.tags + ).toEqual([{ tagId: String(unrelatedId), value: 10 }]); + expect( + await MongoDatasetCollectionTagsV2.collection.countDocuments({ teamId, datasetId }) + ).toBe(2); + + await expect(migrateDatasetTagDefinitions({ datasetId, teamId })).resolves.toMatchObject({ + deletedDefinitionCount: 0, + deletedReferenceCollectionCount: 0 + }); + }); + + it('converts legacy tag ids into the dataset carrier array and preserves typed tags', async () => { + const legacy = await MongoDatasetCollectionTags.create({ + teamId, + datasetId, + tag: 'legacy-name' + }); + const typed = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId, + tag: 'score', + tagType: 'number' + }); + const collection = await MongoDatasetCollection.collection.insertOne({ + teamId, + tmbId, + datasetId, + name: 'file', + type: 'file', + tags: [String(legacy._id), { tagId: String(typed._id), value: 10 }] + }); + + await expect( + migrateCollectionTagValues({ collectionId: collection.insertedId }) + ).resolves.toEqual({ migratedCount: 1 }); + const carrier = await MongoDatasetCollectionTagsV2.collection.findOne({ + teamId, + datasetId, + fromMigration: true + }); + expect(carrier).toMatchObject({ tag: 'default_tag', tagType: 'array' }); + expect( + (await MongoDatasetCollection.collection.findOne({ _id: collection.insertedId }))?.tags + ).toEqual([ + { tagId: String(typed._id), value: 10 }, + { tagId: String(carrier?._id), value: ['legacy-name'] } + ]); + }); + + it('treats default_tag as a normal name and identifies carriers only by fromMigration', async () => { + const ordinary = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId, + tag: 'default_tag', + tagType: 'string' + }); + const carrier = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId, + tag: 'renamed carrier', + tagType: 'string', + fromMigration: true + }); + + await expect(migrateDatasetTagDefinitions({ datasetId, teamId })).resolves.toMatchObject({ + deletedDefinitionCount: 0 + }); + expect(await MongoDatasetCollectionTagsV2.findById(ordinary._id).lean()).toMatchObject({ + tag: 'default_tag', + tagType: 'string', + fromMigration: false + }); + expect(await MongoDatasetCollectionTagsV2.findById(carrier._id).lean()).toMatchObject({ + tag: 'renamed carrier', + tagType: 'array', + fromMigration: true + }); + }); +}); diff --git a/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts b/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts index b9a68ed389fa..88aa1dcf86bf 100644 --- a/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts +++ b/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts @@ -114,4 +114,23 @@ describe('agentForm2AppWorkflow model reference', () => { datasetSearchExtensionModel: 'legacy-extension' }); }); + + it('round-trips the structured collection filter in nested agent params', () => { + const form = getDefaultAppForm(); + form.dataset.collectionFilterMatch = { + logic: 'AND', + conditions: [{ tag: 'price', tagType: 'number', op: '$gte', value: 10 }] + }; + const workflow = agentForm2AppWorkflow(form, (key: string) => key); + const datasetParams = workflow.nodes + .flatMap((node) => node.inputs) + .find((input) => input.key === NodeInputKeyEnum.datasetParams)?.value; + expect(datasetParams).toMatchObject({ + collectionFilterMatch: form.dataset.collectionFilterMatch + }); + expect( + appWorkflow2AgentForm({ nodes: workflow.nodes, chatConfig: workflow.chatConfig }).dataset + .collectionFilterMatch + ).toEqual(form.dataset.collectionFilterMatch); + }); }); diff --git a/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts b/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts index cbb97dedc440..bdccc715cb9a 100644 --- a/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts +++ b/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts @@ -22,6 +22,19 @@ const getModelInputs = ({ modelId, model }: { modelId?: string; model?: string } ); }; +const getFormWithDataset = () => { + const form = getDefaultAppForm(); + form.dataset.datasets = [ + { + datasetId: 'dataset-id', + avatar: 'dataset.svg', + name: 'Dataset', + vectorModel: { model: 'embedding-model' } + } + ]; + return form; +}; + describe('form2AppWorkflow model reference', () => { it.each([ { type: FlowNodeTypeEnum.appModule, hasStreamInput: true }, @@ -115,15 +128,7 @@ describe('form2AppWorkflow model reference', () => { }); it('preserves empty dataset model IDs instead of falling back to legacy fields', () => { - const form = getDefaultAppForm(); - form.dataset.datasets = [ - { - datasetId: 'dataset-id', - avatar: 'dataset.svg', - name: 'Dataset', - vectorModel: { model: 'embedding-model' } - } - ]; + const form = getFormWithDataset(); form.dataset.rerankModelId = ''; form.dataset.rerankModel = 'legacy-rerank'; form.dataset.datasetSearchExtensionModelId = ''; @@ -148,4 +153,61 @@ describe('form2AppWorkflow model reference', () => { ]) ); }); + + it('round-trips collectionFilterMatch through the dataset search node', () => { + const form = getFormWithDataset(); + form.dataset.collectionFilterMatch = { + logic: 'AND', + conditions: [{ tag: 'price', tagType: 'number', op: '$gte', value: 10 }] + }; + + const workflow = form2AppWorkflow(form, (key: string) => key); + const input = workflow.nodes + .flatMap((node) => node.inputs) + .find((item) => item.key === NodeInputKeyEnum.collectionFilterMatch); + + expect(input).toMatchObject({ + key: NodeInputKeyEnum.collectionFilterMatch, + renderTypeList: ['datasetTagFilter', 'reference'], + value: form.dataset.collectionFilterMatch + }); + expect( + appWorkflow2Form({ nodes: workflow.nodes, chatConfig: form.chatConfig }).dataset + .collectionFilterMatch + ).toEqual(form.dataset.collectionFilterMatch); + }); + + it('preserves the explicit legacy filter marker until the user upgrades it', () => { + const form = getFormWithDataset(); + form.dataset[NodeInputKeyEnum.collectionFilterVersion] = 'legacy'; + form.dataset.collectionFilterMatch = '{"tags":{"$and":["legacy"]}}'; + + const workflow = form2AppWorkflow(form, (key: string) => key); + const datasetNode = workflow.nodes.find( + (node) => node.flowNodeType === FlowNodeTypeEnum.datasetSearchNode + ); + expect(datasetNode).toBeTruthy(); + expect( + datasetNode?.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion) + ?.value + ).toBe('legacy'); + expect(appWorkflow2Form({ nodes: workflow.nodes, chatConfig: {} }).dataset).toMatchObject({ + [NodeInputKeyEnum.collectionFilterVersion]: 'legacy', + collectionFilterMatch: form.dataset.collectionFilterMatch + }); + + const unversionedNode = { + ...datasetNode!, + inputs: datasetNode!.inputs + .filter((input) => input.key !== NodeInputKeyEnum.collectionFilterVersion) + .map((input) => + input.key === NodeInputKeyEnum.collectionFilterMatch + ? { ...input, value: undefined } + : input + ) + }; + expect(appWorkflow2Form({ nodes: [unversionedNode], chatConfig: {} }).dataset).toMatchObject({ + [NodeInputKeyEnum.collectionFilterVersion]: 'structured' + }); + }); }); diff --git a/projects/app/test/web/core/app/workflow/utils.test.ts b/projects/app/test/web/core/app/workflow/utils.test.ts index 12a6113b96e2..e0c073bc1f6c 100644 --- a/projects/app/test/web/core/app/workflow/utils.test.ts +++ b/projects/app/test/web/core/app/workflow/utils.test.ts @@ -34,10 +34,7 @@ import { collectWorkflowStartAutoFillRevertPatches, collectWorkflowStartOutputAutoFillRevertPatches } from '@/web/core/workflow/workflowStartAutoFill'; -import type { - FlowNodeInputItemType, - FlowNodeOutputItemType -} from '@fastgpt/global/core/workflow/type/io'; +import type { FlowNodeOutputItemType } from '@fastgpt/global/core/workflow/type/io'; import { NodeOutputKeyEnum, VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants'; import { PluginStatusEnum } from '@fastgpt/global/core/plugin/type'; import { AppErrEnum } from '@fastgpt/global/common/error/code/app'; @@ -53,9 +50,11 @@ import { HttpNode468 } from '@fastgpt/global/core/workflow/template/system/http4 import { LoopStartNode } from '@fastgpt/global/core/workflow/template/system/loop/loopStart'; import { AiChatModule } from '@fastgpt/global/core/workflow/template/system/aiChat'; import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch'; +import { AgentNode } from '@fastgpt/global/core/workflow/template/system/agent'; import { ClassifyQuestionModule } from '@fastgpt/global/core/workflow/template/system/classifyQuestion'; import { ToolCallNode } from '@fastgpt/global/core/workflow/template/system/toolCall'; import { userFilesInput } from '@fastgpt/global/core/workflow/template/system/workflowStart'; +import { Input_Template_Dataset_Tag_Filter_Version } from '@fastgpt/global/core/workflow/template/input'; import { ModelTypeEnum } from '@fastgpt/global/core/ai/constants'; describe('nodeTemplate2FlowNode', () => { @@ -2223,6 +2222,87 @@ describe('workflow model validation', () => { }); describe('storeNode2FlowNode', () => { + it.each([ + { + name: 'unversioned node with metadata', + version: undefined, + filterValue: { logic: 'AND', conditions: [] }, + expectedVersion: 'legacy', + expectedLabel: 'workflow:collection_metadata_filter' + }, + { + name: 'unversioned node without metadata', + version: undefined, + filterValue: undefined, + expectedVersion: 'structured', + expectedLabel: 'workflow:tag_filter' + }, + { + name: 'explicitly structured node with a legacy-shaped value', + version: 'structured', + filterValue: '{"tags":{"$and":["legacy-shape"]}}', + expectedVersion: 'structured', + expectedLabel: 'workflow:tag_filter' + } + ])( + 'hydrates $name as $expectedVersion', + ({ version, filterValue, expectedVersion, expectedLabel }) => { + const storeNode = { + ...DatasetSearchModule, + nodeId: 'dataset-search', + position: { x: 0, y: 0 }, + inputs: DatasetSearchModule.inputs + .filter((input) => input.key !== NodeInputKeyEnum.collectionFilterVersion) + .map((input) => + input.key === NodeInputKeyEnum.collectionFilterMatch + ? { ...input, value: filterValue } + : input + ) + .concat(version ? [{ ...Input_Template_Dataset_Tag_Filter_Version, value: version }] : []) + } as StoreNodeItemType; + + const result = storeNode2FlowNode({ + item: storeNode, + t: ((key: string) => key) as any + }); + + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion) + ?.value + ).toBe(expectedVersion); + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterMatch) + ).toMatchObject({ + label: expectedLabel + }); + } + ); + + it('hydrates Agent V2 with an explicit structured marker', () => { + const storeNode = { + ...AgentNode, + nodeId: 'agent', + position: { x: 0, y: 0 }, + inputs: AgentNode.inputs.filter( + (input) => input.key !== NodeInputKeyEnum.collectionFilterVersion + ) + } as StoreNodeItemType; + + const result = storeNode2FlowNode({ + item: storeNode, + t: ((key: string) => key) as any + }); + + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion) + ?.value + ).toBe('structured'); + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterMatch) + ?.label + ).toBe('workflow:tag_filter'); + }); + it('restores tool set nodes without a source handle', () => { const storeNode = { nodeId: 'tool-set-node', diff --git a/projects/app/test/web/core/workflow/datasetSearchNodeUpgrade.test.ts b/projects/app/test/web/core/workflow/datasetSearchNodeUpgrade.test.ts new file mode 100644 index 000000000000..2bdd1be40728 --- /dev/null +++ b/projects/app/test/web/core/workflow/datasetSearchNodeUpgrade.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + FlowNodeInputTypeEnum, + FlowNodeTypeEnum +} from '@fastgpt/global/core/workflow/node/constant'; +import { + getDatasetSearchFilterVersion, + persistLegacyDatasetSearchNodeUpgrade +} from '@/web/core/workflow/datasetSearchNodeUpgrade'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; + +const legacyNode = { + nodeId: 'dataset-node', + name: 'Dataset search', + flowNodeType: FlowNodeTypeEnum.datasetSearchNode, + version: '4.9.2', + inputs: [ + { key: 'datasetSelectList', label: '', renderTypeList: [], value: ['dataset'] }, + { + key: 'collectionFilterMatch', + label: '', + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter], + value: '{"tags":{"$and":["legacy"]}}' + } + ], + outputs: [{ key: 'quoteQA', label: '', type: 'static' }] +} as any; +const upgradedInput = { + ...legacyNode.inputs[1], + value: { logic: 'AND', conditions: [] } +}; +const versionInput = { + key: 'collectionFilterVersion', + label: '', + renderTypeList: [FlowNodeInputTypeEnum.hidden], + value: 'legacy' +} as any; + +describe('dataset search node upgrade', () => { + it('does not commit the local upgrade when persistence fails', async () => { + const commit = vi.fn(); + const persist = vi.fn().mockRejectedValue(new Error('save failed')); + await expect( + persistLegacyDatasetSearchNodeUpgrade({ + nodes: [legacyNode], + nodeId: legacyNode.nodeId, + filterInput: upgradedInput, + persist, + commit + }) + ).rejects.toThrow('save failed'); + expect(commit).not.toHaveBeenCalled(); + }); + + it('persists and commits one canonical upgraded node without changing shared node data', async () => { + const persist = vi.fn(async () => undefined); + const commit = vi.fn(); + + await persistLegacyDatasetSearchNodeUpgrade({ + nodes: [legacyNode], + nodeId: legacyNode.nodeId, + filterInput: upgradedInput, + persist, + commit + }); + + const persistedNode = persist.mock.calls[0][0][0]; + expect(commit).toHaveBeenCalledWith(persistedNode); + expect(persistedNode).toMatchObject({ + nodeId: legacyNode.nodeId, + name: legacyNode.name, + flowNodeType: FlowNodeTypeEnum.datasetSearchNode, + outputs: legacyNode.outputs + }); + expect(persistedNode.inputs[0]).toEqual(legacyNode.inputs[0]); + expect(persistedNode.inputs.find((input) => input.key === upgradedInput.key)).toEqual( + upgradedInput + ); + expect(getDatasetSearchFilterVersion(persistedNode.inputs)).toBe( + DatasetTagFilterVersionEnum.structured + ); + expect(persistedNode.inputs.filter((input) => input.key === versionInput.key)).toHaveLength(1); + }); +}); From ef078cb5bb6459b40c016d855c99adf6cd345a02 Mon Sep 17 00:00:00 2001 From: DigHuang <114602213+DigHuang@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:00:36 +0800 Subject: [PATCH 4/4] style(dataset/tag): refine tag filtering and management UX --- .../core/dataset/collection/tagFilter.ts | 4 +- .../core/dataset/collection/tagFilter.test.ts | 4 +- .../DateTimePicker/SingleDateTimePicker.tsx | 14 +- .../web/components/common/Icon/button.tsx | 3 +- .../common/Icon/icons/common/calendar.svg | 3 +- .../common/MySelect/MultipleRowSelect.tsx | 13 +- .../common/TagFilter/MultiTagFilter.tsx | 4 + packages/web/i18n/en/dataset.json | 2 +- packages/web/i18n/ko-KR/dataset.json | 2 +- packages/web/i18n/zh-CN/common.json | 2 +- packages/web/i18n/zh-CN/dataset.json | 2 +- packages/web/i18n/zh-Hant/common.json | 2 +- packages/web/i18n/zh-Hant/dataset.json | 2 +- .../core/dataset/DatasetTagFilterRows.tsx | 47 +++--- .../RenderInput/templates/Reference.tsx | 48 +++++-- .../CollectionTagBatchModal.tsx | 11 +- .../CollectionCard/CollectionTagTable.tsx | 7 +- .../detail/CollectionCard/TagCommon.tsx | 134 ++++++++++++------ .../detail/CollectionCard/TagManageModal.tsx | 31 ++-- .../detail/CollectionCard/TagValueInputs.tsx | 82 ++++++----- .../detail/CollectionCard/TagsPopOver.tsx | 53 +++++-- .../dataset/detail/CollectionCard/index.tsx | 49 ++----- 22 files changed, 320 insertions(+), 199 deletions(-) diff --git a/packages/service/core/dataset/collection/tagFilter.ts b/packages/service/core/dataset/collection/tagFilter.ts index 70595ed5d0db..29080912131d 100644 --- a/packages/service/core/dataset/collection/tagFilter.ts +++ b/packages/service/core/dataset/collection/tagFilter.ts @@ -9,7 +9,7 @@ import { MongoDatasetCollection } from './schema'; /** * 按 tagFilters 组装 Collection 列表的标签过滤条件。 - * 同一标签多值为 OR（value $in），不同标签为 AND。 + * 多个标签或同一标签多值均为 OR。 */ export const buildCollectionListTagMatch = (tagFilters: CollectionTagFilterItem[] = []) => { if (tagFilters.length === 0) return {}; @@ -24,7 +24,7 @@ export const buildCollectionListTagMatch = (tagFilters: CollectionTagFilterItem[ })); if (conditions.length === 1) return conditions[0]; - return { $and: conditions }; + return { $or: conditions }; }; type UsedTagValueGroup = { diff --git a/packages/service/test/core/dataset/collection/tagFilter.test.ts b/packages/service/test/core/dataset/collection/tagFilter.test.ts index 89d830ba7e25..923860c3fbc7 100644 --- a/packages/service/test/core/dataset/collection/tagFilter.test.ts +++ b/packages/service/test/core/dataset/collection/tagFilter.test.ts @@ -7,7 +7,7 @@ describe('buildCollectionListTagMatch', () => { expect(buildCollectionListTagMatch([])).toEqual({}); }); - it('uses $elemMatch for one tag and $and across tags', () => { + it('uses $elemMatch for one tag and $or across tags', () => { expect(buildCollectionListTagMatch([{ tagId: 'type', values: ['PRD'] }])).toEqual({ tags: { $elemMatch: { @@ -23,7 +23,7 @@ describe('buildCollectionListTagMatch', () => { { tagId: 'version', values: [2] } ]) ).toEqual({ - $and: [ + $or: [ { tags: { $elemMatch: { diff --git a/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx index 75301123eaf4..66785cb6be32 100644 --- a/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx +++ b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx @@ -122,23 +122,33 @@ export const SingleDateTimePicker = ({ > - + {displayText || placeholder || t('common:datetime_picker.placeholder')} {!hideCalendarIcon && ( diff --git a/packages/web/components/common/Icon/button.tsx b/packages/web/components/common/Icon/button.tsx index 74f83f06fe2d..a5ab26e1d15e 100644 --- a/packages/web/components/common/Icon/button.tsx +++ b/packages/web/components/common/Icon/button.tsx @@ -32,6 +32,7 @@ const MyIconButton = ({ color={'myGray.500'} rounded={'sm'} alignItems={'center'} + justifyContent={'center'} bg={'transparent'} transition={'background 0.1s'} cursor={'pointer'} @@ -47,7 +48,7 @@ const MyIconButton = ({ sx={{ userSelect: 'none' }} {...props} > - + ); diff --git a/packages/web/components/common/Icon/icons/common/calendar.svg b/packages/web/components/common/Icon/icons/common/calendar.svg index 849d9c703730..c7795b95d747 100644 --- a/packages/web/components/common/Icon/icons/common/calendar.svg +++ b/packages/web/components/common/Icon/icons/common/calendar.svg @@ -2,7 +2,6 @@ diff --git a/packages/web/components/common/MySelect/MultipleRowSelect.tsx b/packages/web/components/common/MySelect/MultipleRowSelect.tsx index 2e3f201bbee8..b351d7214512 100644 --- a/packages/web/components/common/MySelect/MultipleRowSelect.tsx +++ b/packages/web/components/common/MySelect/MultipleRowSelect.tsx @@ -227,6 +227,7 @@ export const MultipleRowSelect = ({ size={'lg'} fontSize={'sm'} textAlign={'left'} + overflow={'hidden'} _active={{ transform: 'none' }} @@ -240,13 +241,15 @@ export const MultipleRowSelect = ({ } : {})} > - + {/* data-preserve-width：避免外层 width:auto 冲掉触发器截断 */} + {label ?? placeholder} diff --git a/packages/web/components/common/TagFilter/MultiTagFilter.tsx b/packages/web/components/common/TagFilter/MultiTagFilter.tsx index 23001fe5e649..f9a3bcd647de 100644 --- a/packages/web/components/common/TagFilter/MultiTagFilter.tsx +++ b/packages/web/components/common/TagFilter/MultiTagFilter.tsx @@ -320,6 +320,10 @@ const MultiTagFilter = ({ pointerEvents={'none'} size={'sm'} icon={} + sx={{ + // 自定义勾图标不会走 Chakra 未选中时的隐藏，悬浮会露出白勾 + '.chakra-checkbox__control:not([data-checked]) svg': { opacity: 0 } + }} /> @@ -187,7 +188,7 @@ const TagFilterValueCell = ({ return ( onChange({ value: val === '' ? undefined : val })} @@ -197,7 +198,7 @@ const TagFilterValueCell = ({ if (condition.tagType === DatasetCollectionTagTypeEnum.datetime) { return ( onChange({ value: val })} @@ -206,7 +207,7 @@ const TagFilterValueCell = ({ } return ( {isCollectionId ? ( )} - + {isReference ? ( onChange({ value: e as ReferenceItemValueType })} isArray={false} ButtonProps={{ - ...valueCellEmbeddedStyles, + ...valueCellJoinedStyles, size: 'sm', w: '100%', - px: 3, - borderWidth: 0 + px: 3 }} /> ) : ( diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx index ed3922fdfda1..1738a565b000 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference.tsx @@ -35,6 +35,8 @@ type CommonSelectProps = { list: { label: string | React.ReactNode; value: string; + name?: string; + avatar?: string; children: { label: string; value: string; @@ -93,6 +95,8 @@ export const useReference = ({ ), value: node.nodeId, + name: node.name, + avatar: node.avatar, children: filterSelectableWorkflowNodeOutputs({ outputs: node.outputs, valueType, @@ -183,17 +187,22 @@ const SingleReferenceSelector = ({ }: SelectProps) => { const getSelectValue = useCallback( (value: ReferenceValueType) => { - if (!value) return []; + if (!value) return undefined; const firstColumn = list.find((item) => item.value === value[0]); if (!firstColumn) { - return []; + return undefined; } const secondColumn = firstColumn.children.find((item) => item.value === value[1]); if (!secondColumn) { - return []; + return undefined; } - return [firstColumn.label, secondColumn.label]; + const nodeText = firstColumn.name || ''; + const outputText = secondColumn.label || ''; + return { + avatar: firstColumn.avatar, + text: nodeText && outputText ? `${nodeText} > ${outputText}` : nodeText || outputText + }; }, [list] ); @@ -214,17 +223,34 @@ const SingleReferenceSelector = ({ const ItemSelector = useMemo(() => { const selectorVal = value as ReferenceItemValueType; - const [nodeName, outputName] = getSelectValue(selectorVal); - const isValidSelect = nodeName && outputName; + const selected = getSelectValue(selectorVal); return ( - {nodeName} - - {outputName} + selected ? ( + + {!!selected.avatar && ( + + )} + + {selected.text} + ) : ( diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx index 2a19106bd478..0a457c5f4e01 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx @@ -154,12 +154,19 @@ const CollectionTagBatchModal = ({ tags: body.tags }), { - onSuccess() { + onSuccess(_res, params) { + const requestMode = params?.[0]?.mode; + toast({ + title: + requestMode === BatchCollectionTagModeEnum.remove + ? t('dataset:tag.delete_success') + : t('dataset:tag.setting_success'), + status: 'success' + }); void loadAllDatasetTags(); onSuccess?.(); onClose(); }, - successToast: t('dataset:tag.setting_success'), errorToast: t('dataset:tag.save_failed') } ); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx index 68c4eb422b8c..7eb756c5518c 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx @@ -112,7 +112,12 @@ export const CollectionTagTable = ({ )} - + } diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx index 25e6d800ac07..845ea13cf013 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx @@ -168,19 +168,57 @@ export const formatCollectionTagChipText = ( return valueText ? `${item.tag}: ${valueText}` : item.tag; }; +export const parseCollectionTagParts = ( + item: CollectionTagDisplayItem, + tagDefs: DatasetTagType[] = [] +): { name: string; value?: string } => { + if (typeof item === 'string') return { name: item }; + + const tagType = tagDefs.find((def) => def.tag === item.tag)?.tagType; + const valueText = formatCollectionTagValueText(item.value, tagType); + return { + name: item.tag, + value: valueText || undefined + }; +}; + +export const TagTooltipItem = ({ name, value }: { name: string; value?: string }) => ( + + + {name} + {value ? '：' : ''} + + {value && ( + + {value} + + )} + +); + export const TAG_TOOLTIP_PROPS = { placement: 'bottom' as const, - offset: [0, 10] as [number, number], + offset: [0, 8] as [number, number], hasArrow: true, arrowSize: 10, px: 3, py: 2, - borderRadius: 'sm', + borderRadius: '6px', fontSize: 'xs', lineHeight: '18px', - color: 'myGray.800', - arrowShadowColor: 'rgba(19, 51, 107, 0.1)', - boxShadow: '0px 4px 5px rgba(19, 51, 107, 0.1), 0px 0px 0.5px rgba(19, 51, 107, 0.1)' + color: '#24282C', + bg: 'white', + maxW: '360px', + whiteSpace: 'pre-wrap' as const, + wordBreak: 'break-all' as const, + arrowShadowColor: 'rgba(121, 141, 159, 0.25)', + boxShadow: '0px 2px 4px 0px rgba(161, 167, 179, 0.25), 0px 0px 1px 0px rgba(121, 141, 159, 0.25)' }; export const SaveActionIcon = ({ isEnabled }: { isEnabled: boolean }) => { @@ -213,48 +251,60 @@ export const SaveActionIcon = ({ isEnabled }: { isEnabled: boolean }) => { type TagActionButtonProps = { label: string; icon: React.ReactElement; - onClick?: () => void; + onClick?: (e?: React.MouseEvent) => void; isDisabled?: boolean; color?: string; hoverColor?: string; hoverIconClassName?: string; -}; +} & Omit; -export const TagActionButton = ({ - label, - icon, - onClick, - isDisabled = false, - color = 'myGray.500', - hoverColor, - hoverIconClassName -}: TagActionButtonProps) => ( - - - {icon} - - +export const TagActionButton = React.forwardRef( + function TagActionButton( + { + label, + icon, + onClick, + isDisabled = false, + color = 'myGray.500', + hoverColor, + hoverIconClassName, + ...props + }, + ref + ) { + return ( + + + {icon} + + + ); + } ); export const TagTableContainer = ({ children, ...props }: FlexProps) => ( diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx index 13cbf7e0dfaa..40428fbb7f5b 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx @@ -27,8 +27,11 @@ const TAG_TABLE_COLUMNS = 'minmax(0, 1fr) 180px 100px'; const OPTION_ROW_HEIGHT = 32; const OPTION_ROW_GAP = 2; const OPTION_LIST_MAX_ROWS = 4; +const OPTION_LIST_PADDING = 4; const OPTION_LIST_MAX_H = - OPTION_ROW_HEIGHT * OPTION_LIST_MAX_ROWS + OPTION_ROW_GAP * (OPTION_LIST_MAX_ROWS - 1); + OPTION_ROW_HEIGHT * OPTION_LIST_MAX_ROWS + + OPTION_ROW_GAP * (OPTION_LIST_MAX_ROWS - 1) + + OPTION_LIST_PADDING * 2; const normalizeTagOptions = (nextOptions: string[]) => [ ...new Set(nextOptions.map((option) => option.trim()).filter(Boolean)) @@ -127,7 +130,7 @@ const TagOptionManagePopover = ({ hasArrow={false} offset={[0, 4]} closeOnBlur={true} - w={'152px'} + w={'160px'} p={1.5} borderRadius={'sm'} boxShadow={'md'} @@ -139,23 +142,12 @@ const TagOptionManagePopover = ({ void persistOptions(draftOptionsRef.current); }} Trigger={ - } color={'myGray.600'} - cursor={'pointer'} - _hover={{ - bg: 'myGray.05', - color: 'primary.700' - }} - > - - + hoverColor={'primary.700'} + /> } > {() => ( @@ -183,7 +175,7 @@ const TagOptionManagePopover = ({ overflowY={'auto'} direction={'column'} gap={`${OPTION_ROW_GAP}px`} - mt={0.5} + p={1} > {draftOptions.map((opt, index) => ( @@ -203,6 +195,7 @@ const TagOptionManagePopover = ({ borderColor={'myGray.200'} placeholder={t('dataset:tag.enter_option')} _focus={{ + zIndex: 1, borderColor: 'primary.600', boxShadow: 'focus' }} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx index 196b39b6af16..d7a71ab0edaa 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx @@ -29,6 +29,7 @@ export const tagInputBaseStyles: InputProps = { bg: 'white', fontSize: 'sm', color: 'myGray.900', + px: 3, _hover: { borderColor: 'primary.300' }, @@ -61,28 +62,26 @@ const StringTagInput = ({ ); }; -const embeddedFieldStyles = { - border: 'none', - boxShadow: 'none', - _hover: { border: 'none' }, - _focus: { border: 'none', boxShadow: 'none' } +/** 与左侧切换区拼接时去掉左圆角，保留自身边框与 focus（对齐判断器 / 变量更新）。 */ +const joinedFieldStyles = { + borderLeftRadius: 'none' as const }; export const NumberTagInput = ({ value, onChange, placeholder, - showStepper, - embedded + showStepper = true, + joined }: { value?: number | string; onChange: (val: number | '') => void; placeholder?: string; showStepper?: boolean; - embedded?: boolean; + joined?: boolean; }) => { const { t } = useTranslation(); - const placeholderText = placeholder ?? t('dataset:tag.fill_number'); + const placeholderText = placeholder ?? t('dataset:tag.fill_integer'); if (showStepper) { return ( @@ -91,12 +90,22 @@ export const NumberTagInput = ({ h={'36px'} w={'100%'} fontSize={'sm'} + precision={0} + step={1} inputFieldProps={{ h: '36px', + px: 3, bg: 'white', fontSize: 'sm', color: 'myGray.900', - ...(embedded ? embeddedFieldStyles : {}) + textAlign: 'left', + borderRadius: 'sm', + borderColor: 'myGray.200', + _focus: { + borderColor: 'primary.600', + boxShadow: 'focus' + }, + ...(joined ? joinedFieldStyles : {}) }} value={value === undefined || value === '' ? '' : Number(value)} placeholder={placeholderText} @@ -108,7 +117,7 @@ export const NumberTagInput = ({ return ( void; placeholder?: string; - embedded?: boolean; + joined?: boolean; }) => ( ); @@ -239,7 +246,7 @@ export const TagNameSelect = ({ }} onChange={(e) => setSearch(e.target.value)} /> - + {filteredOptions.length === 0 ? ( {t('common:no_select_data')} @@ -316,10 +323,12 @@ const ArraySelectedChip = ({ h={'24px'} bg={'myGray.100'} color={'myGray.900'} - px={1.5} + px={1} + py={'2px'} borderRadius={'xs'} - fontSize={'xs'} - lineHeight={'16px'} + fontSize={'sm'} + lineHeight={'20px'} + letterSpacing={'0.25px'} flexShrink={0} > @@ -331,14 +340,14 @@ const ArraySelectedChip = ({ type={'button'} alignItems={'center'} justifyContent={'center'} - w={'14px'} - h={'14px'} + w={'16px'} + h={'16px'} cursor={'pointer'} color={'myGray.500'} _hover={{ color: 'myGray.700' }} onClick={(e) => onRemove(opt, e)} > - + )} @@ -349,10 +358,10 @@ const ArrayOverflowChip = ({ count }: { count: number }) => ( data-overflow-chip alignItems={'center'} h={'24px'} - px={1.5} + px={2} bg={'myGray.100'} color={'myGray.600'} - borderRadius={'xs'} + borderRadius={'full'} fontSize={'xs'} lineHeight={'16px'} flexShrink={0} @@ -368,7 +377,7 @@ export const ArrayTagSelect = ({ onCreateOption, allowCreate = true, placeholder, - embedded + joined }: { options: string[]; value?: string[]; @@ -376,7 +385,7 @@ export const ArrayTagSelect = ({ onCreateOption?: (option: string) => void; allowCreate?: boolean; placeholder?: string; - embedded?: boolean; + joined?: boolean; }) => { const { t } = useTranslation(); const { isOpen, onOpen, onClose } = useDisclosure(); @@ -455,6 +464,7 @@ export const ArrayTagSelect = ({ placement={'bottom-start'} closeOnBlur matchWidth + gutter={4} > {value.length === 0 ? ( @@ -520,7 +530,7 @@ export const ArrayTagSelect = ({ - + )} - + {filteredOptions.length === 0 && !canCreate ? ( {t('common:no_select_data')} @@ -611,6 +621,7 @@ export const ArrayTagSelect = ({ px={1} borderRadius={'xs'} cursor={'pointer'} + bg={isChecked ? ARRAY_OPTION_HOVER_BG : 'transparent'} _hover={{ bg: ARRAY_OPTION_HOVER_BG }} onClick={() => handleToggleOption(opt)} > @@ -619,9 +630,14 @@ export const ArrayTagSelect = ({ onChange={() => handleToggleOption(opt)} onClick={(e) => e.stopPropagation()} size={'sm'} + borderRadius={'xs'} icon={} + sx={{ + // 自定义勾图标不会走 Chakra 未选中时的隐藏，悬浮会露出白勾 + '.chakra-checkbox__control:not([data-checked]) svg': { opacity: 0 } + }} /> - + {opt} @@ -671,7 +687,7 @@ export const TagValueField = ({ if (tag.tagType === DatasetCollectionTagTypeEnum.number) { return ( onChange(val)} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx index 550d5d7fb77f..29bd3c35cb25 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx @@ -9,7 +9,9 @@ import MyTag from '@fastgpt/web/components/common/Tag/index'; import { formatCollectionTagChipText, OVERFLOW_CHIP_GAP_PX, + parseCollectionTagParts, TAG_TOOLTIP_PROPS, + TagTooltipItem, useOverflowChipCount } from './TagCommon'; @@ -24,13 +26,22 @@ const TAG_CHIP_PROPS = { borderRadius: 'xs' as const }; +const OVERFLOW_TAG_CHIP_PROPS = { + ...TAG_CHIP_PROPS, + borderRadius: 'full' as const +}; + /** 渲染知识库列表中的标签；单个标签成为唯一可见项时允许收缩并展示完整文本。 */ const TagChip = ({ text, + name, + value, isFlexible = false, withTooltip = false }: { text: string; + name?: string; + value?: string; isFlexible?: boolean; withTooltip?: boolean; }) => { @@ -46,7 +57,11 @@ const TagChip = ({ ); const tagContent = withTooltip ? ( - + } + shouldWrapChildren={false} + {...TAG_TOOLTIP_PROPS} + > {tagText} ) : ( @@ -61,6 +76,8 @@ const TagChip = ({ minW={isFlexible ? 0 : undefined} maxW={isFlexible ? '100%' : undefined} overflow={isFlexible ? 'hidden' : undefined} + cursor={withTooltip ? 'pointer' : undefined} + _hover={withTooltip ? { bg: '#DBF3FF' } : undefined} > {tagContent} @@ -79,10 +96,15 @@ const TagsPopOver = ({ const chipItems = useMemo( () => (currentCollection.tags ?? []) - .map((item, index) => ({ - id: typeof item === 'string' ? item : `${item.tag}-${index}`, - text: formatCollectionTagChipText(item, allDatasetTags) - })) + .map((item, index) => { + const parts = parseCollectionTagParts(item, allDatasetTags); + return { + id: typeof item === 'string' ? item : `${item.tag}-${index}`, + text: formatCollectionTagChipText(item, allDatasetTags), + name: parts.name, + value: parts.value + }; + }) .filter((item) => item.text), [allDatasetTags, currentCollection.tags] ); @@ -140,7 +162,7 @@ const TagsPopOver = ({ {chipItems.map((item) => ( ))} - + {`+${chipItems.length}`} @@ -158,23 +180,36 @@ const TagsPopOver = ({ ))} {overflowTags.length > 0 && ( item.text).join('\n')} + label={ + + {overflowTags.map((item) => ( + + ))} + + } shouldWrapChildren={false} {...TAG_TOOLTIP_PROPS} > e.stopPropagation()} - _hover={{ bg: '#DBF3FF' }} > - + {`+${overflowTags.length}`} diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx index 0afc00317485..3cd7519ea760 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx @@ -471,16 +471,8 @@ const CollectionCard = () => { ...(collectionCanSync(collection.type) ? [ { - label: ( - - - {t('dataset:collection_sync')} - - ), + icon: 'common/refreshLight', + label: t('dataset:collection_sync'), onClick: () => openSyncConfirm({ onConfirm: () => { @@ -491,22 +483,14 @@ const CollectionCard = () => { ] : []), { - label: ( - - - {t('common:Move')} - - ), + icon: 'common/file/move', + label: t('common:Move'), onClick: () => setMoveCollectionData({ collectionId: collection._id }) }, { - label: ( - - - {t('common:Rename')} - - ), + icon: 'edit', + label: t('common:Rename'), onClick: () => onOpenEditTitleModal({ defaultVal: collection.name, @@ -521,12 +505,8 @@ const CollectionCard = () => { datasetDetail.type !== DatasetTypeEnum.websiteDataset ? [ { - label: ( - - - {t('dataset:tag.set')} - - ), + icon: 'core/dataset/tag', + label: t('dataset:tag.set'), onClick: () => setTagSetCollection(collection) } ] @@ -536,18 +516,9 @@ const CollectionCard = () => { { children: [ { - label: ( - - - {t('common:Delete')} - - ), type: 'danger', + icon: 'delete', + label: t('common:Delete'), onClick: () => openDeleteConfirm({ onConfirm: () => onDelCollection([collection._id]),