diff --git a/.agents/design/core/dataset/index.md b/.agents/design/core/dataset/index.md index 1998352d6c96..f2390705c795 100644 --- a/.agents/design/core/dataset/index.md +++ b/.agents/design/core/dataset/index.md @@ -811,7 +811,7 @@ POST /api/core/dataset/collection/create/link # 链接导入 POST /api/core/dataset/collection/create/text # 文本导入 POST /api/core/dataset/collection/create/images # 图片导入 PUT /api/core/dataset/collection/update # 更新集合 -GET /api/core/dataset/collection/list # 集合列表 +POST /api/core/dataset/collection/listV2 # 集合列表 GET /api/core/dataset/collection/detail # 集合详情 POST /api/core/dataset/collection/sync # 同步集合 GET /api/core/dataset/collection/export # 导出集合 @@ -893,9 +893,9 @@ MongoDatasetData.find(query, fields, { ### 3. 分页优化 ```typescript -// 使用 scrollList 而非传统分页 -// 避免深度分页性能问题 -GET /api/core/dataset/collection/scrollList?lastId=xxx&limit=20 +// 使用 offset 分页，避免深度 pageNum 分页性能问题 +POST /api/core/dataset/collection/listV2 +{ datasetId, offset, pageSize } ``` ### 4. 缓存策略 diff --git "a/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 000000000000..04e2e4f0eb67 --- /dev/null +++ "b/.agents/design/\347\237\245\350\257\206\345\272\223\346\240\207\347\255\276-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,616 @@ +# 知识库标签系统设计文档 + +## 1. 文档概述 + +### 1.1 设计目标 + +本设计为 FastGPT 知识库提供统一的类型化标签模型、Collection 标签值管理、标签过滤、历史数据迁移和生命周期清理能力。设计以当前代码实现为准，标签定义存储在 `dataset_collection_tags_v2`，Collection 继续复用 `dataset_collections.tags` 字段。 + +系统支持以下标签类型： + +- `string`：字符串值，支持相等、包含、前缀、后缀和正则匹配。 +- `number`：数值比较。 +- `datetime`：按时间戳进行比较。 +- `array`：字符串数组集合比较，支持集合相等、子集、包含和空值判断。 + +历史标签表 `dataset_collection_tags` 作为迁移来源和兼容数据保留，不再作为新标签管理 API 的写入目标。 + +### 1.2 设计范围 + +| 范围 | 内容 | +|---|---| +| 标签定义 | v2 标签表、标签类型、名称唯一性和标签查询 | +| Collection 标签 | 标签值的创建、转换、设置、批量更新和展示转换 | +| 检索过滤 | 按标签名称、类型和值过滤 Collection，并与创建时间、Collection ID 条件求交集 | +| 历史迁移 | 将旧字符串标签转换为 `default_tag` 数组标签 | +| 生命周期 | dataset 物理删除时清理对应 v2 标签定义 | +| API | FastGPT Collection API、FastGPT 标签过滤 API、fastgpt-pro 标签管理 API | +| 本次不包含 | 删除 v1 标签表、反向迁移 v2 数据 | + +### 1.3 术语 + +| 术语 | 定义 | +|---|---| +| v1 标签表 | `dataset_collection_tags`，历史标签定义表，无 `tagType` | +| v2 标签表 | `dataset_collection_tags_v2`，当前标签定义表，带 `tagType` | +| Collection | 知识库中的文件、链接、文本等集合对象，持久化于 `dataset_collections` | +| 新格式标签 | `{ tagId, value }`，`tagId` 引用 v2 标签表 | +| 旧格式标签 | 字符串形式的历史标签 ID，通常引用 v1 标签表 | +| `default_tag` | 承载记录首次创建时使用的默认名称；作为 API 标签名时按普通字符串处理 | +| `collectionFilterMatch` | 知识库检索使用的 Collection 过滤 JSON 字符串 | + +## 2. 模块职责与边界 + +### 2.1 模块职责 + +1. 创建、查询、更新和删除知识库标签定义。 +2. 根据标签类型校验 Collection 标签值，并将创建入参统一转换为可持久化的新格式。 +3. 为知识库检索提供标签条件解析、MongoDB 粗筛和应用层精确比较。 +4. 在 App 启动阶段按 dataset 维度自动迁移历史 Collection。 +5. 在 dataset 异步物理删除时清理该 dataset 及其子 dataset 的 v2 标签定义。 +6. 在 Collection 列表接口中将存储格式的 `tagId` 转换为对外展示的标签名称。 + +### 2.2 模块边界 + +| 方向 | 依赖或边界 | +|---|---| +| 上游 | 标签管理 API、Collection 创建 API、工作流知识检索节点、Agent V2 搜索 | +| 下游 | MongoDB、MongoDB secondary read、现有 dataset 权限系统 | +| 跨仓库依赖 | fastgpt-pro 提供标签管理 API，FastGPT 提供公共类型、数据模型、Collection 和搜索逻辑 | +| 不负责 | 向量库内部索引实现、旧表下线时间规划 | + +## 3. 系统结构 + +### 3.1 子模块划分 + +| 子模块 | 主要职责 | 关键实现 | +|---|---|---| +| 标签模型 | 定义 v2 标签集合、字段和索引 | `packages/service/core/dataset/tag/schemaV2.ts` | +| Collection 模型 | 保存 Collection 标签值并提供标签索引 | `packages/service/core/dataset/collection/schema.ts` | +| 标签转换工具 | 创建/校验/展示转换标签值 | `packages/service/core/dataset/collection/utils.ts` | +| 标签 API | 提供标签定义和 Collection 标签值管理 | fastgpt-pro `projects/app/src/pages/api/core/dataset/tag/` | +| Collection API | 创建、列表和滚动读取 Collection | `projects/app/src/pages/api/core/dataset/collection/` | +| 搜索过滤 | 解析标签条件并返回 Collection ID | `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | +| 系统迁移 | 分批读取 v1 标签并写入 `default_tag` 新格式 | `projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/` | +| 删除处理器 | 删除 dataset 相关 v1/v2 标签定义 | `packages/service/core/dataset/delete/processor.ts` | +| 全局类型 | Zod Schema、标签类型、常量和错误码 | `packages/global/core/dataset/type.ts` | + +### 3.2 依赖关系 + +```mermaid +flowchart TD + Client[标签管理端或工作流调用方] + ProAPI[fastgpt-pro 标签 API] + CollectionAPI[FastGPT Collection API] + Search[collectionFilter.ts] + Migration[system migration runner] + Delete[datasetDeleteProcessor] + Utils[Collection 标签转换工具] + V2[(dataset_collection_tags_v2)] + V1[(dataset_collection_tags)] + Collections[(dataset_collections)] + Types[Global Zod 类型与常量] + + Client --> ProAPI + Client --> CollectionAPI + ProAPI --> V2 + ProAPI --> Collections + CollectionAPI --> Utils + Utils --> V2 + CollectionAPI --> Collections + Search --> V2 + Search --> Collections + Migration --> V1 + Migration --> V2 + Migration --> Collections + Delete --> V2 + Delete --> V1 + Delete --> Collections + Types -.-> ProAPI + Types -.-> CollectionAPI + Types -.-> Search +``` + +### 3.3 总体架构影响 + +本设计不改变 FastGPT 的分层架构、权限模型和检索主流程，仅在知识库标签子系统内新增 v2 标签模型和过滤逻辑。Collection 的 `tags` 字段不新增字段，通过元素类型兼容旧格式和新格式；标签管理 API 的外部路由保持原有组织方式。 + +## 4. 对外接口设计 + +### 4.1 标签管理接口 + +标签管理接口位于 fastgpt-pro，基础路径为 `/proApi/core/dataset/tag`。接口通过现有 dataset 或 Collection 权限进行鉴权，内部统一操作 v2 标签表。 + +| 路径 | 方法 | 作用 | 关键请求 | +|---|---|---|---| +| `/create` | POST | 创建标签 | `datasetId`、`tag`、可选 `tagType` | +| `/update` | POST | 修改标签名称 | `datasetId`、`tagId`、`tag` | +| `/delete` | DELETE | 删除标签定义 | dataset 和标签标识 | +| `/list` | POST | 分页查询标签 | dataset、分页参数 | +| `/getAllTags` | GET | 查询全部标签 | dataset | +| `/usage` | GET | 查询标签使用情况 | dataset、标签标识 | +| `/batchUpsert` | POST | 批量维护标签定义 | `datasetId`、`tags: [{ tag, tagType }]` | +| `/setCollectionTags` | POST | 设置单个 Collection 标签值 | `datasetId`、`collectionId`、`tags` | +| `/batchSetCollectionTags` | POST | 批量设置 Collection 标签值 | `collectionIds`、`tags` | +| `/addToCollections` | POST | 按 Collection 差集添加或移除标签 | `tag`、`collectionIds`、`originCollectionIds`、`datasetId` | + +`tagType` 的默认值为 `string`。标签值由 `CollectionTagValueSchema` 校验，存储格式为： + +```json +{ + "tagId": "v2-tag-object-id", + "value": "标签值或数组值" +} +``` + +### 4.2 Collection 标签接口 + +所有 Collection 创建入口均可接收混合格式标签： + +```text +(string | { tag: string, value: string | number | string[] })[] +``` + +字符串元素被视为历史名称格式，由 `createOrGetCollectionTags` 聚合到当前 dataset 的 `default_tag` 数组标签；对象元素按 `tag`（标签名）解析到 v2 标签定义并进行存在性和类型校验。最终持久化时统一使用 `{ tagId, value }` 格式。 + +> **契约层次**：Collection 创建/更新入参为 `(string | { tag, value })[]`（按标签名解析，与 listV2 返回格式一致）；持久化存储为 `{ tagId, value }[]`；fastgpt-pro 的标签值设置接口（`setCollectionTags`/`batchSetCollectionTags`）入参为 `{ tagId, value }[]`（按标签 ID 解析）。三者格式不同属有意设计，使用时勿混用。 + +Collection 列表接口 `listV2` 使用 `collectionTagsToTagLabel` 将存储格式转换为： + +```text +(string | { tag: string, value: string | number | string[] })[] +``` + +其中无法在 v2 表中解析的悬空引用不返回给调用方。 + +### 4.3 搜索过滤接口 + +工作流通过 `datasetSearchNode` 的隐藏输入 `collectionFilterVersion` 选择过滤器：显式 `legacy` 调用 `filterLegacyCollectionByMetadata`，显式 `structured` 调用 `filterCollectionByMetadata`；存量节点缺少版本时，仅在存在实际元数据过滤配置时使用 legacy，未配置过滤则直接使用 structured。兼容判断只检查配置是否存在，不根据 `collectionFilterMatch` 的字符串或对象形状推断版本。Agent V2 固定使用结构化过滤。共享能力支持： + +- `tags.$and`：所有条件必须满足。 +- `tags.$or`：至少一个条件满足。 +- `createTime.$gte/$lte`：按 Collection 创建时间过滤。 +- `collectionIds`：按指定 Collection 或文件夹递归展开后过滤。 + +新版入口只接受结构化标签条件；字符串或 `null` 条件直接拒绝。Legacy 入口仅按 `fromMigration=true` 查找承载标签，不调用 `filterCollectionByKeyValueTags` 或 `checkValue`。承载标签可以像普通标签一样改名或删除；删除后依赖它的 legacy 字符串条件无法满足，过滤结果为 `[]`，工作流继续运行但不产生知识库召回结果。 + +### 4.4 自动系统迁移 + +标签迁移不再暴露手工管理员接口。`20260907_migrate_dataset_tags_v2` 注册为阻塞启动任务，由统一 Runner 负责 lease、runId fencing、checkpoint、进度和失败状态。任务失败时 App 不进入 ready。 + +## 5. 内部设计 + +### 5.1 标签数据模型 + +v2 标签定义由 `teamId + datasetId + tag` 确定作用域，该三元组有唯一索引保证并发写入下不重复；`tagType` 决定值校验与过滤比较方式。只有存在旧字符串标签或仍通过兼容 API 写入字符串标签的 dataset 才按需创建一条 `tagType=array`、`fromMigration=true` 的承载定义，首次创建时默认命名为 `default_tag`。`default_tag` 本身是普通标签名称，不承担身份、权限或过滤分流语义；legacy filter 只按 `fromMigration` 定位承载记录。 + +```mermaid +classDiagram + class DatasetCollectionTagV2 { + ObjectId _id + ObjectId teamId + ObjectId datasetId + string tag + string tagType + boolean fromMigration + } + class CollectionTagValue { + string tagId + string_or_number_or_array value + } + class DatasetCollection { + ObjectId _id + ObjectId teamId + ObjectId datasetId + CollectionTagValue[] tags + } + DatasetCollection "1" --> "0..*" CollectionTagValue + CollectionTagValue "*" --> "1" DatasetCollectionTagV2 : tagId +``` + +### 5.2 标签创建与值归一化 + +```mermaid +sequenceDiagram + actor Client as 调用方 + participant API as Collection Create API + participant Utils as createOrGetCollectionTags + participant V2 as v2 标签表 + participant Col as dataset_collections + + Client->>API: 提交 tags + API->>API: Zod 参数校验和 dataset 权限校验 + API->>Utils: 归一化 string/object 混合标签 + alt 存在 string 标签 + Utils->>V2: 查询或创建 default_tag(array) + Utils-->>API: {tagId: defaultTagId, value: string[]} + end + alt 存在对象标签 + Utils->>V2: 查询 tagId 和 tagType + alt 标签不存在或值类型错误 + V2-->>API: tagNotExist/tagValueInvalid + else 校验通过 + Utils-->>API: 保留对象标签值 + end + end + API->>Col: 写入统一的新格式 tags + Col-->>Client: 创建成功 +``` + +处理规则： + +1. 未提供 `tags` 时不写入标签值；空数组保留为空数组。 +2. string 标签名全部合并为一条 `default_tag` 记录，数组值去重。 +3. object 标签必须引用当前 team、dataset 下存在的 v2 标签。 +4. `string`、`number`、`datetime`、`array` 的值类型分别校验： + - `string` 只能为字符串，长度不超过 256。 + - `number` 必须为有限数值；支持输入 number 或可转换为有限数值的字符串，空字符串、纯空白字符串及无法转换的字符串均无效；校验通过后统一按 number 存储。 + - `datetime` 必须为有效的 Unix 时间戳（毫秒）；支持输入 number 或可转换为有限数值的字符串，空字符串、纯空白字符串、无法转换的字符串及超出 JavaScript `Date` 有效范围的数值均无效；日期字符串（如 `2024-01-01`）不作为输入格式，校验通过后统一按 UTC 毫秒时间戳存储。 + - `array` 只能包含字符串，数组长度不超过 64，单元素长度不超过 256。 +5. 多个输入对象引用同一 `tagId` 且 value 不一致时拒绝整个批量操作。 + +### 5.3 标签批量设置 + +`batchSetCollectionTags` 的执行顺序如下： + +1. 校验请求参数及每个 Collection 的写权限。 +2. 对输入标签按 `tagId` 去重并检测冲突。 +3. 读出目标 Collection 的当前 tags，在内存中计算覆盖、累加或移除后的结果。 +4. 用 `bulkWrite` 一次性写回各 Collection 的 tags。 + +删除与写入操作均限定在请求的 `teamId`、`datasetId` 和 `collectionIds` 范围内。无匹配文档视为成功；数据库异常向上抛出，由 API 返回服务端错误。 + +`addToCollections` 计算 `collectionIds - originCollectionIds` 得到添加集合，计算反向差集得到移除集合。添加与移除都先读出当前 tags，在内存中覆盖或去掉该 `tagId`，再用 `bulkWrite` 一次性写回，确保一个 Collection 中同一标签只有一条记录。自动值规则为：`string` 使用空字符串；`number`、`datetime`、`array` 无法通过此接口添加，须走显式值接口。 + +### 5.4 标签检索过滤 + +```mermaid +sequenceDiagram + participant Workflow as 工作流/Agent + participant Dispatcher as Dataset Search Dispatcher + participant Legacy as legacy filter + participant Structured as structured filter + participant TagDB as v2 标签表 + participant ColDB as dataset_collections + + Workflow->>Dispatcher: collectionFilterVersion + collectionFilterMatch + alt 显式 legacy，或缺少版本且存在历史过滤配置 + Dispatcher->>Legacy: 旧 JSON 过滤 + Legacy->>TagDB: 按 dataset 查 fromMigration=true + Legacy->>ColDB: 保持旧 AND/OR/null 语义 + else structured + Dispatcher->>Structured: 结构化条件 + Structured->>TagDB: 按 dataset 和标签名查 tagId/tagType + Structured->>ColDB: 按 tags.tagId 粗筛并使用 hint + Structured->>Structured: 按 tagType 执行 checkValue + end + Dispatcher->>Dispatcher: 与 createTime、collectionIds 求交集 + Dispatcher-->>Workflow: 最终 collectionId 列表 +``` + +过滤实现分为两层： + +- MongoDB 粗筛：按标签 ID 使用 `$all` 或 `$in` 查询，并通过 `.hint({ teamId: 1, datasetId: 1, 'tags.tagId': 1 })` 强制使用新格式标签索引。 +- 应用层精筛：读取候选 Collection 的标签对象，根据 v2 标签的 `tagType` 和操作符调用 `checkValue`。 + +标签不存在、Collection 未配置该标签、值类型不匹配或比较值不可转换时，条件均视为不满足。`$and` 要求全部满足，`$or` 要求至少满足一项；不存在的标签不会被当作满足条件。 + +### 5.5 核心比较算法 + +```text +checkValue(op, target, storedValue, tagType): boolean + 若 op 为 $empty： + array 类型判断值不存在或数组长度为 0；其他类型判断 null、undefined 或空字符串 + 若 op 为 $notEmpty： + array 类型判断为非空数组；其他类型判断为非空值 + target 为 null/undefined 时返回 false + + number/datetime：将存储值和目标值转换为数字，转换失败返回 false，执行 eq/ne/gt/lt/gte/lte + array： + $is -> 两个数组去重后集合相等 + $isNot -> 两个数组去重后集合不相等 + $contains -> 存储数组包含目标字符串 + $notContains -> 存储数组不包含目标字符串 + $in -> 存储数组是目标数组的子集 + $notIn -> 存储数组不是目标数组的子集 + string： + $eq/$ne 使用大小写敏感比较 + $contains/$notContains/$startsWith/$endsWith 忽略大小写 + $regex 使用 RegExp，正则构造失败返回 false +``` + +`$regex` 接受用户可控的 pattern，为防灾难性回溯（ReDoS），pattern 长度上限 64、被测值长度上限 256，并拦截不安全 pattern：嵌套量词类（`(a+)+`）由 `safe-regex` 检出，带分支的量词组（`(a|aa)+`）由首字符重叠检测兜底，超限或不安全均视为不匹配。 + +array 比较的时间复杂度为 `O(n+m)`，其他类型为 `O(1)`；单个 array 值受长度限制，应用层比较不会产生无界内存增长。 + +### 5.6 历史数据迁移 + +迁移由统一 System Migration Runner 按稳定 ObjectId 游标分批执行： + +1. 按 `{ teamId, datasetId, tag }` 整理重复 v2 定义；保留 `_id` 最早项，对其余 ID 先从 Collection 删除引用并校验，再删除定义。 +2. 从 v1 表建立 `旧 tagId -> 标签名称` 映射；遇到旧字符串标签时，按需创建或复用当前 dataset 的系统承载标签并合并名称数组。 +3. 保留其他新格式标签，移除旧字符串元素，使用原始 BSON 快照条件更新。 +4. 建立 v2 唯一索引和 `tags.tagId` 索引，校验无旧字符串标签、无重复定义、无重复或非法承载标签。 + +Collection 子阶段只修改 `tags` 字段，不修改 `_id`、`createTime` 或其他元数据，因此既有 `collectionIds` 和时间过滤仍然有效。工作流记录不参与数据迁移：运行时仅将“缺少版本且实际配置了过滤”的存量节点解释为 legacy，空配置直接使用 structured。重复执行时已转换记录不会再次转换；v1 表在迁移过程中只读。 + +### 5.7 dataset 物理删除 + +异步 `datasetDeleteProcessor` 找到 root dataset 及全部子 dataset 后，删除 v1 标签、v2 标签及其他 dataset 关联资源。v2 清理条件为： + +```text +{ teamId, datasetId: { $in: rootAndChildDatasetIds } } +``` + +该操作不会删除其他 team 或其他 dataset 的标签定义；无匹配是正常成功。v2 标签清理位于本体 Mongo session 事务之外，任务失败时依赖现有队列重试和日志处理，不宣称跨操作原子性。 + +## 6. 数据库设计 + +### 6.1 `dataset_collection_tags_v2` + +```javascript +const DatasetCollectionTagsV2Schema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + tag: { + type: String, + required: true + }, + tagType: { + type: String, + default: 'string', + enum: ['string', 'number', 'datetime', 'array'] + }, + fromMigration: { + type: Boolean, + default: false + } +}); + +DatasetCollectionTagsV2Schema.index({ teamId: 1, datasetId: 1, tag: 1 }, { unique: true }); +``` + +字段说明： + +| 字段 | 类型 | 约束 | 说明 | +|---|---|---|---| +| `_id` | ObjectId | 主键 | 标签定义 ID | +| `teamId` | ObjectId | 必填 | 所属团队 | +| `datasetId` | ObjectId | 必填 | 所属知识库 | +| `tag` | String | 必填 | 标签名称；`default_tag` 按普通字符串处理 | +| `tagType` | String | 枚举 | `string`、`number`、`datetime`、`array`，默认 `string` | +| `fromMigration` | Boolean | 默认 `false` | 标识旧标签数据的系统承载记录；仅 legacy filter 用于定位 | + +`{ teamId, datasetId, tag }` 唯一索引保证同一作用域下标签名不重复，并发写入由数据库兜底；业务层先查后写，撞索引时捕获 E11000 复用已存在记录。 + +### 6.2 `dataset_collections.tags` + +Collection 现有 `tags` 字段继续使用 Mixed 数组，不增加独立字段： + +```javascript +tags: { + type: [], + default: [] +} + +DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, tags: 1 }); +DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, 'tags.tagId': 1 }); +``` + +推荐的新格式： + +```json +[ + { "tagId": "v2-string-id", "value": "产品" }, + { "tagId": "v2-number-id", "value": 2 }, + { "tagId": "v2-datetime-id", "value": 1704067200000 }, + { "tagId": "v2-array-id", "value": ["安全", "高优"] } +] +``` + +旧字符串格式仅用于迁移前历史数据。旧索引保留以支持历史数据读取，新格式过滤使用 `tags.tagId` 索引。 + +### 6.3 v1 表 + +`dataset_collection_tags` 保持原有结构和数据，仅用于历史迁移的只读映射以及现有删除链路的兼容清理。新 API 不读写该表作为当前标签定义来源。 + +## 7. 类型、校验与错误处理 + +### 7.1 全局类型 + +核心定义位于 `packages/global/core/dataset/type.ts`： + +```typescript +DatasetCollectionTagType = 'string' | 'number' | 'datetime' | 'array' +CollectionTagValueType = { + tagId: string; + value: string | number | string[]; +} +DEFAULT_TAG = 'default_tag' +``` + +### 7.2 校验规则 + +| 场景 | 处理 | +|---|---| +| 标签名为空 | 拒绝请求，返回标签名称错误 | +| 标签名重复 | 拒绝请求，返回重复错误 | +| v2 标签不存在 | 返回 `tagNotExist` | +| string 值不是字符串或超长 | 返回 `tagValueInvalid` | +| number 无法转换或超出安全范围 | 返回 `tagValueInvalid` | +| datetime 无法转换为有效时间 | 返回 `tagValueDatetimeInvalid` | +| array 非数组、元素非字符串、长度超过 64 或元素超过 256 | 返回 `arrayTagValueInvalid` | +| Legacy 过滤 JSON 解析失败 | 保持历史降级行为，当前请求不执行该过滤 | +| Structured 过滤不合法 | 拒绝运行，不回退 legacy 或扩大召回范围 | +| MongoDB 查询失败 | 保留异常并由上层统一返回服务端错误 | +| 旧标签 ID 在 v1 表中不存在 | 迁移时忽略该标签，其他可解析标签继续处理 | +| 同一 tagId 出现冲突值 | 拒绝批量写入，不执行覆盖操作 | + +所有读写操作必须带 `teamId` 和 `datasetId` 作用域，API 入口使用现有权限函数，避免跨团队读取或修改标签。 + +## 8. 可靠性、性能与可运维性 + +### 8.1 一致性 + +- v2 标签定义与 Collection 标签值通过 `tagId` 关联，应用层校验引用关系。 +- 写入 Collection 标签先读出当前 tags，在内存中保证同一 `tagId` 只有一条，再用 `bulkWrite` 写回。 +- dataset 物理删除按完整 root/child dataset ID 集合清理，防止子 dataset 标签残留。 +- 迁移按 dataset 维度可重复执行，已转换 Collection 跳过。 + +### 8.2 降级策略 + +- 过滤条件 JSON 不合法时不阻断知识检索。 +- 未知或缺失 `tagType` 按 `string` 处理。 +- 标签定义不存在时条件不匹配，不扩大查询结果。 +- secondary 读取用于列表和过滤等只读路径，写入仍使用主库。 + +### 8.3 性能设计 + +1. v2 标签查询使用 `{ teamId, datasetId, tag }` 复合索引。 +2. 新格式 Collection 过滤使用 `{ teamId, datasetId, 'tags.tagId' }` 复合索引。 +3. `filterCollectionByKeyValueTags` 先按 tagId 粗筛，再在应用层执行类型比较，减少精确比较数据量。 +4. 同时存在旧标签索引和新标签索引时，使用 MongoDB `hint` 强制选择 `tags.tagId` 索引，避免查询规划器误选旧多键索引。 +5. array 值最多 64 项，单项最多 256 字符，控制比较和内存开销。 +6. 标签列表接口使用分页、字段投影和 secondary read；Collection 列表仅返回所需字段。 + +### 8.4 运维要求 + +- 迁移前备份 `dataset_collections`，确认迁移范围和结果。 +- 迁移由启动 Runner 自动执行；运维通迁移页面查看阶段进度，失败时 App 不进入 ready。 +- 唯一索引和 `tags.tagId` 索引在完成校验前创建，运行期可安全使用强制 hint。 +- 关注系统迁移阶段错误、dataset 删除任务日志及 MongoDB 查询慢日志。 +- v1 表的最终下线需另行设计，不属于本模块。 + +## 9. 测试设计 + +### 9.1 单元测试 + +| 类别 | 必测内容 | +|---|---| +| 标签值校验 | 四种 tagType 的合法值、非法值、边界值 | +| `checkValue` | 各类型操作符、空值、类型转换失败、正则异常 | +| array 比较 | `$is`、`$isNot`、`$contains`、`$notContains`、`$in`、`$notIn`、`$empty`、`$notEmpty` | +| 标签归一化 | string、object 和混合输入；对象中的 `default_tag` 按普通标签名解析 | +| 冲突检测 | 相同 tagId 相同值去重、不同值拒绝 | +| Legacy 过滤 | string、null、AND 优先、OR、混合 null/string、多 dataset 承载标签 | +| Structured 过滤 | 拒绝 string/null，按类型执行新版条件 | + +### 9.2 集成测试 + +1. 创建 Collection 后 string 标签被保存为 `default_tag` 新格式。 +2. 创建和更新四种类型标签值均能正确读写。 +3. `listV2` 返回标签名称而非内部 tagId。 +4. 标签过滤与时间、Collection ID、文件夹递归条件正确求交集。 +5. Agent V2 搜索可以透传 `collectionFilterMatch`。 +6. 迁移保留已有新格式标签，旧标签名称正确合并，重复执行和 checkpoint 恢复幂等。 +7. 迁移不修改 Collection `_id` 和 `createTime`。 +8. dataset root/child 物理删除只清理目标 team 和目标 dataset 集合的 v2 标签。 +9. 其他 team、其他 dataset 的同名或同 ID 标签不受影响。 +10. MongoDB 查询命中新格式 `tags.tagId` 索引。 +11. 缺少版本标记且存在历史过滤配置的旧节点调用 legacy filter；历史空配置和显式 structured 节点调用 structured filter。 +12. 节点升级保留 ID、输入输出与连线，且持久化失败时不切换本地节点。 + +### 9.3 性能验证 + +在真实或等价 MongoDB 环境验证 30,000 个 Collection 的标签查询： + +- 有 `tags.tagId` 索引时应明显减少扫描文档数。 +- 两个标签索引并存时应确认查询使用 `hint` 指定的新格式索引。 +- array 应用层比较耗时随数组长度线性增长，且受最大长度限制。 + +## 10. 发布与回滚 + +### 10.1 发布顺序 + +> 当前数据格式不支持旧 Pod 与迁移后数据并存。发布时必须先停止旧版本业务流量再启动带阻塞迁移的新版本；若要求无停机滚动升级，需要先单独发布能双读对象标签的过渡版本。 + +1. App 启动时执行阻塞式 `20260907_migrate_dataset_tags_v2` 系统迁移，失败时不进入 ready。 +2. 迁移按稳定 `_id` 游标分批处理标签定义和 Collection，批次提交后才保存 checkpoint。 +3. 迁移只为存在旧字符串标签的 dataset 按需创建唯一 `fromMigration=true` 承载标签，并把旧标签名称转换为数组值。 +4. 重复 v2 标签定义按 `_id` 保留最早项；删除定义前先从 Collection 中删除指向重复 ID 的标签值，不向保留项合并。 +5. 全部阶段完成后校验无旧字符串标签、无重复定义、无重复 ID 残留引用及非法承载标签。 +6. 存量 `datasetSearchNode` 不改写；缺少 `collectionFilterVersion` 时按是否存在历史过滤配置选择 legacy 或 structured，新建节点显式保存 structured。 +7. 阻塞迁移成功后才开放业务流量，不在运行期同时读取 v1/v2 标签。 + +### 10.2 回滚策略 + +- 未迁移数据仍保留 v1 标签和旧 `tags` 值，可回退到旧代码读取。 +- v2 标签定义和新格式 Collection 数据需通过数据库备份恢复；不通过迁移接口反向回滚。 +- 迁移不改 Collection 主键、创建时间或工作流节点类型；回滚旧版代码前需从备份恢复 `tags` 字段。 +- v1 表不在本次发布中删除，回滚期间保留历史读取基础。 + +## 11. 实现文件索引 + +| 文件 | 作用 | +|---|---| +| `packages/global/core/dataset/type.ts` | 标签类型、标签值 Schema、`DEFAULT_TAG` | +| `packages/global/openapi/core/dataset/collection/tagApi.ts` | 标签 API 请求 Schema | +| `packages/service/core/dataset/tag/schemaV2.ts` | v2 标签 MongoDB Schema | +| `packages/service/core/dataset/collection/schema.ts` | Collection Schema 和标签索引 | +| `packages/service/core/dataset/collection/utils.ts` | 标签创建、校验、存储和展示转换 | +| `packages/service/core/dataset/search/defaultRecall/collectionFilter.ts` | 新版结构化标签过滤 | +| `packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts` | 存量节点的 legacy 过滤语义 | +| `projects/app/src/pages/api/core/dataset/collection/listV2.ts` | Collection 标签过滤列表接口 | +| `projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/` | 历史标签阻塞迁移 | +| `packages/service/core/dataset/delete/processor.ts` | dataset 物理删除和 v2 标签清理 | +| `packages/service/core/dataset/collection/tagFilter.ts` | 详情页标签值筛选：已用值聚合与列表查询条件 | +| `projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts` | 标签筛选选项接口 | +| `projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx` | 知识库详情页双栏标签筛选弹窗 | + +本设计描述的是同一套标签能力的统一实现，不按迭代或变更批次拆分章节。 + +## 12. 知识库详情页标签筛选弹窗 + +Figma：`07页面｜标签筛选`（node `2434:38727`），交互组件标注「两条独立滚动，勾选即生效」。 + +### 12.1 交互 + +1. 触发器默认 `标签 | 全部`；有选中值时展示第一项 `标签名：值`，其余以 `+n` 收起。 +2. 弹窗宽 320px，左右两列独立滚动。 +3. 左侧列出当前知识库全部标签定义；当前列高亮，该标签已选数量以圆形角标展示。 +4. 右侧第一行搜索当前标签的值；下方 checkbox 多选可筛值。左右列固定高度并独立滚动。 +5. 数字/日期/文本：展示当前文件上出现过的全部值。日期展示 `YYYY-MM-DD HH:mm`。 +6. 选项类：展示标签管理里的全部预设 options，并与文件上已用但不在预设里的值取并集。 +7. 勾选即写入列表查询条件，无单独确认按钮。底部展示「已选 N 项」和「清空」（无选中时禁用）。 +8. 右侧无值显示「暂无可筛选值」；搜索无匹配显示「未匹配到相关信息」。 +9. 有筛选条件且列表为空时，空态提供「清空筛选项」。 + +### 12.2 过滤语义 + +- 同一标签多个值：OR（命中任一值）。 +- 不同标签：AND（每个被筛选的标签都要命中）。 +- 选项类按数组包含匹配；文本/数字/日期按等值匹配。 +- `listV2` 只接受 `tagFilters`，按 `{ tagId, values }` 精确匹配；不再兼容旧版 `filterTags` 与字符串标签。 + +### 12.3 接口 + +- `GET /api/core/dataset/collection/tagFilterOptions?datasetId=`：只返回 `{ tagId, values }`（已用值）。标签名和类型复用 `getAllTags`。 +- `POST /api/core/dataset/collection/listV2` 新增 `tagFilters: [{ tagId, values }]`。 + +## 13. 旧新检索节点分流 + +1. `datasetSearchNode` 显式 `legacy` 时调用 legacy filter；显式 `structured` 时调用 structured filter；缺少版本时，有历史过滤配置走 legacy，空配置走 structured。 +2. 节点版本只由隐藏输入决定，不根据 `collectionFilterMatch` 的字符串或对象形状推断。 +3. legacy filter 使用每个 dataset 自己的 `fromMigration=true` 承载标签，保持旧 `$and` 优先于 `$or` 及 `null` 匹配真正空标签 Collection 的语义。 +4. structured filter 只接受结构化条件，字符串或 `null` 标签条件是非法配置，不隐式转换或回退 legacy。 +5. 用户点击升级时保留节点类型、ID、公共参数、输入输出和连线，清空无法安全转换的旧过滤配置，并在保存成功后把版本输入切换为 `structured`。 +6. Agent V2 模板显式保存 `structured`；Agent 搜索服务也显式指定 structured filter，不复用 Dataset Search 的缺省兼容规则。 + +## TODO + +- [x] 新增并注册阻塞式分批标签迁移，包含进度、checkpoint、lease 检查和完成校验。 +- [x] 迁移重复标签定义时先删除 Collection 引用，再删除定义。 +- [x] 使用隐藏版本输入在单一节点类型内分流 legacy/structured 过滤器，删除新链路的 legacy rewrite。 +- [x] 完成节点升级、Agent 配置入口、multipart tags 透传和标签删除引用清理。 +- [x] 承载标签只按 `fromMigration` 定位，`default_tag` 按普通标签名称处理。 +- [x] 补充迁移恢复、旧新语义、API 透传、删除清理和升级行为测试。 +- [x] 运行局部测试、App typecheck、相关 lint 和 `git diff --check`。 +- [ ] 发布前确认采用停机切换，或先发布兼容对象标签的过渡版本；当前实现不支持旧 Pod 与迁移后数据并存。 diff --git a/.gitignore b/.gitignore index f1aa0474d63d..155b596288e0 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ pro/admin/worker/ /pro/llm_benchmark/content_benchmark/eval-runs/ /pro/llm_benchmark/content_benchmark/.cache/ .gstack/ +.worktrees/ diff --git a/packages/global/common/error/code/dataset.ts b/packages/global/common/error/code/dataset.ts index 8b554c8a0d52..de72f57fdaac 100644 --- a/packages/global/common/error/code/dataset.ts +++ b/packages/global/common/error/code/dataset.ts @@ -15,7 +15,15 @@ export enum DatasetErrEnum { notSupportSync = 'notSupportSync', sameApiCollection = 'sameApiCollection', noApiServer = 'noApiServer', - canNotEditAdminPermission = 'canNotEditAdminPermission' + canNotEditAdminPermission = 'canNotEditAdminPermission', + + // Tag errors (501013+) + tagNameDuplicate = 'tagNameDuplicate', + tagNameEmpty = 'tagNameEmpty', + tagNotExist = 'tagNotExist', + tagValueInvalid = 'tagValueInvalid', + tagValueDatetimeInvalid = 'tagValueDatetimeInvalid', + arrayTagValueInvalid = 'arrayTagValueInvalid' } const datasetErr = [ { @@ -69,6 +77,32 @@ const datasetErr = [ { statusText: DatasetErrEnum.noApiServer, message: i18nT('common:core.dataset.error.noApiServer') + }, + + // Tag errors + { + statusText: DatasetErrEnum.tagNameDuplicate, + message: i18nT('common:core.dataset.error.tagNameDuplicate') + }, + { + statusText: DatasetErrEnum.tagNameEmpty, + message: i18nT('common:core.dataset.error.tagNameEmpty') + }, + { + statusText: DatasetErrEnum.tagNotExist, + message: i18nT('common:core.dataset.error.tagNotExist') + }, + { + statusText: DatasetErrEnum.tagValueInvalid, + message: i18nT('common:core.dataset.error.tagValueInvalid') + }, + { + statusText: DatasetErrEnum.tagValueDatetimeInvalid, + message: i18nT('common:core.dataset.error.tagValueDatetimeInvalid') + }, + { + statusText: DatasetErrEnum.arrayTagValueInvalid, + message: i18nT('common:core.dataset.error.arrayTagValueInvalid') } ]; export default datasetErr.reduce((acc, cur, index) => { diff --git a/packages/global/common/string/time.ts b/packages/global/common/string/time.ts index b14573e9140c..d19063ceaf56 100644 --- a/packages/global/common/string/time.ts +++ b/packages/global/common/string/time.ts @@ -83,7 +83,7 @@ export const cronParser2Fields = (cronString: string) => { try { const cronField = cronParser.parseExpression(cronString).fields; return cronField; - } catch (error) { + } catch { return null; } }; diff --git a/packages/global/core/app/formEdit/utils.ts b/packages/global/core/app/formEdit/utils.ts index 67ff5c7debfc..3abdea267543 100644 --- a/packages/global/core/app/formEdit/utils.ts +++ b/packages/global/core/app/formEdit/utils.ts @@ -36,7 +36,8 @@ const agentGeneratedDenyRenderTypes = new Set([ FlowNodeInputTypeEnum.selectTool, FlowNodeInputTypeEnum.selectDataset, FlowNodeInputTypeEnum.selectDatasetParamsModal, - FlowNodeInputTypeEnum.settingDatasetQuotePrompt + FlowNodeInputTypeEnum.settingDatasetQuotePrompt, + FlowNodeInputTypeEnum.datasetTagFilter ]); // 工具配置不能处理依赖文件、知识库、模型或外部动态上下文的输入。 diff --git a/packages/global/core/app/type.ts b/packages/global/core/app/type.ts index b8210555fbaf..4b1172794979 100644 --- a/packages/global/core/app/type.ts +++ b/packages/global/core/app/type.ts @@ -2,6 +2,10 @@ import { StoreNodeItemTypeSchema } from '../workflow/type/node'; import { AppTypeEnum } from './constants'; import { NodeInputKeyEnum } from '../workflow/constants'; import { DatasetSearchModeEnum } from '../dataset/constants'; +import { + DatasetTagFilterValueSchema, + DatasetTagFilterVersionSchema +} from '../dataset/workflowTagFilter'; import type { ReasoningEffort } from '../ai/llm/type'; import { StoreEdgeItemTypeSchema } from '../workflow/type/edge'; import type { AppPermission } from '../../support/permission/app/controller'; @@ -271,7 +275,8 @@ export const AppDatasetSearchParamsTypeSchema = z.object({ datasetSearchExtensionBg: z.string().optional(), [NodeInputKeyEnum.authTmbId]: BoolSchema.optional(), - collectionFilterMatch: z.string().optional() + collectionFilterMatch: z.union([z.string(), DatasetTagFilterValueSchema]).optional(), + [NodeInputKeyEnum.collectionFilterVersion]: DatasetTagFilterVersionSchema.optional() }); export type AppDatasetSearchParamsType = z.infer; diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts index 97cefe73e37d..008eea3326d8 100644 --- a/packages/global/core/app/utils.ts +++ b/packages/global/core/app/utils.ts @@ -6,6 +6,7 @@ import { AppTypeEnum } from './constants'; import appErrList from '../../common/error/code/app'; import pluginErrList from '../../common/error/code/plugin'; import { i18nT } from '../../common/i18n/utils'; +import { DatasetTagFilterVersionEnum } from '../dataset/workflowTagFilter'; const deletedPluginErrorList = new Set([ 'plugin.team_not_installed', @@ -34,6 +35,7 @@ export const getDefaultAppForm = (): AppFormEditFormType => { rerankWeight: 0.5, datasetSearchUsingExtensionQuery: true, datasetSearchExtensionBg: '', + [NodeInputKeyEnum.collectionFilterVersion]: DatasetTagFilterVersionEnum.structured, [NodeInputKeyEnum.authTmbId]: false }, selectedTools: [], diff --git a/packages/global/core/dataset/collection/utils.ts b/packages/global/core/dataset/collection/utils.ts index e2ef77d91494..c4ea83db2933 100644 --- a/packages/global/core/dataset/collection/utils.ts +++ b/packages/global/core/dataset/collection/utils.ts @@ -1,7 +1,12 @@ import { DatasetCollectionTypeEnum } from '../constants'; import { type DatasetCollectionSchemaType } from '../type'; -export const getCollectionSourceData = (collection?: DatasetCollectionSchemaType) => { +export const getCollectionSourceData = ( + collection?: Pick< + DatasetCollectionSchemaType, + 'fileId' | 'rawLink' | 'externalFileId' | 'externalFileUrl' | 'apiFileId' | 'name' + > +) => { return { sourceId: collection?.fileId || diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 04a098adde41..37aac4e45933 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -164,6 +164,29 @@ export const DatasetCollectionSyncResultMap = { } }; +/* ------------ collection tags -------------- */ +export enum DatasetCollectionTagTypeEnum { + string = 'string', + number = 'number', + datetime = 'datetime', + array = 'array' +} + +export const DatasetCollectionTagTypeMap = { + [DatasetCollectionTagTypeEnum.string]: { + label: i18nT('dataset:core.dataset.tags.string') + }, + [DatasetCollectionTagTypeEnum.number]: { + label: i18nT('dataset:core.dataset.tags.number') + }, + [DatasetCollectionTagTypeEnum.datetime]: { + label: i18nT('dataset:core.dataset.tags.date') + }, + [DatasetCollectionTagTypeEnum.array]: { + label: i18nT('dataset:core.dataset.tags.array') + } +}; + export enum DatasetCollectionDataProcessModeEnum { chunk = 'chunk', qa = 'qa', diff --git a/packages/global/core/dataset/tagUtils.ts b/packages/global/core/dataset/tagUtils.ts new file mode 100644 index 000000000000..d4721c7fc4a3 --- /dev/null +++ b/packages/global/core/dataset/tagUtils.ts @@ -0,0 +1,19 @@ +import type { CollectionTagValueType } from './type'; + +/** 同一标签类型固定：数字按大小，其余按字典序。前后端筛选项展示共用。 */ +export const sortCollectionTagValues = (values: T[]): T[] => + [...values].sort((a, b) => { + if (typeof a === 'number' && typeof b === 'number') return a - b; + return String(a).localeCompare(String(b)); + }); + +/** 筛选项只保留非空字符串和有限数字。 */ +export const isUsableCollectionTagFilterValue = (value: unknown): value is string | number => { + if (typeof value === 'string') return value.length > 0; + if (typeof value === 'number') return Number.isFinite(value); + return false; +}; + +/** 新格式 { tagId, value }；旧字符串标签排除。 */ +export const isCollectionTagValue = (item: unknown): item is CollectionTagValueType => + !!item && typeof item === 'object' && !Array.isArray(item) && 'tagId' in item && 'value' in item; diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index a50d68172946..cd0818afef5e 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -10,7 +10,9 @@ import { CollectionTrainingStatusEnum, ChunkSettingModeEnum, ChunkTriggerConfigTypeEnum, - ParagraphChunkAIModeEnum + ParagraphChunkAIModeEnum, + DatasetCollectionTagTypeEnum, + DatasetCollectionTagTypeMap } from './constants'; import { ApiDatasetServerSchema, @@ -27,6 +29,59 @@ import { ObjectIdSchema } from '../../common/type/mongo'; import { PermissionSchema } from '../../support/permission/controller'; import { NumSchema } from '../../common/zod'; +/* ===== Tag Type ===== */ +/** 标签类型枚举 */ +export { DatasetCollectionTagTypeEnum, DatasetCollectionTagTypeMap }; +export type DatasetCollectionTagType = `${DatasetCollectionTagTypeEnum}`; + +/** 选项类标签的预设选项，空选项由前端草稿过滤后再提交。 */ +export const DatasetCollectionTagOptionsSchema = z + .array(z.string().trim().min(1)) + .meta({ description: '选项类标签的预设选项' }); + +/** 旧字符串标签承载记录首次创建时使用的默认名称；身份只由 fromMigration 标识。 */ +export const DEFAULT_TAG = 'default_tag'; + +/** Collection 标签值字段：string/number 存对应值，datetime 存 UTC 毫秒时间戳，array 存 string 数组 */ +export const CollectionTagValueFieldSchema = z.union([z.string(), z.number(), z.array(z.string())]); + +/** Collection API 的兼容标签输入/展示格式：旧标签名或带名称和值的新格式。 */ +export const CollectionTagLabelSchema = z.union([ + z.string(), + z.object({ + tag: z.string(), + value: CollectionTagValueFieldSchema + }) +]); +export type CollectionTagLabelType = z.infer; + +/** Collection 标签值类型（新格式） */ +export const CollectionTagValueSchema = z.object({ + tagId: z.string().meta({ description: '引用 dataset_collection_tags_v2._id' }), + value: CollectionTagValueFieldSchema.meta({ + description: + '标签值。string/number 类型存对应值，datetime 类型存 UTC 毫秒时间戳，array 类型存 string 数组' + }) +}); +export type CollectionTagValueType = z.infer; + +/** 详情页按标签值筛选的单条条件。同一标签多值为 OR，不同标签由调用方做 AND。 */ +export const CollectionTagFilterItemSchema = z.object({ + tagId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '标签 ID' + }), + values: z + .array(z.union([z.string().min(1), z.number().finite()])) + .min(1) + .meta({ + example: ['PRD'], + description: + '选中的标签值。文本/选项为字符串，数字为数值，日期为 UTC 毫秒时间戳。同一标签多值为 OR' + }) +}); +export type CollectionTagFilterItem = z.infer; + /* ===== Chunk ===== */ export const ChunkSettingsSchema = z.object({ trainingType: z @@ -134,7 +189,10 @@ export const DatasetCollectionSchema = ChunkSettingsSchema.omit({ parentId: ParentIdSchema.meta({ description: '父级 ID' }), name: z.string().meta({ description: '名称' }), type: z.enum(DatasetCollectionTypeEnum).meta({ description: '集合类型' }), - tags: z.array(z.string()).optional().meta({ description: '标签' }), + tags: z + .array(z.union([z.string(), CollectionTagValueSchema])) + .optional() + .meta({ description: '标签。支持混合格式：String(ObjectId) 旧格式 | { tagId, value } 新格式' }), createTime: z.coerce.date().meta({ description: '创建时间' }), updateTime: z.coerce.date().meta({ description: '更新时间' }), @@ -165,7 +223,14 @@ export const DatasetCollectionTagsSchema = z.object({ _id: ObjectIdSchema.meta({ description: '标签 ID' }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }), datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), - tag: z.string().meta({ description: '标签' }) + tag: z.string().meta({ description: '标签' }), + tagType: z.enum(DatasetCollectionTagTypeEnum).default(DatasetCollectionTagTypeEnum.string).meta({ + description: '标签类型：string(默认)/number/datetime/array' + }), + options: DatasetCollectionTagOptionsSchema.optional(), + fromMigration: z.boolean().optional().meta({ + description: '标识该记录是旧标签迁移/旧格式输入创建的 default_tag 承载记录' + }) }); export type DatasetCollectionTagsSchemaType = z.infer; @@ -353,7 +418,14 @@ export type DatasetItemType = z.infer; /* ================= tag ===================== */ export const DatasetTagSchema = z.object({ _id: ObjectIdSchema.meta({ description: '标签 ID' }), - tag: z.string().meta({ description: '标签' }) + tag: z.string().meta({ description: '标签' }), + tagType: z.enum(DatasetCollectionTagTypeEnum).default(DatasetCollectionTagTypeEnum.string).meta({ + description: '标签类型：string(默认)/number/datetime/array' + }), + options: DatasetCollectionTagOptionsSchema.optional(), + fromMigration: z.boolean().optional().meta({ + description: '是否为旧标签数据的系统承载定义' + }) }); export type DatasetTagType = z.infer; @@ -365,6 +437,11 @@ export type TagUsageType = z.infer; /* ================= collection ===================== */ export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({ + // 详情接口的 tags 由 collectionTagsToTagLabel 解析为标签名格式（string | { tag, value }），区别于存储格式（string | { tagId, value }） + tags: z + .array(CollectionTagLabelSchema) + .optional() + .meta({ description: '标签。string 为标签名；新格式为 { tag, value }' }), sourceName: z.string().meta({ description: '来源名称' }), sourceId: z.string().optional().meta({ description: '来源 ID' }), file: z diff --git a/packages/global/core/dataset/workflowTagFilter.ts b/packages/global/core/dataset/workflowTagFilter.ts new file mode 100644 index 000000000000..69a224b37b6f --- /dev/null +++ b/packages/global/core/dataset/workflowTagFilter.ts @@ -0,0 +1,430 @@ +import { z } from 'zod'; +import { formatTime2YMDHM } from '../../common/string/time'; +import { DatasetCollectionTagTypeEnum } from './constants'; +import type { DatasetCollectionTagType, DatasetTagType } from './type'; + +export const DatasetTagFilterLogicEnum = { + AND: 'AND', + OR: 'OR' +} as const; + +export const DatasetTagFilterValueModeEnum = { + input: 'input', + reference: 'reference' +} as const; +export type DatasetTagFilterValueMode = + (typeof DatasetTagFilterValueModeEnum)[keyof typeof DatasetTagFilterValueModeEnum]; + +export const DatasetTagFilterVersionEnum = { + legacy: 'legacy', + structured: 'structured' +} as const; +export const DatasetTagFilterVersionSchema = z.enum(DatasetTagFilterVersionEnum); +export type DatasetTagFilterVersion = z.infer; + +const hasDatasetTagFilterConfiguration = (value: unknown) => { + if (typeof value === 'string') return value.trim().length > 0; + if (Array.isArray(value)) return value.length > 0; + return value !== undefined && value !== null; +}; + +/** + * 显式版本优先；存量节点缺少版本时，有过滤配置走 legacy，无配置直接使用 structured。 + * 这里只判断是否配置，不根据过滤值的字符串或对象形状推断版本。 + */ +export const resolveDatasetTagFilterVersion = ({ + version, + filterValue +}: { + version: unknown; + filterValue: unknown; +}): DatasetTagFilterVersion => { + if (version === DatasetTagFilterVersionEnum.structured) { + return DatasetTagFilterVersionEnum.structured; + } + if (version === DatasetTagFilterVersionEnum.legacy) return DatasetTagFilterVersionEnum.legacy; + if (version !== undefined && version !== null && version !== '') { + return DatasetTagFilterVersionEnum.legacy; + } + return hasDatasetTagFilterConfiguration(filterValue) + ? DatasetTagFilterVersionEnum.legacy + : DatasetTagFilterVersionEnum.structured; +}; + +/** 旧编辑器只接收字符串；异常存量值显示为空，不用于推断节点版本。 */ +export const normalizeLegacyDatasetTagFilterValue = (value: unknown) => + typeof value === 'string' ? value : ''; + +/** 条件行字段来源：知识库标签，或固定文件属性。 */ +export const DatasetTagFilterFieldEnum = { + tag: 'tag', + createTime: 'createTime', + collectionId: 'collectionId' +} as const; +export type DatasetTagFilterField = + (typeof DatasetTagFilterFieldEnum)[keyof typeof DatasetTagFilterFieldEnum]; + +/** 工作流标签过滤支持的标签类型。string 不进入条件行下拉。 */ +const WorkflowTagFilterTagTypeSchema = z.enum([ + DatasetCollectionTagTypeEnum.number, + DatasetCollectionTagTypeEnum.datetime, + DatasetCollectionTagTypeEnum.array +] as const); +export type WorkflowTagFilterTagType = z.infer; + +export const DatasetTagFilterConditionSchema = z.object({ + field: z.enum(DatasetTagFilterFieldEnum).optional(), + tag: z.string().optional(), + tagType: WorkflowTagFilterTagTypeSchema.optional(), + op: z.string().optional(), + valueMode: z.enum(DatasetTagFilterValueModeEnum).optional(), + value: z.unknown().optional() +}); +export type DatasetTagFilterCondition = z.infer; + +export const DatasetTagFilterValueSchema = z.object({ + logic: z.enum(DatasetTagFilterLogicEnum), + conditions: z.array(DatasetTagFilterConditionSchema) +}); +export type DatasetTagFilterValue = z.infer; + +export type WorkflowTagFilterOption = { + tag: string; + tagType: WorkflowTagFilterTagType; + options: string[]; +}; + +type TagFilterOperator = { + labelKey: string; + value: string; + icon?: string; + iconFlip?: boolean; +}; + +const emptyValueOperators: TagFilterOperator[] = [ + { labelKey: 'workflow:tag_filter_op_empty', value: '$empty' }, + { labelKey: 'workflow:tag_filter_op_not_empty', value: '$notEmpty' } +]; +const emptyOps = new Set(emptyValueOperators.map((item) => item.value)); + +const tagFilterOperators: Record = { + [DatasetCollectionTagTypeEnum.number]: [ + { labelKey: 'workflow:tag_filter_op_eq', value: '$eq', icon: 'math/equal' }, + { labelKey: 'workflow:tag_filter_op_ne', value: '$ne', icon: 'math/notEqual' }, + { labelKey: 'workflow:tag_filter_op_gt', value: '$gt', icon: 'math/greater' }, + { labelKey: 'workflow:tag_filter_op_lt', value: '$lt', icon: 'math/greater', iconFlip: true }, + { labelKey: 'workflow:tag_filter_op_gte', value: '$gte', icon: 'math/greaterEqual' }, + { + labelKey: 'workflow:tag_filter_op_lte', + value: '$lte', + icon: 'math/greaterEqual', + iconFlip: true + }, + ...emptyValueOperators + ], + [DatasetCollectionTagTypeEnum.datetime]: [ + { labelKey: 'workflow:tag_filter_op_is', value: '$eq' }, + { labelKey: 'workflow:tag_filter_op_is_not', value: '$ne' }, + { labelKey: 'workflow:tag_filter_op_after', value: '$gt' }, + { labelKey: 'workflow:tag_filter_op_before', value: '$lt' }, + ...emptyValueOperators + ], + [DatasetCollectionTagTypeEnum.array]: [ + { labelKey: 'workflow:tag_filter_op_is', value: '$is' }, + { labelKey: 'workflow:tag_filter_op_is_not', value: '$isNot' }, + { labelKey: 'workflow:tag_filter_op_in', value: '$in' }, + { labelKey: 'workflow:tag_filter_op_not_in', value: '$notIn' }, + ...emptyValueOperators + ] +}; + +const createTimeOperators = tagFilterOperators[DatasetCollectionTagTypeEnum.number].filter( + (item) => item.value === '$gte' || item.value === '$lte' +); +const collectionIdOperators = tagFilterOperators[DatasetCollectionTagTypeEnum.array].filter( + (item) => item.value === '$in' +); + +export const createEmptyTagFilterCondition = (): DatasetTagFilterCondition => ({ + tag: '', + op: '', + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined +}); + +export const createEmptyTagFilterValue = (): DatasetTagFilterValue => ({ + logic: DatasetTagFilterLogicEnum.AND, + conditions: [createEmptyTagFilterCondition()] +}); + +export const isWorkflowTagFilterTagType = ( + tagType?: DatasetCollectionTagType +): tagType is WorkflowTagFilterTagType => WorkflowTagFilterTagTypeSchema.safeParse(tagType).success; + +/** + * 判断节点/表单 value 是否为新版条件行结构。 + * 旧版 collectionFilterMatch 是 JSON 字符串（tags/createTime/collectionIds）。 + */ +export const isDatasetTagFilterValue = (value: unknown): value is DatasetTagFilterValue => { + return DatasetTagFilterValueSchema.safeParse(value).success; +}; + +/** 条件不需要右侧值输入（为空 / 不为空）。 */ +export const isTagFilterOpWithoutValue = (op?: string) => !!op && emptyOps.has(op); + +export const isTagFilterAttributeField = (field?: DatasetTagFilterField | string) => + field === DatasetTagFilterFieldEnum.createTime || + field === DatasetTagFilterFieldEnum.collectionId; + +export const getTagFilterOpsByType = (tagType?: WorkflowTagFilterTagType) => { + return tagType ? tagFilterOperators[tagType] : []; +}; + +/** 文件属性只暴露检索载荷能准确表达的操作符，其余字段按标签类型选择。 */ +export const getTagFilterOpsByCondition = (condition: DatasetTagFilterCondition) => { + if (condition.field === DatasetTagFilterFieldEnum.createTime) { + return createTimeOperators; + } + if (condition.field === DatasetTagFilterFieldEnum.collectionId) { + return collectionIdOperators; + } + return getTagFilterOpsByType(condition.tagType); +}; + +const parseMaybeJson = (value: unknown): unknown => { + if (typeof value !== 'string') return value; + const trimmed = value.trim(); + if (!trimmed) return value; + if ( + !( + (trimmed.startsWith('{') && trimmed.endsWith('}')) || + (trimmed.startsWith('[') && trimmed.endsWith(']')) + ) + ) { + return value; + } + try { + return JSON.parse(trimmed); + } catch { + return value; + } +}; + +type TagConditionObject = Record>; + +export const formatTagOptionKey = (tag: string, tagType: string) => `${tag}\0${tagType}`; + +export const parseTagOptionKey = (value: string) => { + const splitIndex = value.indexOf('\0'); + if (splitIndex < 0) return; + return { + tag: value.slice(0, splitIndex), + tagType: value.slice(splitIndex + 1) as WorkflowTagFilterOption['tagType'] + }; +}; + +/** + * 多知识库标签下拉：各库 number/datetime/array 标签按「名称 + 类型」取交集。 + * 只在部分库出现、或同名不同类型的项不进入下拉。array 的 options 取并集去重。 + */ +export const intersectWorkflowTagOptions = ( + tagLists: Pick[][] +): WorkflowTagFilterOption[] => { + if (tagLists.length === 0) return []; + + const maps = tagLists.map((list) => { + const map = new Map(); + for (const item of list) { + if (!isWorkflowTagFilterTagType(item.tagType)) continue; + const key = formatTagOptionKey(item.tag, item.tagType); + const prev = map.get(key); + const options = Array.from( + new Set([...(prev?.options ?? []), ...(item.options ?? []).filter(Boolean)]) + ); + map.set(key, { tag: item.tag, tagType: item.tagType, options }); + } + return map; + }); + + const [first, ...rest] = maps; + if (!first) return []; + + const result: WorkflowTagFilterOption[] = []; + for (const [key, option] of first) { + if (!rest.every((item) => item.has(key))) continue; + const mergedOptions = new Set(option.options); + for (const item of rest) { + const other = item.get(key); + other?.options.forEach((value) => mergedOptions.add(value)); + } + result.push({ + tag: option.tag, + tagType: option.tagType, + options: Array.from(mergedOptions) + }); + } + return result; +}; + +const isTagCondition = (condition: DatasetTagFilterCondition) => + !condition.field || condition.field === DatasetTagFilterFieldEnum.tag; + +const buildTagConditionObject = ( + condition: DatasetTagFilterCondition +): TagConditionObject | undefined => { + if (!isTagCondition(condition)) return; + const tag = condition.tag?.trim(); + const op = condition.op; + if (!tag || !op) return; + if (isTagFilterOpWithoutValue(op)) { + return { [tag]: { [op]: true } }; + } + if (condition.value === undefined || condition.value === null || condition.value === '') { + return; + } + return { [tag]: { [op]: condition.value } }; +}; + +const toCreateTimeString = (value: unknown): string | undefined => { + if (typeof value === 'number' && Number.isFinite(value)) { + return formatTime2YMDHM(value) || undefined; + } + if (typeof value !== 'string') return; + const trimmed = value.trim(); + if (!trimmed) return; + const parsed = Date.parse(trimmed); + return Number.isNaN(parsed) ? undefined : formatTime2YMDHM(parsed) || undefined; +}; + +const toIdList = (value: unknown): string[] => { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof value === 'string') { + return value + .split(/[\s,，]+/) + .map((item) => item.trim()) + .filter(Boolean); + } + if (typeof value === 'number' && Number.isFinite(value)) { + return [String(value)]; + } + return []; +}; + +/** + * 把条件行编成检索入口 JSON：tags + 可选 createTime / collectionIds。 + * logic 只作用于 tags；文件属性在检索协议中是顶层约束，始终与标签结果求交集。 + * 创建时间多行按更严的 $gte/$lte 合并，Collection ID 多行合并为一个去重白名单。 + * 未填完的行会被丢掉；没有任何有效字段时返回 undefined。 + */ +export const serializeDatasetTagFilterValue = ( + value: DatasetTagFilterValue +): string | undefined => { + const tagConditions = value.conditions + .map(buildTagConditionObject) + .filter((item): item is TagConditionObject => Boolean(item)); + + const createTime: { $gte?: string; $lte?: string } = {}; + for (const condition of value.conditions) { + if (condition.field !== DatasetTagFilterFieldEnum.createTime || !condition.op) continue; + const time = toCreateTimeString(condition.value); + if (!time) continue; + if (condition.op === '$gte') { + if (!createTime.$gte || time > createTime.$gte) createTime.$gte = time; + continue; + } + if (condition.op === '$lte') { + if (!createTime.$lte || time < createTime.$lte) createTime.$lte = time; + } + } + + const idLists = value.conditions + .filter( + (condition) => + condition.field === DatasetTagFilterFieldEnum.collectionId && condition.op === '$in' + ) + .map((condition) => toIdList(condition.value)) + .filter((list) => list.length > 0); + const collectionIds = [...new Set(idLists.flat())]; + + const payload: Record = {}; + if (tagConditions.length > 0) { + const key = value.logic === DatasetTagFilterLogicEnum.OR ? '$or' : '$and'; + payload.tags = { [key]: tagConditions }; + } + if (createTime.$gte || createTime.$lte) { + payload.createTime = createTime; + } + if (collectionIds.length > 0) { + payload.collectionIds = collectionIds; + } + if (Object.keys(payload).length === 0) return undefined; + return JSON.stringify(payload); +}; + +/** 条件行引用值是 `[nodeId, outputKey]`，不复用工作流 utils 以免 dataset ↔ workflow 循环依赖。 */ +const isReferenceTuple = (value: unknown): value is [string, string?] => + Array.isArray(value) && + value.length === 2 && + typeof value[0] === 'string' && + (value[1] === undefined || typeof value[1] === 'string'); + +const resolveConditionValue = ( + condition: DatasetTagFilterCondition, + resolveReference: (value: unknown) => unknown +): DatasetTagFilterCondition => { + if (condition.valueMode !== DatasetTagFilterValueModeEnum.reference) return condition; + if (!isReferenceTuple(condition.value)) return { ...condition, value: undefined }; + return { ...condition, value: resolveReference(condition.value) }; +}; + +/** + * 运行时把 collectionFilterMatch 统一成检索 JSON 字符串。 + * 整段引用、旧 JSON 字符串原样（或解析后若是条件行再序列化）；条件行会先解析行内引用。 + */ +export const formatCollectionFilterMatchParam = ({ + value, + resolveReference = () => undefined +}: { + value: unknown; + resolveReference?: (value: unknown) => unknown; +}): string | undefined => { + if (value === undefined || value === null || value === '') return undefined; + + const parsed = parseMaybeJson(value); + const structured = isDatasetTagFilterValue(parsed) ? parsed : undefined; + + if (structured) { + const resolved: DatasetTagFilterValue = { + logic: structured.logic, + conditions: structured.conditions.map((condition) => + resolveConditionValue(condition, resolveReference) + ) + }; + return serializeDatasetTagFilterValue(resolved); + } + + if (typeof value === 'string') return value; + if (typeof value === 'object') return JSON.stringify(value); + return undefined; +}; + +/** + * 已选库变化后，丢掉不在新交集里的标签行。 + * 文件属性和尚未选择字段的空行都保留，否则「添加过滤条件」会被立刻清掉。 + */ +export const pruneTagFilterConditions = ( + value: DatasetTagFilterValue, + options: WorkflowTagFilterOption[] +): DatasetTagFilterValue => { + const valid = new Set(options.map((item) => formatTagOptionKey(item.tag, item.tagType))); + const conditions = value.conditions.filter((condition) => { + if (isTagFilterAttributeField(condition.field) || !condition.tag) return true; + return !!condition.tagType && valid.has(formatTagOptionKey(condition.tag, condition.tagType)); + }); + return { + ...value, + conditions: conditions.length > 0 ? conditions : [createEmptyTagFilterCondition()] + }; +}; diff --git a/packages/global/core/workflow/constants.ts b/packages/global/core/workflow/constants.ts index 7686320e04e2..1c54c52789c3 100644 --- a/packages/global/core/workflow/constants.ts +++ b/packages/global/core/workflow/constants.ts @@ -202,6 +202,7 @@ export enum NodeInputKeyEnum { datasetSearchExtensionModel = 'datasetSearchExtensionModel', datasetSearchExtensionBg = 'datasetSearchExtensionBg', datasetSearchInput = 'datasetSearchInput', + collectionFilterVersion = 'collectionFilterVersion', collectionFilterMatch = 'collectionFilterMatch', authTmbId = 'authTmbId', datasetDeepSearch = 'datasetDeepSearch', diff --git a/packages/global/core/workflow/node/constant.ts b/packages/global/core/workflow/node/constant.ts index d10f6284cf87..a50311d74236 100644 --- a/packages/global/core/workflow/node/constant.ts +++ b/packages/global/core/workflow/node/constant.ts @@ -37,6 +37,8 @@ export enum FlowNodeInputTypeEnum { timeRangeSelect = 'timeRangeSelect', password = 'password', + datasetTagFilter = 'datasetTagFilter', + agentGenerated = 'agentGenerated' // for compatibility for >= v4.16.0 } export const FlowNodeInputMap: Record< @@ -117,6 +119,9 @@ export const FlowNodeInputMap: Record< [FlowNodeInputTypeEnum.password]: { icon: 'core/workflow/inputType/password' }, + [FlowNodeInputTypeEnum.datasetTagFilter]: { + icon: 'core/workflow/inputType/input' + }, [FlowNodeInputTypeEnum.agentGenerated]: { icon: 'core/workflow/inputType/agentGenerated' } diff --git a/packages/global/core/workflow/template/input.ts b/packages/global/core/workflow/template/input.ts index 54024f44e795..86334b4d52db 100644 --- a/packages/global/core/workflow/template/input.ts +++ b/packages/global/core/workflow/template/input.ts @@ -4,6 +4,15 @@ import { WorkflowIOValueTypeEnum } from '../constants'; import { chatNodeSystemPromptTip, systemPromptTip } from './tip'; import { type FlowNodeInputItemType } from '../type/io'; import { i18nT } from '../../../common/i18n/utils'; +import { DatasetTagFilterVersionEnum } from '../../dataset/workflowTagFilter'; + +export const Input_Template_Dataset_Tag_Filter_Version: FlowNodeInputItemType = { + key: NodeInputKeyEnum.collectionFilterVersion, + renderTypeList: [FlowNodeInputTypeEnum.hidden], + label: '', + valueType: WorkflowIOValueTypeEnum.string, + value: DatasetTagFilterVersionEnum.structured +}; export const Input_Template_History: FlowNodeInputItemType = { key: NodeInputKeyEnum.history, diff --git a/packages/global/core/workflow/template/system/agent/index.ts b/packages/global/core/workflow/template/system/agent/index.ts index 6e2c7e183fac..57af857f7995 100644 --- a/packages/global/core/workflow/template/system/agent/index.ts +++ b/packages/global/core/workflow/template/system/agent/index.ts @@ -12,13 +12,14 @@ import { NodeInputKeyEnum } from '../../../constants'; import { + Input_Template_Dataset_Tag_Filter_Version, + Input_Template_File_Link, Input_Template_SettingAiModel, Input_Template_System_Prompt, Input_Template_UserChatInput } from '../../input'; import { chatNodeSystemPromptTip, systemPromptTip } from '../../tip'; import { i18nT } from '../../../../../common/i18n/utils'; -import { Input_Template_File_Link } from '../../input'; import { Output_Template_Error_Message } from '../../output'; import { DatasetSearchModeEnum } from '../../../../dataset/constants'; @@ -246,6 +247,15 @@ export const AgentNode: FlowNodeTemplateType = { label: '', valueType: WorkflowIOValueTypeEnum.boolean, value: false + }, + Input_Template_Dataset_Tag_Filter_Version, + { + key: NodeInputKeyEnum.collectionFilterMatch, + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + label: i18nT('workflow:tag_filter'), + valueType: WorkflowIOValueTypeEnum.string, + isPro: true, + description: i18nT('workflow:tag_filter_description') } ], outputs: [ diff --git a/packages/global/core/workflow/template/system/datasetSearch.ts b/packages/global/core/workflow/template/system/datasetSearch.ts index be6ca3a01c7c..65c14abc6e85 100644 --- a/packages/global/core/workflow/template/system/datasetSearch.ts +++ b/packages/global/core/workflow/template/system/datasetSearch.ts @@ -12,7 +12,7 @@ import { NodeOutputKeyEnum, FlowNodeTemplateTypeEnum } from '../../constants'; -import { Input_Template_UserChatInput } from '../input'; +import { Input_Template_Dataset_Tag_Filter_Version, Input_Template_UserChatInput } from '../input'; import { DatasetSearchModeEnum } from '../../../dataset/constants'; import { i18nT } from '../../../../common/i18n/utils'; import { Output_Template_Error_Message } from '../output'; @@ -34,7 +34,7 @@ export const DatasetSearchModule: FlowNodeTemplateType = { isTool: true, catchError: false, courseUrl: '/guide/build/workflow/nodes/dataset_search', - version: '4.9.2', + version: '4.17.0', inputs: [ { key: NodeInputKeyEnum.datasetSelectList, @@ -131,14 +131,14 @@ export const DatasetSearchModule: FlowNodeTemplateType = { valueType: WorkflowIOValueTypeEnum.arrayString, toolDescription: i18nT('workflow:content_to_search') }, + Input_Template_Dataset_Tag_Filter_Version, { key: NodeInputKeyEnum.collectionFilterMatch, - renderTypeList: [FlowNodeInputTypeEnum.textarea, FlowNodeInputTypeEnum.reference], - label: i18nT('workflow:collection_metadata_filter'), - + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + label: i18nT('workflow:tag_filter'), valueType: WorkflowIOValueTypeEnum.string, isPro: true, - description: i18nT('workflow:filter_description') + description: i18nT('workflow:tag_filter_description') } ], outputs: [ diff --git a/packages/global/openapi/core/dataset/collection/api.ts b/packages/global/openapi/core/dataset/collection/api.ts index 7345d3aac959..e05e008a2e86 100644 --- a/packages/global/openapi/core/dataset/collection/api.ts +++ b/packages/global/openapi/core/dataset/collection/api.ts @@ -10,6 +10,8 @@ import { TrainingModeEnum } from '../../../../core/dataset/constants'; import { + CollectionTagFilterItemSchema, + CollectionTagLabelSchema, CollectionTrainingStatusSchema, DatasetCollectionItemSchema, DatasetCollectionSchema @@ -23,26 +25,15 @@ import { transformOptionalChatAuthTargetInput } from '../../chat/api'; -// ============= Scroll Collections ============= -/** - * @deprecated Use ListCollectionV2BodySchema and /core/dataset/collection/listV2 instead. - */ -export const ScrollCollectionsBodySchema = z.object({ - datasetId: z.string(), - parentId: z.string().nullable().optional().default(null), - searchText: z.string().optional().default(''), - selectFolder: z.boolean().optional().default(false), - filterTags: z.array(z.string()).optional().default([]), - simple: z.boolean().optional().default(false) -}); -export type ScrollCollectionsBodyType = z.infer; - // ============= Update Collection ============= export const UpdateDatasetCollectionBodySchema = z.object({ id: ObjectIdSchema.optional().describe('集合ID，与 datasetId+externalFileId 二选一'), parentId: ParentIdSchema.describe('父级目录ID'), name: z.string().optional().describe('集合名称'), - tags: z.array(z.string()).optional().describe('标签列表（标签名称，非ID）'), + tags: z + .array(CollectionTagLabelSchema) + .optional() + .describe('标签列表（支持 String 旧格式 或 { tag, value } 新格式）'), forbid: z.boolean().optional().describe('是否禁用'), createTime: z.coerce.date().optional().describe('创建时间'), datasetId: z.string().optional().describe('数据集ID，配合 externalFileId 使用'), @@ -155,7 +146,9 @@ export const ListCollectionV2BodySchema = PaginationSchema.extend({ parentId: z.string().nullable().optional().default(null).meta({ description: '父级目录 ID' }), searchText: z.string().max(100).optional().default('').meta({ description: '搜索文本' }), selectFolder: z.boolean().optional().default(false).meta({ description: '只返回文件夹' }), - filterTags: z.array(z.string()).optional().default([]).meta({ description: '过滤标签' }), + tagFilters: z.array(CollectionTagFilterItemSchema).optional().default([]).meta({ + description: '按标签值过滤。同一标签多值为 OR，不同标签为 AND' + }), simple: z.boolean().optional().default(false).meta({ description: '简单模式（不统计数量）' }) }); export type ListCollectionV2BodyType = z.infer; @@ -172,7 +165,7 @@ export const DatasetCollectionsListItemSchema = z updateTime: DatasetCollectionSchema.shape.updateTime, forbid: DatasetCollectionSchema.shape.forbid, trainingType: DatasetCollectionSchema.shape.trainingType, - tags: z.array(z.string()).optional().meta({ description: '标签' }), + tags: DatasetCollectionItemSchema.shape.tags, externalFileId: z.string().optional().meta({ description: '外部文件 ID' }), @@ -256,3 +249,30 @@ export const SyncCollectionResponseSchema = z.enum(DatasetCollectionSyncResultEn description: '同步结果' }); export type SyncCollectionResponseType = z.infer; + +/* ============================================================================ + * API: 获取知识库标签筛选项 + * Route: GET /api/core/dataset/collection/tagFilterOptions + * Method: GET + * Description: 获取知识库下当前已被文件使用的标签值，供详情页双栏筛选使用。标签名和类型由 getAllTags 提供。 + * Tags: ['Dataset'] + * ============================================================================ */ +export const GetTagFilterOptionsQuerySchema = z.object({ + datasetId: ObjectIdSchema.meta({ + example: '68ad85a7463006c963799a05', + description: '数据集 ID' + }) +}); + +const TagFilterOptionItemSchema = z.object({ + tagId: z.string().meta({ example: '68ad85a7463006c963799a05', description: '标签 ID' }), + values: z + .array(z.union([z.string(), z.number()])) + .meta({ example: ['PRD'], description: '当前已被文件使用的标签值' }) +}); +export type TagFilterOptionItemType = z.infer; + +export const GetTagFilterOptionsResponseSchema = z.object({ + list: z.array(TagFilterOptionItemSchema).meta({ description: '已用标签值列表' }) +}); +export type GetTagFilterOptionsResponseType = z.infer; diff --git a/packages/global/openapi/core/dataset/collection/createApi.ts b/packages/global/openapi/core/dataset/collection/createApi.ts index a91175d7a4d8..74f8b8317971 100644 --- a/packages/global/openapi/core/dataset/collection/createApi.ts +++ b/packages/global/openapi/core/dataset/collection/createApi.ts @@ -1,5 +1,5 @@ import z from 'zod'; -import { ChunkSettingsSchema } from '../../../../core/dataset/type'; +import { ChunkSettingsSchema, CollectionTagLabelSchema } from '../../../../core/dataset/type'; import { DatasetCollectionTypeEnum } from '../../../../core/dataset/constants'; import { ParentIdSchema } from '../../../../common/parentFolder/type'; import { ObjectIdSchema } from '../../../../common/type/mongo'; @@ -16,10 +16,15 @@ const DatasetCollectionStoreDataSchema = ChunkSettingsSchema.extend({ customPdfParse: z.boolean().optional().meta({ description: '自定义 PDF 解析' }) }); +const CollectionTagsInputSchema = z.array(CollectionTagLabelSchema).optional().meta({ + description: + '标签列表。字符串元素为旧格式标签名（按名称归并到 default_tag array 标签）；对象元素为 { tag: 标签名, value: 标签值 }，按标签名解析' +}); + // API 创建集合通用基础 Schema export const ApiCreateCollectionBaseSchema = DatasetCollectionStoreDataSchema.extend({ datasetId: z.string().meta({ description: '数据集 ID' }), - tags: z.array(z.string()).optional().meta({ description: '标签列表' }) + tags: CollectionTagsInputSchema }); export type ApiCreateDatasetCollectionParams = z.infer; @@ -50,7 +55,7 @@ export const CreateCollectionBodySchema = z.object({ type: z .enum([DatasetCollectionTypeEnum.folder, DatasetCollectionTypeEnum.virtual]) .meta({ description: '集合类型（folder: 文件夹，virtual: 手动集合）' }), - tags: z.array(z.string()).optional().meta({ description: '标签列表' }) + tags: CollectionTagsInputSchema }); export type CreateCollectionBodyType = z.infer; @@ -154,7 +159,8 @@ export type ImageCreateDatasetCollectionParams = z.infer; // handler 内 parse 用 @@ -179,7 +185,8 @@ export const CreateImageCollectionMultipartSchema = z.object({ // handler 内 parse 用 export const CreateBackupCollectionFormSchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), - parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }) + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + tags: CollectionTagsInputSchema }); export type CreateBackupCollectionFormType = z.infer; @@ -201,7 +208,8 @@ export const CreateBackupCollectionMultipartSchema = z.object({ // handler 内 parse 用 export const CreateTemplateCollectionFormSchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), - parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }) + parentId: ParentIdSchema.optional().meta({ description: '父级目录 ID' }), + tags: CollectionTagsInputSchema }); export type CreateTemplateCollectionFormType = z.infer; diff --git a/packages/global/openapi/core/dataset/collection/index.ts b/packages/global/openapi/core/dataset/collection/index.ts index 00ab9ecd4051..6aec7dfa9748 100644 --- a/packages/global/openapi/core/dataset/collection/index.ts +++ b/packages/global/openapi/core/dataset/collection/index.ts @@ -10,8 +10,9 @@ import { GetCollectionTrainingDetailQuerySchema, GetCollectionTrainingDetailResponseSchema, ListCollectionV2BodySchema, + GetTagFilterOptionsQuerySchema, + GetTagFilterOptionsResponseSchema, ReadCollectionSourceBodyRawSchema, - ScrollCollectionsBodySchema, SyncCollectionBodySchema, UpdateDatasetCollectionBodySchema } from './api'; @@ -75,22 +76,22 @@ export const DatasetCollectionPath: OpenAPIPath = { } } }, - '/core/dataset/collection/scrollList': { - post: { - summary: '获取数据集集合列表（滚动分页）', - description: '已废弃：获取数据集集合列表（滚动分页）。请改用 /core/dataset/collection/listV2', - deprecated: true, + '/core/dataset/collection/tagFilterOptions': { + get: { + summary: '获取知识库标签筛选项', + description: '获取知识库下当前已被文件使用的标签值', tags: [DevApiTagsMap.datasetCollection], - requestBody: { - content: { - 'application/json': { - schema: ScrollCollectionsBodySchema - } - } + requestParams: { + query: GetTagFilterOptionsQuerySchema }, responses: { 200: { - description: '成功返回集合列表' + description: '成功返回标签及已用值列表', + content: { + 'application/json': { + schema: GetTagFilterOptionsResponseSchema + } + } } } } diff --git a/packages/global/openapi/core/dataset/collection/tagApi.ts b/packages/global/openapi/core/dataset/collection/tagApi.ts index d960826f61e1..98342dd13bf1 100644 --- a/packages/global/openapi/core/dataset/collection/tagApi.ts +++ b/packages/global/openapi/core/dataset/collection/tagApi.ts @@ -1,4 +1,19 @@ import z from 'zod'; +import { + DatasetCollectionTagTypeEnum, + CollectionTagValueSchema, + CollectionTagValueFieldSchema, + DatasetCollectionTagOptionsSchema +} from '../../../../core/dataset/type'; + +export const BatchCollectionTagModeEnum = { + add: 'add', + remove: 'remove' +} as const; +const BatchCollectionTagModeSchema = z.enum([ + BatchCollectionTagModeEnum.add, + BatchCollectionTagModeEnum.remove +]); /* ============================================================================ * API: 创建集合标签 @@ -6,7 +21,10 @@ import z from 'zod'; * ============================================================================ */ export const CreateDatasetCollectionTagBodySchema = z.object({ datasetId: z.string().meta({ description: '数据集 ID' }), - tag: z.string().meta({ description: '标签名称' }) + tag: z.string().trim().min(1).meta({ description: '标签名称' }), + tagType: z.enum(DatasetCollectionTagTypeEnum).optional().meta({ + description: '标签类型：string(默认)/number/datetime/array' + }) }); export type CreateDatasetCollectionTagParams = z.infer; @@ -20,9 +38,9 @@ export const AddTagsToCollectionsBodySchema = z.object({ .meta({ description: '来源集合 ID 列表（用于复制标签）' }), collectionIds: z.array(z.string()).meta({ description: '目标集合 ID 列表' }), datasetId: z.string().meta({ description: '数据集 ID' }), - tag: z.string().meta({ description: '标签名称' }) + tag: z.string().trim().meta({ description: '标签名称' }), + value: z.string().optional().meta({ description: '标签值（仅 string 类型标签支持）' }) }); -export type AddTagsToCollectionsParams = z.infer; /* ============================================================================ * API: 更新集合标签 @@ -31,6 +49,75 @@ export type AddTagsToCollectionsParams = z.infer; + +/* ============================================================================ + * API: 删除集合标签 + * Route: DELETE /proApi/core/dataset/tag/delete + * ============================================================================ */ +export const DeleteDatasetCollectionTagQuerySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + id: z.string().meta({ description: '标签 ID' }) +}); + +/* ============================================================================ + * API: 获取知识库全部标签 + * Route: GET /proApi/core/dataset/tag/getAllTags + * ============================================================================ */ +export const GetAllDatasetTagsQuerySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }) +}); + +/* ============================================================================ + * API: 批量 Upsert 标签 + * Route: POST /proApi/core/dataset/tag/batchUpsert + * ============================================================================ */ +const BatchUpsertTagItemSchema = z.object({ + tag: z.string().trim().min(1).meta({ description: '标签名称' }), + tagType: z.enum(DatasetCollectionTagTypeEnum).optional().meta({ description: '标签类型' }) +}); +export const BatchUpsertTagsBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + tags: z.array(BatchUpsertTagItemSchema).min(1).meta({ description: '标签列表' }) +}); + +/* ============================================================================ + * API: 设置单个 Collection 标签值 + * Route: POST /proApi/core/dataset/tag/setCollectionTags + * ============================================================================ */ +export const SetCollectionTagsBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + collectionId: z.string().meta({ description: '集合 ID' }), + tags: z.array(CollectionTagValueSchema).meta({ description: '标签值列表' }) +}); +export type SetCollectionTagsParams = z.infer; + +/* ============================================================================ + * API: 批量设置 Collection 标签值 + * Route: POST /proApi/core/dataset/tag/batchSetCollectionTags + * ============================================================================ */ +const BatchSetCollectionTagItemSchema = z.object({ + tagId: z.string().meta({ description: '引用 dataset_collection_tags_v2._id' }), + value: CollectionTagValueFieldSchema.optional().meta({ + description: 'add：必填标签值。remove：省略则移除整个标签；array 传入 string[] 时只移除这些选项' + }), + append: z.boolean().optional().meta({ + description: '仅 number 类型的 add 生效：true 时把值累加到已有数字，缺省或 false 为覆盖' + }) +}); +export type BatchSetCollectionTagItem = z.infer; + +export const BatchSetCollectionTagsBodySchema = z.object({ + datasetId: z.string().meta({ description: '数据集 ID' }), + collectionIds: z.array(z.string()).min(1).meta({ description: '集合 ID 列表' }), + mode: BatchCollectionTagModeSchema.default(BatchCollectionTagModeEnum.add).meta({ + description: 'add：批量添加/覆盖标签；remove：批量移除标签或选项值' + }), + tags: z.array(BatchSetCollectionTagItemSchema).meta({ description: '标签操作列表' }) +}); +export type BatchSetCollectionTagsParams = z.infer; diff --git a/packages/global/openapi/core/dataset/index.ts b/packages/global/openapi/core/dataset/index.ts index 97a8811d4e84..4ae453975bf6 100644 --- a/packages/global/openapi/core/dataset/index.ts +++ b/packages/global/openapi/core/dataset/index.ts @@ -4,6 +4,7 @@ import { SystemOpenApiTagMap } from '../../tag'; import { DatasetDataPath } from './data'; import { DatasetCollectionPath } from './collection'; import { ApiDatasetPath } from './apiDataset'; +import { DatasetTagPath } from './tag'; import { DatasetFilePath } from './file'; import { DatasetTrainingPath } from './training'; import { DatasetSynonymPath } from './synonym'; @@ -336,6 +337,7 @@ export const DatasetPath: OpenAPIPath = { ...DatasetCollectionPath, ...DatasetDataPath, ...ApiDatasetPath, + ...DatasetTagPath, ...DatasetFilePath, ...DatasetTrainingPath, ...DatasetSynonymPath diff --git a/packages/global/openapi/core/dataset/tag/index.ts b/packages/global/openapi/core/dataset/tag/index.ts new file mode 100644 index 000000000000..ddddda9a522e --- /dev/null +++ b/packages/global/openapi/core/dataset/tag/index.ts @@ -0,0 +1,140 @@ +import type { OpenAPIPath } from '../../../type'; +import { DevApiTagsMap } from '../../../tag'; +import { + BatchSetCollectionTagsBodySchema, + BatchUpsertTagsBodySchema, + CreateDatasetCollectionTagBodySchema, + DeleteDatasetCollectionTagQuerySchema, + GetAllDatasetTagsQuerySchema, + SetCollectionTagsBodySchema, + UpdateDatasetCollectionTagBodySchema +} from '../collection/tagApi'; + +export const DatasetTagPath: OpenAPIPath = { + '/proApi/core/dataset/tag/create': { + post: { + summary: '创建标签', + description: '在指定知识库下创建一个新标签', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: CreateDatasetCollectionTagBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回新创建的标签信息' + } + } + } + }, + '/proApi/core/dataset/tag/update': { + post: { + summary: '更新标签', + description: '更新指定标签的名称', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: UpdateDatasetCollectionTagBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回更新后的标签信息' + } + } + } + }, + '/proApi/core/dataset/tag/delete': { + delete: { + summary: '删除标签', + description: '根据标签 ID 删除指定知识库下的标签', + tags: [DevApiTagsMap.datasetTag], + requestParams: { + query: DeleteDatasetCollectionTagQuerySchema + }, + responses: { + 200: { + description: '成功删除标签' + } + } + } + }, + '/proApi/core/dataset/tag/getAllTags': { + get: { + summary: '获取全部标签', + description: '获取指定知识库下的全部标签', + tags: [DevApiTagsMap.datasetTag], + requestParams: { + query: GetAllDatasetTagsQuerySchema + }, + responses: { + 200: { + description: '成功返回全部标签列表' + } + } + } + }, + '/proApi/core/dataset/tag/batchUpsert': { + post: { + summary: '批量管理标签', + description: + '全量创建标签。已存在的跳过，不存在的创建，缺少的删除；修改类型需要先调用delete接口', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: BatchUpsertTagsBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回批量操作结果' + } + } + } + }, + '/proApi/core/dataset/tag/setCollectionTags': { + post: { + summary: '设置集合标签值', + description: '为单个集合设置标签值', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: SetCollectionTagsBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回操作结果' + } + } + } + }, + '/proApi/core/dataset/tag/batchSetCollectionTags': { + post: { + summary: '批量设置集合标签值', + description: '为多个集合批量添加、覆盖或移除标签值；number 类型支持累加', + tags: [DevApiTagsMap.datasetTag], + requestBody: { + content: { + 'application/json': { + schema: BatchSetCollectionTagsBodySchema + } + } + }, + responses: { + 200: { + description: '成功返回批量操作结果' + } + } + } + } +}; diff --git a/packages/global/openapi/tag.ts b/packages/global/openapi/tag.ts index 673c09971892..34a84849d6c2 100644 --- a/packages/global/openapi/tag.ts +++ b/packages/global/openapi/tag.ts @@ -53,6 +53,7 @@ export const DevApiTagsMap = { datasetData: '数据管理', datasetTraining: '训练管理', datasetApiDataset: 'API 数据集管理', + datasetTag: '标签管理', datasetFile: '知识库文件管理', datasetSynonym: '同义词管理', diff --git a/packages/global/test/core/app/formEdit/utils.test.ts b/packages/global/test/core/app/formEdit/utils.test.ts index 916fbca411f1..162918e35895 100644 --- a/packages/global/test/core/app/formEdit/utils.test.ts +++ b/packages/global/test/core/app/formEdit/utils.test.ts @@ -1026,7 +1026,8 @@ describe('agent generated tool input helpers', () => { FlowNodeInputTypeEnum.hidden, FlowNodeInputTypeEnum.selectDataset, FlowNodeInputTypeEnum.selectLLMModel, - FlowNodeInputTypeEnum.customVariable + FlowNodeInputTypeEnum.customVariable, + FlowNodeInputTypeEnum.datasetTagFilter ])('should hide unsupported parent tool configuration type %s', (renderType) => { expect(canInputBeConfiguredAsToolParam(createMockInput({ renderTypeList: [renderType] }))).toBe( false diff --git a/packages/global/test/core/app/utils.test.ts b/packages/global/test/core/app/utils.test.ts index fbdd8c909dc3..557cf1b9357c 100644 --- a/packages/global/test/core/app/utils.test.ts +++ b/packages/global/test/core/app/utils.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { getDefaultAppForm, getAppType, formatToolError } from '@fastgpt/global/core/app/utils'; import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; describe('getDefaultAppForm', () => { it('should return default app form with correct structure', () => { @@ -34,6 +35,7 @@ describe('getDefaultAppForm', () => { rerankWeight: 0.5, datasetSearchUsingExtensionQuery: true, datasetSearchExtensionBg: '', + [NodeInputKeyEnum.collectionFilterVersion]: 'structured', authTmbId: false }); }); diff --git a/packages/global/test/core/dataset/tagUtils.test.ts b/packages/global/test/core/dataset/tagUtils.test.ts new file mode 100644 index 000000000000..a3f473875c1c --- /dev/null +++ b/packages/global/test/core/dataset/tagUtils.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { + isCollectionTagValue, + isUsableCollectionTagFilterValue, + sortCollectionTagValues +} from '@fastgpt/global/core/dataset/tagUtils'; + +describe('dataset tag utilities', () => { + it('validates stored/filter values and sorts display values', () => { + expect(['PRD', 0].every(isUsableCollectionTagFilterValue)).toBe(true); + expect(['', Number.NaN, [], null].some(isUsableCollectionTagFilterValue)).toBe(false); + expect(isCollectionTagValue({ tagId: 'tag-1', value: ['A'] })).toBe(true); + expect(isCollectionTagValue('legacy-id')).toBe(false); + expect(sortCollectionTagValues([2, 10, 0])).toEqual([0, 2, 10]); + expect(sortCollectionTagValues(['spec', 'PRD'])).toEqual(['PRD', 'spec']); + }); +}); diff --git a/packages/global/test/core/dataset/workflowTagFilter.test.ts b/packages/global/test/core/dataset/workflowTagFilter.test.ts new file mode 100644 index 000000000000..4dfaa6fca552 --- /dev/null +++ b/packages/global/test/core/dataset/workflowTagFilter.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from 'vitest'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { + createEmptyTagFilterValue, + DatasetTagFilterFieldEnum, + DatasetTagFilterLogicEnum, + DatasetTagFilterValueModeEnum, + DatasetTagFilterVersionEnum, + formatCollectionFilterMatchParam, + getTagFilterOpsByCondition, + intersectWorkflowTagOptions, + isDatasetTagFilterValue, + normalizeLegacyDatasetTagFilterValue, + pruneTagFilterConditions, + resolveDatasetTagFilterVersion, + serializeDatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; + +describe('dataset tag filter version', () => { + it('uses structured only when explicitly marked and does not infer from the filter value', () => { + expect( + resolveDatasetTagFilterVersion({ + version: DatasetTagFilterVersionEnum.structured, + filterValue: 'legacy config' + }) + ).toBe(DatasetTagFilterVersionEnum.structured); + expect( + resolveDatasetTagFilterVersion({ version: undefined, filterValue: 'legacy config' }) + ).toBe(DatasetTagFilterVersionEnum.legacy); + expect(resolveDatasetTagFilterVersion({ version: undefined, filterValue: undefined })).toBe( + DatasetTagFilterVersionEnum.structured + ); + expect(resolveDatasetTagFilterVersion({ version: 'invalid', filterValue: undefined })).toBe( + DatasetTagFilterVersionEnum.legacy + ); + expect(normalizeLegacyDatasetTagFilterValue('{"tags":{}}')).toBe('{"tags":{}}'); + expect(normalizeLegacyDatasetTagFilterValue(createEmptyTagFilterValue())).toBe(''); + }); +}); + +describe('dataset tag filter options', () => { + it('keeps supported tags shared by every dataset regardless of migration metadata', () => { + expect( + intersectWorkflowTagOptions([ + [ + { + tag: 'default_tag', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['legacy'], + fromMigration: true + }, + { tag: 'status', tagType: DatasetCollectionTagTypeEnum.array, options: ['open'] }, + { tag: 'title', tagType: DatasetCollectionTagTypeEnum.string, options: [] } + ], + [ + { + tag: 'default_tag', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['archived'], + fromMigration: true + }, + { + tag: 'status', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['closed', 'open'] + }, + { tag: 'only-second', tagType: DatasetCollectionTagTypeEnum.number, options: [] } + ] + ]) + ).toEqual([ + { + tag: 'default_tag', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['legacy', 'archived'] + }, + { tag: 'status', tagType: DatasetCollectionTagTypeEnum.array, options: ['open', 'closed'] } + ]); + }); + + it('limits file attributes to operations supported by the search payload', () => { + expect( + getTagFilterOpsByCondition({ field: DatasetTagFilterFieldEnum.createTime }).map( + (item) => item.value + ) + ).toEqual(['$gte', '$lte']); + expect( + getTagFilterOpsByCondition({ field: DatasetTagFilterFieldEnum.collectionId }).map( + (item) => item.value + ) + ).toEqual(['$in']); + }); +}); + +describe('serializeDatasetTagFilterValue', () => { + it('serializes tag logic and top-level file constraints while dropping incomplete rows', () => { + const result = serializeDatasetTagFilterValue({ + logic: DatasetTagFilterLogicEnum.OR, + conditions: [ + { + tag: 'status', + tagType: DatasetCollectionTagTypeEnum.array, + op: '$in', + value: ['open'] + }, + { + field: DatasetTagFilterFieldEnum.createTime, + op: '$gte', + value: '2026-03-01T08:00:00' + }, + { + field: DatasetTagFilterFieldEnum.collectionId, + op: '$in', + value: 'id-1, id-2, id-1' + }, + { tag: 'incomplete' } + ] + }); + + expect(JSON.parse(result ?? '')).toEqual({ + tags: { $or: [{ status: { $in: ['open'] } }] }, + createTime: { $gte: expect.any(String) }, + collectionIds: ['id-1', 'id-2'] + }); + expect(serializeDatasetTagFilterValue(createEmptyTagFilterValue())).toBeUndefined(); + }); +}); + +describe('formatCollectionFilterMatchParam', () => { + it('resolves structured row references and preserves legacy strings without converting them', () => { + expect( + formatCollectionFilterMatchParam({ + value: { + logic: DatasetTagFilterLogicEnum.AND, + conditions: [ + { + tag: 'price', + tagType: DatasetCollectionTagTypeEnum.number, + op: '$gte', + valueMode: DatasetTagFilterValueModeEnum.reference, + value: ['node', 'price'] + } + ] + }, + resolveReference: () => 10 + }) + ).toBe(JSON.stringify({ tags: { $and: [{ price: { $gte: 10 } }] } })); + + const legacy = '{"tags":{"$and":["legacy"]}}'; + expect(formatCollectionFilterMatchParam({ value: legacy })).toBe(legacy); + expect(formatCollectionFilterMatchParam({ value: undefined })).toBeUndefined(); + }); +}); + +describe('pruneTagFilterConditions', () => { + it('removes unavailable tags but keeps attributes and a usable empty row', () => { + const result = pruneTagFilterConditions( + { + logic: DatasetTagFilterLogicEnum.AND, + conditions: [ + { tag: 'gone', tagType: DatasetCollectionTagTypeEnum.number, op: '$eq', value: 1 }, + { field: DatasetTagFilterFieldEnum.createTime, op: '$gte', value: 1 } + ] + }, + [] + ); + + expect(result.conditions).toEqual([ + { field: DatasetTagFilterFieldEnum.createTime, op: '$gte', value: 1 } + ]); + expect(isDatasetTagFilterValue(result)).toBe(true); + }); +}); diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index 576327238cee..e9788e5a62a0 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -292,6 +292,7 @@ export async function createOneCollection({ session, ...props }: CreateOneCollec apiFileParentId } = props; + // Resolve tags: string names → ObjectId, {tag, value} → {tagId, value} const collectionTags = await createOrGetCollectionTags({ tags, teamId, diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 40228fe7c20c..1383b49374d0 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -43,7 +43,7 @@ const DatasetCollectionSchema = new Schema({ required: true }, tags: { - type: [String], + type: [], default: [] }, @@ -110,6 +110,10 @@ defineIndex(DatasetCollectionSchema, { defineIndex(DatasetCollectionSchema, { key: { teamId: 1, datasetId: 1, tags: 1 } }); +// New format tags.tagId filter +defineIndex(DatasetCollectionSchema, { + key: { teamId: 1, datasetId: 1, 'tags.tagId': 1 } +}); // create time filter defineIndex(DatasetCollectionSchema, { key: { teamId: 1, datasetId: 1, createTime: 1 } diff --git a/packages/service/core/dataset/collection/tagFilter.ts b/packages/service/core/dataset/collection/tagFilter.ts new file mode 100644 index 000000000000..70595ed5d0db --- /dev/null +++ b/packages/service/core/dataset/collection/tagFilter.ts @@ -0,0 +1,89 @@ +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; +import { + isUsableCollectionTagFilterValue, + sortCollectionTagValues +} from '@fastgpt/global/core/dataset/tagUtils'; +import type { TagFilterOptionItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { Types } from '../../../common/mongo'; +import { MongoDatasetCollection } from './schema'; + +/** + * 按 tagFilters 组装 Collection 列表的标签过滤条件。 + * 同一标签多值为 OR（value $in），不同标签为 AND。 + */ +export const buildCollectionListTagMatch = (tagFilters: CollectionTagFilterItem[] = []) => { + if (tagFilters.length === 0) return {}; + + const conditions = tagFilters.map((filter) => ({ + tags: { + $elemMatch: { + tagId: filter.tagId, + value: { $in: filter.values } + } + } + })); + + if (conditions.length === 1) return conditions[0]; + return { $and: conditions }; +}; + +type UsedTagValueGroup = { + _id: string; + values: unknown[]; +}; + +/** + * 聚合当前知识库 Collection 上已被使用的标签值，不含标签定义。 + * Mongo unwind + $addToSet 按 tagId 去重后再拉回 Node，空串和非有限数字在本地丢掉。 + */ +export const getDatasetTagFilterOptions = async ({ + teamId, + datasetId +}: { + teamId: string; + datasetId: string; +}): Promise => { + const grouped = await MongoDatasetCollection.aggregate([ + { + $match: { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(datasetId) + } + }, + { $project: { tags: 1 } }, + { $unwind: '$tags' }, + { + $match: { + 'tags.tagId': { $exists: true, $nin: [null, ''] }, + 'tags.value': { $exists: true } + } + }, + { + $project: { + tagId: { $toString: '$tags.tagId' }, + values: { + $cond: { + if: { $isArray: '$tags.value' }, + then: '$tags.value', + else: ['$tags.value'] + } + } + } + }, + { $unwind: '$values' }, + { + $group: { + _id: '$tagId', + values: { $addToSet: '$values' } + } + } + ]); + + return grouped + .map((item) => ({ + tagId: String(item._id), + values: sortCollectionTagValues(item.values.filter(isUsableCollectionTagFilterValue)) + })) + .filter((item) => item.values.length > 0) + .sort((a, b) => a.tagId.localeCompare(b.tagId)); +}; diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 323e546245bf..a1bbd622cf72 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -1,8 +1,15 @@ import { MongoDatasetCollection } from './schema'; import type { ClientSession } from '../../../common/mongo'; -import { MongoDatasetCollectionTags } from '../tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '../tag/schemaV2'; import { readFromSecondary } from '../../../common/mongo/utils'; -import type { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type'; +import { + DEFAULT_TAG, + type CollectionTagLabelType, + type CollectionTagValueType, + type CollectionWithDatasetType, + type DatasetCollectionTagType +} from '@fastgpt/global/core/dataset/type'; +import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { DatasetCollectionDataProcessModeEnum, DatasetCollectionSyncResultEnum, @@ -10,7 +17,6 @@ import { DatasetSourceReadTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { readDatasetSourceRawText } from '../read'; import { hashStr } from '@fastgpt/global/common/string/tools'; import { mongoSessionRun } from '../../../common/mongo/sessionRun'; @@ -65,58 +71,216 @@ export function getCollectionUpdateTime({ name, time }: { time?: Date; name: str return new Date(); } +export const validateDatasetTagValue = ({ + tagType, + value +}: { + tagType?: DatasetCollectionTagType; + value: string | number | string[]; +}): DatasetErrEnum | undefined => { + return validateAndNormalizeTagValue({ tagType, value }).error; +}; + +/** + * 校验并规范化单个标签值，返回可直接持久化的 value。 + * - number/datetime：统一按 number 存储（字符串转 number、datetime 按 UTC 毫秒时间戳校验） + * - string/array：仅校验，值原样返回 + * Collection 创建路径与 fastgpt-pro 标签值写路径共用，保证两条写链路存储格式一致 + */ +export const validateAndNormalizeTagValue = ({ + tagType, + value +}: { + tagType?: DatasetCollectionTagType; + value: string | number | string[]; +}): { value: string | number | string[]; error?: DatasetErrEnum } => { + const type = tagType ?? 'string'; + + if (type === 'string') { + const error = typeof value !== 'string' || value.length > 256; + return error ? { value, error: DatasetErrEnum.tagValueInvalid } : { value }; + } + if (type === 'array') { + const error = + !Array.isArray(value) || + value.length > 64 || + value.some((item) => typeof item !== 'string' || item.length > 256); + return error ? { value, error: DatasetErrEnum.arrayTagValueInvalid } : { value }; + } + + if (typeof value === 'string' && value.trim() === '') { + return { value, error: DatasetErrEnum.tagValueInvalid }; + } + const numericValue = + typeof value === 'number' ? value : typeof value === 'string' ? Number(value) : NaN; + if (!Number.isFinite(numericValue)) { + return { value, error: DatasetErrEnum.tagValueInvalid }; + } + if (type === 'datetime' && Number.isNaN(new Date(numericValue).getTime())) { + return { value, error: DatasetErrEnum.tagValueDatetimeInvalid }; + } + + return { value: numericValue }; +}; + +const isSameTagValue = (a: string | number | string[], b: string | number | string[]): boolean => { + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) return false; + const bSet = new Set(b); + return a.every((item) => bSet.has(item)); + } + return a === b; +}; + +/** + * 同一 tagId 去重：值相同去重，值冲突拒绝整个批量操作。 + * 所有标签值写入路径（Collection 创建/更新、Pro setCollectionTags/batchSetCollectionTags）复用此逻辑 + */ +export const deduplicateTagValues = async ( + tags: CollectionTagValueType[] +): Promise => { + const seen = new Map(); + const deduped: CollectionTagValueType[] = []; + for (const t of tags) { + if (seen.has(t.tagId)) { + if (!isSameTagValue(seen.get(t.tagId)!, t.value)) { + throw DatasetErrEnum.tagValueInvalid; + } + } else { + seen.set(t.tagId, t.value); + deduped.push(t); + } + } + return deduped; +}; + +/** + * 查找或创建旧字符串标签的承载记录，只按 fromMigration 定位。 + * default_tag 只是首次创建时使用的普通名称，不参与后续身份判断。 + */ +export async function ensureDatasetTagMigrationCarrier({ + datasetId, + teamId, + session +}: { + datasetId: string; + teamId: string; + session?: ClientSession; +}) { + const findDefaultTag = () => + MongoDatasetCollectionTagsV2.findOne({ teamId, datasetId, fromMigration: true }, undefined, { + session + }).lean(); + + const existing = await findDefaultTag(); + if (existing) return existing; + + try { + const [created] = await MongoDatasetCollectionTagsV2.create( + [{ teamId, datasetId, tag: DEFAULT_TAG, tagType: 'array', fromMigration: true }], + { session } + ); + return created.toObject ? created.toObject() : created; + } catch (error: any) { + if (error?.code !== 11000) throw error; + const raced = await findDefaultTag(); + if (raced) return raced; + throw error; + } +} + +/** + * 统一解析 collection 创建时的 tags 入参： + * - string 元素（旧格式标签名）→ 归并到 v2 表 default_tag array 标签 + * - {tag, value} 元素 → 查找 v2 表标签并校验值类型，返回 {tagId, value} + * + * 返回值可直接写入 collection.tags 字段存储。 + * 同一 tagId 多条输入：值相同去重，值冲突拒绝整个操作。 + */ export const createOrGetCollectionTags = async ({ tags, datasetId, teamId, session }: { - tags?: string[]; + tags?: CollectionTagLabelType[]; datasetId: string; teamId: string; session?: ClientSession; -}) => { +}): Promise => { if (!tags) return undefined; - if (tags.length === 0) return []; - const existingTags = await MongoDatasetCollectionTags.find( - { - teamId, - datasetId, - tag: { $in: tags } - }, - undefined, - { session } - ).lean(); - - const existingTagContents = existingTags.map((tag) => tag.tag); - const newTagContents = tags.filter((tag) => !existingTagContents.includes(tag)); - - const newTags = await MongoDatasetCollectionTags.insertMany( - newTagContents.map((tagContent) => ({ - teamId, - datasetId, - tag: tagContent - })), - { session, ordered: true } + const stringNames = tags.filter((item): item is string => typeof item === 'string'); + const objectInputs = tags.filter( + (item): item is { tag: string; value: string | number | string[] } => typeof item !== 'string' ); - return [...existingTags.map((tag) => tag._id), ...newTags.map((tag) => tag._id)]; + const trimmedStringNames = stringNames.map((name) => name.trim()); + if (trimmedStringNames.some((name) => !name)) throw DatasetErrEnum.tagNameEmpty; + + const tagNames = objectInputs.map((item) => item.tag.trim()); + if (tagNames.some((name) => !name)) throw DatasetErrEnum.tagNameEmpty; + + const tagDefinitions = tagNames.length + ? await MongoDatasetCollectionTagsV2.find( + { teamId, datasetId, tag: { $in: tagNames } }, + undefined, + { session } + ).lean() + : []; + const tagDefinitionMap = new Map(tagDefinitions.map((tag) => [tag.tag, tag])); + + const normalizedObjectInputs = objectInputs.map((input) => { + const tagDoc = tagDefinitionMap.get(input.tag.trim()); + if (!tagDoc) { + return { value: input.value, error: DatasetErrEnum.tagNotExist }; + } + const tagType = tagDoc.tagType ?? 'string'; + const { value, error } = validateAndNormalizeTagValue({ tagType, value: input.value }); + return { + tagId: String(tagDoc._id), + value, + error + }; + }); + + for (const { error } of normalizedObjectInputs) { + if (error) throw error; + } + + const defaultValues: string[] = [...new Set(trimmedStringNames)]; + + const result: CollectionTagValueType[] = []; + + if (defaultValues.length > 0) { + const defaultTag = await ensureDatasetTagMigrationCarrier({ datasetId, teamId, session }); + result.push({ tagId: String(defaultTag._id), value: [...new Set(defaultValues)] }); + } + + result.push(...normalizedObjectInputs.map(({ tagId, value }) => ({ tagId: tagId!, value }))); + + return deduplicateTagValues(result); }; +/** + * 将 collection 的 tags（混合格式）解析为可重入的输入格式 + * - 旧格式 ObjectId → 标签名（如 "safety"） + * - 新格式 {tagId, value} → {tag: 标签名, value}（如 {tag: "safety", value: "A"}） + * + * 输出结果可作为 createOrGetCollectionTags 的 tags 参数，用于同步、重建等场景 + */ export const collectionTagsToTagLabel = async ({ datasetId, tags }: { datasetId: string; - tags?: string[]; -}) => { + tags?: (string | CollectionTagValueType)[]; +}): Promise => { if (!tags) return undefined; - if (tags.length === 0) return; + if (tags.length === 0) return []; - // Get all the tags - const collectionTags = await MongoDatasetCollectionTags.find({ datasetId }, undefined, { + const collectionTags = await MongoDatasetCollectionTagsV2.find({ datasetId }, undefined, { ...readFromSecondary }).lean(); const tagsMap = new Map(); @@ -126,9 +290,14 @@ export const collectionTagsToTagLabel = async ({ return tags .map((tag) => { - return tagsMap.get(tag) || ''; + if (typeof tag === 'string') { + const tagName = tagsMap.get(tag); + return tagName ?? null; + } + const tagName = tagsMap.get(tag.tagId); + return tagName ? { tag: tagName, value: tag.value } : null; }) - .filter(Boolean); + .filter((item): item is CollectionTagLabelType => item !== null); }; export const syncCollection = async (collection: CollectionWithDatasetType) => { diff --git a/packages/service/core/dataset/delete/processor.ts b/packages/service/core/dataset/delete/processor.ts index a4876781aafb..f1f65d1467be 100644 --- a/packages/service/core/dataset/delete/processor.ts +++ b/packages/service/core/dataset/delete/processor.ts @@ -2,6 +2,7 @@ import type { Processor } from '@fastgpt/dal/redis/bullmq'; import { addDatasetDeleteJob, type DatasetDeleteJobData } from './index'; import { delDatasetRelevantData, findDatasetAndAllChildren } from '../controller'; import { MongoDatasetCollectionTags } from '../tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '../tag/schemaV2'; import { removeDatasetSyncJobScheduler } from '../datasetSync'; import { mongoSessionRun } from '../../../common/mongo/sessionRun'; import { MongoDataset } from '../schema'; @@ -92,6 +93,11 @@ const deleteDatasets = async ({ datasetId: { $in: datasetIds } }); + await MongoDatasetCollectionTagsV2.deleteMany({ + teamId, + datasetId: { $in: datasetIds } + }); + // Delete dataset avatar for await (const dataset of datasets) { await removeImageByPath(dataset.avatar); diff --git a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts index 70c9fa4a8e9a..3ddbcea603a9 100644 --- a/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts +++ b/packages/service/core/dataset/search/defaultRecall/collectionFilter.ts @@ -1,8 +1,288 @@ import json5 from 'json5'; +import safeRegex from 'safe-regex'; import { MongoDatasetCollection } from '../../collection/schema'; -import { MongoDatasetCollectionTags } from '../../tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '../../tag/schemaV2'; +import { isCollectionTagValue } from '@fastgpt/global/core/dataset/tagUtils'; import { readFromSecondary } from '../../../../common/mongo/utils'; -import { computeFilterIntersection } from '../utils'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { applySharedCollectionMetadataFilters } from './collectionFilterShared'; + +/* ========== New format key-value tag filtering types ========== */ + +/** A single key-value tag condition: { tagName: { $op: value } } */ +type TagCondition = Record>; + +/* ========== checkValue: pure value comparsion ========== */ + +// safe-regex 漏检带分支的量词组（如 (a|aa)+ 在 V8 下呈指数回溯），补充首字符重叠检测： +// 逐层展平分组，量词作用域内含分支且分支首字符重叠 → 不安全 +const hasAmbiguousAlternation = (pattern: string): boolean => { + let s = pattern + .replace(/\\./g, ' ') + .replace(/\[[^\]]*\]/g, ' ') + .replace(/\(\?:|\(\?=/g, '('); + const innerRe = /\(([^()]*)\)([+*?]|\{\d+(?:,\d*)?\})?/; + let m: RegExpExecArray | null; + while ((m = innerRe.exec(s))) { + if (m[2] && m[1].includes('|')) { + const firstChars = new Set(); + for (const alt of m[1].split('|')) { + const c = alt.replace(/^[\\^]/, '').charAt(0); + if (firstChars.has(c)) return true; + firstChars.add(c); + } + } + s = s.slice(0, m.index) + 'x' + s.slice(m.index + m[0].length); + } + return false; +}; + +type CompareOp = + | '$eq' + | '$ne' + | '$gt' + | '$lt' + | '$gte' + | '$lte' + | '$contains' + | '$notContains' + | '$startsWith' + | '$endsWith' + | '$regex' + | '$is' + | '$isNot' + | '$in' + | '$notIn' + | '$empty' + | '$notEmpty'; + +export function checkValue( + op: CompareOp, + target: unknown, + storedVal: string | number | string[] | null | undefined, + tagType: string +): boolean { + if (op === '$empty') { + return tagType === 'array' + ? !Array.isArray(storedVal) || storedVal.length === 0 + : storedVal === null || storedVal === undefined || storedVal === ''; + } + if (op === '$notEmpty') { + return tagType === 'array' + ? Array.isArray(storedVal) && storedVal.length > 0 + : storedVal !== null && storedVal !== undefined && storedVal !== ''; + } + + if (target === null || target === undefined) return false; + + switch (tagType) { + case 'number': + case 'datetime': { + const stored = Number(storedVal); + const t = Number(target); + if (isNaN(stored) || isNaN(t)) return false; + switch (op) { + case '$eq': + return stored === t; + case '$ne': + return stored !== t; + case '$gt': + return stored > t; + case '$lt': + return stored < t; + case '$gte': + return stored >= t; + case '$lte': + return stored <= t; + default: + return false; + } + } + case 'array': { + if (!Array.isArray(storedVal)) return false; + const stored = storedVal; + const targetArray = Array.isArray(target) + ? target.filter((item): item is string => typeof item === 'string') + : []; + const equal = (left: string[], right: string[]) => { + const rightSet = new Set(right); + return new Set(left).size === rightSet.size && left.every((item) => rightSet.has(item)); + }; + const subset = (left: string[], right: string[]) => + left.every((item) => right.includes(item)); + switch (op) { + case '$is': + return Array.isArray(target) && equal(stored, targetArray); + case '$isNot': + return Array.isArray(target) && !equal(stored, targetArray); + case '$contains': + return typeof target === 'string' && stored.includes(target); + case '$notContains': + return typeof target === 'string' && !stored.includes(target); + case '$in': + return Array.isArray(target) && subset(stored, targetArray); + case '$notIn': + return Array.isArray(target) && !subset(stored, targetArray); + default: + return false; + } + } + case 'string': + default: { + const stored = String(storedVal ?? ''); + const t = String(target); + switch (op) { + case '$eq': + return stored === t; + case '$ne': + return stored !== t; + case '$contains': + return stored.toLowerCase().includes(t.toLowerCase()); + case '$notContains': + return !stored.toLowerCase().includes(t.toLowerCase()); + case '$startsWith': + return stored.toLowerCase().startsWith(t.toLowerCase()); + case '$endsWith': + return stored.toLowerCase().endsWith(t.toLowerCase()); + case '$regex': + // 用户可控 pattern：限制长度并拦截灾难性回溯，防止 ReDoS + try { + if (t.length > 64 || stored.length > 256) return false; + if (!safeRegex(t) || hasAmbiguousAlternation(t)) return false; + return new RegExp(t).test(stored); + } catch { + return false; + } + default: + return false; + } + } + } +} + +/* ========== filterCollectionByKeyValueTags ========== */ + +/** + * Filter collections by key-value tag conditions (new format). + * + * AND conditions must all be satisfied; OR conditions need at least one match. + * A condition whose tag does not exist in a dataset fails that condition: + * AND → no match; OR → that condition does not count as a match. + */ +export async function filterCollectionByKeyValueTags({ + $and, + $or, + teamId, + datasetIds +}: { + $and: TagCondition[]; + $or: TagCondition[]; + teamId: string; + datasetIds: string[]; +}): Promise { + const allConditions = [...$and, ...$or]; + const tagNames = new Set(); + for (const cond of allConditions) { + const tagName = Object.keys(cond)[0]; + if (!tagName) continue; + tagNames.add(tagName); + } + if (tagNames.size === 0) return undefined; + + const tagDocs = await MongoDatasetCollectionTagsV2.find( + { + teamId, + datasetId: { $in: datasetIds }, + tag: { $in: [...tagNames] } + }, + '_id datasetId tag tagType', + { ...readFromSecondary } + ).lean(); + + const datasetTagMap = new Map>(); + const addToMap = (dsId: string, tagName: string, id: string, type: string) => { + const tagMap = datasetTagMap.get(dsId); + if (tagMap) { + tagMap.set(tagName, { id, type }); + return; + } + datasetTagMap.set(dsId, new Map([[tagName, { id, type }]])); + }; + for (const doc of tagDocs) { + addToMap(String(doc.datasetId), doc.tag, String(doc._id), doc.tagType ?? 'string'); + } + if (datasetTagMap.size === 0) return []; + + // 3. Check a single value condition against one collection's tags. + // Tag missing in the dataset → not satisfied; entry missing in the collection + // → not satisfied. + const matchCondition = ( + cond: TagCondition, + tagMap: Map, + tagsArr: Array<{ tagId: string; value?: string | number | string[] }> + ): boolean => { + const tagName = Object.keys(cond)[0]; + const tagInfo = tagMap.get(tagName); + if (!tagInfo) return false; + const entry = tagsArr.find((t) => t.tagId === tagInfo.id); + if (!entry) return false; + const opObj = cond[tagName] as Record; + const op = Object.keys(opObj)[0]; + return checkValue(op as CompareOp, opObj[op], entry.value, tagInfo.type); + }; + + const allCollectionIds: string[] = []; + + // 4. Iterate each dataset (the same tag name may map to different tagIds per dataset) + for (const [dsId, tagMap] of datasetTagMap) { + const andTagIds = $and + .map((cond) => tagMap.get(Object.keys(cond)[0])?.id) + .filter((id): id is string => Boolean(id)); + const orTagIds = $or + .map((cond) => tagMap.get(Object.keys(cond)[0])?.id) + .filter((id): id is string => Boolean(id)); + + if (andTagIds.length === 0 && orTagIds.length === 0) continue; + + // Mongo pre-filter by tagId. AND → $all (more precise); pure OR → $in. + // A single 'tags.tagId' predicate keeps the compound index + // { teamId, datasetId, 'tags.tagId' } usable; exact matching happens below. + const tagIdQuery = + andTagIds.length > 0 + ? { 'tags.tagId': { $all: andTagIds } } + : { 'tags.tagId': { $in: orTagIds } }; + + const collections = await MongoDatasetCollection.find( + { teamId, datasetId: dsId, ...tagIdQuery }, + '_id tags', + { ...readFromSecondary } + ) + // 生产环境同时存在 {teamId,datasetId,tags} 与 {teamId,datasetId,'tags.tagId'} 两个多键索引时， + // 查询规划器可能误选前者（在整段 tags 数组上建索引，无法精准定位 tagId），导致全表 FETCH（实测慢约 8x）。 + // 这里用 hint 强制走 tags.tagId 索引，避免依赖规划器的索引选择。 + .hint({ teamId: 1, datasetId: 1, 'tags.tagId': 1 }) + .lean(); + + // 5. Application-layer value comparison + for (const col of collections) { + const tagsArr = (col.tags ?? []).filter(isCollectionTagValue); + + // AND: all must pass + const andOk = $and.every((cond) => matchCondition(cond, tagMap, tagsArr)); + if (!andOk) continue; + + // OR: at least one must pass + if ($or.length > 0) { + const orOk = $or.some((cond) => matchCondition(cond, tagMap, tagsArr)); + if (!orOk) continue; + } + + allCollectionIds.push(String(col._id)); + } + } + + return allCollectionIds.length > 0 ? allCollectionIds : []; +} export const getForbidCollectionIdList = async ({ teamId, @@ -23,12 +303,7 @@ export const getForbidCollectionIdList = async ({ return collections.map((item) => String(item._id)); }; -/** - * 按知识库集合元数据过滤 collectionId。 - * - * 标签过滤保持原有语义：`$and` 优先生效，且 `$and` 中字符串标签和 null 不能共存。 - * 输入 collectionIds 可以是文件夹，会递归展开为实际文件集合。 - */ +/** 新版知识库检索节点元数据过滤，只接受结构化标签条件。 */ export const filterCollectionByMetadata = async ({ teamId, datasetIds, @@ -38,211 +313,46 @@ export const filterCollectionByMetadata = async ({ datasetIds: string[]; collectionFilterMatch?: string; }): Promise => { - const getAllCollectionIds = async ({ - parentCollectionIds - }: { - parentCollectionIds?: string[]; - }): Promise => { - if (!parentCollectionIds) return; - if (parentCollectionIds.length === 0) { - return []; - } - - const collections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - _id: { $in: parentCollectionIds } - }, - '_id type', - { - ...readFromSecondary - } - ).lean(); - - const resultIds = new Set(); - collections.forEach((item) => { - if (item.type !== 'folder') { - resultIds.add(String(item._id)); - } - }); - - const folderIds = collections - .filter((item) => item.type === 'folder') - .map((item) => String(item._id)); - - // Get all child collection ids - if (folderIds.length) { - const childCollections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - parentId: { $in: folderIds } - }, - '_id type', - { - ...readFromSecondary - } - ).lean(); - - const childIds = await getAllCollectionIds({ - parentCollectionIds: childCollections.map((item) => String(item._id)) - }); - - childIds?.forEach((id) => resultIds.add(id)); - } - - return Array.from(resultIds); - }; - if (!collectionFilterMatch || !global.feConfigs.isPlus) return; - let tagCollectionIdList: string[] | undefined = undefined; - let createTimeCollectionIdList: string[] | undefined = undefined; - let inputCollectionIdList: string[] | undefined = undefined; - - try { - const jsonMatch = json5.parse(collectionFilterMatch); - - const andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined; - const orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined; - - if (andTags && andTags.length > 0) { - const uniqueAndTags = Array.from(new Set(andTags)); - if (uniqueAndTags.includes(null) && uniqueAndTags.some((tag) => typeof tag === 'string')) { - return []; - } - if (uniqueAndTags.every((tag) => typeof tag === 'string')) { - const matchedTags = await MongoDatasetCollectionTags.find( - { - teamId, - datasetId: { $in: datasetIds }, - tag: { $in: uniqueAndTags as string[] } - }, - '_id datasetId tag', - { ...readFromSecondary } - ).lean(); - - // Group tags by dataset - const datasetTagMap = new Map }>(); - - matchedTags.forEach((tag) => { - const datasetId = String(tag.datasetId); - if (!datasetTagMap.has(datasetId)) { - datasetTagMap.set(datasetId, { - tagIds: [], - tagNames: new Set() - }); - } - - const datasetData = datasetTagMap.get(datasetId)!; - datasetData.tagIds.push(String(tag._id)); - datasetData.tagNames.add(tag.tag); - }); - - const validDatasetIds = Array.from(datasetTagMap.entries()) - .filter(([, data]) => uniqueAndTags.every((tag) => data.tagNames.has(tag as string))) - .map(([datasetId]) => datasetId); - - if (validDatasetIds.length === 0) return []; - - const collectionsPromises = validDatasetIds.map((datasetId) => { - const { tagIds } = datasetTagMap.get(datasetId)!; - return MongoDatasetCollection.find( - { - teamId, - datasetId, - tags: { $all: tagIds } - }, - '_id', - { ...readFromSecondary } - ).lean(); - }); - - const collectionsResults = await Promise.all(collectionsPromises); - tagCollectionIdList = collectionsResults.flat().map((item) => String(item._id)); - } else if (uniqueAndTags.every((tag) => tag === null)) { - const collections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - $or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }] - }, - '_id', - { ...readFromSecondary } - ).lean(); - tagCollectionIdList = collections.map((item) => String(item._id)); - } - } else if (orTags && orTags.length > 0) { - // Get tagId by tag string - const orTagArray = await MongoDatasetCollectionTags.find( - { - teamId, - datasetId: { $in: datasetIds }, - tag: { $in: orTags.filter((tag) => tag !== null) } - }, - '_id', - { ...readFromSecondary } - ).lean(); - const orTagIds = orTagArray.map((item) => String(item._id)); - - // Get collections by tagId - const collections = await MongoDatasetCollection.find( - { - teamId, - datasetId: { $in: datasetIds }, - $or: [ - { tags: { $in: orTagIds } }, - ...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : []) - ] - }, - '_id', - { ...readFromSecondary } - ).lean(); - - tagCollectionIdList = collections.map((item) => String(item._id)); - } - - // time - const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined; - const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined; - if (getCreateTime || lteCreateTime) { - const collections = await MongoDatasetCollection.find( - { + const metadataMatch = json5.parse(collectionFilterMatch) as { + tags?: { $and?: unknown[]; $or?: unknown[] }; + createTime?: { $gte?: string; $lte?: string }; + collectionIds?: string[]; + }; + const isConditionObject = (item: unknown): item is TagCondition => { + if (typeof item !== 'object' || Array.isArray(item) || item === null) return false; + const tagNames = Object.keys(item); + if (tagNames.length !== 1 || !tagNames[0]) return false; + const operation = Reflect.get(item, tagNames[0]); + return ( + typeof operation === 'object' && + !Array.isArray(operation) && + operation !== null && + Object.keys(operation).length === 1 + ); + }; + const parseConditions = (items?: unknown[]): TagCondition[] => { + if (!items) return []; + if (!items.every(isConditionObject)) throw CommonErrEnum.invalidParams; + return items; + }; + const andTags = parseConditions(metadataMatch.tags?.$and); + const orTags = parseConditions(metadataMatch.tags?.$or); + const tagCollectionIds = + andTags.length > 0 || orTags.length > 0 + ? await filterCollectionByKeyValueTags({ + $and: andTags, + $or: orTags, teamId, - datasetId: { $in: datasetIds }, - createTime: { - ...(getCreateTime && { $gte: new Date(getCreateTime) }), - ...(lteCreateTime && { - $lte: new Date(lteCreateTime) - }) - } - }, - '_id' - ); - createTimeCollectionIdList = collections.map((item) => String(item._id)); - } - - // collectionIds - const inputCollectionIds = jsonMatch?.collectionIds as string[] | undefined; - if (Array.isArray(inputCollectionIds) && inputCollectionIds.length > 0) { - inputCollectionIdList = await getAllCollectionIds({ - parentCollectionIds: inputCollectionIds - }); - if (inputCollectionIdList && inputCollectionIdList.length === 0) { - return []; - } - } + datasetIds + }) + : undefined; - // Concat tag, time and collectionIds - const collectionIds = computeFilterIntersection([ - tagCollectionIdList, - createTimeCollectionIdList, - inputCollectionIdList - ]); - - return await getAllCollectionIds({ - parentCollectionIds: collectionIds - }); - } catch {} + return applySharedCollectionMetadataFilters({ + teamId, + datasetIds, + metadataMatch, + tagCollectionIds + }); }; diff --git a/packages/service/core/dataset/search/defaultRecall/collectionFilterShared.ts b/packages/service/core/dataset/search/defaultRecall/collectionFilterShared.ts new file mode 100644 index 000000000000..a2b972713cb7 --- /dev/null +++ b/packages/service/core/dataset/search/defaultRecall/collectionFilterShared.ts @@ -0,0 +1,113 @@ +import { MongoDatasetCollection } from '../../collection/schema'; +import { readFromSecondary } from '../../../../common/mongo/utils'; +import { computeFilterIntersection } from '../utils'; + +type CollectionMetadataMatch = { + createTime?: { $gte?: string; $lte?: string }; + collectionIds?: string[]; +}; + +/** + * 将文件或文件夹 ID 展开为真正可召回的非文件夹 Collection ID。 + * 查询始终受 teamId + datasetIds 限制，防止跨知识库展开。 + */ +export const expandCollectionIds = async ({ + teamId, + datasetIds, + parentCollectionIds +}: { + teamId: string; + datasetIds: string[]; + parentCollectionIds?: string[]; +}): Promise => { + if (!parentCollectionIds) return; + if (parentCollectionIds.length === 0) return []; + + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + _id: { $in: parentCollectionIds } + }, + '_id type', + { ...readFromSecondary } + ).lean(); + + const resultIds = new Set( + collections.filter((item) => item.type !== 'folder').map((item) => String(item._id)) + ); + const folderIds = collections + .filter((item) => item.type === 'folder') + .map((item) => String(item._id)); + + if (folderIds.length > 0) { + const children = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + parentId: { $in: folderIds } + }, + '_id', + { ...readFromSecondary } + ).lean(); + const childIds = await expandCollectionIds({ + teamId, + datasetIds, + parentCollectionIds: children.map((item) => String(item._id)) + }); + childIds?.forEach((id) => resultIds.add(id)); + } + + return [...resultIds]; +}; + +/** 将标签结果与时间、Collection ID 条件求交，新旧标签链路共用。 */ +export const applySharedCollectionMetadataFilters = async ({ + teamId, + datasetIds, + metadataMatch, + tagCollectionIds +}: { + teamId: string; + datasetIds: string[]; + metadataMatch: CollectionMetadataMatch; + tagCollectionIds?: string[]; +}): Promise => { + const getCreateTime = metadataMatch.createTime?.$gte; + const lteCreateTime = metadataMatch.createTime?.$lte; + const createTimeCollectionIds = + getCreateTime || lteCreateTime + ? ( + await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + createTime: { + ...(getCreateTime ? { $gte: new Date(getCreateTime) } : {}), + ...(lteCreateTime ? { $lte: new Date(lteCreateTime) } : {}) + } + }, + '_id', + { ...readFromSecondary } + ) + ).map((item) => String(item._id)) + : undefined; + + const inputCollectionIds = + Array.isArray(metadataMatch.collectionIds) && metadataMatch.collectionIds.length > 0 + ? await expandCollectionIds({ + teamId, + datasetIds, + parentCollectionIds: metadataMatch.collectionIds + }) + : undefined; + if (inputCollectionIds?.length === 0) return []; + + const collectionIds = computeFilterIntersection([ + tagCollectionIds, + createTimeCollectionIds, + inputCollectionIds + ]); + + return expandCollectionIds({ teamId, datasetIds, parentCollectionIds: collectionIds }); +}; diff --git a/packages/service/core/dataset/search/defaultRecall/index.ts b/packages/service/core/dataset/search/defaultRecall/index.ts index a7b666e9b7b1..67216b040535 100644 --- a/packages/service/core/dataset/search/defaultRecall/index.ts +++ b/packages/service/core/dataset/search/defaultRecall/index.ts @@ -49,7 +49,8 @@ export async function searchDatasetData( rerankModel, rerankWeight = 0.5, datasetIds = [], - collectionFilterMatch + collectionFilterMatch, + collectionFilterMode } = props; const searchMode = DatasetSearchModeMap[inputSearchMode] @@ -98,6 +99,7 @@ export async function searchDatasetData( model, imageQueries, collectionFilterMatch, + collectionFilterMode, embeddingLimit, fullTextLimit, textQueries, diff --git a/packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts b/packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts new file mode 100644 index 000000000000..62d971936a03 --- /dev/null +++ b/packages/service/core/dataset/search/defaultRecall/legacy/collectionFilter.ts @@ -0,0 +1,117 @@ +import json5 from 'json5'; +import { MongoDatasetCollection } from '../../../collection/schema'; +import { MongoDatasetCollectionTagsV2 } from '../../../tag/schemaV2'; +import { readFromSecondary } from '../../../../../common/mongo/utils'; +import { applySharedCollectionMetadataFilters } from '../collectionFilterShared'; +import type { LegacyCollectionFilterMatch } from './type'; + +/** + * 按 dataset 读取迁移承载标签。每个知识库独立解析 tagId，不依赖可修改的标签名。 + */ +const getMigrationTagIds = async ({ + teamId, + datasetIds +}: { + teamId: string; + datasetIds: string[]; +}) => + MongoDatasetCollectionTagsV2.find( + { teamId, datasetId: { $in: datasetIds }, fromMigration: true }, + '_id datasetId', + { ...readFromSecondary } + ).lean(); + +/** + * 存量知识库检索节点的标签过滤。 + * `$and` 存在时完全忽略 `$or`；null 只匹配 tags 真正为空或不存在的 Collection。 + */ +const filterLegacyCollectionByTags = async ({ + teamId, + datasetIds, + tags +}: { + teamId: string; + datasetIds: string[]; + tags?: LegacyCollectionFilterMatch['tags']; +}): Promise => { + const andTags = tags?.$and; + const orTags = tags?.$or; + const activeTags = andTags?.length ? andTags : orTags; + if (!activeTags?.length) return; + + const hasNull = activeTags.includes(null); + const stringTags = [ + ...new Set(activeTags.filter((tag): tag is string => typeof tag === 'string')) + ]; + if (andTags?.length && hasNull && stringTags.length > 0) return []; + + if (andTags?.length && hasNull) { + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + $or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }] + }, + '_id', + { ...readFromSecondary } + ).lean(); + return collections.map((item) => String(item._id)); + } + + const migrationTags = stringTags.length ? await getMigrationTagIds({ teamId, datasetIds }) : []; + const queries = migrationTags.map((migrationTag) => ({ + teamId, + datasetId: migrationTag.datasetId, + tags: { + $elemMatch: { + tagId: String(migrationTag._id), + value: andTags?.length ? { $all: stringTags } : { $in: stringTags } + } + } + })); + + const match = { + teamId, + datasetId: { $in: datasetIds }, + $or: [ + ...queries, + ...(!andTags?.length && hasNull ? [{ tags: { $size: 0 } }, { tags: { $exists: false } }] : []) + ] + }; + if (match.$or.length === 0) return []; + + const collections = await MongoDatasetCollection.find(match, '_id', { + ...readFromSecondary + }).lean(); + return collections.map((item) => String(item._id)); +}; + +/** 旧节点完整元数据过滤入口；旧配置解析失败时保持历史降级行为。 */ +export const filterLegacyCollectionByMetadata = async ({ + teamId, + datasetIds, + collectionFilterMatch +}: { + teamId: string; + datasetIds: string[]; + collectionFilterMatch?: string; +}): Promise => { + if (!collectionFilterMatch || !global.feConfigs.isPlus) return; + + try { + const metadataMatch = json5.parse(collectionFilterMatch) as LegacyCollectionFilterMatch; + const tagCollectionIds = await filterLegacyCollectionByTags({ + teamId, + datasetIds, + tags: metadataMatch.tags + }); + return applySharedCollectionMetadataFilters({ + teamId, + datasetIds, + metadataMatch, + tagCollectionIds + }); + } catch { + return; + } +}; diff --git a/packages/service/core/dataset/search/defaultRecall/legacy/type.ts b/packages/service/core/dataset/search/defaultRecall/legacy/type.ts new file mode 100644 index 000000000000..c000a0c48265 --- /dev/null +++ b/packages/service/core/dataset/search/defaultRecall/legacy/type.ts @@ -0,0 +1,8 @@ +export type LegacyCollectionFilterMatch = { + tags?: { + $and?: Array; + $or?: Array; + }; + createTime?: { $gte?: string; $lte?: string }; + collectionIds?: string[]; +}; diff --git a/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts b/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts index a930ff753a2d..3392bca9bd0d 100644 --- a/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts +++ b/packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts @@ -1,7 +1,10 @@ import { getForbidCollectionIdList, filterCollectionByMetadata } from './collectionFilter'; +import { filterLegacyCollectionByMetadata } from './legacy/collectionFilter'; import { embeddingRecall } from './embeddingRecall'; import { fullTextRecall } from './fullTextRecall'; import type { EmbeddingSystemModelDataType } from '@fastgpt/global/core/ai/model.schema'; +import type { CollectionFilterMode } from '../type'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; /** * 默认召回的并行调度层。 @@ -14,6 +17,7 @@ export const multiQueryRecall = async ({ model, imageQueries, collectionFilterMatch, + collectionFilterMode = DatasetTagFilterVersionEnum.structured, embeddingLimit, fullTextLimit, textQueries, @@ -24,6 +28,7 @@ export const multiQueryRecall = async ({ model: EmbeddingSystemModelDataType; imageQueries: string[]; collectionFilterMatch?: string; + collectionFilterMode?: CollectionFilterMode; embeddingLimit: number; fullTextLimit: number; textQueries: string[]; @@ -34,11 +39,9 @@ export const multiQueryRecall = async ({ teamId, datasetIds }), - filterCollectionByMetadata({ - teamId, - datasetIds, - collectionFilterMatch - }) + collectionFilterMode === DatasetTagFilterVersionEnum.legacy + ? filterLegacyCollectionByMetadata({ teamId, datasetIds, collectionFilterMatch }) + : filterCollectionByMetadata({ teamId, datasetIds, collectionFilterMatch }) ]); const [ diff --git a/packages/service/core/dataset/search/type.ts b/packages/service/core/dataset/search/type.ts index 727ef1c7296a..00941a8afbf3 100644 --- a/packages/service/core/dataset/search/type.ts +++ b/packages/service/core/dataset/search/type.ts @@ -8,6 +8,9 @@ import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/ty import type { ChatItemMiniType } from '@fastgpt/global/core/chat/type'; import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type'; +import type { DatasetTagFilterVersion } from '@fastgpt/global/core/dataset/workflowTagFilter'; + +export type CollectionFilterMode = DatasetTagFilterVersion; export type SearchDatasetDataProps = { histories: ChatItemMiniType[]; @@ -47,6 +50,8 @@ export type SearchDatasetDataProps = { } */ collectionFilterMatch?: string; + /** 由节点 Dispatcher 明确指定，不根据过滤值形状推断。 */ + collectionFilterMode?: CollectionFilterMode; }; export type SearchDatasetDataResponse = { diff --git a/packages/service/core/dataset/tag/schemaV2.ts b/packages/service/core/dataset/tag/schemaV2.ts new file mode 100644 index 000000000000..e60c6c8c9670 --- /dev/null +++ b/packages/service/core/dataset/tag/schemaV2.ts @@ -0,0 +1,52 @@ +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { defineIndex, connectionMongo, getMongoModel } from '../../../common/mongo'; +import { DatasetCollectionName } from '../schema'; +import { type DatasetCollectionTagsSchemaType } from '@fastgpt/global/core/dataset/type'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; +const { Schema } = connectionMongo; + +const DatasetCollectionTagsV2Name = 'dataset_collection_tags_v2'; + +const DatasetCollectionTagsV2Schema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + tag: { + type: String, + required: true + // COMMENT: 标签名称。迁移承载记录的身份只由 fromMigration 标识 + }, + tagType: { + type: String, + default: 'string', + enum: Object.values(DatasetCollectionTagTypeEnum) + // COMMENT: 标签类型。string=字符串比较, number=数值比较, datetime=时间戳比较, array=字符串数组集合比较(V2.0) + }, + options: { + type: [String] + // COMMENT: array 类型标签的预设选项。集合写入选项值时合并新增项，标签管理可覆盖删除 + }, + fromMigration: { + type: Boolean, + default: false + // COMMENT: 迁移/旧格式输入创建的 default_tag 承载记录；过滤按该字段定位而非标签名 + } +}); + +// unique 索引保证并发写入下同一作用域标签名不重复 +defineIndex(DatasetCollectionTagsV2Schema, { + key: { teamId: 1, datasetId: 1, tag: 1 }, + options: { unique: true } +}); + +export const MongoDatasetCollectionTagsV2 = getMongoModel( + DatasetCollectionTagsV2Name, + DatasetCollectionTagsV2Schema +); diff --git a/packages/service/core/workflow/dispatch/ai/agent/index.ts b/packages/service/core/workflow/dispatch/ai/agent/index.ts index 631961f082f4..1a5baeb660e2 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/index.ts @@ -78,6 +78,7 @@ export type DispatchAgentModuleProps = ModuleDispatchProps<{ /** @deprecated */ [NodeInputKeyEnum.datasetSearchExtensionModel]?: string; [NodeInputKeyEnum.datasetSearchExtensionBg]?: string; + [NodeInputKeyEnum.collectionFilterMatch]?: string; [NodeInputKeyEnum.authTmbId]?: boolean; [NodeInputKeyEnum.useAgentSandbox]?: boolean; [NodeInputKeyEnum.sandboxEntrypoint]?: string; diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts index a03f215fbeed..e21fd71b9da4 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts @@ -23,6 +23,7 @@ import { getLogger, LogCategories } from '../../../../../../../common/logger'; import type { DispatchSubAppResponse } from '../../type'; import type { AppFormEditFormType } from '@fastgpt/global/core/app/formEdit/type'; import { DatasetSearchToolSchema } from './utils'; +import { formatCollectionFilterMatchParam } from '@fastgpt/global/core/dataset/workflowTagFilter'; import { parseJsonArgs } from '../../../../../../ai/utils'; import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; @@ -34,6 +35,7 @@ import { import { filterDatasetsByTmbId } from '../../../../../../dataset/utils'; import { normalizeDatasetSearchInput } from '../../../../dataset/utils'; import type { LLMSystemModelDataType } from '@fastgpt/global/core/ai/model.schema'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; const logger = getLogger(LogCategories.MODULE.AI.AGENT); type DatasetSearchParams = { @@ -263,6 +265,10 @@ export const dispatchAgentDatasetSearch = async ({ datasetSearchUsingExtensionQuery: datasetParams.datasetSearchUsingExtensionQuery ?? false, datasetSearchExtensionModel: extensionModelData, datasetSearchExtensionBg: datasetParams.datasetSearchExtensionBg, + collectionFilterMatch: formatCollectionFilterMatchParam({ + value: datasetParams.collectionFilterMatch + }), + collectionFilterMode: DatasetTagFilterVersionEnum.structured, userKey }; const { diff --git a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts index 5790270a6a2d..918bb5149e26 100644 --- a/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts +++ b/packages/service/core/workflow/dispatch/ai/agent/sub/utils.ts @@ -136,6 +136,7 @@ export const getAgentDatasetParams = ( datasetSearchExtensionModelId: params[NodeInputKeyEnum.datasetSearchExtensionModelId], datasetSearchExtensionModel: params[NodeInputKeyEnum.datasetSearchExtensionModel], datasetSearchExtensionBg: params[NodeInputKeyEnum.datasetSearchExtensionBg], + collectionFilterMatch: params[NodeInputKeyEnum.collectionFilterMatch], [NodeInputKeyEnum.authTmbId]: params[NodeInputKeyEnum.authTmbId] }; }; diff --git a/packages/service/core/workflow/dispatch/dataset/search.ts b/packages/service/core/workflow/dispatch/dataset/search.ts index b63212599ce7..ffa01def1555 100644 --- a/packages/service/core/workflow/dispatch/dataset/search.ts +++ b/packages/service/core/workflow/dispatch/dataset/search.ts @@ -25,6 +25,11 @@ import { createQueryExtensionChildNodeResponse } from './nodeResponse'; import { normalizeDatasetSearchInput } from './utils'; +import type { CollectionFilterMode } from '../../../dataset/search/type'; +import { + resolveDatasetTagFilterVersion, + type DatasetTagFilterVersion +} from '@fastgpt/global/core/dataset/workflowTagFilter'; const logger = getLogger(LogCategories.MODULE.WORKFLOW.DATASET); @@ -43,6 +48,7 @@ type DatasetSearchProps = ModuleDispatchProps<{ [NodeInputKeyEnum.datasetSearchRerankWeight]?: number; [NodeInputKeyEnum.collectionFilterMatch]: string; + [NodeInputKeyEnum.collectionFilterVersion]?: DatasetTagFilterVersion; [NodeInputKeyEnum.authTmbId]?: boolean; [NodeInputKeyEnum.datasetSearchUsingExtensionQuery]: boolean; @@ -60,9 +66,10 @@ export type DatasetSearchResponse = DispatchNodeResultType<{ [NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[]; }>; -export async function dispatchDatasetSearch( +/** 根据节点保存的显式版本选择标签过滤语义；缺少版本的存量节点固定走 legacy。 */ +export const dispatchDatasetSearch = async ( props: DatasetSearchProps -): Promise { +): Promise => { const { runningAppInfo: { teamId }, runningUserInfo: { tmbId }, @@ -77,6 +84,7 @@ export async function dispatchDatasetSearch( datasetSearchInput = [], authTmbId = false, collectionFilterMatch, + collectionFilterVersion, searchMode, embeddingWeight, usingReRank, @@ -101,6 +109,11 @@ export async function dispatchDatasetSearch( return Promise.reject(i18nT('chat:dataset_quote_type error')); } + const collectionFilterMode: CollectionFilterMode = resolveDatasetTagFilterVersion({ + version: collectionFilterVersion, + filterValue: collectionFilterMatch + }); + if (datasets.length === 0) { return getNodeErrResponse({ error: i18nT('common:core.chat.error.Select dataset empty') }); } @@ -180,7 +193,8 @@ export async function dispatchDatasetSearch( usingReRank, rerankModel: rerankModelData, rerankWeight, - collectionFilterMatch + collectionFilterMatch, + collectionFilterMode }; const useDeepSearch = datasetDeepSearch && textQueries.length > 0; const { @@ -361,4 +375,4 @@ export async function dispatchDatasetSearch( logger.error('Dataset search dispatch failed', { error }); return getNodeErrResponse({ error }); } -} +}; diff --git a/packages/service/core/workflow/dispatch/utils/runtime.ts b/packages/service/core/workflow/dispatch/utils/runtime.ts index 63a583e0efaa..bf6958436307 100644 --- a/packages/service/core/workflow/dispatch/utils/runtime.ts +++ b/packages/service/core/workflow/dispatch/utils/runtime.ts @@ -4,12 +4,14 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; +import type { ReferenceValueType } from '@fastgpt/global/core/workflow/type/io'; import type { WorkflowVariableStateLike } from '../../types/runtime'; import { getReferenceVariableValue, valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils'; import { nodeInputIsReference } from '@fastgpt/global/core/workflow/utils'; +import { formatCollectionFilterMatchParam } from '@fastgpt/global/core/dataset/workflowTagFilter'; import { replaceEditorVariable } from './replaceEditorVariable'; /** @@ -97,6 +99,46 @@ export const getWorkflowNodeRunParams = ({ } } + if ( + input.key === NodeInputKeyEnum.datasetParams && + value && + typeof value === 'object' && + !Array.isArray(value) + ) { + const datasetParams = value as Record; + value = { + ...datasetParams, + collectionFilterMatch: formatCollectionFilterMatchParam({ + value: datasetParams.collectionFilterMatch, + resolveReference: (refValue) => + getReferenceVariableValue({ + value: refValue as ReferenceValueType, + nodesMap: runtimeNodesMap, + variables: getRuntimeVariables(), + isReferenceVal: true + }) + }) + }; + } + + if (input.key === NodeInputKeyEnum.collectionFilterMatch) { + const formatted = formatCollectionFilterMatchParam({ + value, + resolveReference: (refValue) => + getReferenceVariableValue({ + value: refValue as ReferenceValueType, + nodesMap: runtimeNodesMap, + variables: getRuntimeVariables(), + isReferenceVal: true + }) + }); + if (input.canEdit && dynamicInput && params[dynamicInput.key]) { + params[dynamicInput.key][input.key] = formatted; + } + params[input.key] = formatted; + return; + } + // Dynamic input is stored in the dynamic key if (input.canEdit && dynamicInput && params[dynamicInput.key]) { params[dynamicInput.key][input.key] = valueTypeFormat(value, input.valueType); diff --git a/packages/service/package.json b/packages/service/package.json index 34a4c16f307b..fa9977becd97 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -71,6 +71,7 @@ "proxy-addr": "catalog:", "proxy-agent": "catalog:", "proxy-from-env": "^1.1.0", + "safe-regex": "^2.1.1", "turndown": "^7.1.2", "undici": "catalog:", "winston": "^3.17.0", diff --git a/packages/service/test/core/dataset/collection/tagFilter.test.ts b/packages/service/test/core/dataset/collection/tagFilter.test.ts new file mode 100644 index 000000000000..89d830ba7e25 --- /dev/null +++ b/packages/service/test/core/dataset/collection/tagFilter.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { buildCollectionListTagMatch } from '@fastgpt/service/core/dataset/collection/tagFilter'; + +describe('buildCollectionListTagMatch', () => { + it('returns empty object when no filters are selected', () => { + expect(buildCollectionListTagMatch()).toEqual({}); + expect(buildCollectionListTagMatch([])).toEqual({}); + }); + + it('uses $elemMatch for one tag and $and across tags', () => { + expect(buildCollectionListTagMatch([{ tagId: 'type', values: ['PRD'] }])).toEqual({ + tags: { + $elemMatch: { + tagId: 'type', + value: { $in: ['PRD'] } + } + } + }); + + expect( + buildCollectionListTagMatch([ + { tagId: 'type', values: ['PRD'] }, + { tagId: 'version', values: [2] } + ]) + ).toEqual({ + $and: [ + { + tags: { + $elemMatch: { + tagId: 'type', + value: { $in: ['PRD'] } + } + } + }, + { + tags: { + $elemMatch: { + tagId: 'version', + value: { $in: [2] } + } + } + } + ] + }); + }); +}); diff --git a/packages/service/test/core/dataset/delete/processor.test.ts b/packages/service/test/core/dataset/delete/processor.test.ts index 6c42178bbd82..e1b7ea85bb84 100644 --- a/packages/service/test/core/dataset/delete/processor.test.ts +++ b/packages/service/test/core/dataset/delete/processor.test.ts @@ -7,6 +7,8 @@ import { } from '@fastgpt/global/support/permission/constant'; import { datasetDeleteProcessor } from '@fastgpt/service/core/dataset/delete/processor'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema'; import { getUser } from '@test/datas/users'; import { @@ -57,6 +59,13 @@ describe('datasetDeleteProcessor', () => { name: 'retained dataset', type: DatasetTypeEnum.dataset }); + const otherTeamDataset = await MongoDataset.create({ + teamId: otherTeamUser.teamId, + tmbId: otherTeamUser.tmbId, + name: 'other team dataset', + type: DatasetTypeEnum.dataset, + deleteTime + }); const synonym = await MongoDatasetSynonym.create({ teamId: user.teamId, datasetId: childDataset._id, @@ -78,6 +87,50 @@ describe('datasetDeleteProcessor', () => { fingerprint: 'refund' }); + await MongoDatasetCollectionTags.insertOne({ + teamId: user.teamId, + datasetId: childDataset._id, + tag: 'legacy-child-tag' + }); + await MongoDatasetCollectionTagsV2.insertMany([ + { + teamId: user.teamId, + datasetId: rootDataset._id, + tag: 'root-v2-tag', + tagType: 'string' + }, + { + teamId: user.teamId, + datasetId: childDataset._id, + tag: 'child-v2-tag', + tagType: 'string' + }, + { + teamId: user.teamId, + datasetId: retainedDataset._id, + tag: 'retained-v2-tag-1', + tagType: 'string' + }, + { + teamId: user.teamId, + datasetId: retainedDataset._id, + tag: 'retained-v2-tag-2', + tagType: 'string' + }, + { + teamId: otherTeamUser.teamId, + datasetId: rootDataset._id, + tag: 'other-team-same-dataset-id', + tagType: 'string' + }, + { + teamId: otherTeamUser.teamId, + datasetId: otherTeamDataset._id, + tag: 'other-team-dataset-tag', + tagType: 'string' + } + ]); + await MongoResourcePermission.insertMany([ { teamId: user.teamId, @@ -156,6 +209,36 @@ describe('datasetDeleteProcessor', () => { await MongoDataset.countDocuments({ _id: { $in: [rootDataset._id, childDataset._id] } }) ).toBe(0); expect(await MongoDataset.countDocuments({ _id: retainedDataset._id })).toBe(1); + expect( + await MongoDatasetCollectionTags.countDocuments({ + teamId: user.teamId, + datasetId: childDataset._id + }) + ).toBe(0); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: user.teamId, + datasetId: { $in: [rootDataset._id, childDataset._id] } + }) + ).toBe(0); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: user.teamId, + datasetId: retainedDataset._id + }) + ).toBe(2); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: otherTeamUser.teamId, + datasetId: rootDataset._id + }) + ).toBe(1); + expect( + await MongoDatasetCollectionTagsV2.countDocuments({ + teamId: otherTeamUser.teamId, + datasetId: otherTeamDataset._id + }) + ).toBe(1); await expect(MongoDatasetSynonym.countDocuments({ datasetId: childDataset._id })).resolves.toBe( 0 ); diff --git a/packages/service/test/core/dataset/search/collectionFilter.test.ts b/packages/service/test/core/dataset/search/collectionFilter.test.ts new file mode 100644 index 000000000000..1f9a8fe34c12 --- /dev/null +++ b/packages/service/test/core/dataset/search/collectionFilter.test.ts @@ -0,0 +1,199 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const collectionFindMock = vi.hoisted(() => vi.fn()); +const tagFindMock = vi.hoisted(() => vi.fn()); + +vi.mock('@fastgpt/service/core/dataset/collection/schema', () => ({ + MongoDatasetCollection: { find: collectionFindMock } +})); +vi.mock('@fastgpt/service/core/dataset/tag/schemaV2', () => ({ + MongoDatasetCollectionTagsV2: { find: tagFindMock } +})); + +import { + checkValue, + filterCollectionByKeyValueTags, + filterCollectionByMetadata +} from '../../../../core/dataset/search/defaultRecall/collectionFilter'; + +const findResult = (data: unknown[]) => { + const chain = { hint: () => chain, lean: vi.fn().mockResolvedValue(data) }; + return chain; +}; + +const mockTagsAndCollections = ({ + tags = [], + collections = [] +}: { + tags?: any[]; + collections?: any[]; +}) => { + tagFindMock.mockReturnValue({ lean: vi.fn().mockResolvedValue(tags) }); + collectionFindMock.mockReturnValue(findResult(collections)); +}; + +const filterTags = (params: { $and?: any[]; $or?: any[] }) => + filterCollectionByKeyValueTags({ + $and: params.$and ?? [], + $or: params.$or ?? [], + teamId: 'team-1', + datasetIds: ['dataset-1'] + }); + +describe('filterCollectionByKeyValueTags', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockTagsAndCollections({}); + }); + + it('returns undefined without conditions', async () => { + await expect(filterTags({})).resolves.toBeUndefined(); + }); + + it.each([ + ['string', '$eq', 'Product A', 'Product A'], + ['number', '$gte', 2, 2], + ['datetime', '$lt', 1704153600000, 1704067200000] + ])('filters %s tag values', async (tagType, op, target, stored) => { + mockTagsAndCollections({ + tags: [{ _id: 'tag-1', datasetId: 'dataset-1', tag: 'field', tagType }], + collections: [ + { _id: 'match', tags: [{ tagId: 'tag-1', value: stored }] }, + { _id: 'missing', tags: [] } + ] + }); + + await expect(filterTags({ $and: [{ field: { [op]: target } }] })).resolves.toEqual(['match']); + }); + + it('requires every AND condition and one OR condition', async () => { + mockTagsAndCollections({ + tags: [ + { _id: 'product', datasetId: 'dataset-1', tag: 'product', tagType: 'string' }, + { _id: 'version', datasetId: 'dataset-1', tag: 'version', tagType: 'number' }, + { _id: 'category', datasetId: 'dataset-1', tag: 'category', tagType: 'string' } + ], + collections: [ + { + _id: 'match', + tags: [ + { tagId: 'product', value: 'A' }, + { tagId: 'version', value: 2 }, + { tagId: 'category', value: 'manual' } + ] + }, + { + _id: 'or-miss', + tags: [ + { tagId: 'product', value: 'A' }, + { tagId: 'version', value: 2 } + ] + } + ] + }); + + await expect( + filterTags({ + $and: [{ product: { $eq: 'A' } }, { version: { $gte: 2 } }], + $or: [{ category: { $eq: 'manual' } }] + }) + ).resolves.toEqual(['match']); + }); + + it('matches empty values only when the collection contains that tag', async () => { + mockTagsAndCollections({ + tags: [{ _id: 'tag-1', datasetId: 'dataset-1', tag: 'field', tagType: 'string' }], + collections: [ + { _id: 'empty', tags: [{ tagId: 'tag-1', value: '' }] }, + { _id: 'absent', tags: [] } + ] + }); + + await expect(filterTags({ $and: [{ field: { $empty: true } }] })).resolves.toEqual(['empty']); + }); + + it('rejects missing tags and treats migration metadata as unrelated to structured filtering', async () => { + await expect(filterTags({ $and: [{ missing: { $eq: 'A' } }] })).resolves.toEqual([]); + + mockTagsAndCollections({ + tags: [ + { + _id: 'carrier', + datasetId: 'dataset-1', + tag: 'default_tag', + tagType: 'array', + fromMigration: true + } + ], + collections: [{ _id: 'legacy', tags: [{ tagId: 'carrier', value: ['A'] }] }] + }); + await expect(filterTags({ $and: [{ default_tag: { $contains: 'A' } }] })).resolves.toEqual([ + 'legacy' + ]); + }); +}); + +describe('filterCollectionByMetadata', () => { + beforeEach(() => { + vi.clearAllMocks(); + (global as any).feConfigs = { isPlus: true }; + mockTagsAndCollections({ + tags: [{ _id: 'tag-1', datasetId: 'dataset-1', tag: 'field', tagType: 'string' }], + collections: [{ _id: 'match', tags: [{ tagId: 'tag-1', value: 'A' }] }] + }); + }); + afterEach(() => { + (global as any).feConfigs = {}; + }); + + it('accepts structured conditions and rejects legacy or malformed configurations', async () => { + await expect( + filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['dataset-1'], + collectionFilterMatch: JSON.stringify({ tags: { $and: [{ field: { $eq: 'A' } }] } }) + }) + ).resolves.toEqual(['match']); + + for (const value of [ + JSON.stringify({ tags: { $and: ['legacy'] } }), + JSON.stringify({ tags: { $or: [null] } }), + 'not-json{' + ]) { + await expect( + filterCollectionByMetadata({ + teamId: 'team-1', + datasetIds: ['dataset-1'], + collectionFilterMatch: value + }) + ).rejects.toBeTruthy(); + } + }); +}); + +describe('checkValue', () => { + it.each([ + ['$eq', 'Product A', 'Product A', 'string', true], + ['$contains', 'foo', 'FOOBAR', 'string', true], + ['$gte', 2, '2', 'number', true], + ['$lt', 2, 1, 'datetime', true], + ['$is', ['a', 'b'], ['b', 'a'], 'array', true], + ['$in', ['a', 'b', 'c'], ['a', 'b'], 'array', true], + ['$empty', true, undefined, 'array', true], + ['$unsupported', 'x', 'x', 'string', false] + ])('compares %s for %s values', (op, target, stored, tagType, expected) => { + expect(checkValue(op as any, target, stored as any, tagType)).toBe(expected); + }); + + it('rejects invalid value shapes and unsafe regex patterns', () => { + expect(checkValue('$eq', null, 'x', 'string')).toBe(false); + expect(checkValue('$eq', 2, Number.NaN, 'number')).toBe(false); + expect(checkValue('$is', ['a'], 'not-array', 'array')).toBe(false); + expect(checkValue('$regex', 'foo', 'foobar', 'string')).toBe(true); + expect(checkValue('$regex', '[invalid', 'foobar', 'string')).toBe(false); + expect(checkValue('$regex', '(a+)+$', 'aaaaab', 'string')).toBe(false); + expect(checkValue('$regex', '(a|aa)+$', 'aaaaab', 'string')).toBe(false); + expect(checkValue('$regex', 'a'.repeat(65), 'aaaaa', 'string')).toBe(false); + expect(checkValue('$regex', 'a', 'x'.repeat(257), 'string')).toBe(false); + }); +}); diff --git a/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts b/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts new file mode 100644 index 000000000000..2e533b899b90 --- /dev/null +++ b/packages/service/test/core/dataset/search/collectionTagIndex.benchmark.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { getRootUser } from '@test/datas/users'; + +const COLLECTION_COUNT = 5_000; +const HIT_EVERY = 20; +const TAG_INDEX_KEY = { teamId: 1, datasetId: 1, 'tags.tagId': 1 } as const; +const TAG_INDEX_NAME = 'teamId_1_datasetId_1_tags.tagId_1'; + +const findIndexName = (stage: unknown): string | undefined => { + if (!stage || typeof stage !== 'object') return; + if (Reflect.get(stage, 'stage') === 'IXSCAN') return Reflect.get(stage, 'indexName'); + + const inputStage = findIndexName(Reflect.get(stage, 'inputStage')); + if (inputStage) return inputStage; + + const inputStages = Reflect.get(stage, 'inputStages'); + if (!Array.isArray(inputStages)) return; + return inputStages.map(findIndexName).find(Boolean); +}; + +/** 验证新标签查询使用当前 Schema 声明的复合索引，不比较易受环境影响的墙钟时间。 */ +describe('dataset collection tag index', () => { + it('uses tags.tagId index and scans only matching collections', async () => { + const root = await getRootUser(); + const datasetId = new Types.ObjectId(); + const targetTagId = String(new Types.ObjectId()); + const otherTagId = String(new Types.ObjectId()); + + await MongoDatasetCollection.createIndexes({ background: true }); + for (let start = 0; start < COLLECTION_COUNT; start += 1_000) { + await MongoDatasetCollection.insertMany( + Array.from({ length: Math.min(1_000, COLLECTION_COUNT - start) }, (_, offset) => { + const index = start + offset; + return { + teamId: root.teamId, + tmbId: root.tmbId, + datasetId, + type: 'file', + name: `collection-${index}`, + tags: [{ tagId: index % HIT_EVERY === 0 ? targetTagId : otherTagId, value: index }] + }; + }) + ); + } + + const explain = await MongoDatasetCollection.find( + { teamId: root.teamId, datasetId, 'tags.tagId': targetTagId }, + '_id' + ) + .hint(TAG_INDEX_KEY) + .explain('executionStats') + .then((result: any) => result[0] ?? result); + + expect(findIndexName(explain.queryPlanner?.winningPlan)).toBe(TAG_INDEX_NAME); + expect(explain.executionStats?.nReturned).toBe(COLLECTION_COUNT / HIT_EVERY); + expect(explain.executionStats?.totalDocsExamined).toBeLessThanOrEqual( + COLLECTION_COUNT / HIT_EVERY + ); + }, 300_000); +}); diff --git a/packages/service/test/core/dataset/search/legacy/collectionFilter.test.ts b/packages/service/test/core/dataset/search/legacy/collectionFilter.test.ts new file mode 100644 index 000000000000..91f129713d7c --- /dev/null +++ b/packages/service/test/core/dataset/search/legacy/collectionFilter.test.ts @@ -0,0 +1,139 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { filterLegacyCollectionByMetadata } from '@fastgpt/service/core/dataset/search/defaultRecall/legacy/collectionFilter'; + +const teamId = new Types.ObjectId(); +const tmbId = new Types.ObjectId(); +const datasetA = new Types.ObjectId(); +const datasetB = new Types.ObjectId(); + +const createCollection = async ({ + datasetId, + tags +}: { + datasetId: Types.ObjectId; + tags: unknown[]; +}) => + MongoDatasetCollection.collection.insertOne({ + teamId, + tmbId, + datasetId, + name: 'test', + type: 'file', + tags + }); + +describe('filterLegacyCollectionByMetadata', () => { + beforeEach(async () => { + global.feConfigs = { ...global.feConfigs, isPlus: true }; + await Promise.all([ + MongoDatasetCollection.collection.deleteMany({ teamId }), + MongoDatasetCollectionTagsV2.collection.deleteMany({ teamId }) + ]); + }); + + afterEach(() => { + global.feConfigs = { ...global.feConfigs, isPlus: false }; + }); + + it('uses AND exclusively when AND and OR are both present', async () => { + const carrier = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId: datasetA, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + }); + const [andCollection, orCollection] = await Promise.all([ + createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrier._id), value: ['A'] }] + }), + createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrier._id), value: ['B'] }] + }) + ]); + + const result = await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetA)], + collectionFilterMatch: JSON.stringify({ tags: { $and: ['A'], $or: ['B'] } }) + }); + + expect(result).toEqual([String(andCollection.insertedId)]); + expect(result).not.toContain(String(orCollection.insertedId)); + }); + + it('matches null only against truly empty tags', async () => { + const carrier = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId: datasetA, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + }); + const empty = await createCollection({ datasetId: datasetA, tags: [] }); + await createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrier._id), value: [] }] + }); + + const result = await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetA)], + collectionFilterMatch: JSON.stringify({ tags: { $and: [null] } }) + }); + expect(result).toEqual([String(empty.insertedId)]); + }); + + it('uses each dataset carrier independently and does not widen when one is missing', async () => { + const [carrierA, carrierB] = await MongoDatasetCollectionTagsV2.create([ + { + teamId, + datasetId: datasetA, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + }, + { + teamId, + datasetId: datasetB, + tag: 'default_tag', + tagType: 'array', + fromMigration: true + } + ]); + const [collectionA, collectionB] = await Promise.all([ + createCollection({ + datasetId: datasetA, + tags: [{ tagId: String(carrierA._id), value: ['A'] }] + }), + createCollection({ + datasetId: datasetB, + tags: [{ tagId: String(carrierB._id), value: ['A'] }] + }) + ]); + + expect( + await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetA), String(datasetB)], + collectionFilterMatch: JSON.stringify({ tags: { $or: ['A'] } }) + }) + ).toEqual( + expect.arrayContaining([String(collectionA.insertedId), String(collectionB.insertedId)]) + ); + + await MongoDatasetCollectionTagsV2.deleteOne({ _id: carrierB._id }); + expect( + await filterLegacyCollectionByMetadata({ + teamId: String(teamId), + datasetIds: [String(datasetB)], + collectionFilterMatch: JSON.stringify({ tags: { $or: ['A'] } }) + }) + ).toEqual([]); + }); +}); diff --git a/packages/service/test/core/dataset/search/multiQueryRecall.test.ts b/packages/service/test/core/dataset/search/multiQueryRecall.test.ts new file mode 100644 index 000000000000..101b2e795a75 --- /dev/null +++ b/packages/service/test/core/dataset/search/multiQueryRecall.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + getForbidCollectionIdListMock, + filterCollectionByMetadataMock, + filterLegacyCollectionByMetadataMock, + embeddingRecallMock, + fullTextRecallMock +} = vi.hoisted(() => ({ + getForbidCollectionIdListMock: vi.fn(), + filterCollectionByMetadataMock: vi.fn(), + filterLegacyCollectionByMetadataMock: vi.fn(), + embeddingRecallMock: vi.fn(), + fullTextRecallMock: vi.fn() +})); + +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/collectionFilter', () => ({ + getForbidCollectionIdList: getForbidCollectionIdListMock, + filterCollectionByMetadata: filterCollectionByMetadataMock +})); +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/legacy/collectionFilter', () => ({ + filterLegacyCollectionByMetadata: filterLegacyCollectionByMetadataMock +})); +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/embeddingRecall', () => ({ + embeddingRecall: embeddingRecallMock +})); +vi.mock('@fastgpt/service/core/dataset/search/defaultRecall/fullTextRecall', () => ({ + fullTextRecall: fullTextRecallMock +})); + +import { multiQueryRecall } from '@fastgpt/service/core/dataset/search/defaultRecall/multiQueryRecall'; + +const baseParams = { + teamId: 'team', + datasetIds: ['dataset'], + model: { model: 'embedding' } as any, + imageQueries: [], + collectionFilterMatch: '{}', + embeddingLimit: 10, + fullTextLimit: 10, + textQueries: ['query'], + imageCaptionQueries: [] +}; + +describe('multiQueryRecall collection filter routing', () => { + beforeEach(() => { + vi.clearAllMocks(); + getForbidCollectionIdListMock.mockResolvedValue([]); + filterCollectionByMetadataMock.mockResolvedValue([]); + filterLegacyCollectionByMetadataMock.mockResolvedValue([]); + embeddingRecallMock.mockResolvedValue({ + tokens: 0, + textEmbeddingRecallResults: [], + imageCaptionEmbeddingRecallResults: [], + imageVectorRecallResults: [] + }); + fullTextRecallMock.mockResolvedValue({ + textFullTextRecallResults: [], + imageCaptionFullTextRecallResults: [] + }); + }); + + it.each([ + ['legacy', filterLegacyCollectionByMetadataMock, filterCollectionByMetadataMock], + ['structured', filterCollectionByMetadataMock, filterLegacyCollectionByMetadataMock] + ] as const)('calls only the %s filter', async (mode, expected, unexpected) => { + await multiQueryRecall({ ...baseParams, collectionFilterMode: mode }); + expect(expected).toHaveBeenCalledOnce(); + expect(unexpected).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/service/test/core/dataset/utils.test.ts b/packages/service/test/core/dataset/utils.test.ts index 1d9e6c054be6..1198873241a2 100644 --- a/packages/service/test/core/dataset/utils.test.ts +++ b/packages/service/test/core/dataset/utils.test.ts @@ -12,7 +12,12 @@ import { matchDatasetDataMarkdownImageUrls, uniqueDatasetDataMarkdownImageUrls } from '@fastgpt/service/core/dataset/data/utils'; -import { getTrainingModeByCollection } from '@fastgpt/service/core/dataset/collection/utils'; +import { + createOrGetCollectionTags, + getTrainingModeByCollection, + validateAndNormalizeTagValue, + validateDatasetTagValue +} from '@fastgpt/service/core/dataset/collection/utils'; import { DatasetCollectionDataProcessModeEnum, TrainingModeEnum @@ -22,6 +27,7 @@ import type { EmbeddingSystemModelDataType, LLMSystemModelDataType } from '@fastgpt/global/core/ai/model.schema'; +import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; const mockCreateS3DownloadAccessUrls = vi.hoisted(() => vi.fn(async (params: Array<{ objectKey: string }>) => @@ -31,6 +37,20 @@ const mockCreateS3DownloadAccessUrls = vi.hoisted(() => ) ); +const mockMongoDatasetCollectionTagsFind = vi.hoisted(() => vi.fn()); +const mockMongoDatasetCollectionTagsFindOne = vi.hoisted(() => vi.fn()); +const mockMongoDatasetCollectionTagsCreate = vi.hoisted(() => vi.fn()); + +vi.mock('@fastgpt/service/core/dataset/tag/schemaV2', () => ({ + MongoDatasetCollectionTagsV2: { + find: mockMongoDatasetCollectionTagsFind, + findOne: (...args: unknown[]) => ({ + lean: vi.fn().mockImplementation(() => mockMongoDatasetCollectionTagsFindOne(...args)) + }), + create: mockMongoDatasetCollectionTagsCreate + } +})); + vi.mock('@fastgpt/service/common/s3/utils', () => ({ isS3ObjectKey: vi.fn((key: string, source: string) => { if (!key) return false; @@ -52,6 +72,10 @@ vi.mock('@fastgpt/service/common/s3/contracts/type', () => ({ } })); +vi.mock('@fastgpt/service/core/dataset/tag/schema', () => ({ + MongoDatasetCollectionTags: {} +})); + describe('replaceS3KeyToPreviewUrl', () => { const expiredTime = new Date('2025-12-31'); @@ -688,3 +712,129 @@ describe('getDatasetImageIndexCapability', () => { expect(result.availableVlmModel?.model).toBe('dataset-vlm-model'); }); }); + +describe('validateDatasetTagValue', () => { + it.each([ + ['string', 'value', 'value', undefined], + ['array', ['a'], ['a'], undefined], + ['number', '1.25', 1.25, undefined], + ['datetime', '1704067200000', 1704067200000, undefined], + ['string', 1, 1, DatasetErrEnum.tagValueInvalid], + ['array', ['a'.repeat(257)], ['a'.repeat(257)], DatasetErrEnum.arrayTagValueInvalid], + ['number', 'abc', 'abc', DatasetErrEnum.tagValueInvalid], + ['datetime', Number.MAX_VALUE, Number.MAX_VALUE, DatasetErrEnum.tagValueDatetimeInvalid] + ])('validates and normalizes %s values', (tagType, value, normalized, error) => { + expect(validateAndNormalizeTagValue({ tagType: tagType as any, value: value as any })).toEqual({ + value: normalized, + ...(error ? { error } : {}) + }); + expect(validateDatasetTagValue({ tagType: tagType as any, value: value as any })).toBe(error); + }); +}); + +describe('createOrGetCollectionTags', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue(null); + mockMongoDatasetCollectionTagsCreate.mockResolvedValue([]); + }); + + it('returns without database work when tags are absent or empty', async () => { + await expect( + createOrGetCollectionTags({ tags: undefined, datasetId: 'ds-1', teamId: 'team-1' }) + ).resolves.toBeUndefined(); + await expect( + createOrGetCollectionTags({ tags: [], datasetId: 'ds-1', teamId: 'team-1' }) + ).resolves.toEqual([]); + expect(mockMongoDatasetCollectionTagsFind).not.toHaveBeenCalled(); + }); + + it('creates the migration carrier on demand for legacy string names', async () => { + mockMongoDatasetCollectionTagsCreate.mockResolvedValue([ + { + _id: 'default-tag-id', + toObject: () => ({ _id: 'default-tag-id' }) + } + ]); + + const result = await createOrGetCollectionTags({ + tags: ['safety'], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([{ tagId: 'default-tag-id', value: ['safety'] }]); + expect(mockMongoDatasetCollectionTagsCreate).toHaveBeenCalledWith( + [expect.objectContaining({ tag: 'default_tag', fromMigration: true })], + expect.any(Object) + ); + }); + + it('handles mixed legacy and typed inputs with normalization and deduplication', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id', tag: 'score', tagType: 'number' }]) + }); + mockMongoDatasetCollectionTagsFindOne.mockResolvedValue({ _id: 'default-tag-id' }); + + const result = await createOrGetCollectionTags({ + tags: [' legacy ', 'legacy', { tag: ' score ', value: '2' }], + datasetId: 'ds-1', + teamId: 'team-1' + }); + + expect(result).toEqual([ + { tagId: 'default-tag-id', value: ['legacy'] }, + { tagId: 'tag-id', value: 2 } + ]); + }); + + it('handles an object input named default_tag as a normal typed tag', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi + .fn() + .mockResolvedValue([ + { _id: 'ordinary-default-tag-id', tag: 'default_tag', tagType: 'string' } + ]) + }); + + await expect( + createOrGetCollectionTags({ + tags: [{ tag: 'default_tag', value: 'ordinary value' }], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).resolves.toEqual([{ tagId: 'ordinary-default-tag-id', value: 'ordinary value' }]); + expect(mockMongoDatasetCollectionTagsFindOne).not.toHaveBeenCalled(); + expect(mockMongoDatasetCollectionTagsCreate).not.toHaveBeenCalled(); + }); + + it('rejects missing tags and conflicting duplicate values', async () => { + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([]) + }); + await expect( + createOrGetCollectionTags({ + tags: [{ tag: 'missing', value: 'A' }], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.tagNotExist); + + mockMongoDatasetCollectionTagsFind.mockReturnValue({ + lean: vi.fn().mockResolvedValue([{ _id: 'tag-id', tag: 'tag', tagType: 'string' }]) + }); + await expect( + createOrGetCollectionTags({ + tags: [ + { tag: 'tag', value: 'A' }, + { tag: 'tag', value: 'B' } + ], + datasetId: 'ds-1', + teamId: 'team-1' + }) + ).rejects.toBe(DatasetErrEnum.tagValueInvalid); + }); +}); diff --git a/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts b/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts index cd159e260cc9..5f52a9773d79 100644 --- a/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts +++ b/packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DatasetSearchModeEnum, SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; const { countPromptTokensMock, @@ -175,7 +176,11 @@ describe('dispatchAgentDatasetSearch', () => { embeddingWeight: 0.5, usingReRank: false, rerankWeight: 0.5, - datasetSearchUsingExtensionQuery: true + datasetSearchUsingExtensionQuery: true, + collectionFilterMatch: { + logic: 'AND', + conditions: [{ tag: 'price', tagType: 'number', op: '$gte', value: 10 }] + } } as any }); @@ -185,7 +190,11 @@ describe('dispatchAgentDatasetSearch', () => { expect(result.nodeResponse?.datasetQueries).toEqual(['origin']); expect(defaultSearchDatasetDataMock).toHaveBeenCalledWith( expect.objectContaining({ - textQueries: ['origin'] + textQueries: ['origin'], + collectionFilterMatch: JSON.stringify({ + tags: { $and: [{ price: { $gte: 10 } }] } + }), + collectionFilterMode: DatasetTagFilterVersionEnum.structured }) ); expect(result.nodeResponse?.childrenResponses).toEqual([ diff --git a/packages/service/test/core/workflow/dispatch/dataset/search.test.ts b/packages/service/test/core/workflow/dispatch/dataset/search.test.ts index 55b8c2e115f0..727b4914e2dd 100644 --- a/packages/service/test/core/workflow/dispatch/dataset/search.test.ts +++ b/packages/service/test/core/workflow/dispatch/dataset/search.test.ts @@ -100,6 +100,65 @@ describe('dispatchDatasetSearch', () => { ); }); + it('uses explicit versions and sends unconfigured historical nodes to structured filtering', async () => { + defaultSearchDatasetDataMock.mockResolvedValue({ + searchRes: [], + embeddingTokens: 0, + reRankInputTokens: 0, + usingSimilarityFilter: false, + usingReRank: false + }); + const props = { + runningAppInfo: { teamId: 'team_1' }, + runningUserInfo: { tmbId: 'tmb_1' }, + externalProvider: {}, + histories: [], + node: { name: 'Dataset Search' }, + params: { + datasets: [{ datasetId: 'dataset_1' }], + similarity: 0.4, + limit: 5000, + userChatInput: 'query', + searchMode: DatasetSearchModeEnum.embedding, + usingReRank: false, + datasetSearchUsingExtensionQuery: false, + collectionFilterMatch: '{"tags":{"$and":["legacy"]}}' + }, + usagePush: usagePushMock + } as any; + + await dispatchDatasetSearch(props); + await dispatchDatasetSearch({ + ...props, + params: { ...props.params, collectionFilterVersion: 'structured' } + }); + await dispatchDatasetSearch({ + ...props, + params: { ...props.params, collectionFilterVersion: 'unknown' } + }); + await dispatchDatasetSearch({ + ...props, + params: { + ...props.params, + collectionFilterVersion: undefined, + collectionFilterMatch: undefined + } + }); + + expect(defaultSearchDatasetDataMock.mock.calls[0][0]).toMatchObject({ + collectionFilterMode: 'legacy' + }); + expect(defaultSearchDatasetDataMock.mock.calls[1][0]).toMatchObject({ + collectionFilterMode: 'structured' + }); + expect(defaultSearchDatasetDataMock.mock.calls[2][0]).toMatchObject({ + collectionFilterMode: 'legacy' + }); + expect(defaultSearchDatasetDataMock.mock.calls[3][0]).toMatchObject({ + collectionFilterMode: 'structured' + }); + }); + it('adds query extension as a child node response of dataset search', async () => { defaultSearchDatasetDataMock.mockResolvedValue({ searchRes: [ diff --git a/packages/service/test/core/workflow/dispatch/index.test.ts b/packages/service/test/core/workflow/dispatch/index.test.ts index e57a6fba504e..6fdaf9d7e415 100644 --- a/packages/service/test/core/workflow/dispatch/index.test.ts +++ b/packages/service/test/core/workflow/dispatch/index.test.ts @@ -715,6 +715,81 @@ describe('getWorkflowNodeRunParams', () => { expect(params[NodeInputKeyEnum.addInputParam]).toEqual({ dynamicName: 'Ada' }); expect(params.dynamicName).toBe('Ada'); }); + + it('标签过滤条件行会解析行内引用并编成 tags JSON', () => { + const variableState = createVariableState({ price: 18 }); + const node = createNode('search', FlowNodeTypeEnum.datasetSearchNode); + node.inputs = [ + { + key: NodeInputKeyEnum.collectionFilterMatch, + label: '', + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + valueType: WorkflowIOValueTypeEnum.string, + value: { + logic: 'AND', + conditions: [ + { + tag: 'price', + tagType: 'number', + op: '$gte', + valueMode: 'reference', + value: [VARIABLE_NODE_ID, 'price'] + } + ] + } + } + ]; + + const params = getWorkflowNodeRunParams({ + node, + runtimeNodesMap: new Map(), + variableState: variableState.state + }); + + expect(params[NodeInputKeyEnum.collectionFilterMatch]).toBe( + JSON.stringify({ tags: { $and: [{ price: { $gte: 18 } }] } }) + ); + expect(variableState.getToRuntimeRecordCount()).toBe(1); + }); + + it('Agent 嵌套 datasetParams 会解析标签过滤行内引用', () => { + const variableState = createVariableState({ price: 18 }); + const node = createNode('agent', FlowNodeTypeEnum.agent); + node.inputs = [ + { + key: NodeInputKeyEnum.datasetParams, + label: '', + renderTypeList: [FlowNodeInputTypeEnum.hidden], + valueType: WorkflowIOValueTypeEnum.object, + value: { + datasets: [{ datasetId: 'dataset-1' }], + collectionFilterMatch: { + logic: 'AND', + conditions: [ + { + tag: 'price', + tagType: 'number', + op: '$gte', + valueMode: 'reference', + value: [VARIABLE_NODE_ID, 'price'] + } + ] + } + } + } + ]; + + const params = getWorkflowNodeRunParams({ + node, + runtimeNodesMap: new Map(), + variableState: variableState.state + }); + + expect(params[NodeInputKeyEnum.datasetParams]).toMatchObject({ + datasets: [{ datasetId: 'dataset-1' }], + collectionFilterMatch: JSON.stringify({ tags: { $and: [{ price: { $gte: 18 } }] } }) + }); + }); }); describe('runWorkflow catchError', () => { diff --git a/packages/service/types/safe-regex.d.ts b/packages/service/types/safe-regex.d.ts new file mode 100644 index 000000000000..2be2e2dbc89c --- /dev/null +++ b/packages/service/types/safe-regex.d.ts @@ -0,0 +1,8 @@ +declare module 'safe-regex' { + interface SafeRegexOptions { + /** 解析的 AST 节点数上限，超过即视为不安全（默认 25） */ + limit?: number; + } + function safeRegex(re: RegExp | string, opts?: SafeRegexOptions): boolean; + export default safeRegex; +} diff --git a/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx new file mode 100644 index 000000000000..75301123eaf4 --- /dev/null +++ b/packages/web/components/common/DateTimePicker/SingleDateTimePicker.tsx @@ -0,0 +1,421 @@ +import React, { useState } from 'react'; +import type { FlexProps } from '@chakra-ui/react'; +import { + Box, + Button, + Flex, + HStack, + Input, + Popover, + PopoverContent, + PopoverTrigger, + Portal, + useDisclosure +} from '@chakra-ui/react'; +import { DayPicker } from 'react-day-picker'; +import 'react-day-picker/dist/style.css'; +import { enUS } from 'date-fns/locale/en-US'; +import { ko } from 'date-fns/locale/ko'; +import { zhCN } from 'date-fns/locale/zh-CN'; +import { zhTW } from 'date-fns/locale/zh-TW'; +import type { Locale } from 'date-fns'; +import { addMonths, format, isValid, parse } from 'date-fns'; +import { useTranslation } from 'next-i18next'; +import { LangEnum } from '@fastgpt/global/common/i18n/type'; +import MyIcon from '../Icon'; + +const DATE_INPUT_FORMAT = 'yyyy-MM-dd'; +const TIME_INPUT_FORMAT = 'HH:mm'; + +const DATE_FNS_LOCALE_BY_LANG: Record = { + [LangEnum.zh_CN]: zhCN, + [LangEnum.zh_Hant]: zhTW, + [LangEnum.en]: enUS, + [LangEnum.ko_KR]: ko +}; + +type SingleDateTimePickerProps = { + value?: Date | number; + onChange?: (val: number) => void; + placeholder?: string; + isDisabled?: boolean; + locale?: Locale; + hideCalendarIcon?: boolean; +} & Omit; + +/** 将时间戳或 Date 转为有效 Date；空值或非法日期返回 undefined。 */ +const toValidDate = (value?: Date | number) => { + if (value == null) return undefined; + const date = typeof value === 'number' ? new Date(value) : value; + return isValid(date) ? date : undefined; +}; + +/** 仅在输入完整且合法的 yyyy-MM-dd 时解析为 Date，避免半成品输入把日历跳走。 */ +const parseYmdInput = (text: string) => { + if (text.length !== 10) return undefined; + const parsed = parse(text, DATE_INPUT_FORMAT, new Date()); + return isValid(parsed) ? parsed : undefined; +}; + +/** + * 单日期时间选择器：基于 react-day-picker 二次封装。 + * 打开弹窗时用当前 value 初始化草稿；确认时把日期输入与时间输入合并成时间戳。 + */ +export const SingleDateTimePicker = ({ + value, + onChange, + placeholder, + isDisabled, + locale, + hideCalendarIcon, + ...triggerProps +}: SingleDateTimePickerProps) => { + const { t, i18n } = useTranslation(); + const resolvedLocale = locale ?? DATE_FNS_LOCALE_BY_LANG[i18n.language] ?? zhCN; + const weekStartsOn = resolvedLocale.options?.weekStartsOn ?? 0; + const { isOpen, onOpen, onClose } = useDisclosure(); + + const selectedDate = toValidDate(value); + const [tempDate, setTempDate] = useState(() => selectedDate ?? new Date()); + const [tempMonth, setTempMonth] = useState(() => selectedDate ?? new Date()); + const [dateInputText, setDateInputText] = useState(() => + format(selectedDate ?? new Date(), DATE_INPUT_FORMAT) + ); + const [timeInputText, setTimeInputText] = useState(() => + format(selectedDate ?? new Date(), TIME_INPUT_FORMAT) + ); + + const displayText = selectedDate + ? format(selectedDate, `${DATE_INPUT_FORMAT} ${TIME_INPUT_FORMAT}`) + : ''; + + const applyDraftDate = (date: Date) => { + setTempDate(date); + setTempMonth(date); + setDateInputText(format(date, DATE_INPUT_FORMAT)); + }; + + const handleOpen = () => { + if (isDisabled) return; + const date = toValidDate(value) ?? new Date(); + applyDraftDate(date); + setTimeInputText(format(date, TIME_INPUT_FORMAT)); + onOpen(); + }; + + const handleConfirm = () => { + const result = new Date(parseYmdInput(dateInputText) ?? tempDate); + const [hours = '0', minutes = '0'] = timeInputText.split(':'); + result.setHours(Number(hours), Number(minutes), 0, 0); + onChange?.(result.getTime()); + onClose(); + }; + + return ( + + + + + {displayText || placeholder || t('common:datetime_picker.placeholder')} + + {!hideCalendarIcon && ( + + )} + + + + + + + setTempMonth((prev) => addMonths(prev, -1))} + > + + + + + {format(tempMonth, 'LLLL yyyy', { locale: resolvedLocale })} + + + setTempMonth((prev) => addMonths(prev, 1))} + > + + + + + + { + if (date) applyDraftDate(date); + }} + components={{ + Nav: () => <>, + MonthCaption: () => <> + }} + formatters={{ + formatWeekdayName: (date) => format(date, 'EEEEEE', { locale: resolvedLocale }) + }} + /> + + + + + { + const text = e.target.value; + setDateInputText(text); + const parsed = parseYmdInput(text); + if (parsed) { + setTempDate(parsed); + setTempMonth(parsed); + } + }} + _focus={{ borderColor: 'primary.600', boxShadow: 'focus' }} + /> + setTimeInputText(e.target.value)} + _focus={{ borderColor: 'primary.600', boxShadow: 'focus' }} + /> + + + + + + + + + + + ); +}; + +export default SingleDateTimePicker; diff --git a/packages/web/components/common/DateTimePicker/index.tsx b/packages/web/components/common/DateTimePicker/index.tsx index d84811030afc..22907e2279a8 100644 --- a/packages/web/components/common/DateTimePicker/index.tsx +++ b/packages/web/components/common/DateTimePicker/index.tsx @@ -7,6 +7,8 @@ import 'react-day-picker/dist/style.css'; import { zhCN } from 'date-fns/locale/zh-CN'; import MyIcon from '../Icon'; +export * from './SingleDateTimePicker'; + const DateTimePicker = ({ onChange, popPosition = 'bottom', @@ -25,16 +27,13 @@ const DateTimePicker = ({ } & Omit) => { const containerRef = useRef(null); const popoverRef = useRef(null); - const [selectedDate, setSelectedDate] = useState( + const [internalSelectedDate, setInternalSelectedDate] = useState( selectedDateTime || defaultDate ); + const selectedDate = selectedDateTime ?? internalSelectedDate; const [showSelected, setShowSelected] = useState(false); const [position, setPosition] = useState({ top: 0, left: 0 }); - useEffect(() => { - setSelectedDate(selectedDateTime); - }, [selectedDateTime]); - useEffect(() => { if (!showSelected) return; const updatePosition = () => { @@ -169,7 +168,7 @@ const DateTimePicker = ({ selected={selectedDate} disabled={disabled} onSelect={(date) => { - setSelectedDate(date); + setInternalSelectedDate(date); onChange?.(date); setShowSelected(false); }} diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 90cae73dd5ab..1d4fb6dc127c 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -24,7 +24,12 @@ export const iconPaths = { 'common/backFill': () => import('./icons/common/backFill.svg'), 'common/backLight': () => import('./icons/common/backLight.svg'), 'common/billing': () => import('./icons/common/billing.svg'), + 'common/calendar': () => import('./icons/common/calendar.svg'), 'common/check': () => import('./icons/common/check.svg'), + 'common/checkSquareBroken': () => import('./icons/common/checkSquareBroken.svg'), + 'common/checkSquareBrokenPrimary': () => import('./icons/common/checkSquareBrokenPrimary.svg'), + 'common/checkSquareBrokenPrimaryHover': () => + import('./icons/common/checkSquareBrokenPrimaryHover.svg'), 'common/clearLight': () => import('./icons/common/clearLight.svg'), 'common/closeLight': () => import('./icons/common/closeLight.svg'), 'common/confirm/commonTip': () => import('./icons/common/confirm/commonTip.svg'), @@ -39,16 +44,17 @@ export const iconPaths = { 'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'), 'common/download': () => import('./icons/common/download.svg'), 'common/downloadLine': () => import('./icons/common/downloadLine.svg'), + 'common/edit-filled': () => import('./icons/common/edit-filled.svg'), 'common/edit': () => import('./icons/common/edit.svg'), 'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'), 'common/ellipsis': () => import('./icons/common/ellipsis.svg'), 'common/enable': () => import('./icons/common/enable.svg'), - 'common/filter': () => import('./icons/common/filter.svg'), 'common/error': () => import('./icons/common/error.svg'), 'common/errorFill': () => import('./icons/common/errorFill.svg'), 'common/exclamationMark': () => import('./icons/common/exclamationMark.svg'), 'common/file/move': () => import('./icons/common/file/move.svg'), 'common/fileNotFound': () => import('./icons/common/fileNotFound.svg'), + 'common/filter': () => import('./icons/common/filter.svg'), 'common/first_page': () => import('./icons/common/first_page.svg'), 'common/folderFill': () => import('./icons/common/folderFill.svg'), 'common/folderImport': () => import('./icons/common/folderImport.svg'), @@ -133,8 +139,6 @@ export const iconPaths = { 'core/app/inputGuides': () => import('./icons/core/app/inputGuides.svg'), 'core/app/logsLight': () => import('./icons/core/app/logsLight.svg'), 'core/app/markLight': () => import('./icons/core/app/markLight.svg'), - 'core/app/workflowToolbarAdd': () => import('./icons/core/app/workflowToolbarAdd.svg'), - 'core/app/workflowToolbarSearch': () => import('./icons/core/app/workflowToolbarSearch.svg'), 'core/app/publish/lark': () => import('./icons/core/app/publish/lark.svg'), 'core/app/publish/offiaccount': () => import('./icons/core/app/publish/offiaccount.svg'), 'core/app/publish/wechat': () => import('./icons/core/app/publish/wechat.svg'), @@ -189,6 +193,8 @@ export const iconPaths = { 'core/app/variable/input': () => import('./icons/core/app/variable/input.svg'), 'core/app/workflow/checkPendingImprove': () => import('./icons/core/app/workflow/checkPendingImprove.svg'), + 'core/app/workflowToolbarAdd': () => import('./icons/core/app/workflowToolbarAdd.svg'), + 'core/app/workflowToolbarSearch': () => import('./icons/core/app/workflowToolbarSearch.svg'), 'core/chat/QGFill': () => import('./icons/core/chat/QGFill.svg'), 'core/chat/backText': () => import('./icons/core/chat/backText.svg'), 'core/chat/chatFill': () => import('./icons/core/chat/chatFill.svg'), @@ -198,14 +204,14 @@ export const iconPaths = { 'core/chat/chevronRight': () => import('./icons/core/chat/chevronRight.svg'), 'core/chat/chevronSelector': () => import('./icons/core/chat/chevronSelector.svg'), 'core/chat/chevronUp': () => import('./icons/core/chat/chevronUp.svg'), - 'core/chat/feedback/badLight': () => import('./icons/core/chat/feedback/badLight.svg'), - 'core/chat/feedback/goodLight': () => import('./icons/core/chat/feedback/goodLight.svg'), 'core/chat/deepThinking': () => import('./icons/core/chat/deepThinking.svg'), 'core/chat/dotsHorizontal': () => import('./icons/core/chat/dotsHorizontal.svg'), + 'core/chat/feedback/badLight': () => import('./icons/core/chat/feedback/badLight.svg'), + 'core/chat/feedback/goodLight': () => import('./icons/core/chat/feedback/goodLight.svg'), 'core/chat/fileDownload': () => import('./icons/core/chat/fileDownload.svg'), 'core/chat/fileSelect': () => import('./icons/core/chat/fileSelect.svg'), - 'core/chat/monitor': () => import('./icons/core/chat/monitor.svg'), 'core/chat/markdown': () => import('./icons/core/chat/markdown.svg'), + 'core/chat/monitor': () => import('./icons/core/chat/monitor.svg'), 'core/chat/quoteFill': () => import('./icons/core/chat/quoteFill.svg'), 'core/chat/recordFill': () => import('./icons/core/chat/recordFill.svg'), 'core/chat/sendFill': () => import('./icons/core/chat/sendFill.svg'), @@ -266,9 +272,9 @@ export const iconPaths = { 'core/workflow/debugResult': () => import('./icons/core/workflow/debugResult.svg'), 'core/workflow/edgeArrow': () => import('./icons/core/workflow/edgeArrow.svg'), 'core/workflow/edgeArrowBold': () => import('./icons/core/workflow/edgeArrowBold.svg'), - 'core/workflow/inputType/array': () => import('./icons/core/workflow/inputType/array.svg'), 'core/workflow/inputType/agentGenerated': () => import('./icons/core/workflow/inputType/agentGenerated.svg'), + 'core/workflow/inputType/array': () => import('./icons/core/workflow/inputType/array.svg'), 'core/workflow/inputType/conditional': () => import('./icons/core/workflow/inputType/conditional.svg'), 'core/workflow/inputType/customVariable': () => @@ -430,7 +436,6 @@ export const iconPaths = { empty: () => import('./icons/empty.svg'), export: () => import('./icons/export.svg'), feedback: () => import('./icons/feedback.svg'), - 'common/edit-filled': () => import('./icons/common/edit-filled.svg'), 'file/fill/audio': () => import('./icons/file/fill/audio.svg'), 'file/fill/csv': () => import('./icons/file/fill/csv.svg'), 'file/fill/doc': () => import('./icons/file/fill/doc.svg'), @@ -458,7 +463,10 @@ export const iconPaths = { loading: () => import('./icons/loading.svg'), 'math/divide': () => import('./icons/math/divide.svg'), 'math/equal': () => import('./icons/math/equal.svg'), + 'math/greater': () => import('./icons/math/greater.svg'), + 'math/greaterEqual': () => import('./icons/math/greaterEqual.svg'), 'math/minus': () => import('./icons/math/minus.svg'), + 'math/notEqual': () => import('./icons/math/notEqual.svg'), 'math/plus': () => import('./icons/math/plus.svg'), 'math/times': () => import('./icons/math/times.svg'), mcp: () => import('./icons/mcp.svg'), diff --git a/packages/web/components/common/Icon/icons/common/calendar.svg b/packages/web/components/common/Icon/icons/common/calendar.svg new file mode 100644 index 000000000000..849d9c703730 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/calendar.svg @@ -0,0 +1,8 @@ + + + diff --git a/packages/web/components/common/Icon/icons/common/checkSquareBroken.svg b/packages/web/components/common/Icon/icons/common/checkSquareBroken.svg new file mode 100644 index 000000000000..c03255c8ccff --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/checkSquareBroken.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimary.svg b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimary.svg new file mode 100644 index 000000000000..befa75a5b5d6 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimary.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimaryHover.svg b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimaryHover.svg new file mode 100644 index 000000000000..d1e620884bad --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/checkSquareBrokenPrimaryHover.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/web/components/common/Icon/icons/math/greater.svg b/packages/web/components/common/Icon/icons/math/greater.svg new file mode 100644 index 000000000000..9e176a8e7e00 --- /dev/null +++ b/packages/web/components/common/Icon/icons/math/greater.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/web/components/common/Icon/icons/math/greaterEqual.svg b/packages/web/components/common/Icon/icons/math/greaterEqual.svg new file mode 100644 index 000000000000..5526d2a1a441 --- /dev/null +++ b/packages/web/components/common/Icon/icons/math/greaterEqual.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/web/components/common/Icon/icons/math/notEqual.svg b/packages/web/components/common/Icon/icons/math/notEqual.svg new file mode 100644 index 000000000000..3e83cba5c6e9 --- /dev/null +++ b/packages/web/components/common/Icon/icons/math/notEqual.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/web/components/common/Tabs/FillRowTabs.tsx b/packages/web/components/common/Tabs/FillRowTabs.tsx index ebec2ada9dce..2db032fa6395 100644 --- a/packages/web/components/common/Tabs/FillRowTabs.tsx +++ b/packages/web/components/common/Tabs/FillRowTabs.tsx @@ -141,12 +141,12 @@ const FillRowTabs = ( ? { bg: 'white', boxShadow: '1.5', - color: 'primary.600' + color: 'primary.700' } : { color: 'myGray.500', _hover: { - color: 'primary.600' + color: 'primary.700' }, onClick: () => { if (scrollPositionKey && scrollContainerRef.current) { diff --git a/packages/web/components/common/TagFilter/FilterButton.tsx b/packages/web/components/common/TagFilter/FilterButton.tsx index 2b07db5e2c01..ac382ad5c970 100644 --- a/packages/web/components/common/TagFilter/FilterButton.tsx +++ b/packages/web/components/common/TagFilter/FilterButton.tsx @@ -44,7 +44,7 @@ export const FilterSummaryValue = ({ ); -export type FilterButtonProps = Omit & { +type FilterButtonProps = Omit & { title: ReactNode; value: ReactNode; }; diff --git a/packages/web/components/common/TagFilter/MultiTagFilter.tsx b/packages/web/components/common/TagFilter/MultiTagFilter.tsx new file mode 100644 index 000000000000..23001fe5e649 --- /dev/null +++ b/packages/web/components/common/TagFilter/MultiTagFilter.tsx @@ -0,0 +1,368 @@ +import React, { useMemo, useState, type ReactNode } from 'react'; +import { Box, Button, Checkbox, Flex, Input, type PlacementWithLogical } from '@chakra-ui/react'; +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; +import MyIcon from '../Icon'; +import MyBox from '../MyBox'; +import MyPopover from '../MyPopover'; +import FilterButton from './FilterButton'; + +type MultiTagFilterValue = string | number; + +export type MultiTagFilterGroup = { + tagId: string; + label: string; + values: Array<{ + value: MultiTagFilterValue; + label: string; + }>; +}; + +type MultiTagFilterLabels = { + title: ReactNode; + all: ReactNode; + searchPlaceholder: string; + selected: ReactNode; + item: ReactNode; + clear: ReactNode; + noValues: ReactNode; + noMatch: ReactNode; +}; + +type MultiTagFilterProps = { + groups: MultiTagFilterGroup[]; + selected: CollectionTagFilterItem[]; + onSelectedChange: (next: CollectionTagFilterItem[]) => void; + isLoading?: boolean; + isLoadingValues?: boolean; + labels: MultiTagFilterLabels; + placement?: PlacementWithLogical; + offset?: [number, number]; + onOpen?: () => void; +}; + +const FILTER_LIST_H = '168px'; +const filterListScrollSx = { + overscrollBehavior: 'contain', + scrollbarWidth: 'thin', + scrollbarColor: 'var(--chakra-colors-myGray-200) transparent', + '&::-webkit-scrollbar': { w: '4px' }, + '&::-webkit-scrollbar-thumb': { + bg: 'myGray.200', + borderRadius: 'full' + } +}; +const stopWheelPropagation = (e: React.WheelEvent) => e.stopPropagation(); + +/** + * 双栏标签值筛选：左侧分组、右侧勾选值即生效。同一标签多值为 OR，不同标签由调用方按 AND 解释。 + */ +const toggleMultiTagFilterValue = ( + selected: CollectionTagFilterItem[], + tagId: string, + value: MultiTagFilterValue +): CollectionTagFilterItem[] => { + const current = selected.find((item) => item.tagId === tagId); + if (!current) { + return [...selected, { tagId, values: [value] }]; + } + + const exists = current.values.some((item) => item === value); + const nextValues = exists + ? current.values.filter((item) => item !== value) + : [...current.values, value]; + + if (nextValues.length === 0) { + return selected.filter((item) => item.tagId !== tagId); + } + + return selected.map((item) => (item.tagId === tagId ? { ...item, values: nextValues } : item)); +}; + +const MultiTagFilter = ({ + groups, + selected, + onSelectedChange, + isLoading = false, + isLoadingValues = false, + labels, + placement = 'bottom-start', + offset = [0, 4], + onOpen +}: MultiTagFilterProps) => { + const [activeTagId, setActiveTagId] = useState(''); + const [searchValue, setSearchValue] = useState(''); + const resolvedActiveTagId = groups.some((group) => group.tagId === activeTagId) + ? activeTagId + : (groups[0]?.tagId ?? ''); + + const selectedCountByTagId = useMemo(() => { + const countMap = new Map(); + for (const item of selected) { + countMap.set(item.tagId, item.values.length); + } + return countMap; + }, [selected]); + + const selectedCount = useMemo( + () => selected.reduce((sum, item) => sum + item.values.length, 0), + [selected] + ); + + const summary = useMemo(() => { + if (selectedCount === 0) { + return ( + + {labels.all} + + ); + } + + const firstSelected = selected[0]; + const firstGroup = groups.find((group) => group.tagId === firstSelected.tagId); + const firstValue = firstSelected.values[0]; + const firstValueLabel = + firstGroup?.values.find((item) => item.value === firstValue)?.label ?? String(firstValue); + const firstText = firstGroup ? `${firstGroup.label}: ${firstValueLabel}` : firstValueLabel; + const extraCount = selectedCount - 1; + + return ( + + + {firstText} + + {extraCount > 0 && ( + + +{extraCount} + + )} + + ); + }, [groups, labels.all, selected, selectedCount]); + + const activeGroup = groups.find((group) => group.tagId === resolvedActiveTagId); + const filteredValues = useMemo(() => { + if (!activeGroup) return []; + const keyword = searchValue.trim().toLowerCase(); + if (!keyword) return activeGroup.values; + return activeGroup.values.filter((item) => item.label.toLowerCase().includes(keyword)); + }, [activeGroup, searchValue]); + + const isValueChecked = (tagId: string, value: MultiTagFilterValue) => + selected.find((item) => item.tagId === tagId)?.values.some((item) => item === value) ?? false; + + const emptyRightText = (() => { + if (!activeGroup) return ''; + if (isLoadingValues) return ''; + if (activeGroup.values.length === 0) return labels.noValues; + if (filteredValues.length === 0) return labels.noMatch; + return ''; + })(); + + return ( + } + > + {() => ( + e.stopPropagation()}> + + + {groups.map((group) => { + const isActive = group.tagId === resolvedActiveTagId; + const selectedGroupCount = selectedCountByTagId.get(group.tagId) ?? 0; + + return ( + { + setActiveTagId(group.tagId); + setSearchValue(''); + }} + > + + {group.label} + + + {selectedGroupCount > 0 && ( + + {selectedGroupCount} + + )} + + + + ); + })} + + + + + + + setSearchValue(e.target.value)} + /> + {searchValue && ( + setSearchValue('')} + > + + + )} + + + {emptyRightText ? ( + + {emptyRightText} + + ) : ( + filteredValues.map((item) => { + const checked = isValueChecked(resolvedActiveTagId, item.value); + return ( + + onSelectedChange( + toggleMultiTagFilterValue(selected, resolvedActiveTagId, item.value) + ) + } + > + } + /> + + {item.label} + + + ); + }) + )} + + + + + + + + + {labels.selected} + {selectedCount} + {labels.item} + + + + + )} + + ); +}; + +export default React.memo(MultiTagFilter); diff --git a/packages/web/components/common/TagFilter/index.tsx b/packages/web/components/common/TagFilter/index.tsx index 7544d8ab8c52..045cbaa6b718 100644 --- a/packages/web/components/common/TagFilter/index.tsx +++ b/packages/web/components/common/TagFilter/index.tsx @@ -1,5 +1,5 @@ export { default as FilterButton, FilterSummaryValue, useFilterTriggerWidth } from './FilterButton'; -export type { FilterButtonProps, FilterSummaryValueProps } from './FilterButton'; +export type { FilterSummaryValueProps } from './FilterButton'; export { default as SingleSelectFilter } from './SingleSelectFilter'; export type { SingleSelectFilterOption, SingleSelectFilterProps } from './SingleSelectFilter'; export { default as MultiSelectFilter, useCommonFilterLabels } from './MultiSelectFilter'; @@ -17,6 +17,8 @@ export type { MultiSelectFilterSummary, MultiSelectFilterValue } from './multiSelectFilterUtils'; +export { default as MultiTagFilter } from './MultiTagFilter'; +export type { MultiTagFilterGroup } from './MultiTagFilter'; export { FILTER_SEARCH_THRESHOLD } from './FilterSearchInput'; export { default as FilterSearchInput } from './FilterSearchInput'; export { diff --git a/packages/web/components/core/workflow/NodeInputSelect.tsx b/packages/web/components/core/workflow/NodeInputSelect.tsx index 35f40bfcccd0..eb9bea70811d 100644 --- a/packages/web/components/core/workflow/NodeInputSelect.tsx +++ b/packages/web/components/core/workflow/NodeInputSelect.tsx @@ -157,6 +157,11 @@ const NodeInputSelect = ({ type: FlowNodeInputTypeEnum.password, icon: FlowNodeInputMap[FlowNodeInputTypeEnum.password].icon, title: t('common:core.workflow.inputType.Manual input') + }, + { + type: FlowNodeInputTypeEnum.datasetTagFilter, + icon: FlowNodeInputMap[FlowNodeInputTypeEnum.datasetTagFilter].icon, + title: t('common:core.workflow.inputType.Manual input') } ], [t] @@ -237,6 +242,7 @@ const NodeInputSelect = ({ bg={'myGray.100'} minH={'28px'} h={'28px'} + w={renderType === FlowNodeInputTypeEnum.datasetTagFilter ? '90px' : undefined} > {renderTypeData.title} diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index 0729a61b27b6..7926a935aa68 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -20,6 +20,7 @@ "Delete": "Delete", "Documents": "Documents", "Done": "Done", + "datetime_picker.placeholder": "YYYY-MM-DD --:--", "Download": "Download", "Edit": "Edit", "Error": "Error", @@ -455,6 +456,12 @@ "core.dataset.error.unCreateCollection": "Unauthorized to create dataset collection", "core.dataset.error.unExistDataset": "The dataset does not exist", "core.dataset.error.unLinkCollection": "Unauthorized to operate this external dataset collection", + "core.dataset.error.tagNameDuplicate": "Tag name already exists", + "core.dataset.error.tagNameEmpty": "Tag name cannot be empty", + "core.dataset.error.tagNotExist": "Tag does not exist", + "core.dataset.error.tagValueInvalid": "Tag value format does not match type", + "core.dataset.error.tagValueDatetimeInvalid": "Datetime tag value format is invalid", + "core.dataset.error.arrayTagValueInvalid": "Array tag value invalid (elements must be string, array length ≤ 64, element length ≤ 256)", "core.dataset.externalFile": "External File Library", "core.dataset.file": "File", "core.dataset.folder": "Directory", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 35699a22bc49..432b750ae775 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "Import the CSV file created when you exported a Dataset.", "backup_mode": "Backup import", "backup_template_invalid": "Invalid file format. Check the headers and content. Headers must be q, a, index, metadata (index may repeat; metadata is optional). Excel files must contain a single worksheet with no merged cells", - "batch_delete": "Batch delete", + "batch_delete": "Batch delete files", "chunk_max_tokens": "Max chunk tokens", "chunk_process_params": "Chunk processing parameters", "chunk_size": "Chunk size", @@ -24,6 +24,7 @@ "close_auto_sync": "Turn off auto-sync?", "collection.Create update time": "Creation/Update Time", "collection.export_all_chunks": "Export chunks", + "collection.select_all_filtered": "Select all filtered", "collection.sync.submit": "Sync task submitted", "collection.training_type": "Chunk type", "collection_data_count": "Data amount", @@ -32,6 +33,11 @@ "collection_sync": "Sync data", "collection_sync_confirm_tip": "Start syncing data? FastGPT compares the latest data and, when content has changed, creates a new Collection and deletes the old one.", "collection_tags": "Collection Tags", + "core.dataset.tags.array": "Options", + "core.dataset.tags.date": "Date", + "core.dataset.tags.number": "Number", + "core.dataset.tags.string": "String", + "core.dataset.tags.tagType": "Tag Type", "common.error.unKnow": "Unknown error", "common_dataset": "General Dataset", "common_dataset_desc": "Build a Dataset from files, web pages, or manually entered content", @@ -172,12 +178,57 @@ "tag.Edit_tag": "Edit Tag", "tag.add": "Create", "tag.add_new": "add_new", + "tag.add_option": "Add option", + "tag.add_tag": "Add tag", + "tag.append": "Append", + "tag.attribute": "Attribute", + "tag.batch_add": "Batch add tags", + "tag.batch_edit": "Batch edit tags", + "tag.batch_remove": "Batch remove tags", + "tag.batch_selected_files": "({{num}}) files selected", "tag.cancel": "Cancel", - "tag.delete_tag_confirm": "Confirm to delete the tag?", + "tag.create_failed": "Failed to create tag, please try again", + "tag.create_option": "Create option", + "tag.create_success": "Tag created successfully", + "tag.delete_failed": "Failed to delete tag, please try again", + "tag.delete_success": "Tag deleted successfully", + "tag.delete_tag_confirm_content": "Once deleted, this tag and its tag values added to files will be completely deleted and cannot be undone.", + "tag.delete_tag_confirm_title": "Confirm to delete the tag?", + "tag.enter_name": "Enter tag name", + "tag.enter_option": "Enter option", + "tag.fill_integer": "Enter an integer", + "tag.fill_number": "Enter number", + "tag.fill_value": "Enter tag value", + "tag.filter_clear": "Clear", + "tag.filter_clear_items": "Clear filters", + "tag.filter_empty_prefix": "No files match the current filters. ", + "tag.filter_item": "items", + "tag.filter_no_match": "No matching results", + "tag.filter_no_values": "No filterable values", + "tag.filter_search": "Search", + "tag.filter_selected": "Selected", "tag.manage": "Tagging", + "tag.manage_options": "Manage options", + "tag.name": "Tag name", + "tag.overwrite": "Overwrite", + "tag.overwrite_existing": "Overwrite existing values", + "tag.overwrite_tip": "The value is applied to all selected files. Existing values for this tag are replaced; files without this tag receive the new value.", + "tag.save_failed": "Failed to save tag, please try again", + "tag.save_success": "Tag saved successfully", "tag.searchOrAddTag": "Search or Add Tag", + "tag.search_or_create_option": "Search or create option", + "tag.select_attribute": "Select attribute", + "tag.select_options": "Select options", + "tag.select_tag": "Select tag", + "tag.select_tag_first": "Select a tag first", + "tag.select_tag_value": "Select tag value", + "tag.set": "Tag Settings", + "tag.setting_success": "Tags set successfully", + "tag.setting_tip": "Set tags and corresponding attribute values for the file, used as filter conditions during retrieval.", + "tag.setting_title": "Set Tags", "tag.tags": "Tags", "tag.total_tags": "Total {{total}} tags", + "tag.value": "Tag Value", "template_dataset": "Template import", "template_file_invalid": "Invalid file format. Check the headers and content. Headers must be q, a, index, metadata (index may repeat; metadata is optional). Excel files must contain a single worksheet with no merged cells", "template_mode": "Template import", diff --git a/packages/web/i18n/en/system_migration.json b/packages/web/i18n/en/system_migration.json index 72d2483a1bec..78ae6f1d1666 100644 --- a/packages/web/i18n/en/system_migration.json +++ b/packages/web/i18n/en/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "Checking dataset owner permissions", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "Checking agent skill owner permissions", "migrations.20260905_backfill_resource_owner_acl.validation": "Validating resource owner permissions", - "migrations.20260905_backfill_resource_owner_acl.result": "Checked {{appsProcessedCount}} apps, {{datasetsProcessedCount}} datasets, and {{agentSkillsProcessedCount}} skills. Backfilled owner permissions for {{appsUpdatedCount}} apps, {{datasetsUpdatedCount}} datasets, and {{agentSkillsUpdatedCount}} skills." + "migrations.20260905_backfill_resource_owner_acl.result": "Checked {{appsProcessedCount}} apps, {{datasetsProcessedCount}} datasets, and {{agentSkillsProcessedCount}} skills. Backfilled owner permissions for {{appsUpdatedCount}} apps, {{datasetsUpdatedCount}} datasets, and {{agentSkillsUpdatedCount}} skills.", + "migrations.20260907_migrate_dataset_tags_v2.name": "Upgrade dataset tag data", + "migrations.20260907_migrate_dataset_tags_v2.description": "Migrates legacy collection tags to typed tags and cleans duplicate tag definitions.", + "migrations.20260907_migrate_dataset_tags_v2.result": "Checked {{datasetsProcessedCount}} datasets, migrated {{collectionsMigratedCount}} collections, and removed {{duplicateDefinitionsDeletedCount}} duplicate tag definitions.", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "Normalize dataset tag definitions", + "migrations.20260907_migrate_dataset_tags_v2.collections": "Migrate collection tag values", + "migrations.20260907_migrate_dataset_tags_v2.validation": "Validate tag migration" } diff --git a/packages/web/i18n/en/workflow.json b/packages/web/i18n/en/workflow.json index 0dc240393ad2..ef9fb58d379d 100644 --- a/packages/web/i18n/en/workflow.json +++ b/packages/web/i18n/en/workflow.json @@ -39,6 +39,33 @@ "code_execution": "Code run", "code_sandbox_intro": "Run a script in the Sandbox for complex data processing. Available syntax and dependencies are limited.", "collection_metadata_filter": "Collection Metadata Filter", + "tag_filter": "Tag filter", + "tag_filter_add_condition": "Add filter", + "tag_filter_description": "Currently supports filtering by tags, creation time, and collection IDs to precisely locate matching files and improve retrieval accuracy and efficiency. For example, if a user asks “How long is the validity period?” and the question is known to be about product A on an order, metadata filtering can lock onto files related to product A and avoid searching other files that also mention “validity period.”", + "tag_filter_file_attrs": "File attributes", + "tag_filter_file_tags": "File tags", + "tag_filter_input_value": "Enter a value", + "tag_filter_op_after": "After", + "tag_filter_op_before": "Before", + "tag_filter_op_empty": "Is empty", + "tag_filter_op_eq": "Equals", + "tag_filter_op_gt": "Greater than", + "tag_filter_op_gte": "Greater than or equal to", + "tag_filter_op_in": "Is in", + "tag_filter_op_is": "Is", + "tag_filter_op_is_not": "Is not", + "tag_filter_op_lt": "Less than", + "tag_filter_op_lte": "Less than or equal to", + "tag_filter_op_ne": "Not equal to", + "tag_filter_op_not_empty": "Is not empty", + "tag_filter_op_not_in": "Is not in", + "tag_filter_select_condition": "Select condition", + "tag_filter_select_option": "Select options", + "tag_filter_select_tag": "Select tag", + "tag_filter_select_time": "Select time", + "tag_filter_upgrade_content": "You can’t switch back after upgrading, and the current filter settings will be cleared. (Copy this node first if you want a backup.)", + "tag_filter_upgrade_cta": "Deprecated. Upgrade to the latest version", + "tag_filter_upgrade_title": "Upgrade to the new version?", "complete_extraction_result": "Complete Extraction Result", "complete_extraction_result_description": "A JSON string, e.g., {\"name\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "Concatenation Result", diff --git a/packages/web/i18n/ko-KR/common.json b/packages/web/i18n/ko-KR/common.json index a49d0ae247c5..43634701a4f4 100644 --- a/packages/web/i18n/ko-KR/common.json +++ b/packages/web/i18n/ko-KR/common.json @@ -20,6 +20,7 @@ "Delete": "삭제", "Documents": "문서", "Done": "완료", + "datetime_picker.placeholder": "연-월-일 --:--", "Download": "다운로드", "Edit": "편집", "Error": "오류", diff --git a/packages/web/i18n/ko-KR/dataset.json b/packages/web/i18n/ko-KR/dataset.json index 8a77009c91bb..85b9ea41283e 100644 --- a/packages/web/i18n/ko-KR/dataset.json +++ b/packages/web/i18n/ko-KR/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "데이터셋을 내보낼 때 다운로드한 CSV 파일을 다시 가져옵니다.", "backup_mode": "백업 가져오기", "backup_template_invalid": "파일 형식이 올바르지 않습니다. 헤더와 콘텐츠가 요구사항에 맞는지 확인하세요. 헤더는 q, a, index, metadata여야 합니다(index는 반복 가능, metadata는 선택 사항). Excel 파일은 워크시트 하나만 지원하며 병합된 셀을 포함할 수 없습니다", - "batch_delete": "일괄 삭제", + "batch_delete": "파일 일괄 삭제", "chunk_max_tokens": "최대 청크 토큰", "chunk_process_params": "청크 처리 파라미터", "chunk_size": "청크 크기", @@ -24,6 +24,7 @@ "close_auto_sync": "자동 동기화를 끄시겠습니까?", "collection.Create update time": "생성/수정 시간", "collection.export_all_chunks": "청크 내보내기", + "collection.select_all_filtered": "필터 항목 전체 선택", "collection.sync.submit": "동기화 작업이 제출되었습니다", "collection.training_type": "청크 유형", "collection_data_count": "데이터 양", @@ -32,6 +33,11 @@ "collection_sync": "데이터 동기화", "collection_sync_confirm_tip": "데이터 동기화를 시작하시겠습니까? 시스템이 최신 데이터를 가져와 비교합니다. 내용이 다르면 새 컬렉션이 생성되고 기존 컬렉션은 삭제됩니다. 확인해 주세요!", "collection_tags": "컬렉션 태그", + "core.dataset.tags.array": "옵션", + "core.dataset.tags.number": "숫자", + "core.dataset.tags.string": "텍스트", + "core.dataset.tags.date": "날짜", + "core.dataset.tags.tagType": "태그 유형", "common.error.unKnow": "알 수 없는 오류", "common_dataset": "일반 데이터셋", "common_dataset_desc": "파일, 웹 링크 또는 수동 입력을 통해 데이터셋을 구축합니다", @@ -172,12 +178,57 @@ "tag.Edit_tag": "태그 편집", "tag.add": "생성", "tag.add_new": "추가", + "tag.add_option": "옵션 추가", + "tag.add_tag": "태그 추가", + "tag.append": "추가", + "tag.attribute": "속성", + "tag.batch_add": "태그 일괄 추가", + "tag.batch_edit": "태그 일괄 수정", + "tag.batch_remove": "태그 일괄 제거", + "tag.batch_selected_files": "선택됨 ({{num}})개 파일", "tag.cancel": "취소", - "tag.delete_tag_confirm": "태그를 삭제하시겠습니까?", + "tag.create_failed": "태그 추가 실패, 다시 시도해 주세요", + "tag.create_option": "옵션 생성", + "tag.create_success": "태그가 추가되었습니다", + "tag.delete_failed": "태그 삭제 실패, 다시 시도해 주세요", + "tag.delete_success": "태그가 삭제되었습니다", + "tag.delete_tag_confirm_content": "삭제 후 파일에 추가된 해당 태그 및 태그 값이 모두 삭제되며 취소할 수 없습니다.", + "tag.delete_tag_confirm_title": "태그 삭제 확인", + "tag.enter_name": "태그 이름을 입력하세요", + "tag.enter_option": "옵션을 입력하세요", + "tag.fill_integer": "정수를 입력하세요", + "tag.fill_number": "숫자를 입력하세요", + "tag.fill_value": "태그 값을 입력하세요", + "tag.filter_clear": "지우기", + "tag.filter_clear_items": "필터 지우기", + "tag.filter_empty_prefix": "조건에 맞는 파일을 찾지 못했습니다. ", + "tag.filter_item": "개", + "tag.filter_no_match": "일치하는 정보가 없습니다", + "tag.filter_no_values": "필터링할 값이 없습니다", + "tag.filter_search": "검색", + "tag.filter_selected": "선택됨", "tag.manage": "태그 관리", + "tag.manage_options": "옵션 관리", + "tag.name": "태그 이름", + "tag.overwrite": "덮어쓰기", + "tag.overwrite_existing": "기존 값 덮어쓰기", + "tag.overwrite_tip": "시간 또는 숫자 태그를 일괄 설정하면 선택한 모든 항목에 값이 적용됩니다. 해당 태그가 이미 있으면 기존 값이 대체되고, 없으면 새 값이 추가됩니다.", + "tag.save_failed": "태그 저장 실패, 다시 시도해 주세요", + "tag.save_success": "태그가 저장되었습니다", "tag.searchOrAddTag": "태그 검색 또는 추가", + "tag.search_or_create_option": "옵션 검색 또는 생성", + "tag.select_attribute": "속성을 선택하세요", + "tag.select_options": "옵션을 선택하세요", + "tag.select_tag": "태그를 선택하세요", + "tag.select_tag_first": "먼저 태그를 선택하세요", + "tag.select_tag_value": "태그 값을 선택하세요", + "tag.set": "태그 설정", + "tag.setting_success": "태그가 설정되었습니다", + "tag.setting_tip": "검색 시 필터 조건으로 사용할 파일의 태그 및 속성 값을 설정합니다.", + "tag.setting_title": "태그 설정", "tag.tags": "태그", "tag.total_tags": "총 {{total}}개 태그", + "tag.value": "태그 값", "template_dataset": "템플릿 가져오기", "template_file_invalid": "파일 형식이 올바르지 않습니다. 헤더와 콘텐츠가 요구사항에 맞는지 확인하세요. 헤더는 q, a, index, metadata여야 합니다(index는 반복 가능, metadata는 선택 사항). Excel 파일은 워크시트 하나만 지원하며 병합된 셀을 포함할 수 없습니다", "template_mode": "템플릿 가져오기", diff --git a/packages/web/i18n/ko-KR/system_migration.json b/packages/web/i18n/ko-KR/system_migration.json index c6431a94c98e..bd2bf1cfa583 100644 --- a/packages/web/i18n/ko-KR/system_migration.json +++ b/packages/web/i18n/ko-KR/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "지식 베이스 소유자 권한 확인 중", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "Agent Skill 소유자 권한 확인 중", "migrations.20260905_backfill_resource_owner_acl.validation": "리소스 소유자 권한 검증 중", - "migrations.20260905_backfill_resource_owner_acl.result": "앱 {{appsProcessedCount}}개, 지식 베이스 {{datasetsProcessedCount}}개, 스킬 {{agentSkillsProcessedCount}}개를 확인했습니다. 소유자 권한 보완: 앱 {{appsUpdatedCount}}개, 지식 베이스 {{datasetsUpdatedCount}}개, 스킬 {{agentSkillsUpdatedCount}}개." + "migrations.20260905_backfill_resource_owner_acl.result": "앱 {{appsProcessedCount}}개, 지식 베이스 {{datasetsProcessedCount}}개, 스킬 {{agentSkillsProcessedCount}}개를 확인했습니다. 소유자 권한 보완: 앱 {{appsUpdatedCount}}개, 지식 베이스 {{datasetsUpdatedCount}}개, 스킬 {{agentSkillsUpdatedCount}}개.", + "migrations.20260907_migrate_dataset_tags_v2.name": "지식 베이스 태그 데이터 업그레이드", + "migrations.20260907_migrate_dataset_tags_v2.description": "기존 Collection 태그를 유형화된 태그로 마이그레이션하고 중복 태그 정의를 정리합니다.", + "migrations.20260907_migrate_dataset_tags_v2.result": "지식 베이스 {{datasetsProcessedCount}}개를 확인하고 Collection {{collectionsMigratedCount}}개를 마이그레이션했으며, 중복 태그 정의 {{duplicateDefinitionsDeletedCount}}개를 삭제했습니다.", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "지식 베이스 태그 정의 정리", + "migrations.20260907_migrate_dataset_tags_v2.collections": "Collection 태그 값 마이그레이션", + "migrations.20260907_migrate_dataset_tags_v2.validation": "태그 마이그레이션 검증" } diff --git a/packages/web/i18n/ko-KR/workflow.json b/packages/web/i18n/ko-KR/workflow.json index f84c69ef5dde..555828363c36 100644 --- a/packages/web/i18n/ko-KR/workflow.json +++ b/packages/web/i18n/ko-KR/workflow.json @@ -39,6 +39,33 @@ "code_execution": "코드 샌드박스", "code_sandbox_intro": "샌드박스에서 스크립트 코드를 실행하면 복잡한 데이터 처리를 수행할 수 있지만, 문법과 사용 가능한 의존성에는 제한이 있습니다.", "collection_metadata_filter": "컬렉션 메타데이터 필터", + "tag_filter": "태그 필터", + "tag_filter_add_condition": "필터 조건 추가", + "tag_filter_description": "현재 태그, 생성 시간, 컬렉션 ID 필터를 지원하며, 해당 파일을 정확하게 찾아 검색 정확도와 효율을 높일 수 있습니다. 예: 사용자가 “유효기간은 얼마인가요?”라고 묻고, 해당 질문이 주문의 제품 A에 대한 것임이 확인되면 메타데이터 필터로 제품 A 관련 파일만 잠가, 다른 “유효기간”이 포함된 파일에서 검색하는 것을 피할 수 있습니다.", + "tag_filter_file_attrs": "파일 속성", + "tag_filter_file_tags": "파일 태그", + "tag_filter_input_value": "값 입력", + "tag_filter_op_after": "이후", + "tag_filter_op_before": "이전", + "tag_filter_op_empty": "비어 있음", + "tag_filter_op_eq": "같음", + "tag_filter_op_gt": "보다 큼", + "tag_filter_op_gte": "크거나 같음", + "tag_filter_op_in": "포함됨", + "tag_filter_op_is": "일치", + "tag_filter_op_is_not": "불일치", + "tag_filter_op_lt": "보다 작음", + "tag_filter_op_lte": "작거나 같음", + "tag_filter_op_ne": "같지 않음", + "tag_filter_op_not_empty": "비어 있지 않음", + "tag_filter_op_not_in": "포함되지 않음", + "tag_filter_select_condition": "조건 선택", + "tag_filter_select_option": "옵션을 선택하세요", + "tag_filter_select_tag": "태그 선택", + "tag_filter_select_time": "시간 선택", + "tag_filter_upgrade_content": "전환 후에는 이전 버전으로 돌아갈 수 없으며 현재 필터 설정이 삭제됩니다. (백업이 필요하면 현재 노드를 복사하세요.)", + "tag_filter_upgrade_cta": "더 이상 사용되지 않습니다. 최신 버전으로 업그레이드", + "tag_filter_upgrade_title": "새 버전으로 업그레이드할까요?", "complete_extraction_result": "전체 추출 결과", "complete_extraction_result_description": "JSON 문자열, 예: {\"name\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "연결 결과", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index 9cff264ac8c2..a94ff2871592 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -20,6 +20,7 @@ "Delete": "删除", "Documents": "文档", "Done": "完成", + "datetime_picker.placeholder": "年-月-日 --:--", "Download": "下载", "Edit": "编辑", "Error": "错误", @@ -455,6 +456,12 @@ "core.dataset.error.unCreateCollection": "无权创建知识库集合", "core.dataset.error.unExistDataset": "知识库不存在", "core.dataset.error.unLinkCollection": "无权操作该外部知识库集合", + "core.dataset.error.tagNameDuplicate": "标签名称已存在", + "core.dataset.error.tagNameEmpty": "标签名称不能为空", + "core.dataset.error.tagNotExist": "标签不存在", + "core.dataset.error.tagValueInvalid": "标签值格式不符合类型要求", + "core.dataset.error.tagValueDatetimeInvalid": "日期时间标签值格式无效", + "core.dataset.error.arrayTagValueInvalid": "数组标签值格式无效（元素须为字符串，数组长度不超过 64，单元素不超过 256 字符）", "core.dataset.externalFile": "外部文件库", "core.dataset.file": "文件", "core.dataset.folder": "目录", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index eeed3f2f42d2..11ad0889df51 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "可以将导出知识库时，下载的 csv 文件重新导入。", "backup_mode": "备份导入", "backup_template_invalid": "文件格式异常，请检查表头和内容是否符合要求。表头应为 q、a、index、metadata（index 可重复，metadata 可选）；Excel 文件仅支持单个工作表，且不能包含合并单元格", - "batch_delete": "批量删除", + "batch_delete": "批量删除文件", "chunk_max_tokens": "分块上限", "chunk_process_params": "分块处理参数", "chunk_size": "分块大小", @@ -24,6 +24,7 @@ "close_auto_sync": "确认关闭自动同步功能？", "collection.Create update time": "创建/更新时间", "collection.export_all_chunks": "导出分块", + "collection.select_all_filtered": "全选筛选项", "collection.sync.submit": "已提交同步任务", "collection.training_type": "处理模式", "collection_data_count": "数据量", @@ -32,6 +33,11 @@ "collection_sync": "立即同步", "collection_sync_confirm_tip": "确认开始同步数据？系统将会拉取最新数据进行比较，如果内容不相同，则会创建一个新的集合并删除旧的集合，请确认！", "collection_tags": "集合标签", + "core.dataset.tags.array": "选项", + "core.dataset.tags.date": "日期", + "core.dataset.tags.number": "数字", + "core.dataset.tags.string": "文本", + "core.dataset.tags.tagType": "标签类型", "common.error.unKnow": "未知错误", "common_dataset": "通用知识库", "common_dataset_desc": "通过导入文件、网页链接或手动录入形式构建知识库", @@ -172,12 +178,57 @@ "tag.Edit_tag": "编辑标签", "tag.add": "创建", "tag.add_new": "新建", + "tag.add_option": "添加选项", + "tag.add_tag": "添加标签", + "tag.append": "追加", + "tag.attribute": "属性", + "tag.batch_add": "批量添加标签", + "tag.batch_edit": "批量修改标签", + "tag.batch_remove": "批量移除标签", + "tag.batch_selected_files": "已选 ({{num}}) 个文件", "tag.cancel": "取消选择", - "tag.delete_tag_confirm": "确定删除标签？", + "tag.create_failed": "标签新增失败，请重试", + "tag.create_option": "创建选项", + "tag.create_success": "标签新增成功", + "tag.delete_failed": "标签删除失败，请重试", + "tag.delete_success": "标签删除成功", + "tag.delete_tag_confirm_content": "删除后，已添加到文件中的该标签及标签值将全部删除，且无法撤销。", + "tag.delete_tag_confirm_title": "确认删除标签？", + "tag.enter_name": "请输入标签名称", + "tag.enter_option": "请输入选项", + "tag.fill_integer": "请输入整数值", + "tag.fill_number": "请填写数字", + "tag.fill_value": "请填写标签值", + "tag.filter_clear": "清空", + "tag.filter_clear_items": "清空筛选项", + "tag.filter_empty_prefix": "未找到符合条件的文件，", + "tag.filter_item": "项", + "tag.filter_no_match": "未匹配到相关信息", + "tag.filter_no_values": "暂无可筛选值", + "tag.filter_search": "搜索", + "tag.filter_selected": "已选", "tag.manage": "标签管理", + "tag.manage_options": "管理选项", + "tag.name": "标签名称", + "tag.overwrite": "覆盖", + "tag.overwrite_existing": "覆盖原有值", + "tag.overwrite_tip": "批量设置时间或数值标签时，所设置的值会应用到全部选中内容。若部分内容已存在该标签值，原有值将被替换；未设置该标签的内容将直接添加该值。", + "tag.save_failed": "标签保存失败，请重试", + "tag.save_success": "标签保存成功", "tag.searchOrAddTag": "搜索或添加标签", + "tag.search_or_create_option": "搜索或创建选项", + "tag.select_attribute": "请选择属性", + "tag.select_options": "请选择选项", + "tag.select_tag": "请选择标签", + "tag.select_tag_first": "请先选择标签", + "tag.select_tag_value": "请选择标签值", + "tag.set": "标签设置", + "tag.setting_success": "标签设置成功", + "tag.setting_tip": "设置文件的标签和对应属性值，用于检索时的过滤条件。", + "tag.setting_title": "设置标签", "tag.tags": "标签", "tag.total_tags": "共{{total}}个标签", + "tag.value": "标签值", "template_dataset": "模版导入", "template_file_invalid": "文件格式异常，请检查表头和内容是否符合要求。表头应为 q、a、index、metadata（index 可重复，metadata 可选）；Excel 文件仅支持单个工作表，且不能包含合并单元格", "template_mode": "模板导入", diff --git a/packages/web/i18n/zh-CN/system_migration.json b/packages/web/i18n/zh-CN/system_migration.json index 381d3ef399e6..f618174124b4 100644 --- a/packages/web/i18n/zh-CN/system_migration.json +++ b/packages/web/i18n/zh-CN/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "检查知识库所有者权限", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "检查 Agent Skill 所有者权限", "migrations.20260905_backfill_resource_owner_acl.validation": "校验资源所有者权限", - "migrations.20260905_backfill_resource_owner_acl.result": "已检查 {{appsProcessedCount}} 个应用、{{datasetsProcessedCount}} 个知识库和 {{agentSkillsProcessedCount}} 个技能。补齐所有者权限：{{appsUpdatedCount}} 个应用、{{datasetsUpdatedCount}} 个知识库和 {{agentSkillsUpdatedCount}} 个技能。" + "migrations.20260905_backfill_resource_owner_acl.result": "已检查 {{appsProcessedCount}} 个应用、{{datasetsProcessedCount}} 个知识库和 {{agentSkillsProcessedCount}} 个技能。补齐所有者权限：{{appsUpdatedCount}} 个应用、{{datasetsUpdatedCount}} 个知识库和 {{agentSkillsUpdatedCount}} 个技能。", + "migrations.20260907_migrate_dataset_tags_v2.name": "升级知识库标签数据", + "migrations.20260907_migrate_dataset_tags_v2.description": "将历史 Collection 标签迁移为类型化标签，并清理重复标签定义。", + "migrations.20260907_migrate_dataset_tags_v2.result": "已检查 {{datasetsProcessedCount}} 个知识库，迁移 {{collectionsMigratedCount}} 个 Collection，删除 {{duplicateDefinitionsDeletedCount}} 个重复标签定义。", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "整理知识库标签定义", + "migrations.20260907_migrate_dataset_tags_v2.collections": "迁移 Collection 标签值", + "migrations.20260907_migrate_dataset_tags_v2.validation": "校验标签迁移结果" } diff --git a/packages/web/i18n/zh-CN/workflow.json b/packages/web/i18n/zh-CN/workflow.json index fa5a06de458d..fb2c22773dd2 100644 --- a/packages/web/i18n/zh-CN/workflow.json +++ b/packages/web/i18n/zh-CN/workflow.json @@ -39,6 +39,33 @@ "code_execution": "代码运行", "code_sandbox_intro": "在沙盒里执行一段脚本代码，可用于进行复杂的数据处理，语法和可用依赖会受到限制。", "collection_metadata_filter": "集合元数据过滤", + "tag_filter": "标签过滤", + "tag_filter_add_condition": "添加过滤条件", + "tag_filter_description": "目前支持标签、创建时间和集合ID过滤，可精确查找对应文件，进而提升检索准确率和效率。如:用户提问“有效期是多久?”，已确定用户询问订单对应产品A，通过元数据过滤，可锁定产品A相关文件，避免在其他含有“有效期”的文件中检索。", + "tag_filter_file_attrs": "文件属性", + "tag_filter_file_tags": "文件标签", + "tag_filter_input_value": "输入值", + "tag_filter_op_after": "晚于", + "tag_filter_op_before": "早于", + "tag_filter_op_empty": "为空", + "tag_filter_op_eq": "等于", + "tag_filter_op_gt": "大于", + "tag_filter_op_gte": "大于等于", + "tag_filter_op_in": "属于", + "tag_filter_op_is": "是", + "tag_filter_op_is_not": "不是", + "tag_filter_op_lt": "小于", + "tag_filter_op_lte": "小于等于", + "tag_filter_op_ne": "不等于", + "tag_filter_op_not_empty": "不为空", + "tag_filter_op_not_in": "不属于", + "tag_filter_select_condition": "选择条件", + "tag_filter_select_option": "请选择选项", + "tag_filter_select_tag": "选择标签", + "tag_filter_select_time": "请选择时间", + "tag_filter_upgrade_content": "切换后将无法返回旧版，且当前过滤设置将被清除。（建议复制当前节点作为备份）", + "tag_filter_upgrade_cta": "已弃用，升级到最新版本", + "tag_filter_upgrade_title": "确认升级到新版？", "complete_extraction_result": "完整提取结果", "complete_extraction_result_description": "一个 JSON 字符串，例如：{\"name:\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "拼接结果", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index c9f3cc8e3ade..ff3955ee056e 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -20,6 +20,7 @@ "Delete": "刪除", "Documents": "文件", "Done": "完成", + "datetime_picker.placeholder": "年-月-日 --:--", "Download": "下載", "Edit": "編輯", "Error": "錯誤", @@ -455,6 +456,12 @@ "core.dataset.error.unCreateCollection": "無權建立知識庫集合", "core.dataset.error.unExistDataset": "知識庫不存在", "core.dataset.error.unLinkCollection": "無權操作該外部知識庫集合", + "core.dataset.error.tagNameDuplicate": "標籤名稱已存在", + "core.dataset.error.tagNameEmpty": "標籤名稱不能為空", + "core.dataset.error.tagNotExist": "標籤不存在", + "core.dataset.error.tagValueInvalid": "標籤值格式不符合類型要求", + "core.dataset.error.tagValueDatetimeInvalid": "日期時間標籤值格式無效", + "core.dataset.error.arrayTagValueInvalid": "陣列標籤值格式無效（元素須為字串，陣列長度不超過 64，單元素不超過 256 字元）", "core.dataset.externalFile": "外部檔案庫", "core.dataset.file": "檔案", "core.dataset.folder": "目錄", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index b71899472cc1..4638fbbe8284 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -12,7 +12,7 @@ "backup_dataset_tip": "可以將導出知識庫時，下載的 csv 文件重新導入。", "backup_mode": "備份導入", "backup_template_invalid": "文件格式異常，請檢查表頭和內容是否符合要求。表頭應為 q、a、index、metadata（index 可重複，metadata 可選）；Excel 文件僅支持單個工作表，且不能包含合併單元格", - "batch_delete": "批量刪除", + "batch_delete": "批量刪除檔案", "chunk_max_tokens": "分塊上限", "chunk_process_params": "分塊處理參數", "chunk_size": "分塊大小", @@ -24,6 +24,7 @@ "close_auto_sync": "確認關閉自動同步功能？", "collection.Create update time": "建立／更新時間", "collection.export_all_chunks": "導出分塊", + "collection.select_all_filtered": "全選篩選項", "collection.sync.submit": "已提交同步任務", "collection.training_type": "處理模式", "collection_data_count": "資料量", @@ -32,6 +33,11 @@ "collection_sync": "立即同步", "collection_sync_confirm_tip": "確定要開始同步資料嗎？系統會比對最新資料；若內容不同，將建立新集合並刪除舊集合。", "collection_tags": "集合標籤", + "core.dataset.tags.array": "選項", + "core.dataset.tags.date": "日期", + "core.dataset.tags.number": "數字", + "core.dataset.tags.string": "文字", + "core.dataset.tags.tagType": "標籤類型", "common.error.unKnow": "未知錯誤", "common_dataset": "通用資料集", "common_dataset_desc": "通過導入文件、網頁鏈接或手動錄入形式構建知識庫", @@ -172,12 +178,57 @@ "tag.Edit_tag": "編輯標籤", "tag.add": "建立", "tag.add_new": "新增", + "tag.add_option": "新增選項", + "tag.add_tag": "新增標籤", + "tag.append": "追加", + "tag.attribute": "屬性", + "tag.batch_add": "批量新增標籤", + "tag.batch_edit": "批量修改標籤", + "tag.batch_remove": "批量移除標籤", + "tag.batch_selected_files": "已選 ({{num}}) 個檔案", "tag.cancel": "取消", - "tag.delete_tag_confirm": "確定要刪除標籤嗎？", + "tag.create_failed": "標籤新增失敗，請重試", + "tag.create_option": "建立選項", + "tag.create_success": "標籤新增成功", + "tag.delete_failed": "標籤刪除失敗，請重試", + "tag.delete_success": "標籤刪除成功", + "tag.delete_tag_confirm_content": "刪除後，已新增至檔案中的該標籤及標籤值將全部刪除，且無法復原。", + "tag.delete_tag_confirm_title": "確認刪除標籤？", + "tag.enter_name": "請輸入標籤名稱", + "tag.enter_option": "請輸入選項", + "tag.fill_integer": "請輸入整數值", + "tag.fill_number": "請填寫數字", + "tag.fill_value": "請填寫標籤值", + "tag.filter_clear": "清空", + "tag.filter_clear_items": "清空篩選項", + "tag.filter_empty_prefix": "未找到符合條件的檔案，", + "tag.filter_item": "項", + "tag.filter_no_match": "未匹配到相關資訊", + "tag.filter_no_values": "暫無可篩選值", + "tag.filter_search": "搜尋", + "tag.filter_selected": "已選", "tag.manage": "標籤管理", + "tag.manage_options": "管理選項", + "tag.name": "標籤名稱", + "tag.overwrite": "覆蓋", + "tag.overwrite_existing": "覆蓋原有值", + "tag.overwrite_tip": "批次設定時間或數值標籤時，所設定的值會套用到全部選取內容。若部分內容已存在該標籤值，原有值將被取代；未設定該標籤的內容將直接新增該值。", + "tag.save_failed": "標籤儲存失敗，請重試", + "tag.save_success": "標籤儲存成功", "tag.searchOrAddTag": "搜尋或新增標籤", + "tag.search_or_create_option": "搜尋或建立選項", + "tag.select_attribute": "請選擇屬性", + "tag.select_options": "請選擇選項", + "tag.select_tag": "請選擇標籤", + "tag.select_tag_first": "請先選擇標籤", + "tag.select_tag_value": "請選擇標籤值", + "tag.set": "標籤設定", + "tag.setting_success": "標籤設定成功", + "tag.setting_tip": "設定檔案的標籤與對應屬性值，用於檢索時的過濾條件。", + "tag.setting_title": "設定標籤", "tag.tags": "標籤", "tag.total_tags": "共 {{total}} 個標籤", + "tag.value": "標籤值", "template_dataset": "模版導入", "template_file_invalid": "文件格式異常，請檢查表頭和內容是否符合要求。表頭應為 q、a、index、metadata（index 可重複，metadata 可選）；Excel 文件僅支持單個工作表，且不能包含合併單元格", "template_mode": "模板導入", diff --git a/packages/web/i18n/zh-Hant/system_migration.json b/packages/web/i18n/zh-Hant/system_migration.json index d50a9449fafd..69ce49ec7f87 100644 --- a/packages/web/i18n/zh-Hant/system_migration.json +++ b/packages/web/i18n/zh-Hant/system_migration.json @@ -76,5 +76,11 @@ "migrations.20260905_backfill_resource_owner_acl.datasets": "檢查知識庫擁有者權限", "migrations.20260905_backfill_resource_owner_acl.agent_skills": "檢查 Agent Skill 擁有者權限", "migrations.20260905_backfill_resource_owner_acl.validation": "驗證資源擁有者權限", - "migrations.20260905_backfill_resource_owner_acl.result": "已檢查 {{appsProcessedCount}} 個應用、{{datasetsProcessedCount}} 個知識庫和 {{agentSkillsProcessedCount}} 個技能。補齊擁有者權限：{{appsUpdatedCount}} 個應用、{{datasetsUpdatedCount}} 個知識庫和 {{agentSkillsUpdatedCount}} 個技能。" + "migrations.20260905_backfill_resource_owner_acl.result": "已檢查 {{appsProcessedCount}} 個應用、{{datasetsProcessedCount}} 個知識庫和 {{agentSkillsProcessedCount}} 個技能。補齊擁有者權限：{{appsUpdatedCount}} 個應用、{{datasetsUpdatedCount}} 個知識庫和 {{agentSkillsUpdatedCount}} 個技能。", + "migrations.20260907_migrate_dataset_tags_v2.name": "升級知識庫標籤資料", + "migrations.20260907_migrate_dataset_tags_v2.description": "將歷史 Collection 標籤遷移為類型化標籤，並清理重複標籤定義。", + "migrations.20260907_migrate_dataset_tags_v2.result": "已檢查 {{datasetsProcessedCount}} 個知識庫，遷移 {{collectionsMigratedCount}} 個 Collection，刪除 {{duplicateDefinitionsDeletedCount}} 個重複標籤定義。", + "migrations.20260907_migrate_dataset_tags_v2.datasets": "整理知識庫標籤定義", + "migrations.20260907_migrate_dataset_tags_v2.collections": "遷移 Collection 標籤值", + "migrations.20260907_migrate_dataset_tags_v2.validation": "驗證標籤遷移結果" } diff --git a/packages/web/i18n/zh-Hant/workflow.json b/packages/web/i18n/zh-Hant/workflow.json index 3fd1792641be..9a8231130f12 100644 --- a/packages/web/i18n/zh-Hant/workflow.json +++ b/packages/web/i18n/zh-Hant/workflow.json @@ -39,6 +39,33 @@ "code_execution": "程式碼執行", "code_sandbox_intro": "在沙盒裡執行一段腳本程式碼，可用於進行複雜的資料處理，語法和可用依賴會受到限制。", "collection_metadata_filter": "資料集詮釋資料篩選器", + "tag_filter": "標籤過濾", + "tag_filter_add_condition": "新增過濾條件", + "tag_filter_description": "目前支援標籤、建立時間和集合ID過濾，可精確查找對應檔案，進而提升檢索準確率和效率。如：使用者提問「有效期是多久？」，已確定使用者詢問訂單對應產品A，透過詮釋資料過濾，可鎖定產品A相關檔案，避免在其他含有「有效期」的檔案中檢索。", + "tag_filter_file_attrs": "檔案屬性", + "tag_filter_file_tags": "檔案標籤", + "tag_filter_input_value": "輸入值", + "tag_filter_op_after": "晚於", + "tag_filter_op_before": "早於", + "tag_filter_op_empty": "為空", + "tag_filter_op_eq": "等於", + "tag_filter_op_gt": "大於", + "tag_filter_op_gte": "大於等於", + "tag_filter_op_in": "屬於", + "tag_filter_op_is": "是", + "tag_filter_op_is_not": "不是", + "tag_filter_op_lt": "小於", + "tag_filter_op_lte": "小於等於", + "tag_filter_op_ne": "不等於", + "tag_filter_op_not_empty": "不為空", + "tag_filter_op_not_in": "不屬於", + "tag_filter_select_condition": "選擇條件", + "tag_filter_select_option": "請選擇選項", + "tag_filter_select_tag": "選擇標籤", + "tag_filter_select_time": "請選擇時間", + "tag_filter_upgrade_content": "切換後將無法返回舊版，且目前過濾設定將被清除。（建議複製目前節點作為備份）", + "tag_filter_upgrade_cta": "已棄用，升級到最新版本", + "tag_filter_upgrade_title": "確認升級到新版？", "complete_extraction_result": "完整擷取結果", "complete_extraction_result_description": "一個 JSON 字串，例如：{\"name\":\"YY\",\"Time\":\"2023/7/2 18:00\"}", "concatenation_result": "串接結果", diff --git a/packages/web/styles/theme.ts b/packages/web/styles/theme.ts index 7c6be15928dd..34b212a208bc 100644 --- a/packages/web/styles/theme.ts +++ b/packages/web/styles/theme.ts @@ -654,6 +654,16 @@ const Checkbox = checkBoxMultiStyle({ } } }, + _indeterminate: { + bg: 'primary.50', + borderColor: 'primary.600', + borderWidth: '1px', + color: 'primary.600', + boxShadow: `${shadowLight} !important`, + _hover: { + bg: 'primary.50' + } + }, _hover: { borderColor: 'primary.400' }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6bd89b67bb2b..56572d910f88 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -704,6 +704,9 @@ importers: proxy-from-env: specifier: ^1.1.0 version: 1.1.0 + safe-regex: + specifier: ^2.1.1 + version: 2.1.1 turndown: specifier: ^7.1.2 version: 7.2.0 @@ -12979,6 +12982,10 @@ packages: regex@6.1.0: resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==} + regexp-tree@0.1.27: + resolution: {integrity: sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==} + hasBin: true + regexp.prototype.flags@1.5.4: resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==} engines: {node: '>= 0.4'} @@ -13256,6 +13263,9 @@ packages: resolution: {integrity: sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==} engines: {node: '>= 0.4'} + safe-regex@2.1.1: + resolution: {integrity: sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A==} + safe-stable-stringify@2.5.0: resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} engines: {node: '>=10'} @@ -29021,6 +29031,8 @@ snapshots: dependencies: regex-utilities: 2.3.0 + regexp-tree@0.1.27: {} + regexp.prototype.flags@1.5.4: dependencies: call-bind: 1.0.8 @@ -29426,6 +29438,10 @@ snapshots: es-errors: 1.3.0 is-regex: 1.2.1 + safe-regex@2.1.1: + dependencies: + regexp-tree: 0.1.27 + safe-stable-stringify@2.5.0: {} safer-buffer@2.1.2: {} diff --git a/pro b/pro index 26544a3004ff..92078141bdd2 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit 26544a3004ff56b22bdc39d703a7a0340f5b2644 +Subproject commit 92078141bdd27042e5fa99a0e37fc0c902a40e06 diff --git a/projects/app/src/components/core/dataset/DatasetTagFilterRows.tsx b/projects/app/src/components/core/dataset/DatasetTagFilterRows.tsx new file mode 100644 index 000000000000..3b7466b02185 --- /dev/null +++ b/projects/app/src/components/core/dataset/DatasetTagFilterRows.tsx @@ -0,0 +1,487 @@ +import React, { useEffect, useMemo } from 'react'; +import { Box, Button, Flex, HStack, Input } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyIconButton from '@fastgpt/web/components/common/Icon/button'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { + createEmptyTagFilterCondition, + createEmptyTagFilterValue, + DatasetTagFilterLogicEnum, + DatasetTagFilterValueModeEnum, + DatasetTagFilterFieldEnum, + getTagFilterOpsByCondition, + intersectWorkflowTagOptions, + isDatasetTagFilterValue, + isTagFilterOpWithoutValue, + isTagFilterAttributeField, + pruneTagFilterConditions, + type DatasetTagFilterCondition, + type DatasetTagFilterValue, + type WorkflowTagFilterOption +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import type { ReferenceItemValueType } from '@fastgpt/global/core/workflow/type/io'; +import type { WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants'; +import { getAllTags } from '@/web/core/dataset/api/collection'; +import { ReferSelector } from '@/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/Reference'; +import { + ArrayTagSelect, + DateTimeTagInput, + NumberTagInput, + tagInputBaseStyles +} from '@/pageComponents/dataset/detail/CollectionCard/TagValueInputs'; +import PromptEditor from '@fastgpt/web/components/common/Textarea/PromptEditor'; +import type { + EditorVariableLabelPickerType, + EditorVariablePickerType +} from '@fastgpt/web/components/common/Textarea/PromptEditor/type'; +import { TagFilterFieldSelect, TagFilterOpSelect } from './TagFilterSelects'; + +export type TagFilterReferenceList = { + label: string | React.ReactNode; + value: string; + children: { + label: string; + value: string; + valueType?: WorkflowIOValueTypeEnum; + }[]; +}[]; + +/** AND/OR 切换，贴在标题右侧，对齐判断器条件组。 */ +export const TagFilterLogicToggle = ({ + value, + onChange +}: { + value?: DatasetTagFilterValue; + onChange: (value: DatasetTagFilterValue) => void; +}) => { + const logic = value?.logic ?? DatasetTagFilterLogicEnum.AND; + + return ( + { + onChange({ + ...(value ?? createEmptyTagFilterValue()), + logic: + logic === DatasetTagFilterLogicEnum.AND + ? DatasetTagFilterLogicEnum.OR + : DatasetTagFilterLogicEnum.AND + }); + }} + > + {logic} + + + ); +}; + +/** 旧版本升级按钮，贴在标题栏右侧。 */ +export const DatasetTagFilterUpgradeButton = ({ + onUpgrade +}: { + onUpgrade: () => Promise; +}) => { + const { t } = useTranslation(); + const { openConfirm, ConfirmModal } = useConfirm({ + type: 'delete', + title: t('workflow:tag_filter_upgrade_title'), + content: t('workflow:tag_filter_upgrade_content') + }); + const { runAsync: runUpgrade, loading } = useRequest(onUpgrade); + + return ( + <> + openConfirm({ onConfirm: runUpgrade })()} + > + {t('workflow:tag_filter_upgrade_cta')} + + + + ); +}; + +/** 旧 JSON 字符串过滤：保留 PromptEditor 编辑，支持变量标签渲染。 */ +export const DatasetTagFilterDeprecated = ({ + value, + onChange, + variables, + variableLabels +}: { + value: string; + onChange: (value: string) => void; + variables?: EditorVariablePickerType[]; + variableLabels?: EditorVariableLabelPickerType[]; +}) => { + const { t } = useTranslation(); + + return ( + + ); +}; + +const valueCellEmbeddedStyles = { + border: 'none', + boxShadow: 'none', + h: '36px', + minH: '36px', + maxH: '36px', + borderRadius: 0, + _hover: { border: 'none' }, + _focus: { border: 'none', boxShadow: 'none' } +}; + +const TagFilterValueCell = ({ + condition, + option, + referenceList, + onChange +}: { + condition: DatasetTagFilterCondition; + option?: WorkflowTagFilterOption; + referenceList: TagFilterReferenceList; + onChange: (patch: Partial) => void; +}) => { + const { t } = useTranslation(); + const isCollectionId = condition.field === DatasetTagFilterFieldEnum.collectionId; + const isReference = + isCollectionId || condition.valueMode === DatasetTagFilterValueModeEnum.reference; + + const literalInput = (() => { + if (!condition.tagType) { + return ( + + ); + } + if (condition.tagType === DatasetCollectionTagTypeEnum.number) { + return ( + onChange({ value: val === '' ? undefined : val })} + /> + ); + } + if (condition.tagType === DatasetCollectionTagTypeEnum.datetime) { + return ( + onChange({ value: val })} + /> + ); + } + return ( + onChange({ value: val })} + /> + ); + })(); + + return ( + + {isCollectionId ? ( + + + + ) : ( + + { + if (isReference) { + onChange({ + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined + }); + return; + } + onChange({ + valueMode: DatasetTagFilterValueModeEnum.reference, + value: ['', undefined] + }); + }} + > + + + + + )} + + {isReference ? ( + onChange({ value: e as ReferenceItemValueType })} + isArray={false} + ButtonProps={{ + ...valueCellEmbeddedStyles, + size: 'sm', + w: '100%', + px: 3, + borderWidth: 0 + }} + /> + ) : ( + literalInput + )} + + + ); +}; + +const TagFilterConditionRow = ({ + condition, + options, + referenceList, + onChange, + onRemove +}: { + condition: DatasetTagFilterCondition; + options: WorkflowTagFilterOption[]; + referenceList: TagFilterReferenceList; + onChange: (patch: Partial) => void; + onRemove: () => void; +}) => { + const option = options.find( + (item) => + !isTagFilterAttributeField(condition.field) && + item.tag === condition.tag && + item.tagType === condition.tagType + ); + const ops = getTagFilterOpsByCondition(condition); + const hideValue = isTagFilterOpWithoutValue(condition.op); + + return ( + + + { + onChange({ + field: next.field, + tag: next.tag ?? '', + tagType: next.tagType, + op: '', + valueMode: next.valueMode ?? DatasetTagFilterValueModeEnum.input, + value: next.value + }); + }} + /> + onChange({ op, value: undefined })} + /> + {!hideValue && ( + + )} + + + + ); +}; + +/** + * 知识库搜索的标签过滤条件行。工作流节点和简易模式共用。 + * 旧 JSON 字符串请走 DatasetTagFilterDeprecated，不要传入本组件。 + */ +const DatasetTagFilterRows = ({ + value, + onChange, + datasetIds, + referenceList +}: { + value: unknown; + onChange: (value: DatasetTagFilterValue) => void; + datasetIds: string[]; + referenceList: TagFilterReferenceList; +}) => { + const { t } = useTranslation(); + const datasetIdsKey = datasetIds.join(','); + const filterValue = isDatasetTagFilterValue(value) ? value : createEmptyTagFilterValue(); + + const { + data: tagLists = [], + loading, + error + } = useRequest( + async () => { + if (datasetIds.length === 0) return []; + const results = await Promise.all(datasetIds.map(getAllTags)); + return results.map(({ list }) => list); + }, + { + manual: false, + refreshDeps: [datasetIdsKey], + errorToast: '' + } + ); + + const options = useMemo(() => intersectWorkflowTagOptions(tagLists), [tagLists]); + + useEffect(() => { + if (loading || error || datasetIds.length === 0 || !isDatasetTagFilterValue(value)) return; + const next = pruneTagFilterConditions(value, options); + if (JSON.stringify(next) === JSON.stringify(value)) return; + onChange(next); + }, [datasetIds.length, error, loading, onChange, options, value]); + + return ( + + + {filterValue.conditions.map((condition, index) => ( + { + onChange({ + ...filterValue, + conditions: filterValue.conditions.map((item, itemIndex) => + itemIndex === index ? { ...item, ...patch } : item + ) + }); + }} + onRemove={() => { + if (filterValue.conditions.length <= 1) { + onChange({ + ...filterValue, + conditions: [createEmptyTagFilterCondition()] + }); + return; + } + onChange({ + ...filterValue, + conditions: filterValue.conditions.filter((_, itemIndex) => itemIndex !== index) + }); + }} + /> + ))} + + + + ); +}; + +export default React.memo(DatasetTagFilterRows); diff --git a/projects/app/src/components/core/dataset/TagFilterSelects.tsx b/projects/app/src/components/core/dataset/TagFilterSelects.tsx new file mode 100644 index 000000000000..25c3816081ff --- /dev/null +++ b/projects/app/src/components/core/dataset/TagFilterSelects.tsx @@ -0,0 +1,379 @@ +import React, { useMemo, useState } from 'react'; +import { + Box, + Button, + Flex, + Input, + Popover, + PopoverContent, + PopoverTrigger, + useDisclosure +} from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import type { IconNameType } from '@fastgpt/web/components/common/Icon/type'; +import { + DatasetTagFilterFieldEnum, + DatasetTagFilterValueModeEnum, + formatTagOptionKey, + parseTagOptionKey, + type DatasetTagFilterCondition, + type DatasetTagFilterField, + type DatasetTagFilterValueMode, + type WorkflowTagFilterOption +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/constants'; + +const FILE_TAGS = 'fileTags'; +const FILE_ATTRS = 'fileAttrs'; + +type FieldSelectValue = { + field: DatasetTagFilterField; + tag?: string; + tagType?: WorkflowTagFilterOption['tagType']; + valueMode?: DatasetTagFilterValueMode; + value?: unknown; +}; + +/** 选择标签：左侧文件标签 / 文件属性，右侧可搜索。选中后触发器只展示叶子名称。 */ +export const TagFilterFieldSelect = ({ + condition, + options, + onChange +}: { + condition: DatasetTagFilterCondition; + options: WorkflowTagFilterOption[]; + onChange: (value: FieldSelectValue) => void; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const [search, setSearch] = useState(''); + const selectedLabel = (() => { + if (condition.field === DatasetTagFilterFieldEnum.createTime) { + return t('common:core.dataset.collection.metadata.Createtime' as any); + } + if (condition.field === DatasetTagFilterFieldEnum.collectionId) { + return t('common:core.dataset.collection.id' as any); + } + return condition.tag ?? ''; + })(); + const selectedValue = + condition.field === DatasetTagFilterFieldEnum.createTime + ? DatasetTagFilterFieldEnum.createTime + : condition.field === DatasetTagFilterFieldEnum.collectionId + ? DatasetTagFilterFieldEnum.collectionId + : condition.tag && condition.tagType + ? formatTagOptionKey(condition.tag, condition.tagType) + : ''; + const activeGroup = + condition.field === DatasetTagFilterFieldEnum.createTime || + condition.field === DatasetTagFilterFieldEnum.collectionId + ? FILE_ATTRS + : FILE_TAGS; + + const [group, setGroup] = useState(activeGroup); + + const rightItems = useMemo(() => { + const keyword = search.trim().toLowerCase(); + if (group === FILE_ATTRS) { + return [ + { + label: t('common:core.dataset.collection.metadata.Createtime'), + value: DatasetTagFilterFieldEnum.createTime + }, + { + label: t('common:core.dataset.collection.id'), + value: DatasetTagFilterFieldEnum.collectionId + } + ].filter((item) => !keyword || String(item.label).toLowerCase().includes(keyword)); + } + return options + .filter((item) => !keyword || item.tag.toLowerCase().includes(keyword)) + .map((item) => ({ + label: item.tag, + value: formatTagOptionKey(item.tag, item.tagType) + })); + }, [group, options, search, t]); + + const handleOpen = () => { + setGroup(activeGroup); + setSearch(''); + onOpen(); + }; + + const handleSelect = (value: string) => { + if (value === DatasetTagFilterFieldEnum.createTime) { + onChange({ + field: DatasetTagFilterFieldEnum.createTime, + tagType: DatasetCollectionTagTypeEnum.datetime, + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined + }); + } else if (value === DatasetTagFilterFieldEnum.collectionId) { + onChange({ + field: DatasetTagFilterFieldEnum.collectionId, + tagType: DatasetCollectionTagTypeEnum.array, + valueMode: DatasetTagFilterValueModeEnum.reference, + value: ['', undefined] + }); + } else { + const parsed = parseTagOptionKey(value); + onChange({ + field: DatasetTagFilterFieldEnum.tag, + tag: parsed?.tag, + tagType: parsed?.tagType, + valueMode: DatasetTagFilterValueModeEnum.input, + value: undefined + }); + } + onClose(); + }; + + return ( + + + + + + + + + {[ + { value: FILE_TAGS, label: t('workflow:tag_filter_file_tags') }, + { value: FILE_ATTRS, label: t('workflow:tag_filter_file_attrs') } + ].map((item) => { + const selected = group === item.value; + return ( + { + setGroup(item.value); + setSearch(''); + }} + > + {item.label} + + + ); + })} + + + + setSearch(e.target.value)} + /> + + {rightItems.length === 0 ? ( + + {t('common:no_select_data')} + + ) : ( + rightItems.map((item) => { + const selected = group === activeGroup && item.value === selectedValue; + return ( + handleSelect(item.value)} + > + {item.label} + + ); + }) + )} + + + + + + + ); +}; + +type OpItem = { + labelKey: string; + value: string; + icon?: string; + iconFlip?: boolean; +}; + +/** 条件操作符。数字类用稿上的 16px 符号图标 + 12px 文案。 */ +export const TagFilterOpSelect = ({ + value, + list, + onChange +}: { + value?: string; + list: OpItem[]; + onChange: (value: string) => void; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const selected = list.find((item) => item.value === value); + + return ( + + + + + + + {list.map((item) => ( + { + onChange(item.value); + onClose(); + }} + > + {item.icon && ( + + )} + {t(item.labelKey)} + + ))} + + + + ); +}; diff --git a/projects/app/src/global/core/api/datasetReq.ts b/projects/app/src/global/core/api/datasetReq.ts index 264a3801dbd5..fd727c09490b 100644 --- a/projects/app/src/global/core/api/datasetReq.ts +++ b/projects/app/src/global/core/api/datasetReq.ts @@ -1,6 +1,6 @@ -import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import type { PaginationProps } from '@fastgpt/global/openapi/api'; import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; /* ===== dataset ===== */ @@ -9,7 +9,7 @@ export type GetDatasetCollectionsProps = PaginationProps<{ datasetId: string; parentId?: ParentIdType; searchText?: string; - filterTags?: string[]; + tagFilters?: CollectionTagFilterItem[]; simple?: boolean; selectFolder?: boolean; }>; diff --git a/projects/app/src/migration/registry.ts b/projects/app/src/migration/registry.ts index 9b6affe2501e..d73e3af725a9 100644 --- a/projects/app/src/migration/registry.ts +++ b/projects/app/src/migration/registry.ts @@ -16,6 +16,7 @@ import { backfillResourceCreateTime } from './tasks/4170/20260903_backfill_resou import { backfillBillMetadata } from './tasks/4170/20260905_backfill_bill_metadata'; import { backfillResourceOwnerAcl } from './tasks/4170/20260905_backfill_resource_owner_acl'; import { cleanupTeamMemberRoles } from './tasks/4170/20260907_cleanup_team_member_roles'; +import { migrateDatasetTagsV2 } from './tasks/20260907_migrate_dataset_tags_v2'; export type SystemMigrationLogger = { info: (message: string, metadata?: Record) => void; @@ -309,6 +310,32 @@ export const systemMigrations = [ blockStartup: false, onFailure: SystemMigrationFailurePolicyEnum.continue, run: backfillResourceOwnerAcl + }, + { + id: '20260907_migrate_dataset_tags_v2', + version: '4.17.0', + nameKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.name'), + descriptionKey: i18nT( + 'system_migration:migrations.20260907_migrate_dataset_tags_v2.description' + ), + resultKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.result'), + progressSteps: [ + { + key: 'datasets', + labelKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.datasets') + }, + { + key: 'collections', + labelKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.collections') + }, + { + key: 'validation', + labelKey: i18nT('system_migration:migrations.20260907_migrate_dataset_tags_v2.validation') + } + ], + blockStartup: true, + onFailure: SystemMigrationFailurePolicyEnum.stop, + run: migrateDatasetTagsV2 } ] as const satisfies readonly SystemMigration[]; diff --git a/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/index.ts b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/index.ts new file mode 100644 index 000000000000..481032480c73 --- /dev/null +++ b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/index.ts @@ -0,0 +1,198 @@ +import { z } from 'zod'; +import { SystemMigrationStatusEnum } from '@fastgpt/global/migration/constants'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { systemMigrationBatchSize } from '@/migration/constants'; +import type { SystemMigrationContext } from '@/migration/registry'; +import { Types } from '@fastgpt/service/common/mongo'; +import { + migrateCollectionTagValues, + migrateDatasetTagDefinitions, + migrationCollections, + validateDatasetTagMigration, + type DatasetTagMigrationRecord, + type MigrationRecordResult +} from './service'; + +const StageKeySchema = z.enum(['datasets', 'collections']); +type StageKey = z.infer; + +const StageStateSchema = z.object({ + initialized: z.boolean(), + completed: z.boolean(), + endId: z.string().nullable(), + lastId: z.string().nullable(), + processedCount: z.number().int().nonnegative(), + migratedCount: z.number().int().nonnegative(), + total: z.number().int().nonnegative(), + deletedDefinitionCount: z.number().int().nonnegative(), + deletedReferenceCollectionCount: z.number().int().nonnegative() +}); +type StageState = z.infer; + +const CheckpointSchema = z.object({ + version: z.literal(1), + stages: z.record(StageKeySchema, StageStateSchema) +}); +type Checkpoint = z.infer; + +const createStageState = (): StageState => ({ + initialized: false, + completed: false, + endId: null, + lastId: null, + processedCount: 0, + migratedCount: 0, + total: 0, + deletedDefinitionCount: 0, + deletedReferenceCollectionCount: 0 +}); + +const stageKeys = StageKeySchema.options; +const createCheckpoint = (): Checkpoint => ({ + version: 1, + stages: Object.fromEntries( + stageKeys.map((key) => [key, createStageState()]) + ) as Checkpoint['stages'] +}); + +const stageQueries: Record> = { + datasets: { type: { $ne: DatasetTypeEnum.folder } }, + collections: {} +}; + +/** + * 4.17.0 阻塞升级：标签定义和 Collection 标签值是新版运行时的前置条件。 + * 任务按两个原始 Mongo 集合的 ObjectId 固定上界分批扫描；单记录写入幂等，业务提交后才推进 checkpoint。 + * 失败直接交给 Runner 阻止 ready，不使用阻塞任务禁止的 failedRecords。 + */ +export const migrateDatasetTagsV2 = async (context: SystemMigrationContext) => { + let checkpoint = (await context.getCheckpoint(CheckpointSchema)) ?? createCheckpoint(); + + const saveStage = async (key: StageKey, state: StageState) => { + checkpoint = { ...checkpoint, stages: { ...checkpoint.stages, [key]: state } }; + await context.saveCheckpoint(checkpoint); + }; + + const runStage = async ( + key: StageKey, + processRecord: (record: DatasetTagMigrationRecord) => Promise + ) => { + const collection = migrationCollections[key]; + const query = stageQueries[key]; + let state = checkpoint.stages[key]; + + if (!state.initialized) { + const lastRecord = await collection + .find({ ...query, _id: { $type: 'objectId' } }, { projection: { _id: 1 } }) + .sort({ _id: -1 }) + .limit(1) + .next(); + const endId = lastRecord ? String(lastRecord._id) : null; + const total = endId + ? await collection.countDocuments({ + ...query, + _id: { $type: 'objectId', $lte: lastRecord!._id } + }) + : 0; + state = { ...state, initialized: true, endId, total }; + await saveStage(key, state); + } + + await context.reportProgress({ + key, + status: state.completed + ? SystemMigrationStatusEnum.succeeded + : SystemMigrationStatusEnum.running, + current: state.processedCount, + total: state.total + }); + if (state.completed) return state; + + while (state.endId && state.lastId !== state.endId) { + context.signal.throwIfAborted(); + await context.assertActive(); + const records = (await collection + .find( + { + ...query, + _id: { + $type: 'objectId', + ...(state.lastId ? { $gt: new Types.ObjectId(state.lastId) } : {}), + $lte: new Types.ObjectId(state.endId) + } + }, + { projection: { _id: 1, teamId: 1 } } + ) + .sort({ _id: 1 }) + .limit(systemMigrationBatchSize) + .toArray()) as DatasetTagMigrationRecord[]; + if (records.length === 0) break; + + let batchResult: MigrationRecordResult = {}; + for (const record of records) { + const result = await processRecord(record); + batchResult = { + migratedCount: (batchResult.migratedCount ?? 0) + (result.migratedCount ?? 0), + deletedDefinitionCount: + (batchResult.deletedDefinitionCount ?? 0) + (result.deletedDefinitionCount ?? 0), + deletedReferenceCollectionCount: + (batchResult.deletedReferenceCollectionCount ?? 0) + + (result.deletedReferenceCollectionCount ?? 0) + }; + } + await context.assertActive(); + state = { + ...state, + lastId: String(records.at(-1)!._id), + processedCount: state.processedCount + records.length, + migratedCount: state.migratedCount + (batchResult.migratedCount ?? 0), + deletedDefinitionCount: + state.deletedDefinitionCount + (batchResult.deletedDefinitionCount ?? 0), + deletedReferenceCollectionCount: + state.deletedReferenceCollectionCount + (batchResult.deletedReferenceCollectionCount ?? 0) + }; + await saveStage(key, state); + await context.reportProgress({ + key, + status: SystemMigrationStatusEnum.running, + current: state.processedCount, + total: state.total + }); + } + + state = { ...state, completed: true }; + await saveStage(key, state); + await context.reportProgress({ + key, + status: SystemMigrationStatusEnum.succeeded, + current: state.processedCount, + total: state.total + }); + return state; + }; + + const datasets = await runStage('datasets', async (record) => { + if (!record.teamId) throw new Error(`Dataset ${String(record._id)} has no teamId`); + return migrateDatasetTagDefinitions({ datasetId: record._id, teamId: record.teamId }); + }); + const collections = await runStage('collections', (record) => + migrateCollectionTagValues({ collectionId: record._id }) + ); + + await context.reportProgress({ key: 'validation', status: SystemMigrationStatusEnum.running }); + await context.assertActive(); + const validation = await validateDatasetTagMigration().catch(async (error) => { + checkpoint = createCheckpoint(); + await context.saveCheckpoint(checkpoint); + throw error; + }); + await context.reportProgress({ key: 'validation', status: SystemMigrationStatusEnum.succeeded }); + + return { + datasetsProcessedCount: datasets.processedCount, + collectionsMigratedCount: collections.migratedCount, + duplicateDefinitionsDeletedCount: datasets.deletedDefinitionCount, + duplicateReferenceCollectionsCleanedCount: datasets.deletedReferenceCollectionCount, + legacyCollectionCount: validation.legacyCollectionCount + }; +}; diff --git a/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/service.ts b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/service.ts new file mode 100644 index 000000000000..1db4608b8a91 --- /dev/null +++ b/projects/app/src/migration/tasks/20260907_migrate_dataset_tags_v2/service.ts @@ -0,0 +1,271 @@ +import { DatasetCollectionTagTypeEnum } from '@fastgpt/global/core/dataset/type'; +import { Types, type ClientSession } from '@fastgpt/service/common/mongo'; +import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { ensureDatasetTagMigrationCarrier } from '@fastgpt/service/core/dataset/collection/utils'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; + +export type DatasetTagMigrationRecord = { + _id: Types.ObjectId; + teamId?: Types.ObjectId; +}; + +export type MigrationRecordResult = { + migratedCount?: number; + deletedDefinitionCount?: number; + deletedReferenceCollectionCount?: number; +}; + +const asObjectId = (value: unknown) => { + const stringValue = String(value ?? ''); + return Types.ObjectId.isValid(stringValue) ? new Types.ObjectId(stringValue) : undefined; +}; + +/** + * 整理单个知识库的 v2 标签定义：保留最早的同名定义，重复项先删引用再删定义， + * 并整理已有的 fromMigration 承载记录。没有旧字符串标签的知识库不预建承载记录。 + */ +export const migrateDatasetTagDefinitions = async ({ + datasetId, + teamId, + session +}: { + datasetId: Types.ObjectId; + teamId: Types.ObjectId; + session?: ClientSession; +}): Promise => { + const migrate = async (activeSession: ClientSession): Promise => { + const definitions = await MongoDatasetCollectionTagsV2.collection + .find({ teamId, datasetId }, { session: activeSession }) + .sort({ _id: 1 }) + .toArray(); + const deleteIds = new Map(); + + const definitionsByName = new Map(); + for (const definition of definitions) { + if (deleteIds.has(String(definition._id))) continue; + const group = definitionsByName.get(String(definition.tag)) ?? []; + group.push(definition); + definitionsByName.set(String(definition.tag), group); + } + for (const group of definitionsByName.values()) { + group.slice(1).forEach((definition) => { + deleteIds.set(String(definition._id), definition._id); + }); + } + + const migrationDefinitions = definitions.filter( + (definition) => definition.fromMigration === true && !deleteIds.has(String(definition._id)) + ); + const carrier = migrationDefinitions[0]; + migrationDefinitions.slice(1).forEach((definition) => { + deleteIds.set(String(definition._id), definition._id); + }); + + const duplicateIds = [...deleteIds.values()]; + let deletedReferenceCollectionCount = 0; + if (duplicateIds.length > 0) { + const duplicateTagIds: Array = [ + ...duplicateIds, + ...duplicateIds.map(String) + ]; + const duplicateTagIdStrings = duplicateIds.map(String); + const pullResult = await MongoDatasetCollection.collection.updateMany( + { teamId, datasetId, 'tags.tagId': { $in: duplicateTagIds } }, + [ + { + $set: { + tags: { + $filter: { + input: '$tags', + as: 'tag', + cond: { + $not: { + $in: [ + { + $convert: { + input: '$$tag.tagId', + to: 'string', + onError: null, + onNull: null + } + }, + duplicateTagIdStrings + ] + } + } + } + } + } + } + ], + { session: activeSession } + ); + deletedReferenceCollectionCount = pullResult.modifiedCount; + const remainingReferences = await MongoDatasetCollection.collection.countDocuments( + { teamId, datasetId, 'tags.tagId': { $in: duplicateTagIds } }, + { session: activeSession } + ); + if (remainingReferences > 0) { + throw new Error('Duplicate dataset tag references remain after cleanup'); + } + await MongoDatasetCollectionTagsV2.collection.deleteMany( + { _id: { $in: duplicateIds }, teamId, datasetId }, + { session: activeSession } + ); + } + + if (carrier) { + await MongoDatasetCollectionTagsV2.collection.updateOne( + { _id: carrier._id, teamId, datasetId }, + { + $set: { + tagType: DatasetCollectionTagTypeEnum.array, + fromMigration: true + } + }, + { session: activeSession } + ); + } + + return { + migratedCount: 1, + deletedDefinitionCount: duplicateIds.length, + deletedReferenceCollectionCount + }; + }; + + return session ? migrate(session) : mongoSessionRun(migrate); +}; + +/** 将单个 Collection 的旧标签 ID 转换为当前 dataset 承载标签的名称数组。 */ +export const migrateCollectionTagValues = async ({ + collectionId, + session +}: { + collectionId: Types.ObjectId; + session?: ClientSession; +}): Promise => { + const migrate = async (activeSession: ClientSession): Promise => { + const collection = await MongoDatasetCollection.collection.findOne( + { _id: collectionId }, + { projection: { teamId: 1, datasetId: 1, tags: 1 }, session: activeSession } + ); + if (!collection) return {}; + + const currentTags = Array.isArray(collection.tags) ? collection.tags : []; + const legacyTagIds = currentTags.filter((tag): tag is string => typeof tag === 'string'); + if (legacyTagIds.length === 0) return {}; + + const objectIds = legacyTagIds + .map(asObjectId) + .filter((id): id is Types.ObjectId => Boolean(id)); + const legacyDefinitions = objectIds.length + ? await MongoDatasetCollectionTags.collection + .find( + { _id: { $in: objectIds }, teamId: collection.teamId, datasetId: collection.datasetId }, + { projection: { tag: 1 }, session: activeSession } + ) + .toArray() + : []; + const carrier = await ensureDatasetTagMigrationCarrier({ + teamId: String(collection.teamId), + datasetId: String(collection.datasetId), + session: activeSession + }); + + const carrierId = String(carrier._id); + const existingCarrier = currentTags.find( + (tag) => tag && typeof tag === 'object' && String(tag.tagId) === carrierId + ); + const existingValues = Array.isArray(existingCarrier?.value) + ? existingCarrier.value.filter((value: unknown): value is string => typeof value === 'string') + : []; + const migratedValues = legacyDefinitions + .map((definition) => definition.tag) + .filter((tag): tag is string => typeof tag === 'string' && tag.length > 0); + const mergedValues = [...new Set([...existingValues, ...migratedValues])]; + const nextTags = currentTags.filter( + (tag) => typeof tag !== 'string' && String(tag?.tagId ?? '') !== carrierId + ); + if (mergedValues.length > 0) nextTags.push({ tagId: carrierId, value: mergedValues }); + + const result = await MongoDatasetCollection.collection.updateOne( + { _id: collectionId, tags: currentTags }, + { $set: { tags: nextTags } }, + { session: activeSession } + ); + if (result.matchedCount !== 1) { + throw new Error('Collection tags changed concurrently during migration'); + } + return { migratedCount: 1 }; + }; + + return session ? migrate(session) : mongoSessionRun(migrate); +}; + +export const migrationCollections = { + datasets: MongoDataset.collection, + collections: MongoDatasetCollection.collection +}; + +/** 创建标签唯一索引并在迁移成功前校验所有必要不变量。 */ +export const validateDatasetTagMigration = async () => { + await Promise.all([ + MongoDatasetCollectionTagsV2.createIndexes({ background: true }), + MongoDatasetCollection.createIndexes({ background: true }) + ]); + + const [legacyCollectionCount, duplicateGroups, duplicateCarrierGroups, invalidCarrierCount] = + await Promise.all([ + MongoDatasetCollection.collection.countDocuments({ + tags: { $elemMatch: { $type: 'string' } } + }), + MongoDatasetCollectionTagsV2.collection + .aggregate([ + { + $group: { + _id: { teamId: '$teamId', datasetId: '$datasetId', tag: '$tag' }, + n: { $sum: 1 } + } + }, + { $match: { n: { $gt: 1 } } }, + { $limit: 1 } + ]) + .toArray(), + MongoDatasetCollectionTagsV2.collection + .aggregate([ + { + $match: { fromMigration: true } + }, + { + $group: { + _id: { teamId: '$teamId', datasetId: '$datasetId' }, + n: { $sum: 1 } + } + }, + { $match: { n: { $gt: 1 } } }, + { $limit: 1 } + ]) + .toArray(), + MongoDatasetCollectionTagsV2.collection.countDocuments({ + fromMigration: true, + tagType: { $ne: DatasetCollectionTagTypeEnum.array } + }) + ]); + + if ( + legacyCollectionCount > 0 || + duplicateGroups.length > 0 || + duplicateCarrierGroups.length > 0 || + invalidCarrierCount > 0 + ) { + throw new Error( + `Dataset tag migration validation failed: legacyCollections=${legacyCollectionCount}, duplicateTagGroups=${duplicateGroups.length}, duplicateCarrierGroups=${duplicateCarrierGroups.length}, invalidCarriers=${invalidCarrierCount}` + ); + } + + return { legacyCollectionCount }; +}; diff --git a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx index 72ecbac44eaa..85638ebd049a 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx +++ b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/EditForm.tsx @@ -44,6 +44,16 @@ import { findClientModelByReference, resolveClientModelReferenceId } from '@/web/core/ai/model/modelReference'; +import DatasetTagFilterRows, { + TagFilterLogicToggle +} from '@/components/core/dataset/DatasetTagFilterRows'; +import { formatEditorVariablePickerIcon } from '@fastgpt/global/core/workflow/utils'; +import { workflowSystemVariables } from '@/web/core/app/utils'; +import { VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants'; +import { + isDatasetTagFilterValue, + type DatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; const DatasetSelectModal = dynamic(() => import('@/components/core/app/DatasetSelectModal')); const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal')); @@ -78,6 +88,40 @@ const EditForm = ({ const appId = useContextSelector(AppContext, (v) => v.appId); const selectDatasets = useMemo(() => appForm?.dataset?.datasets, [appForm]); + const formatVariables = useMemo( + () => + formatEditorVariablePickerIcon([ + ...workflowSystemVariables.filter( + (variable) => + !['appId', 'chatId', 'responseChatItemId', 'histories'].includes(variable.key) + ), + ...(appForm.chatConfig.variables ?? []) + ]).map((item) => ({ ...item, label: t(item.label as any) })), + [appForm.chatConfig.variables, t] + ); + const tagFilterReferenceList = useMemo( + () => [ + { + label: t('common:core.module.Variable'), + value: VARIABLE_NODE_ID, + children: formatVariables.map((item) => ({ + label: item.label, + value: item.key, + valueType: item.valueType + })) + } + ], + [formatVariables, t] + ); + const onCollectionFilterMatchChange = useCallback( + (value: DatasetTagFilterValue) => { + setAppForm((state) => ({ + ...state, + dataset: { ...state.dataset, collectionFilterMatch: value } + })); + }, + [setAppForm] + ); const { isWelcomeTextFolded, toggleWelcomeTextFold } = useWelcomeTextFoldState(appId); const { @@ -628,6 +672,30 @@ const EditForm = ({ /> ))} + {appForm.dataset.datasets.length > 0 && feConfigs?.isPlus && ( + + + {t('workflow:tag_filter')} + + + + + + item.datasetId)} + referenceList={tagFilterReferenceList} + /> + + )} {/* File select */} diff --git a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts index 7b750e468e70..c6948f074e6b 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts +++ b/projects/app/src/pageComponents/app/detail/Edit/ChatAgent/utils.ts @@ -351,6 +351,7 @@ export function agentForm2AppWorkflow( datasetSearchExtensionModelId: data.dataset.datasetSearchExtensionModelId, datasetSearchExtensionModel: data.dataset.datasetSearchExtensionModel, datasetSearchExtensionBg: data.dataset.datasetSearchExtensionBg, + collectionFilterMatch: data.dataset.collectionFilterMatch, [NodeInputKeyEnum.authTmbId]: data.dataset.authTmbId }) }, diff --git a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx index a939e094d6a8..356f4bdcf836 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx +++ b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/EditForm.tsx @@ -37,11 +37,26 @@ import { SANDBOX_ICON } from '@fastgpt/global/core/ai/sandbox/tools'; import SandboxConfigButton from '../../components/SandboxConfigButton'; import { useUserStore } from '@/web/support/user/useUserStore'; import DatasetCard from '@/components/core/app/DatasetCard'; +import DatasetTagFilterRows, { + DatasetTagFilterDeprecated, + DatasetTagFilterUpgradeButton, + TagFilterLogicToggle +} from '@/components/core/dataset/DatasetTagFilterRows'; import { useWelcomeTextFoldState } from '@/components/core/app/useAppEditorUIState'; import { findClientModelByReference, resolveClientModelReferenceId } from '@/web/core/ai/model/modelReference'; +import { NodeInputKeyEnum, VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants'; +import { + createEmptyTagFilterValue, + DatasetTagFilterVersionEnum, + isDatasetTagFilterValue, + normalizeLegacyDatasetTagFilterValue, + resolveDatasetTagFilterVersion, + type DatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { form2AppWorkflow } from './utils'; const DatasetSelectModal = dynamic(() => import('@/components/core/app/DatasetSelectModal')); const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal')); @@ -82,7 +97,7 @@ const EditForm = ({ const showSandbox = feConfigs.show_agent_sandbox; const { teamPlanStatus } = useUserStore(); const enableSandbox = !teamPlanStatus?.standard || !!teamPlanStatus?.standard?.enableSandbox; - const { appDetail } = useContextSelector(AppContext, (v) => v); + const { appDetail, onSaveApp } = useContextSelector(AppContext, (v) => v); const selectDatasets = useMemo(() => appForm?.dataset?.datasets, [appForm]); const [, startTst] = useTransition(); const isAgentSandboxEnabled = !!appForm.aiSettings.useAgentSandbox; @@ -111,7 +126,7 @@ const EditForm = ({ ...item, label: t(item.label as any), parent: { - id: 'VARIABLE_NODE_ID', + id: VARIABLE_NODE_ID, label: t('common:core.module.Variable'), avatar: 'core/workflow/template/variable' } @@ -119,6 +134,39 @@ const EditForm = ({ [appForm.chatConfig.variables, t] ); + const tagFilterReferenceList = useMemo( + () => [ + { + label: t('common:core.module.Variable'), + value: VARIABLE_NODE_ID, + children: formatVariables.map((item) => ({ + label: item.label, + value: item.key, + valueType: item.valueType + })) + } + ], + [formatVariables, t] + ); + + const onCollectionFilterMatchChange = useCallback( + (value: DatasetTagFilterValue | string) => { + setAppForm((state) => ({ + ...state, + dataset: { + ...state.dataset, + collectionFilterMatch: value + } + })); + }, + [setAppForm] + ); + const isLegacyCollectionFilter = + resolveDatasetTagFilterVersion({ + version: appForm.dataset[NodeInputKeyEnum.collectionFilterVersion], + filterValue: appForm.dataset.collectionFilterMatch + }) === DatasetTagFilterVersionEnum.legacy; + const { llmModelList, reRankModelList } = useUserModelLists(); const selectedModel = findClientModelByReference({ @@ -486,6 +534,78 @@ const EditForm = ({ /> ))} + {appForm.dataset.datasets?.length > 0 && feConfigs?.isPlus && ( + + + + {isLegacyCollectionFilter + ? t('workflow:collection_metadata_filter') + : t('workflow:tag_filter')} + + + {isLegacyCollectionFilter ? ( + <> + + { + const nextForm = { + ...appForm, + dataset: { + ...appForm.dataset, + [NodeInputKeyEnum.collectionFilterVersion]: + DatasetTagFilterVersionEnum.structured, + collectionFilterMatch: createEmptyTagFilterValue() + } + }; + const workflow = form2AppWorkflow(nextForm, t); + await onSaveApp({ + ...workflow, + isPublish: false, + chatConfig: nextForm.chatConfig + }); + setAppForm(nextForm); + }} + /> + + ) : ( + + + + )} + + {isLegacyCollectionFilter ? ( + + ) : ( + item.datasetId)} + referenceList={tagFilterReferenceList} + /> + )} + + )} {/* File select */} diff --git a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts index ba7cc684d699..d18c76d39b64 100644 --- a/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts +++ b/projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts @@ -29,6 +29,7 @@ import { AiChatQuoteTemplate } from '@fastgpt/global/core/workflow/template/system/aiChat/index'; import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch'; +import { resolveDatasetTagFilterVersion } from '@fastgpt/global/core/dataset/workflowTagFilter'; import { i18nT } from '@fastgpt/global/common/i18n/utils'; import { Input_Template_File_Link, @@ -121,6 +122,11 @@ export const appWorkflow2Form = ({ NodeInputKeyEnum.sandboxEntrypoint ); } else if (node.flowNodeType === FlowNodeTypeEnum.datasetSearchNode) { + defaultAppForm.dataset[NodeInputKeyEnum.collectionFilterVersion] = + resolveDatasetTagFilterVersion({ + version: findInputValueByKey(node.inputs, NodeInputKeyEnum.collectionFilterVersion), + filterValue: findInputValueByKey(node.inputs, NodeInputKeyEnum.collectionFilterMatch) + }); defaultAppForm.dataset.datasets = findInputValueByKey( node.inputs, NodeInputKeyEnum.datasetSelectList @@ -178,6 +184,10 @@ export const appWorkflow2Form = ({ node.inputs, NodeInputKeyEnum.authTmbId ); + defaultAppForm.dataset.collectionFilterMatch = findInputValueByKey( + node.inputs, + NodeInputKeyEnum.collectionFilterMatch + ); } else if ( node.flowNodeType === FlowNodeTypeEnum.pluginModule || node.flowNodeType === FlowNodeTypeEnum.appModule || @@ -579,6 +589,25 @@ export function form2AppWorkflow( valueType: WorkflowIOValueTypeEnum.boolean, value: formData.dataset.authTmbId }, + { + key: NodeInputKeyEnum.collectionFilterVersion, + renderTypeList: [FlowNodeInputTypeEnum.hidden], + label: '', + valueType: WorkflowIOValueTypeEnum.string, + value: resolveDatasetTagFilterVersion({ + version: formData.dataset[NodeInputKeyEnum.collectionFilterVersion], + filterValue: formData.dataset.collectionFilterMatch + }) + }, + { + key: NodeInputKeyEnum.collectionFilterMatch, + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter, FlowNodeInputTypeEnum.reference], + label: i18nT('workflow:tag_filter'), + valueType: WorkflowIOValueTypeEnum.string, + isPro: true, + description: i18nT('workflow:tag_filter_description'), + value: formData.dataset.collectionFilterMatch + }, { ...Input_Template_UserChatInput, key: NodeInputKeyEnum.datasetSearchInput, diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx index 92c7d83aa99b..67b70eef488f 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/Label.tsx @@ -21,17 +21,23 @@ type Props = { nodeId: string; input: FlowNodeInputItemType; RightComponent?: React.JSX.Element; + rightInline?: boolean; isTool?: boolean; }; -const InputLabel = ({ nodeId, input, RightComponent, isTool }: Props) => { +const InputLabel = ({ nodeId, input, RightComponent, rightInline, isTool }: Props) => { const { t } = useSafeTranslation(); + const labelText = t(input.label as any); + const descriptionText = input.description ? t(input.description as any) : undefined; + const onChangeNode = useContextSelector(WorkflowActionsContext, (v) => v.onChangeNode); - const { description, required, label, renderTypeList, valueType, valueDesc } = input; + const { required, renderTypeList, valueType, valueDesc } = input; const renderType = getSelectedInputRenderType(input) ?? renderTypeList?.[0] ?? FlowNodeInputTypeEnum.input; + const shouldRenderRightInline = + rightInline ?? renderType === FlowNodeInputTypeEnum.datasetTagFilter; const displayRenderTypeList = useMemo( () => getToolInputDisplayRenderTypeList({ @@ -65,9 +71,9 @@ const InputLabel = ({ nodeId, input, RightComponent, isTool }: Props) => { - {t(label as any)} + {labelText} - {description && } + {descriptionText && } {/* value type */} {[FlowNodeInputTypeEnum.reference, FlowNodeInputTypeEnum.fileSelect].includes(renderType) && ( @@ -122,8 +128,8 @@ const InputLabel = ({ nodeId, input, RightComponent, isTool }: Props) => { {/* Right Component */} {!input.deprecated && RightComponent && ( <> - - {RightComponent} + {!shouldRenderRightInline && } + {shouldRenderRightInline ? {RightComponent} : RightComponent} )} diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx index 9d79c206af14..fcea88faae70 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx @@ -16,6 +16,7 @@ import MyTag from '@fastgpt/web/components/common/Tag/index'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { getSelectedInputRenderType } from '@fastgpt/global/core/workflow/utils'; +import { datasetSearchUsesLegacyFilter } from '@/web/core/workflow/datasetSearchNodeUpgrade'; const RenderList: Record< FlowNodeInputTypeEnum, @@ -91,6 +92,12 @@ const RenderList: Record< [FlowNodeInputTypeEnum.password]: { Component: CommonInputForm }, + [FlowNodeInputTypeEnum.datasetTagFilter]: { + Component: dynamic(() => import('./templates/DatasetTagFilter')), + LableRightComponent: dynamic(() => + import('./templates/DatasetTagFilter').then((mod) => mod.DatasetTagFilterLogic) + ) + }, [FlowNodeInputTypeEnum.agentGenerated]: undefined, [FlowNodeInputTypeEnum.customVariable]: undefined, @@ -261,6 +268,10 @@ const RenderInput = ({ nodeId={nodeId} input={input} RightComponent={RenderComponent?.LableRightComponent} + rightInline={ + renderType === FlowNodeInputTypeEnum.datasetTagFilter && + !datasetSearchUsesLegacyFilter(filterProInputs) + } isTool={isTool} /> )} diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/DatasetTagFilter.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/DatasetTagFilter.tsx new file mode 100644 index 000000000000..5df94c0734f3 --- /dev/null +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/DatasetTagFilter.tsx @@ -0,0 +1,186 @@ +import React, { useCallback, useMemo } from 'react'; +import type { RenderInputProps } from '../type'; +import { useContextSelector } from 'use-context-selector'; +import { NodeInputKeyEnum, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants'; +import { + createEmptyTagFilterValue, + isDatasetTagFilterValue, + normalizeLegacyDatasetTagFilterValue, + type DatasetTagFilterValue +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { WorkflowActionsContext } from '@/pageComponents/app/detail/WorkflowComponents/context/workflowActionsContext'; +import { useReference } from './Reference'; +import DatasetTagFilterRows, { + DatasetTagFilterDeprecated, + DatasetTagFilterUpgradeButton, + TagFilterLogicToggle +} from '@/components/core/dataset/DatasetTagFilterRows'; +import { WorkflowBufferDataContext } from '../../../../../context/workflowInitContext'; +import { AppContext } from '@/pageComponents/app/detail/context'; +import { getEditorVariables } from '@/pageComponents/app/detail/WorkflowComponents/utils'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { useMemoEnhance } from '@fastgpt/web/hooks/useMemoEnhance'; +import { useTranslation } from 'next-i18next'; +import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; +import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch'; +import { WorkflowUtilsContext } from '@/pageComponents/app/detail/WorkflowComponents/context/workflowUtilsContext'; +import { + datasetSearchUsesLegacyFilter, + persistLegacyDatasetSearchNodeUpgrade +} from '@/web/core/workflow/datasetSearchNodeUpgrade'; + +const DatasetTagFilterRender = ({ inputs = [], item, nodeId }: RenderInputProps) => { + const { t } = useTranslation(); + const onChangeNode = useContextSelector(WorkflowActionsContext, (v) => v.onChangeNode); + const { getNodeById, edges } = useContextSelector(WorkflowBufferDataContext, (v) => v); + const { appDetail } = useContextSelector(AppContext, (v) => v); + const { feConfigs } = useSystemStore(); + const isLegacyNode = datasetSearchUsesLegacyFilter(inputs); + + const { referenceList } = useReference({ + nodeId, + valueType: WorkflowIOValueTypeEnum.any + }); + const datasetIds = useMemo(() => { + const datasetValue = inputs.find( + (input) => input.key === NodeInputKeyEnum.datasetSelectList + )?.value; + if (!Array.isArray(datasetValue)) return []; + return datasetValue + .map((dataset) => + dataset && typeof dataset === 'object' && 'datasetId' in dataset + ? String(dataset.datasetId ?? '') + : '' + ) + .filter(Boolean); + }, [inputs]); + + const editorVariables = useMemoEnhance(() => { + return getEditorVariables({ + nodeId, + getNodeById, + edges, + appDetail, + t + }); + }, [nodeId, getNodeById, edges, appDetail, t]); + + const externalVariables = useMemo(() => { + return ( + feConfigs?.externalProviderWorkflowVariables?.map((item) => ({ + key: item.key, + label: item.name + })) ?? [] + ); + }, [feConfigs?.externalProviderWorkflowVariables]); + + const allVariables = useMemo( + () => [...(editorVariables ?? []), ...externalVariables], + [editorVariables, externalVariables] + ); + + const onChange = useCallback( + (value: DatasetTagFilterValue | string) => { + onChangeNode({ + nodeId, + type: 'updateInput', + key: item.key, + value: { ...item, value } + }); + }, + [item, nodeId, onChangeNode] + ); + + if (isLegacyNode) { + return ( + + ); + } + + return ( + + ); +}; + +/** 标题右侧组件：新版显示 AND/OR 切换；旧版显示「已弃用，升级到最新版本」 */ +export const DatasetTagFilterLogic = React.memo(function DatasetTagFilterLogic({ + inputs = [], + item, + nodeId +}: RenderInputProps) { + const onChangeNode = useContextSelector(WorkflowActionsContext, (v) => v.onChangeNode); + const flowData2StoreData = useContextSelector(WorkflowUtilsContext, (v) => v.flowData2StoreData); + const { appDetail, onSaveApp } = useContextSelector(AppContext, (v) => v); + const isLegacyNode = datasetSearchUsesLegacyFilter(inputs); + + if (isLegacyNode) { + return ( + { + const upgradedInput = { + ...item, + renderTypeList: [ + FlowNodeInputTypeEnum.datasetTagFilter, + FlowNodeInputTypeEnum.reference + ], + selectedType: FlowNodeInputTypeEnum.datasetTagFilter, + label: + DatasetSearchModule.inputs.find((input) => input.key === item.key)?.label ?? + item.label, + description: + DatasetSearchModule.inputs.find((input) => input.key === item.key)?.description ?? + item.description, + value: createEmptyTagFilterValue() + }; + const workflow = flowData2StoreData(); + if (!workflow) throw new Error('Workflow data is unavailable'); + await persistLegacyDatasetSearchNodeUpgrade({ + nodes: workflow.nodes, + nodeId, + filterInput: upgradedInput, + persist: (nodes) => + onSaveApp({ + ...workflow, + nodes, + isPublish: false, + chatConfig: appDetail.chatConfig + }), + commit: (upgradedNode) => + onChangeNode({ + nodeId, + type: 'attr', + key: 'inputs', + value: upgradedNode.inputs + }) + }); + }} + /> + ); + } + + return ( + { + onChangeNode({ + nodeId, + type: 'updateInput', + key: item.key, + value: { ...item, value } + }); + }} + /> + ); +}); + +export default React.memo(DatasetTagFilterRender); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx new file mode 100644 index 000000000000..2a19106bd478 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagBatchModal.tsx @@ -0,0 +1,578 @@ +import React, { useMemo, useState } from 'react'; +import { Box, Button, Checkbox, Flex, HStack, useDisclosure } from '@chakra-ui/react'; +import MyModal from '@fastgpt/web/components/v2/common/MyModal'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs'; +import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { postBatchSetCollectionTags } from '@/web/core/dataset/api/collection'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { + BatchCollectionTagModeEnum, + type BatchSetCollectionTagItem +} from '@fastgpt/global/openapi/core/dataset/collection/tagApi'; +import { + DatasetCollectionTagTypeEnum, + type CollectionTagValueType, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; +import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; +import { resolveDisplayedCollectionTag, TagTableContainer } from './TagCommon'; +import TagManageModal from './TagManageModal'; +import { useAppendDatasetTagOption } from './useAppendDatasetTagOption'; +import { useCollectionTagRows } from './useCollectionTagRows'; +import { CollectionTagTable } from './CollectionTagTable'; +import { createEmptyTagRow, type CollectionTagRow } from './tagForm'; + +type BatchTagMode = (typeof BatchCollectionTagModeEnum)[keyof typeof BatchCollectionTagModeEnum]; + +type TagRowType = CollectionTagRow & { + append?: boolean; +}; + +type CollectionTagBatchModalProps = { + collections: DatasetCollectionsListItemType[]; + onClose: () => void; + onSuccess?: () => void; +}; + +type RemoveTagGroup = { + tagId: string; + tagName: string; + tagType: DatasetTagType['tagType']; + values: string[]; +}; + +const createEmptyBatchTagRow = (): TagRowType => ({ + ...createEmptyTagRow(), + append: true +}); + +/** 从已选集合聚合可移除的标签：选项类收集去重后的值，其余类型只展示标签名。 */ +const buildRemoveTagGroups = ( + collections: DatasetCollectionsListItemType[], + tags: DatasetTagType[] +): RemoveTagGroup[] => { + const groups = new Map(); + + for (const collection of collections) { + for (const item of collection.tags ?? []) { + const resolved = resolveDisplayedCollectionTag(item, tags); + if (!resolved) continue; + + const tagId = String(resolved.tagDoc._id); + const tagType = resolved.tagDoc.tagType ?? DatasetCollectionTagTypeEnum.string; + const group = groups.get(tagId) ?? { + tagId, + tagName: resolved.tagDoc.tag, + tagType, + values: [] + }; + + if (tagType === DatasetCollectionTagTypeEnum.array) { + const list = Array.isArray(resolved.value) + ? resolved.value + : resolved.value !== '' + ? [String(resolved.value)] + : []; + for (const value of list) { + if (value && !group.values.includes(value)) { + group.values.push(value); + } + } + } + + groups.set(tagId, group); + } + } + + return [...groups.values()]; +}; + +const CollectionTagBatchModal = ({ + collections, + onClose, + onSuccess +}: CollectionTagBatchModalProps) => { + const { t } = useTranslation(); + const { toast } = useToast(); + const { datasetDetail, allDatasetTags, isLoadingAllDatasetTags, loadAllDatasetTags } = + useContextSelector(DatasetPageContext, (v) => v); + const { + isOpen: isTagManageOpen, + onOpen: onOpenTagManage, + onClose: onCloseTagManage + } = useDisclosure(); + + const [mode, setMode] = useState(BatchCollectionTagModeEnum.add); + const [initialRows] = useState(() => [createEmptyBatchTagRow()]); + const [expandedTagIds, setExpandedTagIds] = useState>(new Set()); + const [selectedTagIds, setSelectedTagIds] = useState>(new Set()); + const [selectedValues, setSelectedValues] = useState>>(new Map()); + + const { + rows, + hasIncompleteRow, + isAddDisabled, + updateRows, + handleAddRow, + handleDeleteRow, + handleTagChange, + handleValueChange + } = useCollectionTagRows({ + tags: allDatasetTags, + initialRows, + createRow: createEmptyBatchTagRow, + patchOnTagChange: (tagDoc) => ({ + append: + (tagDoc?.tagType ?? DatasetCollectionTagTypeEnum.string) === + DatasetCollectionTagTypeEnum.number + }), + onRowDeleted: () => { + toast({ + title: t('dataset:tag.delete_success'), + status: 'success' + }); + } + }); + + const removeGroups = useMemo( + () => buildRemoveTagGroups(collections, allDatasetTags), + [collections, allDatasetTags] + ); + + const { runAsync: onSave, loading: isSaving } = useRequest( + (body: { mode: typeof mode; tags: BatchSetCollectionTagItem[] }) => + postBatchSetCollectionTags({ + datasetId: datasetDetail._id, + collectionIds: collections.map((item) => item._id), + mode: body.mode, + tags: body.tags + }), + { + onSuccess() { + void loadAllDatasetTags(); + onSuccess?.(); + onClose(); + }, + successToast: t('dataset:tag.setting_success'), + errorToast: t('dataset:tag.save_failed') + } + ); + + const { runAsync: onAppendTagOption } = useAppendDatasetTagOption(); + + const removePayload = useMemo(() => { + const tags: BatchSetCollectionTagItem[] = []; + for (const group of removeGroups) { + if (group.tagType === DatasetCollectionTagTypeEnum.array) { + const selected = selectedValues.get(group.tagId); + if (!selected || selected.size === 0) continue; + if (selected.size === group.values.length) { + tags.push({ tagId: group.tagId }); + } else { + tags.push({ tagId: group.tagId, value: [...selected] }); + } + continue; + } + if (selectedTagIds.has(group.tagId)) { + tags.push({ tagId: group.tagId }); + } + } + return tags; + }, [removeGroups, selectedTagIds, selectedValues]); + + const isConfirmDisabled = + isSaving || + (mode === BatchCollectionTagModeEnum.add + ? rows.length === 0 || hasIncompleteRow + : removePayload.length === 0); + + const toggleExpand = (tagId: string) => { + setExpandedTagIds((prev) => { + const next = new Set(prev); + if (next.has(tagId)) next.delete(tagId); + else next.add(tagId); + return next; + }); + }; + + const toggleNonArrayTag = (tagId: string) => { + setSelectedTagIds((prev) => { + const next = new Set(prev); + if (next.has(tagId)) next.delete(tagId); + else next.add(tagId); + return next; + }); + }; + + const toggleArrayValue = (tagId: string, value: string) => { + setSelectedValues((prev) => { + const next = new Map(prev); + const current = new Set(next.get(tagId) ?? []); + if (current.has(value)) current.delete(value); + else current.add(value); + if (current.size === 0) next.delete(tagId); + else next.set(tagId, current); + return next; + }); + }; + + const toggleArrayParent = (tagId: string, allValues: string[]) => { + setSelectedValues((prev) => { + const next = new Map(prev); + const current = next.get(tagId); + const isAllSelected = current && current.size === allValues.length && allValues.length > 0; + if (isAllSelected) { + next.delete(tagId); + } else { + next.set(tagId, new Set(allValues)); + } + return next; + }); + }; + + const handleSubmit = async () => { + if (isConfirmDisabled) return; + + if (mode === BatchCollectionTagModeEnum.remove) { + await onSave({ mode, tags: removePayload }); + return; + } + + const tags: BatchSetCollectionTagItem[] = rows.map((row) => { + const tagDoc = allDatasetTags.find((tag) => String(tag._id) === row.tagId); + const tagType = tagDoc?.tagType ?? DatasetCollectionTagTypeEnum.string; + return { + tagId: row.tagId, + value: row.value as CollectionTagValueType['value'], + ...(tagType === DatasetCollectionTagTypeEnum.number ? { append: Boolean(row.append) } : {}) + }; + }); + + await onSave({ mode: BatchCollectionTagModeEnum.add, tags }); + }; + + const renderAddValueExtra = (row: TagRowType, selectedTag?: DatasetTagType) => { + const tagType = selectedTag?.tagType ?? DatasetCollectionTagTypeEnum.string; + if (tagType === DatasetCollectionTagTypeEnum.number) { + return ( + + {( + [ + { label: t('dataset:tag.append'), append: true }, + { label: t('dataset:tag.overwrite'), append: false } + ] as const + ).map((item) => { + const checked = Boolean(row.append) === item.append; + return ( + + updateRows((current) => + current.map((currentRow) => + currentRow.id === row.id ? { ...currentRow, append: item.append } : currentRow + ) + ) + } + > + + + + + {item.label} + + + ); + })} + + ); + } + if ( + tagType === DatasetCollectionTagTypeEnum.array || + tagType === DatasetCollectionTagTypeEnum.datetime + ) { + return ( + + {t('dataset:tag.overwrite_existing')} + + + ); + } + return null; + }; + + return ( + <> + + {t('dataset:tag.batch_edit')} + {t('dataset:tag.batch_selected_files', { num: collections.length })} + + } + closeOnOverlayClick={false} + bodyStyles={{ + flex: 1, + minH: 0, + pb: 0, + overflow: 'hidden' + }} + footerStyles={{ + justifyContent: 'space-between', + px: 8, + pt: 2, + pb: 8 + }} + footer={ + <> + + + + + + + + } + borderRadius={'10px'} + sx={{ + '.chakra-modal__close-btn': { + top: 2, + right: 2, + w: '36px', + h: '36px' + } + }} + > + + + list={[ + { label: t('dataset:tag.batch_add'), value: BatchCollectionTagModeEnum.add }, + { label: t('dataset:tag.batch_remove'), value: BatchCollectionTagModeEnum.remove } + ]} + value={mode} + onChange={(next) => { + setMode(next); + setExpandedTagIds(new Set()); + setSelectedTagIds(new Set()); + setSelectedValues(new Map()); + }} + outerHeight={'40px'} + itemHeight={'32px'} + px={'12px'} + alignSelf={'flex-start'} + flexShrink={0} + /> + + {mode === BatchCollectionTagModeEnum.add ? ( + { + void onAppendTagOption({ tagId, option }); + }} + renderValueExtra={renderAddValueExtra} + valueFieldProps={{ + disabledPlaceholder: t('dataset:tag.select_tag_first'), + numberShowStepper: true, + numberPlaceholder: t('dataset:tag.fill_integer'), + arrayPlaceholder: t('dataset:tag.select_tag_value') + }} + /> + ) : ( + + + {removeGroups.length === 0 ? ( + + ) : ( + + {removeGroups.map((group) => { + const isArray = group.tagType === DatasetCollectionTagTypeEnum.array; + const selected = selectedValues.get(group.tagId) ?? new Set(); + const isAllSelected = + isArray && group.values.length > 0 && selected.size === group.values.length; + const isIndeterminate = + isArray && selected.size > 0 && selected.size < group.values.length; + const isExpanded = expandedTagIds.has(group.tagId); + const count = isArray ? group.values.length : 1; + + return ( + + isArray + ? toggleArrayParent(group.tagId, group.values) + : toggleNonArrayTag(group.tagId) + } + > + + + + + {`${group.tagName} (${count})`} + + + {isArray && group.values.length > 0 && ( + { + e.stopPropagation(); + toggleExpand(group.tagId); + }} + > + + + )} + + {isArray && + isExpanded && + group.values.map((value) => ( + { + e.stopPropagation(); + toggleArrayValue(group.tagId, value); + }} + > + + + {value} + + + ))} + + ); + })} + + )} + + + )} + + + {isTagManageOpen && } + + ); +}; + +export default React.memo(CollectionTagBatchModal); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagSetModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagSetModal.tsx new file mode 100644 index 000000000000..ae799df67bf2 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagSetModal.tsx @@ -0,0 +1,183 @@ +import React, { useMemo } from 'react'; +import { Box, Button, Flex, HStack, useDisclosure } from '@chakra-ui/react'; +import MyModal from '@fastgpt/web/components/v2/common/MyModal'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { postSetCollectionTags } from '@/web/core/dataset/api/collection'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { type CollectionTagValueType } from '@fastgpt/global/core/dataset/type'; +import TagManageModal from './TagManageModal'; +import { useAppendDatasetTagOption } from './useAppendDatasetTagOption'; +import { useCollectionTagRows } from './useCollectionTagRows'; +import { CollectionTagTable } from './CollectionTagTable'; +import { collectionTagsToRows, createEmptyTagRow } from './tagForm'; + +type CollectionTagSetModalProps = { + collection: DatasetCollectionsListItemType; + onClose: () => void; + onSuccess?: () => void; +}; + +const CollectionTagSetModal = ({ collection, onClose, onSuccess }: CollectionTagSetModalProps) => { + const { t } = useTranslation(); + const { toast } = useToast(); + const { datasetDetail, allDatasetTags, isLoadingAllDatasetTags, loadAllDatasetTags } = + useContextSelector(DatasetPageContext, (v) => v); + const { + isOpen: isTagManageOpen, + onOpen: onOpenTagManage, + onClose: onCloseTagManage + } = useDisclosure(); + + const initialRows = useMemo( + () => collectionTagsToRows(collection.tags, allDatasetTags), + [collection.tags, allDatasetTags] + ); + + const { + rows, + hasIncompleteRow, + isAddDisabled, + handleAddRow, + handleDeleteRow, + handleTagChange, + handleValueChange + } = useCollectionTagRows({ + tags: allDatasetTags, + initialRows, + createRow: createEmptyTagRow, + onRowDeleted: () => { + toast({ + title: t('dataset:tag.delete_success'), + status: 'success' + }); + } + }); + + const { runAsync: onSaveTags, loading: isSaving } = useRequest( + (tags: CollectionTagValueType[]) => + postSetCollectionTags({ + datasetId: datasetDetail._id, + collectionId: collection._id, + tags + }), + { + onSuccess() { + void loadAllDatasetTags(); + onSuccess?.(); + onClose(); + }, + successToast: t('dataset:tag.setting_success'), + errorToast: t('dataset:tag.save_failed') + } + ); + + const { runAsync: onAppendTagOption } = useAppendDatasetTagOption(); + + const isConfirmDisabled = hasIncompleteRow || isSaving; + + const handleSubmit = async () => { + if (isConfirmDisabled) return; + + await onSaveTags(rows.map((row) => ({ tagId: row.tagId, value: row.value }))); + }; + + return ( + <> + + {t('dataset:tag.setting_title')} + + + } + closeOnOverlayClick={false} + bodyStyles={{ + flex: 1, + minH: 0, + pb: 0, + overflow: 'hidden' + }} + footerStyles={{ + justifyContent: 'space-between', + px: 8, + pt: 2, + pb: 8 + }} + footer={ + <> + + + + + + + + } + borderRadius={'10px'} + sx={{ + '.chakra-modal__close-btn': { + top: 2, + right: 2, + w: '36px', + h: '36px' + } + }} + > + { + void onAppendTagOption({ tagId, option }); + }} + /> + + {isTagManageOpen && } + + ); +}; + +export default React.memo(CollectionTagSetModal); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx new file mode 100644 index 000000000000..68c4eb422b8c --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/CollectionTagTable.tsx @@ -0,0 +1,130 @@ +import React from 'react'; +import { Box, Flex } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; +import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; +import { TagActionButton, TagTableContainer, TagTableHeader } from './TagCommon'; +import { TagNameSelect, TagValueField } from './TagValueInputs'; +import { SET_TAG_TABLE_COLUMNS, unusedTagSelectOptions, type CollectionTagRow } from './tagForm'; + +type CollectionTagTableProps = { + rows: T[]; + tags: DatasetTagType[]; + onTagChange: (rowId: string, tagId: string) => void; + onValueChange: (rowId: string, value: string | number | string[]) => void; + onDeleteRow: (rowId: string) => void; + onManage: () => void; + onCreateOption?: (tagId: string, option: string) => void; + /** 批量场景在标签值下方预留固定 16px，避免选类型时行高抖动。 */ + renderValueExtra?: (row: T, tag?: DatasetTagType) => React.ReactNode; + valueFieldProps?: { + disabledPlaceholder?: string; + numberShowStepper?: boolean; + numberPlaceholder?: string; + arrayPlaceholder?: string; + }; +}; + +/** 设置标签 / 批量添加共用的标签表，行高只由是否预留 extra 槽决定，不随当前类型变化。 */ +export const CollectionTagTable = ({ + rows, + tags, + onTagChange, + onValueChange, + onDeleteRow, + onManage, + onCreateOption, + renderValueExtra, + valueFieldProps +}: CollectionTagTableProps) => { + const { t } = useTranslation(); + const hasExtraSlot = Boolean(renderValueExtra); + const rowH = hasExtraSlot ? '96px' : '80px'; + + return ( + + + {t('dataset:tag.name')} + {t('dataset:tag.value')} + {t('common:Operation')} + + + + {rows.length === 0 ? ( + + ) : ( + rows.map((row) => { + const selectedTag = tags.find((tag) => String(tag._id) === row.tagId); + + return ( + + + onTagChange(row.id, newTagId)} + onManage={onManage} + /> + + + + + onValueChange(row.id, val)} + onCreateOption={(option) => { + if (!selectedTag) return; + onCreateOption?.(String(selectedTag._id), option); + }} + {...valueFieldProps} + /> + + {hasExtraSlot && ( + + {renderValueExtra?.(row, selectedTag)} + + )} + + + + } + hoverColor={'red.600'} + onClick={() => onDeleteRow(row.id)} + /> + + + ); + }) + )} + + + ); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx index 983641a9ba1a..550c5d25abed 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx @@ -21,6 +21,7 @@ import dynamic from 'next/dynamic'; import { usePagination } from '@fastgpt/web/hooks/usePagination'; import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { type CollectionTagFilterItem } from '@fastgpt/global/core/dataset/type'; import { useRouter } from 'next/router'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { type WebsiteConfigFormType } from './WebsiteConfig'; @@ -40,8 +41,8 @@ type CollectionPageContextType = { scrollContainerRef: RefObject; searchText: string; setSearchText: Dispatch>; - filterTags: string[]; - setFilterTags: Dispatch>; + tagFilters: CollectionTagFilterItem[]; + setTagFilters: Dispatch>; }; export const CollectionPageContext = createContext({ @@ -67,8 +68,8 @@ export const CollectionPageContext = createContext({ setSearchText: function (_value: SetStateAction): void { throw new Error('Function not implemented.'); }, - filterTags: [], - setFilterTags: function (_value: SetStateAction): void { + tagFilters: [], + setTagFilters: function (_value: SetStateAction): void { throw new Error('Function not implemented.'); } }); @@ -85,7 +86,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => // collection list const [searchText, setSearchText] = useState(''); - const [filterTags, setFilterTags] = useState([]); + const [tagFilters, setTagFilters] = useState([]); const scrollContainerRef = useRef(null); const { data: collections, @@ -103,9 +104,9 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => datasetId, parentId, searchText, - filterTags + tagFilters }, - refreshDeps: [parentId, searchText, filterTags], + refreshDeps: [parentId, searchText, tagFilters], scrollContainerRef }); @@ -161,8 +162,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => searchText, setSearchText, - filterTags, - setFilterTags, + tagFilters, + setTagFilters, collections, Pagination, total, @@ -175,7 +176,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => [ Pagination, collections, - filterTags, + tagFilters, getData, isGetting, onOpenWebsiteModal, diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx new file mode 100644 index 000000000000..9cb55ee7a90e --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/DatasetTagFilter.tsx @@ -0,0 +1,89 @@ +import { useContextSelector } from 'use-context-selector'; +import { useTranslation } from 'next-i18next'; +import { useEffect, useMemo } from 'react'; +import { MultiTagFilter, type MultiTagFilterGroup } from '@fastgpt/web/components/common/TagFilter'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { CollectionPageContext } from './Context'; +import { getDatasetTagFilterOptions } from '@/web/core/dataset/api/collection'; +import { buildTagFilterValues, formatCollectionTagValueText } from './TagCommon'; + +const DatasetTagFilter = () => { + const { t } = useTranslation(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags); + const isLoadingAllDatasetTags = useContextSelector( + DatasetPageContext, + (v) => v.isLoadingAllDatasetTags + ); + const tagFilters = useContextSelector(CollectionPageContext, (v) => v.tagFilters); + const setTagFilters = useContextSelector(CollectionPageContext, (v) => v.setTagFilters); + + const { + runAsync: loadUsedValues, + data: usedValuesRes, + loading: isLoadingUsedValues + } = useRequest(() => getDatasetTagFilterOptions(datasetDetail._id), { + manual: false, + ready: !!datasetDetail._id, + refreshDeps: [datasetDetail._id] + }); + + const usedValuesByTagId = useMemo(() => { + const valueMap = new Map>(); + for (const item of usedValuesRes?.list ?? []) { + valueMap.set(item.tagId, item.values); + } + return valueMap; + }, [usedValuesRes]); + + const groups = useMemo( + () => + allDatasetTags.map((tag) => { + const tagId = String(tag._id); + return { + tagId, + label: tag.tag, + values: buildTagFilterValues(tag, usedValuesByTagId.get(tagId) ?? []).map((value) => ({ + value, + label: formatCollectionTagValueText(value, tag.tagType) + })) + }; + }), + [allDatasetTags, usedValuesByTagId] + ); + + useEffect(() => { + if (allDatasetTags.length === 0) return; + const validTagIds = new Set(allDatasetTags.map((tag) => String(tag._id))); + setTagFilters((prev) => { + const next = prev.filter((item) => validTagIds.has(item.tagId)); + return next.length === prev.length ? prev : next; + }); + }, [allDatasetTags, setTagFilters]); + + return ( + { + void loadUsedValues(); + }} + labels={{ + title: t('dataset:tag.tags'), + all: t('common:All'), + searchPlaceholder: t('dataset:tag.filter_search'), + selected: t('dataset:tag.filter_selected'), + item: t('dataset:tag.filter_item'), + clear: t('dataset:tag.filter_clear'), + noValues: t('dataset:tag.filter_no_values'), + noMatch: t('dataset:tag.filter_no_match') + }} + /> + ); +}; + +export default DatasetTagFilter; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx index 30c098e71671..ef1d6e1f39a9 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx @@ -10,8 +10,31 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex const EmptyCollectionTip = () => { const { t } = useTranslation(); const onOpenWebsiteModal = useContextSelector(CollectionPageContext, (v) => v.onOpenWebsiteModal); + const tagFilters = useContextSelector(CollectionPageContext, (v) => v.tagFilters); + const setTagFilters = useContextSelector(CollectionPageContext, (v) => v.setTagFilters); const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + if (tagFilters.length > 0) { + return ( + + {t('dataset:tag.filter_empty_prefix')} + { + setTagFilters([]); + }} + > + {t('dataset:tag.filter_clear_items')} + + + } + /> + ); + } + return ( <> {(datasetDetail.type === DatasetTypeEnum.dataset || diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx index 7b38541fc506..09f71a0e1a74 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Box, Flex, MenuButton, Button, Link, useDisclosure, HStack } from '@chakra-ui/react'; +import { Box, Flex, Button, Link, useDisclosure, HStack } from '@chakra-ui/react'; import { getDatasetCollectionPathById, postDatasetCollection, @@ -30,16 +30,33 @@ import { useContextSelector } from 'use-context-selector'; import { CollectionPageContext } from './Context'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { useSystem } from '@fastgpt/web/hooks/useSystem'; -import HeaderTagPopOver from './HeaderTagPopOver'; +import DatasetTagFilter from './DatasetTagFilter'; import MyBox from '@fastgpt/web/components/common/MyBox'; import Icon from '@fastgpt/web/components/common/Icon'; import MyTag from '@fastgpt/web/components/common/Tag/index'; import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import TagManageModal from './TagManageModal'; const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector')); const BackupImportModal = dynamic(() => import('./BackupImportModal')); const TemplateImportModal = dynamic(() => import('./TemplateImportModal')); +const HeaderImportButton = ({ children }: { children: React.ReactNode }) => ( + +); + const Header = ({ hasTrainingData, hasTrainingError, @@ -99,6 +116,11 @@ const Header = ({ onOpen: onOpenTemplateImportModal, onClose: onCloseTemplateImportModal } = useDisclosure(); + const { + isOpen: isTagManageModalOpen, + onOpen: onOpenTagManageModal, + onClose: onCloseTagManageModal + } = useDisclosure(); const { runAsync: onCreateCollection } = useRequest( async ({ @@ -128,9 +150,9 @@ const Header = ({ const isWebSite = datasetDetail?.type === DatasetTypeEnum.websiteDataset; return ( - - - + + + ({ parentId: path.parentId, @@ -180,8 +202,10 @@ const Header = ({ {/* search input */} {isPc && ( } + {datasetDetail.type !== DatasetTypeEnum.websiteDataset && feConfigs?.isPlus && ( + + )} + - {hasTrainingError && ( + {hasTrainingError && ( + + )} + {datasetDetail.type !== DatasetTypeEnum.websiteDataset && + datasetDetail.permission.hasWritePer && + feConfigs?.isPlus && ( )} - {/* diff collection button */} {datasetDetail.permission.hasWritePer && ( @@ -230,35 +268,9 @@ const Header = ({ - - - - - - {t('common:dataset.collections.Create And Import')} - - - + + {t('common:dataset.collections.Create And Import')} + } menuList={[ { @@ -421,35 +433,9 @@ const Header = ({ - - - - - - {t('common:dataset.collections.Create And Import')} - - - + + {t('common:dataset.collections.Create And Import')} + } menuList={[ { @@ -618,6 +604,14 @@ const Header = ({ onClose={onCloseTemplateImportModal} /> )} + {isTagManageModalOpen && ( + { + onCloseTagManageModal(); + getData(1); + }} + /> + )} ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx deleted file mode 100644 index c945557dec18..000000000000 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx +++ /dev/null @@ -1,211 +0,0 @@ -import { Box, Button, Checkbox, Flex, Input, useDisclosure } from '@chakra-ui/react'; -import MyPopover from '@fastgpt/web/components/common/MyPopover'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import MyBox from '@fastgpt/web/components/common/MyBox'; -import { useContextSelector } from 'use-context-selector'; -import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { useTranslation } from 'next-i18next'; -import { CollectionPageContext } from './Context'; -import { isEqual } from 'lodash-es'; -import TagManageModal from './TagManageModal'; -import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; - -const HeaderTagPopOver = () => { - const { t } = useTranslation(); - - const { - searchDatasetTagsResult, - searchTagKey, - setSearchTagKey, - checkedDatasetTag, - setCheckedDatasetTag, - onCreateCollectionTag, - isCreateCollectionTagLoading - } = useContextSelector(DatasetPageContext, (v) => v); - - const { filterTags, setFilterTags, getData } = useContextSelector( - CollectionPageContext, - (v) => v - ); - - const checkedTags = filterTags; - - const { - isOpen: isTagManageModalOpen, - onOpen: onOpenTagManageModal, - onClose: onCloseTagManageModal - } = useDisclosure(); - - const checkTags = (tag: DatasetTagType) => { - let currentCheckedTags = []; - if (checkedTags.includes(tag._id)) { - currentCheckedTags = checkedTags.filter((t) => t !== tag._id); - setCheckedDatasetTag(checkedDatasetTag.filter((t) => t._id !== tag._id)); - } else { - currentCheckedTags = [...checkedTags, tag._id]; - setCheckedDatasetTag([...checkedDatasetTag, tag]); - } - if (isEqual(currentCheckedTags, filterTags)) return; - setFilterTags(currentCheckedTags); - }; - - return ( - <> - - - {t('dataset:tag.tags')} - - {checkedTags.length > 0 && ( - - {`(${checkedTags.length})`} - - )} - - - - - } - > - {({ onClose }) => ( - e.stopPropagation()}> - - setSearchTagKey(e.target.value)} - /> - - - - {searchTagKey && - !searchDatasetTagsResult.map((item) => item.tag).includes(searchTagKey) && ( - onCreateCollectionTag(searchTagKey)} - > - - - {t('dataset:tag.add') + ` "${searchTagKey}"`} - - - )} - - {[ - ...new Map( - [...checkedDatasetTag, ...searchDatasetTagsResult].map((item) => [item._id, item]) - ).values() - ].map((item) => { - const checked = checkedTags.includes(item._id); - return ( - { - e.preventDefault(); - checkTags(item); - }} - > - { - checkTags(item); - }} - size={'md'} - icon={} - /> - {item.tag} - - ); - })} - - - - - - - - )} - - {isTagManageModalOpen && ( - { - onCloseTagManageModal(); - getData(1); - }} - /> - )} - - ); -}; - -export default HeaderTagPopOver; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx new file mode 100644 index 000000000000..25e6d800ac07 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagCommon.tsx @@ -0,0 +1,299 @@ +import React, { useLayoutEffect, useRef, useState } from 'react'; +import { Box, Flex, type FlexProps } from '@chakra-ui/react'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import dayjs from 'dayjs'; +import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; +import { + DatasetCollectionTagTypeEnum, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; +import { + isUsableCollectionTagFilterValue, + sortCollectionTagValues +} from '@fastgpt/global/core/dataset/tagUtils'; + +export type CollectionTagDisplayItem = string | { tag: string; value?: string | number | string[] }; + +export const OVERFLOW_CHIP_GAP_PX = 8; + +/** + * 根据测量宽度计算可见 chip 数量。空间不够时至少留 1 个，其余用 +n。 + */ +const countVisibleOverflowChips = ({ + chipWidths, + overflowWidth, + containerWidth, + gapPx = OVERFLOW_CHIP_GAP_PX +}: { + chipWidths: number[]; + overflowWidth: number; + containerWidth: number; + gapPx?: number; +}): number => { + if (chipWidths.length === 0) return 0; + + let usedWidth = 0; + let visibleCount = chipWidths.length; + + for (let i = 0; i < chipWidths.length; i++) { + const isLast = i === chipWidths.length - 1; + const gap = isLast ? 0 : gapPx; + const reserved = isLast ? 0 : overflowWidth; + + if (usedWidth + chipWidths[i] + gap + reserved <= containerWidth) { + usedWidth += chipWidths[i] + gap; + continue; + } + + visibleCount = i; + break; + } + + return visibleCount === 0 ? 1 : visibleCount; +}; + +/** 列表/选项输入共用的 chip 溢出测量。itemKey 变化时重新量。 */ +export const useOverflowChipCount = ({ + itemKey, + itemCount, + gapPx = OVERFLOW_CHIP_GAP_PX +}: { + itemKey: unknown; + itemCount: number; + gapPx?: number; +}) => { + const containerRef = useRef(null); + const measureRef = useRef(null); + const [visibleCount, setVisibleCount] = useState(itemCount); + + useLayoutEffect(() => { + const container = containerRef.current; + const measure = measureRef.current; + if (!container || !measure) { + setVisibleCount(itemCount); + return; + } + + const calculate = () => { + const chipEls = Array.from(measure.querySelectorAll('[data-tag-chip]')) as HTMLElement[]; + const overflowEl = measure.querySelector('[data-overflow-chip]') as HTMLElement | null; + setVisibleCount( + countVisibleOverflowChips({ + chipWidths: chipEls.map((el) => el.offsetWidth), + overflowWidth: overflowEl?.offsetWidth ?? 0, + containerWidth: container.offsetWidth, + gapPx + }) + ); + }; + + calculate(); + const observer = new ResizeObserver(calculate); + observer.observe(container); + return () => observer.disconnect(); + }, [gapPx, itemCount, itemKey]); + + return { containerRef, measureRef, visibleCount }; +}; + +export const formatCollectionTagValueText = ( + value: string | number | string[] | undefined, + tagType?: DatasetTagType['tagType'] +): string => { + if (value == null || value === '') return ''; + if (Array.isArray(value)) return value.filter(Boolean).join(', '); + if (tagType === 'datetime') { + const d = dayjs(value); + return d.isValid() ? formatTime2YMDHM(d.valueOf()) : String(value); + } + return String(value); +}; + +/** + * 把列表返回的展示格式（名称或 { tag, value }）解析成标签定义和值。 + * 旧字符串按名称或 ID 找回定义；选项类旧字符串当作单个选项。 + */ +export const resolveDisplayedCollectionTag = ( + item: CollectionTagDisplayItem, + tags: DatasetTagType[] +) => { + const tagNameOrId = typeof item === 'string' ? item : item.tag; + const tagDoc = tags.find((tag) => tag.tag === tagNameOrId || String(tag._id) === tagNameOrId); + if (!tagDoc) return; + + const value = (() => { + if (typeof item !== 'string') { + if (item.value != null && item.value !== '') return item.value; + return tagDoc.tagType === DatasetCollectionTagTypeEnum.array ? [] : ''; + } + return tagDoc.tagType === DatasetCollectionTagTypeEnum.array ? [item] : item; + })(); + + return { tagDoc, value }; +}; + +/** + * 组装筛选弹窗右侧的值列表。 + * 选项类用标签管理里的预设 options，并与文件上已用但不在预设里的值取并集； + * 文本/数字/日期没有预设列表，只展示已用值。 + */ +export const buildTagFilterValues = ( + tag: Pick, + usedValues: Array = [] +): Array => { + const values = new Map(); + const addValue = (value: string | number) => { + if (!isUsableCollectionTagFilterValue(value)) return; + values.set(String(value), value); + }; + + if (tag.tagType === DatasetCollectionTagTypeEnum.array) { + for (const option of tag.options ?? []) addValue(option); + } + for (const value of usedValues) addValue(value); + + return sortCollectionTagValues(Array.from(values.values())); +}; + +/** 新格式展示「名称：值」；旧字符串标签只展示名称。 */ +export const formatCollectionTagChipText = ( + item: CollectionTagDisplayItem, + tagDefs: DatasetTagType[] = [] +): string => { + if (typeof item === 'string') return item; + + const tagType = tagDefs.find((def) => def.tag === item.tag)?.tagType; + const valueText = formatCollectionTagValueText(item.value, tagType); + return valueText ? `${item.tag}: ${valueText}` : item.tag; +}; + +export const TAG_TOOLTIP_PROPS = { + placement: 'bottom' as const, + offset: [0, 10] as [number, number], + hasArrow: true, + arrowSize: 10, + px: 3, + py: 2, + borderRadius: 'sm', + fontSize: 'xs', + lineHeight: '18px', + color: 'myGray.800', + arrowShadowColor: 'rgba(19, 51, 107, 0.1)', + boxShadow: '0px 4px 5px rgba(19, 51, 107, 0.1), 0px 0px 0.5px rgba(19, 51, 107, 0.1)' +}; + +export const SaveActionIcon = ({ isEnabled }: { isEnabled: boolean }) => { + if (!isEnabled) { + return ; + } + + return ( + + + + + ); +}; + +type TagActionButtonProps = { + label: string; + icon: React.ReactElement; + onClick?: () => void; + isDisabled?: boolean; + color?: string; + hoverColor?: string; + hoverIconClassName?: string; +}; + +export const TagActionButton = ({ + label, + icon, + onClick, + isDisabled = false, + color = 'myGray.500', + hoverColor, + hoverIconClassName +}: TagActionButtonProps) => ( + + + {icon} + + +); + +export const TagTableContainer = ({ children, ...props }: FlexProps) => ( + + {children} + +); + +export const TagTableHeader = ({ + columns, + children +}: { + columns: string; + children: React.ReactNode; +}) => ( + + {children} + +); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx index bf3a71ea334b..13cbf7e0dfaa 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx @@ -1,516 +1,616 @@ -import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { Input, Button, Flex, Box, Checkbox } from '@chakra-ui/react'; -import MyModal from '@fastgpt/web/components/common/MyModal'; +import React, { useEffect, useRef, useState } from 'react'; +import { Box, Button, Flex, Input } from '@chakra-ui/react'; +import MyModal from '@fastgpt/web/components/v2/common/MyModal'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; +import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; +import MySelect from '@fastgpt/web/components/common/MySelect'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { CollectionPageContext } from './Context'; -import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; import { delDatasetCollectionTag, - getDatasetCollectionTags, - getDatasetCollections, - getTagUsage, - postAddTagsToCollections, + postCreateDatasetCollectionTag, updateDatasetCollectionTag } from '@/web/core/dataset/api/collection'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import MyInput from '@/components/MyInput'; -import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; -import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; +import { + DatasetCollectionTagTypeEnum, + DatasetCollectionTagTypeMap, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; -import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm'; -import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; +import MyPopover from '@fastgpt/web/components/common/MyPopover'; +import { SaveActionIcon, TagActionButton, TagTableContainer, TagTableHeader } from './TagCommon'; + +const TAG_TABLE_COLUMNS = 'minmax(0, 1fr) 180px 100px'; +const OPTION_ROW_HEIGHT = 32; +const OPTION_ROW_GAP = 2; +const OPTION_LIST_MAX_ROWS = 4; +const OPTION_LIST_MAX_H = + OPTION_ROW_HEIGHT * OPTION_LIST_MAX_ROWS + OPTION_ROW_GAP * (OPTION_LIST_MAX_ROWS - 1); + +const normalizeTagOptions = (nextOptions: string[]) => [ + ...new Set(nextOptions.map((option) => option.trim()).filter(Boolean)) +]; + +const isSameTagOptions = (left: string[], right: string[]) => + left.length === right.length && left.every((option, index) => option === right[index]); + +/** + * 选项类标签的预设管理。 + * 输入框失焦、回车和关闭弹层时静默保存；Enter 跳到下一行，最后一行 Enter 追加空输入；列表固定 4 行，超出滚动。 + * 保存成功后不要用 options 内容做 React key，否则弹层会卸载，换行新增会丢。 + */ +const TagOptionManagePopover = ({ + options, + onSave +}: { + options: string[]; + onSave: (options: string[]) => Promise; +}) => { + const { t } = useTranslation(); + const [draftOptions, setDraftOptions] = useState(options); + const savedOptionsRef = useRef(options); + const draftOptionsRef = useRef(draftOptions); + const inputRefs = useRef<(HTMLInputElement | null)[]>([]); + + useEffect(() => { + draftOptionsRef.current = draftOptions; + }, [draftOptions]); + + const resetDraft = (nextOptions: string[]) => { + setDraftOptions(nextOptions); + savedOptionsRef.current = nextOptions; + }; + + const persistOptions = async (nextOptions: string[]) => { + const normalizedOptions = normalizeTagOptions(nextOptions); + if (isSameTagOptions(normalizedOptions, savedOptionsRef.current)) return; + + try { + await onSave(normalizedOptions); + savedOptionsRef.current = normalizedOptions; + } catch { + setDraftOptions(savedOptionsRef.current); + } + }; + + const focusOption = (index: number) => { + setTimeout(() => { + inputRefs.current[index]?.focus(); + }, 50); + }; + + const handleAddOption = () => { + setDraftOptions((prev) => { + if (prev.length > 0 && !prev[prev.length - 1]?.trim()) { + focusOption(prev.length - 1); + return prev; + } + const next = [...prev, '']; + focusOption(next.length - 1); + return next; + }); + }; + + const handleUpdateOption = (index: number, value: string) => { + setDraftOptions((prev) => { + const next = [...prev]; + next[index] = value; + return next; + }); + }; + + const handleRemoveOption = (index: number) => { + const next = draftOptionsRef.current.filter((_, i) => i !== index); + setDraftOptions(next); + void persistOptions(next); + }; + + const handleKeyDown = (index: number, e: React.KeyboardEvent) => { + if (e.key !== 'Enter') return; + e.preventDefault(); + const currentDraft = draftOptionsRef.current; + void persistOptions(currentDraft); + if (index === currentDraft.length - 1) { + handleAddOption(); + return; + } + inputRefs.current[index + 1]?.focus(); + }; + + return ( + resetDraft(options)} + onCloseFunc={() => { + void persistOptions(draftOptionsRef.current); + }} + Trigger={ + + + + } + > + {() => ( + + + + + {t('dataset:tag.add_option')} + + + + {draftOptions.length > 0 && ( + + {draftOptions.map((opt, index) => ( + + { + inputRefs.current[index] = el; + }} + value={opt} + flex={1} + minW={0} + h={`${OPTION_ROW_HEIGHT}px`} + px={3} + fontSize={'xs'} + lineHeight={'16px'} + borderRadius={'sm'} + border={'1px solid'} + borderColor={'myGray.200'} + placeholder={t('dataset:tag.enter_option')} + _focus={{ + borderColor: 'primary.600', + boxShadow: 'focus' + }} + onChange={(e) => handleUpdateOption(index, e.target.value)} + onBlur={() => { + void persistOptions(draftOptionsRef.current); + }} + onKeyDown={(e) => handleKeyDown(index, e)} + /> + e.preventDefault()} + onClick={() => handleRemoveOption(index)} + > + + + + ))} + + )} + + )} + + ); +}; const TagManageModal = ({ onClose }: { onClose: () => void }) => { const { t } = useTranslation(); - const { datasetDetail, onCreateCollectionTag, loadAllDatasetTags, setSearchTagKey } = - useContextSelector(DatasetPageContext, (v) => v); - const { getData, pageNum, collections } = useContextSelector(CollectionPageContext, (v) => v); + const { datasetDetail, allDatasetTags, loadAllDatasetTags } = useContextSelector( + DatasetPageContext, + (v) => v + ); const tagInputRef = useRef(null); const editInputRef = useRef(null); - - const [currentAddTag, setCurrentAddTag] = useState< - (DatasetTagType & { collections: string[] }) | undefined - >(undefined); - const [newTag, setNewTag] = useState(undefined); - const [searchText, setSearchText] = useState(''); - + const [newTagType, setNewTagType] = useState(undefined); const [currentEditTagContent, setCurrentEditTagContent] = useState(undefined); const [currentEditTag, setCurrentEditTag] = useState(undefined); useEffect(() => { - if (newTag !== undefined && tagInputRef.current) { + if (newTag !== undefined) { tagInputRef.current?.focus(); } }, [newTag]); useEffect(() => { - if (currentEditTag !== undefined && editInputRef.current) { + if (currentEditTag !== undefined) { editInputRef.current?.focus(); } }, [currentEditTag]); + const { openConfirm: openDeleteConfirm, ConfirmModal: DeleteConfirmModal } = useConfirm({ + type: 'delete', + title: t('dataset:tag.delete_tag_confirm_title'), + content: t('dataset:tag.delete_tag_confirm_content') + }); + const { runAsync: onDeleteCollectionTag } = useRequest( - (tag: string) => + (tagId: string) => delDatasetCollectionTag({ datasetId: datasetDetail._id, - id: tag + id: tagId }), { onSuccess() { - refreshList(); - setSearchTagKey(''); loadAllDatasetTags(); }, - successToast: t('common:delete_success'), - errorToast: t('common:delete_failed') + successToast: t('dataset:tag.delete_success'), + errorToast: t('dataset:tag.delete_failed') + } + ); + + const { runAsync: onCreateCollectionTag } = useRequest( + ({ tag, tagType }: { tag: string; tagType: DatasetCollectionTagTypeEnum }) => + postCreateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tag, + tagType + }), + { + onSuccess() { + loadAllDatasetTags(); + }, + successToast: t('dataset:tag.create_success'), + errorToast: t('dataset:tag.create_failed') } ); const { runAsync: onUpdateCollectionTag } = useRequest( - async (tag: DatasetTagType) => { - return updateDatasetCollectionTag({ + (tag: DatasetTagType) => + updateDatasetCollectionTag({ datasetId: datasetDetail._id, tagId: tag._id, tag: tag.tag - }); - }, + }), { onSuccess() { - refreshList(); - setSearchTagKey(''); loadAllDatasetTags(); - } + }, + successToast: t('dataset:tag.save_success'), + errorToast: t('dataset:tag.save_failed') } ); - const { runAsync: onSaveCollectionTag } = useRequest( - async ({ - tag, - originCollectionIds, - collectionIds - }: { - tag: string; - originCollectionIds: string[]; - collectionIds: string[]; - }) => { - return postAddTagsToCollections({ - tag, - originCollectionIds, - collectionIds, - datasetId: datasetDetail._id - }); - }, + const { runAsync: onSaveTagOptions } = useRequest( + ({ tag, options }: { tag: DatasetTagType; options: string[] }) => + updateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tagId: tag._id, + tag: tag.tag, + options + }), { - onFinally() { - getData(pageNum); - }, - successToast: t('common:save_success'), - errorToast: t('common:save_failed') + onSuccess: loadAllDatasetTags, + errorToast: t('dataset:tag.save_failed') } ); - // Tags list - const { - data: collectionTags, - ScrollData, - refreshList, - total: tagsTotal - } = useScrollPagination(getDatasetCollectionTags, { - pageSize: 10, - params: { - datasetId: datasetDetail._id, - searchText - }, - refreshDeps: [searchText], - EmptyTip: - }); + const tagTypeOptions = [ + DatasetCollectionTagTypeEnum.array, + DatasetCollectionTagTypeEnum.number, + DatasetCollectionTagTypeEnum.datetime + ].map((tagType) => ({ + label: t(DatasetCollectionTagTypeMap[tagType].label), + value: tagType + })); - // Collections list - const { data: collectionsList, ScrollData: ScrollDataCollections } = useScrollPagination( - getDatasetCollections, - { - pageSize: 30, - params: { - datasetId: datasetDetail._id, - simple: true, - searchText - }, - refreshDeps: [searchText], - EmptyTip: + const submitNewTag = async () => { + const tag = newTag?.trim(); + if (!tag || !newTagType || allDatasetTags.some((item) => item.tag === tag)) return; + + await onCreateCollectionTag({ tag, tagType: newTagType }); + setNewTag(undefined); + setNewTagType(undefined); + }; + + const submitUpdatedTag = async (tag: DatasetTagType) => { + const content = currentEditTagContent?.trim(); + if ( + content && + content !== tag.tag && + !allDatasetTags.some((item) => item._id !== tag._id && item.tag === content) + ) { + await onUpdateCollectionTag({ ...tag, tag: content }); } - ); + setCurrentEditTag(undefined); + setCurrentEditTagContent(undefined); + }; - const { data: tagUsages } = useRequest(() => getTagUsage(datasetDetail._id), { - manual: false, - refreshDeps: [collections] - }); + const cancelNewTag = () => { + setNewTag(undefined); + setNewTagType(undefined); + }; + + const canSaveNewTag = Boolean( + newTag?.trim() && newTagType && !allDatasetTags.some((item) => item.tag === newTag.trim()) + ); return ( + {t('dataset:tag.manage')} + + + } closeOnOverlayClick={false} + bodyStyles={{ + flex: 1, + minH: 0, + pb: 0, + overflow: 'hidden' + }} + footerStyles={{ + justifyContent: 'flex-start', + px: 8, + pt: 2, + pb: 8 + }} + footer={ + + } + borderRadius={'10px'} + sx={{ + '.chakra-modal__close-btn': { + top: 2, + right: 2, + w: '36px', + h: '36px' + } + }} > - {currentAddTag === undefined ? ( - <> - - - - {t('dataset:tag.total_tags', { - total: tagsTotal - })} - - { - setSearchText(e.target.value); - }} - /> - - - - {newTag !== undefined && ( - - setNewTag(e.target.value)} - ref={tagInputRef} - w={'200px'} - onBlur={async () => { - if (newTag && !collectionTags.map((item) => item.tag).includes(newTag)) { - await onCreateCollectionTag(newTag); - refreshList(); - } - setNewTag(undefined); - }} - /> - - )} - - - {collectionTags.map((item) => { - const tagUsage = tagUsages?.find((tagUsage) => tagUsage.tagId === item._id); - const collections = tagUsage?.collections || []; - const usage = collections.length; + + + {t('dataset:tag.name')} + {t('dataset:tag.attribute')} + {t('common:Operation')} + + + + {allDatasetTags.length === 0 ? ( + + ) : ( + allDatasetTags.map((tag) => { + const isEditing = currentEditTag?._id === tag._id; + const tagType = tag.tagType ?? DatasetCollectionTagTypeEnum.string; + const editedTagContent = currentEditTagContent?.trim(); + const canSaveEditedTag = Boolean( + editedTagContent && + editedTagContent !== tag.tag && + !allDatasetTags.some( + (item) => item._id !== tag._id && item.tag === editedTagContent + ) + ); return ( - - - { - setCurrentAddTag({ ...item, collections }); - }} - cursor={'pointer'} - > - {currentEditTag?._id !== item._id ? ( - - {item.tag} - - ) : ( - setCurrentEditTagContent(e.target.value)} - ref={editInputRef} - w={'200px'} - onBlur={() => { - if ( - currentEditTagContent && - !collectionTags - .map((item) => item.tag) - .includes(currentEditTagContent) - ) { - onUpdateCollectionTag({ - tag: currentEditTagContent, - _id: item._id - }); + + {isEditing ? ( + setCurrentEditTagContent(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + if (canSaveEditedTag) { + void submitUpdatedTag(tag); } + } + }} + /> + ) : ( + + {tag.tag} + + )} + + + {t(DatasetCollectionTagTypeMap[tagType].label)} + {tagType === DatasetCollectionTagTypeEnum.array && ( + onSaveTagOptions({ tag, options: nextOptions })} + /> + )} + + + {isEditing ? ( + <> + } + isDisabled={!canSaveEditedTag} + hoverIconClassName={canSaveEditedTag ? 'tag-save-icon-hover' : undefined} + onClick={() => void submitUpdatedTag(tag)} + /> + } + hoverColor={'myGray.700'} + onClick={() => { setCurrentEditTag(undefined); setCurrentEditTagContent(undefined); }} /> - )} - {`(${usage})`} - - { - setCurrentAddTag({ ...item, collections }); - setSearchText(''); - }} - cursor={'pointer'} - > - - - { - setCurrentEditTag(item); - editInputRef.current?.focus(); - }} - > - - - - - - } - onConfirm={() => onDeleteCollectionTag(item._id)} - /> + + ) : ( + <> + } + hoverColor={'primary.700'} + onClick={() => { + setCurrentEditTag(tag); + setCurrentEditTagContent(tag.tag); + }} + /> + } + hoverColor={'red.600'} + onClick={() => { + openDeleteConfirm({ + onConfirm: () => onDeleteCollectionTag(tag._id) + })(); + }} + /> + + )} - + ); - })} - - - ) : ( - - )} - - ); -}; + }) + )} -export default TagManageModal; - -const AddTagToCollections = ({ - currentAddTag, - setCurrentAddTag, - onSaveCollectionTag, - setSearchText, - collectionsList, - ScrollDataCollections -}: { - currentAddTag: DatasetTagType & { collections: string[] }; - setCurrentAddTag: (tag: (DatasetTagType & { collections: string[] }) | undefined) => void; - onSaveCollectionTag: ({ - tag, - originCollectionIds, - collectionIds - }: { - tag: string; - originCollectionIds: string[]; - collectionIds: string[]; - }) => void; - setSearchText: (text: string) => void; - collectionsList: DatasetCollectionsListItemType[]; - ScrollDataCollections: ReturnType['ScrollData']; -}) => { - const { t } = useTranslation(); - - const [selectedCollections, setSelectedCollections] = useState( - currentAddTag.collections - ); - const [originCollections, setOriginCollections] = useState(currentAddTag.collections); - - const formatCollections = useMemo( - () => - collectionsList.map((collection) => { - const icon = getCollectionIcon({ type: collection.type, name: collection.name }); - return { - id: collection._id, - tags: collection.tags, - name: collection.name, - icon - }; - }), - [collectionsList] - ); - - return ( - <> - - { - setCurrentAddTag(undefined); - setSearchText(''); - }} - /> - { - - - {currentAddTag.tag} - + {newTag !== undefined && ( {`(${selectedCollections.length})`} - - } - - { - setSearchText(e.target.value); - }} - /> - - - - {formatCollections.map((collection) => { - return ( - { - setSelectedCollections((prev) => { - if (prev.includes(collection.id)) { - return prev.filter((id) => id !== collection.id); - } else { - return [...prev, collection.id]; - } - }); - }} + h={'80px'} + flexShrink={0} + borderBottom={'1px solid'} + borderColor={'myGray.150'} > - } - onChange={() => { - setSelectedCollections((prev) => { - if (prev.includes(collection.id)) { - return prev.filter((id) => id !== collection.id); - } else { - return [...prev, collection.id]; - } - }); - }} - isChecked={selectedCollections.includes(collection.id)} - /> - - - {collection.name} + + setNewTag(e.target.value)} + /> - - ); - })} - - + + + value={newTagType} + placeholder={t('dataset:tag.select_attribute')} + list={tagTypeOptions} + width={'100%'} + h={'36px'} + fontSize={'sm'} + lineHeight={'20px'} + letterSpacing={'0.25px'} + onChange={(val) => setNewTagType(val)} + menuPlacement={'bottom-start'} + /> + + + } + isDisabled={!canSaveNewTag} + hoverIconClassName={canSaveNewTag ? 'tag-save-icon-hover' : undefined} + onClick={() => void submitNewTag()} + /> + } + hoverColor={'myGray.700'} + onClick={cancelNewTag} + /> + + + )} + + + + ); }; + +export default React.memo(TagManageModal); diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx new file mode 100644 index 000000000000..196b39b6af16 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagValueInputs.tsx @@ -0,0 +1,704 @@ +import React, { useMemo, useState } from 'react'; +import { + Box, + Checkbox, + Flex, + Input, + type InputProps, + Popover, + PopoverContent, + PopoverTrigger, + Portal, + useDisclosure +} from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { SingleDateTimePicker } from '@fastgpt/web/components/common/DateTimePicker'; +import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput'; +import { + DatasetCollectionTagTypeEnum, + type DatasetTagType +} from '@fastgpt/global/core/dataset/type'; +import { OVERFLOW_CHIP_GAP_PX, useOverflowChipCount } from './TagCommon'; + +export const tagInputBaseStyles: InputProps = { + h: '36px', + borderRadius: 'sm', + border: '1px solid', + borderColor: 'myGray.200', + bg: 'white', + fontSize: 'sm', + color: 'myGray.900', + _hover: { + borderColor: 'primary.300' + }, + _focus: { + borderColor: 'primary.600', + boxShadow: 'focus' + }, + _placeholder: { + color: 'myGray.500' + } +}; +const StringTagInput = ({ + value, + onChange, + placeholder +}: { + value?: string; + onChange: (val: string) => void; + placeholder?: string; +}) => { + const { t } = useTranslation(); + + return ( + onChange(e.target.value)} + /> + ); +}; + +const embeddedFieldStyles = { + border: 'none', + boxShadow: 'none', + _hover: { border: 'none' }, + _focus: { border: 'none', boxShadow: 'none' } +}; + +export const NumberTagInput = ({ + value, + onChange, + placeholder, + showStepper, + embedded +}: { + value?: number | string; + onChange: (val: number | '') => void; + placeholder?: string; + showStepper?: boolean; + embedded?: boolean; +}) => { + const { t } = useTranslation(); + const placeholderText = placeholder ?? t('dataset:tag.fill_number'); + + if (showStepper) { + return ( + onChange(val === undefined ? '' : val)} + /> + ); + } + + return ( + { + const val = e.target.value; + if (val === '') { + onChange(''); + } else { + const num = Number(val); + if (!Number.isNaN(num)) { + onChange(num); + } + } + }} + /> + ); +}; + +export const DateTimeTagInput = ({ + value, + onChange, + placeholder, + embedded +}: { + value?: number | string; + onChange: (val: number) => void; + placeholder?: string; + embedded?: boolean; +}) => ( + +); + +/** 设置标签弹窗里的标签名称下拉：搜索已有标签并单选，底部「标签管理」打开标签管理弹窗。 */ +export const TagNameSelect = ({ + value, + options, + onChange, + onManage +}: { + value?: string; + options: { label: string; value: string }[]; + onChange: (val: string) => void; + onManage: () => void; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const [search, setSearch] = useState(''); + + const selected = options.find((opt) => opt.value === value); + const filteredOptions = useMemo(() => { + const trimmed = search.trim().toLowerCase(); + if (!trimmed) return options; + return options.filter((opt) => opt.label.toLowerCase().includes(trimmed)); + }, [options, search]); + + const handleClose = () => { + setSearch(''); + onClose(); + }; + + return ( + + + + + {selected?.label || t('dataset:tag.select_tag')} + + + + + + + + setSearch(e.target.value)} + /> + + {filteredOptions.length === 0 ? ( + + {t('common:no_select_data')} + + ) : ( + filteredOptions.map((opt) => { + const isSelected = opt.value === value; + return ( + { + if (opt.value !== value) onChange(opt.value); + handleClose(); + }} + > + {opt.label} + + ); + }) + )} + + + { + handleClose(); + onManage(); + }} + > + + + {t('dataset:tag.manage')} + + + + + + + ); +}; + +const ARRAY_OPTION_HOVER_BG = 'rgba(17, 24, 36, 0.05)'; + +const ArraySelectedChip = ({ + opt, + onRemove +}: { + opt: string; + onRemove?: (opt: string, e: React.MouseEvent) => void; +}) => ( + + + {opt} + + {onRemove && ( + onRemove(opt, e)} + > + + + )} + +); + +const ArrayOverflowChip = ({ count }: { count: number }) => ( + + {`+${count}`} + +); + +export const ArrayTagSelect = ({ + options, + value = [], + onChange, + onCreateOption, + allowCreate = true, + placeholder, + embedded +}: { + options: string[]; + value?: string[]; + onChange: (val: string[]) => void; + onCreateOption?: (option: string) => void; + allowCreate?: boolean; + placeholder?: string; + embedded?: boolean; +}) => { + const { t } = useTranslation(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const [search, setSearch] = useState(''); + const { containerRef, measureRef, visibleCount } = useOverflowChipCount({ + itemKey: value, + itemCount: value.length + }); + + const allOptions = useMemo(() => { + const list = [...options]; + for (const v of value) { + if (v && !list.includes(v)) { + list.push(v); + } + } + return list.filter(Boolean); + }, [options, value]); + + const filteredOptions = useMemo(() => { + const trimmed = search.trim().toLowerCase(); + if (!trimmed) return allOptions; + return allOptions.filter((opt) => opt.toLowerCase().includes(trimmed)); + }, [search, allOptions]); + + const canCreate = Boolean( + allowCreate && + search.trim() && + !allOptions.some((opt) => opt.toLowerCase() === search.trim().toLowerCase()) + ); + + const handleCreateOption = () => { + const trimmed = search.trim(); + if (!trimmed) return; + if (!value.includes(trimmed)) { + onChange([...value, trimmed]); + } + onCreateOption?.(trimmed); + setSearch(''); + }; + + const handleToggleOption = (opt: string) => { + if (value.includes(opt)) { + onChange(value.filter((v) => v !== opt)); + } else { + onChange([...value, opt]); + } + }; + + const handleToggleAll = () => { + if (allOptions.length === 0) return; + const isAllSelected = allOptions.every((opt) => value.includes(opt)); + if (isAllSelected) { + onChange([]); + } else { + onChange(allOptions); + } + }; + + const handleRemoveOption = (opt: string, e: React.MouseEvent) => { + e.stopPropagation(); + onChange(value.filter((v) => v !== opt)); + }; + + const visibleValues = value.slice(0, visibleCount); + const overflowCount = Math.max(value.length - visibleCount, 0); + + return ( + { + setSearch(''); + onClose(); + }} + placement={'bottom-start'} + closeOnBlur + matchWidth + > + + + + {value.length === 0 ? ( + + {placeholder ?? t('dataset:tag.select_options')} + + ) : ( + <> + + {value.map((opt) => ( + + ))} + + + + {visibleValues.map((opt) => ( + + ))} + {overflowCount > 0 && } + + + )} + + + + + + + + setSearch(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + if (allowCreate) handleCreateOption(); + } + }} + /> + + {canCreate && ( + + + {t('dataset:tag.create_option')} + + + {search.trim()} + + + )} + + {!search.trim() && allOptions.length > 0 && ( + <> + + + {t('common:All')} + + + + + )} + + + {filteredOptions.length === 0 && !canCreate ? ( + + {t('common:no_select_data')} + + ) : ( + filteredOptions.map((opt) => { + const isChecked = value.includes(opt); + return ( + handleToggleOption(opt)} + > + handleToggleOption(opt)} + onClick={(e) => e.stopPropagation()} + size={'sm'} + icon={} + /> + + {opt} + + + ); + }) + )} + + + + + + ); +}; + +/** 按标签类型渲染对应输入。未选标签时展示禁用占位。 */ +export const TagValueField = ({ + tag, + value, + onChange, + onCreateOption, + disabledPlaceholder, + numberShowStepper, + numberPlaceholder, + arrayPlaceholder +}: { + tag?: DatasetTagType; + value: string | number | string[]; + onChange: (val: string | number | string[]) => void; + onCreateOption?: (option: string) => void; + disabledPlaceholder?: string; + numberShowStepper?: boolean; + numberPlaceholder?: string; + arrayPlaceholder?: string; +}) => { + const { t } = useTranslation(); + + if (!tag) { + return ( + + ); + } + + if (tag.tagType === DatasetCollectionTagTypeEnum.number) { + return ( + onChange(val)} + /> + ); + } + + if (tag.tagType === DatasetCollectionTagTypeEnum.datetime) { + return onChange(val)} />; + } + + if (tag.tagType === DatasetCollectionTagTypeEnum.array) { + return ( + onChange(val)} + onCreateOption={onCreateOption} + /> + ); + } + + return ( + onChange(val)} + /> + ); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx index be3311698060..550d5d7fb77f 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx @@ -1,255 +1,187 @@ -import { Box, Checkbox, Flex, Input } from '@chakra-ui/react'; -import MyPopover from '@fastgpt/web/components/common/MyPopover'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import MyBox from '@fastgpt/web/components/common/MyBox'; -import { putDatasetCollectionById } from '@/web/core/dataset/api/collection'; +import { Box, Flex } from '@chakra-ui/react'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -import { useTranslation } from 'next-i18next'; -import { useMemo, useRef, useState } from 'react'; -import { useDeepCompareEffect } from 'ahooks'; -import { - type DatasetCollectionItemType, - type DatasetTagType -} from '@fastgpt/global/core/dataset/type'; -import { isEqual } from 'lodash-es'; +import { useLayoutEffect, useMemo, useState } from 'react'; +import { type DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type'; import { type DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import MyTag from '@fastgpt/web/components/common/Tag/index'; +import { + formatCollectionTagChipText, + OVERFLOW_CHIP_GAP_PX, + TAG_TOOLTIP_PROPS, + useOverflowChipCount +} from './TagCommon'; + +const TAG_CHIP_PROPS = { + colorSchema: 'cyan' as const, + type: 'fill' as const, + h: '20px', + px: 2, + flexShrink: 0, + fontSize: 'mini', + fontWeight: 'medium', + borderRadius: 'xs' as const +}; + +/** 渲染知识库列表中的标签；单个标签成为唯一可见项时允许收缩并展示完整文本。 */ +const TagChip = ({ + text, + isFlexible = false, + withTooltip = false +}: { + text: string; + isFlexible?: boolean; + withTooltip?: boolean; +}) => { + const tagText = ( + + {text} + + ); + + const tagContent = withTooltip ? ( + + {tagText} + + ) : ( + tagText + ); + + const chip = ( + + {tagContent} + + ); + + return chip; +}; const TagsPopOver = ({ - currentCollection, - hoverBg = 'myGray.50' + currentCollection }: { currentCollection: DatasetCollectionItemType | DatasetCollectionsListItemType; - hoverBg?: string; }) => { - const { t } = useTranslation(); - const { - searchTagKey, - setSearchTagKey, - searchDatasetTagsResult, - allDatasetTags, - onCreateCollectionTag, - isCreateCollectionTagLoading - } = useContextSelector(DatasetPageContext, (v) => v); + const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags); - const [collectionTags, setCollectionTags] = useState(currentCollection.tags ?? []); - const [checkedTags, setCheckedTags] = useState([]); - const [showTagManage, setShowTagManage] = useState(false); - const [isUpdateLoading, setIsUpdateLoading] = useState(false); - - const tagList = useMemo( + const chipItems = useMemo( () => - (collectionTags - ?.map((item) => { - const tagObject = allDatasetTags.find((tag) => tag.tag === item); - return tagObject ? { _id: tagObject._id, tag: tagObject.tag } : null; - }) - .filter((tag) => tag !== null) as { - _id: string; - tag: string; - }[]) || [], - [collectionTags, allDatasetTags] + (currentCollection.tags ?? []) + .map((item, index) => ({ + id: typeof item === 'string' ? item : `${item.tag}-${index}`, + text: formatCollectionTagChipText(item, allDatasetTags) + })) + .filter((item) => item.text), + [allDatasetTags, currentCollection.tags] ); - const [visibleTags, setVisibleTags] = useState(tagList); - const [overflowTags, setOverflowTags] = useState([]); - const containerRef = useRef(null); + const { containerRef, measureRef, visibleCount } = useOverflowChipCount({ + itemKey: chipItems, + itemCount: chipItems.length + }); - useDeepCompareEffect(() => { - const calculateTags = () => { - if (!containerRef.current || !tagList) return; + const [shouldShrinkFirstTag, setShouldShrinkFirstTag] = useState(false); + useLayoutEffect(() => { + const container = containerRef.current; + const measure = measureRef.current; + if (!container || !measure) return; - const containerWidth = containerRef.current.offsetWidth; - const tagWidth = 11; - let totalWidth = 30; - let visibleCount = 0; + const calculate = () => { + const firstTag = measure.querySelector('[data-tag-chip]') as HTMLElement | null; + const overflowChip = container.querySelector('[data-overflow-chip]') as HTMLElement | null; + const hasOnlyOneVisibleTag = visibleCount === 1 && chipItems.length > 1; - for (let i = 0; i < tagList.length; i++) { - const tag = tagList[i]; - const estimatedWidth = tag.tag.length * tagWidth + 16; // 加上左右 padding 的宽度 - if (totalWidth + estimatedWidth <= containerWidth) { - totalWidth += estimatedWidth; - visibleCount++; - } else { - break; - } + if (!firstTag || !overflowChip || !hasOnlyOneVisibleTag) { + setShouldShrinkFirstTag(false); + return; } - setVisibleTags(tagList.slice(0, visibleCount)); - setOverflowTags(tagList.slice(visibleCount)); + const availableWidth = + container.offsetWidth - overflowChip.offsetWidth - OVERFLOW_CHIP_GAP_PX; + setShouldShrinkFirstTag(firstTag.offsetWidth > availableWidth); }; - setTimeout(calculateTags, 100); - setCheckedTags(tagList); + calculate(); + const observer = new ResizeObserver(calculate); + observer.observe(container); + return () => observer.disconnect(); + }, [chipItems.length, containerRef, measureRef, visibleCount]); - window.addEventListener('resize', calculateTags); + if (chipItems.length === 0) return null; - return () => { - window.removeEventListener('resize', calculateTags); - }; - }, [tagList]); + const visibleTags = chipItems.slice(0, visibleCount); + const overflowTags = chipItems.slice(visibleCount); return ( - { - e.stopPropagation(); - setShowTagManage(true); - }} - cursor={'pointer'} - > - - {visibleTags.map((item, index) => ( - - {item.tag} - - ))} - - {overflowTags.length > 0 && ( - - {`+${overflowTags.length}`} - - )} - - } - onCloseFunc={async () => { - setSearchTagKey(''); - - setShowTagManage(false); - if (isEqual(checkedTags, tagList) || !showTagManage) return; - setIsUpdateLoading(true); - await putDatasetCollectionById({ - id: currentCollection._id, - tags: checkedTags.map((tag) => tag.tag) - }); - setCollectionTags(checkedTags.map((tag) => tag.tag)); - setIsUpdateLoading(false); - }} - display={showTagManage || overflowTags.length > 0 ? 'block' : 'none'} - > - {({}) => ( - <> - {showTagManage ? ( - e.stopPropagation()}> - - setSearchTagKey(e.target.value)} - /> - - - {searchTagKey && - !searchDatasetTagsResult.map((item) => item.tag).includes(searchTagKey) && ( - onCreateCollectionTag(searchTagKey)} - > - - - {t('dataset:tag.add') + ` "${searchTagKey}"`} - - - )} - {searchDatasetTagsResult?.map((item) => { - const tagsList = checkedTags.map((tag) => tag.tag); - return ( - { - e.preventDefault(); - if (tagsList.includes(item.tag)) { - setCheckedTags(checkedTags.filter((t) => t.tag !== item.tag)); - } else { - setCheckedTags([...checkedTags, item]); - } - }} - > - { - if (e.target.checked) { - setCheckedTags([...checkedTags, item]); - } else { - setCheckedTags(checkedTags.filter((t) => t._id !== item._id)); - } - }} - icon={} - /> - {item.tag} - - ); - })} - - - ) : ( - - {overflowTags.map((tag, index) => ( - - {tag.tag} - - ))} + + + {chipItems.map((item) => ( + + ))} + + {`+${chipItems.length}`} + + + + {visibleTags.map((item, index) => ( + + ))} + {overflowTags.length > 0 && ( + item.text).join('\n')} + shouldWrapChildren={false} + {...TAG_TOOLTIP_PROPS} + > + e.stopPropagation()} + _hover={{ bg: '#DBF3FF' }} + > + + {`+${overflowTags.length}`} + - )} - - )} - + + )} + + ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx index 0c9aaa8450bf..0afc00317485 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx @@ -29,6 +29,7 @@ import MyMenu from '@fastgpt/web/components/common/MyMenu'; import { useEditTitle } from '@/web/common/hooks/useEditTitle'; import { DatasetCollectionTypeEnum, + DatasetTypeEnum, DatasetStatusEnum, DatasetCollectionSyncResultMap, DatasetCollectionDataProcessModeMap @@ -62,6 +63,8 @@ import { hasDatasetTrainingError as checkDatasetTrainingError } from '@/web/core const Header = dynamic(() => import('./Header')); const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip')); +const CollectionTagSetModal = dynamic(() => import('./CollectionTagSetModal')); +const CollectionTagBatchModal = dynamic(() => import('./CollectionTagBatchModal')); const CollectionCard = () => { const BoxRef = useRef(null); @@ -77,6 +80,8 @@ const CollectionCard = () => { }>(); const [isTrainingErrorModalOpen, setIsTrainingErrorModalOpen] = useState(false); const [hasDatasetTrainingError, setHasDatasetTrainingError] = useState(false); + const [tagSetCollection, setTagSetCollection] = useState(); + const [isBatchTagModalOpen, setIsBatchTagModalOpen] = useState(false); const { collections, @@ -113,9 +118,9 @@ const CollectionCard = () => { isSelected, getRowSelectionProps, setSelectedItems, - FloatingActionBar, isSelecteAll, - selectAllTrigger + selectAllTrigger, + hasSelections } = useTableMultipleSelect({ list: formatCollections, getItemId: (e) => e._id @@ -190,6 +195,10 @@ const CollectionCard = () => { () => !!formatCollections.find((item) => item.trainingAmount > 0), [formatCollections] ); + const enabledCount = useMemo( + () => formatCollections.filter((item) => !item.forbid).length, + [formatCollections] + ); useRequest( async () => { @@ -229,10 +238,16 @@ const CollectionCard = () => { } }); - const isLoading = isUpdating || isSyncing || isGetting || isDropping; + const isPageLoading = isUpdating || isSyncing || isDropping; return ( - + {/* header */}
{ /> {/* collection table */} - - - - - - - - - - - - - - - {formatCollections.map((collection) => ( - { - if (collection.type === DatasetCollectionTypeEnum.folder) { - router.push({ - query: { - datasetId: datasetDetail._id, - parentId: collection._id - } - }); - } else { - router.push({ - query: { - datasetId: datasetDetail._id, - collectionId: collection._id, - currentTab: TabEnum.dataCard - } - }); - } + + +
- - - {t('common:Name')} - - {t('dataset:collection.training_type')}{t('dataset:collection_data_count')}{t('dataset:collection.Create update time')}{t('common:Status')}{t('dataset:Enable')} -
- + + - - - + + + + + + + + + {formatCollections.map((collection) => ( + { + if (collection.type === DatasetCollectionTypeEnum.folder) { + router.push({ + query: { + datasetId: datasetDetail._id, + parentId: collection._id + } + }); + } else { + router.push({ + query: { + datasetId: datasetDetail._id, + collectionId: collection._id, + currentTab: TabEnum.dataCard + } }); - }} - > - - {t(collection.statusText as any)} - - - - - - - + + + + + - - ))} - -
- - e.stopPropagation()}> - toggleSelect(collection)} - /> - - - - - - - {collection.name} - - - - {feConfigs?.isPlus && !!collection.tags?.length && ( - - )} - + } + }, + tbody: { + tr: { + h: '80px', + td: { + h: '80px', + py: 0, + px: 6, + borderBottom: 'sm', + borderLeftRadius: 0, + borderRightRadius: 0 + } + } + } + }} + > +
+ + + {t('common:Name')} - - - {collection.trainingType - ? t( - (DatasetCollectionDataProcessModeMap[collection.trainingType]?.label || - '-') as any - ) - : '-'} - {collection.dataAmount || '-'} - {formatTime2YMDHM(collection.createTime)} - {formatTime2YMDHM(collection.updateTime)} - - - { - e.stopPropagation(); - setTrainingStatesCollection({ - collectionId: collection._id, - permission: collection.permission + + {t('dataset:collection.training_type')}{t('dataset:collection_data_count')}{t('dataset:collection.Create update time')}{t('common:Status')} + {t('dataset:Enable')}({enabledCount}) + {t('common:Operation')}
e.stopPropagation()}> - - onUpdateCollection({ - id: collection._id, - forbid: !e.target.checked - }) + } } - /> - e.stopPropagation()}> - {collection.permission.hasWritePer && ( - + })} + > + + + e.stopPropagation()}> + toggleSelect(collection)} + /> + + + - + + + {collection.name} + + + + {feConfigs?.isPlus && !!collection.tags?.length && ( + + + + )} + + + + {collection.trainingType + ? t( + (DatasetCollectionDataProcessModeMap[collection.trainingType]?.label || + '-') as any + ) + : '-'} + {collection.dataAmount || '-'} + {formatTime2YMDHM(collection.createTime)} + {formatTime2YMDHM(collection.updateTime)} + + + { + e.stopPropagation(); + setTrainingStatesCollection({ + collectionId: collection._id, + permission: collection.permission + }); + }} + > + + {t(collection.statusText as any)} + + + + + e.stopPropagation()}> + + onUpdateCollection({ + id: collection._id, + forbid: !e.target.checked + }) } - menuList={[ - { - children: [ - ...(collectionCanSync(collection.type) - ? [ - { - label: ( - - - {t('dataset:collection_sync')} - - ), - onClick: () => - openSyncConfirm({ - onConfirm: () => { - onclickStartSync(collection._id); - } - })() - } - ] - : []), - { - label: ( - - - {t('common:Move')} - - ), - onClick: () => - setMoveCollectionData({ collectionId: collection._id }) - }, - { - label: ( - - - {t('common:Rename')} - - ), - onClick: () => - onOpenEditTitleModal({ - defaultVal: collection.name, - onSuccess: (newName) => - onUpdateCollection({ - id: collection._id, - name: newName - }) - }) - } - ] - }, - { - children: [ - { - label: ( - - - {t('common:Delete')} - - ), - type: 'danger', - onClick: () => - openDeleteConfirm({ - onConfirm: () => onDelCollection([collection._id]), - customContent: - collection.type === DatasetCollectionTypeEnum.folder - ? t( - 'common:dataset.collections.Confirm to delete the folder' - ) - : t('common:dataset.Confirm to delete the file') - })() - } - ] - } - ]} /> - )} -
+ + e.stopPropagation()}> + {collection.permission.hasWritePer && ( + + + + } + menuList={[ + { + children: [ + ...(collectionCanSync(collection.type) + ? [ + { + label: ( + + + {t('dataset:collection_sync')} + + ), + onClick: () => + openSyncConfirm({ + onConfirm: () => { + onclickStartSync(collection._id); + } + })() + } + ] + : []), + { + label: ( + + + {t('common:Move')} + + ), + onClick: () => + setMoveCollectionData({ collectionId: collection._id }) + }, + { + label: ( + + + {t('common:Rename')} + + ), + onClick: () => + onOpenEditTitleModal({ + defaultVal: collection.name, + onSuccess: (newName) => + onUpdateCollection({ + id: collection._id, + name: newName + }) + }) + }, + ...(feConfigs?.isPlus && + datasetDetail.type !== DatasetTypeEnum.websiteDataset + ? [ + { + label: ( + + + {t('dataset:tag.set')} + + ), + onClick: () => setTagSetCollection(collection) + } + ] + : []) + ] + }, + { + children: [ + { + label: ( + + + {t('common:Delete')} + + ), + type: 'danger', + onClick: () => + openDeleteConfirm({ + onConfirm: () => onDelCollection([collection._id]), + customContent: + collection.type === DatasetCollectionTypeEnum.folder + ? t( + 'common:dataset.collections.Confirm to delete the folder' + ) + : t('common:dataset.Confirm to delete the file') + })() + } + ] + } + ]} + /> + )} + + + ))} + + - {total === 0 && } - + {total === 0 && } + + - + {total > pageSize && !hasSelections && ( + + + + )} + + {/* 勾选后浮在卡片底部中央，距底边 12px */} + {hasSelections && ( + + + + + {t('dataset:collection.select_all_filtered')} + + + {t('dataset:tag.filter_selected')} + + {selectedItems.length} + + {t('dataset:tag.filter_item')} + + + + {datasetDetail.permission.hasWritePer && + datasetDetail.type !== DatasetTypeEnum.websiteDataset && + feConfigs?.isPlus && ( + + )} - } - > - {total > pageSize && ( - - - - )} - + + )} + {!!tagSetCollection && ( + setTagSetCollection(undefined)} + onSuccess={() => { + getData(pageNum); + setTagSetCollection(undefined); + }} + /> + )} + + {isBatchTagModalOpen && ( + setIsBatchTagModalOpen(false)} + onSuccess={() => { + getData(pageNum); + setSelectedItems([]); + setIsBatchTagModalOpen(false); + }} + /> + )} + {!!trainingStatesCollection && ( + tagType === DatasetCollectionTagTypeEnum.array ? [] : ''; + +export const createEmptyTagRow = (): CollectionTagRow => ({ + id: getNanoid(), + tagId: '', + value: '' +}); + +/** 把集合上的展示格式标签灌进设置弹窗表格；没有可解析项时给一行空行。 */ +export const collectionTagsToRows = ( + items: CollectionTagDisplayItem[] | undefined, + tags: DatasetTagType[] +): CollectionTagRow[] => { + const rows: CollectionTagRow[] = []; + for (const item of items ?? []) { + const resolved = resolveDisplayedCollectionTag(item, tags); + if (!resolved) continue; + rows.push({ + id: getNanoid(), + tagId: String(resolved.tagDoc._id), + value: resolved.value + }); + } + return rows.length > 0 ? rows : [createEmptyTagRow()]; +}; + +/** 行数据是否已选标签且填了对应类型的值，用于禁用「添加」和底部确定。 */ +export const isTagRowComplete = ( + row: Pick, + tags: DatasetTagType[] +) => { + if (!row.tagId) return false; + const tagDoc = tags.find((tag) => String(tag._id) === row.tagId); + if (!tagDoc) return false; + + const tagType = tagDoc.tagType ?? DatasetCollectionTagTypeEnum.string; + if (tagType === DatasetCollectionTagTypeEnum.string) { + return typeof row.value === 'string' && row.value.trim().length > 0; + } + if ( + tagType === DatasetCollectionTagTypeEnum.number || + tagType === DatasetCollectionTagTypeEnum.datetime + ) { + return typeof row.value === 'number' && Number.isFinite(row.value); + } + if (tagType === DatasetCollectionTagTypeEnum.array) { + return Array.isArray(row.value) && row.value.length > 0; + } + return false; +}; + +/** 当前行可选的标签：排除其它行已选中的 tagId。 */ +export const unusedTagSelectOptions = ( + tags: DatasetTagType[], + rows: Array<{ id: string; tagId: string }>, + currentRowId: string +) => { + const otherSelectedTagIds = new Set( + rows.filter((row) => row.id !== currentRowId && Boolean(row.tagId)).map((row) => row.tagId) + ); + + return tags + .filter((tag) => !otherSelectedTagIds.has(String(tag._id))) + .map((tag) => ({ + label: tag.tag, + value: String(tag._id) + })); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts new file mode 100644 index 000000000000..5425cb620436 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useAppendDatasetTagOption.ts @@ -0,0 +1,36 @@ +import { useContextSelector } from 'use-context-selector'; +import { useTranslation } from 'next-i18next'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { updateDatasetCollectionTag } from '@/web/core/dataset/api/collection'; + +/** 给选项类标签追加一个预设 option，并刷新当前知识库标签定义。 */ +export const useAppendDatasetTagOption = () => { + const { t } = useTranslation(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags); + const loadAllDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadAllDatasetTags); + + return useRequest( + ({ tagId, option }: { tagId: string; option: string }) => { + const tag = allDatasetTags.find((item) => String(item._id) === tagId); + if (!tag) return Promise.resolve(); + + const nextOptions = [...new Set([...(tag.options ?? []), option.trim()].filter(Boolean))]; + if (nextOptions.length === (tag.options ?? []).length) { + return Promise.resolve(); + } + return updateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tagId: tag._id, + tag: tag.tag, + options: nextOptions + }); + }, + { + refreshDeps: [datasetDetail._id], + onSuccess: loadAllDatasetTags, + errorToast: t('dataset:tag.save_failed') + } + ); +}; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/useCollectionTagRows.ts b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useCollectionTagRows.ts new file mode 100644 index 000000000000..d44ccc6fc1c9 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/useCollectionTagRows.ts @@ -0,0 +1,76 @@ +import { useMemo, useState } from 'react'; +import { type DatasetTagType } from '@fastgpt/global/core/dataset/type'; +import { emptyTagRowValue, isTagRowComplete, type CollectionTagRow } from './tagForm'; + +/** + * 设置标签 / 批量添加共用的表格行状态。 + * 未编辑时跟随 initialRows（设置弹窗等标签定义加载完再灌入已有值）；编辑后锁在 draft 上。 + */ +export const useCollectionTagRows = ({ + tags, + initialRows, + createRow, + patchOnTagChange, + onRowDeleted +}: { + tags: DatasetTagType[]; + initialRows: T[]; + createRow: () => T; + patchOnTagChange?: (tagDoc?: DatasetTagType) => Partial; + onRowDeleted?: () => void; +}) => { + const [draftRows, setDraftRows] = useState(); + const rows = draftRows ?? initialRows; + + const hasIncompleteRow = useMemo( + () => rows.some((row) => !isTagRowComplete(row, tags)), + [rows, tags] + ); + + const isAddDisabled = hasIncompleteRow || tags.length === 0 || rows.length >= tags.length; + + const updateRows = (updater: (current: T[]) => T[]) => { + setDraftRows((prev) => updater(prev ?? rows)); + }; + + const handleAddRow = () => { + if (isAddDisabled) return; + updateRows((current) => [...current, createRow()]); + }; + + const handleDeleteRow = (id: string) => { + updateRows((current) => current.filter((row) => row.id !== id)); + onRowDeleted?.(); + }; + + const handleTagChange = (rowId: string, newTagId: string) => { + const tagDoc = tags.find((tag) => String(tag._id) === newTagId); + updateRows((current) => + current.map((row) => + row.id === rowId + ? { + ...row, + tagId: newTagId, + value: emptyTagRowValue(tagDoc?.tagType), + ...patchOnTagChange?.(tagDoc) + } + : row + ) + ); + }; + + const handleValueChange = (rowId: string, value: string | number | string[]) => { + updateRows((current) => current.map((row) => (row.id === rowId ? { ...row, value } : row))); + }; + + return { + rows, + hasIncompleteRow, + isAddDisabled, + updateRows, + handleAddRow, + handleDeleteRow, + handleTagChange, + handleValueChange + }; +}; diff --git a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx index 7c8e37124d42..ebd3c22b931a 100644 --- a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx +++ b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx @@ -177,7 +177,9 @@ const DataCard = () => { )} {feConfigs?.isPlus && !!collection?.tags?.length && ( - + + + )} diff --git a/projects/app/src/pages/api/core/dataset/collection/create/backup.ts b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts index bb9cb81c04aa..88c1fae72ce7 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/backup.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts @@ -29,7 +29,7 @@ async function handler(req: ApiRequestProps) { }); filepaths.push(result.fileMetadata.path); const filename = decodeMultipartFilename(result.fileMetadata.originalname); - const { datasetId, parentId } = CreateBackupCollectionFormSchema.parse(result.data); + const { datasetId, parentId, tags } = CreateBackupCollectionFormSchema.parse(result.data); const { teamId, tmbId, dataset } = await authDataset({ req, @@ -82,7 +82,8 @@ async function handler(req: ApiRequestProps) { name: filename, type: DatasetCollectionTypeEnum.file, fileId, - trainingType: DatasetCollectionDataProcessModeEnum.backup + trainingType: DatasetCollectionDataProcessModeEnum.backup, + tags } }); promoted = true; diff --git a/projects/app/src/pages/api/core/dataset/collection/create/images.ts b/projects/app/src/pages/api/core/dataset/collection/create/images.ts index 7e0ba396f2dc..6405cfaa343c 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/images.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/images.ts @@ -35,7 +35,7 @@ async function handler(req: ApiRequestProps): Promise item.path)); - const { parentId, datasetId, collectionName } = CreateImageCollectionFormSchema.parse( + const { parentId, datasetId, collectionName, tags } = CreateImageCollectionFormSchema.parse( result.data ); @@ -93,6 +93,7 @@ async function handler(req: ApiRequestProps): Promise ({ - ...item, - tags: await collectionTagsToTagLabel({ - datasetId, - tags: item.tags - }), - dataAmount: 0, - trainingAmount: 0, - permission - })) - ), - total: await MongoDatasetCollection.countDocuments(match) - }; - } - - const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([ - MongoDatasetCollection.aggregate([ - { - $match: match - }, - { - $sort: { updateTime: -1 } - }, - { - $skip: (pageNum - 1) * pageSize - }, - { - $limit: pageSize - }, - // count training data - { - $lookup: { - from: DatasetTrainingCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }] - } - } - }, - { $count: 'count' } - ], - as: 'trainingCount' - } - }, - // count collection total data - { - $lookup: { - from: DatasetDataCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [ - { $eq: ['$teamId', '$$team_id'] }, - { $eq: ['$datasetId', '$$dataset_id'] }, - { $eq: ['$collectionId', '$$id'] } - ] - } - } - }, - { $count: 'count' } - ], - as: 'dataCount' - } - }, - { - $project: { - ...selectField, - dataAmount: { - $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] - }, - trainingAmount: { - $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] - } - } - } - ]), - MongoDatasetCollection.countDocuments(match, { - ...readFromSecondary - }) - ]); - - const data = await Promise.all( - collections.map(async (item) => ({ - ...item, - tags: await collectionTagsToTagLabel({ - datasetId, - tags: item.tags - }), - permission - })) - ); - - // count collections - return { - pageNum, - pageSize, - data, - total - }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/listV2.ts b/projects/app/src/pages/api/core/dataset/collection/listV2.ts index 76024de032f9..1992221a9210 100644 --- a/projects/app/src/pages/api/core/dataset/collection/listV2.ts +++ b/projects/app/src/pages/api/core/dataset/collection/listV2.ts @@ -6,6 +6,7 @@ import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; +import { buildCollectionListTagMatch } from '@fastgpt/service/core/dataset/collection/tagFilter'; import { type DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; @@ -77,7 +78,7 @@ async function handler(req: ApiRequestProps): Promise> { - const { datasetId, parentId, searchText, selectFolder, filterTags, simple } = parseApiInput({ - req, - bodySchema: ScrollCollectionsBodySchema - }).body; - const { offset, pageSize: rawPageSize } = parsePaginationRequest(req); - - const regexText = searchText ? replaceRegChars(searchText) : ''; - const pageSize = Math.min(rawPageSize, 30); - - // auth dataset and get my role - const { teamId, permission } = await authDataset({ - req, - authToken: true, - authApiKey: true, - datasetId, - per: ReadPermissionVal - }); - - const match = { - teamId: new Types.ObjectId(teamId), - datasetId: new Types.ObjectId(datasetId), - parentId: parentId ? new Types.ObjectId(parentId) : null, - ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}), - ...(regexText - ? { - name: { $regex: regexText, $options: 'i' } - } - : {}), - ...(filterTags.length ? { tags: { $all: filterTags } } : {}) - }; - - const selectField = { - _id: 1, - parentId: 1, - tmbId: 1, - name: 1, - type: 1, - forbid: 1, - createTime: 1, - updateTime: 1, - trainingType: 1, - fileId: 1, - rawLink: 1, - tags: 1 - }; - - // not count data amount - if (simple) { - const collections = await MongoDatasetCollection.find(match) - .select(selectField) - .sort({ - updateTime: -1 - }) - .skip(offset) - .limit(pageSize) - .lean(); - - return { - list: await Promise.all( - collections.map(async (item) => ({ - ...item, - dataAmount: 0, - trainingAmount: 0, - ...defaultCollectionTrainingStatus, - indexAmount: 0, - permission - })) - ), - total: await MongoDatasetCollection.countDocuments(match) - }; - } - - const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([ - MongoDatasetCollection.aggregate([ - { - $match: match - }, - { - $sort: { updateTime: -1 } - }, - { - $skip: offset - }, - { - $limit: pageSize - }, - // count training data - { - $lookup: { - from: DatasetTrainingCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }] - } - } - }, - { $count: 'count' } - ], - as: 'trainingCount' - } - }, - // count collection total data - { - $lookup: { - from: DatasetDataCollectionName, - let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, - pipeline: [ - { - $match: { - $expr: { - $and: [ - { $eq: ['$teamId', '$$team_id'] }, - { $eq: ['$datasetId', '$$dataset_id'] }, - { $eq: ['$collectionId', '$$id'] } - ] - } - } - }, - { $count: 'count' } - ], - as: 'dataCount' - } - }, - { - $project: { - ...selectField, - dataAmount: { - $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] - }, - trainingAmount: { - $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] - }, - activeTrainingAmount: { $literal: 0 }, - finalErrorAmount: { $literal: 0 }, - hasError: { $literal: false }, - slowestTrainingStatus: { $literal: CollectionTrainingStatusEnum.ready } - } - } - ]), - MongoDatasetCollection.countDocuments(match) - ]); - - const data = await Promise.all( - collections.map(async (item) => ({ - ...item, - permission - })) - ); - - // count collections - return { - list: data, - total - }; -} - -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts b/projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts new file mode 100644 index 000000000000..1f68b903fdb1 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/tagFilterOptions.ts @@ -0,0 +1,32 @@ +import { NextAPI } from '@/service/middleware/entry'; +import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; +import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; +import { getDatasetTagFilterOptions } from '@fastgpt/service/core/dataset/collection/tagFilter'; +import type { ApiRequestProps } from '@fastgpt/next/type'; +import { + GetTagFilterOptionsQuerySchema, + GetTagFilterOptionsResponseSchema, + type GetTagFilterOptionsResponseType +} from '@fastgpt/global/openapi/core/dataset/collection/api'; + +async function handler(req: ApiRequestProps): Promise { + const { datasetId } = parseApiInput({ + req, + querySchema: GetTagFilterOptionsQuerySchema + }).query; + + const { teamId } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: ReadPermissionVal + }); + + const list = await getDatasetTagFilterOptions({ teamId, datasetId }); + + return GetTagFilterOptionsResponseSchema.parse({ list }); +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/app/detail/index.tsx b/projects/app/src/pages/app/detail/index.tsx index 87bd3c135d0b..e34d3425cbf1 100644 --- a/projects/app/src/pages/app/detail/index.tsx +++ b/projects/app/src/pages/app/detail/index.tsx @@ -94,7 +94,8 @@ export async function getServerSideProps(context: any) { 'publish', 'apikey', 'workflow', - 'skill' + 'skill', + 'dataset' ])) } }; diff --git a/projects/app/src/web/core/dataset/api/collection.ts b/projects/app/src/web/core/dataset/api/collection.ts index c8a550280347..df8854fc90a1 100644 --- a/projects/app/src/web/core/dataset/api/collection.ts +++ b/projects/app/src/web/core/dataset/api/collection.ts @@ -1,13 +1,9 @@ -import { GET, POST, PUT, DELETE } from '@/web/common/api/request'; +import { GET, POST, DELETE } from '@/web/common/api/request'; import type { ParentTreePathItemType, ParentIdType } from '@fastgpt/global/common/parentFolder/type'; -import type { - DatasetCollectionItemType, - DatasetTagType, - TagUsageType -} from '@fastgpt/global/core/dataset/type'; +import type { DatasetCollectionItemType, DatasetTagType } from '@fastgpt/global/core/dataset/type'; import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; import type { CreateApiCollectionV2BodyType, @@ -19,19 +15,21 @@ import type { CreateCollectionBodyType } from '@fastgpt/global/openapi/core/dataset/collection/createApi'; import type { - AddTagsToCollectionsParams, + BatchSetCollectionTagsParams, CreateDatasetCollectionTagParams, + SetCollectionTagsParams, UpdateDatasetCollectionTagParams } from '@fastgpt/global/openapi/core/dataset/collection/tagApi'; import type { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants'; import type { DatasetCollectionsListItemType, DeleteCollectionBodyType, + GetTagFilterOptionsResponseType, ReadCollectionSourceBodyType, ReadCollectionSourceResponseType, UpdateDatasetCollectionBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api'; -import type { PaginationProps, PaginationResponse } from '@fastgpt/global/openapi/api'; +import type { PaginationResponse } from '@fastgpt/global/openapi/api'; import type { GetCollectionTrainingDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api'; /* ============================= collections ==================================== */ @@ -132,22 +130,20 @@ export const postTemplateDatasetCollection = ({ /* =============================== tag ==================================== */ export const postCreateDatasetCollectionTag = (data: CreateDatasetCollectionTagParams) => POST(`/proApi/core/dataset/tag/create`, data); -export const postAddTagsToCollections = (data: AddTagsToCollectionsParams) => - POST(`/proApi/core/dataset/tag/addToCollections`, data); export const delDatasetCollectionTag = (data: { id: string; datasetId: string }) => DELETE(`/proApi/core/dataset/tag/delete`, data); export const updateDatasetCollectionTag = (data: UpdateDatasetCollectionTagParams) => POST(`/proApi/core/dataset/tag/update`, data); -export const getDatasetCollectionTags = ( - data: PaginationProps<{ - datasetId: string; - searchText?: string; - }> -) => POST>(`/proApi/core/dataset/tag/list`, data); -export const getTagUsage = (datasetId: string) => - GET(`/proApi/core/dataset/tag/tagUsage?datasetId=${datasetId}`); export const getAllTags = (datasetId: string) => GET<{ list: DatasetTagType[] }>(`/proApi/core/dataset/tag/getAllTags?datasetId=${datasetId}`); +export const getDatasetTagFilterOptions = (datasetId: string) => + GET( + `/core/dataset/collection/tagFilterOptions?datasetId=${datasetId}` + ); +export const postSetCollectionTags = (data: SetCollectionTagsParams) => + POST(`/proApi/core/dataset/tag/setCollectionTags`, data); +export const postBatchSetCollectionTags = (data: BatchSetCollectionTagsParams) => + POST(`/proApi/core/dataset/tag/batchSetCollectionTags`, data); /* ================== read source ======================== */ export const getCollectionSource = (data: ReadCollectionSourceBodyType) => diff --git a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx index 2275aa433b57..e5e9039cef40 100644 --- a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx +++ b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx @@ -1,13 +1,8 @@ import { useQuery } from '@tanstack/react-query'; -import { type Dispatch, type ReactNode, type SetStateAction, useState } from 'react'; -import { useTranslation } from 'next-i18next'; +import { type ReactNode, useState } from 'react'; import { createContext } from 'use-context-selector'; import { getDatasetById, getDatasetPaths, putDatasetById } from '../api'; -import { - getAllTags, - getDatasetCollectionTags, - postCreateDatasetCollectionTag -} from '../api/collection'; +import { getAllTags } from '../api/collection'; import { getDatasetTrainingQueue } from '../api/training'; import { defaultDatasetDetail } from '../constants'; import { type UpdateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api'; @@ -23,15 +18,9 @@ type DatasetPageContextType = { loadDatasetDetail: (id: string) => Promise; updateDataset: (data: UpdateDatasetBody) => Promise; - searchDatasetTagsResult: DatasetTagType[]; allDatasetTags: DatasetTagType[]; + isLoadingAllDatasetTags: boolean; loadAllDatasetTags: () => Promise; - checkedDatasetTag: DatasetTagType[]; - setCheckedDatasetTag: React.Dispatch>; - onCreateCollectionTag: (tag: string) => Promise; - isCreateCollectionTagLoading: boolean; - searchTagKey: string; - setSearchTagKey: Dispatch>; paths: ParentTreePathItemType[]; refetchPaths: () => void; @@ -48,29 +37,17 @@ export const DatasetPageContext = createContext({ }, datasetId: '', datasetDetail: defaultDatasetDetail, - loadDatasetDetail: function (id: string): Promise { + loadDatasetDetail: function (_id: string): Promise { throw new Error('Function not implemented.'); }, - updateDataset: function (data: UpdateDatasetBody): Promise { + updateDataset: function (_data: UpdateDatasetBody): Promise { throw new Error('Function not implemented.'); }, - searchDatasetTagsResult: [], allDatasetTags: [], - checkedDatasetTag: [], - setCheckedDatasetTag: function (): void { - throw new Error('Function not implemented.'); - }, + isLoadingAllDatasetTags: false, loadAllDatasetTags: function (): Promise { throw new Error('Function not implemented.'); }, - onCreateCollectionTag: function (tag: string): Promise { - throw new Error('Function not implemented.'); - }, - isCreateCollectionTagLoading: false, - searchTagKey: '', - setSearchTagKey: function (value: SetStateAction): void { - throw new Error('Function not implemented.'); - }, paths: [], refetchPaths: () => {} }); @@ -82,7 +59,6 @@ export const DatasetPageContextProvider = ({ children: ReactNode; datasetId: string; }) => { - const { t } = useTranslation(); const { feConfigs } = useSystemStore(); // dataset detail @@ -105,10 +81,11 @@ export const DatasetPageContextProvider = ({ }; // dataset tags - const [checkedDatasetTag, setCheckedDatasetTag] = useState([]); - const [searchTagKey, setSearchTagKey] = useState(''); - - const { runAsync: loadAllDatasetTags, data: allDatasetTags = [] } = useRequest( + const { + runAsync: loadAllDatasetTags, + data: allDatasetTags = [], + loading: isLoadingAllDatasetTags + } = useRequest( async () => { if (!feConfigs?.isPlus || !datasetDetail._id) return []; @@ -120,38 +97,6 @@ export const DatasetPageContextProvider = ({ refreshDeps: [datasetDetail._id] } ); - const { data: searchDatasetTagsResult = [] } = useRequest( - async () => { - if (!searchTagKey) return allDatasetTags; - const { list } = await getDatasetCollectionTags({ - datasetId: datasetDetail._id, - searchText: searchTagKey, - offset: 0, - pageSize: 15 - }); - return list; - }, - { - manual: false, - throttleWait: 300, - refreshDeps: [datasetDetail._id, searchTagKey, allDatasetTags] - } - ); - const { runAsync: onCreateCollectionTag, loading: isCreateCollectionTagLoading } = useRequest( - (tag: string) => - postCreateDatasetCollectionTag({ - datasetId: datasetDetail._id, - tag - }), - { - refreshDeps: [datasetDetail._id], - onSuccess() { - loadAllDatasetTags(); - }, - successToast: t('common:create_success'), - errorToast: t('common:create_failed') - } - ); // training and rebuild queue const { data: { rebuildingCount = 0, trainingCount = 0 } = {}, refetch: refetchDatasetTraining } = @@ -189,15 +134,9 @@ export const DatasetPageContextProvider = ({ trainingCount, refetchDatasetTraining, - searchDatasetTagsResult, - checkedDatasetTag, - setCheckedDatasetTag, allDatasetTags, - loadAllDatasetTags, - onCreateCollectionTag, - isCreateCollectionTagLoading, - searchTagKey, - setSearchTagKey + isLoadingAllDatasetTags, + loadAllDatasetTags }; return {children}; diff --git a/projects/app/src/web/core/workflow/datasetSearchNodeUpgrade.ts b/projects/app/src/web/core/workflow/datasetSearchNodeUpgrade.ts new file mode 100644 index 000000000000..1221101b4a7a --- /dev/null +++ b/projects/app/src/web/core/workflow/datasetSearchNodeUpgrade.ts @@ -0,0 +1,73 @@ +import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io'; +import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node'; +import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; +import { + DatasetTagFilterVersionEnum, + resolveDatasetTagFilterVersion +} from '@fastgpt/global/core/dataset/workflowTagFilter'; +import { Input_Template_Dataset_Tag_Filter_Version } from '@fastgpt/global/core/workflow/template/input'; + +/** 从节点输入读取唯一的过滤版本来源，不检查 collectionFilterMatch 的值形状。 */ +export const getDatasetSearchFilterVersion = (inputs: FlowNodeInputItemType[]) => + resolveDatasetTagFilterVersion({ + version: inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion)?.value, + filterValue: inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterMatch)?.value + }); + +export const datasetSearchUsesLegacyFilter = (inputs: FlowNodeInputItemType[]) => + getDatasetSearchFilterVersion(inputs) === DatasetTagFilterVersionEnum.legacy; + +/** 只更新过滤版本和过滤输入，nodeId、节点类型、其他参数及连线引用保持不变。 */ +export const upgradeLegacyDatasetSearchNode = ({ + node, + filterInput +}: { + node: StoreNodeItemType; + filterInput: FlowNodeInputItemType; +}): StoreNodeItemType => { + if (filterInput.key !== NodeInputKeyEnum.collectionFilterMatch) { + throw new Error('Dataset search filter upgrade received an invalid filter input'); + } + const structuredVersionInput = { + ...Input_Template_Dataset_Tag_Filter_Version, + value: DatasetTagFilterVersionEnum.structured + }; + const inputs = node.inputs + .filter( + (input) => + input.key !== NodeInputKeyEnum.collectionFilterVersion && + input.key !== NodeInputKeyEnum.collectionFilterMatch + ) + .concat(structuredVersionInput, filterInput); + + return { + ...node, + inputs + }; +}; + +/** 先持久化再提交本地升级；持久化失败时 commit 不会执行。 */ +export const persistLegacyDatasetSearchNodeUpgrade = async ({ + nodes, + nodeId, + filterInput, + persist, + commit +}: { + nodes: StoreNodeItemType[]; + nodeId: string; + filterInput: FlowNodeInputItemType; + persist: (nodes: StoreNodeItemType[]) => Promise; + commit: (node: StoreNodeItemType) => void; +}) => { + let upgradedNode: StoreNodeItemType | undefined; + const upgradedNodes = nodes.map((node) => { + if (node.nodeId !== nodeId) return node; + upgradedNode = upgradeLegacyDatasetSearchNode({ node, filterInput }); + return upgradedNode; + }); + if (!upgradedNode) throw new Error(`Dataset search node not found: ${nodeId}`); + + await persist(upgradedNodes); + commit(upgradedNode); +}; diff --git a/projects/app/src/web/core/workflow/utils.ts b/projects/app/src/web/core/workflow/utils.ts index 253065c3c81c..d580d32d332e 100644 --- a/projects/app/src/web/core/workflow/utils.ts +++ b/projects/app/src/web/core/workflow/utils.ts @@ -38,6 +38,10 @@ import { workflowSystemVariables } from '../app/utils'; import type { WorkflowDataContextType } from '@/pageComponents/app/detail/WorkflowComponents/context/workflowInitContext'; import type { MyLLMModelItemType } from '@fastgpt/global/openapi/core/ai/model/api'; import { normalizeFlowNodeInputType } from '@fastgpt/global/core/app/formEdit/utils'; +import { + DatasetTagFilterVersionEnum, + resolveDatasetTagFilterVersion +} from '@fastgpt/global/core/dataset/workflowTagFilter'; /** * 将节点模板转换为画布节点，并按创建时语言初始化可编辑文本。 @@ -198,6 +202,20 @@ export const storeNode2FlowNode = ({ isWorkflowSystemModelInput({ node: storeNode, input: templateInput }) ); }; + const collectionFilterVersion = + storeNode.flowNodeType === FlowNodeTypeEnum.datasetSearchNode + ? resolveDatasetTagFilterVersion({ + version: adaptedStoreInputs.find( + (input) => input.key === NodeInputKeyEnum.collectionFilterVersion + )?.value, + filterValue: adaptedStoreInputs.find( + (input) => input.key === NodeInputKeyEnum.collectionFilterMatch + )?.value + }) + : DatasetTagFilterVersionEnum.structured; + const usesLegacyDatasetSearchFilter = + storeNode.flowNodeType === FlowNodeTypeEnum.datasetSearchNode && + collectionFilterVersion === DatasetTagFilterVersionEnum.legacy; // replace item data const nodeItem: FlowNodeItemType = { @@ -223,8 +241,30 @@ export const storeNode2FlowNode = ({ debugLabel: t(inputTemplate.debugLabel ?? (storeInput.debugLabel as any)), toolDescription: t(inputTemplate.toolDescription ?? (storeInput.toolDescription as any)), key: storeInput.key, - selectedType: storeInput.selectedType ?? inputTemplate.selectedType, - value: storeInput.value + label: + usesLegacyDatasetSearchFilter && + inputTemplate.key === NodeInputKeyEnum.collectionFilterMatch + ? 'workflow:collection_metadata_filter' + : inputTemplate.label, + description: + usesLegacyDatasetSearchFilter && + inputTemplate.key === NodeInputKeyEnum.collectionFilterMatch + ? 'workflow:filter_description' + : inputTemplate.description, + selectedType: (() => { + // 旧节点用 textarea 手写 JSON；切到条件行渲染类型，但保留字符串 value 以便展示升级 UI。 + if ( + inputTemplate.key === NodeInputKeyEnum.collectionFilterMatch && + storeInput.selectedType === FlowNodeInputTypeEnum.textarea + ) { + return FlowNodeInputTypeEnum.datasetTagFilter; + } + return storeInput.selectedType ?? inputTemplate.selectedType; + })(), + value: + inputTemplate.key === NodeInputKeyEnum.collectionFilterVersion + ? collectionFilterVersion + : storeInput.value }; }) .concat( diff --git a/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts b/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts index 0ef688b0f18c..230148decbe1 100644 --- a/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts +++ b/projects/app/test/api/core/dataset/collection/create/fileSourceLifecycle.test.ts @@ -91,7 +91,7 @@ describe('dataset multipart file source lifecycle', () => { vi.clearAllMocks(); source.materialize.mockReset(); mockResolveFormData.mockResolvedValue({ - data: { datasetId }, + data: { datasetId, tags: [{ tag: 'score', value: 10 }] }, fileMetadata: { path: '/tmp/source.csv', originalname: 'source.csv', @@ -140,6 +140,13 @@ describe('dataset multipart file source lifecycle', () => { mockParseDatasetImportFile.mock.invocationCallOrder[0] ); expect(mockCleanupPendingDatasetFile).not.toHaveBeenCalled(); + expect(mockCreateCollectionAndInsertData).toHaveBeenCalledWith( + expect.objectContaining({ + createCollectionParams: expect.objectContaining({ + tags: [{ tag: 'score', value: 10 }] + }) + }) + ); } ); diff --git a/projects/app/test/api/core/dataset/collection/create/images.test.ts b/projects/app/test/api/core/dataset/collection/create/images.test.ts index dd4d7b895b77..ca6a8070fff2 100644 --- a/projects/app/test/api/core/dataset/collection/create/images.test.ts +++ b/projects/app/test/api/core/dataset/collection/create/images.test.ts @@ -108,7 +108,8 @@ describe('POST /api/core/dataset/collection/create/images', () => { data: { parentId, datasetId, - collectionName: 'Native image embedding collection' + collectionName: 'Native image embedding collection', + tags: [{ tag: 'score', value: 10 }] }, fileMetadata: [ { @@ -174,6 +175,7 @@ describe('POST /api/core/dataset/collection/create/images', () => { datasetId, type: DatasetCollectionTypeEnum.images, name: 'Native image embedding collection', + tags: [{ tag: 'score', value: 10 }], trainingType: DatasetCollectionDataProcessModeEnum.chunk } }); diff --git a/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts b/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts new file mode 100644 index 000000000000..d20490e6f0e4 --- /dev/null +++ b/projects/app/test/api/core/dataset/collection/tagFilterOptions.test.ts @@ -0,0 +1,99 @@ +import listHandler from '@/pages/api/core/dataset/collection/listV2'; +import tagFilterOptionsHandler from '@/pages/api/core/dataset/collection/tagFilterOptions'; +import { + DatasetCollectionTagTypeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { getRootUser } from '@test/datas/users'; +import { Call } from '@test/utils/request'; +import { describe, expect, it } from 'vitest'; + +describe('collection tag filter options and listV2 tagFilters', () => { + it('returns used values and filters collections by tag values', async () => { + const root = await getRootUser(); + const dataset = await MongoDataset.create({ + name: 'tag-filter-dataset', + teamId: root.teamId, + tmbId: root.tmbId, + vectorModel: 'test', + agentModel: 'test' + }); + + const [docType, version] = await MongoDatasetCollectionTagsV2.create([ + { + teamId: root.teamId, + datasetId: dataset._id, + tag: '文档类型', + tagType: DatasetCollectionTagTypeEnum.array, + options: ['PRD', 'unused'] + }, + { + teamId: root.teamId, + datasetId: dataset._id, + tag: '版本', + tagType: DatasetCollectionTagTypeEnum.number + } + ]); + + await MongoDatasetCollection.create([ + { + name: 'prd-file', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [ + { tagId: String(docType._id), value: ['PRD'] }, + { tagId: String(version._id), value: 2 } + ] + }, + { + name: 'spec-file', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [{ tagId: String(docType._id), value: ['spec'] }] + }, + { + name: 'noise-file', + type: DatasetCollectionTypeEnum.file, + teamId: root.teamId, + tmbId: root.tmbId, + datasetId: dataset._id, + tags: [ + { tagId: String(docType._id), value: [''] }, + { tagId: String(version._id), value: 0 }, + 'legacy-id' + ] + } + ]); + + const optionsRes = await Call(tagFilterOptionsHandler, { + auth: root, + query: { datasetId: String(dataset._id) } + }); + + expect(optionsRes.code).toBe(200); + const docTypeOption = optionsRes.data.list.find((item) => item.tagId === String(docType._id)); + const versionOption = optionsRes.data.list.find((item) => item.tagId === String(version._id)); + expect(docTypeOption?.values).toEqual(['PRD', 'spec']); + expect(versionOption?.values).toEqual([0, 2]); + + const listRes = await Call(listHandler, { + auth: root, + body: { + datasetId: String(dataset._id), + pageSize: 10, + offset: 0, + tagFilters: [{ tagId: String(docType._id), values: ['PRD'] }] + } + }); + + expect(listRes.code).toBe(200); + expect(listRes.data.list.map((item) => item.name)).toEqual(['prd-file']); + }); +}); diff --git a/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts b/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts index b71fd531e800..bbc80c80d90a 100644 --- a/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts +++ b/projects/app/test/api/core/dataset/collection/trainingStatus.test.ts @@ -1,5 +1,4 @@ import listHandler from '@/pages/api/core/dataset/collection/listV2'; -import scrollListHandler from '@/pages/api/core/dataset/collection/scrollList'; import detailHandler from '@/pages/api/core/dataset/collection/detail'; import trainingDetailHandler from '@/pages/api/core/dataset/collection/trainingDetail'; import { @@ -7,7 +6,6 @@ import { DatasetCollectionTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { DatasetCollectionsListItemSchema } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; @@ -60,8 +58,7 @@ describe('collection training status api', () => { body: { datasetId: dataset._id, pageSize: 10, - offset: 0, - filterTags: [] + offset: 0 } }); @@ -230,50 +227,4 @@ describe('collection training status api', () => { expect(res.data.queuedCounts.parse).toBe(1); expect(res.data.queuedCounts.chunk).toBe(0); }); - - it('should keep deprecated scrollList compatible with the collection list item schema', async () => { - const root = await getRootUser(); - const dataset = await MongoDataset.create({ - name: 'test', - teamId: root.teamId, - tmbId: root.tmbId, - vectorModel: 'test', - agentModel: 'test' - }); - const collection = await MongoDatasetCollection.create({ - name: 'test', - type: DatasetCollectionTypeEnum.file, - teamId: root.teamId, - tmbId: root.tmbId, - datasetId: dataset._id - }); - - await MongoDatasetTraining.create({ - teamId: root.teamId, - tmbId: root.tmbId, - datasetId: dataset._id, - collectionId: collection._id, - billId: 'test', - mode: TrainingModeEnum.chunk, - retryCount: 0, - errorMsg: 'final failed' - }); - - const res = await Call(scrollListHandler, { - auth: root, - body: { - datasetId: dataset._id, - pageSize: 10, - offset: 0, - filterTags: [] - } - }); - - expect(res.code).toBe(200); - expect(() => DatasetCollectionsListItemSchema.parse(res.data.list[0])).not.toThrow(); - expect(res.data.list[0]).toMatchObject({ - trainingAmount: 1, - slowestTrainingStatus: CollectionTrainingStatusEnum.ready - }); - }); }); diff --git a/projects/app/test/migration/registry.test.ts b/projects/app/test/migration/registry.test.ts index 63b253b5c6e3..f256a55472cc 100644 --- a/projects/app/test/migration/registry.test.ts +++ b/projects/app/test/migration/registry.test.ts @@ -24,7 +24,7 @@ describe('validateSystemMigrationRegistry', () => { onFailure: SystemMigrationFailurePolicyEnum.stop, progressSteps: [{ key: 'members' }, { key: 'validation' }] }); - expect(systemMigrations.slice(2).map((migration) => migration.id)).toEqual([ + expect(systemMigrations.slice(2, -1).map((migration) => migration.id)).toEqual([ '20260903_backfill_model_permissions', '20260903_backfill_dataset_model_references', '20260903_backfill_evaluation_model_references', @@ -33,10 +33,10 @@ describe('validateSystemMigrationRegistry', () => { '20260905_backfill_bill_metadata', '20260905_backfill_resource_owner_acl' ]); - expect(systemMigrations.slice(2).every((migration) => !migration.blockStartup)).toBe(true); + expect(systemMigrations.slice(2, -1).every((migration) => !migration.blockStartup)).toBe(true); expect( systemMigrations - .slice(2) + .slice(2, -1) .every((migration) => migration.onFailure === SystemMigrationFailurePolicyEnum.continue) ).toBe(true); expect(systemMigrations[1]).toMatchObject({ @@ -45,6 +45,13 @@ describe('validateSystemMigrationRegistry', () => { blockStartup: true, onFailure: SystemMigrationFailurePolicyEnum.stop }); + expect(systemMigrations.at(-1)).toMatchObject({ + id: '20260907_migrate_dataset_tags_v2', + version: '4.17.0', + blockStartup: true, + onFailure: SystemMigrationFailurePolicyEnum.stop, + progressSteps: [{ key: 'datasets' }, { key: 'collections' }, { key: 'validation' }] + }); }); it('accepts an ordered registry with stable IDs', () => { diff --git a/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/index.test.ts b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/index.test.ts new file mode 100644 index 000000000000..9e2c3eb03935 --- /dev/null +++ b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/index.test.ts @@ -0,0 +1,172 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { SystemMigrationContext } from '@/migration/registry'; +import type { SystemMigrationProgressInput } from '@fastgpt/global/migration/schema'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { migrateDatasetTagsV2 } from '@/migration/tasks/20260907_migrate_dataset_tags_v2'; + +vi.mock('@/migration/constants', () => ({ systemMigrationBatchSize: 1 })); + +const createContext = () => { + let checkpoint: Record | undefined; + const context = { + migrationId: '20260907_migrate_dataset_tags_v2', + runId: 'test-run', + signal: new AbortController().signal, + getCheckpoint: async (schema) => + checkpoint === undefined ? undefined : schema.parse(checkpoint), + saveCheckpoint: vi.fn(async (value: Record) => { + checkpoint = structuredClone(value); + }), + assertActive: vi.fn(async () => undefined), + reportProgress: vi.fn(async (_value: SystemMigrationProgressInput) => undefined), + getFailedRecords: vi.fn(async () => []), + reportFailedRecords: vi.fn(async () => undefined), + fail: vi.fn(async () => { + throw new Error('Blocking tasks must throw'); + }), + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } + } satisfies SystemMigrationContext; + return { context, getCheckpoint: () => checkpoint }; +}; + +const seedMigrationData = async () => { + const teamId = new Types.ObjectId(); + const tmbId = new Types.ObjectId(); + const datasetId = new Types.ObjectId(); + await MongoDataset.collection.insertOne({ + _id: datasetId, + teamId, + tmbId, + name: 'dataset', + type: 'dataset' + }); + const legacyTagId = new Types.ObjectId(); + await MongoDatasetCollectionTags.collection.insertOne({ + _id: legacyTagId, + teamId, + datasetId, + tag: 'legacy' + }); + const collectionId = new Types.ObjectId(); + await MongoDatasetCollection.collection.insertOne({ + _id: collectionId, + teamId, + tmbId, + datasetId, + name: 'file', + type: 'file', + tags: [String(legacyTagId)] + }); + return { datasetId, collectionId }; +}; + +describe('migrateDatasetTagsV2', () => { + beforeEach(async () => { + await Promise.all([ + MongoDataset.collection.deleteMany({}), + MongoDatasetCollection.collection.deleteMany({}), + MongoDatasetCollectionTags.collection.deleteMany({}), + MongoDatasetCollectionTagsV2.collection.deleteMany({}) + ]); + }); + + it('migrates legacy collection tags before validation succeeds', async () => { + const ids = await seedMigrationData(); + const { context } = createContext(); + + await expect(migrateDatasetTagsV2(context)).resolves.toMatchObject({ + datasetsProcessedCount: 1, + collectionsMigratedCount: 1, + legacyCollectionCount: 0 + }); + const carrier = await MongoDatasetCollectionTagsV2.collection.findOne({ + datasetId: ids.datasetId, + fromMigration: true + }); + expect(carrier).toMatchObject({ tag: 'default_tag', tagType: 'array' }); + expect( + (await MongoDatasetCollection.collection.findOne({ _id: ids.collectionId }))?.tags + ).toEqual([{ tagId: String(carrier?._id), value: ['legacy'] }]); + expect(context.getFailedRecords).not.toHaveBeenCalled(); + expect(context.reportFailedRecords).not.toHaveBeenCalled(); + }); + + it('can rerun without duplicating carriers or changing migrated data', async () => { + const ids = await seedMigrationData(); + await migrateDatasetTagsV2(createContext().context); + const firstCollection = await MongoDatasetCollection.collection.findOne({ + _id: ids.collectionId + }); + + await expect(migrateDatasetTagsV2(createContext().context)).resolves.toMatchObject({ + collectionsMigratedCount: 0, + duplicateDefinitionsDeletedCount: 0, + legacyCollectionCount: 0 + }); + expect( + await MongoDatasetCollectionTagsV2.collection.countDocuments({ + datasetId: ids.datasetId, + fromMigration: true + }) + ).toBe(1); + expect(await MongoDatasetCollection.collection.findOne({ _id: ids.collectionId })).toEqual( + firstCollection + ); + }); + + it('resumes from the saved checkpoint and safely replays after interruption', async () => { + await seedMigrationData(); + const state = createContext(); + state.context.reportProgress.mockImplementation(async (progress) => { + if (progress.key === 'datasets' && progress.status === 'running' && progress.current === 1) { + throw new Error('interrupted'); + } + }); + + await expect(migrateDatasetTagsV2(state.context)).rejects.toThrow('interrupted'); + expect(state.getCheckpoint()).toMatchObject({ + version: 1, + stages: { datasets: { lastId: expect.any(String), processedCount: 1 } } + }); + state.context.reportProgress.mockResolvedValue(undefined); + await expect(migrateDatasetTagsV2(state.context)).resolves.toMatchObject({ + datasetsProcessedCount: 1, + collectionsMigratedCount: 1 + }); + }); + + it('does not complete when final validation observes a late legacy write', async () => { + const ids = await seedMigrationData(); + const state = createContext(); + state.context.reportProgress.mockImplementation(async (progress) => { + if (progress.key !== 'validation' || progress.status !== 'running') return; + const dataset = await MongoDataset.collection.findOne({ _id: ids.datasetId }); + await MongoDatasetCollection.collection.insertOne({ + teamId: dataset?.teamId, + tmbId: new Types.ObjectId(), + datasetId: ids.datasetId, + name: 'late legacy write', + type: 'file', + tags: [String(new Types.ObjectId())] + }); + }); + + await expect(migrateDatasetTagsV2(state.context)).rejects.toThrow( + 'Dataset tag migration validation failed' + ); + expect(state.getCheckpoint()).toMatchObject({ + stages: { + datasets: { initialized: false, completed: false }, + collections: { initialized: false, completed: false } + } + }); + expect(state.context.reportProgress).not.toHaveBeenCalledWith({ + key: 'validation', + status: 'succeeded' + }); + }); +}); diff --git a/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/service.test.ts b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/service.test.ts new file mode 100644 index 000000000000..5b21ee4a6971 --- /dev/null +++ b/projects/app/test/migration/tasks/20260907_migrate_dataset_tags_v2/service.test.ts @@ -0,0 +1,143 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { Types } from '@fastgpt/service/common/mongo'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema'; +import { MongoDatasetCollectionTagsV2 } from '@fastgpt/service/core/dataset/tag/schemaV2'; +import { + migrateCollectionTagValues, + migrateDatasetTagDefinitions +} from '@/migration/tasks/20260907_migrate_dataset_tags_v2/service'; + +const tagUniqueIndexName = 'teamId_1_datasetId_1_tag_1'; +const teamId = new Types.ObjectId(); +const datasetId = new Types.ObjectId(); +const tmbId = new Types.ObjectId(); + +describe('dataset tag v2 migration service', () => { + beforeEach(async () => { + await Promise.all([ + MongoDatasetCollection.collection.deleteMany({ teamId }), + MongoDatasetCollectionTags.collection.deleteMany({ teamId }), + MongoDatasetCollectionTagsV2.collection.deleteMany({ teamId }) + ]); + }); + + afterEach(async () => { + await Promise.all([ + MongoDatasetCollection.collection.deleteMany({ teamId }), + MongoDatasetCollectionTags.collection.deleteMany({ teamId }), + MongoDatasetCollectionTagsV2.collection.deleteMany({ teamId }) + ]); + await MongoDatasetCollectionTagsV2.createIndexes({ background: true }); + }); + + it('removes duplicate references before definitions and remains idempotent', async () => { + await MongoDatasetCollectionTagsV2.collection.dropIndex(tagUniqueIndexName).catch(() => {}); + const keptId = new Types.ObjectId(); + const duplicateId = new Types.ObjectId(); + const unrelatedId = new Types.ObjectId(); + await MongoDatasetCollectionTagsV2.collection.insertMany([ + { _id: keptId, teamId, datasetId, tag: 'product', tagType: 'string' }, + { _id: duplicateId, teamId, datasetId, tag: 'product', tagType: 'string' }, + { _id: unrelatedId, teamId, datasetId, tag: 'score', tagType: 'number' } + ]); + const collection = await MongoDatasetCollection.collection.insertOne({ + teamId, + tmbId, + datasetId, + name: 'file', + type: 'file', + tags: [ + { tagId: String(duplicateId), value: 'FastGPT' }, + { tagId: String(unrelatedId), value: 10 } + ] + }); + + await expect(migrateDatasetTagDefinitions({ datasetId, teamId })).resolves.toMatchObject({ + migratedCount: 1, + deletedDefinitionCount: 1, + deletedReferenceCollectionCount: 1 + }); + expect(await MongoDatasetCollectionTagsV2.collection.findOne({ _id: keptId })).toBeTruthy(); + expect(await MongoDatasetCollectionTagsV2.collection.findOne({ _id: duplicateId })).toBeNull(); + expect( + (await MongoDatasetCollection.collection.findOne({ _id: collection.insertedId }))?.tags + ).toEqual([{ tagId: String(unrelatedId), value: 10 }]); + expect( + await MongoDatasetCollectionTagsV2.collection.countDocuments({ teamId, datasetId }) + ).toBe(2); + + await expect(migrateDatasetTagDefinitions({ datasetId, teamId })).resolves.toMatchObject({ + deletedDefinitionCount: 0, + deletedReferenceCollectionCount: 0 + }); + }); + + it('converts legacy tag ids into the dataset carrier array and preserves typed tags', async () => { + const legacy = await MongoDatasetCollectionTags.create({ + teamId, + datasetId, + tag: 'legacy-name' + }); + const typed = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId, + tag: 'score', + tagType: 'number' + }); + const collection = await MongoDatasetCollection.collection.insertOne({ + teamId, + tmbId, + datasetId, + name: 'file', + type: 'file', + tags: [String(legacy._id), { tagId: String(typed._id), value: 10 }] + }); + + await expect( + migrateCollectionTagValues({ collectionId: collection.insertedId }) + ).resolves.toEqual({ migratedCount: 1 }); + const carrier = await MongoDatasetCollectionTagsV2.collection.findOne({ + teamId, + datasetId, + fromMigration: true + }); + expect(carrier).toMatchObject({ tag: 'default_tag', tagType: 'array' }); + expect( + (await MongoDatasetCollection.collection.findOne({ _id: collection.insertedId }))?.tags + ).toEqual([ + { tagId: String(typed._id), value: 10 }, + { tagId: String(carrier?._id), value: ['legacy-name'] } + ]); + }); + + it('treats default_tag as a normal name and identifies carriers only by fromMigration', async () => { + const ordinary = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId, + tag: 'default_tag', + tagType: 'string' + }); + const carrier = await MongoDatasetCollectionTagsV2.create({ + teamId, + datasetId, + tag: 'renamed carrier', + tagType: 'string', + fromMigration: true + }); + + await expect(migrateDatasetTagDefinitions({ datasetId, teamId })).resolves.toMatchObject({ + deletedDefinitionCount: 0 + }); + expect(await MongoDatasetCollectionTagsV2.findById(ordinary._id).lean()).toMatchObject({ + tag: 'default_tag', + tagType: 'string', + fromMigration: false + }); + expect(await MongoDatasetCollectionTagsV2.findById(carrier._id).lean()).toMatchObject({ + tag: 'renamed carrier', + tagType: 'array', + fromMigration: true + }); + }); +}); diff --git a/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts b/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts index b9a68ed389fa..88aa1dcf86bf 100644 --- a/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts +++ b/projects/app/test/pageComponents/app/detail/Edit/ChatAgent/utils.test.ts @@ -114,4 +114,23 @@ describe('agentForm2AppWorkflow model reference', () => { datasetSearchExtensionModel: 'legacy-extension' }); }); + + it('round-trips the structured collection filter in nested agent params', () => { + const form = getDefaultAppForm(); + form.dataset.collectionFilterMatch = { + logic: 'AND', + conditions: [{ tag: 'price', tagType: 'number', op: '$gte', value: 10 }] + }; + const workflow = agentForm2AppWorkflow(form, (key: string) => key); + const datasetParams = workflow.nodes + .flatMap((node) => node.inputs) + .find((input) => input.key === NodeInputKeyEnum.datasetParams)?.value; + expect(datasetParams).toMatchObject({ + collectionFilterMatch: form.dataset.collectionFilterMatch + }); + expect( + appWorkflow2AgentForm({ nodes: workflow.nodes, chatConfig: workflow.chatConfig }).dataset + .collectionFilterMatch + ).toEqual(form.dataset.collectionFilterMatch); + }); }); diff --git a/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts b/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts index cbb97dedc440..bdccc715cb9a 100644 --- a/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts +++ b/projects/app/test/pageComponents/app/detail/Edit/SimpleApp/utils.test.ts @@ -22,6 +22,19 @@ const getModelInputs = ({ modelId, model }: { modelId?: string; model?: string } ); }; +const getFormWithDataset = () => { + const form = getDefaultAppForm(); + form.dataset.datasets = [ + { + datasetId: 'dataset-id', + avatar: 'dataset.svg', + name: 'Dataset', + vectorModel: { model: 'embedding-model' } + } + ]; + return form; +}; + describe('form2AppWorkflow model reference', () => { it.each([ { type: FlowNodeTypeEnum.appModule, hasStreamInput: true }, @@ -115,15 +128,7 @@ describe('form2AppWorkflow model reference', () => { }); it('preserves empty dataset model IDs instead of falling back to legacy fields', () => { - const form = getDefaultAppForm(); - form.dataset.datasets = [ - { - datasetId: 'dataset-id', - avatar: 'dataset.svg', - name: 'Dataset', - vectorModel: { model: 'embedding-model' } - } - ]; + const form = getFormWithDataset(); form.dataset.rerankModelId = ''; form.dataset.rerankModel = 'legacy-rerank'; form.dataset.datasetSearchExtensionModelId = ''; @@ -148,4 +153,61 @@ describe('form2AppWorkflow model reference', () => { ]) ); }); + + it('round-trips collectionFilterMatch through the dataset search node', () => { + const form = getFormWithDataset(); + form.dataset.collectionFilterMatch = { + logic: 'AND', + conditions: [{ tag: 'price', tagType: 'number', op: '$gte', value: 10 }] + }; + + const workflow = form2AppWorkflow(form, (key: string) => key); + const input = workflow.nodes + .flatMap((node) => node.inputs) + .find((item) => item.key === NodeInputKeyEnum.collectionFilterMatch); + + expect(input).toMatchObject({ + key: NodeInputKeyEnum.collectionFilterMatch, + renderTypeList: ['datasetTagFilter', 'reference'], + value: form.dataset.collectionFilterMatch + }); + expect( + appWorkflow2Form({ nodes: workflow.nodes, chatConfig: form.chatConfig }).dataset + .collectionFilterMatch + ).toEqual(form.dataset.collectionFilterMatch); + }); + + it('preserves the explicit legacy filter marker until the user upgrades it', () => { + const form = getFormWithDataset(); + form.dataset[NodeInputKeyEnum.collectionFilterVersion] = 'legacy'; + form.dataset.collectionFilterMatch = '{"tags":{"$and":["legacy"]}}'; + + const workflow = form2AppWorkflow(form, (key: string) => key); + const datasetNode = workflow.nodes.find( + (node) => node.flowNodeType === FlowNodeTypeEnum.datasetSearchNode + ); + expect(datasetNode).toBeTruthy(); + expect( + datasetNode?.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion) + ?.value + ).toBe('legacy'); + expect(appWorkflow2Form({ nodes: workflow.nodes, chatConfig: {} }).dataset).toMatchObject({ + [NodeInputKeyEnum.collectionFilterVersion]: 'legacy', + collectionFilterMatch: form.dataset.collectionFilterMatch + }); + + const unversionedNode = { + ...datasetNode!, + inputs: datasetNode!.inputs + .filter((input) => input.key !== NodeInputKeyEnum.collectionFilterVersion) + .map((input) => + input.key === NodeInputKeyEnum.collectionFilterMatch + ? { ...input, value: undefined } + : input + ) + }; + expect(appWorkflow2Form({ nodes: [unversionedNode], chatConfig: {} }).dataset).toMatchObject({ + [NodeInputKeyEnum.collectionFilterVersion]: 'structured' + }); + }); }); diff --git a/projects/app/test/web/core/app/workflow/utils.test.ts b/projects/app/test/web/core/app/workflow/utils.test.ts index 12a6113b96e2..e0c073bc1f6c 100644 --- a/projects/app/test/web/core/app/workflow/utils.test.ts +++ b/projects/app/test/web/core/app/workflow/utils.test.ts @@ -34,10 +34,7 @@ import { collectWorkflowStartAutoFillRevertPatches, collectWorkflowStartOutputAutoFillRevertPatches } from '@/web/core/workflow/workflowStartAutoFill'; -import type { - FlowNodeInputItemType, - FlowNodeOutputItemType -} from '@fastgpt/global/core/workflow/type/io'; +import type { FlowNodeOutputItemType } from '@fastgpt/global/core/workflow/type/io'; import { NodeOutputKeyEnum, VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants'; import { PluginStatusEnum } from '@fastgpt/global/core/plugin/type'; import { AppErrEnum } from '@fastgpt/global/common/error/code/app'; @@ -53,9 +50,11 @@ import { HttpNode468 } from '@fastgpt/global/core/workflow/template/system/http4 import { LoopStartNode } from '@fastgpt/global/core/workflow/template/system/loop/loopStart'; import { AiChatModule } from '@fastgpt/global/core/workflow/template/system/aiChat'; import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch'; +import { AgentNode } from '@fastgpt/global/core/workflow/template/system/agent'; import { ClassifyQuestionModule } from '@fastgpt/global/core/workflow/template/system/classifyQuestion'; import { ToolCallNode } from '@fastgpt/global/core/workflow/template/system/toolCall'; import { userFilesInput } from '@fastgpt/global/core/workflow/template/system/workflowStart'; +import { Input_Template_Dataset_Tag_Filter_Version } from '@fastgpt/global/core/workflow/template/input'; import { ModelTypeEnum } from '@fastgpt/global/core/ai/constants'; describe('nodeTemplate2FlowNode', () => { @@ -2223,6 +2222,87 @@ describe('workflow model validation', () => { }); describe('storeNode2FlowNode', () => { + it.each([ + { + name: 'unversioned node with metadata', + version: undefined, + filterValue: { logic: 'AND', conditions: [] }, + expectedVersion: 'legacy', + expectedLabel: 'workflow:collection_metadata_filter' + }, + { + name: 'unversioned node without metadata', + version: undefined, + filterValue: undefined, + expectedVersion: 'structured', + expectedLabel: 'workflow:tag_filter' + }, + { + name: 'explicitly structured node with a legacy-shaped value', + version: 'structured', + filterValue: '{"tags":{"$and":["legacy-shape"]}}', + expectedVersion: 'structured', + expectedLabel: 'workflow:tag_filter' + } + ])( + 'hydrates $name as $expectedVersion', + ({ version, filterValue, expectedVersion, expectedLabel }) => { + const storeNode = { + ...DatasetSearchModule, + nodeId: 'dataset-search', + position: { x: 0, y: 0 }, + inputs: DatasetSearchModule.inputs + .filter((input) => input.key !== NodeInputKeyEnum.collectionFilterVersion) + .map((input) => + input.key === NodeInputKeyEnum.collectionFilterMatch + ? { ...input, value: filterValue } + : input + ) + .concat(version ? [{ ...Input_Template_Dataset_Tag_Filter_Version, value: version }] : []) + } as StoreNodeItemType; + + const result = storeNode2FlowNode({ + item: storeNode, + t: ((key: string) => key) as any + }); + + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion) + ?.value + ).toBe(expectedVersion); + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterMatch) + ).toMatchObject({ + label: expectedLabel + }); + } + ); + + it('hydrates Agent V2 with an explicit structured marker', () => { + const storeNode = { + ...AgentNode, + nodeId: 'agent', + position: { x: 0, y: 0 }, + inputs: AgentNode.inputs.filter( + (input) => input.key !== NodeInputKeyEnum.collectionFilterVersion + ) + } as StoreNodeItemType; + + const result = storeNode2FlowNode({ + item: storeNode, + t: ((key: string) => key) as any + }); + + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterVersion) + ?.value + ).toBe('structured'); + expect( + result.data.inputs.find((input) => input.key === NodeInputKeyEnum.collectionFilterMatch) + ?.label + ).toBe('workflow:tag_filter'); + }); + it('restores tool set nodes without a source handle', () => { const storeNode = { nodeId: 'tool-set-node', diff --git a/projects/app/test/web/core/workflow/datasetSearchNodeUpgrade.test.ts b/projects/app/test/web/core/workflow/datasetSearchNodeUpgrade.test.ts new file mode 100644 index 000000000000..2bdd1be40728 --- /dev/null +++ b/projects/app/test/web/core/workflow/datasetSearchNodeUpgrade.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + FlowNodeInputTypeEnum, + FlowNodeTypeEnum +} from '@fastgpt/global/core/workflow/node/constant'; +import { + getDatasetSearchFilterVersion, + persistLegacyDatasetSearchNodeUpgrade +} from '@/web/core/workflow/datasetSearchNodeUpgrade'; +import { DatasetTagFilterVersionEnum } from '@fastgpt/global/core/dataset/workflowTagFilter'; + +const legacyNode = { + nodeId: 'dataset-node', + name: 'Dataset search', + flowNodeType: FlowNodeTypeEnum.datasetSearchNode, + version: '4.9.2', + inputs: [ + { key: 'datasetSelectList', label: '', renderTypeList: [], value: ['dataset'] }, + { + key: 'collectionFilterMatch', + label: '', + renderTypeList: [FlowNodeInputTypeEnum.datasetTagFilter], + value: '{"tags":{"$and":["legacy"]}}' + } + ], + outputs: [{ key: 'quoteQA', label: '', type: 'static' }] +} as any; +const upgradedInput = { + ...legacyNode.inputs[1], + value: { logic: 'AND', conditions: [] } +}; +const versionInput = { + key: 'collectionFilterVersion', + label: '', + renderTypeList: [FlowNodeInputTypeEnum.hidden], + value: 'legacy' +} as any; + +describe('dataset search node upgrade', () => { + it('does not commit the local upgrade when persistence fails', async () => { + const commit = vi.fn(); + const persist = vi.fn().mockRejectedValue(new Error('save failed')); + await expect( + persistLegacyDatasetSearchNodeUpgrade({ + nodes: [legacyNode], + nodeId: legacyNode.nodeId, + filterInput: upgradedInput, + persist, + commit + }) + ).rejects.toThrow('save failed'); + expect(commit).not.toHaveBeenCalled(); + }); + + it('persists and commits one canonical upgraded node without changing shared node data', async () => { + const persist = vi.fn(async () => undefined); + const commit = vi.fn(); + + await persistLegacyDatasetSearchNodeUpgrade({ + nodes: [legacyNode], + nodeId: legacyNode.nodeId, + filterInput: upgradedInput, + persist, + commit + }); + + const persistedNode = persist.mock.calls[0][0][0]; + expect(commit).toHaveBeenCalledWith(persistedNode); + expect(persistedNode).toMatchObject({ + nodeId: legacyNode.nodeId, + name: legacyNode.name, + flowNodeType: FlowNodeTypeEnum.datasetSearchNode, + outputs: legacyNode.outputs + }); + expect(persistedNode.inputs[0]).toEqual(legacyNode.inputs[0]); + expect(persistedNode.inputs.find((input) => input.key === upgradedInput.key)).toEqual( + upgradedInput + ); + expect(getDatasetSearchFilterVersion(persistedNode.inputs)).toBe( + DatasetTagFilterVersionEnum.structured + ); + expect(persistedNode.inputs.filter((input) => input.key === versionInput.key)).toHaveLength(1); + }); +});