Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .agents/design/core/dataset/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -811,7 +811,7 @@ POST /api/core/dataset/collection/create/link # 链接导入
POST /api/core/dataset/collection/create/text # 文本导入
POST /api/core/dataset/collection/create/images # 图片导入
PUT /api/core/dataset/collection/update # 更新集合
GET /api/core/dataset/collection/list # 集合列表
POST /api/core/dataset/collection/listV2 # 集合列表
GET /api/core/dataset/collection/detail # 集合详情
POST /api/core/dataset/collection/sync # 同步集合
GET /api/core/dataset/collection/export # 导出集合
Expand Down Expand Up @@ -893,9 +893,9 @@ MongoDatasetData.find(query, fields, {

### 3. 分页优化
```typescript
// 使用 scrollList 而非传统分页
// 避免深度分页性能问题
GET /api/core/dataset/collection/scrollList?lastId=xxx&limit=20
// 使用 offset 分页,避免深度 pageNum 分页性能问题
POST /api/core/dataset/collection/listV2
{ datasetId, offset, pageSize }
```

### 4. 缓存策略
Expand Down
616 changes: 616 additions & 0 deletions .agents/design/知识库标签-设计文档.md

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,4 @@ pro/admin/worker/
/pro/llm_benchmark/content_benchmark/eval-runs/
/pro/llm_benchmark/content_benchmark/.cache/
.gstack/
.worktrees/
36 changes: 35 additions & 1 deletion packages/global/common/error/code/dataset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,15 @@ export enum DatasetErrEnum {
notSupportSync = 'notSupportSync',
sameApiCollection = 'sameApiCollection',
noApiServer = 'noApiServer',
canNotEditAdminPermission = 'canNotEditAdminPermission'
canNotEditAdminPermission = 'canNotEditAdminPermission',

// Tag errors (501013+)
tagNameDuplicate = 'tagNameDuplicate',
tagNameEmpty = 'tagNameEmpty',
tagNotExist = 'tagNotExist',
tagValueInvalid = 'tagValueInvalid',
tagValueDatetimeInvalid = 'tagValueDatetimeInvalid',
arrayTagValueInvalid = 'arrayTagValueInvalid'
}
const datasetErr = [
{
Expand Down Expand Up @@ -69,6 +77,32 @@ const datasetErr = [
{
statusText: DatasetErrEnum.noApiServer,
message: i18nT('common:core.dataset.error.noApiServer')
},

// Tag errors
{
statusText: DatasetErrEnum.tagNameDuplicate,
message: i18nT('common:core.dataset.error.tagNameDuplicate')
},
{
statusText: DatasetErrEnum.tagNameEmpty,
message: i18nT('common:core.dataset.error.tagNameEmpty')
},
{
statusText: DatasetErrEnum.tagNotExist,
message: i18nT('common:core.dataset.error.tagNotExist')
},
{
statusText: DatasetErrEnum.tagValueInvalid,
message: i18nT('common:core.dataset.error.tagValueInvalid')
},
{
statusText: DatasetErrEnum.tagValueDatetimeInvalid,
message: i18nT('common:core.dataset.error.tagValueDatetimeInvalid')
},
{
statusText: DatasetErrEnum.arrayTagValueInvalid,
message: i18nT('common:core.dataset.error.arrayTagValueInvalid')
}
];
export default datasetErr.reduce((acc, cur, index) => {
Expand Down
2 changes: 1 addition & 1 deletion packages/global/common/string/time.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ export const cronParser2Fields = (cronString: string) => {
try {
const cronField = cronParser.parseExpression(cronString).fields;
return cronField;
} catch (error) {
} catch {
return null;
}
};
Expand Down
3 changes: 2 additions & 1 deletion packages/global/core/app/formEdit/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ const agentGeneratedDenyRenderTypes = new Set<FlowNodeInputTypeEnum>([
FlowNodeInputTypeEnum.selectTool,
FlowNodeInputTypeEnum.selectDataset,
FlowNodeInputTypeEnum.selectDatasetParamsModal,
FlowNodeInputTypeEnum.settingDatasetQuotePrompt
FlowNodeInputTypeEnum.settingDatasetQuotePrompt,
FlowNodeInputTypeEnum.datasetTagFilter
]);

// 工具配置不能处理依赖文件、知识库、模型或外部动态上下文的输入。
Expand Down
7 changes: 6 additions & 1 deletion packages/global/core/app/type.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ import { StoreNodeItemTypeSchema } from '../workflow/type/node';
import { AppTypeEnum } from './constants';
import { NodeInputKeyEnum } from '../workflow/constants';
import { DatasetSearchModeEnum } from '../dataset/constants';
import {
DatasetTagFilterValueSchema,
DatasetTagFilterVersionSchema
} from '../dataset/workflowTagFilter';
import type { ReasoningEffort } from '../ai/llm/type';
import { StoreEdgeItemTypeSchema } from '../workflow/type/edge';
import type { AppPermission } from '../../support/permission/app/controller';
Expand Down Expand Up @@ -271,7 +275,8 @@ export const AppDatasetSearchParamsTypeSchema = z.object({
datasetSearchExtensionBg: z.string().optional(),
[NodeInputKeyEnum.authTmbId]: BoolSchema.optional(),

collectionFilterMatch: z.string().optional()
collectionFilterMatch: z.union([z.string(), DatasetTagFilterValueSchema]).optional(),
[NodeInputKeyEnum.collectionFilterVersion]: DatasetTagFilterVersionSchema.optional()
});
export type AppDatasetSearchParamsType = z.infer<typeof AppDatasetSearchParamsTypeSchema>;

Expand Down
2 changes: 2 additions & 0 deletions packages/global/core/app/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { AppTypeEnum } from './constants';
import appErrList from '../../common/error/code/app';
import pluginErrList from '../../common/error/code/plugin';
import { i18nT } from '../../common/i18n/utils';
import { DatasetTagFilterVersionEnum } from '../dataset/workflowTagFilter';

const deletedPluginErrorList = new Set([
'plugin.team_not_installed',
Expand Down Expand Up @@ -34,6 +35,7 @@ export const getDefaultAppForm = (): AppFormEditFormType => {
rerankWeight: 0.5,
datasetSearchUsingExtensionQuery: true,
datasetSearchExtensionBg: '',
[NodeInputKeyEnum.collectionFilterVersion]: DatasetTagFilterVersionEnum.structured,
[NodeInputKeyEnum.authTmbId]: false
},
selectedTools: [],
Expand Down
7 changes: 6 additions & 1 deletion packages/global/core/dataset/collection/utils.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import { DatasetCollectionTypeEnum } from '../constants';
import { type DatasetCollectionSchemaType } from '../type';

export const getCollectionSourceData = (collection?: DatasetCollectionSchemaType) => {
export const getCollectionSourceData = (
collection?: Pick<
DatasetCollectionSchemaType,
'fileId' | 'rawLink' | 'externalFileId' | 'externalFileUrl' | 'apiFileId' | 'name'
>
) => {
return {
sourceId:
collection?.fileId ||
Expand Down
23 changes: 23 additions & 0 deletions packages/global/core/dataset/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,29 @@ export const DatasetCollectionSyncResultMap = {
}
};

/* ------------ collection tags -------------- */
export enum DatasetCollectionTagTypeEnum {
string = 'string',
number = 'number',
datetime = 'datetime',
array = 'array'
}

export const DatasetCollectionTagTypeMap = {
[DatasetCollectionTagTypeEnum.string]: {
label: i18nT('dataset:core.dataset.tags.string')
},
[DatasetCollectionTagTypeEnum.number]: {
label: i18nT('dataset:core.dataset.tags.number')
},
[DatasetCollectionTagTypeEnum.datetime]: {
label: i18nT('dataset:core.dataset.tags.date')
},
[DatasetCollectionTagTypeEnum.array]: {
label: i18nT('dataset:core.dataset.tags.array')
}
};

export enum DatasetCollectionDataProcessModeEnum {
chunk = 'chunk',
qa = 'qa',
Expand Down
19 changes: 19 additions & 0 deletions packages/global/core/dataset/tagUtils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import type { CollectionTagValueType } from './type';

/** 同一标签类型固定:数字按大小,其余按字典序。前后端筛选项展示共用。 */
export const sortCollectionTagValues = <T extends string | number>(values: T[]): T[] =>
[...values].sort((a, b) => {
if (typeof a === 'number' && typeof b === 'number') return a - b;
return String(a).localeCompare(String(b));
});

/** 筛选项只保留非空字符串和有限数字。 */
export const isUsableCollectionTagFilterValue = (value: unknown): value is string | number => {
if (typeof value === 'string') return value.length > 0;
if (typeof value === 'number') return Number.isFinite(value);
return false;
};

/** 新格式 { tagId, value };旧字符串标签排除。 */
export const isCollectionTagValue = (item: unknown): item is CollectionTagValueType =>
!!item && typeof item === 'object' && !Array.isArray(item) && 'tagId' in item && 'value' in item;
85 changes: 81 additions & 4 deletions packages/global/core/dataset/type.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import {
CollectionTrainingStatusEnum,
ChunkSettingModeEnum,
ChunkTriggerConfigTypeEnum,
ParagraphChunkAIModeEnum
ParagraphChunkAIModeEnum,
DatasetCollectionTagTypeEnum,
DatasetCollectionTagTypeMap
} from './constants';
import {
ApiDatasetServerSchema,
Expand All @@ -27,6 +29,59 @@ import { ObjectIdSchema } from '../../common/type/mongo';
import { PermissionSchema } from '../../support/permission/controller';
import { NumSchema } from '../../common/zod';

/* ===== Tag Type ===== */
/** 标签类型枚举 */
export { DatasetCollectionTagTypeEnum, DatasetCollectionTagTypeMap };
export type DatasetCollectionTagType = `${DatasetCollectionTagTypeEnum}`;

/** 选项类标签的预设选项,空选项由前端草稿过滤后再提交。 */
export const DatasetCollectionTagOptionsSchema = z
.array(z.string().trim().min(1))
.meta({ description: '选项类标签的预设选项' });

/** 旧字符串标签承载记录首次创建时使用的默认名称;身份只由 fromMigration 标识。 */
export const DEFAULT_TAG = 'default_tag';

/** Collection 标签值字段:string/number 存对应值,datetime 存 UTC 毫秒时间戳,array 存 string 数组 */
export const CollectionTagValueFieldSchema = z.union([z.string(), z.number(), z.array(z.string())]);

/** Collection API 的兼容标签输入/展示格式:旧标签名或带名称和值的新格式。 */
export const CollectionTagLabelSchema = z.union([
z.string(),
z.object({
tag: z.string(),
value: CollectionTagValueFieldSchema
})
]);
export type CollectionTagLabelType = z.infer<typeof CollectionTagLabelSchema>;

/** Collection 标签值类型(新格式) */
export const CollectionTagValueSchema = z.object({
tagId: z.string().meta({ description: '引用 dataset_collection_tags_v2._id' }),
value: CollectionTagValueFieldSchema.meta({
description:
'标签值。string/number 类型存对应值,datetime 类型存 UTC 毫秒时间戳,array 类型存 string 数组'
})
});
export type CollectionTagValueType = z.infer<typeof CollectionTagValueSchema>;

/** 详情页按标签值筛选的单条条件。同一标签多值为 OR,不同标签由调用方做 AND。 */
export const CollectionTagFilterItemSchema = z.object({
tagId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '标签 ID'
}),
values: z
.array(z.union([z.string().min(1), z.number().finite()]))
.min(1)
.meta({
example: ['PRD'],
description:
'选中的标签值。文本/选项为字符串,数字为数值,日期为 UTC 毫秒时间戳。同一标签多值为 OR'
})
});
export type CollectionTagFilterItem = z.infer<typeof CollectionTagFilterItemSchema>;

/* ===== Chunk ===== */
export const ChunkSettingsSchema = z.object({
trainingType: z
Expand Down Expand Up @@ -134,7 +189,10 @@ export const DatasetCollectionSchema = ChunkSettingsSchema.omit({
parentId: ParentIdSchema.meta({ description: '父级 ID' }),
name: z.string().meta({ description: '名称' }),
type: z.enum(DatasetCollectionTypeEnum).meta({ description: '集合类型' }),
tags: z.array(z.string()).optional().meta({ description: '标签' }),
tags: z
.array(z.union([z.string(), CollectionTagValueSchema]))
.optional()
.meta({ description: '标签。支持混合格式:String(ObjectId) 旧格式 | { tagId, value } 新格式' }),

createTime: z.coerce.date().meta({ description: '创建时间' }),
updateTime: z.coerce.date().meta({ description: '更新时间' }),
Expand Down Expand Up @@ -165,7 +223,14 @@ export const DatasetCollectionTagsSchema = z.object({
_id: ObjectIdSchema.meta({ description: '标签 ID' }),
teamId: ObjectIdSchema.meta({ description: '团队 ID' }),
datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }),
tag: z.string().meta({ description: '标签' })
tag: z.string().meta({ description: '标签' }),
tagType: z.enum(DatasetCollectionTagTypeEnum).default(DatasetCollectionTagTypeEnum.string).meta({
description: '标签类型:string(默认)/number/datetime/array'
}),
options: DatasetCollectionTagOptionsSchema.optional(),
fromMigration: z.boolean().optional().meta({
description: '标识该记录是旧标签迁移/旧格式输入创建的 default_tag 承载记录'
})
});
export type DatasetCollectionTagsSchemaType = z.infer<typeof DatasetCollectionTagsSchema>;

Expand Down Expand Up @@ -353,7 +418,14 @@ export type DatasetItemType = z.infer<typeof DatasetItemSchema>;
/* ================= tag ===================== */
export const DatasetTagSchema = z.object({
_id: ObjectIdSchema.meta({ description: '标签 ID' }),
tag: z.string().meta({ description: '标签' })
tag: z.string().meta({ description: '标签' }),
tagType: z.enum(DatasetCollectionTagTypeEnum).default(DatasetCollectionTagTypeEnum.string).meta({
description: '标签类型:string(默认)/number/datetime/array'
}),
options: DatasetCollectionTagOptionsSchema.optional(),
fromMigration: z.boolean().optional().meta({
description: '是否为旧标签数据的系统承载定义'
})
});
export type DatasetTagType = z.infer<typeof DatasetTagSchema>;

Expand All @@ -365,6 +437,11 @@ export type TagUsageType = z.infer<typeof TagUsageSchema>;

/* ================= collection ===================== */
export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({
// 详情接口的 tags 由 collectionTagsToTagLabel 解析为标签名格式(string | { tag, value }),区别于存储格式(string | { tagId, value })
tags: z
.array(CollectionTagLabelSchema)
.optional()
.meta({ description: '标签。string 为标签名;新格式为 { tag, value }' }),
sourceName: z.string().meta({ description: '来源名称' }),
sourceId: z.string().optional().meta({ description: '来源 ID' }),
file: z
Expand Down
Loading
Loading