diff --git a/packages/dal/redis/bullmq/binding.ts b/packages/dal/redis/bullmq/binding.ts index ebcda41371b0..a05a2555e0cf 100644 --- a/packages/dal/redis/bullmq/binding.ts +++ b/packages/dal/redis/bullmq/binding.ts @@ -1,7 +1,7 @@ import { getRedisRuntime } from '../runtime'; import { getRedisBullMQRuntime } from './context'; import type { QueueNames } from './names'; -import type { Processor, Queue, QueueOptions, Worker, WorkerOptions } from './types'; +import type { FlowProducer, Processor, Queue, QueueOptions, Worker, WorkerOptions } from './types'; const defaultWorkerOpts: Omit = { removeOnComplete: { @@ -59,6 +59,11 @@ export class BullMQBinding { ...opts }); } + + /** Returns the Runtime-owned FlowProducer for atomically creating dependent job trees. */ + getFlowProducer(): FlowProducer { + return this.getRuntime().getFlowProducer(); + } } /** 进程级 BullMQ 绑定。 */ diff --git a/packages/dal/redis/bullmq/flow-producer-manager.ts b/packages/dal/redis/bullmq/flow-producer-manager.ts new file mode 100644 index 000000000000..a8bca36c8761 --- /dev/null +++ b/packages/dal/redis/bullmq/flow-producer-manager.ts @@ -0,0 +1,61 @@ +import { FlowProducer } from 'bullmq'; +import type { RedisRuntime } from '../runtime/connection'; +import type { RedisRuntimeLogger } from '../types'; +import { closeWithTimeout, forceDisconnect } from './close'; + +/** Manages the shared FlowProducer lifecycle for a BullMQ Runtime. */ +export class BullMQFlowProducerManager { + private flowProducer: FlowProducer | undefined; + + constructor( + private readonly options: { + redisRuntime: RedisRuntime; + logger: RedisRuntimeLogger; + closeTimeoutMs: number; + } + ) {} + + /** Creates one FlowProducer and reuses it for all flows in this Runtime. */ + getFlowProducer() { + if (this.flowProducer) return this.flowProducer; + + const connection = this.options.redisRuntime.createQueueConnection(); + try { + const flowProducer = new FlowProducer({ connection }); + const errorHandler = (error: Error) => { + this.options.logger.error('BullMQ flow producer error', { error }); + }; + flowProducer.on('error', errorHandler); + this.flowProducer = flowProducer; + return flowProducer; + } catch (error) { + void this.options.redisRuntime.releaseConnection(connection).catch((releaseError) => { + this.options.logger.warn( + 'Failed to release Redis connection after FlowProducer creation error', + { error: releaseError } + ); + }); + throw error; + } + } + + async close() { + const flowProducer = this.flowProducer; + this.flowProducer = undefined; + if (!flowProducer) return; + + await closeWithTimeout({ + operation: () => flowProducer.close(), + resource: 'BullMQ flow producer', + timeoutMs: this.options.closeTimeoutMs + }).catch((error) => { + this.options.logger.warn('BullMQ flow producer close failed', { error }); + forceDisconnect({ + name: 'flow-producer', + resource: 'flow producer connection', + disconnect: () => flowProducer.disconnect(), + logger: this.options.logger + }); + }); + } +} diff --git a/packages/dal/redis/bullmq/index.ts b/packages/dal/redis/bullmq/index.ts index 4854bd99ce6b..ff86db9cc47c 100644 --- a/packages/dal/redis/bullmq/index.ts +++ b/packages/dal/redis/bullmq/index.ts @@ -2,6 +2,7 @@ export { bullMQ, BullMQBinding } from './binding'; export { getConfiguredRedisBullMQRuntime, getRedisBullMQRuntime } from './context'; export { QueueNames } from './names'; export { RedisBullMQRuntime } from './runtime'; +export { BullMQFlowProducerManager } from './flow-producer-manager'; export { addOrRequeueFailedJob } from './job-recovery'; export { DelayedError, UnrecoverableError } from 'bullmq'; export * from './services'; @@ -9,7 +10,10 @@ export type { BullMQRuntimeState, BullMQWorkerLifecycleOptions, ConnectionOptions, + FlowJob, + FlowProducer, Job, + JobNode, JobSchedulerJson, Processor, Queue, diff --git a/packages/dal/redis/bullmq/runtime.ts b/packages/dal/redis/bullmq/runtime.ts index 618d22c77f75..2cc2480557c9 100644 --- a/packages/dal/redis/bullmq/runtime.ts +++ b/packages/dal/redis/bullmq/runtime.ts @@ -1,4 +1,4 @@ -import type { Queue, Processor, QueueOptions, Worker, WorkerOptions } from 'bullmq'; +import type { FlowProducer, Processor, Queue, QueueOptions, Worker, WorkerOptions } from 'bullmq'; import type { RedisRuntime } from '../runtime/connection'; import { DEFAULT_CLOSE_TIMEOUT_MS, @@ -6,6 +6,7 @@ import { silentBullMQLogger } from './constants'; import { BullMQQueueManager } from './queue-manager'; +import { BullMQFlowProducerManager } from './flow-producer-manager'; import type { BullMQRuntimeState, BullMQWorkerLifecycleOptions, @@ -20,6 +21,7 @@ export class RedisBullMQRuntime { private readonly logger: RedisRuntimeLogger; private readonly queueManager: BullMQQueueManager; + private readonly flowProducerManager: BullMQFlowProducerManager; private readonly workerManager: BullMQWorkerManager; private readonly unregisterBeforeCloseHook: () => void; private state: BullMQRuntimeState = 'running'; @@ -44,6 +46,11 @@ export class RedisBullMQRuntime { logger, closeTimeoutMs }); + this.flowProducerManager = new BullMQFlowProducerManager({ + redisRuntime, + logger, + closeTimeoutMs + }); this.workerManager = new BullMQWorkerManager({ redisRuntime, logger, @@ -83,6 +90,12 @@ export class RedisBullMQRuntime { return this.workerManager.getWorker(name, processor, opts); } + /** Returns the Runtime-shared FlowProducer, which owns a dedicated queue connection. */ + getFlowProducer(): FlowProducer { + this.assertRunning(); + return this.flowProducerManager.getFlowProducer(); + } + close() { if (this.closePromise) return this.closePromise; @@ -101,6 +114,13 @@ export class RedisBullMQRuntime { } // Worker 关闭失败也不能跳过 Queue,否则队列连接会被 Redis Runtime 强制回收。 + try { + await this.flowProducerManager.close(); + } catch (error) { + firstError = error; + hasError = true; + } + try { await this.queueManager.close(); } catch (error) { diff --git a/packages/dal/redis/bullmq/services/appDelete.ts b/packages/dal/redis/bullmq/services/appDelete.ts index cf8817e1c400..e964bc1dbd82 100644 --- a/packages/dal/redis/bullmq/services/appDelete.ts +++ b/packages/dal/redis/bullmq/services/appDelete.ts @@ -1,11 +1,14 @@ import { bullMQ, type BullMQBinding } from '../binding'; import { addOrRequeueFailedJob } from '../job-recovery'; import { QueueNames } from '../names'; -import type { Processor, Queue, Worker } from '../types'; +import type { FlowJob, JobNode, Processor, Queue, Worker } from '../types'; export type AppDeleteJobData = { teamId: string; appId: string; + /** Distinguishes task roots, internal Flow steps, and pre-Flow jobs. */ + jobType?: 'task' | 'step' | 'root' | 'app'; + taskId?: string; }; const appDeleteQueueOptions = { @@ -20,6 +23,22 @@ const appDeleteQueueOptions = { } }; +const appDeleteFlowJobOptions = appDeleteQueueOptions.defaultJobOptions; + +/** Apply the App deletion queue retry and retention policy to every node in a Flow tree. */ +const applyFlowJobOptions = (flow: FlowJob): FlowJob => ({ + ...flow, + opts: { + ...appDeleteFlowJobOptions, + ...flow.opts + }, + ...(flow.children + ? { + children: flow.children.map((child) => applyFlowJobOptions(child)) + } + : {}) +}); + /** App 删除队列的业务合同和生命周期入口。 */ export class AppDeleteMQService { constructor(private readonly binding: BullMQBinding = bullMQ) {} @@ -40,19 +59,31 @@ export class AppDeleteMQService { }); } - /** 投递幂等的 App 删除任务,并延迟一秒让请求先完成。 */ + /** Add a root deletion job and delay it by one second for request completion. */ addJob(data: AppDeleteJobData) { const jobId = `${String(data.teamId)}-${String(data.appId)}`; return addOrRequeueFailedJob({ queue: this.getQueue(), name: 'delete_app', - data, + data: { ...data, jobType: 'root' }, opts: { jobId, delay: 1000 } }); } + + /** Returns the FlowProducer used to atomically submit deletion task Flows. */ + getFlowProducer() { + return this.binding.getFlowProducer(); + } + + /** Atomically submits task Flows; callers define step order through the dependency chain. */ + addFlows(flows: FlowJob[]): Promise { + if (flows.length === 0) return Promise.resolve([]); + + return this.getFlowProducer().addBulk(flows.map((flow) => applyFlowJobOptions(flow))); + } } export const appDeleteMQService = new AppDeleteMQService(); diff --git a/packages/dal/redis/bullmq/types.ts b/packages/dal/redis/bullmq/types.ts index 2f6b07b91d92..2e8ea13d5a82 100644 --- a/packages/dal/redis/bullmq/types.ts +++ b/packages/dal/redis/bullmq/types.ts @@ -1,6 +1,9 @@ import type { ConnectionOptions, + FlowJob, + FlowProducer, Job, + JobNode, JobSchedulerJson, Processor, Queue, @@ -42,7 +45,10 @@ export type WorkerListenerSnapshot = { export type { ConnectionOptions, + FlowJob, + FlowProducer, Job, + JobNode, JobSchedulerJson, Processor, Queue, diff --git a/packages/dal/test/redis/bullmq-binding.test.ts b/packages/dal/test/redis/bullmq-binding.test.ts index b7c06477ed50..10c4b216d9f0 100644 --- a/packages/dal/test/redis/bullmq-binding.test.ts +++ b/packages/dal/test/redis/bullmq-binding.test.ts @@ -4,6 +4,7 @@ vi.unmock('@fastgpt/dal/redis/bullmq'); const bullMQMocks = vi.hoisted(() => { const queue = { id: 'queue' }; + const flowProducer = { id: 'flow-producer' }; const worker = { id: 'worker' }; const redisRuntime = { id: 'redis-runtime', @@ -11,12 +12,14 @@ const bullMQMocks = vi.hoisted(() => { }; const runtime = { getQueue: vi.fn(() => queue), + getFlowProducer: vi.fn(() => flowProducer), getWorker: vi.fn(() => worker), getLogger: vi.fn(() => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() })) }; return { queue, + flowProducer, worker, redisRuntime, runtime, @@ -38,6 +41,7 @@ describe('DAL BullMQ binding', () => { beforeEach(async () => { vi.resetModules(); bullMQMocks.runtime.getQueue.mockClear(); + bullMQMocks.runtime.getFlowProducer.mockClear(); bullMQMocks.runtime.getWorker.mockClear(); bullMQMocks.getRedisBullMQRuntime.mockClear(); bullMQ = await import('@fastgpt/dal/redis/bullmq'); @@ -78,4 +82,9 @@ describe('DAL BullMQ binding', () => { }) ); }); + + it('delegates FlowProducer access to the DAL BullMQ Runtime', () => { + expect(bullMQ.bullMQ.getFlowProducer()).toBe(bullMQMocks.flowProducer); + expect(bullMQMocks.runtime.getFlowProducer).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/dal/test/redis/bullmq-services.test.ts b/packages/dal/test/redis/bullmq-services.test.ts index c9b89c06f10c..1ef007eeccc0 100644 --- a/packages/dal/test/redis/bullmq-services.test.ts +++ b/packages/dal/test/redis/bullmq-services.test.ts @@ -25,11 +25,13 @@ describe('BullMQ business services', () => { it('allows queue binding injection while keeping queue contracts in the service class', async () => { const queue = { add: vi.fn().mockResolvedValue({ id: 'job-1' }), + addBulk: vi.fn().mockResolvedValue([{ id: 'job-1' }]), getJob: vi.fn().mockResolvedValue(null) }; const binding = { getQueue: vi.fn(() => queue), getWorker: vi.fn(), + getFlowProducer: vi.fn(), getLogger: vi.fn(() => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() })) } as unknown as BullMQBinding; const service = new AppDeleteMQService(binding); @@ -47,13 +49,67 @@ describe('BullMQ business services', () => { removeOnFail: { age: 30 * 24 * 60 * 60 } } }); - expect(queue.add).toHaveBeenCalledWith('delete_app', data, { - jobId: 'team-1-app-1', - delay: 1000 - }); + expect(queue.add).toHaveBeenCalledWith( + 'delete_app', + { ...data, jobType: 'root' }, + { + jobId: 'team-1-app-1', + delay: 1000 + } + ); expect(queue.getJob).toHaveBeenCalledWith('team-1-app-1'); }); + it('adds one task Flow with retry options on every node', async () => { + const flowProducer = { + addBulk: vi.fn().mockResolvedValue([{ job: { id: 'task-1' } }]) + }; + const binding = { + getQueue: vi.fn(), + getWorker: vi.fn(), + getFlowProducer: vi.fn(() => flowProducer), + getLogger: vi.fn(() => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() })) + } as unknown as BullMQBinding; + const service = new AppDeleteMQService(binding); + + await expect( + service.addFlows([ + { + name: 'delete_app_task', + queueName: 'appDelete', + data: { jobType: 'task' }, + opts: { jobId: 'task-1' }, + children: [ + { + name: 'delete_app_step', + queueName: 'appDelete', + data: { jobType: 'step' }, + opts: { jobId: 'task-1:step:app-1' } + } + ] + } + ]) + ).resolves.toEqual([{ job: { id: 'task-1' } }]); + + expect(flowProducer.addBulk).toHaveBeenCalledWith([ + expect.objectContaining({ + opts: expect.objectContaining({ + jobId: 'task-1', + attempts: 10, + removeOnComplete: true + }), + children: [ + expect.objectContaining({ + opts: expect.objectContaining({ + jobId: 'task-1:step:app-1', + attempts: 10 + }) + }) + ] + }) + ]); + }); + it('uses failed-job recovery for Dataset deletion jobs', async () => { const queue = { add: vi.fn().mockResolvedValue({ id: 'job-2' }), diff --git a/packages/dal/test/redis/bullmq.test.ts b/packages/dal/test/redis/bullmq.test.ts index 2741f59fc246..dfea63790549 100644 --- a/packages/dal/test/redis/bullmq.test.ts +++ b/packages/dal/test/redis/bullmq.test.ts @@ -10,9 +10,11 @@ import { RedisRuntime, type RedisClient } from '../../redis/runtime'; const bullMQMocks = vi.hoisted(() => ({ queues: [] as Array<{ name: string; options: Record }>, + flowProducers: [] as Array<{ options: Record }>, workers: [] as Array<{ name: string; options: Record }>, closeOrder: [] as string[], queueConstructorFailures: 0, + flowProducerConstructorFailures: 0, workerConstructorFailures: 0, onWorkerClose: undefined as (() => void) | undefined })); @@ -71,7 +73,27 @@ vi.mock('bullmq', async () => { } } + class MockFlowProducer extends EventEmitter { + readonly close = vi.fn(async () => { + bullMQMocks.closeOrder.push('flow-producer'); + }); + readonly disconnect = vi.fn(async () => undefined); + readonly connection = { + disconnect: vi.fn(async () => undefined) + }; + + constructor(readonly options: Record) { + super(); + if (bullMQMocks.flowProducerConstructorFailures > 0) { + bullMQMocks.flowProducerConstructorFailures -= 1; + throw new Error('flow producer constructor failed'); + } + bullMQMocks.flowProducers.push(this); + } + } + return { + FlowProducer: MockFlowProducer, Queue: MockQueue, UnrecoverableError: class UnrecoverableError extends Error {}, Worker: MockWorker @@ -102,9 +124,11 @@ const createBullMQRuntime = (options?: Partial) => { describe('RedisBullMQRuntime', () => { beforeEach(() => { bullMQMocks.queues.length = 0; + bullMQMocks.flowProducers.length = 0; bullMQMocks.workers.length = 0; bullMQMocks.closeOrder.length = 0; bullMQMocks.queueConstructorFailures = 0; + bullMQMocks.flowProducerConstructorFailures = 0; bullMQMocks.workerConstructorFailures = 0; bullMQMocks.onWorkerClose = undefined; }); @@ -122,6 +146,16 @@ describe('RedisBullMQRuntime', () => { expect(bullMQMocks.workers[0]?.options).toMatchObject({ concurrency: 2 }); }); + it('creates and reuses one FlowProducer per Runtime', () => { + const { redisRuntime, runtime } = createBullMQRuntime(); + + const flowProducer = runtime.getFlowProducer(); + + expect(runtime.getFlowProducer()).toBe(flowProducer); + expect(bullMQMocks.flowProducers).toHaveLength(1); + expect(redisRuntime.createQueueConnection).toHaveBeenCalledTimes(1); + }); + it('releases the Runtime connection when a Queue or Worker constructor fails', async () => { const { redisRuntime, runtime } = createBullMQRuntime(); bullMQMocks.queueConstructorFailures = 1; @@ -129,15 +163,19 @@ describe('RedisBullMQRuntime', () => { bullMQMocks.workerConstructorFailures = 1; expect(() => runtime.getWorker('worker-failure', vi.fn())).toThrow('worker constructor failed'); + + bullMQMocks.flowProducerConstructorFailures = 1; + expect(() => runtime.getFlowProducer()).toThrow('flow producer constructor failed'); await Promise.resolve(); - expect(redisRuntime.releaseConnection).toHaveBeenCalledTimes(2); + expect(redisRuntime.releaseConnection).toHaveBeenCalledTimes(3); }); it('registers the Redis hook and closes workers before queues', async () => { const { redisRuntime, runtime } = createBullMQRuntime(); const processor = vi.fn(); const queue = runtime.getQueue('datasetSync'); + runtime.getFlowProducer(); const worker = runtime.getWorker('datasetSync', processor); const queueError = vi.fn(); queue.on('error', queueError); @@ -150,7 +188,11 @@ describe('RedisBullMQRuntime', () => { expect(runtime.close()).toBe(closePromise); await closePromise; - expect(bullMQMocks.closeOrder).toEqual(['worker:datasetSync:true', 'queue:datasetSync']); + expect(bullMQMocks.closeOrder).toEqual([ + 'worker:datasetSync:true', + 'flow-producer', + 'queue:datasetSync' + ]); expect(worker.listenerCount('error')).toBe(0); expect(queue.listenerCount('error')).toBe(1); expect(queue.listeners('error')).toContain(queueError); @@ -181,7 +223,8 @@ describe('RedisBullMQRuntime', () => { const redisRuntime = new RedisRuntime({ redisUrl: 'redis://localhost', clientFactory: (options) => { - const client = new RuntimeRedisClient(options, clients.length === 0 ? 'queue' : 'worker'); + const role = clients.length === 0 ? 'queue' : clients.length === 1 ? 'flow' : 'worker'; + const client = new RuntimeRedisClient(options, role); clients.push(client); return client as unknown as RedisClient; } @@ -189,14 +232,17 @@ describe('RedisBullMQRuntime', () => { const bullMQRuntime = new RedisBullMQRuntime({ redisRuntime }); bullMQRuntime.getQueue('runtime-order'); + bullMQRuntime.getFlowProducer(); bullMQRuntime.getWorker('runtime-order', vi.fn()); await redisRuntime.close(); expect(bullMQMocks.closeOrder).toEqual([ 'worker:runtime-order:true', + 'flow-producer', 'queue:runtime-order', 'redis:queue', + 'redis:flow', 'redis:worker' ]); expect(bullMQRuntime.getState()).toBe('closed'); diff --git a/packages/service/core/app/delete/index.ts b/packages/service/core/app/delete/index.ts index 05f4365e78bd..aef8e729fcec 100644 --- a/packages/service/core/app/delete/index.ts +++ b/packages/service/core/app/delete/index.ts @@ -1,12 +1,290 @@ import { appDeleteProcessor } from './processor'; -import { appDeleteMQService, type AppDeleteJobData } from '@fastgpt/dal/redis/bullmq'; +import { + appDeleteMQService, + QueueNames, + type AppDeleteJobData, + type FlowJob, + type Job +} from '@fastgpt/dal/redis/bullmq'; +import { findAppAndAllChildren } from '../controller'; +import { MongoApp } from '../schema'; +import { batchRunSettled } from '@fastgpt/global/common/system/utils'; +import { getLogger, LogCategories } from '../../../common/logger'; +import { setCron } from '../../../common/system/cron'; export type { AppDeleteJobData } from '@fastgpt/dal/redis/bullmq'; -// 创建工作进程 +type AppDeleteTaskInput = { + teamId: string; + appId: string; +}; + +type MarkedApp = { + _id: unknown; + teamId: unknown; + parentId?: unknown; +}; + +const APP_DELETE_RESUME_BATCH_SIZE = 200; +const APP_DELETE_RESUME_CONCURRENCY = 5; +const APP_DELETE_FLOW_BULK_SIZE = 50; +const APP_DELETE_RESUME_CRON = '*/5 * * * *'; +const logger = getLogger(LogCategories.MODULE.APP.FOLDER); + +let recoveryCronRegistered = false; +let recoveryPromise: Promise | undefined; + +const getTaskId = ({ teamId, appId }: AppDeleteTaskInput) => + `app-delete-task-${String(teamId)}-${String(appId)}`; + +/** + * Build one task Flow as a linked list. BullMQ processes children before parents, so reversing + * the resource list makes descendants complete before their parents and keeps the task root last. + */ +export const buildAppDeleteFlow = ({ + teamId, + appId, + apps +}: AppDeleteTaskInput & { apps: Array<{ _id: unknown }> }): FlowJob => { + const taskId = getTaskId({ teamId, appId }); + const stepChain = apps + .slice() + .reverse() + .reduce((nextStep, app) => { + const resourceId = String(app._id); + const step: FlowJob = { + name: 'delete_app_step', + queueName: QueueNames.appDelete, + data: { + teamId, + appId: resourceId, + taskId, + jobType: 'step' + }, + opts: { + jobId: `${taskId}:step:${resourceId}`, + failParentOnFailure: true, + ...(nextStep ? {} : { delay: 1000 }) + } + }; + + if (nextStep) step.children = [nextStep]; + return step; + }, undefined); + + return { + name: 'delete_app_task', + queueName: QueueNames.appDelete, + data: { + teamId, + appId, + taskId, + jobType: 'task' + }, + opts: { + jobId: taskId + }, + ...(stepChain ? { children: [stepChain] } : {}) + }; +}; + +/** + * Remove only a terminal task Flow before rebuilding it. Active and waiting tasks are left alone, + * allowing multiple recovery callers to converge on the same stable task ID. + */ +const prepareTaskFlow = async (data: AppDeleteTaskInput) => { + const taskId = getTaskId(data); + const queue = appDeleteMQService.getQueue(); + const loadDeleteTaskFlow = async () => { + const apps = await findAppAndAllChildren({ + teamId: data.teamId, + appId: data.appId, + fields: '_id teamId parentId deleteTime' + }); + + const unmarkedApps = apps.filter((app) => !app.deleteTime); + if (unmarkedApps.length > 0) { + logger.warn('App delete safety check mismatch', { + markedCount: apps.length - unmarkedApps.length, + totalCount: apps.length, + unmarkedCount: unmarkedApps.length + }); + throw new Error('App delete safety check mismatch'); + } + + return { + flow: buildAppDeleteFlow({ ...data, apps }) + }; + }; + + const getExistingJob = async () => { + const job = await queue.getJob(taskId); + if (!job) return; + + const state = await job.getState(); + if (state !== 'unknown') return { job, state }; + + const latestJob = await queue.getJob(taskId); + if (!latestJob) return; + + const latestState = await latestJob.getState(); + if (latestState === 'unknown') { + throw new Error(`BullMQ app delete task is in an unknown state: ${taskId}`); + } + return { job: latestJob, state: latestState }; + }; + + const existing = await getExistingJob(); + if (existing) { + if ( + ['waiting-children', 'waiting', 'active', 'delayed', 'prioritized', 'paused'].includes( + existing.state + ) + ) { + return { job: existing.job }; + } + await existing.job.remove({ removeChildren: true }); + } + + return loadDeleteTaskFlow(data); +}; + +/** Submit task Flows in bounded atomic batches so recovery cannot create one resource job at a time. */ +const addAppDeleteTasks = async (tasks: AppDeleteTaskInput[]) => { + const flows: FlowJob[] = []; + const jobs: Job[] = []; + for (const task of tasks) { + const prepared = await prepareTaskFlow(task); + if ('job' in prepared) { + jobs.push(prepared.job); + } else { + flows.push(prepared.flow); + } + } + + for (let index = 0; index < flows.length; index += APP_DELETE_FLOW_BULK_SIZE) { + const nodes = await appDeleteMQService.addFlows( + flows.slice(index, index + APP_DELETE_FLOW_BULK_SIZE) + ); + jobs.push(...nodes.map((node) => node.job)); + } + return jobs; +}; + +/** Add one deletion task Flow. The public contract remains root App based, not resource based. */ +export const addAppDeleteJob = async (data: AppDeleteTaskInput) => { + const [job] = await addAppDeleteTasks([data]); + return job; +}; + +/** Add multiple root deletion tasks through BullMQ FlowProducer.addBulk. */ +export const addAppDeleteJobs = (data: AppDeleteTaskInput[]) => addAppDeleteTasks(data); + +/** + * Initialize the App deletion worker and asynchronously resume soft-deleted task roots whose Flow + * was not created or reached a terminal failure. Recovery does not block worker creation. + */ export const initAppDeleteWorker = () => { - return appDeleteMQService.getWorker(appDeleteProcessor); + const worker = appDeleteMQService.getWorker(appDeleteProcessor); + + registerAppDeleteRecoveryCron(); + + resumeMarkedAppDeleteJobs().catch((error) => { + logger.error('Failed to resume marked app delete jobs', { error }); + }); + + return worker; +}; + +/** Keep recovery alive after transient Redis or Mongo failures during startup. */ +const registerAppDeleteRecoveryCron = () => { + if (recoveryCronRegistered) return; + + recoveryCronRegistered = true; + setCron(APP_DELETE_RESUME_CRON, () => { + resumeMarkedAppDeleteJobs().catch((error) => { + logger.error('Failed to resume marked app delete jobs', { error }); + }); + }); }; -// 添加删除任务 -export const addAppDeleteJob = (data: AppDeleteJobData) => appDeleteMQService.addJob(data); +/** Scan only marked tree roots and rebuild missing or failed task Flows with bounded concurrency. */ +async function resumeMarkedAppDeleteJobsInternal(): Promise { + const cursor = MongoApp.find( + { + deleteTime: { + $exists: true, + $ne: null + } + }, + { + _id: 1, + teamId: 1, + parentId: 1 + } + ) + .lean() + .cursor({ batchSize: APP_DELETE_RESUME_BATCH_SIZE }); + + let totalMarked = 0; + let rootCount = 0; + let resumedCount = 0; + let failedCount = 0; + const markedApps: MarkedApp[] = []; + + for await (const app of cursor) { + totalMarked += 1; + markedApps.push(app); + } + + const getMarkedTaskRoots = (apps: MarkedApp[]) => { + const markedIds = new Set(apps.map((app) => `${String(app.teamId)}:${String(app._id)}`)); + return apps.filter((app) => { + if (!app.parentId) return true; + return !markedIds.has(`${String(app.teamId)}:${String(app.parentId)}`); + }); + }; + + const roots = getMarkedTaskRoots(markedApps); + rootCount = roots.length; + const tasks = roots.map((root) => ({ + teamId: String(root.teamId), + appId: String(root._id) + })); + + const taskBatches = []; + for (let index = 0; index < tasks.length; index += APP_DELETE_FLOW_BULK_SIZE) { + taskBatches.push(tasks.slice(index, index + APP_DELETE_FLOW_BULK_SIZE)); + } + + const results = await batchRunSettled( + taskBatches, + (batch) => addAppDeleteJobs(batch), + APP_DELETE_RESUME_CONCURRENCY + ); + + for (const [index, result] of results.entries()) { + if (result.success) { + resumedCount += taskBatches[index].length; + } else { + failedCount += taskBatches[index].length; + } + } + + logger.info('Marked app delete tasks resumed', { + totalMarked, + rootCount, + resumedCount, + failedCount + }); +} + +/** Process-local overlap protection; distributed callers converge on stable Flow task IDs. */ +export function resumeMarkedAppDeleteJobs(): Promise { + if (recoveryPromise) return recoveryPromise; + + recoveryPromise = resumeMarkedAppDeleteJobsInternal().finally(() => { + recoveryPromise = undefined; + }); + return recoveryPromise; +} diff --git a/packages/service/core/app/delete/processor.ts b/packages/service/core/app/delete/processor.ts index c19cecfbe1b6..84cc13220a1f 100644 --- a/packages/service/core/app/delete/processor.ts +++ b/packages/service/core/app/delete/processor.ts @@ -1,80 +1,81 @@ import type { Processor } from '@fastgpt/dal/redis/bullmq'; import type { AppDeleteJobData } from './index'; -import { findAppAndAllChildren, deleteAppDataProcessor } from '../controller'; -import { batchRun } from '@fastgpt/global/common/system/utils'; -import type { AppSchemaType } from '@fastgpt/global/core/app/type'; +import { deleteAppDataProcessor } from '../controller'; import { MongoApp } from '../schema'; import { getLogger, LogCategories } from '../../../common/logger'; +import { addAppDeleteJob } from './index'; const logger = getLogger(LogCategories.MODULE.APP.FOLDER); -const deleteApps = async ({ teamId, apps }: { teamId: string; apps: AppSchemaType[] }) => { - // 每个 App 使用独立 Source Lease;任务内串行用于限制外部资源清理压力。 - const results = await batchRun( - apps, - async (app) => { - await deleteAppDataProcessor({ app, teamId }); +/** + * Clean one app using only the fields required by the existing deletion processor. + * Missing apps are treated as an idempotent completion; live apps fail the safety check. + */ +const deleteSingleApp = async ({ teamId, appId }: { teamId: string; appId: string }) => { + const startTime = Date.now(); + const app = await MongoApp.findOne( + { + _id: appId, + teamId }, - 1 - ); + '_id teamId type avatar deleteTime' + ).lean(); + + if (!app) { + logger.warn('App not found for deletion', { teamId, appId }); + return; + } - return results.flat(); + // Recheck the soft-delete marker so stale jobs cannot remove a live app. + if (!app.deleteTime) { + logger.warn('App delete safety check mismatch', { + teamId, + appId, + markedCount: 0, + totalCount: 1 + }); + throw new Error('App delete safety check mismatch'); + } + + await deleteAppDataProcessor({ app, teamId }); + + logger.info('App delete completed', { + teamId, + appId, + durationMs: Date.now() - startTime + }); }; export const appDeleteProcessor: Processor = async (job) => { - const { teamId, appId } = job.data; + const { teamId, appId, jobType = 'root' } = job.data; const startTime = Date.now(); logger.info('App delete started', { teamId, appId }); try { - // 1. 查找应用及其所有子应用 - const apps = await findAppAndAllChildren({ - teamId, - appId - }); - - if (!apps || apps.length === 0) { - logger.warn('App not found for deletion', { teamId, appId }); - return; - } - - // 2. 安全检查:确保所有要删除的应用都已标记为 deleteTime - const markedForDelete = await MongoApp.find( - { - _id: { $in: apps.map((app) => app._id) }, + if (jobType === 'task') { + logger.info('App delete task completed', { teamId, - deleteTime: { $ne: null } - }, - { _id: 1 } - ).lean(); - - if (markedForDelete.length !== apps.length) { - logger.warn('App delete safety check mismatch', { - markedCount: markedForDelete.length, - totalCount: apps.length, - markedAppIds: markedForDelete.map((app) => app._id), - totalAppIds: apps.map((app) => app._id) + appId, + taskId: job.data.taskId, + durationMs: Date.now() - startTime }); - throw new Error('App delete safety check mismatch'); + return; } - const childrenLen = apps.length - 1; - const appIds = apps.map((app) => app._id); + if (jobType === 'step' || jobType === 'app') { + await deleteSingleApp({ teamId, appId }); + return; + } - // 3. 执行真正的删除操作(只删除已经标记为 deleteTime 的数据) - await deleteApps({ - teamId, - apps - }); + // Legacy root jobs become a bridge to the task Flow. New requests enqueue the Flow directly. + await addAppDeleteJob({ teamId, appId }); logger.info('App delete completed', { teamId, appId, - childCount: childrenLen, durationMs: Date.now() - startTime, - totalApps: appIds.length, - appIds + legacy: true }); } catch (error: any) { logger.error('App delete failed', { teamId, appId, error }); diff --git a/packages/service/support/user/team/delete/utils.ts b/packages/service/support/user/team/delete/utils.ts index 4be050b174c3..e875109df881 100644 --- a/packages/service/support/user/team/delete/utils.ts +++ b/packages/service/support/user/team/delete/utils.ts @@ -1,9 +1,9 @@ import { MongoApp } from '../../../../core/app/schema'; import { deleteAppsImmediate } from '../../../../core/app/controller'; -import { addAppDeleteJob } from '../../../../core/app/delete'; +import { addAppDeleteJobs } from '../../../../core/app/delete'; export const onDelAllApp = async (teamId: string) => { - // 正常只投递根应用;如果历史数据留下孤立子应用,则把孤立应用作为自己的根补偿投递。 + // Normally only roots are submitted; orphaned children become compensating roots. const apps = await MongoApp.find( { teamId @@ -32,11 +32,11 @@ export const onDelAllApp = async (teamId: string) => { } ); - // 添加到删除队列 - for (const app of deleteRootApps) { - await addAppDeleteJob({ + // Add all root task Flows atomically in bounded batches. + await addAppDeleteJobs( + deleteRootApps.map((app) => ({ teamId, appId: String(app._id) - }); - } + })) + ); }; diff --git a/packages/service/test/core/app/delete/processor.test.ts b/packages/service/test/core/app/delete/processor.test.ts new file mode 100644 index 000000000000..46767e77a80e --- /dev/null +++ b/packages/service/test/core/app/delete/processor.test.ts @@ -0,0 +1,279 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Job } from '@fastgpt/dal/redis/bullmq'; +import type { AppDeleteJobData } from '@fastgpt/dal/redis/bullmq'; + +const mocks = vi.hoisted(() => ({ + findAppAndAllChildren: vi.fn(), + deleteAppDataProcessor: vi.fn(), + findOne: vi.fn(), + find: vi.fn(), + addFlows: vi.fn(), + getQueue: vi.fn(), + getWorker: vi.fn(), + setCron: vi.fn(), + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn() + } +})); + +vi.mock('@fastgpt/service/core/app/controller', () => ({ + findAppAndAllChildren: mocks.findAppAndAllChildren, + deleteAppDataProcessor: mocks.deleteAppDataProcessor +})); + +vi.mock('@fastgpt/service/core/app/schema', () => ({ + MongoApp: { + findOne: mocks.findOne, + find: mocks.find + } +})); + +vi.mock('@fastgpt/service/common/logger', () => ({ + getLogger: () => mocks.logger, + LogCategories: { MODULE: { APP: { FOLDER: 'app.folder' } } } +})); + +vi.mock('@fastgpt/service/common/system/cron', () => ({ + setCron: mocks.setCron +})); + +vi.mock('@fastgpt/dal/redis/bullmq', () => ({ + appDeleteMQService: { + addFlows: mocks.addFlows, + getQueue: mocks.getQueue, + getWorker: mocks.getWorker + }, + QueueNames: { appDelete: 'appDelete' } +})); + +import { appDeleteProcessor } from '../../../../core/app/delete/processor'; +import { + addAppDeleteJob, + initAppDeleteWorker, + resumeMarkedAppDeleteJobs +} from '../../../../core/app/delete'; + +const createJob = (id: string, data: AppDeleteJobData) => + ({ id, data }) as unknown as Job; + +describe('appDeleteProcessor', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.findAppAndAllChildren.mockResolvedValue([ + { _id: 'root', teamId: 'team-1', parentId: null, deleteTime: new Date() }, + { _id: 'child-1', teamId: 'team-1', parentId: 'root', deleteTime: new Date() }, + { _id: 'child-2', teamId: 'team-1', parentId: 'root', deleteTime: new Date() } + ]); + mocks.addFlows.mockResolvedValue([{ job: { id: 'app-delete-task' } }]); + mocks.getQueue.mockReturnValue({ getJob: vi.fn().mockResolvedValue(null) }); + mocks.findOne.mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + _id: 'app-1', + teamId: 'team-1', + type: 'simple', + avatar: '', + deleteTime: new Date() + }) + }); + mocks.deleteAppDataProcessor.mockResolvedValue(undefined); + }); + + it('bridges a legacy root deletion job into one task Flow', async () => { + await appDeleteProcessor( + createJob('root-job', { teamId: 'team-1', appId: 'root', jobType: 'root' }) + ); + + expect(mocks.findAppAndAllChildren).toHaveBeenCalledWith({ + teamId: 'team-1', + appId: 'root', + fields: '_id teamId parentId deleteTime' + }); + expect(mocks.addFlows).toHaveBeenCalledWith([ + expect.objectContaining({ + name: 'delete_app_task', + data: expect.objectContaining({ jobType: 'task' }), + children: [ + expect.objectContaining({ + data: expect.objectContaining({ appId: 'root', jobType: 'step' }), + children: [ + expect.objectContaining({ + data: expect.objectContaining({ appId: 'child-1', jobType: 'step' }), + children: [ + expect.objectContaining({ + data: expect.objectContaining({ appId: 'child-2', jobType: 'step' }) + }) + ] + }) + ] + }) + ] + }) + ]); + expect(mocks.findOne).not.toHaveBeenCalled(); + expect(mocks.deleteAppDataProcessor).not.toHaveBeenCalled(); + }); + + it('treats a historical job without jobType as a root job', async () => { + await appDeleteProcessor(createJob('legacy-root-job', { teamId: 'team-1', appId: 'root' })); + + expect(mocks.addFlows).toHaveBeenCalledTimes(1); + expect(mocks.deleteAppDataProcessor).not.toHaveBeenCalled(); + }); + + it('completes a task root after all child steps have finished', async () => { + await appDeleteProcessor( + createJob('task-job', { + teamId: 'team-1', + appId: 'root', + taskId: 'task-1', + jobType: 'task' + }) + ); + + expect(mocks.findAppAndAllChildren).not.toHaveBeenCalled(); + expect(mocks.findOne).not.toHaveBeenCalled(); + expect(mocks.deleteAppDataProcessor).not.toHaveBeenCalled(); + }); + + it('does not log all subtree IDs when the root safety check fails', async () => { + mocks.findAppAndAllChildren.mockResolvedValue([ + { _id: 'root', teamId: 'team-1', parentId: null, deleteTime: null }, + { _id: 'child-1', teamId: 'team-1', parentId: 'root', deleteTime: null } + ]); + + await expect( + appDeleteProcessor( + createJob('unsafe-root-job', { teamId: 'team-1', appId: 'root', jobType: 'root' }) + ) + ).rejects.toThrow('App delete safety check mismatch'); + + expect(mocks.logger.warn).toHaveBeenCalledWith('App delete safety check mismatch', { + markedCount: 0, + totalCount: 2, + unmarkedCount: 2 + }); + expect(mocks.logger.warn.mock.calls[0][1]).not.toHaveProperty('unmarkedAppIds'); + }); + + it('cleans exactly one marked app for a single-app job', async () => { + await appDeleteProcessor( + createJob('app-job', { teamId: 'team-1', appId: 'app-1', jobType: 'app' }) + ); + + expect(mocks.findOne).toHaveBeenCalledWith( + { _id: 'app-1', teamId: 'team-1' }, + '_id teamId type avatar deleteTime' + ); + expect(mocks.deleteAppDataProcessor).toHaveBeenCalledWith({ + app: expect.objectContaining({ _id: 'app-1' }), + teamId: 'team-1' + }); + expect(mocks.findAppAndAllChildren).not.toHaveBeenCalled(); + }); + + it('rejects an unmarked single-app job before deleting external data', async () => { + mocks.findOne.mockReturnValue({ + lean: vi.fn().mockResolvedValue({ + _id: 'app-1', + teamId: 'team-1', + type: 'simple', + avatar: '', + deleteTime: null + }) + }); + + await expect( + appDeleteProcessor( + createJob('unmarked-app-job', { teamId: 'team-1', appId: 'app-1', jobType: 'app' }) + ) + ).rejects.toThrow('App delete safety check mismatch'); + expect(mocks.deleteAppDataProcessor).not.toHaveBeenCalled(); + }); + + it('resumes marked apps through a cursor with bounded batch processing', async () => { + const cursor = (async function* () { + yield { _id: 'app-1', teamId: 'team-1', parentId: null }; + yield { _id: 'child-1', teamId: 'team-1', parentId: 'app-1' }; + yield { _id: 'app-2', teamId: 'team-2', parentId: null }; + })(); + const lean = vi.fn().mockReturnValue({ + cursor: vi.fn().mockReturnValue(cursor) + }); + mocks.find.mockReturnValue({ lean }); + + await resumeMarkedAppDeleteJobs(); + + expect(mocks.find).toHaveBeenCalledWith( + { deleteTime: { $exists: true, $ne: null } }, + { _id: 1, teamId: 1, parentId: 1 } + ); + expect(mocks.addFlows).toHaveBeenCalledTimes(1); + expect(mocks.addFlows.mock.calls[0]?.[0]).toHaveLength(2); + expect(mocks.logger.info).toHaveBeenCalledWith('Marked app delete tasks resumed', { + totalMarked: 3, + rootCount: 2, + resumedCount: 2, + failedCount: 0 + }); + }); + + it('reuses an active task instead of building another Flow', async () => { + const existingJob = { + getState: vi.fn().mockResolvedValue('waiting-children') + }; + const getJob = vi.fn().mockResolvedValue(existingJob); + mocks.getQueue.mockReturnValue({ getJob }); + + await expect(addAppDeleteJob({ teamId: 'team-1', appId: 'root' })).resolves.toBe(existingJob); + + expect(getJob).toHaveBeenCalledWith('app-delete-task-team-1-root'); + expect(mocks.findAppAndAllChildren).not.toHaveBeenCalled(); + expect(mocks.addFlows).not.toHaveBeenCalled(); + }); + + it('removes a failed task Flow before rebuilding it', async () => { + const existingJob = { + getState: vi.fn().mockResolvedValue('failed'), + remove: vi.fn().mockResolvedValue(undefined) + }; + mocks.getQueue.mockReturnValue({ getJob: vi.fn().mockResolvedValue(existingJob) }); + + await addAppDeleteJob({ teamId: 'team-1', appId: 'root' }); + + expect(existingJob.remove).toHaveBeenCalledWith({ removeChildren: true }); + expect(mocks.findAppAndAllChildren).toHaveBeenCalledTimes(1); + expect(mocks.addFlows).toHaveBeenCalledTimes(1); + }); + + it('does not rebuild a task while its state is still unknown', async () => { + const existingJob = { + getState: vi.fn().mockResolvedValue('unknown') + }; + const getJob = vi.fn().mockResolvedValue(existingJob); + mocks.getQueue.mockReturnValue({ getJob }); + + await expect(addAppDeleteJob({ teamId: 'team-1', appId: 'root' })).rejects.toThrow( + 'BullMQ app delete task is in an unknown state' + ); + + expect(getJob).toHaveBeenCalledTimes(2); + expect(mocks.findAppAndAllChildren).not.toHaveBeenCalled(); + expect(mocks.addFlows).not.toHaveBeenCalled(); + }); + + it('registers periodic recovery when the worker starts', async () => { + const cursor = (async function* () {})(); + mocks.find.mockReturnValue({ + lean: vi.fn().mockReturnValue({ cursor: vi.fn().mockReturnValue(cursor) }) + }); + const worker = { name: 'app-delete-worker' }; + mocks.getWorker.mockReturnValue(worker); + + expect(initAppDeleteWorker()).toBe(worker); + expect(mocks.setCron).toHaveBeenCalledWith('*/5 * * * *', expect.any(Function)); + + await resumeMarkedAppDeleteJobs(); + }); +}); diff --git a/projects/app/src/pages/api/core/app/del.ts b/projects/app/src/pages/api/core/app/del.ts index 1ba2dae351fc..8c158a3f1e48 100644 --- a/projects/app/src/pages/api/core/app/del.ts +++ b/projects/app/src/pages/api/core/app/del.ts @@ -54,14 +54,11 @@ async function handler(req: NextApiRequest): Promise { teamId, appIds: deleteAppsList.map((app) => app._id) }); - - // Add to delete queue for async cleanup - await addAppDeleteJob({ - teamId, - appId - }); }); + // Add the task Flow after the soft-delete transaction commits. + await addAppDeleteJob({ teamId, appId }); + (async () => { addAuditLog({ tmbId, diff --git a/projects/app/test/api/core/app/delete.test.ts b/projects/app/test/api/core/app/delete.test.ts index c9b6434db4f8..e5af0b09a55a 100644 --- a/projects/app/test/api/core/app/delete.test.ts +++ b/projects/app/test/api/core/app/delete.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it, beforeEach, vi, afterEach } from 'vitest'; -import type { AppDeleteJobData } from '@fastgpt/service/core/app/delete'; -import { addAppDeleteJob } from '@fastgpt/service/core/app/delete'; +import { buildAppDeleteFlow } from '@fastgpt/service/core/app/delete'; import { appDeleteProcessor } from '@fastgpt/service/core/app/delete/processor'; import handler from '@/pages/api/core/app/del'; import { MongoApp } from '@fastgpt/service/core/app/schema'; @@ -34,23 +33,19 @@ vi.mock('@fastgpt/dal/redis/bullmq', () => { return { bullMQ, appDeleteMQService: { - addJob: (data: AppDeleteJobData) => - bullMQ - .getQueue('appDelete', { - defaultJobOptions: { - attempts: 10, - backoff: { - type: 'exponential', - delay: 5000 - }, - removeOnComplete: true, - removeOnFail: { age: 30 * 24 * 60 * 60 } - } - }) - .add('delete_app', data, { - jobId: `${data.teamId}-${data.appId}`, - delay: 1000 - }) + getQueue: () => + bullMQ.getQueue('appDelete', { + defaultJobOptions: { + attempts: 10, + backoff: { + type: 'exponential', + delay: 5000 + }, + removeOnComplete: true, + removeOnFail: { age: 30 * 24 * 60 * 60 } + } + }), + addFlows: vi.fn().mockResolvedValue([{ job: { id: 'app-delete-task' } }]) }, QueueNames: { appDelete: 'appDelete' @@ -70,9 +65,10 @@ vi.mock('@fastgpt/service/common/file/image/controller', () => ({ })); // Import mocked modules for type access -import { bullMQ, QueueNames } from '@fastgpt/dal/redis/bullmq'; +import { bullMQ, QueueNames, appDeleteMQService } from '@fastgpt/dal/redis/bullmq'; const mockGetQueue = vi.mocked(bullMQ.getQueue); +const mockAddFlows = vi.mocked(appDeleteMQService.addFlows); describe('App Delete Queue', () => { beforeEach(() => { @@ -83,60 +79,32 @@ describe('App Delete Queue', () => { vi.restoreAllMocks(); }); - describe('addAppDeleteJob', () => { - it('should add job to queue with correct parameters', async () => { - const mockQueue = { - add: vi.fn().mockResolvedValue({ id: 'job-123' }) - }; - mockGetQueue.mockReturnValue(mockQueue as any); - - const jobData: AppDeleteJobData = { - teamId: 'team-123', - appId: 'app-123' - }; - - const result = await addAppDeleteJob(jobData); - - expect(mockGetQueue).toHaveBeenCalledWith(QueueNames.appDelete, { - defaultJobOptions: { - attempts: 10, - backoff: { - type: 'exponential', - delay: 5000 - }, - removeOnComplete: true, - removeOnFail: { age: 30 * 24 * 60 * 60 } - } + describe('buildAppDeleteFlow', () => { + it('builds a task flow with descendants before parents', () => { + const flow = buildAppDeleteFlow({ + teamId: 'team-1', + appId: 'root', + apps: [{ _id: 'root' }, { _id: 'child' }, { _id: 'grandchild' }] }); - expect(mockQueue.add).toHaveBeenCalledWith('delete_app', jobData, { - jobId: 'team-123-app-123', - delay: 1000 + expect(flow).toMatchObject({ + name: 'delete_app_task', + queueName: QueueNames.appDelete, + data: { teamId: 'team-1', appId: 'root', jobType: 'task' }, + opts: { jobId: 'app-delete-task-team-1-root' }, + children: [ + { + opts: { failParentOnFailure: true }, + data: { appId: 'root', jobType: 'step' }, + children: [ + { + data: { appId: 'child', jobType: 'step' }, + children: [{ data: { appId: 'grandchild', jobType: 'step' } }] + } + ] + } + ] }); - - expect(result).toEqual({ id: 'job-123' }); - }); - - it('should use correct jobId format for preventing duplicates', async () => { - const mockQueue = { - add: vi.fn().mockResolvedValue({ id: 'job-456' }) - }; - mockGetQueue.mockReturnValue(mockQueue as any); - - const jobData: AppDeleteJobData = { - teamId: 'team-xyz', - appId: 'app-abc' - }; - - await addAppDeleteJob(jobData); - - expect(mockQueue.add).toHaveBeenCalledWith( - 'delete_app', - jobData, - expect.objectContaining({ - jobId: 'team-xyz-app-abc' - }) - ); }); }); }); @@ -161,7 +129,7 @@ describe('App Delete API Integration', () => { // Mock the queue to avoid actual background deletion const mockQueue = { - add: vi.fn().mockResolvedValue({ id: 'job-123' }) + getJob: vi.fn().mockResolvedValue(null) }; mockGetQueue.mockReturnValue(mockQueue as any); @@ -179,17 +147,16 @@ describe('App Delete API Integration', () => { expect(deletedApp?.deleteTime).not.toBeNull(); // Verify queue job was added - expect(mockQueue.add).toHaveBeenCalledWith( - 'delete_app', - { - teamId: rootUser.teamId, - appId: String(testApp._id) - }, - { - jobId: `${rootUser.teamId}-${testApp._id}`, - delay: 1000 - } - ); + expect(mockAddFlows).toHaveBeenCalledWith([ + expect.objectContaining({ + name: 'delete_app_task', + data: expect.objectContaining({ + teamId: rootUser.teamId, + appId: String(testApp._id), + jobType: 'task' + }) + }) + ]); // Cleanup await MongoApp.deleteOne({ _id: testApp._id }); @@ -217,7 +184,7 @@ describe('App Delete API Integration', () => { // Mock the queue const mockQueue = { - add: vi.fn().mockResolvedValue({ id: 'job-folder' }) + getJob: vi.fn().mockResolvedValue(null) }; mockGetQueue.mockReturnValue(mockQueue as any); @@ -262,7 +229,7 @@ describe('App Delete API Integration', () => { }); const mockQueue = { - add: vi.fn().mockResolvedValue({ id: 'job-workflow-tool' }) + getJob: vi.fn().mockResolvedValue(null) }; mockGetQueue.mockReturnValue(mockQueue as any); @@ -345,7 +312,7 @@ describe('App Delete Data Cleanup Verification', () => { // 3. 执行删除处理器(模拟队列任务执行) const mockJob = { - data: { teamId, appId }, + data: { teamId, appId, jobType: 'app' }, id: 'test-job-id' }; @@ -387,13 +354,18 @@ describe('App Delete Data Cleanup Verification', () => { { deleteTime: new Date() } ); - // 执行删除(应该级联删除子应用) - const mockJob = { - data: { teamId, appId: String(parentApp._id) }, + // Each app is cleaned by its own queue job after the root job is split. + const parentJob = { + data: { teamId, appId: String(parentApp._id), jobType: 'app' as const }, id: 'test-nested-job' }; + const childJob = { + data: { teamId, appId: String(childApp._id), jobType: 'app' as const }, + id: 'test-nested-child-job' + }; - await appDeleteProcessor(mockJob); + await appDeleteProcessor(parentJob); + await appDeleteProcessor(childJob); // 验证父应用和子应用都被删除 expect(await MongoApp.countDocuments({ _id: parentApp._id })).toBe(0); @@ -409,7 +381,7 @@ describe('App Delete Data Cleanup Verification', () => { }); }); - it('should reject external cleanup when a nested app is not marked for deletion', async () => { + it('should reject external cleanup when the target app is not marked for deletion', async () => { const parentApp = await MongoApp.create({ name: 'Safety Parent App', teamId, @@ -425,11 +397,9 @@ describe('App Delete Data Cleanup Verification', () => { parentId: parentApp._id, modules: [] }); - await MongoApp.updateOne({ _id: parentApp._id }, { deleteTime: new Date() }); - await expect( appDeleteProcessor({ - data: { teamId, appId: String(parentApp._id) }, + data: { teamId, appId: String(parentApp._id), jobType: 'app' }, id: 'test-delete-safety-job' }) ).rejects.toThrow('App delete safety check mismatch'); @@ -457,7 +427,7 @@ describe('App Delete Data Cleanup Verification', () => { // 删除第一个应用 const mockJob1 = { - data: { teamId, appId }, + data: { teamId, appId, jobType: 'app' }, id: 'test-batch-job-1' }; @@ -498,7 +468,7 @@ describe('App Delete Data Cleanup Verification', () => { await MongoApp.updateOne({ _id: workflowToolAppId }, { deleteTime: new Date() }); await appDeleteProcessor({ - data: { teamId, appId: workflowToolAppId }, + data: { teamId, appId: workflowToolAppId, jobType: 'app' }, id: 'test-workflow-tool-cleanup-job' });