|
| 1 | +/** |
| 2 | + * What the Home Workspaces probe reads when a workspace has thousands of unanswered time pauses, the |
| 3 | + * shape of a schedule waiting on many runs. Each pause point is a JSON read the planner cannot index |
| 4 | + * or estimate, so a probe with no match reads every live pause of the workspaces it was asked about. |
| 5 | + * The bound this holds is that it reads no more than that: the plan is measured on real PostgreSQL |
| 6 | + * with the schema's own indexes, and a neighbor's pauses are never touched. Written as a plain join |
| 7 | + * under `LIMIT 1`, this reads the whole table instead. |
| 8 | + */ |
| 9 | +import { db } from '@sim/db' |
| 10 | +import { user, workflow, workspace } from '@sim/db/schema' |
| 11 | +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' |
| 12 | +import { generateId } from '@sim/utils/id' |
| 13 | +import { eq, inArray, sql } from 'drizzle-orm' |
| 14 | +import { afterAll, beforeAll, describe, expect, it } from 'vitest' |
| 15 | +import { readReviewAndLastRun } from '@/lib/organizations/home/workspace-activity' |
| 16 | + |
| 17 | +const LIVE_PAUSES = 5_000 |
| 18 | +const WORKFLOWS_PER_WORKSPACE = 20 |
| 19 | +const PAUSES_PER_WORKFLOW = LIVE_PAUSES / WORKFLOWS_PER_WORKSPACE |
| 20 | +/** |
| 21 | + * The rest of the deployment. In a table this workspace dominates a sequential scan is the right |
| 22 | + * plan, and it would say nothing about the table production has. |
| 23 | + */ |
| 24 | +const NEIGHBOR_WORKFLOWS = 100 |
| 25 | +const PAUSES_PER_NEIGHBOR_WORKFLOW = 1_000 |
| 26 | +/** Each loop of a plan node reports its rows rounded, so a total can only be held this closely. */ |
| 27 | +const ROUNDING_SLACK = 1.05 |
| 28 | +/** Far above what this costs, so only a change of plan or a pathological runner can reach it. */ |
| 29 | +const EXECUTION_CEILING_MS = 5_000 |
| 30 | + |
| 31 | +const ownerId = generateId() |
| 32 | +const timeOnly = generateId() |
| 33 | +const neighbor = generateId() |
| 34 | +const oneWaiting = generateId() |
| 35 | +const workspaces = [timeOnly, neighbor, oneWaiting] |
| 36 | + |
| 37 | +const timePausePoints = { |
| 38 | + context: { |
| 39 | + contextId: 'context', |
| 40 | + blockId: 'wait', |
| 41 | + response: { data: { resumeAt: '2030-01-01T00:00:00.000Z' } }, |
| 42 | + registeredAt: '2026-01-01T00:00:00.000Z', |
| 43 | + resumeStatus: 'paused', |
| 44 | + snapshotReady: true, |
| 45 | + pauseKind: 'time', |
| 46 | + resumeAt: '2030-01-01T00:00:00.000Z', |
| 47 | + }, |
| 48 | +} |
| 49 | +const humanPausePoints = { |
| 50 | + context: { ...timePausePoints.context, blockId: 'approval', pauseKind: 'human', resumeAt: null }, |
| 51 | +} |
| 52 | + |
| 53 | +const workflowIdsOf = (workspaceId: string) => |
| 54 | + Array.from( |
| 55 | + { length: workspaceId === neighbor ? NEIGHBOR_WORKFLOWS : WORKFLOWS_PER_WORKSPACE }, |
| 56 | + (_, index) => `${workspaceId}-${index}` |
| 57 | + ) |
| 58 | + |
| 59 | +/** Inserted in one statement: these are the fixture, not the behavior. */ |
| 60 | +function insertPauses(workflowId: string, count: number, pausePoints: object) { |
| 61 | + return db.execute(sql` |
| 62 | + insert into paused_executions ( |
| 63 | + id, workflow_id, execution_id, execution_snapshot, pause_points, total_pause_count |
| 64 | + ) |
| 65 | + select gen_random_uuid()::text, ${workflowId}, gen_random_uuid()::text, '{}'::jsonb, |
| 66 | + ${JSON.stringify(pausePoints)}::jsonb, 1 |
| 67 | + from generate_series(1, ${count}) |
| 68 | + `) |
| 69 | +} |
| 70 | + |
| 71 | +interface PlanNode { |
| 72 | + 'Node Type': string |
| 73 | + 'Relation Name'?: string |
| 74 | + 'Actual Rows': number |
| 75 | + 'Actual Loops': number |
| 76 | + 'Rows Removed by Filter'?: number |
| 77 | + Plans?: PlanNode[] |
| 78 | +} |
| 79 | + |
| 80 | +function collectNodes(node: PlanNode): PlanNode[] { |
| 81 | + return [node, ...(node.Plans ?? []).flatMap(collectNodes)] |
| 82 | +} |
| 83 | + |
| 84 | +/** The planner's real choice for the real query, with its parameters written in. */ |
| 85 | +async function measureProbe(workspaceIds: string[]) { |
| 86 | + const { sql: text, params } = readReviewAndLastRun(workspaceIds).toSQL() |
| 87 | + const inlined = text.replace(/\$(\d+)/g, (_, position: string) => { |
| 88 | + const value = params[Number(position) - 1] |
| 89 | + return typeof value === 'number' ? String(value) : `'${String(value).replaceAll("'", "''")}'` |
| 90 | + }) |
| 91 | + const rows = await db.execute(sql.raw(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${inlined}`)) |
| 92 | + const [plan] = (rows[0] as { 'QUERY PLAN': { Plan: PlanNode; 'Execution Time': number }[] })[ |
| 93 | + 'QUERY PLAN' |
| 94 | + ] |
| 95 | + if (!plan) throw new Error('EXPLAIN returned no plan') |
| 96 | + const pauseReads = collectNodes(plan.Plan).filter( |
| 97 | + (node) => node['Relation Name'] === 'paused_executions' |
| 98 | + ) |
| 99 | + return { |
| 100 | + executionMs: plan['Execution Time'], |
| 101 | + scans: pauseReads.map((node) => node['Node Type']), |
| 102 | + rowsRead: pauseReads.reduce( |
| 103 | + (total, node) => |
| 104 | + total + |
| 105 | + (node['Actual Rows'] + (node['Rows Removed by Filter'] ?? 0)) * node['Actual Loops'], |
| 106 | + 0 |
| 107 | + ), |
| 108 | + } |
| 109 | +} |
| 110 | + |
| 111 | +beforeAll(async () => { |
| 112 | + const now = new Date() |
| 113 | + await db.insert(user).values({ |
| 114 | + id: ownerId, |
| 115 | + name: 'Workspace activity fixture', |
| 116 | + email: `${ownerId}@workspace-activity.test`, |
| 117 | + emailVerified: true, |
| 118 | + createdAt: now, |
| 119 | + updatedAt: now, |
| 120 | + }) |
| 121 | + await insertWorkspaceFixture( |
| 122 | + db, |
| 123 | + workspaces.map((id) => ({ |
| 124 | + id, |
| 125 | + name: 'Workspace activity fixture', |
| 126 | + ownerId, |
| 127 | + billedAccountUserId: ownerId, |
| 128 | + })) |
| 129 | + ) |
| 130 | + await db.insert(workflow).values( |
| 131 | + workspaces.flatMap((workspaceId) => |
| 132 | + workflowIdsOf(workspaceId).map((id) => ({ |
| 133 | + id, |
| 134 | + workspaceId, |
| 135 | + name: id, |
| 136 | + userId: ownerId, |
| 137 | + lastSynced: now, |
| 138 | + createdAt: now, |
| 139 | + updatedAt: now, |
| 140 | + })) |
| 141 | + ) |
| 142 | + ) |
| 143 | + for (const workspaceId of workspaces) { |
| 144 | + for (const workflowId of workflowIdsOf(workspaceId)) { |
| 145 | + // Before its time pauses, so a probe that stops at the first match stops right away. |
| 146 | + if (workspaceId === oneWaiting) await insertPauses(workflowId, 1, humanPausePoints) |
| 147 | + await insertPauses( |
| 148 | + workflowId, |
| 149 | + workspaceId === neighbor ? PAUSES_PER_NEIGHBOR_WORKFLOW : PAUSES_PER_WORKFLOW, |
| 150 | + timePausePoints |
| 151 | + ) |
| 152 | + } |
| 153 | + } |
| 154 | + await db.execute(sql`analyze paused_executions`) |
| 155 | + await db.execute(sql`analyze workflow`) |
| 156 | +}, 120_000) |
| 157 | + |
| 158 | +afterAll(async () => { |
| 159 | + await deleteWorkspaceFixture(db, inArray(workspace.id, workspaces)) |
| 160 | + await db.delete(user).where(eq(user.id, ownerId)) |
| 161 | +}, 120_000) |
| 162 | + |
| 163 | +describe('the waiting probe under thousands of time pauses', () => { |
| 164 | + it('does not report a time pause as waiting on a person', async () => { |
| 165 | + const [row] = await readReviewAndLastRun([timeOnly]) |
| 166 | + expect(row?.waitingWorkflowName).toBeNull() |
| 167 | + }, 30_000) |
| 168 | + |
| 169 | + it('with no match reads this workspace’s pauses through an index, and nobody else’s', async () => { |
| 170 | + const { rowsRead, scans, executionMs } = await measureProbe([timeOnly]) |
| 171 | + expect(rowsRead).toBeGreaterThanOrEqual(LIVE_PAUSES / ROUNDING_SLACK) |
| 172 | + expect(rowsRead).toBeLessThanOrEqual(LIVE_PAUSES * ROUNDING_SLACK) |
| 173 | + expect(scans).not.toContain('Seq Scan') |
| 174 | + expect(executionMs).toBeLessThan(EXECUTION_CEILING_MS) |
| 175 | + }, 30_000) |
| 176 | + |
| 177 | + it('stops at the first pause waiting on a person', async () => { |
| 178 | + const [row] = await readReviewAndLastRun([oneWaiting]) |
| 179 | + expect(row?.waitingWorkflowName).not.toBeNull() |
| 180 | + |
| 181 | + const { rowsRead } = await measureProbe([oneWaiting]) |
| 182 | + expect(rowsRead).toBeLessThanOrEqual(PAUSES_PER_WORKFLOW * ROUNDING_SLACK) |
| 183 | + }, 30_000) |
| 184 | + |
| 185 | + it('costs each workspace only its own pauses when asked about several', async () => { |
| 186 | + const { rowsRead } = await measureProbe([timeOnly, oneWaiting]) |
| 187 | + expect(rowsRead).toBeLessThanOrEqual((LIVE_PAUSES + PAUSES_PER_WORKFLOW) * ROUNDING_SLACK) |
| 188 | + }, 30_000) |
| 189 | +}) |
0 commit comments