fix(db): avoid repeated dev index builds and search backfills

Author: Sg312

.github/workflows/migrations.yml

@@ -80,7 +80,7 @@ jobs:
             # terminal" but still exits 0, so the job goes green without applying the
             # change. tee keeps the output live in the log; we then fail on drizzle's
             # own TTY error. pipefail also preserves a non-zero db:push exit through tee.
-            bun run db:push --force < /dev/null 2>&1 | tee /tmp/db-push.log
+            SIM_DEV_DB_PUSH=1 bun run db:push --force < /dev/null 2>&1 | tee /tmp/db-push.log
             if grep -q "Interactive prompts require a TTY terminal" /tmp/db-push.log; then
               echo "ERROR: db:push needs an interactive rename decision; land it as a versioned migration instead of relying on push." >&2
               exit 1

packages/db/dev-push.test.ts

@@ -0,0 +1,48 @@
+import * as schema from '@sim/db/schema'
+import { generateDrizzleJson, generateMigration } from 'drizzle-kit/api'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+interface SnapshotTable {
+  indexes: Record<string, { method: string; with?: Record<string, unknown> }>
+}
+
+afterEach(() => vi.unstubAllEnvs())
+
+describe('dev push index options', () => {
+  it('matches introspected options without changing ordinary migration snapshots', async () => {
+    vi.stubEnv('SIM_DEV_DB_PUSH', undefined)
+    const ordinary = generateDrizzleJson(schema)
+    const introspected = structuredClone(ordinary)
+    const hnswIndexes = Object.values<SnapshotTable>(introspected.tables).flatMap((table) =>
+      Object.values(table.indexes).filter((index) => index.method === 'hnsw')
+    )
+    expect(hnswIndexes).toHaveLength(22)
+    for (const index of hnswIndexes) {
+      expect(index.with).toEqual({ m: 16, ef_construction: 64 })
+      index.with = { m: '16', ef_construction: '64' }
+    }
+
+    vi.stubEnv('SIM_DEV_DB_PUSH', '1')
+    const dev = generateDrizzleJson(schema)
+    expect(await generateMigration(introspected, dev)).toEqual([])
+    expect({ ...dev, id: ordinary.id }).toEqual(introspected)
+
+    vi.stubEnv('SIM_DEV_DB_PUSH', undefined)
+    const restored = generateDrizzleJson(schema)
+    expect({ ...restored, id: ordinary.id }).toEqual(ordinary)
+  })
+
+  it('still detects a real index-option change in dev', async () => {
+    vi.stubEnv('SIM_DEV_DB_PUSH', '1')
+    const current = generateDrizzleJson(schema)
+    const previous = structuredClone(current)
+    previous.tables['public.embedding'].indexes.embedding_vector_hnsw_idx.with = {
+      m: '8',
+      ef_construction: '64',
+    }
+    const statements = await generateMigration(previous, current)
+    expect(statements).toHaveLength(2)
+    expect(statements[0]).toContain('DROP INDEX "embedding_vector_hnsw_idx"')
+    expect(statements[1]).toContain('WITH (m=16,ef_construction=64)')
+  })
+})

packages/db/schema.ts

@@ -27,6 +27,17 @@ import {
 } from 'drizzle-orm/pg-core'
 import { DEFAULT_FREE_CREDITS, TAG_SLOTS } from './constants'
 
+/**
+ * Drizzle push compares index options as JSON, while Postgres introspection
+ * returns strings. Normalize only dev pushes so unchanged HNSW indexes survive
+ * the diff; versioned migration snapshots retain their original numeric values.
+ */
+function hnswIndexOptions() {
+  return process.env.SIM_DEV_DB_PUSH === '1'
+    ? { m: '16', ef_construction: '64' }
+    : { m: 16, ef_construction: 64 }
+}
+
 // Custom tsvector type for full-text search
 export const tsvector = customType<{
   data: string
@@ -3194,28 +3205,16 @@ export const embedding = pgTable(
     // Vector similarity search indexes (HNSW), one per stored width
     embeddingVectorHnswIdx: index('embedding_vector_hnsw_idx')
       .using('hnsw', table.embedding.op('vector_cosine_ops'))
-      .with({
-        m: 16,
-        ef_construction: 64,
-      }),
+      .with(hnswIndexOptions()),
     embedding384VectorHnswIdx: index('embedding_384_vector_hnsw_idx')
       .using('hnsw', table.embedding384.op('vector_cosine_ops'))
-      .with({
-        m: 16,
-        ef_construction: 64,
-      }),
+      .with(hnswIndexOptions()),
     embedding768VectorHnswIdx: index('embedding_768_vector_hnsw_idx')
       .using('hnsw', table.embedding768.op('vector_cosine_ops'))
-      .with({
-        m: 16,
-        ef_construction: 64,
-      }),
+      .with(hnswIndexOptions()),
     embedding1024VectorHnswIdx: index('embedding_1024_vector_hnsw_idx')
       .using('hnsw', table.embedding1024.op('vector_cosine_ops'))
-      .with({
-        m: 16,
-        ef_construction: 64,
-      }),
+      .with(hnswIndexOptions()),
     /**
      * pgvector indexes `vector` only up to 2,000 dimensions and `halfvec` up to
      * 4,000, so the 3,072 column is indexed through a `halfvec` cast — the
@@ -3236,27 +3235,24 @@ export const embedding = pgTable(
      */
     embedding3072VectorHnswIdx: index('embedding_3072_vector_hnsw_idx')
       .using('hnsw', sql`(${table.embedding3072}::halfvec(3072)) halfvec_cosine_ops`)
-      .with({
-        m: 16,
-        ef_construction: 64,
-      }),
+      .with(hnswIndexOptions()),
 
     /** contract-pending(after stored candidate retrieval is fully deployed): remove these expression indexes; old app versions still query them during rollout. */
     embeddingBinaryHnswIdx: index('embedding_binary_hnsw_idx')
       .using('hnsw', sql`(binary_quantize(${table.embedding})::bit(1536)) bit_hamming_ops`)
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     embedding384BinaryHnswIdx: index('embedding_384_binary_hnsw_idx')
       .using('hnsw', sql`(binary_quantize(${table.embedding384})::bit(384)) bit_hamming_ops`)
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     embedding768BinaryHnswIdx: index('embedding_768_binary_hnsw_idx')
       .using('hnsw', sql`(binary_quantize(${table.embedding768})::bit(768)) bit_hamming_ops`)
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     embedding1024BinaryHnswIdx: index('embedding_1024_binary_hnsw_idx')
       .using('hnsw', sql`(binary_quantize(${table.embedding1024})::bit(1024)) bit_hamming_ops`)
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     embedding3072BinaryHnswIdx: index('embedding_3072_binary_hnsw_idx')
       .using('hnsw', sql`(binary_quantize(${table.embedding3072})::bit(3072)) bit_hamming_ops`)
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
 
     // Text tag indexes
     tag1Idx: index('emb_kb_tag1_lower_idx').on(table.knowledgeBaseId, sql`lower(${table.tag1})`),
@@ -3349,37 +3345,37 @@ export const embeddingSearch = pgTable(
       .where(sql`${table.enabled}`),
     binaryIdx: index('embedding_search_binary_hnsw_idx')
       .using('hnsw', table.binary.op('bit_hamming_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     binary384Idx: index('embedding_search_384_binary_hnsw_idx')
       .using('hnsw', table.binary384.op('bit_hamming_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     binary768Idx: index('embedding_search_768_binary_hnsw_idx')
       .using('hnsw', table.binary768.op('bit_hamming_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     binary1024Idx: index('embedding_search_1024_binary_hnsw_idx')
       .using('hnsw', table.binary1024.op('bit_hamming_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     binary3072Idx: index('embedding_search_3072_binary_hnsw_idx')
       .using('hnsw', table.binary3072.op('bit_hamming_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     vectorIdx: index('embedding_search_cosine_hnsw_idx')
       .using('hnsw', table.vector.op('halfvec_cosine_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     vector512Idx: index('embedding_search_512_cosine_hnsw_idx')
       .using('hnsw', table.vector512.op('halfvec_cosine_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     vector384Idx: index('embedding_search_384_cosine_hnsw_idx')
       .using('hnsw', table.vector384.op('halfvec_cosine_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     vector768Idx: index('embedding_search_768_cosine_hnsw_idx')
       .using('hnsw', table.vector768.op('halfvec_cosine_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     vector1024Idx: index('embedding_search_1024_cosine_hnsw_idx')
       .using('hnsw', table.vector1024.op('halfvec_cosine_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     vector3072Idx: index('embedding_search_3072_cosine_hnsw_idx')
       .using('hnsw', table.vector3072.op('halfvec_cosine_ops'))
-      .with({ m: 16, ef_construction: 64 }),
+      .with(hnswIndexOptions()),
     widthCheck: check(
       'embedding_search_width_check',
       sql`num_nonnulls("binary", "binary_384", "binary_768", "binary_1024", "binary_3072") = 1`
@@ -3456,10 +3452,7 @@ export const docsEmbeddings = pgTable(
     // Vector similarity search indexes (HNSW) - optimized for documentation embeddings
     embeddingVectorHnswIdx: index('docs_embedding_vector_hnsw_idx')
       .using('hnsw', table.embedding.op('vector_cosine_ops'))
-      .with({
-        m: 16,
-        ef_construction: 64,
-      }),
+      .with(hnswIndexOptions()),
 
     // GIN index for JSONB metadata queries
     metadataGinIdx: index('docs_emb_metadata_gin_idx').using('gin', table.metadata),

packages/db/script-migrations/0016_backfill_search_vectors.test.ts

@@ -0,0 +1,65 @@
+import {
+  backfillSearchVectorsMigration,
+  runDevSearchBackfill,
+} from '@sim/db/script-migrations/0016_backfill_search_vectors'
+import type { Sql } from 'postgres'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+/** Models the ledger while keeping the expensive projection work mocked. */
+function createLedger(appliedNames: string[] = []) {
+  const names = new Set(appliedNames)
+  const query = vi.fn(async (parts: TemplateStringsArray, name?: string) => {
+    const statement = parts.join('?').trim()
+    if (statement.startsWith('SELECT name')) {
+      return name && names.has(name) ? [{ name }] : []
+    }
+    if (statement.startsWith('INSERT INTO script_migrations') && name) names.add(name)
+    return []
+  })
+  const begin = vi.fn(async (callback: (tx: typeof query) => Promise<void>) => callback(query))
+  return { names, begin, sql: Object.assign(query, { begin }) as unknown as Sql }
+}
+
+afterEach(() => vi.restoreAllMocks())
+
+describe('dev search backfill tracking', () => {
+  it('records success and skips projection work on the next deploy', async () => {
+    const up = vi.spyOn(backfillSearchVectorsMigration, 'up').mockResolvedValue(undefined)
+    const { sql, names } = createLedger()
+    await runDevSearchBackfill(sql)
+    expect(names).toEqual(
+      new Set(['0015_backfill_embedding_search', '0016_backfill_search_vectors'])
+    )
+    await runDevSearchBackfill(sql)
+    expect(up).toHaveBeenCalledTimes(1)
+  })
+
+  it('retries failed work without recording completion', async () => {
+    const up = vi
+      .spyOn(backfillSearchVectorsMigration, 'up')
+      .mockRejectedValueOnce(new Error('backfill interrupted'))
+      .mockResolvedValue(undefined)
+    const { sql, names, begin } = createLedger()
+    await expect(runDevSearchBackfill(sql)).rejects.toThrow('backfill interrupted')
+    expect(names.size).toBe(0)
+    expect(begin).not.toHaveBeenCalled()
+    await runDevSearchBackfill(sql)
+    expect(names.has(backfillSearchVectorsMigration.name)).toBe(true)
+    expect(up).toHaveBeenCalledTimes(2)
+  })
+
+  it('runs the vector upgrade even when the older binary backfill is recorded', async () => {
+    const up = vi.spyOn(backfillSearchVectorsMigration, 'up').mockResolvedValue(undefined)
+    const { sql } = createLedger(['0015_backfill_embedding_search'])
+    await runDevSearchBackfill(sql)
+    expect(up).toHaveBeenCalledOnce()
+  })
+
+  it('skips a backfill already recorded by the tracked migration runner', async () => {
+    const up = vi.spyOn(backfillSearchVectorsMigration, 'up').mockResolvedValue(undefined)
+    const { sql, begin } = createLedger([backfillSearchVectorsMigration.name])
+    await runDevSearchBackfill(sql)
+    expect(up).not.toHaveBeenCalled()
+    expect(begin).not.toHaveBeenCalled()
+  })
+})

packages/db/script-migrations/0016_backfill_search_vectors.ts

@@ -246,12 +246,43 @@ export const backfillSearchVectorsMigration: ScriptMigration = {
   },
 }
 
+/** Records the dev push backfill only after success, so later deploys skip both full scans. */
+export async function runDevSearchBackfill(sql: Sql): Promise<void> {
+  await sql`
+    CREATE TABLE IF NOT EXISTS script_migrations (
+      name text PRIMARY KEY,
+      applied_at timestamptz NOT NULL DEFAULT now()
+    )
+  `
+  const [applied] = await sql`
+    SELECT name FROM script_migrations WHERE name = ${backfillSearchVectorsMigration.name}
+  `
+  if (applied) {
+    logger.info('Dev search backfill already applied; skipping')
+    return
+  }
+
+  await backfillSearchVectorsMigration.up(sql)
+  await sql.begin(async (tx) => {
+    for (const name of [
+      backfillSearchVectorsMigration.name,
+      ...(backfillSearchVectorsMigration.supersedes ?? []),
+    ]) {
+      await tx`INSERT INTO script_migrations (name) VALUES (${name}) ON CONFLICT (name) DO NOTHING`
+    }
+  })
+}
+
 if (import.meta.main) {
   const url = resolveMigrationDatabaseUrl()
   if (!url) throw new Error('DATABASE_URL is required to initialize search vectors')
   const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => undefined })
   try {
-    await backfillSearchVectorsMigration.up(sql)
+    if (process.env.SIM_DEV_DB_PUSH === '1') {
+      await runDevSearchBackfill(sql)
+    } else {
+      await backfillSearchVectorsMigration.up(sql)
+    }
   } finally {
     await sql.end()
   }