fix(docs): keep search responsive and contain sidebar scrolling (#9786)

* fix(docs): keep search responsive and contain sidebar scrolling

* fix(docs): strengthen search cancellation and IPv6 checks

Author: waleedlatif1

apps/docs/components/docs/navigation-drawer.tsx

@@ -54,7 +54,7 @@ export function DocsNavigationDrawer({ children }: DocsNavigationDrawerProps) {
             }}
             className={cn(
               docsScrollAreaClass,
-              '-mx-1 mt-5 min-h-0 flex-1 overflow-y-auto overscroll-contain px-1 pb-6'
+              '-mx-1 mt-5 min-h-0 flex-1 overflow-y-auto overflow-x-hidden overscroll-contain px-1 pb-6'
             )}
           >
             <SidebarNavigation

apps/docs/components/docs/shell.tsx

@@ -98,7 +98,7 @@ export function DocsShell({ children, navigationBySection }: DocsShellProps) {
               key={section}
               className={cn(
                 docsScrollAreaClass,
-                '-mx-1 sticky top-24 hidden max-h-[calc(100dvh-120px)] min-w-0 space-y-5 overflow-y-auto overscroll-contain px-1 pb-5 lg:block'
+                '-mx-1 sticky top-24 hidden max-h-[calc(100dvh-120px)] min-w-0 space-y-5 overflow-y-auto overflow-x-hidden overscroll-contain px-1 pb-5 lg:block'
               )}
             >
               <DocsSearchTrigger />

apps/docs/hooks/queries/search.ts

@@ -5,6 +5,7 @@ import { z } from 'zod'
 
 const SEARCH_STALE_TIME = 60_000
 const SEARCH_GC_TIME = 300_000
+const SEARCH_TIMEOUT_MS = 5000
 const searchKeys = {
   all: ['docs-search'] as const,
   query: (query: string) => [...searchKeys.all, query] as const,
@@ -28,9 +29,21 @@ export function useDocsSearch(query: string) {
     gcTime: SEARCH_GC_TIME,
     retry: false,
     queryFn: async ({ signal }) => {
-      const response = await fetch(`/api/search?query=${encodeURIComponent(query)}`, { signal })
-      if (!response.ok) throw new Error('Search is unavailable. Try again.')
-      return resultsSchema.parse(await response.json())
+      const controller = new AbortController()
+      const abort = () => controller.abort(signal.reason)
+      signal.addEventListener('abort', abort, { once: true })
+      if (signal.aborted) abort()
+      const timeout = setTimeout(() => controller.abort(), SEARCH_TIMEOUT_MS)
+      try {
+        const response = await fetch(`/api/search?query=${encodeURIComponent(query)}`, {
+          signal: controller.signal,
+        })
+        if (!response.ok) throw new Error('Search is unavailable. Try again.')
+        return resultsSchema.parse(await response.json())
+      } finally {
+        clearTimeout(timeout)
+        signal.removeEventListener('abort', abort)
+      }
     },
   })
 }

apps/docs/lib/search/build.ts

@@ -1,9 +1,40 @@
-import type { DocsPage } from '@/lib/content/types'
+import { findNavigationGroup } from '@/lib/content/navigation'
+import type { ContentCatalog, DocsPage } from '@/lib/content/types'
 import { pageSearchChunks } from '@/lib/search/chunks'
-import type { SearchIndex } from '@/lib/search/types'
+import type { SearchIndex, SearchPage } from '@/lib/search/types'
 import { normalizeSearchText, searchCatalogHash } from '@/lib/search/utils'
 
 const MAX_SEARCH_INDEX_BYTES = 64 * 1024 * 1024
+const MAX_SEARCH_CATALOG_BYTES = 4 * 1024 * 1024
+const SECTION_LABELS = { docs: 'Documentation', api: 'API reference', cli: 'CLI', mcp: 'MCP' }
+
+/** Search citations need document identity and navigation, without loading document bodies. */
+export function buildSearchCatalog(catalog: ContentCatalog): SearchPage[] {
+  const pages: SearchPage[] = []
+  let bytes = 0
+  for (const page of catalog.pages) {
+    if (page.url === '/') continue
+    const section = SECTION_LABELS[page.section]
+    const group = findNavigationGroup(catalog.navigation[page.section], page.url)
+    const entry: SearchPage = {
+      id: page.id,
+      url: page.url,
+      title: page.title,
+      contentHash: page.contentHash,
+      headings: page.headings.map((heading) => heading.id),
+      breadcrumbs: [
+        section,
+        ...(group && group !== section && group !== page.title ? [group] : []),
+        page.title,
+      ],
+    }
+    bytes += Buffer.byteLength(JSON.stringify(entry)) + 1
+    if (bytes > MAX_SEARCH_CATALOG_BYTES - 2)
+      throw new Error('Generated search catalog exceeds the 4 MiB size budget')
+    pages.push(entry)
+  }
+  return pages
+}
 
 /** Build-time parsing keeps the public search cold path independent of corpus Markdown size. */
 export function buildSearchIndex(pages: DocsPage[]): SearchIndex {

apps/docs/lib/search/precomputed.ts

@@ -1,14 +1,20 @@
 import 'server-only'
-import { getPages } from '@/lib/content/catalog'
-import type { SearchIndex } from '@/lib/search/types'
+import type { SearchIndex, SearchPage } from '@/lib/search/types'
 import { searchCatalogHash } from '@/lib/search/utils'
 import generatedIndex from '@/.content/search.json'
+import generatedCatalog from '@/.content/search-catalog.json'
 
 const index = generatedIndex as SearchIndex
-if (index.version !== 1 || index.catalogHash !== searchCatalogHash(getPages()))
+const pages = generatedCatalog as SearchPage[]
+const pagesByUrl = new Map(pages.map((page) => [page.url, page]))
+if (index.version !== 1 || index.catalogHash !== searchCatalogHash(pages))
   throw new Error('Search index does not match the documentation catalog. Run content:generate.')
 
 /** An immutable server-only index, generated and published with the canonical catalog. */
 export function getSearchIndex() {
   return index.entries
 }
+
+export function getSearchPage(path: string) {
+  return pagesByUrl.get(path)
+}

apps/docs/lib/search/service.ts

@@ -1,11 +1,8 @@
 import { createLogger } from '@sim/logger'
 import { compareStrings, truncateAtCodePoint } from '@sim/utils/string'
-import { getNavigation, getPage } from '@/lib/content/catalog'
-import { findNavigationGroup } from '@/lib/content/navigation'
-import type { DocsPage } from '@/lib/content/types'
 import { generateSearchEmbedding } from '@/lib/embeddings'
-import { getSearchIndex } from '@/lib/search/precomputed'
-import type { SearchResult } from '@/lib/search/types'
+import { getSearchIndex, getSearchPage } from '@/lib/search/precomputed'
+import type { SearchPage, SearchResult } from '@/lib/search/types'
 import { normalizeSearchText } from '@/lib/search/utils'
 import { DOCS_BASE_URL } from '@/lib/urls'
 
@@ -24,22 +21,16 @@ const STOP_WORDS = new Set([
   'the',
   'to',
 ])
-const SECTION_LABELS = { docs: 'Documentation', api: 'API reference', cli: 'CLI', mcp: 'MCP' }
+const HYBRID_TIMEOUT_MS = 1000
 let hybridRequests = 0
 
-function resultFor(page: DocsPage, id: string, url: string, heading: string): SearchResult {
-  const section = SECTION_LABELS[page.section]
-  const group = findNavigationGroup(getNavigation(page.section), page.url)
+function resultFor(page: SearchPage, id: string, url: string, heading: string): SearchResult {
   return {
     id,
     type: 'page',
     url,
     content: heading || page.title,
-    breadcrumbs: [
-      section,
-      ...(group && group !== section && group !== page.title ? [group] : []),
-      page.title,
-    ],
+    breadcrumbs: [...page.breadcrumbs],
   }
 }
 
@@ -74,7 +65,7 @@ function lexicalSearch(query: string, limit: number): SearchResult[] {
     score += body.includes(phrase) ? 12 : 0
     const existing = scores.get(chunk.pageId)
     if (existing && existing.score >= score) continue
-    const page = getPage(new URL(chunk.sourceLink).pathname)
+    const page = getSearchPage(new URL(chunk.sourceLink).pathname)
     if (!page) continue
     scores.set(chunk.pageId, {
       score,
@@ -97,76 +88,94 @@ interface DatabaseResult {
   metadata: { contentHash?: string }
 }
 
-async function hybridSearch(query: string, limit: number, signal?: AbortSignal) {
+function hybridResults(lists: DatabaseResult[][], limit: number): SearchResult[] {
+  const scores = new Map<string, { score: number; result: SearchResult }>()
+  for (const list of lists) {
+    const seenPages = new Set<string>()
+    list.forEach((row, index) => {
+      let citation: URL
+      try {
+        citation = new URL(row.source_link)
+      } catch {
+        return
+      }
+      if (citation.origin !== new URL(DOCS_BASE_URL).origin) return
+      const page = getSearchPage(citation.pathname)
+      if (!page || row.metadata?.contentHash !== page.contentHash) return
+      if (citation.hash && !page.headings.includes(citation.hash.slice(1))) return
+      if (seenPages.has(page.id)) return
+      seenPages.add(page.id)
+      const existing = scores.get(page.id)
+      const score = (existing?.score ?? 0) + 1 / (60 + index + 1)
+      scores.set(page.id, {
+        score,
+        result: existing?.result ?? resultFor(page, row.chunk_id, citation.href, row.header_text),
+      })
+    })
+  }
+  return [...scores.values()]
+    .sort(
+      (left, right) => right.score - left.score || compareStrings(left.result.url, right.result.url)
+    )
+    .slice(0, limit)
+    .map(({ result }) => result)
+}
+
+async function hybridSearch(
+  query: string,
+  limit: number,
+  signal: AbortSignal,
+  onResults: (results: SearchResult[]) => void
+) {
   const { searchDatabase } = await import('@/lib/search/database')
+  if (signal.aborted) return []
   const sql = searchDatabase()
-  const embedding = await generateSearchEmbedding(query, signal).catch(() => null)
-  if (signal?.aborted) return []
   const keyword = sql<DatabaseResult[]>`
     SELECT chunk_id, source_document, source_link, header_text, metadata
     FROM docs_embeddings
     WHERE chunk_text_tsv @@ plainto_tsquery('english', ${query})
     ORDER BY ts_rank(chunk_text_tsv, plainto_tsquery('english', ${query})) DESC, source_link
     LIMIT ${limit * 3}
   `
-  const vector = embedding
-    ? sql<DatabaseResult[]>`
+  let vector: typeof keyword | undefined
+  const sources: DatabaseResult[][] = [[], []]
+  async function collect(source: number, request: PromiseLike<DatabaseResult[]>) {
+    const rows = await request
+    if (signal.aborted) return
+    sources[source] = rows
+    onResults(hybridResults(sources, limit))
+  }
+  async function collectVector() {
+    const embedding = await generateSearchEmbedding(query, signal).catch(() => null)
+    if (!embedding || signal.aborted) return
+    vector = sql<DatabaseResult[]>`
         SELECT chunk_id, source_document, source_link, header_text, metadata
         FROM docs_embeddings
         WHERE 1 - (embedding <=> ${JSON.stringify(embedding)}::vector) >= 0.6
         ORDER BY embedding <=> ${JSON.stringify(embedding)}::vector, source_link
         LIMIT ${limit * 3}
       `
-    : undefined
+    await collect(1, vector)
+  }
   const cancel = () => {
     keyword.cancel()
     vector?.cancel()
   }
-  signal?.addEventListener('abort', cancel, { once: true })
+  signal.addEventListener('abort', cancel, { once: true })
   try {
-    const lists = await Promise.allSettled([keyword, vector ?? Promise.resolve([])])
-    const scores = new Map<string, { score: number; result: SearchResult }>()
+    const lists = await Promise.allSettled([collect(0, keyword), collectVector()])
     for (const [source, list] of lists.entries()) {
       if (list.status === 'rejected') {
-        if (!signal?.aborted)
+        if (!signal.aborted)
           logger.warn('Hybrid search query unavailable; serving remaining results', {
             source: source === 0 ? 'keyword' : 'vector',
             error: list.reason,
           })
-        continue
       }
-      const seenPages = new Set<string>()
-      list.value.forEach((row, index) => {
-        let citation: URL
-        try {
-          citation = new URL(row.source_link)
-        } catch {
-          return
-        }
-        if (citation.origin !== new URL(DOCS_BASE_URL).origin) return
-        const page = getPage(citation.pathname)
-        if (!page || row.metadata?.contentHash !== page.contentHash) return
-        if (citation.hash && !page.headings.some((heading) => `#${heading.id}` === citation.hash))
-          return
-        if (seenPages.has(page.id)) return
-        seenPages.add(page.id)
-        const existing = scores.get(page.id)
-        const score = (existing?.score ?? 0) + 1 / (60 + index + 1)
-        scores.set(page.id, {
-          score,
-          result: existing?.result ?? resultFor(page, row.chunk_id, citation.href, row.header_text),
-        })
-      })
     }
-    return [...scores.values()]
-      .sort(
-        (left, right) =>
-          right.score - left.score || compareStrings(left.result.url, right.result.url)
-      )
-      .slice(0, limit)
-      .map(({ result }) => result)
+    return hybridResults(sources, limit)
   } finally {
-    signal?.removeEventListener('abort', cancel)
+    signal.removeEventListener('abort', cancel)
   }
 }
 
@@ -182,35 +191,34 @@ export async function searchDocs(
     : 10
   if (!query) return []
   const lexical = lexicalSearch(query, limit)
-  if (!process.env.DATABASE_URL || hybridRequests >= 2 || signal?.aborted) return lexical
+  if (lexical.length || !process.env.DATABASE_URL || hybridRequests >= 2 || signal?.aborted)
+    return lexical
+  const controller = new AbortController()
+  const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal
+  let partial: SearchResult[] = []
+  let onAbort: (() => void) | undefined
+  const aborted = new Promise<SearchResult[]>((resolve) => {
+    onAbort = () => resolve(partial)
+    combined.addEventListener('abort', onAbort, { once: true })
+    if (combined.aborted) onAbort()
+  })
+  const timeout = setTimeout(() => controller.abort(), HYBRID_TIMEOUT_MS)
   hybridRequests += 1
+  const pending = hybridSearch(query, limit, combined, (results) => {
+    partial = results
+  })
+    .catch((error) => {
+      if (!combined.aborted)
+        logger.warn('Hybrid search unavailable; serving local results', { error })
+      return partial
+    })
+    .finally(() => {
+      hybridRequests -= 1
+    })
   try {
-    const hybrid = await hybridSearch(query, limit, signal)
-    const results = new Map<string, { score: number; result: SearchResult }>()
-    for (const list of [lexical, hybrid]) {
-      list.forEach((result, rank) => {
-        const path = new URL(result.url).pathname
-        const existing = results.get(path)
-        const title = getPage(path)?.title
-        const exactTitle =
-          title && normalizeSearchText(title) === normalizeSearchText(query) ? 0.05 : 0
-        results.set(path, {
-          score: (existing?.score ?? exactTitle) + 1 / (60 + rank + 1),
-          result: existing?.result ?? result,
-        })
-      })
-    }
-    return [...results.values()]
-      .sort(
-        (left, right) =>
-          right.score - left.score || compareStrings(left.result.url, right.result.url)
-      )
-      .slice(0, limit)
-      .map(({ result }) => result)
-  } catch (error) {
-    if (!signal?.aborted) logger.warn('Hybrid search unavailable; serving local results', { error })
-    return lexical
+    return await Promise.race([pending, aborted])
   } finally {
-    hybridRequests -= 1
+    clearTimeout(timeout)
+    if (onAbort) combined.removeEventListener('abort', onAbort)
   }
 }

apps/docs/lib/search/types.ts

@@ -6,6 +6,15 @@ export interface SearchResult {
   breadcrumbs: string[]
 }
 
+export interface SearchPage {
+  id: string
+  url: string
+  title: string
+  contentHash: string
+  headings: string[]
+  breadcrumbs: string[]
+}
+
 export interface SearchChunk {
   id: string
   pageId: string

apps/docs/lib/search/utils.ts

@@ -7,7 +7,7 @@ export function normalizeSearchText(value: string): string {
 }
 
 /** Couples generated search data to exactly the catalog content it cites. */
-export function searchCatalogHash(pages: DocsPage[]): string {
+export function searchCatalogHash(pages: Pick<DocsPage, 'id' | 'url' | 'contentHash'>[]): string {
   const identities = pages
     .filter((page) => page.url !== '/')
     .map((page) => `${page.id}:${page.contentHash}`)