Skip to content

Commit d5ef6c8

Browse files
waleedlatif1waleedlatif1
committed
improvement(knowledge): page search results, take a custom date window, and rerank a person's search
- the Search page asks for 50 chunks, collapses them to one card per document, shows ten at a time and reveals more on request - `updated` gains a custom range: `from`/`to` days in the URL, sent as an inclusive `modifiedAfter`/`modifiedBefore` window; `modifiedBefore` joins the filter contract, the filter intersection, the document conditions, the on-row date test, the filtered-set estimate and the bounded probe - the dashboard search opts into the platform's cross-encoder reranker whenever a Cohere key is configured; reranking stays best-effort
1 parent 1b59f04 commit d5ef6c8

13 files changed

Lines changed: 192 additions & 29 deletions

File tree

‎apps/sim/app/api/knowledge/search/route.test.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ import { NextRequest } from 'next/server'
66
import { beforeEach, describe, expect, it, vi } from 'vitest'
77

88
const mocks = vi.hoisted(() => ({ search: vi.fn() }))
9+
const { mockHasRotatingApiKey } = vi.hoisted(() => ({ mockHasRotatingApiKey: vi.fn(() => true) }))
10+
vi.mock('@/lib/core/config/api-keys', () => ({ hasRotatingApiKey: mockHasRotatingApiKey }))
11+
912
vi.mock('@/lib/knowledge/application/workspace-search', () => ({
1013
searchScopedKnowledge: { operation: { id: 'knowledge.search' }, execute: mocks.search },
1114
}))
@@ -47,6 +50,10 @@ describe('workspace search route', () => {
4750
expect(call.input.signal).toBe(request.signal)
4851
expect(call.input.allowPartialResults).toBe(true)
4952
expect(call.input.vectorBudgetMs).toBe(3000)
53+
/** A person's search is reranked by the platform's cross-encoder whenever one is configured. */
54+
expect(mockHasRotatingApiKey).toHaveBeenCalledWith('cohere')
55+
expect(call.input.rerankerEnabled).toBe(true)
56+
expect(call.input.rerankerModel).toBe('rerank-v4.0-fast')
5057
controller.abort()
5158
expect(call.input.signal.aborted).toBe(true)
5259
await expect(response.json()).resolves.toEqual({

‎apps/sim/app/api/knowledge/search/route.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import {
44
internalRateLimits,
55
internalSessionAuth,
66
} from '@/lib/api/server/routes'
7+
import { hasRotatingApiKey } from '@/lib/core/config/api-keys'
78
import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies'
89
import { knowledgeOperations } from '@/lib/knowledge/application/operations'
910
import { searchScopedKnowledge } from '@/lib/knowledge/application/workspace-search'
11+
import { DEFAULT_RERANKER_MODEL } from '@/lib/knowledge/reranker-models'
1012
import { sourceAuthor } from '@/lib/knowledge/search/author'
1113

1214
const DIRECT_SEARCH_VECTOR_BUDGET_MS = 3000
@@ -28,6 +30,12 @@ export const POST = defineInternalJsonRoute({
2830
topK: body.topK,
2931
allowPartialResults: true,
3032
vectorBudgetMs: DIRECT_SEARCH_VECTOR_BUDGET_MS,
33+
/**
34+
* A person's search is reranked by the platform's cross-encoder when one is configured;
35+
* reranking is best-effort, so a provider outage leaves the fused order in place.
36+
*/
37+
rerankerEnabled: hasRotatingApiKey('cohere'),
38+
rerankerModel: DEFAULT_RERANKER_MODEL,
3139
surface: 'dashboard' as const,
3240
signal: request.signal,
3341
}),

‎apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,13 @@ afterEach(() => {
4848
act(() => root.unmount())
4949
vi.unstubAllGlobals()
5050
})
51-
async function render(scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' }) {
51+
async function render(
52+
scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' },
53+
searchParams = ''
54+
) {
5255
await act(async () =>
5356
root.render(
54-
<NuqsTestingAdapter>
57+
<NuqsTestingAdapter searchParams={searchParams}>
5558
<KnowledgeSearchResults scope={scope} query='launch' onSummarize={vi.fn()} />
5659
</NuqsTestingAdapter>
5760
)
@@ -148,3 +151,53 @@ describe('source setup navigation', () => {
148151
expect(container.querySelector('a')?.getAttribute('href')).toBe(href)
149152
})
150153
})
154+
155+
describe('result paging and the custom window', () => {
156+
const result = (n: number) => ({
157+
documentId: `doc-${n}`,
158+
knowledgeBaseId: 'kb',
159+
knowledgeBaseName: 'Index',
160+
documentName: `Document ${n}`,
161+
sourceUrl: null,
162+
connectorType: 'slack',
163+
sourceModifiedAt: null,
164+
author: null,
165+
content: 'launch notes',
166+
chunkIndex: 0,
167+
similarity: 0.5,
168+
})
169+
170+
it('shows ten documents at a time and reveals more on request', async () => {
171+
mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
172+
mocks.search.mockReturnValue({
173+
data: {
174+
query: 'launch',
175+
results: Array.from({ length: 25 }, (_, n) => result(n)),
176+
retrieval: { status: 'complete', timedOutLegs: [] },
177+
},
178+
isPending: false,
179+
isFetching: false,
180+
isPlaceholderData: false,
181+
isError: false,
182+
refetch: mocks.retry,
183+
})
184+
await render()
185+
expect(container.textContent).toContain('Document 9')
186+
expect(container.textContent).not.toContain('Document 10')
187+
const more = [...container.querySelectorAll('button')].find(
188+
(b) => b.textContent === 'Show more'
189+
)!
190+
expect(more).toBeDefined()
191+
await act(async () => more.click())
192+
expect(container.textContent).toContain('Document 19')
193+
expect(container.textContent).not.toContain('Document 20')
194+
})
195+
196+
it('searches a custom window as an inclusive range of days', async () => {
197+
mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } })
198+
await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10')
199+
const filters = mocks.search.mock.calls.at(-1)![2]
200+
expect(filters.modifiedAfter).toBe('2026-09-01T00:00:00.000Z')
201+
expect(filters.modifiedBefore).toBe('2026-09-10T23:59:59.999Z')
202+
})
203+
})

‎apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
'use client'
22

33
import { useState } from 'react'
4-
import { Chip, ChipLink, cn } from '@sim/emcn'
4+
import { Chip, ChipDatePicker, ChipLink, cn } from '@sim/emcn'
55
import { useQueryStates } from 'nuqs'
66
import { ActivityStatus } from '@/components/ui/activity-status'
77
import type {
@@ -27,6 +27,8 @@ import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/conn
2727
import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge'
2828

2929
const DAY_MS = 24 * 60 * 60 * 1000
30+
/** Cards shown before the reader asks for more; the search itself returns several pages' worth. */
31+
const RESULTS_PAGE_SIZE = 10
3032
/** Every result without a connector is an upload; the filter names them so. */
3133
const UPLOAD_SOURCE = 'upload'
3234

@@ -127,6 +129,7 @@ interface SearchResultsProps {
127129
function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
128130
const [hasShownFilters, setHasShownFilters] = useState(false)
129131
const [searchedAt] = useState(Date.now)
132+
const [shown, setShown] = useState(RESULTS_PAGE_SIZE)
130133
const {
131134
data: index,
132135
isPending: basesPending,
@@ -136,11 +139,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
136139
} = useSearchIndex(scope)
137140
const [filters, setFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys)
138141
const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated)
142+
/** A custom window is inclusive of both days; `to` runs to the end of its day. */
143+
const custom = filters.updated === 'custom'
139144
const searchFilters: WorkspaceSearchFilters = {
140145
...(filters.source ? { source: filters.source } : {}),
141146
...(window?.days
142147
? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() }
143148
: {}),
149+
...(custom && filters.from ? { modifiedAfter: filters.from.toISOString() } : {}),
150+
...(custom && filters.to
151+
? { modifiedBefore: new Date(filters.to.getTime() + DAY_MS - 1).toISOString() }
152+
: {}),
144153
}
145154
const {
146155
data: search,
@@ -262,6 +271,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
262271
{window.label}
263272
</Chip>
264273
))}
274+
{custom && (
275+
<ChipDatePicker
276+
mode='range'
277+
label='Updated between'
278+
startDate={filters.from?.toISOString()}
279+
endDate={filters.to?.toISOString()}
280+
onRangeChange={(start, end) =>
281+
void setFilters({ from: new Date(start), to: new Date(end) })
282+
}
283+
/>
284+
)}
265285
</div>
266286
)}
267287
{showResults && (
@@ -272,7 +292,7 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
272292
className={cn('flex flex-col', isPlaceholderData && 'opacity-60')}
273293
onKeyDown={handleResultsKeyDown}
274294
>
275-
{documents.map((result) => {
295+
{documents.slice(0, shown).map((result) => {
276296
const source = toSource(result, query, scope)
277297
return (
278298
<SourceCard
@@ -291,6 +311,13 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
291311
/>
292312
)
293313
})}
314+
{documents.length > shown && (
315+
<div className='flex px-2 py-2'>
316+
<Chip variant='border' onClick={() => setShown((count) => count + RESULTS_PAGE_SIZE)}>
317+
Show more
318+
</Chip>
319+
</div>
320+
)}
294321
</div>
295322
)}
296323
</div>

‎apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ describe('search refinement with the real query cache and URL state', () => {
212212
expect(requests.at(-1)?.body).toEqual({
213213
organizationId: 'organization',
214214
query: 'launch',
215+
topK: 50,
215216
filters: expectedFilters,
216217
})
217218
expect(container.querySelector('h1')).toBeNull()

‎apps/sim/app/workspace/[workspaceId]/home/search-params.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { parseAsString, parseAsStringLiteral } from 'nuqs/server'
1+
import { parseAsIsoDate, parseAsString, parseAsStringLiteral } from 'nuqs/server'
22

33
/**
44
* Co-located, typed URL query-param definition for the home/Chat surface.
@@ -26,19 +26,23 @@ export const resourceUrlKeys = {
2626
clearOnDefault: true,
2727
} as const
2828

29-
/** The recency windows a search can be narrowed to. */
29+
/** The recency windows a search can be narrowed to; `custom` reads its bounds from `from` and `to`. */
3030
export const UPDATED_WINDOWS = [
3131
{ id: 'any', label: 'Any time', days: null },
3232
{ id: '7d', label: 'Past week', days: 7 },
3333
{ id: '30d', label: 'Past month', days: 30 },
34+
{ id: 'custom', label: 'Custom range', days: null },
3435
] as const
3536
const UPDATED_WINDOW_IDS = UPDATED_WINDOWS.map((window) => window.id)
3637

3738
/**
3839
* Shared result filters for organization search. `source` is a connector type
39-
* or `upload`, absent for every source.
40+
* or `upload`, absent for every source; `from` and `to` are the days of a custom
41+
* window, inclusive, and mean nothing unless `updated` is `custom`.
4042
*/
4143
export const searchFilterParsers = {
4244
source: parseAsString,
4345
updated: parseAsStringLiteral(UPDATED_WINDOW_IDS).withDefault('any'),
46+
from: parseAsIsoDate,
47+
to: parseAsIsoDate,
4448
} as const

‎apps/sim/hooks/queries/kb/knowledge.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ export const KNOWLEDGE_DOCUMENT_LIST_STALE_TIME = 60 * 1000
8989
export const KNOWLEDGE_CHUNK_LIST_STALE_TIME = 60 * 1000
9090
export const KNOWLEDGE_CHUNK_SEARCH_STALE_TIME = 60 * 1000
9191
export const WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME = 60 * 1000
92+
/** Chunks one search asks for: several pages of documents once collapsed to one card each. */
93+
export const WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT = 50
9294
export const KNOWLEDGE_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000
9395
export const KNOWLEDGE_TAG_USAGE_STALE_TIME = 60 * 1000
9496
export const KNOWLEDGE_DOCUMENT_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000
@@ -1229,6 +1231,7 @@ export function useWorkspaceKnowledgeSearch(
12291231
...(scope ? resourceScopeFields(scope) : {}),
12301232
query: trimmed,
12311233
filters,
1234+
topK: WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT,
12321235
},
12331236
signal
12341237
),

‎apps/sim/lib/api/contracts/knowledge/search.ts‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -172,11 +172,26 @@ export const workspaceKnowledgeSearchResultSchema = z.object({
172172
})
173173
export type WorkspaceKnowledgeSearchResult = z.output<typeof workspaceKnowledgeSearchResultSchema>
174174

175-
export const workspaceSearchFiltersSchema = z.object({
176-
source: z.string().trim().min(1, 'Source cannot be empty').max(100).optional(),
177-
modifiedAfter: z.string().datetime({ offset: true }).optional(),
178-
documentIds: z.array(z.string().min(1).max(200)).min(1).max(20).optional(),
179-
})
175+
export const workspaceSearchFiltersSchema = z
176+
.object({
177+
source: z.string().trim().min(1, 'Source cannot be empty').max(100).optional(),
178+
modifiedAfter: z.string().datetime({ offset: true }).optional(),
179+
modifiedBefore: z.string().datetime({ offset: true }).optional(),
180+
documentIds: z.array(z.string().min(1).max(200)).min(1).max(20).optional(),
181+
})
182+
.superRefine((filters, ctx) => {
183+
if (
184+
filters.modifiedAfter &&
185+
filters.modifiedBefore &&
186+
Date.parse(filters.modifiedBefore) < Date.parse(filters.modifiedAfter)
187+
) {
188+
ctx.addIssue({
189+
code: 'custom',
190+
path: ['modifiedBefore'],
191+
message: 'modifiedBefore must not precede modifiedAfter',
192+
})
193+
}
194+
})
180195
export type WorkspaceSearchFilters = z.output<typeof workspaceSearchFiltersSchema>
181196

182197
export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema.safeExtend({

‎apps/sim/lib/core/config/api-keys.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
import { env } from '@/lib/core/config/env'
22
import { LLM_KEY_POOLS } from '@/lib/core/config/env-capabilities'
33

4+
/** Whether the platform holds at least one key for a provider, without selecting one. */
5+
export function hasRotatingApiKey(provider: string): boolean {
6+
if (!(provider in LLM_KEY_POOLS)) return false
7+
const definition = LLM_KEY_POOLS[provider as keyof typeof LLM_KEY_POOLS]
8+
if (definition.keys.some((key) => Boolean(env[key]))) return true
9+
return 'fallbackKey' in definition && Boolean(env[definition.fallbackKey])
10+
}
11+
412
/**
513
* Rotates through available API keys for a provider
614
* @param provider - The provider to get a key for (e.g., 'openai')

‎apps/sim/lib/knowledge/search/filter-conditions.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { document, knowledgeConnector } from '@sim/db/schema'
2-
import { eq, gte, inArray, isNull, type SQL, sql } from 'drizzle-orm'
2+
import { eq, gte, inArray, isNull, lte, type SQL, sql } from 'drizzle-orm'
33
import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
44

55
/** Filters the document in every retrieval leg, alongside its current ACL. */
@@ -9,6 +9,9 @@ export function workspaceSearchFilterConditions(filters?: WorkspaceSearchFilters
99
if (filters?.modifiedAfter) {
1010
conditions.push(gte(document.sourceModifiedAt, new Date(filters.modifiedAfter)))
1111
}
12+
if (filters?.modifiedBefore) {
13+
conditions.push(lte(document.sourceModifiedAt, new Date(filters.modifiedBefore)))
14+
}
1215
if (filters?.source === 'upload') conditions.push(isNull(document.connectorId))
1316
else if (filters?.source) {
1417
conditions.push(

0 commit comments

Comments
 (0)