Skip to content

Commit eb0dfcf

Browse files
Vikhyath MondretiSg312
Vikhyath Mondreti
authored and
Sg312
committed
Add workspace specification benchmarks with isolated Plan evaluation
1 parent 389797c commit eb0dfcf

52 files changed

Lines changed: 34951 additions & 33 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/sim/.env.example‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ NEXT_PUBLIC_APP_URL=http://localhost:3000
2828
# COPILOT_API_KEY= # Mint one at https://sim.ai. Without it the Sim Chat block, prompt jobs, and Inbox cannot run
2929
# MSHIP_SYSPROMPT_OVERRIDE= # Highest-priority instructions for Mothership; honored only when the validated API key owner is enterprise
3030
# NEXT_PUBLIC_CHAT_DISABLED=true # Hides the Chat module: the workspace lands on your first workflow, and the chats list, scheduled tasks, and editor Chat panel are absent. Chat is shown when unset; `bun run setup` sets this for you if you skip the chat key
31+
# MOTHERSHIP_BENCHMARK_ENABLED=true # Shows the organization Benchmark tab and enables its authenticated APIs
32+
# COPILOT_DEV_URL=http://localhost:8080 # Benchmark worker endpoint; requires the worker benchmark profile and the normal COPILOT_API_KEY
3133

3234
# Remote Function sandboxes (Optional)
3335
# Build a dedicated Function base before enabling JavaScript imports, Python, or Shell.

‎apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/route.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import {
2+
deleteBenchmarkContract,
3+
getBenchmarkContract,
4+
updateBenchmarkContract,
5+
} from '@/lib/api/contracts/benchmarks'
6+
import {
7+
defineInternalJsonRoute,
8+
internalOrchestrationErrorPolicy,
9+
internalRateLimits,
10+
internalSessionAuth,
11+
} from '@/lib/api/server/routes'
12+
import { deleteBenchmark, getBenchmark, updateBenchmark } from '@/lib/benchmarks/application/cases'
13+
import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
14+
import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
15+
16+
export const GET = defineInternalJsonRoute({
17+
contract: getBenchmarkContract,
18+
auth: internalSessionAuth,
19+
operation: benchmarkOperations.read,
20+
rateLimit: internalRateLimits.none({
21+
reason: 'The owner polls bounded benchmark artifacts under current source access',
22+
}),
23+
errorPolicy: internalOrchestrationErrorPolicy,
24+
beforeParse: () => requireBenchmarkEnabled(),
25+
mapInput: ({ params }) => ({ organizationId: params.id, benchmarkId: params.benchmarkId }),
26+
useCase: getBenchmark,
27+
})
28+
29+
export const PATCH = defineInternalJsonRoute({
30+
contract: updateBenchmarkContract,
31+
auth: internalSessionAuth,
32+
operation: benchmarkOperations.update,
33+
rateLimit: internalRateLimits.user({ bucketName: 'benchmark-update' }),
34+
errorPolicy: internalOrchestrationErrorPolicy,
35+
beforeParse: () => requireBenchmarkEnabled(),
36+
mapInput: ({ params, body: { version, ...patch } }) => ({
37+
organizationId: params.id,
38+
benchmarkId: params.benchmarkId,
39+
version,
40+
patch,
41+
}),
42+
useCase: updateBenchmark,
43+
})
44+
45+
export const DELETE = defineInternalJsonRoute({
46+
contract: deleteBenchmarkContract,
47+
auth: internalSessionAuth,
48+
operation: benchmarkOperations.delete,
49+
rateLimit: internalRateLimits.user({ bucketName: 'benchmark-delete' }),
50+
errorPolicy: internalOrchestrationErrorPolicy,
51+
beforeParse: () => requireBenchmarkEnabled(),
52+
mapInput: ({ params, body }) => ({
53+
organizationId: params.id,
54+
benchmarkId: params.benchmarkId,
55+
version: body.version,
56+
}),
57+
useCase: deleteBenchmark,
58+
})

‎apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/run/route.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { runBenchmarkStageContract } from '@/lib/api/contracts/benchmarks'
2+
import {
3+
defineInternalJsonRoute,
4+
internalOrchestrationErrorPolicy,
5+
internalRateLimits,
6+
internalSessionAuth,
7+
} from '@/lib/api/server/routes'
8+
import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
9+
import { runBenchmarkStage } from '@/lib/benchmarks/application/run-stage'
10+
import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
11+
12+
export const maxDuration = 660
13+
14+
export const POST = defineInternalJsonRoute({
15+
contract: runBenchmarkStageContract,
16+
auth: internalSessionAuth,
17+
operation: benchmarkOperations.run,
18+
rateLimit: internalRateLimits.user({
19+
bucketName: 'benchmark-run',
20+
config: { maxTokens: 10, refillRate: 2, refillIntervalMs: 60_000 },
21+
}),
22+
errorPolicy: internalOrchestrationErrorPolicy,
23+
beforeParse: () => requireBenchmarkEnabled(),
24+
mapInput: ({ params, body }) => ({
25+
organizationId: params.id,
26+
benchmarkId: params.benchmarkId,
27+
...body,
28+
}),
29+
useCase: runBenchmarkStage,
30+
})

‎apps/sim/app/api/organizations/[id]/benchmarks/route.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { createBenchmarkContract, listBenchmarksContract } from '@/lib/api/contracts/benchmarks'
2+
import {
3+
defineInternalJsonRoute,
4+
internalOrchestrationErrorPolicy,
5+
internalRateLimits,
6+
internalSessionAuth,
7+
} from '@/lib/api/server/routes'
8+
import { createBenchmark, listBenchmarks } from '@/lib/benchmarks/application/cases'
9+
import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
10+
import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
11+
12+
export const GET = defineInternalJsonRoute({
13+
contract: listBenchmarksContract,
14+
auth: internalSessionAuth,
15+
operation: benchmarkOperations.list,
16+
rateLimit: internalRateLimits.none({
17+
reason: 'Private benchmark metadata uses bounded pagination under current membership',
18+
}),
19+
errorPolicy: internalOrchestrationErrorPolicy,
20+
beforeParse: () => requireBenchmarkEnabled(),
21+
mapInput: ({ params, query }) => ({ organizationId: params.id, ...query }),
22+
useCase: listBenchmarks,
23+
})
24+
25+
export const POST = defineInternalJsonRoute({
26+
contract: createBenchmarkContract,
27+
auth: internalSessionAuth,
28+
operation: benchmarkOperations.create,
29+
rateLimit: internalRateLimits.user({ bucketName: 'benchmark-create' }),
30+
errorPolicy: internalOrchestrationErrorPolicy,
31+
beforeParse: () => requireBenchmarkEnabled(),
32+
mapInput: ({ params, body }) => ({ organizationId: params.id, ...body }),
33+
useCase: createBenchmark,
34+
})

‎apps/sim/app/o/[organizationId]/benchmark/benchmark.tsx‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
'use client'
2+
3+
import { Chip, ChipCombobox } from '@sim/emcn'
4+
import { Plus } from '@sim/emcn/icons'
5+
import { useQueryStates } from 'nuqs'
6+
import { HEADER_ACTION_CLUSTER, PAGE_HEADER_BAR } from '@/components/page-header-bar'
7+
import { BenchmarkDetail, CreateBenchmark } from '@/app/o/[organizationId]/benchmark/components'
8+
import {
9+
benchmarkParams,
10+
benchmarkUrlOptions,
11+
} from '@/app/o/[organizationId]/benchmark/search-params'
12+
import { useBenchmarks } from '@/hooks/queries/benchmarks'
13+
14+
interface BenchmarkProps {
15+
organizationId: string
16+
canPlan: boolean
17+
}
18+
19+
export function Benchmark({ organizationId, canPlan }: BenchmarkProps) {
20+
const [{ benchmarkId }, setParams] = useQueryStates(benchmarkParams, benchmarkUrlOptions)
21+
const benchmarks = useBenchmarks(organizationId)
22+
const records = benchmarks.data?.pages.flatMap((page) => page.benchmarks) ?? []
23+
24+
return (
25+
<div className='flex h-full flex-col bg-[var(--bg)]'>
26+
<div className={PAGE_HEADER_BAR}>
27+
<div className={HEADER_ACTION_CLUSTER} />
28+
</div>
29+
<div className='min-h-0 flex-1 overflow-y-auto [scrollbar-gutter:stable_both-edges]'>
30+
<div className='mx-auto flex w-full max-w-chat flex-col gap-7 px-6 pt-8 pb-12'>
31+
<div>
32+
<h1 className='text-[var(--text-primary)] text-lg'>Benchmark</h1>
33+
<p className='mt-1 text-[var(--text-muted)] text-small'>
34+
Measure how well a plan captures the details needed to build a workspace.
35+
</p>
36+
</div>
37+
<div className='flex flex-wrap items-center gap-2'>
38+
<ChipCombobox
39+
aria-label='Saved benchmarks'
40+
className='min-w-[200px] flex-1'
41+
value={benchmarkId}
42+
options={records.map((record) => ({ value: record.id, label: record.name }))}
43+
onChange={(value) => setParams({ benchmarkId: value })}
44+
placeholder='Saved benchmarks'
45+
searchable
46+
isLoading={benchmarks.isLoading}
47+
error={benchmarks.error?.message}
48+
/>
49+
<Chip leftIcon={Plus} onClick={() => setParams({ benchmarkId: null })}>
50+
New benchmark
51+
</Chip>
52+
{benchmarks.hasNextPage && (
53+
<Chip
54+
disabled={benchmarks.isFetchingNextPage}
55+
onClick={() => benchmarks.fetchNextPage()}
56+
>
57+
{benchmarks.isFetchingNextPage ? 'Loading…' : 'Load older'}
58+
</Chip>
59+
)}
60+
</div>
61+
{benchmarkId ? (
62+
<BenchmarkDetail
63+
key={benchmarkId}
64+
organizationId={organizationId}
65+
benchmarkId={benchmarkId}
66+
canPlan={canPlan}
67+
onDeleted={() => setParams({ benchmarkId: null }, { history: 'replace' })}
68+
/>
69+
) : (
70+
<CreateBenchmark
71+
organizationId={organizationId}
72+
onCreated={(value) => setParams({ benchmarkId: value })}
73+
/>
74+
)}
75+
</div>
76+
</div>
77+
</div>
78+
)
79+
}

0 commit comments

Comments
 (0)