Add workspace specification benchmarks with isolated Plan evaluation

Author: Vikhyath Mondreti

apps/sim/.env.example

@@ -28,6 +28,8 @@ NEXT_PUBLIC_APP_URL=http://localhost:3000
 # COPILOT_API_KEY=                      # Mint one at https://sim.ai. Without it the Sim Chat block, prompt jobs, and Inbox cannot run
 # MSHIP_SYSPROMPT_OVERRIDE=             # Highest-priority instructions for Mothership; honored only when the validated API key owner is enterprise
 # NEXT_PUBLIC_CHAT_DISABLED=true        # Hides the Chat module: the workspace lands on your first workflow, and the chats list, scheduled tasks, and editor Chat panel are absent. Chat is shown when unset; `bun run setup` sets this for you if you skip the chat key
+# MOTHERSHIP_BENCHMARK_ENABLED=true      # Shows the organization Benchmark tab and enables its authenticated APIs
+# COPILOT_DEV_URL=http://localhost:8080  # Benchmark worker endpoint; requires the worker benchmark profile and the normal COPILOT_API_KEY
 
 # Remote Function sandboxes (Optional)
 # Build a dedicated Function base before enabling JavaScript imports, Python, or Shell.

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/route.ts

@@ -0,0 +1,58 @@
+import {
+  deleteBenchmarkContract,
+  getBenchmarkContract,
+  updateBenchmarkContract,
+} from '@/lib/api/contracts/benchmarks'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import { deleteBenchmark, getBenchmark, updateBenchmark } from '@/lib/benchmarks/application/cases'
+import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
+import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
+
+export const GET = defineInternalJsonRoute({
+  contract: getBenchmarkContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.read,
+  rateLimit: internalRateLimits.none({
+    reason: 'The owner polls bounded benchmark artifacts under current source access',
+  }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  beforeParse: () => requireBenchmarkEnabled(),
+  mapInput: ({ params }) => ({ organizationId: params.id, benchmarkId: params.benchmarkId }),
+  useCase: getBenchmark,
+})
+
+export const PATCH = defineInternalJsonRoute({
+  contract: updateBenchmarkContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.update,
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-update' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  beforeParse: () => requireBenchmarkEnabled(),
+  mapInput: ({ params, body: { version, ...patch } }) => ({
+    organizationId: params.id,
+    benchmarkId: params.benchmarkId,
+    version,
+    patch,
+  }),
+  useCase: updateBenchmark,
+})
+
+export const DELETE = defineInternalJsonRoute({
+  contract: deleteBenchmarkContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.delete,
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-delete' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  beforeParse: () => requireBenchmarkEnabled(),
+  mapInput: ({ params, body }) => ({
+    organizationId: params.id,
+    benchmarkId: params.benchmarkId,
+    version: body.version,
+  }),
+  useCase: deleteBenchmark,
+})

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/run/route.ts

@@ -0,0 +1,30 @@
+import { runBenchmarkStageContract } from '@/lib/api/contracts/benchmarks'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
+import { runBenchmarkStage } from '@/lib/benchmarks/application/run-stage'
+import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
+
+export const maxDuration = 660
+
+export const POST = defineInternalJsonRoute({
+  contract: runBenchmarkStageContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.run,
+  rateLimit: internalRateLimits.user({
+    bucketName: 'benchmark-run',
+    config: { maxTokens: 10, refillRate: 2, refillIntervalMs: 60_000 },
+  }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  beforeParse: () => requireBenchmarkEnabled(),
+  mapInput: ({ params, body }) => ({
+    organizationId: params.id,
+    benchmarkId: params.benchmarkId,
+    ...body,
+  }),
+  useCase: runBenchmarkStage,
+})

apps/sim/app/api/organizations/[id]/benchmarks/route.ts

@@ -0,0 +1,34 @@
+import { createBenchmarkContract, listBenchmarksContract } from '@/lib/api/contracts/benchmarks'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import { createBenchmark, listBenchmarks } from '@/lib/benchmarks/application/cases'
+import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
+import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
+
+export const GET = defineInternalJsonRoute({
+  contract: listBenchmarksContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.list,
+  rateLimit: internalRateLimits.none({
+    reason: 'Private benchmark metadata uses bounded pagination under current membership',
+  }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  beforeParse: () => requireBenchmarkEnabled(),
+  mapInput: ({ params, query }) => ({ organizationId: params.id, ...query }),
+  useCase: listBenchmarks,
+})
+
+export const POST = defineInternalJsonRoute({
+  contract: createBenchmarkContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.create,
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-create' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  beforeParse: () => requireBenchmarkEnabled(),
+  mapInput: ({ params, body }) => ({ organizationId: params.id, ...body }),
+  useCase: createBenchmark,
+})

apps/sim/app/o/[organizationId]/benchmark/benchmark.tsx

@@ -0,0 +1,79 @@
+'use client'
+
+import { Chip, ChipCombobox } from '@sim/emcn'
+import { Plus } from '@sim/emcn/icons'
+import { useQueryStates } from 'nuqs'
+import { HEADER_ACTION_CLUSTER, PAGE_HEADER_BAR } from '@/components/page-header-bar'
+import { BenchmarkDetail, CreateBenchmark } from '@/app/o/[organizationId]/benchmark/components'
+import {
+  benchmarkParams,
+  benchmarkUrlOptions,
+} from '@/app/o/[organizationId]/benchmark/search-params'
+import { useBenchmarks } from '@/hooks/queries/benchmarks'
+
+interface BenchmarkProps {
+  organizationId: string
+  canPlan: boolean
+}
+
+export function Benchmark({ organizationId, canPlan }: BenchmarkProps) {
+  const [{ benchmarkId }, setParams] = useQueryStates(benchmarkParams, benchmarkUrlOptions)
+  const benchmarks = useBenchmarks(organizationId)
+  const records = benchmarks.data?.pages.flatMap((page) => page.benchmarks) ?? []
+
+  return (
+    <div className='flex h-full flex-col bg-[var(--bg)]'>
+      <div className={PAGE_HEADER_BAR}>
+        <div className={HEADER_ACTION_CLUSTER} />
+      </div>
+      <div className='min-h-0 flex-1 overflow-y-auto [scrollbar-gutter:stable_both-edges]'>
+        <div className='mx-auto flex w-full max-w-chat flex-col gap-7 px-6 pt-8 pb-12'>
+          <div>
+            <h1 className='text-[var(--text-primary)] text-lg'>Benchmark</h1>
+            <p className='mt-1 text-[var(--text-muted)] text-small'>
+              Measure how well a plan captures the details needed to build a workspace.
+            </p>
+          </div>
+          <div className='flex flex-wrap items-center gap-2'>
+            <ChipCombobox
+              aria-label='Saved benchmarks'
+              className='min-w-[200px] flex-1'
+              value={benchmarkId}
+              options={records.map((record) => ({ value: record.id, label: record.name }))}
+              onChange={(value) => setParams({ benchmarkId: value })}
+              placeholder='Saved benchmarks'
+              searchable
+              isLoading={benchmarks.isLoading}
+              error={benchmarks.error?.message}
+            />
+            <Chip leftIcon={Plus} onClick={() => setParams({ benchmarkId: null })}>
+              New benchmark
+            </Chip>
+            {benchmarks.hasNextPage && (
+              <Chip
+                disabled={benchmarks.isFetchingNextPage}
+                onClick={() => benchmarks.fetchNextPage()}
+              >
+                {benchmarks.isFetchingNextPage ? 'Loading…' : 'Load older'}
+              </Chip>
+            )}
+          </div>
+          {benchmarkId ? (
+            <BenchmarkDetail
+              key={benchmarkId}
+              organizationId={organizationId}
+              benchmarkId={benchmarkId}
+              canPlan={canPlan}
+              onDeleted={() => setParams({ benchmarkId: null }, { history: 'replace' })}
+            />
+          ) : (
+            <CreateBenchmark
+              organizationId={organizationId}
+              onCreated={(value) => setParams({ benchmarkId: value })}
+            />
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}