Add superuser benchmark console with selected-user execution

Author: icecrasher321

apps/sim/app/api/benchmarks/availability/route.ts

@@ -0,0 +1,19 @@
+import { benchmarkAvailabilityContract } from '@/lib/api/contracts/benchmarks'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
+import { benchmarkAvailability } from '@/lib/benchmarks/application/selection'
+
+export const GET = defineInternalJsonRoute({
+  contract: benchmarkAvailabilityContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.availability,
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-selection' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  mapInput: () => ({}),
+  useCase: benchmarkAvailability,
+})

apps/sim/app/api/benchmarks/organizations/[id]/users/route.ts

@@ -0,0 +1,19 @@
+import { benchmarkUsersContract } from '@/lib/api/contracts/benchmarks'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
+import { listBenchmarkUsers } from '@/lib/benchmarks/application/selection'
+
+export const GET = defineInternalJsonRoute({
+  contract: benchmarkUsersContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.users,
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-selection' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  mapInput: ({ params, query }) => ({ organizationId: params.id, ...query }),
+  useCase: listBenchmarkUsers,
+})

apps/sim/app/api/benchmarks/organizations/[id]/workspaces/route.ts

@@ -0,0 +1,19 @@
+import { benchmarkWorkspacesContract } from '@/lib/api/contracts/benchmarks'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
+import { listBenchmarkWorkspaces } from '@/lib/benchmarks/application/selection'
+
+export const GET = defineInternalJsonRoute({
+  contract: benchmarkWorkspacesContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.workspaces,
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-selection' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  mapInput: ({ params, query }) => ({ organizationId: params.id, ...query }),
+  useCase: listBenchmarkWorkspaces,
+})

apps/sim/app/api/benchmarks/organizations/route.ts

@@ -0,0 +1,19 @@
+import { benchmarkOrganizationsContract } from '@/lib/api/contracts/benchmarks'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
+import { listBenchmarkOrganizations } from '@/lib/benchmarks/application/selection'
+
+export const GET = defineInternalJsonRoute({
+  contract: benchmarkOrganizationsContract,
+  auth: internalSessionAuth,
+  operation: benchmarkOperations.organizations,
+  rateLimit: internalRateLimits.user({ bucketName: 'benchmark-selection' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  mapInput: ({ query }) => query,
+  useCase: listBenchmarkOrganizations,
+})

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/route.ts

@@ -9,9 +9,9 @@ import {
   internalRateLimits,
   internalSessionAuth,
 } from '@/lib/api/server/routes'
+import { requireBenchmarkOperator } from '@/lib/benchmarks/application/access'
 import { deleteBenchmark, getBenchmark, updateBenchmark } from '@/lib/benchmarks/application/cases'
 import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
-import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
 
 export const GET = defineInternalJsonRoute({
   contract: getBenchmarkContract,
@@ -21,7 +21,9 @@ export const GET = defineInternalJsonRoute({
     reason: 'The owner polls bounded benchmark artifacts under current source access',
   }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params }) => ({ organizationId: params.id, benchmarkId: params.benchmarkId }),
   useCase: getBenchmark,
 })
@@ -32,7 +34,9 @@ export const PATCH = defineInternalJsonRoute({
   operation: benchmarkOperations.update,
   rateLimit: internalRateLimits.user({ bucketName: 'benchmark-update' }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params, body: { version, ...patch } }) => ({
     organizationId: params.id,
     benchmarkId: params.benchmarkId,
@@ -48,7 +52,9 @@ export const DELETE = defineInternalJsonRoute({
   operation: benchmarkOperations.delete,
   rateLimit: internalRateLimits.user({ bucketName: 'benchmark-delete' }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params, body }) => ({
     organizationId: params.id,
     benchmarkId: params.benchmarkId,

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/run/route.ts

@@ -5,9 +5,9 @@ import {
   internalRateLimits,
   internalSessionAuth,
 } from '@/lib/api/server/routes'
+import { requireBenchmarkOperator } from '@/lib/benchmarks/application/access'
 import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
 import { runBenchmarkStage } from '@/lib/benchmarks/application/run-stage'
-import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
 
 export const maxDuration = 660
 
@@ -20,7 +20,9 @@ export const POST = defineInternalJsonRoute({
     config: { maxTokens: 10, refillRate: 2, refillIntervalMs: 60_000 },
   }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params, body }) => ({
     organizationId: params.id,
     benchmarkId: params.benchmarkId,

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/runs/[runId]/route.ts

@@ -5,17 +5,19 @@ import {
   internalRateLimits,
   internalSessionAuth,
 } from '@/lib/api/server/routes'
+import { requireBenchmarkOperator } from '@/lib/benchmarks/application/access'
 import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
 import { getBenchmarkRun, reviewBenchmarkRun } from '@/lib/benchmarks/application/runs'
-import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
 
 export const GET = defineInternalJsonRoute({
   contract: getBenchmarkRunContract,
   auth: internalSessionAuth,
   operation: benchmarkOperations.readRun,
   rateLimit: internalRateLimits.none({ reason: 'Owner-only bounded benchmark run snapshot' }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params }) => ({
     organizationId: params.id,
     benchmarkId: params.benchmarkId,
@@ -30,7 +32,9 @@ export const PATCH = defineInternalJsonRoute({
   operation: benchmarkOperations.reviewRun,
   rateLimit: internalRateLimits.user({ bucketName: 'benchmark-review' }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params, body }) => ({
     organizationId: params.id,
     benchmarkId: params.benchmarkId,

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/runs/route.ts

@@ -5,17 +5,19 @@ import {
   internalRateLimits,
   internalSessionAuth,
 } from '@/lib/api/server/routes'
+import { requireBenchmarkOperator } from '@/lib/benchmarks/application/access'
 import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
 import { listBenchmarkRuns } from '@/lib/benchmarks/application/runs'
-import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
 
 export const GET = defineInternalJsonRoute({
   contract: listBenchmarkRunsContract,
   auth: internalSessionAuth,
   operation: benchmarkOperations.listRuns,
   rateLimit: internalRateLimits.none({ reason: 'Owner-only paginated benchmark run summaries' }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params, query }) => ({
     organizationId: params.id,
     benchmarkId: params.benchmarkId,

apps/sim/app/api/organizations/[id]/benchmarks/route.ts

@@ -5,9 +5,9 @@ import {
   internalRateLimits,
   internalSessionAuth,
 } from '@/lib/api/server/routes'
+import { requireBenchmarkOperator } from '@/lib/benchmarks/application/access'
 import { createBenchmark, listBenchmarks } from '@/lib/benchmarks/application/cases'
 import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
-import { requireBenchmarkEnabled } from '@/lib/benchmarks/config'
 
 export const GET = defineInternalJsonRoute({
   contract: listBenchmarksContract,
@@ -17,7 +17,9 @@ export const GET = defineInternalJsonRoute({
     reason: 'Private benchmark metadata uses bounded pagination under current membership',
   }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params, query }) => ({ organizationId: params.id, ...query }),
   useCase: listBenchmarks,
 })
@@ -28,7 +30,9 @@ export const POST = defineInternalJsonRoute({
   operation: benchmarkOperations.create,
   rateLimit: internalRateLimits.user({ bucketName: 'benchmark-create' }),
   errorPolicy: internalOrchestrationErrorPolicy,
-  beforeParse: () => requireBenchmarkEnabled(),
+  beforeParse: async ({ principal }) => {
+    await requireBenchmarkOperator(principal)
+  },
   mapInput: ({ params, body }) => ({ organizationId: params.id, ...body }),
   useCase: createBenchmark,
 })

apps/sim/app/benchmark/benchmark-console.tsx

@@ -0,0 +1,134 @@
+'use client'
+
+import { useState } from 'react'
+import { Chip, ChipCombobox, ChipLink } from '@sim/emcn'
+import { useQueryStates } from 'nuqs'
+import {
+  benchmarkConsoleParams,
+  benchmarkConsoleUrlOptions,
+  emptyBenchmarkSelection,
+} from '@/app/benchmark/search-params'
+import { Benchmark } from '@/app/o/[organizationId]/benchmark/benchmark'
+import {
+  useBenchmarkOrganizations,
+  useBenchmarkUsers,
+  useBenchmarkWorkspaces,
+} from '@/hooks/queries/benchmarks'
+import { useDebounce } from '@/hooks/use-debounce'
+
+export function BenchmarkConsole() {
+  const [{ organizationId, runAsUserId }, setParams] = useQueryStates(
+    benchmarkConsoleParams,
+    benchmarkConsoleUrlOptions
+  )
+  const [organizationSearch, setOrganizationSearch] = useState('')
+  const [userSearch, setUserSearch] = useState('')
+  const organizations = useBenchmarkOrganizations(
+    useDebounce(organizationSearch, 250),
+    organizationId
+  )
+  const users = useBenchmarkUsers(organizationId, useDebounce(userSearch, 250), runAsUserId)
+  const workspaces = useBenchmarkWorkspaces(organizationId, runAsUserId)
+  const selectedOrganization = organizations.data?.pages[0]?.selected
+  const organizationOptions = [
+    ...(selectedOrganization ? [selectedOrganization] : []),
+    ...(organizations.data?.pages
+      .flatMap((page) => page.organizations)
+      .filter((org) => org.id !== selectedOrganization?.id) ?? []),
+  ]
+  const selectedUser = users.data?.pages[0]?.selected
+  const userOptions = [
+    ...(selectedUser ? [selectedUser] : []),
+    ...(users.data?.pages
+      .flatMap((page) => page.users)
+      .filter((user) => user.id !== selectedUser?.id) ?? []),
+  ]
+  const error = organizations.error ?? users.error ?? workspaces.error
+
+  return (
+    <main className='workspace-root min-h-screen bg-[var(--bg)] text-[var(--text-body)]'>
+      <div className='mx-auto flex w-full max-w-chat flex-col gap-5 px-6 pt-8'>
+        <div className='flex items-center justify-between gap-4'>
+          <h1 className='text-[var(--text-primary)] text-lg'>Benchmarks</h1>
+          <ChipLink href='/account/settings/admin'>Back to settings</ChipLink>
+        </div>
+        <p className='text-[var(--text-muted)] text-small'>
+          Run a benchmark with a user’s enterprise access. Your admin session stays unchanged.
+        </p>
+        <div className='grid gap-4 sm:grid-cols-2'>
+          <div className='flex flex-col gap-2'>
+            <label htmlFor='benchmark-organization' className='text-small'>
+              Organization
+            </label>
+            <ChipCombobox
+              id='benchmark-organization'
+              aria-label='Organization'
+              value={organizationId}
+              options={organizationOptions.map((org) => ({ value: org.id, label: org.name }))}
+              placeholder='Select an organization'
+              searchable
+              isLoading={organizations.isLoading}
+              onSearchChange={setOrganizationSearch}
+              onChange={(value) =>
+                setParams({ organizationId: value, runAsUserId: null, ...emptyBenchmarkSelection })
+              }
+            />
+            {organizations.hasNextPage && (
+              <Chip
+                disabled={organizations.isFetchingNextPage}
+                onClick={() => organizations.fetchNextPage()}
+              >
+                Load more organizations
+              </Chip>
+            )}
+          </div>
+          <div className='flex flex-col gap-2'>
+            <label htmlFor='benchmark-user' className='text-small'>
+              Run as user
+            </label>
+            <ChipCombobox
+              key={organizationId}
+              id='benchmark-user'
+              aria-label='Run as user'
+              value={runAsUserId}
+              options={userOptions.map((user) => ({
+                value: user.id,
+                label: `${user.name} (${user.email})`,
+              }))}
+              placeholder='Select a user'
+              searchable
+              isLoading={users.isLoading}
+              disabled={!organizationId}
+              onSearchChange={setUserSearch}
+              onChange={(value) => setParams({ runAsUserId: value, ...emptyBenchmarkSelection })}
+            />
+            {users.hasNextPage && (
+              <Chip disabled={users.isFetchingNextPage} onClick={() => users.fetchNextPage()}>
+                Load more users
+              </Chip>
+            )}
+          </div>
+        </div>
+        {runAsUserId && (
+          <p className='text-[var(--text-muted)] text-small'>
+            Running as {selectedUser?.email ?? runAsUserId}. Normal permissions and billing for this
+            organization apply. Reports are private to you.
+          </p>
+        )}
+        {error && (
+          <p role='alert' className='text-[var(--text-error)] text-small'>
+            {error.message}
+          </p>
+        )}
+      </div>
+      {organizationId && runAsUserId && workspaces.data && !workspaces.error && (
+        <Benchmark
+          key={`${organizationId}:${runAsUserId}`}
+          organizationId={organizationId}
+          runAsUserId={runAsUserId}
+          canPlan={workspaces.data.pages[0]?.canPlan ?? false}
+        />
+      )}
+    </main>
+  )
+}