feat(evals): compare agent tool-use across models

Run the same live scenarios across a list of models and write a scenario x
model matrix. models.ts resolves provider:model specs (DeepSeek, OpenAI, Groq,
OpenRouter) and reads each provider's key from <PROVIDER>_API_KEY.

- agent-tool-use.compare.live.test.ts: EVAL_MODELS x scenarios x trials
- report.ts: buildLiveComparisonReport + JSON/Markdown matrix
- report.test.ts: key-free aggregation coverage
- test:evals:compare script; README documents the spec format

Author: sudoKrishna

apps/sim/evals/README.md

@@ -75,6 +75,23 @@ example, an exact retry count). The report is at
 `test-results/evals/agent-tool-use-live.{json,md}` with pass rates, average
 iterations, latency, and the failed check names.
 
+### Compare models
+
+Run the same scenarios across several models and get a scenario × model matrix:
+
+```sh
+cd apps/sim
+EVAL_MODELS=deepseek:deepseek-chat,deepseek:deepseek-reasoner \
+  DEEPSEEK_API_KEY=... bun run test:evals:compare
+```
+
+A spec is `provider:model`; a bare model id defaults to DeepSeek. Providers are
+DeepSeek, OpenAI, Groq, and OpenRouter, each reading its key from
+`<PROVIDER>_API_KEY`. The report is
+`test-results/evals/agent-tool-use-compare.{json,md}`: per-model pass rate,
+iterations, latency, and tokens, plus a per-scenario pass-rate matrix.
+`EVAL_MIN_PASS_RATE` fails a model below a floor.
+
 ## Add a case
 
 1. Open [`agent-tool-use/scenarios.ts`](./agent-tool-use/scenarios.ts) and add

apps/sim/evals/agent-tool-use/agent-tool-use.compare.live.test.ts

@@ -0,0 +1,92 @@
+import { providersMock } from '@sim/testing/mocks/providers.mock'
+import { providersConversationHistoryMock } from '@sim/testing/mocks/providers-conversation-history.mock'
+import { providersUtilsMock } from '@sim/testing/mocks/providers-utils.mock'
+import { toolsMock } from '@sim/testing/mocks/tools.mock'
+import { afterAll, describe, expect, it, vi } from 'vitest'
+import { runScenario } from '@/evals/agent-tool-use/harness'
+import { createLiveModelCompletion, parseLiveModels } from '@/evals/agent-tool-use/models'
+import { type LiveModelRun, writeLiveComparisonReport } from '@/evals/agent-tool-use/report'
+import { AGENT_TOOL_USE_SCENARIOS } from '@/evals/agent-tool-use/scenarios'
+
+vi.mock('@/providers/conversation-history', () => providersConversationHistoryMock)
+vi.mock('@/tools', () => toolsMock)
+vi.mock('@/providers/utils', () => providersUtilsMock)
+vi.mock('@/providers', () => providersMock)
+
+/**
+ * Compare the same scenarios across several models. Opt-in, never in CI:
+ *
+ *   EVAL_LIVE=1 EVAL_MODELS=deepseek:deepseek-chat,deepseek:deepseek-reasoner \
+ *     bun run --cwd apps/sim test --mode live evals/agent-tool-use/agent-tool-use.compare.live.test.ts
+ *
+ * Each provider reads its key from `<PROVIDER>_API_KEY`; a bare model id
+ * defaults to DeepSeek. The report is a scenario × model matrix plus per-model
+ * pass rate, iterations, latency, and tokens. Set `EVAL_MIN_PASS_RATE` (0–1) to
+ * fail a model below a pass-rate floor.
+ */
+const LIVE = process.env.EVAL_LIVE === '1'
+const TRIALS = Number(process.env.EVAL_TRIALS ?? '3')
+const MIN_PASS_RATE = Number(process.env.EVAL_MIN_PASS_RATE ?? '0')
+const TIMEOUT_MS = Number(process.env.EVAL_TIMEOUT_MS ?? '180000')
+const models = parseLiveModels(process.env.EVAL_MODELS)
+const liveScenarios = AGENT_TOOL_USE_SCENARIOS.filter((scenario) => !scenario.scriptedOnly)
+
+const runs: LiveModelRun[] = []
+
+const cases = models.flatMap((spec) =>
+  liveScenarios.map((scenario) => ({
+    id: `${spec.provider.id}/${spec.model} · ${scenario.id}`,
+    spec,
+    scenario,
+  }))
+)
+
+afterAll(() => {
+  if (!LIVE || runs.length === 0) return
+  writeLiveComparisonReport(
+    runs,
+    process.env.EVAL_COMPARE_REPORT_PATH ?? 'test-results/evals/agent-tool-use-compare.json'
+  )
+})
+
+describe.skipIf(!LIVE)('agent tool-use model comparison', () => {
+  it.each(cases)(
+    '$id',
+    async ({ spec, scenario }) => {
+      const completion = createLiveModelCompletion(spec)
+      for (let trial = 0; trial < TRIALS; trial++) {
+        const result = await runScenario(scenario, {
+          completion,
+          mode: 'live',
+          model: spec.model,
+          providerName: spec.provider.label,
+        })
+        runs.push({
+          provider: spec.provider.id,
+          model: spec.model,
+          scenarioId: scenario.id,
+          result,
+        })
+      }
+    },
+    TIMEOUT_MS
+  )
+
+  it('meets the per-model pass-rate floor', () => {
+    if (MIN_PASS_RATE <= 0 || runs.length === 0) return
+    const byModel = new Map<string, { passed: number; total: number }>()
+    for (const run of runs) {
+      const key = `${run.provider}/${run.model}`
+      const entry = byModel.get(key) ?? { passed: 0, total: 0 }
+      entry.total += 1
+      if (run.result.passed) entry.passed += 1
+      byModel.set(key, entry)
+    }
+    for (const [model, entry] of byModel) {
+      expect(
+        entry.passed / entry.total,
+        `${model} passed ${entry.passed}/${entry.total}`
+      ).toBeGreaterThanOrEqual(MIN_PASS_RATE)
+    }
+  })
+})

apps/sim/evals/agent-tool-use/models.ts

@@ -0,0 +1,85 @@
+import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop'
+import { createOpenAICompatLiveCompletion } from './live'
+
+/**
+ * Provider registry for the model-comparison live suite.
+ *
+ * A model spec is `provider:model` (or a bare model id, which defaults to
+ * DeepSeek). Each provider reads its key from the matching environment
+ * variable, so adding a provider is one entry plus its key.
+ */
+export interface LiveProviderSpec {
+  id: string
+  label: string
+  baseURL?: string
+  keyEnv: string
+  baseURLEnv?: string
+}
+
+const PROVIDERS: Record<string, LiveProviderSpec> = {
+  deepseek: {
+    id: 'deepseek',
+    label: 'DeepSeek',
+    baseURL: 'https://api.deepseek.com',
+    baseURLEnv: 'DEEPSEEK_BASE_URL',
+    keyEnv: 'DEEPSEEK_API_KEY',
+  },
+  openai: { id: 'openai', label: 'OpenAI', keyEnv: 'OPENAI_API_KEY' },
+  groq: {
+    id: 'groq',
+    label: 'Groq',
+    baseURL: 'https://api.groq.com/openai/v1',
+    keyEnv: 'GROQ_API_KEY',
+  },
+  openrouter: {
+    id: 'openrouter',
+    label: 'OpenRouter',
+    baseURL: 'https://openrouter.ai/api/v1',
+    keyEnv: 'OPENROUTER_API_KEY',
+  },
+}
+
+export interface LiveModelSpec {
+  provider: LiveProviderSpec
+  model: string
+}
+
+/** Resolves `provider:model`, or a bare model id against DeepSeek. */
+export function resolveLiveModelSpec(spec: string): LiveModelSpec {
+  const separator = spec.indexOf(':')
+  if (separator === -1) {
+    return { provider: PROVIDERS.deepseek, model: spec.trim() }
+  }
+  const providerId = spec.slice(0, separator).trim().toLowerCase()
+  const provider = PROVIDERS[providerId]
+  if (!provider) {
+    throw new Error(`Unknown eval provider "${providerId}" in spec "${spec}"`)
+  }
+  return { provider, model: spec.slice(separator + 1).trim() }
+}
+
+export function parseLiveModels(value: string | undefined): LiveModelSpec[] {
+  return (value ?? 'deepseek:deepseek-chat')
+    .split(',')
+    .map((spec) => spec.trim())
+    .filter(Boolean)
+    .map(resolveLiveModelSpec)
+}
+
+/** Builds a real completion for one resolved `provider:model` spec. */
+export function createLiveModelCompletion(spec: LiveModelSpec): OpenAICompatCreateCompletion {
+  const apiKey = process.env[spec.provider.keyEnv]
+  if (!apiKey) {
+    throw new Error(`${spec.provider.keyEnv} is required for ${spec.provider.label}/${spec.model}`)
+  }
+  const baseURL = spec.provider.baseURLEnv
+    ? (process.env[spec.provider.baseURLEnv] ?? spec.provider.baseURL)
+    : spec.provider.baseURL
+
+  return createOpenAICompatLiveCompletion({
+    apiKey,
+    model: spec.model,
+    ...(baseURL ? { baseURL } : {}),
+    ...(process.env.EVAL_TIMEOUT_MS ? { timeoutMs: Number(process.env.EVAL_TIMEOUT_MS) } : {}),
+  })
+}

apps/sim/evals/agent-tool-use/report.test.ts

@@ -0,0 +1,71 @@
+import { existsSync, mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import {
+  buildLiveComparisonReport,
+  type LiveModelRun,
+  writeLiveComparisonReport,
+} from '@/evals/agent-tool-use/report'
+import type { AgentToolUseResult } from '@/evals/agent-tool-use/types'
+
+function result(passed: boolean): AgentToolUseResult {
+  return {
+    id: 'scenario',
+    name: 'scenario',
+    category: 'recovery',
+    passed,
+    checks: [],
+    finalContent: '',
+    toolInvocations: [],
+    metrics: {
+      iterations: 1,
+      toolCalls: 1,
+      successfulToolCalls: 1,
+      erroredToolCalls: 0,
+      latencyMs: 10,
+      modelTimeMs: 0,
+      toolsTimeMs: 0,
+      firstResponseTimeMs: 0,
+      inputTokens: 1,
+      outputTokens: 1,
+      totalTokens: 2,
+    },
+  }
+}
+
+function run(model: string, passed: boolean): LiveModelRun {
+  return { provider: 'deepseek', model, scenarioId: 'scenario', result: result(passed) }
+}
+
+describe('model comparison report', () => {
+  it('groups runs by model, sorts by pass rate, and fills the scenario matrix', () => {
+    const report = buildLiveComparisonReport([
+      run('chat', true),
+      run('chat', true),
+      run('reasoner', false),
+      run('reasoner', true),
+    ])
+
+    expect(report.models.map((model) => model.label)).toEqual([
+      'deepseek/chat',
+      'deepseek/reasoner',
+    ])
+    expect(report.models[0]).toMatchObject({ passRate: 1, passed: 2, trials: 2 })
+    expect(report.models[1]).toMatchObject({ passRate: 0.5, passed: 1, trials: 2 })
+    expect(report.matrix.scenario['deepseek/chat']).toBe(1)
+    expect(report.matrix.scenario['deepseek/reasoner']).toBe(0.5)
+  })
+
+  it('writes JSON and Markdown reports', () => {
+    const directory = mkdtempSync(join(tmpdir(), 'eval-compare-'))
+    try {
+      const path = join(directory, 'compare.json')
+      writeLiveComparisonReport([run('chat', true)], path)
+      expect(existsSync(path)).toBe(true)
+      expect(existsSync(join(directory, 'compare.md'))).toBe(true)
+    } finally {
+      rmSync(directory, { recursive: true, force: true })
+    }
+  })
+})

apps/sim/evals/agent-tool-use/report.ts

@@ -127,3 +127,123 @@ export function writeLiveEvalReport(summaries: LiveScenarioSummary[], reportPath
   writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`)
   writeFileSync(reportPath.replace(/\.json$/, '.md'), renderLiveMarkdown(report))
 }
+
+/** One trial of one model on one scenario, for the comparison report. */
+export interface LiveModelRun {
+  provider: string
+  model: string
+  scenarioId: string
+  result: AgentToolUseResult
+}
+
+/** Aggregate behavior of one model across the suite. */
+export interface LiveModelSummary {
+  provider: string
+  model: string
+  label: string
+  trials: number
+  passed: number
+  passRate: number
+  avgIterations: number
+  avgLatencyMs: number
+  totalTokens: number
+}
+
+/** Side-by-side comparison of several models on the same scenarios. */
+export interface LiveComparisonReport {
+  suite: 'agent-tool-use-compare'
+  generatedAt: string
+  scenarios: string[]
+  models: LiveModelSummary[]
+  /** scenarioId -> model label -> pass rate. */
+  matrix: Record<string, Record<string, number>>
+}
+
+function modelLabel(run: LiveModelRun): string {
+  return `${run.provider}/${run.model}`
+}
+
+export function buildLiveComparisonReport(runs: LiveModelRun[]): LiveComparisonReport {
+  const byModel = new Map<string, LiveModelRun[]>()
+  for (const run of runs) {
+    const key = modelLabel(run)
+    const list = byModel.get(key) ?? []
+    list.push(run)
+    byModel.set(key, list)
+  }
+
+  const scenarios = [...new Set(runs.map((run) => run.scenarioId))].sort()
+  const models: LiveModelSummary[] = []
+  const matrix: Record<string, Record<string, number>> = {}
+
+  for (const [label, modelRuns] of byModel) {
+    const results = modelRuns.map((run) => run.result)
+    const passed = results.filter((result) => result.passed).length
+    const [provider, model] = label.split('/')
+    models.push({
+      provider,
+      model,
+      label,
+      trials: results.length,
+      passed,
+      passRate: results.length === 0 ? 0 : passed / results.length,
+      avgIterations: average(results.map((result) => result.metrics.iterations)),
+      avgLatencyMs: average(results.map((result) => result.metrics.latencyMs)),
+      totalTokens: results.reduce((sum, result) => sum + result.metrics.totalTokens, 0),
+    })
+
+    for (const scenario of scenarios) {
+      const scenarioRuns = modelRuns.filter((run) => run.scenarioId === scenario)
+      const scenarioPassed = scenarioRuns.filter((run) => run.result.passed).length
+      matrix[scenario] ??= {}
+      matrix[scenario][label] = scenarioRuns.length === 0 ? 0 : scenarioPassed / scenarioRuns.length
+    }
+  }
+
+  models.sort((a, b) => b.passRate - a.passRate)
+  return {
+    suite: 'agent-tool-use-compare',
+    generatedAt: new Date().toISOString(),
+    scenarios,
+    models,
+    matrix,
+  }
+}
+
+function renderComparisonMarkdown(report: LiveComparisonReport): string {
+  const labels = report.models.map((model) => model.label)
+  const lines = [
+    '# Agent tool-use model comparison',
+    '',
+    `Generated: ${report.generatedAt}`,
+    '',
+    '| Model | Pass rate | Trials | Avg iterations | Avg latency | Tokens |',
+    '| --- | ---: | ---: | ---: | ---: | ---: |',
+    ...report.models.map(
+      (model) =>
+        `| ${model.label} | ${(model.passRate * 100).toFixed(0)}% (${model.passed}/${model.trials}) | ${model.trials} | ${model.avgIterations.toFixed(1)} | ${Math.round(model.avgLatencyMs)}ms | ${model.totalTokens} |`
+    ),
+    '',
+    `| Scenario | ${labels.join(' | ')} |`,
+    `| --- | ${labels.map(() => '---:').join(' | ')} |`,
+    ...report.scenarios.map(
+      (scenario) =>
+        `| ${escapeCell(scenario)} | ${labels
+          .map((label) => `${((report.matrix[scenario]?.[label] ?? 0) * 100).toFixed(0)}%`)
+          .join(' | ')} |`
+    ),
+    '',
+  ]
+  return lines.join('\n')
+}
+
+/**
+ * Writes the model-comparison JSON report and a sibling Markdown table.
+ * Unlike the single-model report, this is a matrix: scenario × model pass rates.
+ */
+export function writeLiveComparisonReport(runs: LiveModelRun[], reportPath: string): void {
+  const report = buildLiveComparisonReport(runs)
+  mkdirSync(dirname(reportPath), { recursive: true })
+  writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`)
+  writeFileSync(reportPath.replace(/\.json$/, '.md'), renderComparisonMarkdown(report))
+}