fix(stats): use capability fallbacks in comparison radar (anomalyco#51161)

Author: adamdotdevin

.github/actions/setup-bun/action.yml

@@ -15,12 +15,15 @@ runs:
       with:
         node-version: "24"
 
-    - name: Get baseline download URL
+    - name: Get Bun version and baseline download URL
       id: bun-url
       shell: bash
       run: |
+        V=$(node -p "require('./package.json').packageManager.split('@')[1]")
+        # Bun 1.4.2 includes the patched peer-variant fix in oven-sh/bun#33646.
+        if [ "$RUNNER_OS" = "Windows" ]; then V=1.4.2; fi
+        echo "version=$V" >> "$GITHUB_OUTPUT"
         if [ "$RUNNER_ARCH" = "X64" ]; then
-          V=$(node -p "require('./package.json').packageManager.split('@')[1]")
           case "$RUNNER_OS" in
             macOS)   OS=darwin ;;
             Linux)   OS=linux ;;
@@ -30,9 +33,10 @@ runs:
         fi
 
     - name: Setup Bun
+      id: setup-bun
       uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
       with:
-        bun-version-file: ${{ !steps.bun-url.outputs.url && 'package.json' || '' }}
+        bun-version: ${{ !steps.bun-url.outputs.url && steps.bun-url.outputs.version || '' }}
         bun-download-url: ${{ steps.bun-url.outputs.url }}
 
     - name: Get cache directory
@@ -45,29 +49,21 @@ runs:
       uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:
         path: ${{ steps.cache.outputs.dir }}
-        key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+        key: ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-${{ hashFiles('**/bun.lock') }}
         restore-keys: |
-          ${{ runner.os }}-bun-
+          ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-
 
     - name: Install setuptools for distutils compatibility
       run: python3 -m pip install setuptools || pip install setuptools || true
       shell: bash
 
     - name: Install dependencies
-      run: |
-        # Workaround for patched peer variants
-        # e.g. ./patches/ for standard-openapi
-        # https://github.com/oven-sh/bun/issues/28147
-        if [ "$RUNNER_OS" = "Windows" ]; then
-          bun install --linker hoisted ${{ inputs.install-flags }}
-        else
-          bun install ${{ inputs.install-flags }}
-        fi
+      run: bun install ${{ inputs.install-flags }}
       shell: bash
 
     - name: Save Bun dependencies
       if: steps.bun-cache.outputs.cache-hit != 'true' && github.event_name != 'pull_request' && github.event_name != 'pull_request_target'
       uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:
         path: ${{ steps.cache.outputs.dir }}
-        key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+        key: ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-${{ hashFiles('**/bun.lock') }}

packages/stats/app/src/routes/compare-radar.tsx

@@ -22,6 +22,12 @@ type RadarAxis = {
   label: string
   description: string
   score: (model: ModelCatalogEntry) => number | undefined
+  capability?: "reasoning" | "toolCall"
+}
+
+type RadarScore = {
+  value: number
+  fallback?: string
 }
 
 type RadarPoint = {
@@ -37,7 +43,7 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
       name: model.name,
       labName: model.labName,
       color: radarColors[index % radarColors.length],
-      scores: axes().map((axis) => (model.catalog ? axis.score(model.catalog) : undefined)),
+      scores: axes().map((axis) => resolveRadarScore(axis, model.catalog)),
     })),
   )
   const accessibleDescription = createMemo(() =>
@@ -62,6 +68,9 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
               <span>
                 <strong>{model.name}</strong>
                 <Show when={model.labName}>{(name) => <small>{name()}</small>}</Show>
+                <Show when={model.scores.some((score) => score.fallback)}>
+                  <small data-slot="compare-radar-coverage">Hollow points use fallbacks · hover for details</small>
+                </Show>
               </span>
             </li>
           )}
@@ -89,10 +98,16 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
                   <polygon data-slot="compare-radar-area" points={radarSeriesPolygon(model.scores)} />
                   <For each={model.scores}>
                     {(score, index) => {
-                      const point = () => radarPoint(index(), axes().length, score ?? 0)
+                      const point = () => radarPoint(index(), axes().length, score.value)
                       return (
                         <>
-                          <circle data-slot="compare-radar-point" cx={point().x} cy={point().y} r="0.95" />
+                          <circle
+                            data-slot="compare-radar-point"
+                            data-fallback={score.fallback ? "true" : undefined}
+                            cx={point().x}
+                            cy={point().y}
+                            r="0.95"
+                          />
                           <circle
                             data-slot="compare-radar-point-hit"
                             cx={point().x}
@@ -138,6 +153,13 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
           >
             <strong>{axes()[activeAxis() ?? 0]?.label}</strong>
             <p>{axes()[activeAxis() ?? 0]?.description}</p>
+            <For each={series()}>
+              {(model) => (
+                <p>
+                  {model.name}: {formatRadarScore(model.scores[activeAxis() ?? 0])}
+                </p>
+              )}
+            </For>
           </div>
         </Show>
       </div>
@@ -166,7 +188,7 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
   )
 }
 
-function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[] {
+export function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[] {
   const benchmarks = benchmarkScoreGroups(catalogModels)
   const toolUseBenchmarks = benchmarkScoreGroups(catalogModels, true)
   const costs = catalogModels.flatMap((model) => {
@@ -180,9 +202,10 @@ function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[
   return [
     {
       label: "Reasoning",
-      description: "Ability to solve complex, multi-step problems. Based on reasoning benchmarks when available.",
-      score: (model) =>
-        benchmarkPercentile(model, benchmarks, reasoningBenchmarkPattern) ?? (model.reasoning ? 100 : 0),
+      capability: "reasoning",
+      description:
+        "Ability to solve complex, multi-step problems. Benchmarks take priority; reasoning support defaults to 50/100.",
+      score: (model) => benchmarkPercentile(model, benchmarks, reasoningBenchmarkPattern),
     },
     {
       label: "Coding",
@@ -218,7 +241,8 @@ function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[
     },
     {
       label: "Tool use",
-      description: "Performance on agent benchmarks including Terminal-Bench, Tau3, and Claw-Eval.",
+      capability: "toolCall",
+      description: "Agent benchmark performance. Benchmarks take priority; tool calling support defaults to 50/100.",
       score: (model) =>
         benchmarkPercentile(model, toolUseBenchmarks, toolUseBenchmarkPattern, {
           aggregate: "average",
@@ -324,9 +348,9 @@ function radarPolygonPoints(count: number, score: number) {
     .join(" ")
 }
 
-function radarSeriesPolygon(scores: (number | undefined)[]) {
+function radarSeriesPolygon(scores: RadarScore[]) {
   return scores
-    .map((score, index) => radarPoint(index, scores.length, score ?? 0))
+    .map((score, index) => radarPoint(index, scores.length, score.value))
     .map((point) => `${point.x},${point.y}`)
     .join(" ")
 }
@@ -355,6 +379,17 @@ function roundRadarCoordinate(value: number) {
   return Math.round(value * 1000) / 1000
 }
 
-function formatRadarScore(score: number | undefined) {
-  return score === undefined ? "No data" : `${Math.round(score)}/100`
+function formatRadarScore(score: RadarScore) {
+  return `${Math.round(score.value)}/100${score.fallback ? ` — ${score.fallback}` : ""}`
+}
+
+export function resolveRadarScore(axis: RadarAxis, model: ModelCatalogEntry | null): RadarScore {
+  const score = model ? axis.score(model) : undefined
+  if (score !== undefined) return { value: score }
+  const supported = axis.capability ? model?.[axis.capability] : undefined
+  if (supported === undefined) return { value: 50, fallback: "No data; neutral placeholder" }
+  const capability = axis.capability === "toolCall" ? "Tool calling" : "Reasoning"
+  return supported
+    ? { value: 50, fallback: `${capability} supported; no comparable benchmark` }
+    : { value: 0, fallback: `${capability} not supported` }
 }

packages/stats/app/src/routes/index.css

@@ -6291,8 +6291,7 @@ body {
   vector-effect: non-scaling-stroke;
 }
 
-[data-page="stats"] [data-slot="compare-radar-area"],
-[data-page="stats"] [data-slot="compare-radar-line"] {
+[data-page="stats"] [data-slot="compare-radar-area"] {
   stroke: currentColor;
   stroke-width: 1.5px;
   stroke-linejoin: round;
@@ -6304,17 +6303,17 @@ body {
   fill-opacity: 0.09;
 }
 
-[data-page="stats"] [data-slot="compare-radar-line"] {
-  fill: none;
-}
-
 [data-page="stats"] [data-slot="compare-radar-point"] {
   fill: currentColor;
   stroke: currentColor;
   stroke-width: 1px;
   vector-effect: non-scaling-stroke;
 }
 
+[data-page="stats"] [data-slot="compare-radar-point"][data-fallback="true"] {
+  fill: var(--stats-bg);
+}
+
 [data-page="stats"] [data-slot="compare-radar-point-hit"] {
   fill: transparent;
   cursor: pointer;

packages/stats/app/src/routes/model-catalog.ts

@@ -25,8 +25,8 @@ export type ModelCatalogEntry = {
   limit?: { context?: number; output?: number }
   modalities: { input: string[]; output: string[] }
   openWeights: boolean
-  reasoning: boolean
-  toolCall: boolean
+  reasoning?: boolean
+  toolCall?: boolean
   attachment: boolean
   temperature: boolean
   cost?: ModelCatalogCost
@@ -54,6 +54,7 @@ export type ModelCatalogLab = {
 
 export type ModelCatalog = {
   models: ModelCatalogEntry[]
+  aliases?: ModelCatalogEntry[]
   labs: ModelCatalogLab[]
 }
 
@@ -80,7 +81,8 @@ export function findModelCatalogEntry(catalog: ModelCatalog, model: string, lab?
   return (
     catalog.models.find((entry) => entry.id.toLowerCase() === normalizedId) ??
     catalog.models.find((entry) => (lab ? entry.lab === catalogLabSlug(lab) : true) && entry.slug === leaf) ??
-    catalog.models.find((entry) => entry.slug === leaf)
+    catalog.models.find((entry) => entry.slug === leaf) ??
+    catalog.aliases?.find((entry) => (lab ? entry.lab === catalogLabSlug(lab) : true) && entry.slug === leaf)
   )
 }
 
@@ -133,7 +135,7 @@ export function catalogSlug(value: string) {
     .replace(/-{2,}/g, "-")
 }
 
-function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayload?: unknown): ModelCatalog {
+export function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayload?: unknown): ModelCatalog {
   const costs = readCatalogCosts(pricingPayload)
   const labDescriptions = readCatalogLabDescriptions(payload, pricingPayload, labPayload)
   const models = readCatalogModels(payload)
@@ -149,6 +151,25 @@ function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayloa
     .toSorted((a, b) => a.lab.localeCompare(b.lab) || displayDateTime(b.releaseDate) - displayDateTime(a.releaseDate))
   return {
     models,
+    // Contributor is a serving tier of these Muse models, with its own pricing.
+    // Keep aliases out of the model population used to normalize benchmark scores.
+    aliases: ["meta/muse-spark-1.2", "meta/muse-spark-1.3"].flatMap((id) => {
+      const model = models.find((entry) => entry.id === id)
+      if (!model) return []
+      const alias = `${id}-contributor`
+      return [
+        {
+          ...model,
+          id: alias,
+          slug: `${model.slug}-contributor`,
+          name: `${model.name} Contributor`,
+          cost:
+            costs.get(catalogIdKey(alias)) ??
+            costs.get(`${model.lab}/${model.slug}-contributor`) ??
+            costs.get(`${model.slug}-contributor`),
+        },
+      ]
+    }),
     labs: Object.values(
       models.reduce<Record<string, ModelCatalogLab>>((result, model) => {
         result[model.lab] = {
@@ -184,8 +205,8 @@ function readModelCatalogEntry(value: unknown): ModelCatalogEntry[] {
       limit: readCatalogLimit(value.limit),
       modalities: readCatalogModalities(value.modalities),
       openWeights: booleanValue(value.open_weights),
-      reasoning: booleanValue(value.reasoning),
-      toolCall: booleanValue(value.tool_call),
+      reasoning: typeof value.reasoning === "boolean" ? value.reasoning : undefined,
+      toolCall: typeof value.tool_call === "boolean" ? value.tool_call : undefined,
       attachment: booleanValue(value.attachment),
       temperature: booleanValue(value.temperature),
       cost: readCatalogCost(value.cost),

packages/stats/app/test/compare-radar.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test"
+import { buildRadarAxes, resolveRadarScore } from "../src/routes/compare-radar"
+import type { ModelCatalogEntry } from "../src/routes/model-catalog"
+
+const model: ModelCatalogEntry = {
+  id: "meta/muse-spark-1.3",
+  lab: "meta",
+  slug: "muse-spark-1-3",
+  name: "Muse Spark 1.3",
+  modalities: { input: ["text", "image"], output: ["text"] },
+  reasoning: true,
+  toolCall: true,
+  openWeights: false,
+  attachment: true,
+  temperature: true,
+  weights: [],
+  benchmarks: [],
+}
+
+function scores(entry: ModelCatalogEntry | null, catalog = [model]) {
+  return Object.fromEntries(buildRadarAxes(catalog).map((axis) => [axis.label, resolveRadarScore(axis, entry)]))
+}
+
+describe("radar capability fallbacks", () => {
+  test("supported capabilities have visible baselines without benchmarks", () => {
+    const result = scores(model)
+    expect(result["Tool use"]).toEqual({ value: 50, fallback: "Tool calling supported; no comparable benchmark" })
+    expect(result.Reasoning).toEqual({ value: 50, fallback: "Reasoning supported; no comparable benchmark" })
+    expect(result.Coding).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
+  })
+
+  test("explicitly unsupported capabilities remain zero", () => {
+    const result = scores({ ...model, reasoning: false, toolCall: false })
+    expect(result["Tool use"]).toEqual({ value: 0, fallback: "Tool calling not supported" })
+    expect(result.Reasoning).toEqual({ value: 0, fallback: "Reasoning not supported" })
+  })
+
+  test("unknown capabilities and unmatched models use neutral placeholders", () => {
+    const result = scores({ ...model, reasoning: undefined, toolCall: undefined })
+    expect(result["Tool use"]).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
+    expect(result.Reasoning).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
+    expect(Object.values(scores(null))).toEqual(Array(6).fill({ value: 50, fallback: "No data; neutral placeholder" }))
+  })
+
+  test("measured benchmark percentiles override fallbacks, including zero", () => {
+    const low = {
+      ...model,
+      benchmarks: [
+        { name: "Tau3", score: 20 },
+        { name: "GPQA", score: 40 },
+      ],
+    }
+    const high = {
+      ...model,
+      id: "other/model",
+      benchmarks: [
+        { name: "Tau3", score: 80 },
+        { name: "GPQA", score: 90 },
+      ],
+    }
+    expect(scores(low, [low, high])["Tool use"]).toEqual({ value: 0 })
+    expect(scores(low, [low, high]).Reasoning).toEqual({ value: 0 })
+    expect(scores(high, [low, high])["Tool use"]).toEqual({ value: 100 })
+    expect(scores(high, [low, high]).Reasoning).toEqual({ value: 100 })
+  })
+
+  test("a benchmark without comparison peers retains the capability baseline", () => {
+    const entry = { ...model, benchmarks: [{ name: "Tau3", score: 90 }] }
+    expect(scores(entry, [entry])["Tool use"]).toEqual({
+      value: 50,
+      fallback: "Tool calling supported; no comparable benchmark",
+    })
+  })
+})