feat(evals): query_logs source discovery + function-stream disambiguation

apps/web/src/data/regression-eval-results.json

@@ -1331,5 +1331,240 @@
     "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md",
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5-no-skills/resolve-storage-001-upsert-missing-update-policy.json"
+  },
+  {
+    "experiment": "mcp357-claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-logs-002-undescribed-source-discovery",
+    "stage": "investigate",
+    "product": [
+      "data-api"
+    ],
+    "topic": [
+      "observability"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": true,
+    "checks": [
+      {
+        "name": "identified the Data API permission failure on the orders table",
+        "passed": true
+      },
+      {
+        "name": "enumerated available log sources before concluding",
+        "passed": true
+      },
+      {
+        "name": "referenced the postgrest_logs source by name",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "Our auto-generated REST API (the Data API) is failing for some requests in production, but our edge functions and database both look healthy. Look through the project's logs and tell me what's actually failing, and why.",
+    "promptSourcePath": "evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md",
+    "attempts": 1,
+    "sourcePath": "mcp357-claude-code-sonnet-5/investigate-logs-002-undescribed-source-discovery.json"
+  },
+  {
+    "experiment": "mcp357-claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-logs-003-function-stream-disambiguation",
+    "stage": "investigate",
+    "product": [
+      "edge-functions"
+    ],
+    "topic": [
+      "observability"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": true,
+    "checks": [
+      {
+        "name": "surfaced the function's runtime finding (dollars-vs-cents charge bug)",
+        "passed": true
+      },
+      {
+        "name": "queried the function_logs runtime console stream",
+        "passed": true
+      },
+      {
+        "name": "did not report the function as healthy based on HTTP 200s",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "Our `process-payment` edge function returns HTTP 200 for every request, so at the HTTP layer everything looks healthy — but customers are being charged the wrong amounts. I need to see what the function is actually logging from inside while it runs. What's going wrong?",
+    "promptSourcePath": "evals/investigate-logs-003-function-stream-disambiguation/PROMPT.md",
+    "attempts": 1,
+    "sourcePath": "mcp357-claude-code-sonnet-5/investigate-logs-003-function-stream-disambiguation.json"
+  },
+  {
+    "experiment": "mcp357-codex-gpt-5.4-mini",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-5.4-mini",
+      "reasoningEffort": "medium"
+    },
+    "eval": "investigate-logs-002-undescribed-source-discovery",
+    "stage": "investigate",
+    "product": [
+      "data-api"
+    ],
+    "topic": [
+      "observability"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": true,
+    "checks": [
+      {
+        "name": "identified the Data API permission failure on the orders table",
+        "passed": true
+      },
+      {
+        "name": "enumerated available log sources before concluding",
+        "passed": true
+      },
+      {
+        "name": "referenced the postgrest_logs source by name",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ]
+    },
+    "docs": {
+      "calls": [
+        {
+          "source": "search_docs",
+          "query": "query {\n  searchDocs(query: \"Exposing a Table to the Data API grants anon authenticated permission denied 42501\", limit: 5) {\n    nodes {\n      title\n      href\n      content\n    }\n  }\n}",
+          "hasContent": true,
+          "pages": [
+            {
+              "url": "https://supabase.com/docs/guides/api/securing-your-api",
+              "title": "Securing your API"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/troubleshooting/database-api-42501-errors",
+              "title": "Database API 42501 errors"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/deployment/managing-environments",
+              "title": "Managing Environments"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/troubleshooting/42501--permission-denied-for-table-httprequestqueue-KnozmQ",
+              "title": "42501 : permission denied for table http_request_queue"
+            },
+            {
+              "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0026_pg_graphql_anon_table_exposed",
+              "title": "Database Advisor: Lint 0026_pg_graphql_anon_table_exposed"
+            }
+          ],
+          "resultChars": 48999
+        }
+      ]
+    },
+    "prompt": "Our auto-generated REST API (the Data API) is failing for some requests in production, but our edge functions and database both look healthy. Look through the project's logs and tell me what's actually failing, and why.",
+    "promptSourcePath": "evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md",
+    "attempts": 1,
+    "sourcePath": "mcp357-codex-gpt-5.4-mini/investigate-logs-002-undescribed-source-discovery.json"
+  },
+  {
+    "experiment": "mcp357-codex-gpt-5.4-mini",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-5.4-mini",
+      "reasoningEffort": "medium"
+    },
+    "eval": "investigate-logs-003-function-stream-disambiguation",
+    "stage": "investigate",
+    "product": [
+      "edge-functions"
+    ],
+    "topic": [
+      "observability"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": true,
+    "checks": [
+      {
+        "name": "surfaced the function's runtime finding (dollars-vs-cents charge bug)",
+        "passed": true
+      },
+      {
+        "name": "queried the function_logs runtime console stream",
+        "passed": false
+      },
+      {
+        "name": "did not report the function as healthy based on HTTP 200s",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ]
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "Our `process-payment` edge function returns HTTP 200 for every request, so at the HTTP layer everything looks healthy — but customers are being charged the wrong amounts. I need to see what the function is actually logging from inside while it runs. What's going wrong?",
+    "promptSourcePath": "evals/investigate-logs-003-function-stream-disambiguation/PROMPT.md",
+    "attempts": 1,
+    "sourcePath": "mcp357-codex-gpt-5.4-mini/investigate-logs-003-function-stream-disambiguation.json"
   }
 ]

evals/investigate-logs-002-undescribed-source-discovery/EVAL.ts

@@ -0,0 +1,72 @@
+import type {
+  CheckResult,
+  ToolEvalContext,
+  ToolScorer,
+} from '@supabase-evals/core';
+
+// The `sql` arg lands directly on `body` (claude-code / ai-sdk) or nested under
+// `body.arguments` (codex's mcp_tool_call shape).
+function extractSql(body: Record<string, unknown> | undefined): string {
+  if (!body) return '';
+  if (typeof body.sql === 'string') return body.sql;
+  const args = body.arguments;
+  if (args && typeof args === 'object') {
+    const nested = (args as Record<string, unknown>).sql;
+    if (typeof nested === 'string') return nested;
+  }
+  return '';
+}
+
+/**
+ * All SQL the agent sent to the `query_logs` tool, concatenated and lowercased.
+ * `endpoint` is the raw tool name, which is bare (`query_logs`) for codex and
+ * prefixed (`mcp__supabase-mcp__query_logs`) for claude-code, so match on the
+ * suffix.
+ */
+function queryLogsSql(ctx: ToolEvalContext): string {
+  return ctx.toolCalls
+    .filter((tc) => tc.endpoint?.endsWith('query_logs'))
+    .map((tc) => extractSql(tc.body))
+    .join('\n')
+    .toLowerCase();
+}
+
+const scorer: ToolScorer = async (ctx) => {
+  const report = ctx.agentReport ?? '';
+  const sql = queryLogsSql(ctx);
+
+  // The root cause (a permission failure on the `orders` table) lives ONLY in
+  // postgrest_logs — a source the query_logs description does not enumerate.
+  const identifiedRootCause =
+    /\borders\b/i.test(report) &&
+    /(permission denied|permission|forbidden|\b403\b|42501|not authoriz|\brls\b|grant)/i.test(
+      report
+    );
+
+  const checks: CheckResult[] = [
+    {
+      name: 'identified the Data API permission failure on the orders table',
+      passed: identifiedRootCause,
+    },
+    {
+      // Diagnostic: did the agent enumerate available sources (discovery-first)
+      // rather than assume the ones named in the description? (Matt Q1.)
+      name: 'enumerated available log sources before concluding',
+      passed:
+        /group\s+by\s+source/.test(sql) ||
+        /distinct\s+source/.test(sql) ||
+        /select\s+source\b/.test(sql),
+    },
+    {
+      // Diagnostic: did it use the exact (undescribed) source name, vs. reaching
+      // the data by scanning the unified stream unfiltered? Either path is valid;
+      // this just records which the agent took.
+      name: 'referenced the postgrest_logs source by name',
+      passed: /postgrest_logs/.test(sql),
+    },
+  ];
+
+  return { passed: identifiedRootCause, checks };
+};
+
+export default scorer;

evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md

@@ -0,0 +1,16 @@
+---
+stage: investigate
+suite: regression
+interface: mcp
+product:
+  - data-api
+topic:
+  - observability
+motivation: >-
+  supabase/mcp#357 (Matt Rossman review) — the query_logs tool description lists
+  example log sources but the set is non-exhaustive (e.g. postgrest_logs exists
+  on real projects but isn't listed). Does the agent discover available sources,
+  or does it only query the ones named in the description and miss/hallucinate?
+---
+
+Our auto-generated REST API (the Data API) is failing for some requests in production, but our edge functions and database both look healthy. Look through the project's logs and tell me what's actually failing, and why.

evals/investigate-logs-002-undescribed-source-discovery/remote/logs.jsonl

@@ -0,0 +1,12 @@
+{"id":"pr-01","ts":"2026-08-05T14:00:00Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}}
+{"id":"pr-02","ts":"2026-08-05T14:00:12Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}}
+{"id":"pr-03","ts":"2026-08-05T14:00:31Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"POST","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}}
+{"id":"pr-04","ts":"2026-08-05T14:01:05Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}}
+{"id":"pr-05","ts":"2026-08-05T14:02:18Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}}
+{"id":"pr-06","ts":"2026-08-05T14:00:45Z","source":"postgrest","level":"info","message":"request completed","metadata":{"method":"GET","path":"/rest/v1/products","status_code":200}}
+{"id":"pr-07","ts":"2026-08-05T14:01:50Z","source":"postgrest","level":"info","message":"request completed","metadata":{"method":"GET","path":"/rest/v1/products","status_code":200}}
+{"id":"eg-01","ts":"2026-08-05T14:00:20Z","source":"edge","level":"info","message":"GET | 200 | /rest/v1/products","metadata":{"method":"GET","path":"/rest/v1/products","status_code":200}}
+{"id":"eg-02","ts":"2026-08-05T14:01:10Z","source":"edge","level":"info","message":"GET | 200 | /auth/v1/user","metadata":{"method":"GET","path":"/auth/v1/user","status_code":200}}
+{"id":"pg-01","ts":"2026-08-05T14:00:05Z","source":"postgres","level":"info","message":"statement: select * from products limit 20","metadata":{"error_severity":"LOG","user_name":"authenticator"}}
+{"id":"pg-02","ts":"2026-08-05T14:00:40Z","source":"postgres","level":"info","message":"statement: select * from products where id = $1","metadata":{"error_severity":"LOG","user_name":"authenticator"}}
+{"id":"pg-03","ts":"2026-08-05T14:01:30Z","source":"postgres","level":"info","message":"connection authorized: user=authenticator database=postgres","metadata":{"error_severity":"LOG","user_name":"authenticator"}}