diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 637ed7c1..b87af8ef 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1331,5 +1331,240 @@ "promptSourcePath": "evals/resolve-storage-001-upsert-missing-update-policy/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-storage-001-upsert-missing-update-policy.json" + }, + { + "experiment": "mcp357-claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-002-undescribed-source-discovery", + "stage": "investigate", + "product": [ + "data-api" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "identified the Data API permission failure on the orders table", + "passed": true + }, + { + "name": "enumerated available log sources before concluding", + "passed": true + }, + { + "name": "referenced the postgrest_logs source by name", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our auto-generated REST API (the Data API) is failing for some requests in production, but our edge functions and database both look healthy. Look through the project's logs and tell me what's actually failing, and why.", + "promptSourcePath": "evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md", + "attempts": 1, + "sourcePath": "mcp357-claude-code-sonnet-5/investigate-logs-002-undescribed-source-discovery.json" + }, + { + "experiment": "mcp357-claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-003-function-stream-disambiguation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "surfaced the function's runtime finding (dollars-vs-cents charge bug)", + "passed": true + }, + { + "name": "queried the function_logs runtime console stream", + "passed": true + }, + { + "name": "did not report the function as healthy based on HTTP 200s", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `process-payment` edge function returns HTTP 200 for every request, so at the HTTP layer everything looks healthy — but customers are being charged the wrong amounts. I need to see what the function is actually logging from inside while it runs. What's going wrong?", + "promptSourcePath": "evals/investigate-logs-003-function-stream-disambiguation/PROMPT.md", + "attempts": 1, + "sourcePath": "mcp357-claude-code-sonnet-5/investigate-logs-003-function-stream-disambiguation.json" + }, + { + "experiment": "mcp357-codex-gpt-5.4-mini", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-logs-002-undescribed-source-discovery", + "stage": "investigate", + "product": [ + "data-api" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "identified the Data API permission failure on the orders table", + "passed": true + }, + { + "name": "enumerated available log sources before concluding", + "passed": true + }, + { + "name": "referenced the postgrest_logs source by name", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Exposing a Table to the Data API grants anon authenticated permission denied 42501\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/database-api-42501-errors", + "title": "Database API 42501 errors" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/42501--permission-denied-for-table-httprequestqueue-KnozmQ", + "title": "42501 : permission denied for table http_request_queue" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0026_pg_graphql_anon_table_exposed", + "title": "Database Advisor: Lint 0026_pg_graphql_anon_table_exposed" + } + ], + "resultChars": 48999 + } + ] + }, + "prompt": "Our auto-generated REST API (the Data API) is failing for some requests in production, but our edge functions and database both look healthy. Look through the project's logs and tell me what's actually failing, and why.", + "promptSourcePath": "evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md", + "attempts": 1, + "sourcePath": "mcp357-codex-gpt-5.4-mini/investigate-logs-002-undescribed-source-discovery.json" + }, + { + "experiment": "mcp357-codex-gpt-5.4-mini", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-logs-003-function-stream-disambiguation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "surfaced the function's runtime finding (dollars-vs-cents charge bug)", + "passed": true + }, + { + "name": "queried the function_logs runtime console stream", + "passed": false + }, + { + "name": "did not report the function as healthy based on HTTP 200s", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our `process-payment` edge function returns HTTP 200 for every request, so at the HTTP layer everything looks healthy — but customers are being charged the wrong amounts. I need to see what the function is actually logging from inside while it runs. What's going wrong?", + "promptSourcePath": "evals/investigate-logs-003-function-stream-disambiguation/PROMPT.md", + "attempts": 1, + "sourcePath": "mcp357-codex-gpt-5.4-mini/investigate-logs-003-function-stream-disambiguation.json" } ] diff --git a/evals/investigate-logs-002-undescribed-source-discovery/EVAL.ts b/evals/investigate-logs-002-undescribed-source-discovery/EVAL.ts new file mode 100644 index 00000000..8b0d09f6 --- /dev/null +++ b/evals/investigate-logs-002-undescribed-source-discovery/EVAL.ts @@ -0,0 +1,72 @@ +import type { + CheckResult, + ToolEvalContext, + ToolScorer, +} from '@supabase-evals/core'; + +// The `sql` arg lands directly on `body` (claude-code / ai-sdk) or nested under +// `body.arguments` (codex's mcp_tool_call shape). +function extractSql(body: Record | undefined): string { + if (!body) return ''; + if (typeof body.sql === 'string') return body.sql; + const args = body.arguments; + if (args && typeof args === 'object') { + const nested = (args as Record).sql; + if (typeof nested === 'string') return nested; + } + return ''; +} + +/** + * All SQL the agent sent to the `query_logs` tool, concatenated and lowercased. + * `endpoint` is the raw tool name, which is bare (`query_logs`) for codex and + * prefixed (`mcp__supabase-mcp__query_logs`) for claude-code, so match on the + * suffix. + */ +function queryLogsSql(ctx: ToolEvalContext): string { + return ctx.toolCalls + .filter((tc) => tc.endpoint?.endsWith('query_logs')) + .map((tc) => extractSql(tc.body)) + .join('\n') + .toLowerCase(); +} + +const scorer: ToolScorer = async (ctx) => { + const report = ctx.agentReport ?? ''; + const sql = queryLogsSql(ctx); + + // The root cause (a permission failure on the `orders` table) lives ONLY in + // postgrest_logs — a source the query_logs description does not enumerate. + const identifiedRootCause = + /\borders\b/i.test(report) && + /(permission denied|permission|forbidden|\b403\b|42501|not authoriz|\brls\b|grant)/i.test( + report + ); + + const checks: CheckResult[] = [ + { + name: 'identified the Data API permission failure on the orders table', + passed: identifiedRootCause, + }, + { + // Diagnostic: did the agent enumerate available sources (discovery-first) + // rather than assume the ones named in the description? (Matt Q1.) + name: 'enumerated available log sources before concluding', + passed: + /group\s+by\s+source/.test(sql) || + /distinct\s+source/.test(sql) || + /select\s+source\b/.test(sql), + }, + { + // Diagnostic: did it use the exact (undescribed) source name, vs. reaching + // the data by scanning the unified stream unfiltered? Either path is valid; + // this just records which the agent took. + name: 'referenced the postgrest_logs source by name', + passed: /postgrest_logs/.test(sql), + }, + ]; + + return { passed: identifiedRootCause, checks }; +}; + +export default scorer; diff --git a/evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md b/evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md new file mode 100644 index 00000000..3c2ca6b0 --- /dev/null +++ b/evals/investigate-logs-002-undescribed-source-discovery/PROMPT.md @@ -0,0 +1,16 @@ +--- +stage: investigate +suite: regression +interface: mcp +product: + - data-api +topic: + - observability +motivation: >- + supabase/mcp#357 (Matt Rossman review) — the query_logs tool description lists + example log sources but the set is non-exhaustive (e.g. postgrest_logs exists + on real projects but isn't listed). Does the agent discover available sources, + or does it only query the ones named in the description and miss/hallucinate? +--- + +Our auto-generated REST API (the Data API) is failing for some requests in production, but our edge functions and database both look healthy. Look through the project's logs and tell me what's actually failing, and why. diff --git a/evals/investigate-logs-002-undescribed-source-discovery/remote/logs.jsonl b/evals/investigate-logs-002-undescribed-source-discovery/remote/logs.jsonl new file mode 100644 index 00000000..47f45557 --- /dev/null +++ b/evals/investigate-logs-002-undescribed-source-discovery/remote/logs.jsonl @@ -0,0 +1,12 @@ +{"id":"pr-01","ts":"2026-08-05T14:00:00Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}} +{"id":"pr-02","ts":"2026-08-05T14:00:12Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}} +{"id":"pr-03","ts":"2026-08-05T14:00:31Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"POST","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}} +{"id":"pr-04","ts":"2026-08-05T14:01:05Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}} +{"id":"pr-05","ts":"2026-08-05T14:02:18Z","source":"postgrest","level":"error","message":"permission denied for table orders","metadata":{"method":"GET","path":"/rest/v1/orders","status_code":403,"error_code":"42501"}} +{"id":"pr-06","ts":"2026-08-05T14:00:45Z","source":"postgrest","level":"info","message":"request completed","metadata":{"method":"GET","path":"/rest/v1/products","status_code":200}} +{"id":"pr-07","ts":"2026-08-05T14:01:50Z","source":"postgrest","level":"info","message":"request completed","metadata":{"method":"GET","path":"/rest/v1/products","status_code":200}} +{"id":"eg-01","ts":"2026-08-05T14:00:20Z","source":"edge","level":"info","message":"GET | 200 | /rest/v1/products","metadata":{"method":"GET","path":"/rest/v1/products","status_code":200}} +{"id":"eg-02","ts":"2026-08-05T14:01:10Z","source":"edge","level":"info","message":"GET | 200 | /auth/v1/user","metadata":{"method":"GET","path":"/auth/v1/user","status_code":200}} +{"id":"pg-01","ts":"2026-08-05T14:00:05Z","source":"postgres","level":"info","message":"statement: select * from products limit 20","metadata":{"error_severity":"LOG","user_name":"authenticator"}} +{"id":"pg-02","ts":"2026-08-05T14:00:40Z","source":"postgres","level":"info","message":"statement: select * from products where id = $1","metadata":{"error_severity":"LOG","user_name":"authenticator"}} +{"id":"pg-03","ts":"2026-08-05T14:01:30Z","source":"postgres","level":"info","message":"connection authorized: user=authenticator database=postgres","metadata":{"error_severity":"LOG","user_name":"authenticator"}} diff --git a/evals/investigate-logs-003-function-stream-disambiguation/EVAL.ts b/evals/investigate-logs-003-function-stream-disambiguation/EVAL.ts new file mode 100644 index 00000000..9a4a27dc --- /dev/null +++ b/evals/investigate-logs-003-function-stream-disambiguation/EVAL.ts @@ -0,0 +1,71 @@ +import type { + CheckResult, + ToolEvalContext, + ToolScorer, +} from '@supabase-evals/core'; + +// The `sql` arg lands directly on `body` (claude-code / ai-sdk) or nested under +// `body.arguments` (codex's mcp_tool_call shape). +function extractSql(body: Record | undefined): string { + if (!body) return ''; + if (typeof body.sql === 'string') return body.sql; + const args = body.arguments; + if (args && typeof args === 'object') { + const nested = (args as Record).sql; + if (typeof nested === 'string') return nested; + } + return ''; +} + +/** + * All SQL the agent sent to the `query_logs` tool, concatenated and lowercased. + * `endpoint` is bare (`query_logs`) for codex and prefixed + * (`mcp__supabase-mcp__query_logs`) for claude-code, so match on the suffix. + */ +function queryLogsSql(ctx: ToolEvalContext): string { + return ctx.toolCalls + .filter((tc) => tc.endpoint?.endsWith('query_logs')) + .map((tc) => extractSql(tc.body)) + .join('\n') + .toLowerCase(); +} + +const scorer: ToolScorer = async (ctx) => { + const report = ctx.agentReport ?? ''; + const sql = queryLogsSql(ctx); + + // The bug is only visible in the runtime console stream (function_logs), not + // in the request/invocation stream (function_edge_logs, which shows healthy + // 200s). The console lines are about charging in dollars instead of cents. + const surfacedInternalFinding = + /\bcents?\b/i.test(report) && + /(dollar|convert|conversion|100x|100\s*times|overcharg|amount)/i.test( + report + ); + + const checks: CheckResult[] = [ + { + name: "surfaced the function's runtime finding (dollars-vs-cents charge bug)", + passed: surfacedInternalFinding, + }, + { + // Matt Q2: does the agent know function_logs (runtime console) is the + // right stream vs function_edge_logs (request/invocation)? + name: 'queried the function_logs runtime console stream', + passed: /function_logs/.test(sql), + }, + { + // Guard against being fooled by the healthy 200s in function_edge_logs. + name: 'did not report the function as healthy based on HTTP 200s', + passed: + surfacedInternalFinding && + !/(function is (healthy|fine|working)|no (issues|errors|problems)\b|nothing (wrong|abnormal)|looks healthy)/i.test( + report + ), + }, + ]; + + return { passed: surfacedInternalFinding, checks }; +}; + +export default scorer; diff --git a/evals/investigate-logs-003-function-stream-disambiguation/PROMPT.md b/evals/investigate-logs-003-function-stream-disambiguation/PROMPT.md new file mode 100644 index 00000000..6f04c683 --- /dev/null +++ b/evals/investigate-logs-003-function-stream-disambiguation/PROMPT.md @@ -0,0 +1,16 @@ +--- +stage: investigate +suite: regression +interface: mcp +product: + - edge-functions +topic: + - observability +motivation: >- + supabase/mcp#357 (Matt Rossman review) — the query_logs description lists both + function_edge_logs and function_logs as sources but doesn't explain what each + represents. Does the agent understand function_edge_logs (request/invocation) + vs function_logs (runtime console output) well enough to pick the right one? +--- + +Our `process-payment` edge function returns HTTP 200 for every request, so at the HTTP layer everything looks healthy — but customers are being charged the wrong amounts. I need to see what the function is actually logging from inside while it runs. What's going wrong? diff --git a/evals/investigate-logs-003-function-stream-disambiguation/remote/logs.jsonl b/evals/investigate-logs-003-function-stream-disambiguation/remote/logs.jsonl new file mode 100644 index 00000000..5d82cdc6 --- /dev/null +++ b/evals/investigate-logs-003-function-stream-disambiguation/remote/logs.jsonl @@ -0,0 +1,11 @@ +{"id":"fe-01","ts":"2026-08-05T15:00:00Z","source":"edge-function","level":"info","message":"POST | 200 | /process-payment","metadata":{"function_id":"process-payment","method":"POST","status":200,"execution_time_ms":140}} +{"id":"fe-02","ts":"2026-08-05T15:00:20Z","source":"edge-function","level":"info","message":"POST | 200 | /process-payment","metadata":{"function_id":"process-payment","method":"POST","status":200,"execution_time_ms":132}} +{"id":"fe-03","ts":"2026-08-05T15:00:41Z","source":"edge-function","level":"info","message":"POST | 200 | /process-payment","metadata":{"function_id":"process-payment","method":"POST","status":200,"execution_time_ms":151}} +{"id":"fe-04","ts":"2026-08-05T15:01:03Z","source":"edge-function","level":"info","message":"POST | 200 | /process-payment","metadata":{"function_id":"process-payment","method":"POST","status":200,"execution_time_ms":128}} +{"id":"fe-05","ts":"2026-08-05T15:01:25Z","source":"edge-function","level":"info","message":"POST | 200 | /process-payment","metadata":{"function_id":"process-payment","method":"POST","status":200,"execution_time_ms":139}} +{"id":"fr-01","ts":"2026-08-05T15:00:01Z","source":"edge-function-runtime","level":"warning","message":"charge amount 4999 computed in dollars but Stripe expects cents; sending without conversion","metadata":{"function_id":"process-payment","event_type":"Log","execution_id":"exec-fr-01"}} +{"id":"fr-02","ts":"2026-08-05T15:00:21Z","source":"edge-function-runtime","level":"warning","message":"charge amount 2500 computed in dollars but Stripe expects cents; sending without conversion","metadata":{"function_id":"process-payment","event_type":"Log","execution_id":"exec-fr-02"}} +{"id":"fr-03","ts":"2026-08-05T15:00:42Z","source":"edge-function-runtime","level":"warning","message":"charge amount 9900 computed in dollars but Stripe expects cents; sending without conversion","metadata":{"function_id":"process-payment","event_type":"Log","execution_id":"exec-fr-03"}} +{"id":"fr-04","ts":"2026-08-05T15:01:04Z","source":"edge-function-runtime","level":"error","message":"customer charged 100x expected amount: passed 4999 as cents-equivalent without dividing","metadata":{"function_id":"process-payment","event_type":"Log","execution_id":"exec-fr-04"}} +{"id":"fr-05","ts":"2026-08-05T15:01:26Z","source":"edge-function-runtime","level":"info","message":"payment intent created","metadata":{"function_id":"send-receipt","event_type":"Log","execution_id":"exec-fr-05"}} +{"id":"fr-06","ts":"2026-08-05T15:01:40Z","source":"edge-function-runtime","level":"info","message":"receipt email queued","metadata":{"function_id":"send-receipt","event_type":"Log","execution_id":"exec-fr-06"}} diff --git a/experiments/mcp357-claude-code-sonnet-5.ts b/experiments/mcp357-claude-code-sonnet-5.ts new file mode 100644 index 00000000..959f0556 --- /dev/null +++ b/experiments/mcp357-claude-code-sonnet-5.ts @@ -0,0 +1,25 @@ +import { + claudeCodeAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +// Pins the MCP server to the supabase/mcp#357 preview build (query_logs + +// logsDialect) so the query_logs description/source-list under review is what +// the agent actually sees. Swap to a released version once #357 ships. +const MCP_357 = 'https://pkg.pr.new/@supabase/mcp-server-supabase@8665e14'; + +export default defineExperiment({ + suite: ['regression'], + agent: claudeCodeAgent({ + model: 'claude-sonnet-5', + reasoningEffort: 'high', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer({ version: MCP_357 })], + }), + localStack: localStackRuntime(), + skills: ['supabase', 'supabase-postgres-best-practices'], +}); diff --git a/experiments/mcp357-codex-gpt-5.4-mini.ts b/experiments/mcp357-codex-gpt-5.4-mini.ts new file mode 100644 index 00000000..c517bb3b --- /dev/null +++ b/experiments/mcp357-codex-gpt-5.4-mini.ts @@ -0,0 +1,25 @@ +import { + codexAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +// Pins the MCP server to the supabase/mcp#357 preview build (query_logs + +// logsDialect) so the query_logs description/source-list under review is what +// the agent actually sees. Swap to a released version once #357 ships. +const MCP_357 = 'https://pkg.pr.new/@supabase/mcp-server-supabase@8665e14'; + +export default defineExperiment({ + suite: ['regression'], + agent: codexAgent({ + model: 'gpt-5.4-mini', + reasoningEffort: 'medium', + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer({ version: MCP_357 })], + }), + localStack: localStackRuntime(), + skills: ['supabase', 'supabase-postgres-best-practices'], +}); diff --git a/packages/platform-lite/src/management-api/debugging.ts b/packages/platform-lite/src/management-api/debugging.ts index 3b057836..e8ecd43c 100644 --- a/packages/platform-lite/src/management-api/debugging.ts +++ b/packages/platform-lite/src/management-api/debugging.ts @@ -198,7 +198,7 @@ export function createDebuggingRoutes( */ const UNMODELED_SOURCES = /\b(workflow_run_logs|realtime_logs)\b/i; const PHYSICAL_RELATIONS = - /\b(?:from|join)\s+(edge_logs|function_edge_logs|function_logs|postgres_logs|auth_logs|storage_logs)\b/i; + /\b(?:from|join)\s+(edge_logs|function_edge_logs|function_logs|postgres_logs|postgrest_logs|auth_logs|storage_logs)\b/i; const WALL_CLOCK_FNS = /\b(now|now64|today|yesterday|current_timestamp|current_date|localtimestamp)\b/i; // 'logs' as a relation; log_attributes and other log_* identifiers do not match. diff --git a/packages/platform-lite/src/project/log-seeding.ts b/packages/platform-lite/src/project/log-seeding.ts index 6ad584f8..d431bd3e 100644 --- a/packages/platform-lite/src/project/log-seeding.ts +++ b/packages/platform-lite/src/project/log-seeding.ts @@ -113,6 +113,28 @@ CREATE TABLE IF NOT EXISTS storage_logs ( metadata jsonb NOT NULL DEFAULT '{}'::jsonb ); +-- The Data API (PostgREST) request stream. A real hosted source that current +-- mcp's query_logs description does NOT enumerate in its example source list, +-- so it exercises whether an agent DISCOVERS available sources vs assumes the +-- listed ones (see investigate-observability-002). Seed it with source +-- 'postgrest' / 'postgrest_logs'. +CREATE TABLE IF NOT EXISTS postgrest_logs ( + id text PRIMARY KEY, + identifier text, + timestamp timestamptz NOT NULL DEFAULT now(), + ts timestamptz, + event_message text, + message text, + source text, + level text, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + method text, + path text, + status_code integer, + error_code text, + error text +); + -- Unified ClickHouse-shaped stream: the hosted /analytics/endpoints/logs -- endpoint exposes one 'logs' relation with a 'source' discriminator and a -- log_attributes map. Mirror it so ClickHouse-dialect SQL from current mcp @@ -176,7 +198,11 @@ CREATE VIEW logs AS FROM auth_logs UNION ALL SELECT id, identifier, timestamp, ts, event_message, message, level, level, 'storage_logs', metadata - FROM storage_logs; + FROM storage_logs + UNION ALL + SELECT id, identifier, timestamp, ts, event_message, message, level, level, 'postgrest_logs', + metadata || jsonb_strip_nulls(jsonb_build_object('request.method', method, 'request.path', path, 'response.status_code', status_code, 'error.code', error_code, 'error.message', error)) + FROM postgrest_logs; -- Enforcement of "only the unified stream is queryable" lives HERE, not in a -- regex: the ClickHouse route's transaction runs SET LOCAL ROLE logs_reader, @@ -283,11 +309,19 @@ export async function seedLogRow(logsDb: PGlite, row: LogRow): Promise { return; } + if ( + normalizedSource === 'postgrest' || + normalizedSource === 'postgrest_logs' + ) { + await seedPostgrestLog(logsDb, log); + return; + } + // Loud failure over a silent no-op: a dropped seed surfaces later as a false // "no logs" query result, which reads as a passing scenario. Same doctrine as // the unmodeled-source guard in debugging.ts. throw new Error( - `unknown log seed source '${row.source}' — expected edge-function, edge-function-runtime, edge, postgres/database, auth, or storage` + `unknown log seed source '${row.source}' — expected edge-function, edge-function-runtime, edge, postgres/database, auth, storage, or postgrest` ); } @@ -451,6 +485,34 @@ async function seedAuthLog( ); } +async function seedPostgrestLog( + logsDb: PGlite, + log: NormalizedLogSeed +): Promise { + await logsDb.query( + `INSERT INTO postgrest_logs + ( + id, identifier, timestamp, ts, event_message, message, source, level, metadata, + method, path, status_code, error_code, error + ) + VALUES ($1, $2, $3, $3, $4, $4, $5, $6, $7::jsonb, $8, $9, $10, $11, $12)`, + [ + log.id, + metadataText(log.metadata, ['identifier']), + log.ts, + log.message, + log.source, + log.level, + log.metadataJson, + metadataText(log.metadata, ['method']), + metadataText(log.metadata, ['path']), + metadataNumber(log.metadata, ['status_code', 'status']), + metadataText(log.metadata, ['error_code', 'code']), + metadataText(log.metadata, ['error']) ?? log.message, + ] + ); +} + async function seedStorageLog( logsDb: PGlite, log: NormalizedLogSeed diff --git a/packages/platform-lite/test/clickhouse-logs.test.ts b/packages/platform-lite/test/clickhouse-logs.test.ts index 7b388a63..6131d24f 100644 --- a/packages/platform-lite/test/clickhouse-logs.test.ts +++ b/packages/platform-lite/test/clickhouse-logs.test.ts @@ -58,6 +58,21 @@ for (const [id, functionId] of [ }, }); } +// A source the mcp query_logs description does NOT enumerate: it must still be +// seedable and queryable through the unified stream once modeled. +await seedLogRow(logsDb, { + id: 'pr1', + ts: new Date('2026-04-28T10:00:00Z'), + source: 'postgrest', + level: 'error', + message: 'permission denied for table orders', + metadata: { + method: 'GET', + path: '/rest/v1/orders', + status_code: 403, + error_code: '42501', + }, +}); afterAll(() => logsDb.close()); // verbatim from mcp getClickHouseLogQuery('edge-function') @@ -103,6 +118,30 @@ describe('compileClickHouseLogsSql + unified logs view', () => { expect(Number(result.rows[1]!.error_count)).toBe(1); }); + it('serves a modeled-but-undescribed source (postgrest_logs) through the unified stream', async () => { + const sql = `select id, event_message, + log_attributes['request.path'] as path, + log_attributes['response.status_code'] as status_code, + log_attributes['error.code'] as error_code + from logs + where source = 'postgrest_logs' + order by timestamp desc + limit 10`; + const result = await logsDb.query<{ + event_message: string; + path: string; + status_code: string; + error_code: string; + }>(compileClickHouseLogsSql(sql)); + expect(result.rows).toHaveLength(1); + expect(result.rows[0]).toMatchObject({ + path: '/rest/v1/orders', + status_code: '403', + error_code: '42501', + }); + expect(result.rows[0]!.event_message).toMatch(/permission denied/i); + }); + it('surfaces an error for a bare numeric comparison on a map value (hosted-faithful)', async () => { // Hosted ClickHouse map values are String, so a bare `>= 500` comparison is // a type error there — it must error here too. The surfaced error is the @@ -241,12 +280,13 @@ limit 100` ['edge_logs', 5], ['function_edge_logs', 5], ['function_logs', 2], + ['postgrest_logs', 1], ['storage_logs', 1], ]); const total = await logsDb.query<{ n: string }>( compileClickHouseLogsSql('select count(*) as n from logs') ); - expect(Number(total.rows[0]!.n)).toBe(13); + expect(Number(total.rows[0]!.n)).toBe(14); }); it('matches an absent map key against the empty string, as a ClickHouse Map does', async () => { @@ -258,7 +298,7 @@ limit 100` "select count(*) as n from logs where log_attributes['nonexistent'] = ''" ) ); - expect(Number(result.rows[0]!.n)).toBe(13); + expect(Number(result.rows[0]!.n)).toBe(14); }); it('translates map access with whitespace inside the subscript', async () => {