From 0c58d12aaab5c41e9765147a34dc1f5144935517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:25:06 -0700 Subject: [PATCH 01/13] docs(mcp): plan Jev-based Core schema domain selection --- ...-09-29-jev-core-schema-domain-selection.md | 962 ++++++++++++++++++ 1 file changed, 962 insertions(+) create mode 100644 use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md diff --git a/use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md b/use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md new file mode 100644 index 0000000..89ba853 --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md @@ -0,0 +1,962 @@ +# Jev Core Schema Domain Selection Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let `fetch-core-graphql-schema` take a natural-language `intent` and return the compacted schema of every Core API domain that intent needs, chosen by TypeSafe's Jev model on Workers AI, in one call. + +**Architecture:** A new `coreDomainSelector` service asks Jev one Noul ("is this domain needed?") per domain in a single request, with the intent and the domain index as state. It keeps the domains above a threshold (capped), and the splitter compacts their union into one schema. `domain` stays as an exact override; any Jev failure or an empty selection falls back to today's domain index, so Jev can never break the tool. + +**Tech Stack:** Cloudflare Workers + Workers AI binding (`typesafe/jev`), Hono, Awilix, Zod v4, Bun test. + +**Spec:** No separate spec document. The design was agreed in conversation on 2026-09-29 and is restated in Goal/Architecture above and in Global Constraints below. + +## Global Constraints + +- Package root for every path below: `use-crystallize/mcp-servers/crystallize/`. Run all commands from there. +- Model id: `typesafe/jev` (Workers AI, 32,000-token context, $0.042 / 1M input tokens, output free). +- Jev Noul answer shape: `{ "type": "noul", "noul": <0..1> }` under `response.answers[]`. +- Selection policy (initial, tuned in Task 6): keep domains with probability `>= 0.3`, sorted descending, at most `4`. +- `domain` input wins over `intent` when both are given (exact, no Jev call). +- A Jev failure, a blank intent, or an empty selection must return the domain index, never an error. +- No new npm dependencies. +- Code style: 4-space indent, 120-char width, Zod v4, ESM. Run `bun lint` before each commit. +- Work on a branch, not `main`: `git checkout -b feat/jev-core-domain-selection`. + +## Review Focus + +1. **Jev unavailable** (binding missing in local dev, outage, 32k context exceeded on a huge schema) → tool returns the domain index with a note, not "Failed to fetch core schema". Pinned in Task 4 (`falls back to the index when the selector throws`). +2. **Blank or whitespace `intent`** → treated as no intent (domain index), and Jev is not called. Pinned in Task 2 and Task 4. +3. **Both `domain` and `intent` given** → exact `domain` wins, selector not called. Pinned in Task 4. +4. **Jev response missing some answer ids** → those domains are skipped, the rest still used. Pinned in Task 2. +5. **Malformed Jev response** (no `answers`) → treated as failure → fallback. Pinned in Task 1 (client throws) + Task 4 (fallback on throw). + +--- + +### Task 1: Jev client + +**Files:** +- Create: `src/contracts/jev.ts` +- Create: `src/core/services/jev-client.ts` +- Test: `tests/core/services/jev-client.test.ts` + +**Interfaces:** +- Produces: + - `type JevNoulQuestion = { type: "noul"; instructions: string; criteria?: { true: string; false: string } }` + - `type JevRequest = { state: unknown; questions: Record }` + - `type JevResponse = { model: string; answers: Record; usage?: { input_tokens: number; output_tokens: number } }` + - `type JevClient = (request: JevRequest) => Promise` + - `const JEV_MODEL = "typesafe/jev"` + - `type JevBinding = { run(model: typeof JEV_MODEL, input: JevRequest): Promise }` + - `createJevClient({ ai }: { ai: JevBinding }): JevClient` + +- [ ] **Step 1: Write the failing test** + +`tests/core/services/jev-client.test.ts`: + +```ts +import { describe, it, expect, mock } from "bun:test"; +import { createJevClient, JEV_MODEL, type JevBinding } from "../../../src/core/services/jev-client"; +import type { JevRequest } from "../../../src/contracts/jev"; + +const request: JevRequest = { + state: { intent: "list orders" }, + questions: { d0: { type: "noul", instructions: "Is the order domain needed?" } }, +}; + +describe("createJevClient", () => { + it("runs typesafe/jev with the request and returns the response", async () => { + const response = { model: "jev-1.13.0", answers: { d0: { type: "noul", noul: 0.9 } } }; + const run = mock(async () => response); + const client = createJevClient({ ai: { run } as JevBinding }); + + expect(await client(request)).toEqual(response); + expect(run).toHaveBeenCalledWith(JEV_MODEL, request); + }); + + it("throws when the response has no answers", async () => { + const client = createJevClient({ ai: { run: async () => ({}) } as JevBinding }); + await expect(client(request)).rejects.toThrow("Jev returned no answers"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/core/services/jev-client.test.ts` +Expected: FAIL, cannot resolve `../../../src/core/services/jev-client`. + +- [ ] **Step 3: Write minimal implementation** + +`src/contracts/jev.ts`: + +```ts +export type JevNoulQuestion = { + type: "noul"; + instructions: string; + criteria?: { true: string; false: string }; +}; + +export type JevRequest = { + state: unknown; + questions: Record; +}; + +export type JevResponse = { + model: string; + answers: Record; + usage?: { input_tokens: number; output_tokens: number }; +}; + +export type JevClient = (request: JevRequest) => Promise; +``` + +`src/core/services/jev-client.ts`: + +```ts +import type { JevClient, JevRequest, JevResponse } from "../../contracts/jev"; + +export const JEV_MODEL = "typesafe/jev"; + +// The generated `Ai` types predate typesafe/jev, so the binding is narrowed to the one call we make. +export type JevBinding = { + run(model: typeof JEV_MODEL, input: JevRequest): Promise; +}; + +export const createJevClient = + ({ ai }: { ai: JevBinding }): JevClient => + async (request) => { + const response = (await ai.run(JEV_MODEL, request)) as Partial | null; + if (!response || typeof response !== "object" || !response.answers) { + throw new Error("Jev returned no answers"); + } + return response as JevResponse; + }; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/core/services/jev-client.test.ts` +Expected: PASS (2 tests). + +- [ ] **Step 5: Commit** + +```bash +bun lint +git add src/contracts/jev.ts src/core/services/jev-client.ts tests/core/services/jev-client.test.ts +git commit -m "feat(mcp): add a Jev client over the Workers AI binding" +``` + +--- + +### Task 2: Core domain selector + +**Files:** +- Create: `src/contracts/core-domain-selector.ts` +- Create: `src/core/services/core-domain-selector.ts` +- Test: `tests/core/services/core-domain-selector.test.ts` + +**Interfaces:** +- Consumes: `JevClient`, `JevRequest` (Task 1); `DomainIndex` from `src/contracts/core-schema-domain-splitter.ts`. +- Produces: + - `type SelectedDomain = { name: string; probability: number }` + - `type CoreDomainSelector = { select(index: DomainIndex, intent: string): Promise }` + - `const DOMAIN_THRESHOLD = 0.3`, `const MAX_DOMAINS = 4` + - `buildDomainRequest(index: DomainIndex, intent: string): { request: JevRequest; domainByQuestionId: Map }` + - `scoreDomains(jevClient: JevClient, index: DomainIndex, intent: string): Promise`: every answered domain, unsorted, unfiltered (used by the Task 6 eval). + - `pickDomains(scored: SelectedDomain[], threshold?: number, max?: number): SelectedDomain[]` + - `createCoreDomainSelector({ jevClient }: { jevClient: JevClient }): CoreDomainSelector` + +- [ ] **Step 1: Write the failing test** + +`tests/core/services/core-domain-selector.test.ts`: + +```ts +import { describe, it, expect, mock } from "bun:test"; +import { + buildDomainRequest, + createCoreDomainSelector, + pickDomains, + scoreDomains, +} from "../../../src/core/services/core-domain-selector"; +import type { DomainIndex } from "../../../src/contracts/core-schema-domain-splitter"; +import type { JevClient, JevResponse } from "../../../src/contracts/jev"; + +const index: DomainIndex = { + domains: [ + { name: "customer", queries: ["customer", "customers"], mutations: ["createCustomer"] }, + { name: "order", queries: ["order", "orders"], mutations: ["createOrder"] }, + { name: "subscription", queries: ["subscription"], mutations: [] }, + ], +}; + +const answers = (nouls: Record): JevResponse => ({ + model: "jev-1.13.0", + answers: Object.fromEntries(Object.entries(nouls).map(([id, noul]) => [id, { type: "noul", noul }])), +}); + +describe("buildDomainRequest", () => { + it("puts the intent and the domain index in state and asks one noul per domain", () => { + const { request, domainByQuestionId } = buildDomainRequest(index, "create an order"); + + expect(request.state).toEqual({ intent: "create an order", domains: index.domains }); + expect(Object.keys(request.questions)).toEqual(["d0", "d1", "d2"]); + expect(domainByQuestionId.get("d1")).toBe("order"); + expect(request.questions.d1.type).toBe("noul"); + expect(request.questions.d1.instructions).toContain('"order"'); + expect(request.questions.d1.criteria?.true).toContain("order"); + }); +}); + +describe("pickDomains", () => { + it("keeps domains at or above the threshold, most likely first, capped", () => { + const picked = pickDomains( + [ + { name: "a", probability: 0.2 }, + { name: "b", probability: 0.9 }, + { name: "c", probability: 0.3 }, + { name: "d", probability: 0.6 }, + ], + 0.3, + 2, + ); + expect(picked).toEqual([ + { name: "b", probability: 0.9 }, + { name: "d", probability: 0.6 }, + ]); + }); +}); + +describe("scoreDomains", () => { + it("maps answers back to domain names and skips missing answers", async () => { + const jevClient: JevClient = async () => answers({ d0: 0.4, d1: 0.95 }); + expect(await scoreDomains(jevClient, index, "create an order")).toEqual([ + { name: "customer", probability: 0.4 }, + { name: "order", probability: 0.95 }, + ]); + }); +}); + +describe("createCoreDomainSelector", () => { + it("returns the picked domains for an intent", async () => { + const jevClient: JevClient = async () => answers({ d0: 0.7, d1: 0.95, d2: 0.05 }); + const selector = createCoreDomainSelector({ jevClient }); + + expect(await selector.select(index, "create an order for a customer")).toEqual([ + { name: "order", probability: 0.95 }, + { name: "customer", probability: 0.7 }, + ]); + }); + + it("does not call Jev for a blank intent", async () => { + const jevClient = mock(async () => answers({})); + const selector = createCoreDomainSelector({ jevClient }); + + expect(await selector.select(index, " ")).toEqual([]); + expect(jevClient).not.toHaveBeenCalled(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/core/services/core-domain-selector.test.ts` +Expected: FAIL, cannot resolve `../../../src/core/services/core-domain-selector`. + +- [ ] **Step 3: Write minimal implementation** + +`src/contracts/core-domain-selector.ts`: + +```ts +import type { DomainIndex } from "./core-schema-domain-splitter"; + +export type SelectedDomain = { + name: string; + probability: number; +}; + +export type CoreDomainSelector = { + select(index: DomainIndex, intent: string): Promise; +}; +``` + +`src/core/services/core-domain-selector.ts`: + +```ts +import type { CoreDomainSelector, SelectedDomain } from "../../contracts/core-domain-selector"; +import type { DomainIndex } from "../../contracts/core-schema-domain-splitter"; +import type { JevClient, JevRequest } from "../../contracts/jev"; + +/** + * Missing a needed domain costs the agent another round-trip; an extra domain only costs output size. + * So the threshold leans low and the cap bounds the output. + */ +export const DOMAIN_THRESHOLD = 0.3; +export const MAX_DOMAINS = 4; + +/** + * One Noul per domain over a shared state, sent as a single request (Jev answers them in parallel). + * Question ids are opaque to the model, so each instruction names its domain in full. + */ +export function buildDomainRequest( + index: DomainIndex, + intent: string, +): { request: JevRequest; domainByQuestionId: Map } { + const domainByQuestionId = new Map(); + const questions: JevRequest["questions"] = {}; + index.domains.forEach((domain, i) => { + const id = `d${i}`; + domainByQuestionId.set(id, domain.name); + questions[id] = { + type: "noul", + instructions: + `Does accomplishing \`intent\` against the Crystallize Core API require at least one of the ` + + `queries or mutations listed for the "${domain.name}" domain in \`domains\`?`, + criteria: { + true: `At least one "${domain.name}" query or mutation is needed to read or write what the intent is about`, + false: `The intent can be accomplished without any "${domain.name}" query or mutation`, + }, + }; + }); + return { request: { state: { intent, domains: index.domains }, questions }, domainByQuestionId }; +} + +export async function scoreDomains( + jevClient: JevClient, + index: DomainIndex, + intent: string, +): Promise { + const { request, domainByQuestionId } = buildDomainRequest(index, intent); + const response = await jevClient(request); + const scored: SelectedDomain[] = []; + for (const [id, name] of domainByQuestionId) { + const answer = response.answers[id]; + if (answer) scored.push({ name, probability: answer.noul }); + } + return scored; +} + +export function pickDomains( + scored: SelectedDomain[], + threshold = DOMAIN_THRESHOLD, + max = MAX_DOMAINS, +): SelectedDomain[] { + return scored + .filter((d) => d.probability >= threshold) + .sort((a, b) => b.probability - a.probability) + .slice(0, max); +} + +export const createCoreDomainSelector = ({ jevClient }: { jevClient: JevClient }): CoreDomainSelector => ({ + async select(index, intent) { + if (!intent.trim()) return []; + return pickDomains(await scoreDomains(jevClient, index, intent)); + }, +}); +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/core/services/core-domain-selector.test.ts` +Expected: PASS (5 tests). + +- [ ] **Step 5: Commit** + +```bash +bun lint +git add src/contracts/core-domain-selector.ts src/core/services/core-domain-selector.ts tests/core/services/core-domain-selector.test.ts +git commit -m "feat(mcp): select Core schema domains for an intent with Jev" +``` + +--- + +### Task 3: Compact several domains at once + +**Files:** +- Modify: `src/contracts/core-schema-domain-splitter.ts:13-20` +- Modify: `src/core/services/core-schema-domain-splitter.ts:123-147` +- Test: `tests/core/services/core-schema-domain-splitter.test.ts` (append to the `createCoreSchemaDomainSplitter` describe) + +**Interfaces:** +- Produces: `CoreSchemaDomainSplitter.getCompactedDomainsSchema(introspection: IntrospectionResult, domains: string[], operations: "queries" | "mutations" | "both"): string`. `getCompactedDomainSchema` keeps its signature and delegates to it. + +- [ ] **Step 1: Write the failing test** + +Append inside `describe("createCoreSchemaDomainSplitter", ...)`, after the `getCompactedDomainSchema` describe: + +```ts + describe("getCompactedDomainsSchema", () => { + it("returns the union of the requested domains' root fields", () => { + const schema = splitter.getCompactedDomainsSchema(introspection, ["order", "subscription"], "both"); + + expect(schema).toContain("createOrder"); + expect(schema).toContain("Subscription"); + expect(schema).not.toContain("createCustomer"); + }); + + it("names every unknown domain", () => { + const result = splitter.getCompactedDomainsSchema(introspection, ["order", "nope", "nada"], "both"); + expect(result).toContain('Unknown domain "nope", "nada"'); + expect(result).toContain("Available domains:"); + }); + }); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/core/services/core-schema-domain-splitter.test.ts` +Expected: FAIL, `splitter.getCompactedDomainsSchema is not a function`. + +- [ ] **Step 3: Write minimal implementation** + +In `src/contracts/core-schema-domain-splitter.ts`, add to the `CoreSchemaDomainSplitter` type: + +```ts + getCompactedDomainsSchema( + introspection: IntrospectionResult, + domains: string[], + operations: "queries" | "mutations" | "both", + ): string; +``` + +In `src/core/services/core-schema-domain-splitter.ts`, replace the whole `getCompactedDomainSchema` method (lines 123-147) with: + +```ts + getCompactedDomainSchema( + introspection: IntrospectionResult, + domain: string, + operations: "queries" | "mutations" | "both", + ): string { + return this.getCompactedDomainsSchema(introspection, [domain], operations); + }, + + getCompactedDomainsSchema( + introspection: IntrospectionResult, + domains: string[], + operations: "queries" | "mutations" | "both", + ): string { + const index = this.listDomains(introspection); + const unknown = domains.filter((name) => !index.domains.some((d) => d.name === name)); + if (unknown.length > 0) { + const available = index.domains.map((d) => d.name).join(", "); + return `Unknown domain "${unknown.join('", "')}". Available domains: ${available}`; + } + + const fieldNames = new Set(); + for (const domainInfo of index.domains.filter((d) => domains.includes(d.name))) { + if (operations === "queries" || operations === "both") { + for (const f of domainInfo.queries) fieldNames.add(f); + } + if (operations === "mutations" || operations === "both") { + for (const f of domainInfo.mutations) fieldNames.add(f); + } + } + + return compactSchemaFromIntrospection(introspection, { + operations, + rootFieldFilter: fieldNames, + }); + }, +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/core/services/core-schema-domain-splitter.test.ts` +Expected: PASS, including the existing `returns error for unknown domain` test (message is unchanged for one domain). + +- [ ] **Step 5: Commit** + +```bash +bun lint +git add src/contracts/core-schema-domain-splitter.ts src/core/services/core-schema-domain-splitter.ts tests/core/services/core-schema-domain-splitter.test.ts +git commit -m "feat(mcp): compact several Core schema domains in one schema" +``` + +--- + +### Task 4: `intent` input on `fetch-core-graphql-schema` + +**Files:** +- Modify: `src/core/mcp/tools/fetch-core-graphql-schema.ts` +- Test: `tests/core/mcp/tools/fetch-core-graphql-schema.test.ts` + +**Interfaces:** +- Consumes: `CoreDomainSelector`, `SelectedDomain` (Task 2); `getCompactedDomainsSchema` (Task 3). +- Produces: tool `Deps` gains `coreDomainSelector: CoreDomainSelector` (Task 5 registers it); input schema gains `intent?: string`. + +- [ ] **Step 1: Write the failing test** + +`tests/core/mcp/tools/fetch-core-graphql-schema.test.ts`: + +```ts +import { describe, it, expect, mock, beforeEach, afterEach } from "bun:test"; +import { createFetchCoreGraphqlSchemaToolWrapper } from "../../../../src/core/mcp/tools/fetch-core-graphql-schema"; +import { createCoreSchemaDomainSplitter } from "../../../../src/core/services/core-schema-domain-splitter"; +import type { CoreDomainSelector } from "../../../../src/contracts/core-domain-selector"; +import type { TenantMatcher } from "../../../../src/contracts/tenant-matcher"; +import type { AuthContextResolver } from "../../../../src/contracts/auth-context-resolver"; +import { buildIntrospectionFromSDL, testAuthContext, testTenants } from "../../../utils/fixtures"; + +const SDL = ` + type Query { + order(id: ID!): Order + customer(id: ID!): Customer + subscription(id: ID!): Subscription + } + type Mutation { + createOrder(total: Float!): Order! + createCustomer(name: String!): Customer! + } + type Order { id: ID! total: Float! } + type Customer { id: ID! name: String! } + type Subscription { id: ID! plan: String! } +`; + +const originalFetch = globalThis.fetch; + +describe("fetch-core-graphql-schema", () => { + let select: ReturnType; + let tool: ReturnType; + + const run = async (input: { domain?: string; intent?: string }) => { + const result = await tool.handler({ tenant: "shop", ...input, authContext: testAuthContext }); + return result.content[0].text; + }; + + beforeEach(() => { + const introspection = buildIntrospectionFromSDL(SDL); + globalThis.fetch = mock(async () => new Response(JSON.stringify(introspection))) as unknown as typeof fetch; + select = mock(async () => []); + tool = createFetchCoreGraphqlSchemaToolWrapper({ + coreSchemaDomainSplitter: createCoreSchemaDomainSplitter(), + coreDomainSelector: { select } as unknown as CoreDomainSelector, + tenantMatcher: (() => testTenants[0]) as unknown as TenantMatcher, + authContextResolver: { getAuthHeaders: () => ({}) } as unknown as AuthContextResolver, + }); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("returns the domain index when neither domain nor intent is given", async () => { + const text = await run({}); + expect(text).toContain("# Core API Schema Domains"); + expect(select).not.toHaveBeenCalled(); + }); + + it("returns the schema of every domain the selector picks for an intent", async () => { + select.mockImplementation(async () => [ + { name: "order", probability: 0.97 }, + { name: "customer", probability: 0.81 }, + ]); + const text = await run({ intent: "create an order for a customer" }); + + expect(select.mock.calls[0][1]).toBe("create an order for a customer"); + expect(text).toContain("order (97%), customer (81%)"); + expect(text).toContain("createOrder"); + expect(text).toContain("createCustomer"); + expect(text).not.toContain("Subscription"); + }); + + it("falls back to the index when the selector picks nothing", async () => { + const text = await run({ intent: "bake a cake" }); + expect(text).toContain("Could not pick domains for this intent"); + expect(text).toContain("# Core API Schema Domains"); + }); + + it("falls back to the index when the selector throws", async () => { + select.mockImplementation(async () => { + throw new Error("Workers AI is down"); + }); + const text = await run({ intent: "create an order" }); + expect(text).toContain("Could not pick domains for this intent"); + expect(text).not.toContain("Failed to fetch core schema"); + }); + + it("treats a blank intent as no intent", async () => { + const text = await run({ intent: " " }); + expect(text).toContain("# Core API Schema Domains"); + expect(text).not.toContain("Could not pick domains"); + expect(select).not.toHaveBeenCalled(); + }); + + it("prefers an exact domain over an intent", async () => { + const text = await run({ domain: "subscription", intent: "create an order" }); + expect(text).toContain("Subscription"); + expect(text).not.toContain("createOrder"); + expect(select).not.toHaveBeenCalled(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test tests/core/mcp/tools/fetch-core-graphql-schema.test.ts` +Expected: FAIL. The intent tests get the plain index back (no `order (97%)`, no `Could not pick domains`), and `tsc` would flag the unknown `coreDomainSelector` dep. + +- [ ] **Step 3: Write minimal implementation** + +In `src/core/mcp/tools/fetch-core-graphql-schema.ts`: + +1. Add imports and the dep: + +```ts +import type { CoreDomainSelector, SelectedDomain } from "../../../contracts/core-domain-selector"; +``` + +```ts +type Deps = { + coreSchemaDomainSplitter: CoreSchemaDomainSplitter; + coreDomainSelector: CoreDomainSelector; + tenantMatcher: TenantMatcher; + authContextResolver: AuthContextResolver; +}; +``` + +2. Add below `formatDomainIndex`: + +```ts +function formatSelectionHeader(selected: SelectedDomain[]): string { + const picked = selected.map((d) => `${d.name} (${Math.round(d.probability * 100)}%)`).join(", "); + return ( + `# Core API schema for: ${selected.map((d) => d.name).join(", ")}\n\n` + + `Domains picked for your intent: ${picked}. If something is missing, call again with \`domain\` ` + + "set to the one you need, or with neither `intent` nor `domain` for the full list.\n\n" + ); +} + +const NO_SELECTION_NOTE = + "Could not pick domains for this intent automatically. Call again with `domain` set to one of the following.\n\n"; +``` + +3. Replace the factory's destructuring, `description`, `inputSchema` and `handler` with: + +```ts +export const createFetchCoreGraphqlSchemaToolWrapper = ({ + coreSchemaDomainSplitter, + coreDomainSelector, + tenantMatcher, + authContextResolver, +}: Deps) => { + return defineToolWrapper({ + description: + "Fetch the compacted GraphQL schema of the Crystallize Core API (aka Core Next) for a given tenant. " + + "BEFORE calling this tool, call the `skills` tool first to get documentation and query examples — " + + "skills often provide enough context to build queries without needing the full schema. " + + "The Core API schema is large, so it is split into domains. Prefer passing `intent`: a one-sentence " + + "description of what you want to do (e.g. 'create an order for an existing customer'); the server " + + "then returns the schema of every domain that task needs, in one call. " + + "Pass `domain` instead when you already know the exact domain. " + + "Common domains: order, customer, subscription, subscriptionPlan, pricelist, pipeline, flow, app, user, webhook, stockLocation, invite. " + + "Call with neither to get the full list of domains. " + + "The Core API is the admin API — use it for orders, customers, price lists, users, subscriptions, " + + "subscription plans, pipelines, flows, apps, and other back-office/admin resources. " + + "Do NOT use this for fetching items or products for storefronts — use Catalogue or Discovery APIs instead.", + inputSchema: z.object({ + tenant: tenantSchema, + domain: z + .string() + .optional() + .describe( + "The exact domain to fetch the schema for (e.g. 'order', 'customer', 'subscription'). " + + "Takes precedence over `intent`.", + ), + intent: z + .string() + .max(1000) + .optional() + .describe( + "What you want to do with the Core API, in one sentence " + + "(e.g. 'list the orders of a customer and their subscriptions'). " + + "The server picks the relevant domains and returns their combined schema.", + ), + }), + annotations: { + readOnlyHint: true, + }, + handler: async ({ tenant, domain, intent, authContext }) => { + const matchedTenant = tenantMatcher(authContext.tenants, { identifier: tenant }); + const url = buildAtApiUrl("https://api.crystallize.com", matchedTenant.identifier, ""); + const headers: Record = authContextResolver.getAuthHeaders(authContext); + const text = (value: string) => ({ content: [{ type: "text" as const, text: value }] }); + try { + const introspection = await fetchIntrospection(url, headers); + const index = coreSchemaDomainSplitter.listDomains(introspection); + + if (domain) { + if (!index.domains.some((d) => d.name === domain)) { + return text(`Unknown domain "${domain}". Here are the available domains:\n\n${formatDomainIndex(index)}`); + } + return text(coreSchemaDomainSplitter.getCompactedDomainSchema(introspection, domain, "both")); + } + + if (intent?.trim()) { + let selected: SelectedDomain[] = []; + try { + selected = await coreDomainSelector.select(index, intent); + } catch (error) { + // Selection is an optimization: when Jev is unavailable, the index still works. + console.warn(`Core domain selection failed: ${sanitizeErrorMessage(error)}`); + } + if (selected.length === 0) { + return text(NO_SELECTION_NOTE + formatDomainIndex(index)); + } + const schema = coreSchemaDomainSplitter.getCompactedDomainsSchema( + introspection, + selected.map((d) => d.name), + "both", + ); + return text(formatSelectionHeader(selected) + schema); + } + + return text(formatDomainIndex(index)); + } catch (error) { + return text(`Failed to fetch core schema: ${sanitizeErrorMessage(error)}`); + } + }, + }); +}; +``` + +Wrap the `Unknown domain` line if `bun lint` or the 120-char rule complains. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test tests/core/mcp/tools/fetch-core-graphql-schema.test.ts` +Expected: PASS (6 tests). Then run `bun test` for the whole suite: all green. + +- [ ] **Step 5: Commit** + +```bash +bun lint +git add src/core/mcp/tools/fetch-core-graphql-schema.ts tests/core/mcp/tools/fetch-core-graphql-schema.test.ts +git commit -m "feat(mcp): accept an intent on fetch-core-graphql-schema" +``` + +--- + +### Task 5: Wire the Workers AI binding and the services + +**Files:** +- Modify: `wrangler.jsonc` (add `ai` binding) +- Modify: `worker-configuration.d.ts` (regenerated, do not hand-edit) +- Modify: `src/core/container.ts` +- Modify: `CLAUDE.md` (container registrations, tool table, platform bindings) + +**Interfaces:** +- Consumes: `createJevClient`, `JevBinding` (Task 1); `createCoreDomainSelector` (Task 2); tool `Deps.coreDomainSelector` (Task 4). +- Produces: container keys `ai`, `jevClient`, `coreDomainSelector`. + +- [ ] **Step 1: Add the binding** + +In `wrangler.jsonc`, after the `"placement"` block, add: + +```jsonc + // Workers AI, for TypeSafe's Jev model (typesafe/jev): picks Core schema domains for an intent. + "ai": { + "binding": "AI", + }, +``` + +- [ ] **Step 2: Regenerate the bindings type** + +Run: `bun cf-typegen` +Expected: `worker-configuration.d.ts` now has `AI: Ai;` in `interface Env`. + +- [ ] **Step 3: Register the services** + +In `src/core/container.ts`: + +```ts +import { asFunction, asValue, createContainer, InferCradleFromContainer, InjectionMode } from "awilix"; +``` + +```ts +import { createJevClient, type JevBinding } from "./services/jev-client"; +import { createCoreDomainSelector } from "./services/core-domain-selector"; +``` + +Change `const build = () =>` to `const build = (env: CloudflareBindings) =>` and add to the services block, after `coreSchemaDomainSplitter`: + +```ts + // Bindings are fixed for the isolate's lifetime, unlike request-derived values, so the + // cached container can hold them. The cast: the generated `Ai` types predate typesafe/jev. + ai: asValue(env.AI as unknown as JevBinding), + jevClient: asFunction(createJevClient).singleton(), + coreDomainSelector: asFunction(createCoreDomainSelector).singleton(), +``` + +Replace the `buildContainer` line with: + +```ts +export const buildContainer = (env: CloudflareBindings) => (container ??= build(env)); +``` + +- [ ] **Step 4: Update CLAUDE.md** + +- Under **Services** (singletons), add after `coreSchemaDomainSplitter`: + - `` `ai` — the Workers AI binding (`env.AI`), held by the cached container because bindings are isolate-stable `` + - `` `jevClient` — runs TypeSafe's Jev (`typesafe/jev`) on Workers AI `` + - `` `coreDomainSelector` — picks the Core schema domains an `intent` needs (one Jev Noul per domain; falls back to the index on failure) `` +- In the Tool Registry table, change the `fetch-core-graphql-schema` purpose to `Get compacted Core schema (by domain or intent)`. +- Under **Platform**, add: `` **Bindings**: `AI` (Workers AI; used for `typesafe/jev`). Local dev calls the remote model, so `wrangler login` is needed for intent selection; without it the tool falls back to the domain index. `` +- In the project-structure tree, add `jev.ts`, `core-domain-selector.ts` under `contracts/` and `jev-client.ts`, `core-domain-selector.ts` under `services/`, one-line comments each. + +- [ ] **Step 5: Verify** + +Run: `bun type-check && bun test && bun lint` +Expected: no type errors (the known `@ts-expect-error` in `services-provider.ts` stays used), all tests pass, lint clean. + +Run: `bun run build` +Expected: build succeeds and `dist/crystallize_mcp_server/wrangler.json` contains `"ai": { "binding": "AI" }`. + +- [ ] **Step 6: Commit** + +```bash +git add wrangler.jsonc worker-configuration.d.ts src/core/container.ts CLAUDE.md +git commit -m "feat(mcp): bind Workers AI and register the Jev domain selector" +``` + +--- + +### Task 6: Evaluate against real Jev and tune the threshold + +**Files:** +- Create: `bin/eval-domain-selection.ts` +- Create: `bin/eval-domain-selection.cases.json` + +**Interfaces:** +- Consumes: `scoreDomains`, `pickDomains`, `MAX_DOMAINS` (Task 2); `createCoreSchemaDomainSplitter`; `fetchIntrospection`; `buildAtApiUrl`; `JevClient` (Task 1). + +This needs real credentials. Ask the user for them; never commit them. Required env: `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN` (Workers AI permission), `CRYSTALLIZE_TENANT`, `CRYSTALLIZE_TOKEN_ID`, `CRYSTALLIZE_TOKEN_SECRET`. + +- [ ] **Step 1: Write the cases** + +`bin/eval-domain-selection.cases.json`, with each `expected` listing the domains the intent truly needs: + +```json +[ + { "intent": "list the last 10 orders", "expected": ["order"] }, + { "intent": "create an order for an existing customer", "expected": ["order", "customer"] }, + { "intent": "find a customer by email and show their subscriptions", "expected": ["customer", "subscription"] }, + { "intent": "create a monthly subscription plan with a 14 day trial", "expected": ["subscriptionPlan"] }, + { "intent": "add a price list with 20% off for B2B customers", "expected": ["pricelist"] }, + { "intent": "move an order to the shipped stage of the fulfilment pipeline", "expected": ["pipeline", "order"] }, + { "intent": "set up a webhook that fires when an order is created", "expected": ["webhook"] }, + { "intent": "invite a new colleague to the tenant as an admin", "expected": ["invite"] }, + { "intent": "list all users and their roles", "expected": ["user"] }, + { "intent": "create a new warehouse for stock", "expected": ["stockLocation"] }, + { "intent": "cancel a customer's subscription", "expected": ["subscription"] }, + { "intent": "install an app on the tenant", "expected": ["app"] } +] +``` + +- [ ] **Step 2: Write the script** + +`bin/eval-domain-selection.ts`: + +```ts +import { readFileSync } from "node:fs"; +import type { JevClient, JevResponse } from "../src/contracts/jev"; +import { JEV_MODEL } from "../src/core/services/jev-client"; +import { MAX_DOMAINS, pickDomains, scoreDomains } from "../src/core/services/core-domain-selector"; +import { createCoreSchemaDomainSplitter } from "../src/core/services/core-schema-domain-splitter"; +import { fetchIntrospection } from "../src/core/services/compact-schema-builder"; +import { buildAtApiUrl } from "../src/core/security"; + +const env = (name: string) => { + const value = process.env[name]; + if (!value) { + console.error(`Missing env var ${name}`); + process.exit(1); + } + return value; +}; + +const accountId = env("CLOUDFLARE_ACCOUNT_ID"); +const apiToken = env("CLOUDFLARE_API_TOKEN"); + +// Same request shape as the binding, over the REST API, so the eval runs outside a Worker. +const jevClient: JevClient = async (request) => { + const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run`, { + method: "POST", + headers: { Authorization: `Bearer ${apiToken}`, "Content-Type": "application/json" }, + body: JSON.stringify({ model: JEV_MODEL, input: request }), + }); + const json = (await response.json()) as { success: boolean; result: JevResponse; errors?: unknown }; + if (!json.success) throw new Error(`Workers AI error: ${JSON.stringify(json.errors)}`); + return json.result; +}; + +const url = buildAtApiUrl("https://api.crystallize.com", env("CRYSTALLIZE_TENANT"), ""); +const introspection = await fetchIntrospection(url, { + "X-Crystallize-Access-Token-Id": env("CRYSTALLIZE_TOKEN_ID"), + "X-Crystallize-Access-Token-Secret": env("CRYSTALLIZE_TOKEN_SECRET"), +}); +const index = createCoreSchemaDomainSplitter().listDomains(introspection); +const names = new Set(index.domains.map((d) => d.name)); +console.log(`${index.domains.length} domains: ${[...names].join(", ")}\n`); + +const cases = JSON.parse(readFileSync(new URL("./eval-domain-selection.cases.json", import.meta.url), "utf-8")) as { + intent: string; + expected: string[]; +}[]; + +const THRESHOLDS = [0.2, 0.3, 0.5]; +const totals = new Map(THRESHOLDS.map((t) => [t, { fullRecall: 0, returned: 0 }])); + +for (const c of cases) { + const missingFromIndex = c.expected.filter((e) => !names.has(e)); + if (missingFromIndex.length > 0) console.warn(`! case expects unknown domains: ${missingFromIndex.join(", ")}`); + + const scored = await scoreDomains(jevClient, index, c.intent); + const top = [...scored].sort((a, b) => b.probability - a.probability).slice(0, 6); + console.log(`- ${c.intent}\n expected: ${c.expected.join(", ")}`); + console.log(` top: ${top.map((d) => `${d.name} ${d.probability.toFixed(2)}`).join(", ")}`); + + for (const t of THRESHOLDS) { + const picked = pickDomains(scored, t, MAX_DOMAINS).map((d) => d.name); + const totalsForT = totals.get(t)!; + if (c.expected.every((e) => picked.includes(e))) totalsForT.fullRecall++; + totalsForT.returned += picked.length; + } +} + +console.log("\nthreshold full-recall avg-domains-returned"); +for (const [t, { fullRecall, returned }] of totals) { + console.log(`${t.toFixed(2)} ${fullRecall}/${cases.length} ${(returned / cases.length).toFixed(2)}`); +} +``` + +- [ ] **Step 3: Run it** + +Run (the user provides the values): + +```bash +CLOUDFLARE_ACCOUNT_ID=… CLOUDFLARE_API_TOKEN=… CRYSTALLIZE_TENANT=… \ +CRYSTALLIZE_TOKEN_ID=… CRYSTALLIZE_TOKEN_SECRET=… bun run bin/eval-domain-selection.ts +``` + +Expected: the domain list prints first. Fix any case whose `expected` names are not in it (the `!` warnings), then rerun. Then a per-case block and the threshold table. + +- [ ] **Step 4: Tune** + +Pick the highest threshold whose full-recall equals the best full-recall in the table (fewest extra domains without losing needed ones). If it differs from `0.3`, update `DOMAIN_THRESHOLD` in `src/core/services/core-domain-selector.ts` and the value in this plan's Global Constraints. If full-recall is below 10/12 at every threshold, stop and report the failing cases to the user; the fix is in the question wording or domain naming, not the threshold. + +Also note one Jev response's input-token usage for the prompt, to confirm it sits well under the 32k context (log `(await jevClient(...)).usage` once if needed). + +- [ ] **Step 5: Commit** + +```bash +bun lint && bun test +git add bin/eval-domain-selection.ts bin/eval-domain-selection.cases.json src/core/services/core-domain-selector.ts +git commit -m "chore(mcp): add a Jev domain-selection eval and tune the threshold" +``` + +--- + +## Out of scope (follow-ups) + +- **Skills reference selection with Jev** (a `task` input on the `skills` tool picking which `references/*.md` to return). Separate plan once this one ships. +- **Caching Core introspection** (fetched live on every call today). +- **Splitting the Catalogue/Discovery/Shop schemas.** From 9420a498a959093267bd40c0ec6403b344ea9ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:25:27 -0700 Subject: [PATCH 02/13] feat(mcp): add a Jev client over the Workers AI binding --- .../crystallize/src/contracts/jev.ts | 18 ++++++++++++++ .../src/core/services/jev-client.ts | 18 ++++++++++++++ .../tests/core/services/jev-client.test.ts | 24 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 use-crystallize/mcp-servers/crystallize/src/contracts/jev.ts create mode 100644 use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts create mode 100644 use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts diff --git a/use-crystallize/mcp-servers/crystallize/src/contracts/jev.ts b/use-crystallize/mcp-servers/crystallize/src/contracts/jev.ts new file mode 100644 index 0000000..9991a75 --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/src/contracts/jev.ts @@ -0,0 +1,18 @@ +export type JevNoulQuestion = { + type: "noul"; + instructions: string; + criteria?: { true: string; false: string }; +}; + +export type JevRequest = { + state: unknown; + questions: Record; +}; + +export type JevResponse = { + model: string; + answers: Record; + usage?: { input_tokens: number; output_tokens: number }; +}; + +export type JevClient = (request: JevRequest) => Promise; diff --git a/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts b/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts new file mode 100644 index 0000000..6d17230 --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts @@ -0,0 +1,18 @@ +import type { JevClient, JevRequest, JevResponse } from "../../contracts/jev"; + +export const JEV_MODEL = "typesafe/jev"; + +// The generated `Ai` types predate typesafe/jev, so the binding is narrowed to the one call we make. +export type JevBinding = { + run(model: typeof JEV_MODEL, input: JevRequest): Promise; +}; + +export const createJevClient = + ({ ai }: { ai: JevBinding }): JevClient => + async (request) => { + const response = (await ai.run(JEV_MODEL, request)) as Partial | null; + if (!response || typeof response !== "object" || !response.answers) { + throw new Error("Jev returned no answers"); + } + return response as JevResponse; + }; diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts new file mode 100644 index 0000000..e9d6ae9 --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts @@ -0,0 +1,24 @@ +import { describe, it, expect, mock } from "bun:test"; +import { createJevClient, JEV_MODEL, type JevBinding } from "../../../src/core/services/jev-client"; +import type { JevRequest } from "../../../src/contracts/jev"; + +const request: JevRequest = { + state: { intent: "list orders" }, + questions: { d0: { type: "noul", instructions: "Is the order domain needed?" } }, +}; + +describe("createJevClient", () => { + it("runs typesafe/jev with the request and returns the response", async () => { + const response = { model: "jev-1.13.0", answers: { d0: { type: "noul", noul: 0.9 } } }; + const run = mock(async () => response); + const client = createJevClient({ ai: { run } as JevBinding }); + + expect(await client(request)).toEqual(response); + expect(run).toHaveBeenCalledWith(JEV_MODEL, request); + }); + + it("throws when the response has no answers", async () => { + const client = createJevClient({ ai: { run: async () => ({}) } as JevBinding }); + await expect(client(request)).rejects.toThrow("Jev returned no answers"); + }); +}); From 2fc7c6ab96013ec426f47afe643577cf1d1c7fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:25:57 -0700 Subject: [PATCH 03/13] feat(mcp): select Core schema domains for an intent with Jev --- .../src/contracts/core-domain-selector.ts | 10 +++ .../src/core/services/core-domain-selector.ts | 70 ++++++++++++++++ .../services/core-domain-selector.test.ts | 84 +++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 use-crystallize/mcp-servers/crystallize/src/contracts/core-domain-selector.ts create mode 100644 use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts create mode 100644 use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts diff --git a/use-crystallize/mcp-servers/crystallize/src/contracts/core-domain-selector.ts b/use-crystallize/mcp-servers/crystallize/src/contracts/core-domain-selector.ts new file mode 100644 index 0000000..b88ab03 --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/src/contracts/core-domain-selector.ts @@ -0,0 +1,10 @@ +import type { DomainIndex } from "./core-schema-domain-splitter"; + +export type SelectedDomain = { + name: string; + probability: number; +}; + +export type CoreDomainSelector = { + select(index: DomainIndex, intent: string): Promise; +}; diff --git a/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts b/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts new file mode 100644 index 0000000..48de794 --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts @@ -0,0 +1,70 @@ +import type { CoreDomainSelector, SelectedDomain } from "../../contracts/core-domain-selector"; +import type { DomainIndex } from "../../contracts/core-schema-domain-splitter"; +import type { JevClient, JevRequest } from "../../contracts/jev"; + +/** + * Missing a needed domain costs the agent another round-trip; an extra domain only costs output size. + * So the threshold leans low and the cap bounds the output. + */ +export const DOMAIN_THRESHOLD = 0.3; +export const MAX_DOMAINS = 4; + +/** + * One Noul per domain over a shared state, sent as a single request (Jev answers them in parallel). + * Question ids are opaque to the model, so each instruction names its domain in full. + */ +export function buildDomainRequest( + index: DomainIndex, + intent: string, +): { request: JevRequest; domainByQuestionId: Map } { + const domainByQuestionId = new Map(); + const questions: JevRequest["questions"] = {}; + index.domains.forEach((domain, i) => { + const id = `d${i}`; + domainByQuestionId.set(id, domain.name); + questions[id] = { + type: "noul", + instructions: + `Does accomplishing \`intent\` against the Crystallize Core API require at least one of the ` + + `queries or mutations listed for the "${domain.name}" domain in \`domains\`?`, + criteria: { + true: `At least one "${domain.name}" query or mutation is needed to read or write what the intent is about`, + false: `The intent can be accomplished without any "${domain.name}" query or mutation`, + }, + }; + }); + return { request: { state: { intent, domains: index.domains }, questions }, domainByQuestionId }; +} + +export async function scoreDomains( + jevClient: JevClient, + index: DomainIndex, + intent: string, +): Promise { + const { request, domainByQuestionId } = buildDomainRequest(index, intent); + const response = await jevClient(request); + const scored: SelectedDomain[] = []; + for (const [id, name] of domainByQuestionId) { + const answer = response.answers[id]; + if (answer) scored.push({ name, probability: answer.noul }); + } + return scored; +} + +export function pickDomains( + scored: SelectedDomain[], + threshold = DOMAIN_THRESHOLD, + max = MAX_DOMAINS, +): SelectedDomain[] { + return scored + .filter((d) => d.probability >= threshold) + .sort((a, b) => b.probability - a.probability) + .slice(0, max); +} + +export const createCoreDomainSelector = ({ jevClient }: { jevClient: JevClient }): CoreDomainSelector => ({ + async select(index, intent) { + if (!intent.trim()) return []; + return pickDomains(await scoreDomains(jevClient, index, intent)); + }, +}); diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts new file mode 100644 index 0000000..ab1d0d5 --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect, mock } from "bun:test"; +import { + buildDomainRequest, + createCoreDomainSelector, + pickDomains, + scoreDomains, +} from "../../../src/core/services/core-domain-selector"; +import type { DomainIndex } from "../../../src/contracts/core-schema-domain-splitter"; +import type { JevClient, JevResponse } from "../../../src/contracts/jev"; + +const index: DomainIndex = { + domains: [ + { name: "customer", queries: ["customer", "customers"], mutations: ["createCustomer"] }, + { name: "order", queries: ["order", "orders"], mutations: ["createOrder"] }, + { name: "subscription", queries: ["subscription"], mutations: [] }, + ], +}; + +const answers = (nouls: Record): JevResponse => ({ + model: "jev-1.13.0", + answers: Object.fromEntries(Object.entries(nouls).map(([id, noul]) => [id, { type: "noul", noul }])), +}); + +describe("buildDomainRequest", () => { + it("puts the intent and the domain index in state and asks one noul per domain", () => { + const { request, domainByQuestionId } = buildDomainRequest(index, "create an order"); + + expect(request.state).toEqual({ intent: "create an order", domains: index.domains }); + expect(Object.keys(request.questions)).toEqual(["d0", "d1", "d2"]); + expect(domainByQuestionId.get("d1")).toBe("order"); + expect(request.questions.d1.type).toBe("noul"); + expect(request.questions.d1.instructions).toContain('"order"'); + expect(request.questions.d1.criteria?.true).toContain("order"); + }); +}); + +describe("pickDomains", () => { + it("keeps domains at or above the threshold, most likely first, capped", () => { + const picked = pickDomains( + [ + { name: "a", probability: 0.2 }, + { name: "b", probability: 0.9 }, + { name: "c", probability: 0.3 }, + { name: "d", probability: 0.6 }, + ], + 0.3, + 2, + ); + expect(picked).toEqual([ + { name: "b", probability: 0.9 }, + { name: "d", probability: 0.6 }, + ]); + }); +}); + +describe("scoreDomains", () => { + it("maps answers back to domain names and skips missing answers", async () => { + const jevClient: JevClient = async () => answers({ d0: 0.4, d1: 0.95 }); + expect(await scoreDomains(jevClient, index, "create an order")).toEqual([ + { name: "customer", probability: 0.4 }, + { name: "order", probability: 0.95 }, + ]); + }); +}); + +describe("createCoreDomainSelector", () => { + it("returns the picked domains for an intent", async () => { + const jevClient: JevClient = async () => answers({ d0: 0.7, d1: 0.95, d2: 0.05 }); + const selector = createCoreDomainSelector({ jevClient }); + + expect(await selector.select(index, "create an order for a customer")).toEqual([ + { name: "order", probability: 0.95 }, + { name: "customer", probability: 0.7 }, + ]); + }); + + it("does not call Jev for a blank intent", async () => { + const jevClient = mock(async () => answers({})); + const selector = createCoreDomainSelector({ jevClient }); + + expect(await selector.select(index, " ")).toEqual([]); + expect(jevClient).not.toHaveBeenCalled(); + }); +}); From 8563bb46b007ea83a19fb4813011aef93bc21730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:26:19 -0700 Subject: [PATCH 04/13] feat(mcp): compact several Core schema domains in one schema --- .../contracts/core-schema-domain-splitter.ts | 5 ++++ .../services/core-schema-domain-splitter.ts | 26 +++++++++++++------ .../core-schema-domain-splitter.test.ts | 16 ++++++++++++ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/use-crystallize/mcp-servers/crystallize/src/contracts/core-schema-domain-splitter.ts b/use-crystallize/mcp-servers/crystallize/src/contracts/core-schema-domain-splitter.ts index 5fe4dca..0e4a7f7 100644 --- a/use-crystallize/mcp-servers/crystallize/src/contracts/core-schema-domain-splitter.ts +++ b/use-crystallize/mcp-servers/crystallize/src/contracts/core-schema-domain-splitter.ts @@ -17,4 +17,9 @@ export type CoreSchemaDomainSplitter = { domain: string, operations: "queries" | "mutations" | "both", ): string; + getCompactedDomainsSchema( + introspection: IntrospectionResult, + domains: string[], + operations: "queries" | "mutations" | "both", + ): string; }; diff --git a/use-crystallize/mcp-servers/crystallize/src/core/services/core-schema-domain-splitter.ts b/use-crystallize/mcp-servers/crystallize/src/core/services/core-schema-domain-splitter.ts index 662812b..9f77337 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/services/core-schema-domain-splitter.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/services/core-schema-domain-splitter.ts @@ -124,20 +124,30 @@ export const createCoreSchemaDomainSplitter = (): CoreSchemaDomainSplitter => { introspection: IntrospectionResult, domain: string, operations: "queries" | "mutations" | "both", + ): string { + return this.getCompactedDomainsSchema(introspection, [domain], operations); + }, + + getCompactedDomainsSchema( + introspection: IntrospectionResult, + domains: string[], + operations: "queries" | "mutations" | "both", ): string { const index = this.listDomains(introspection); - const domainInfo = index.domains.find((d) => d.name === domain); - if (!domainInfo) { + const unknown = domains.filter((name) => !index.domains.some((d) => d.name === name)); + if (unknown.length > 0) { const available = index.domains.map((d) => d.name).join(", "); - return `Unknown domain "${domain}". Available domains: ${available}`; + return `Unknown domain "${unknown.join('", "')}". Available domains: ${available}`; } const fieldNames = new Set(); - if (operations === "queries" || operations === "both") { - for (const f of domainInfo.queries) fieldNames.add(f); - } - if (operations === "mutations" || operations === "both") { - for (const f of domainInfo.mutations) fieldNames.add(f); + for (const domainInfo of index.domains.filter((d) => domains.includes(d.name))) { + if (operations === "queries" || operations === "both") { + for (const f of domainInfo.queries) fieldNames.add(f); + } + if (operations === "mutations" || operations === "both") { + for (const f of domainInfo.mutations) fieldNames.add(f); + } } return compactSchemaFromIntrospection(introspection, { diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/services/core-schema-domain-splitter.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/services/core-schema-domain-splitter.test.ts index b967779..94c2510 100644 --- a/use-crystallize/mcp-servers/crystallize/tests/core/services/core-schema-domain-splitter.test.ts +++ b/use-crystallize/mcp-servers/crystallize/tests/core/services/core-schema-domain-splitter.test.ts @@ -142,4 +142,20 @@ describe("createCoreSchemaDomainSplitter", () => { expect(schema).not.toContain("# Mutations"); }); }); + + describe("getCompactedDomainsSchema", () => { + it("returns the union of the requested domains' root fields", () => { + const schema = splitter.getCompactedDomainsSchema(introspection, ["order", "subscription"], "both"); + + expect(schema).toContain("createOrder"); + expect(schema).toContain("Subscription"); + expect(schema).not.toContain("createCustomer"); + }); + + it("names every unknown domain", () => { + const result = splitter.getCompactedDomainsSchema(introspection, ["order", "nope", "nada"], "both"); + expect(result).toContain('Unknown domain "nope", "nada"'); + expect(result).toContain("Available domains:"); + }); + }); }); From ec956880ec42cb6f1e6124310ffb2901eb50c3fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:27:28 -0700 Subject: [PATCH 05/13] feat(mcp): accept an intent on fetch-core-graphql-schema --- .../mcp/tools/fetch-core-graphql-schema.ts | 98 ++++++++++------- .../tools/fetch-core-graphql-schema.test.ts | 101 ++++++++++++++++++ 2 files changed, 161 insertions(+), 38 deletions(-) create mode 100644 use-crystallize/mcp-servers/crystallize/tests/core/mcp/tools/fetch-core-graphql-schema.test.ts diff --git a/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts b/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts index f9e4cd3..9eee997 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts @@ -3,11 +3,13 @@ import { defineToolWrapper } from "../../../contracts/tool"; import { TenantMatcher } from "../../../contracts/tenant-matcher"; import { AuthContextResolver } from "../../../contracts/auth-context-resolver"; import { CoreSchemaDomainSplitter, DomainIndex } from "../../../contracts/core-schema-domain-splitter"; +import type { CoreDomainSelector, SelectedDomain } from "../../../contracts/core-domain-selector"; import { fetchIntrospection } from "../../services/compact-schema-builder"; import { tenantSchema, sanitizeErrorMessage, buildAtApiUrl } from "../../security"; type Deps = { coreSchemaDomainSplitter: CoreSchemaDomainSplitter; + coreDomainSelector: CoreDomainSelector; tenantMatcher: TenantMatcher; authContextResolver: AuthContextResolver; }; @@ -33,8 +35,21 @@ function formatDomainIndex(index: DomainIndex): string { return lines.join("\n"); } +function formatSelectionHeader(selected: SelectedDomain[]): string { + const picked = selected.map((d) => `${d.name} (${Math.round(d.probability * 100)}%)`).join(", "); + return ( + `# Core API schema for: ${selected.map((d) => d.name).join(", ")}\n\n` + + `Domains picked for your intent: ${picked}. If something is missing, call again with \`domain\` ` + + "set to the one you need, or with neither `intent` nor `domain` for the full list.\n\n" + ); +} + +const NO_SELECTION_NOTE = + "Could not pick domains for this intent automatically. Call again with `domain` set to one of the following.\n\n"; + export const createFetchCoreGraphqlSchemaToolWrapper = ({ coreSchemaDomainSplitter, + coreDomainSelector, tenantMatcher, authContextResolver, }: Deps) => { @@ -43,9 +58,12 @@ export const createFetchCoreGraphqlSchemaToolWrapper = ({ "Fetch the compacted GraphQL schema of the Crystallize Core API (aka Core Next) for a given tenant. " + "BEFORE calling this tool, call the `skills` tool first to get documentation and query examples — " + "skills often provide enough context to build queries without needing the full schema. " + - "The Core API schema is large, so it is split into domains. You MUST provide a `domain` parameter. " + + "The Core API schema is large, so it is split into domains. Prefer passing `intent`: a one-sentence " + + "description of what you want to do (e.g. 'create an order for an existing customer'); the server " + + "then returns the schema of every domain that task needs, in one call. " + + "Pass `domain` instead when you already know the exact domain. " + "Common domains: order, customer, subscription, subscriptionPlan, pricelist, pipeline, flow, app, user, webhook, stockLocation, invite. " + - "If the domain you need is not listed, call without `domain` to get the full list. " + + "Call with neither to get the full list of domains. " + "The Core API is the admin API — use it for orders, customers, price lists, users, subscriptions, " + "subscription plans, pipelines, flows, apps, and other back-office/admin resources. " + "Do NOT use this for fetching items or products for storefronts — use Catalogue or Discovery APIs instead.", @@ -55,58 +73,62 @@ export const createFetchCoreGraphqlSchemaToolWrapper = ({ .string() .optional() .describe( - "The domain to fetch the schema for (e.g. 'order', 'customer', 'subscription'). " + - "Omit to get the full list of available domains.", + "The exact domain to fetch the schema for (e.g. 'order', 'customer', 'subscription'). " + + "Takes precedence over `intent`.", + ), + intent: z + .string() + .max(1000) + .optional() + .describe( + "What you want to do with the Core API, in one sentence " + + "(e.g. 'list the orders of a customer and their subscriptions'). " + + "The server picks the relevant domains and returns their combined schema.", ), }), annotations: { readOnlyHint: true, }, - handler: async ({ tenant, domain, authContext }) => { + handler: async ({ tenant, domain, intent, authContext }) => { const matchedTenant = tenantMatcher(authContext.tenants, { identifier: tenant }); const url = buildAtApiUrl("https://api.crystallize.com", matchedTenant.identifier, ""); const headers: Record = authContextResolver.getAuthHeaders(authContext); + const text = (value: string) => ({ content: [{ type: "text" as const, text: value }] }); try { const introspection = await fetchIntrospection(url, headers); + const index = coreSchemaDomainSplitter.listDomains(introspection); - if (!domain) { - return { - content: [ - { - type: "text", - text: formatDomainIndex(coreSchemaDomainSplitter.listDomains(introspection)), - }, - ], - }; + if (domain) { + if (!index.domains.some((d) => d.name === domain)) { + return text( + `Unknown domain "${domain}". Here are the available domains:\n\n${formatDomainIndex(index)}`, + ); + } + return text(coreSchemaDomainSplitter.getCompactedDomainSchema(introspection, domain, "both")); } - // Check if domain exists, if not return available domains - const index = coreSchemaDomainSplitter.listDomains(introspection); - const domainExists = index.domains.some((d) => d.name === domain); - if (!domainExists) { - return { - content: [ - { - type: "text", - text: `Unknown domain "${domain}". Here are the available domains:\n\n${formatDomainIndex(index)}`, - }, - ], - }; + if (intent?.trim()) { + let selected: SelectedDomain[] = []; + try { + selected = await coreDomainSelector.select(index, intent); + } catch (error) { + // Selection is an optimization: when Jev is unavailable, the index still works. + console.warn(`Core domain selection failed: ${sanitizeErrorMessage(error)}`); + } + if (selected.length === 0) { + return text(NO_SELECTION_NOTE + formatDomainIndex(index)); + } + const schema = coreSchemaDomainSplitter.getCompactedDomainsSchema( + introspection, + selected.map((d) => d.name), + "both", + ); + return text(formatSelectionHeader(selected) + schema); } - const schema = coreSchemaDomainSplitter.getCompactedDomainSchema(introspection, domain, "both"); - return { - content: [{ type: "text", text: schema }], - }; + return text(formatDomainIndex(index)); } catch (error) { - return { - content: [ - { - type: "text", - text: `Failed to fetch core schema: ${sanitizeErrorMessage(error)}`, - }, - ], - }; + return text(`Failed to fetch core schema: ${sanitizeErrorMessage(error)}`); } }, }); diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/mcp/tools/fetch-core-graphql-schema.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/mcp/tools/fetch-core-graphql-schema.test.ts new file mode 100644 index 0000000..1b7354e --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/tests/core/mcp/tools/fetch-core-graphql-schema.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect, mock, beforeEach } from "bun:test"; + +// Module-level, like the other schema tests: Bun's mock.module is process-wide, so stubbing +// globalThis.fetch would be bypassed whenever another file's module mock is still in place. +const mockFetchIntrospection = mock(); +mock.module("../../../../src/core/services/compact-schema-builder", () => { + const actual = require("../../../../src/core/services/compact-schema-builder"); + return { ...actual, fetchIntrospection: mockFetchIntrospection }; +}); + +import { createFetchCoreGraphqlSchemaToolWrapper } from "../../../../src/core/mcp/tools/fetch-core-graphql-schema"; +import { createCoreSchemaDomainSplitter } from "../../../../src/core/services/core-schema-domain-splitter"; +import type { CoreDomainSelector } from "../../../../src/contracts/core-domain-selector"; +import type { TenantMatcher } from "../../../../src/contracts/tenant-matcher"; +import type { AuthContextResolver } from "../../../../src/contracts/auth-context-resolver"; +import { buildIntrospectionFromSDL, testAuthContext, testTenants } from "../../../utils/fixtures"; + +const SDL = ` + type Query { + order(id: ID!): Order + customer(id: ID!): Customer + subscription(id: ID!): Subscription + } + type Mutation { + createOrder(total: Float!): Order! + createCustomer(name: String!): Customer! + } + type Order { id: ID! total: Float! } + type Customer { id: ID! name: String! } + type Subscription { id: ID! plan: String! } +`; + +describe("fetch-core-graphql-schema", () => { + let select: ReturnType; + let tool: ReturnType; + + const run = async (input: { domain?: string; intent?: string }) => { + const result = await tool.handler({ tenant: "shop", ...input, authContext: testAuthContext }); + return result.content[0].text; + }; + + beforeEach(() => { + mockFetchIntrospection.mockImplementation(async () => buildIntrospectionFromSDL(SDL)); + select = mock(async () => []); + tool = createFetchCoreGraphqlSchemaToolWrapper({ + coreSchemaDomainSplitter: createCoreSchemaDomainSplitter(), + coreDomainSelector: { select } as unknown as CoreDomainSelector, + tenantMatcher: (() => testTenants[0]) as unknown as TenantMatcher, + authContextResolver: { getAuthHeaders: () => ({}) } as unknown as AuthContextResolver, + }); + }); + + it("returns the domain index when neither domain nor intent is given", async () => { + const text = await run({}); + expect(text).toContain("# Core API Schema Domains"); + expect(select).not.toHaveBeenCalled(); + }); + + it("returns the schema of every domain the selector picks for an intent", async () => { + select.mockImplementation(async () => [ + { name: "order", probability: 0.97 }, + { name: "customer", probability: 0.81 }, + ]); + const text = await run({ intent: "create an order for a customer" }); + + expect(select.mock.calls[0][1]).toBe("create an order for a customer"); + expect(text).toContain("order (97%), customer (81%)"); + expect(text).toContain("createOrder"); + expect(text).toContain("createCustomer"); + expect(text).not.toContain("Subscription"); + }); + + it("falls back to the index when the selector picks nothing", async () => { + const text = await run({ intent: "bake a cake" }); + expect(text).toContain("Could not pick domains for this intent"); + expect(text).toContain("# Core API Schema Domains"); + }); + + it("falls back to the index when the selector throws", async () => { + select.mockImplementation(async () => { + throw new Error("Workers AI is down"); + }); + const text = await run({ intent: "create an order" }); + expect(text).toContain("Could not pick domains for this intent"); + expect(text).not.toContain("Failed to fetch core schema"); + }); + + it("treats a blank intent as no intent", async () => { + const text = await run({ intent: " " }); + expect(text).toContain("# Core API Schema Domains"); + expect(text).not.toContain("Could not pick domains"); + expect(select).not.toHaveBeenCalled(); + }); + + it("prefers an exact domain over an intent", async () => { + const text = await run({ domain: "subscription", intent: "create an order" }); + expect(text).toContain("Subscription"); + expect(text).not.toContain("createOrder"); + expect(select).not.toHaveBeenCalled(); + }); +}); From 04b4cb814435571d7d7bcf53c14cb8d783455b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:28:33 -0700 Subject: [PATCH 06/13] feat(mcp): bind Workers AI and register the Jev domain selector --- .../mcp-servers/crystallize/CLAUDE.md | 19 +++++++++++++++---- .../crystallize/src/core/container.ts | 13 ++++++++++--- .../tests/core/services/jev-client.test.ts | 4 ++-- .../crystallize/worker-configuration.d.ts | 3 ++- .../mcp-servers/crystallize/wrangler.jsonc | 4 ++++ 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/use-crystallize/mcp-servers/crystallize/CLAUDE.md b/use-crystallize/mcp-servers/crystallize/CLAUDE.md index 5d2a342..2b3f4a4 100644 --- a/use-crystallize/mcp-servers/crystallize/CLAUDE.md +++ b/use-crystallize/mcp-servers/crystallize/CLAUDE.md @@ -28,9 +28,11 @@ src/ │ ├── analytics-tracker.ts # AnalyticsTracker / AnalyticsEvent / AnalyticsRequestContext types │ ├── app-context.ts # AppContext type (Bindings + Variables) │ ├── auth-context-resolver.ts # AuthContextResolver type (resolve client credentials) +│ ├── core-domain-selector.ts # CoreDomainSelector / SelectedDomain types (intent → domains) │ ├── core-schema-domain-splitter.ts # Core schema domain-splitting types │ ├── graphql-query-corrector.ts # Query correction types (CorrectionResult, CorrectionLog) │ ├── graphql-schema-compacter.ts # Schema compaction types (GraphqlSchemaCompacter, options) +│ ├── jev.ts # Jev request/response types + JevClient │ ├── mass-operation-runner.ts # MassOperationRunner types (task + status) │ ├── mutation-executor.ts # MutationExecutor type (execute-once, no retry) │ ├── query-executor.ts # Query execution types (QueryExecutor, options, result) @@ -48,7 +50,7 @@ src/ │ │ ├── build-mass-operation.ts # Validate a mass-operation file (read-only) │ │ ├── fetch-catalog-graphql-schema.ts # Fetch compacted Catalogue API schema │ │ ├── fetch-content-model.ts # Fetch tenant shapes/content model -│ │ ├── fetch-core-graphql-schema.ts # Fetch compacted Core API schema (by domain) +│ │ ├── fetch-core-graphql-schema.ts # Fetch compacted Core API schema (by domain or intent) │ │ ├── fetch-discovery-graphql-schema.ts # Fetch compacted Discovery API schema │ │ ├── fetch-shop-cart-graphql-schema.ts # Fetch compacted Shop Cart API schema │ │ ├── get-mass-operation-status.ts # Poll a mass-operation bulk task (read-only) @@ -65,9 +67,11 @@ src/ │ └── services/ │ ├── auth-context-helpers.ts # Resolve client credentials from auth (token/session/bearer) │ ├── compact-schema-builder.ts # GraphQL schema compaction (introspection → compact text) +│ ├── core-domain-selector.ts # Pick the Core schema domains an intent needs (Jev Nouls) │ ├── core-schema-domain-splitter.ts # Split the Core schema into queryable domains │ ├── execute-mutation.ts # Execute-once mutation executor (no correction, no retry) │ ├── graphql-query-corrector.ts # Auto-correct malformed GraphQL queries (Levenshtein) +│ ├── jev-client.ts # Run TypeSafe's Jev (typesafe/jev) on the Workers AI binding │ ├── mass-operation-runner.ts # Upload + create + start bulk tasks; read task status │ ├── query-with-correction.ts # Execute queries with auto-correction on failure │ └── tenant-matcher.ts # Match tenant by id/identifier from auth context @@ -111,6 +115,10 @@ The app uses **Awilix** for dependency injection. The container is built once (s - `tenantMatcher` — resolve tenant from auth context - `graphqlSchemaCompacter` — compact introspection schemas - `coreSchemaDomainSplitter` — split the Core schema into queryable domains +- `ai` — the Workers AI binding (`env.AI`), held by the cached container because bindings are isolate-stable +- `jevClient` — runs TypeSafe's Jev (`typesafe/jev`) on Workers AI +- `coreDomainSelector` — picks the Core schema domains an `intent` needs (one Jev Noul per domain; the tool falls + back to the domain index when it fails or picks nothing) - `graphqlQueryCorrector` — fix malformed GraphQL queries - `queryExecutor` — execute queries with auto-correction - `mutationExecutor` — execute mutations exactly once (no correction, no retry) @@ -192,7 +200,7 @@ Goals panel has fewer `tool:` rows than that list, the difference is what to cre | `fetch-content-model` | `fetchContentModelToolWrapper` | Fetch tenant shapes / content model | | `fetch-catalog-graphql-schema` | `fetchCatalogGraphqlSchemaToolWrapper` | Get compacted Catalogue schema | | `fetch-discovery-graphql-schema` | `fetchDiscoveryGraphqlSchemaToolWrapper` | Get compacted Discovery schema | -| `fetch-core-graphql-schema` | `fetchCoreGraphqlSchemaToolWrapper` | Get compacted Core schema (by domain) | +| `fetch-core-graphql-schema` | `fetchCoreGraphqlSchemaToolWrapper` | Compacted Core schema (domain or intent) | | `fetch-shop-cart-graphql-schema` | `fetchShopCartGraphqlSchemaToolWrapper` | Get compacted Shop Cart schema | | `build-mass-operation` | `buildMassOperationToolWrapper` | Validate a mass-operation file | | `get-mass-operation-status` | `getMassOperationStatusToolWrapper` | Poll a mass-operation bulk task | @@ -409,8 +417,9 @@ retroactive, so create that goal before shipping. - Visits, visit duration, bounce rate and entry/exit pages are **meaningless** for this traffic (they derive from a rolling 30-minute window over that coarse hash). That is precisely why session config lives in the path. - `analyticsTracker` is registered **`.scoped()`**, and `defer` / `analyticsRequestContext` are registered on the - request scope. `buildContainer` caches one container per isolate and ignores `env`, so anything request-derived - registered as a singleton would freeze at the first request. + request scope. `buildContainer` caches one container per isolate (built from the first request's `env`), so + anything request-derived registered as a singleton would freeze at the first request. Bindings like `AI` are + isolate-stable, which is why the container may hold them. - Localhost **and `*.workers.dev`** are skipped. `PLAUSIBLE_DOMAIN` sits in top-level `vars`, so it binds to every deploy including preview URLs; without that guard a preview would file its traffic as production. - `PLAUSIBLE_API_ENDPOINT` drives **both** sides: the server-side POST target, and the landing page's `data-api` @@ -460,6 +469,8 @@ Use `c.set()` / `c.get()` in middleware/handlers. Extend `Variables` when adding - **Deployment**: Wrangler CLI - **Smart placement** enabled in wrangler config - Bindings type generated via `bun cf-typegen` → `worker-configuration.d.ts` +- **Bindings**: `AI` (Workers AI; used for `typesafe/jev`). Local dev calls the remote model, so `wrangler login` is + needed for intent selection; without it `fetch-core-graphql-schema` falls back to the domain index. - **Vars** (plain `vars` in `wrangler.jsonc`, none are secrets): `PLAUSIBLE_DOMAIN` (site ID for server-side MCP events; unset to disable them), `PLAUSIBLE_SCRIPT_URL` (the landing page's `pa-XXXX.js`; unset to drop the script) and `PLAUSIBLE_API_ENDPOINT` (override for a self-hosted Plausible or a first-party proxy — it feeds diff --git a/use-crystallize/mcp-servers/crystallize/src/core/container.ts b/use-crystallize/mcp-servers/crystallize/src/core/container.ts index 276380a..3fbcdb3 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/container.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/container.ts @@ -1,5 +1,5 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; -import { asFunction, createContainer, InferCradleFromContainer, InjectionMode } from "awilix"; +import { asFunction, asValue, createContainer, InferCradleFromContainer, InjectionMode } from "awilix"; import packageJson from "../../package.json"; import { createFetchCatalogGraphqlSchemaToolWrapper } from "./mcp/tools/fetch-catalog-graphql-schema"; import { createFetchContentModelToolWrapper } from "./mcp/tools/fetch-content-model"; @@ -27,6 +27,8 @@ import { createMutationExecutor } from "./services/execute-mutation"; import { createMassOperationRunner } from "./services/mass-operation-runner"; import { createAuthContextResolver } from "./services/auth-context-helpers"; import { createCoreSchemaDomainSplitter } from "./services/core-schema-domain-splitter"; +import { createJevClient, type JevBinding } from "./services/jev-client"; +import { createCoreDomainSelector } from "./services/core-domain-selector"; import { createPlausibleAnalyticsTracker } from "./services/plausible-analytics-tracker"; import { AnalyticsTracker } from "../contracts/analytics-tracker"; @@ -36,7 +38,7 @@ export type Services = { analyticsTracker: AnalyticsTracker; }; -const build = () => +const build = (env: CloudflareBindings) => createContainer({ injectionMode: InjectionMode.PROXY, strict: true, @@ -46,6 +48,11 @@ const build = () => tenantMatcher: asFunction(createTenantMatcher).singleton(), graphqlSchemaCompacter: asFunction(createGraphlSchemaCompacter).singleton(), coreSchemaDomainSplitter: asFunction(createCoreSchemaDomainSplitter).singleton(), + // Bindings are fixed for the isolate's lifetime, unlike request-derived values, so the + // cached container can hold them. The cast: the generated `Ai` types predate typesafe/jev. + ai: asValue(env.AI as unknown as JevBinding), + jevClient: asFunction(createJevClient).singleton(), + coreDomainSelector: asFunction(createCoreDomainSelector).singleton(), graphqlQueryCorrector: asFunction(createGraphqlQueryCorrector).singleton(), queryExecutor: asFunction(createQueryExecutor).singleton(), mutationExecutor: asFunction(createMutationExecutor).singleton(), @@ -78,7 +85,7 @@ const build = () => }); let container: ReturnType | null = null; -export const buildContainer = (_env: CloudflareBindings) => (container ??= build()); +export const buildContainer = (env: CloudflareBindings) => (container ??= build(env)); type Container = InferCradleFromContainer>; export const toolRegistry = { diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts index e9d6ae9..4e125c9 100644 --- a/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts +++ b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, mock } from "bun:test"; import { createJevClient, JEV_MODEL, type JevBinding } from "../../../src/core/services/jev-client"; -import type { JevRequest } from "../../../src/contracts/jev"; +import type { JevRequest, JevResponse } from "../../../src/contracts/jev"; const request: JevRequest = { state: { intent: "list orders" }, @@ -9,7 +9,7 @@ const request: JevRequest = { describe("createJevClient", () => { it("runs typesafe/jev with the request and returns the response", async () => { - const response = { model: "jev-1.13.0", answers: { d0: { type: "noul", noul: 0.9 } } }; + const response: JevResponse = { model: "jev-1.13.0", answers: { d0: { type: "noul", noul: 0.9 } } }; const run = mock(async () => response); const client = createJevClient({ ai: { run } as JevBinding }); diff --git a/use-crystallize/mcp-servers/crystallize/worker-configuration.d.ts b/use-crystallize/mcp-servers/crystallize/worker-configuration.d.ts index bc09e32..247e940 100644 --- a/use-crystallize/mcp-servers/crystallize/worker-configuration.d.ts +++ b/use-crystallize/mcp-servers/crystallize/worker-configuration.d.ts @@ -1,11 +1,12 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface CloudflareBindings` (hash: 43f0b43e42c338c36ab98e28d304fc04) +// Generated by Wrangler by running `wrangler types --env-interface CloudflareBindings` (hash: f4a14dec248caaa4a8b0313b5f3f2760) // Runtime types generated with workerd@1.20260511.1 2025-08-03 nodejs_compat declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/index"); } interface Env { + AI: Ai; PLAUSIBLE_DOMAIN: "mcp.crystallize.com"; PLAUSIBLE_SCRIPT_URL: "https://plausible.io/js/pa-VtwNguh3SIy5jYVmDZF5N.js"; PLAUSIBLE_API_ENDPOINT: "https://plausible.io/api/event"; diff --git a/use-crystallize/mcp-servers/crystallize/wrangler.jsonc b/use-crystallize/mcp-servers/crystallize/wrangler.jsonc index 9e56f3f..4af5df6 100644 --- a/use-crystallize/mcp-servers/crystallize/wrangler.jsonc +++ b/use-crystallize/mcp-servers/crystallize/wrangler.jsonc @@ -16,6 +16,10 @@ "placement": { "mode": "smart", }, + // Workers AI, for TypeSafe's Jev model (typesafe/jev): picks Core schema domains for an intent. + "ai": { + "binding": "AI", + }, "vars": { // Plausible site ID for the SERVER-side MCP events. The Events API needs // no key — the site is identified by this alone, so it is a plain var, not From bfc38bb8608a37117cfb35cf0f37cae054d806f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:29:18 -0700 Subject: [PATCH 07/13] chore(mcp): add a Jev domain-selection eval --- .../bin/eval-domain-selection.cases.json | 14 ++++ .../crystallize/bin/eval-domain-selection.ts | 70 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json create mode 100644 use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts diff --git a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json new file mode 100644 index 0000000..f6d69ea --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json @@ -0,0 +1,14 @@ +[ + { "intent": "list the last 10 orders", "expected": ["order"] }, + { "intent": "create an order for an existing customer", "expected": ["order", "customer"] }, + { "intent": "find a customer by email and show their subscriptions", "expected": ["customer", "subscription"] }, + { "intent": "create a monthly subscription plan with a 14 day trial", "expected": ["subscriptionPlan"] }, + { "intent": "add a price list with 20% off for B2B customers", "expected": ["pricelist"] }, + { "intent": "move an order to the shipped stage of the fulfilment pipeline", "expected": ["pipeline", "order"] }, + { "intent": "set up a webhook that fires when an order is created", "expected": ["webhook"] }, + { "intent": "invite a new colleague to the tenant as an admin", "expected": ["invite"] }, + { "intent": "list all users and their roles", "expected": ["user"] }, + { "intent": "create a new warehouse for stock", "expected": ["stockLocation"] }, + { "intent": "cancel a customer's subscription", "expected": ["subscription"] }, + { "intent": "install an app on the tenant", "expected": ["app"] } +] diff --git a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts new file mode 100644 index 0000000..01100bb --- /dev/null +++ b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts @@ -0,0 +1,70 @@ +import { readFileSync } from "node:fs"; +import type { JevClient, JevResponse } from "../src/contracts/jev"; +import { JEV_MODEL } from "../src/core/services/jev-client"; +import { MAX_DOMAINS, pickDomains, scoreDomains } from "../src/core/services/core-domain-selector"; +import { createCoreSchemaDomainSplitter } from "../src/core/services/core-schema-domain-splitter"; +import { fetchIntrospection } from "../src/core/services/compact-schema-builder"; +import { buildAtApiUrl } from "../src/core/security"; + +const env = (name: string) => { + const value = process.env[name]; + if (!value) { + console.error(`Missing env var ${name}`); + process.exit(1); + } + return value; +}; + +const accountId = env("CLOUDFLARE_ACCOUNT_ID"); +const apiToken = env("CLOUDFLARE_API_TOKEN"); + +// Same request shape as the binding, over the REST API, so the eval runs outside a Worker. +const jevClient: JevClient = async (request) => { + const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run`, { + method: "POST", + headers: { Authorization: `Bearer ${apiToken}`, "Content-Type": "application/json" }, + body: JSON.stringify({ model: JEV_MODEL, input: request }), + }); + const json = (await response.json()) as { success: boolean; result: JevResponse; errors?: unknown }; + if (!json.success) throw new Error(`Workers AI error: ${JSON.stringify(json.errors)}`); + return json.result; +}; + +const url = buildAtApiUrl("https://api.crystallize.com", env("CRYSTALLIZE_TENANT"), ""); +const introspection = await fetchIntrospection(url, { + "X-Crystallize-Access-Token-Id": env("CRYSTALLIZE_TOKEN_ID"), + "X-Crystallize-Access-Token-Secret": env("CRYSTALLIZE_TOKEN_SECRET"), +}); +const index = createCoreSchemaDomainSplitter().listDomains(introspection); +const names = new Set(index.domains.map((d) => d.name)); +console.log(`${index.domains.length} domains: ${[...names].join(", ")}\n`); + +const cases = JSON.parse(readFileSync(new URL("./eval-domain-selection.cases.json", import.meta.url), "utf-8")) as { + intent: string; + expected: string[]; +}[]; + +const THRESHOLDS = [0.2, 0.3, 0.5]; +const totals = new Map(THRESHOLDS.map((t) => [t, { fullRecall: 0, returned: 0 }])); + +for (const c of cases) { + const missingFromIndex = c.expected.filter((e) => !names.has(e)); + if (missingFromIndex.length > 0) console.warn(`! case expects unknown domains: ${missingFromIndex.join(", ")}`); + + const scored = await scoreDomains(jevClient, index, c.intent); + const top = [...scored].sort((a, b) => b.probability - a.probability).slice(0, 6); + console.log(`- ${c.intent}\n expected: ${c.expected.join(", ")}`); + console.log(` top: ${top.map((d) => `${d.name} ${d.probability.toFixed(2)}`).join(", ")}`); + + for (const t of THRESHOLDS) { + const picked = pickDomains(scored, t, MAX_DOMAINS).map((d) => d.name); + const totalsForT = totals.get(t)!; + if (c.expected.every((e) => picked.includes(e))) totalsForT.fullRecall++; + totalsForT.returned += picked.length; + } +} + +console.log("\nthreshold full-recall avg-domains-returned"); +for (const [t, { fullRecall, returned }] of totals) { + console.log(`${t.toFixed(2)} ${fullRecall}/${cases.length} ${(returned / cases.length).toFixed(2)}`); +} From 7d019b09dd60de65737b92a714c45c055fed4291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:31:00 -0700 Subject: [PATCH 08/13] fix(mcp): use the Core schema's real domain names in the tool description and eval cases --- .../bin/eval-domain-selection.cases.json | 20 +++++++++---------- .../mcp/tools/fetch-core-graphql-schema.ts | 4 +++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json index f6d69ea..59f2ef8 100644 --- a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json +++ b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.cases.json @@ -1,14 +1,14 @@ [ { "intent": "list the last 10 orders", "expected": ["order"] }, - { "intent": "create an order for an existing customer", "expected": ["order", "customer"] }, - { "intent": "find a customer by email and show their subscriptions", "expected": ["customer", "subscription"] }, - { "intent": "create a monthly subscription plan with a 14 day trial", "expected": ["subscriptionPlan"] }, - { "intent": "add a price list with 20% off for B2B customers", "expected": ["pricelist"] }, - { "intent": "move an order to the shipped stage of the fulfilment pipeline", "expected": ["pipeline", "order"] }, + { "intent": "create an order for an existing customer", "expected": ["registerOrder", "customer"] }, + { "intent": "find a customer by email and show their subscriptions", "expected": ["customer", "subscriptionContract"] }, + { "intent": "pause a customer's subscription until next month", "expected": ["pauseSubscriptionContract"] }, + { "intent": "add a price list with 20% off for B2B customers", "expected": ["priceList"] }, + { "intent": "move an order to the shipped stage of its pipeline", "expected": ["orderPipelineStage"] }, { "intent": "set up a webhook that fires when an order is created", "expected": ["webhook"] }, - { "intent": "invite a new colleague to the tenant as an admin", "expected": ["invite"] }, - { "intent": "list all users and their roles", "expected": ["user"] }, - { "intent": "create a new warehouse for stock", "expected": ["stockLocation"] }, - { "intent": "cancel a customer's subscription", "expected": ["subscription"] }, - { "intent": "install an app on the tenant", "expected": ["app"] } + { "intent": "invite a new colleague to the tenant", "expected": ["inviteToken"] }, + { "intent": "give an existing user the admin role", "expected": ["userRole"] }, + { "intent": "set the stock of a product variant to 12", "expected": ["modifyProductVariantStock"] }, + { "intent": "cancel a customer's subscription", "expected": ["subscriptionContract"] }, + { "intent": "install a plugin on the tenant", "expected": ["pluginInstallation"] } ] diff --git a/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts b/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts index 9eee997..4668447 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/mcp/tools/fetch-core-graphql-schema.ts @@ -62,7 +62,9 @@ export const createFetchCoreGraphqlSchemaToolWrapper = ({ "description of what you want to do (e.g. 'create an order for an existing customer'); the server " + "then returns the schema of every domain that task needs, in one call. " + "Pass `domain` instead when you already know the exact domain. " + - "Common domains: order, customer, subscription, subscriptionPlan, pricelist, pipeline, flow, app, user, webhook, stockLocation, invite. " + + "Domains are fine-grained (verbs outside create/update/delete get their own domain). Common ones: order, " + + "registerOrder, customer, subscriptionContract, priceList, orderPipeline, orderPipelineStage, flow, " + + "webhook, role, userRole, inviteToken, pluginInstallation. " + "Call with neither to get the full list of domains. " + "The Core API is the admin API — use it for orders, customers, price lists, users, subscriptions, " + "subscription plans, pipelines, flows, apps, and other back-office/admin resources. " + From c69f106ed4baf371ec01f0322b86bf278c69d321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:37:14 -0700 Subject: [PATCH 09/13] fix(mcp): time out slow Jev calls and reject unusable Jev answers --- .../src/core/services/core-domain-selector.ts | 8 +++++-- .../src/core/services/jev-client.ts | 21 ++++++++++++++----- .../services/core-domain-selector.test.ts | 19 +++++++++++++++++ .../tests/core/services/jev-client.test.ts | 5 +++++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts b/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts index 48de794..cc8875d 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts @@ -45,8 +45,12 @@ export async function scoreDomains( const response = await jevClient(request); const scored: SelectedDomain[] = []; for (const [id, name] of domainByQuestionId) { - const answer = response.answers[id]; - if (answer) scored.push({ name, probability: answer.noul }); + const noul = response.answers[id]?.noul; + if (typeof noul === "number" && Number.isFinite(noul)) scored.push({ name, probability: noul }); + } + // Zero usable answers means the response shape changed, not that nothing is relevant: fail loudly. + if (scored.length === 0 && domainByQuestionId.size > 0) { + throw new Error("Jev returned no usable answers"); } return scored; } diff --git a/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts b/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts index 6d17230..bb4bd06 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts @@ -2,17 +2,28 @@ import type { JevClient, JevRequest, JevResponse } from "../../contracts/jev"; export const JEV_MODEL = "typesafe/jev"; +// A slow Workers AI must not hold the tool call: failing fast lets the caller fall back. +export const JEV_TIMEOUT_MS = 8000; + // The generated `Ai` types predate typesafe/jev, so the binding is narrowed to the one call we make. export type JevBinding = { run(model: typeof JEV_MODEL, input: JevRequest): Promise; }; export const createJevClient = - ({ ai }: { ai: JevBinding }): JevClient => + ({ ai }: { ai: JevBinding }, timeoutMs = JEV_TIMEOUT_MS): JevClient => async (request) => { - const response = (await ai.run(JEV_MODEL, request)) as Partial | null; - if (!response || typeof response !== "object" || !response.answers) { - throw new Error("Jev returned no answers"); + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Jev timed out after ${timeoutMs}ms`)), timeoutMs); + }); + try { + const response = (await Promise.race([ai.run(JEV_MODEL, request), timeout])) as Partial | null; + if (!response || typeof response !== "object" || !response.answers) { + throw new Error("Jev returned no answers"); + } + return response as JevResponse; + } finally { + clearTimeout(timer); } - return response as JevResponse; }; diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts index ab1d0d5..4f6dab8 100644 --- a/use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts +++ b/use-crystallize/mcp-servers/crystallize/tests/core/services/core-domain-selector.test.ts @@ -63,6 +63,25 @@ describe("scoreDomains", () => { }); }); +describe("scoreDomains with unusable answers", () => { + it("skips answers without a numeric noul", async () => { + const jevClient: JevClient = async () => + ({ + model: "jev-1.13.0", + answers: { d0: { type: "noul", noul: Number.NaN }, d1: { type: "noul", noul: 0.8 } }, + }) as JevResponse; + expect(await scoreDomains(jevClient, index, "create an order")).toEqual([{ name: "order", probability: 0.8 }]); + }); + + it("throws when no answer is usable, so a shape change is not mistaken for 'nothing relevant'", async () => { + const jevClient: JevClient = async () => + ({ model: "jev-1.13.0", answers: { d0: { type: "noul", probability: 0.9 } } }) as unknown as JevResponse; + await expect(scoreDomains(jevClient, index, "create an order")).rejects.toThrow( + "Jev returned no usable answers", + ); + }); +}); + describe("createCoreDomainSelector", () => { it("returns the picked domains for an intent", async () => { const jevClient: JevClient = async () => answers({ d0: 0.7, d1: 0.95, d2: 0.05 }); diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts index 4e125c9..ab5793a 100644 --- a/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts +++ b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts @@ -17,6 +17,11 @@ describe("createJevClient", () => { expect(run).toHaveBeenCalledWith(JEV_MODEL, request); }); + it("gives up when Jev does not answer in time", async () => { + const client = createJevClient({ ai: { run: () => new Promise(() => {}) } as JevBinding }, 20); + await expect(client(request)).rejects.toThrow("Jev timed out after 20ms"); + }); + it("throws when the response has no answers", async () => { const client = createJevClient({ ai: { run: async () => ({}) } as JevBinding }); await expect(client(request)).rejects.toThrow("Jev returned no answers"); From efaa17eeb29b07274a141a24b3ce7bae2fa9a4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:43:36 -0700 Subject: [PATCH 10/13] fix(mcp): unwrap the AI Gateway envelope around Jev answers --- .../crystallize/src/core/services/jev-client.ts | 13 ++++++++++++- .../tests/core/services/jev-client.test.ts | 13 +++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts b/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts index bb4bd06..8945a55 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/services/jev-client.ts @@ -10,6 +10,17 @@ export type JevBinding = { run(model: typeof JEV_MODEL, input: JevRequest): Promise; }; +/** + * Third-party models on Workers AI answer through AI Gateway as `{ state, result, gatewayMetadata }` + * (verified live for typesafe/jev), while the model page documents the bare `{ model, answers }`. Accept both. + */ +function unwrapGatewayEnvelope(raw: unknown): Partial | null { + if (!raw || typeof raw !== "object" || !("state" in raw)) return raw as Partial | null; + const { state, result } = raw as { state: unknown; result?: unknown }; + if (state !== "Completed") throw new Error(`Jev request ended in state "${String(state)}"`); + return result as Partial | null; +} + export const createJevClient = ({ ai }: { ai: JevBinding }, timeoutMs = JEV_TIMEOUT_MS): JevClient => async (request) => { @@ -18,7 +29,7 @@ export const createJevClient = timer = setTimeout(() => reject(new Error(`Jev timed out after ${timeoutMs}ms`)), timeoutMs); }); try { - const response = (await Promise.race([ai.run(JEV_MODEL, request), timeout])) as Partial | null; + const response = unwrapGatewayEnvelope(await Promise.race([ai.run(JEV_MODEL, request), timeout])); if (!response || typeof response !== "object" || !response.answers) { throw new Error("Jev returned no answers"); } diff --git a/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts index ab5793a..7fd528e 100644 --- a/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts +++ b/use-crystallize/mcp-servers/crystallize/tests/core/services/jev-client.test.ts @@ -17,6 +17,19 @@ describe("createJevClient", () => { expect(run).toHaveBeenCalledWith(JEV_MODEL, request); }); + it("unwraps the AI Gateway envelope that third-party models come back in", async () => { + const inner: JevResponse = { model: "jev-1.13.0", answers: { d0: { type: "noul", noul: 0.99 } } }; + const envelope = { state: "Completed", result: inner, gatewayMetadata: { keySource: "Unified" } }; + const client = createJevClient({ ai: { run: async () => envelope } as JevBinding }); + + expect(await client(request)).toEqual(inner); + }); + + it("throws when the gateway envelope is not completed", async () => { + const client = createJevClient({ ai: { run: async () => ({ state: "Failed", result: {} }) } as JevBinding }); + await expect(client(request)).rejects.toThrow('Jev request ended in state "Failed"'); + }); + it("gives up when Jev does not answer in time", async () => { const client = createJevClient({ ai: { run: () => new Promise(() => {}) } as JevBinding }, 20); await expect(client(request)).rejects.toThrow("Jev timed out after 20ms"); From cf9ca4fcf7f0e5129c96cbedbcfd63f51ce2ffe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:43:36 -0700 Subject: [PATCH 11/13] chore(mcp): run the eval through the production Jev client and tune the threshold to 0.5 --- .../crystallize/bin/eval-domain-selection.ts | 40 ++++++++++++++----- ...-09-29-jev-core-schema-domain-selection.md | 2 +- .../src/core/services/core-domain-selector.ts | 5 ++- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts index 01100bb..e587d55 100644 --- a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts +++ b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; -import type { JevClient, JevResponse } from "../src/contracts/jev"; -import { JEV_MODEL } from "../src/core/services/jev-client"; +import type { JevClient } from "../src/contracts/jev"; +import { createJevClient, type JevBinding } from "../src/core/services/jev-client"; import { MAX_DOMAINS, pickDomains, scoreDomains } from "../src/core/services/core-domain-selector"; import { createCoreSchemaDomainSplitter } from "../src/core/services/core-schema-domain-splitter"; import { fetchIntrospection } from "../src/core/services/compact-schema-builder"; @@ -18,16 +18,29 @@ const env = (name: string) => { const accountId = env("CLOUDFLARE_ACCOUNT_ID"); const apiToken = env("CLOUDFLARE_API_TOKEN"); -// Same request shape as the binding, over the REST API, so the eval runs outside a Worker. +// A stand-in for the AI binding over the REST API, so the eval runs outside a Worker but still goes +// through the production client (envelope unwrapping, validation). REST's `result` is what the binding returns. +const restBinding: JevBinding = { + async run(model, input) { + const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run`, { + method: "POST", + headers: { Authorization: `Bearer ${apiToken}`, "Content-Type": "application/json" }, + body: JSON.stringify({ model, input }), + }); + const json = (await response.json()) as { success: boolean; result: unknown; errors?: unknown }; + if (!json.success) throw new Error(`Workers AI error: ${JSON.stringify(json.errors)}`); + return json.result; + }, +}; +const productionClient = createJevClient({ ai: restBinding }, 30_000); +const usage = { inputTokens: 0, calls: 0, ms: 0 }; const jevClient: JevClient = async (request) => { - const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run`, { - method: "POST", - headers: { Authorization: `Bearer ${apiToken}`, "Content-Type": "application/json" }, - body: JSON.stringify({ model: JEV_MODEL, input: request }), - }); - const json = (await response.json()) as { success: boolean; result: JevResponse; errors?: unknown }; - if (!json.success) throw new Error(`Workers AI error: ${JSON.stringify(json.errors)}`); - return json.result; + const started = Date.now(); + const response = await productionClient(request); + usage.ms += Date.now() - started; + usage.calls++; + usage.inputTokens += response.usage?.input_tokens ?? 0; + return response; }; const url = buildAtApiUrl("https://api.crystallize.com", env("CRYSTALLIZE_TENANT"), ""); @@ -68,3 +81,8 @@ console.log("\nthreshold full-recall avg-domains-returned"); for (const [t, { fullRecall, returned }] of totals) { console.log(`${t.toFixed(2)} ${fullRecall}/${cases.length} ${(returned / cases.length).toFixed(2)}`); } + +console.log( + `\nper call: ${Math.round(usage.inputTokens / usage.calls)} input tokens, ` + + `${Math.round(usage.ms / usage.calls)}ms average latency (REST, from this machine)`, +); diff --git a/use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md b/use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md index 89ba853..1970834 100644 --- a/use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md +++ b/use-crystallize/mcp-servers/crystallize/docs/superpowers/plans/2026-09-29-jev-core-schema-domain-selection.md @@ -15,7 +15,7 @@ - Package root for every path below: `use-crystallize/mcp-servers/crystallize/`. Run all commands from there. - Model id: `typesafe/jev` (Workers AI, 32,000-token context, $0.042 / 1M input tokens, output free). - Jev Noul answer shape: `{ "type": "noul", "noul": <0..1> }` under `response.answers[]`. -- Selection policy (initial, tuned in Task 6): keep domains with probability `>= 0.3`, sorted descending, at most `4`. +- Selection policy (tuned in Task 6 on 12 live cases: 12/12 full recall at 0.2, 0.3 and 0.5): keep domains with probability `>= 0.5`, sorted descending, at most `4`. - `domain` input wins over `intent` when both are given (exact, no Jev call). - A Jev failure, a blank intent, or an empty selection must return the domain index, never an error. - No new npm dependencies. diff --git a/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts b/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts index cc8875d..c69b0d3 100644 --- a/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts +++ b/use-crystallize/mcp-servers/crystallize/src/core/services/core-domain-selector.ts @@ -4,9 +4,10 @@ import type { JevClient, JevRequest } from "../../contracts/jev"; /** * Missing a needed domain costs the agent another round-trip; an extra domain only costs output size. - * So the threshold leans low and the cap bounds the output. + * Tuned with bin/eval-domain-selection.ts on the live schema: 0.5 kept 12/12 cases at full recall with + * 1.4 domains returned on average (0.3 returned 2.1). The cap bounds the output. */ -export const DOMAIN_THRESHOLD = 0.3; +export const DOMAIN_THRESHOLD = 0.5; export const MAX_DOMAINS = 4; /** From ffc94b9762a96562f4695ed10528d62f10d8ccb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:52:53 -0700 Subject: [PATCH 12/13] chore(mcp): read the Cloudflare account id from wrangler.jsonc in the Jev eval --- .../mcp-servers/crystallize/bin/eval-domain-selection.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts index e587d55..657a7c8 100644 --- a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts +++ b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts @@ -15,7 +15,11 @@ const env = (name: string) => { return value; }; -const accountId = env("CLOUDFLARE_ACCOUNT_ID"); +// The account is pinned in wrangler.jsonc (not a secret); CLOUDFLARE_ACCOUNT_ID overrides it. +const pinnedAccountId = readFileSync(new URL("../wrangler.jsonc", import.meta.url), "utf-8").match( + /"account_id"\s*:\s*"([0-9a-f]+)"/, +)?.[1]; +const accountId = process.env.CLOUDFLARE_ACCOUNT_ID || pinnedAccountId || env("CLOUDFLARE_ACCOUNT_ID"); const apiToken = env("CLOUDFLARE_API_TOKEN"); // A stand-in for the AI binding over the REST API, so the eval runs outside a Worker but still goes From 2b20af7757f776cc5f77ce611121ebfa9b680d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Tue, 29 Sep 2026 15:54:25 -0700 Subject: [PATCH 13/13] chore(mcp): default the Jev eval to the wrangler login and the furnitut tenant --- .../crystallize/bin/eval-domain-selection.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts index 657a7c8..798e847 100644 --- a/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts +++ b/use-crystallize/mcp-servers/crystallize/bin/eval-domain-selection.ts @@ -20,7 +20,16 @@ const pinnedAccountId = readFileSync(new URL("../wrangler.jsonc", import.meta.ur /"account_id"\s*:\s*"([0-9a-f]+)"/, )?.[1]; const accountId = process.env.CLOUDFLARE_ACCOUNT_ID || pinnedAccountId || env("CLOUDFLARE_ACCOUNT_ID"); -const apiToken = env("CLOUDFLARE_API_TOKEN"); +// Defaults to the current `wrangler login`, so a logged-in developer needs no token at all. +const wranglerToken = () => { + const out = Bun.spawnSync(["bunx", "wrangler", "auth", "token", "--json"], { stderr: "ignore" }); + try { + return (JSON.parse(out.stdout.toString()) as { token?: string }).token; + } catch { + return undefined; + } +}; +const apiToken = process.env.CLOUDFLARE_API_TOKEN || wranglerToken() || env("CLOUDFLARE_API_TOKEN"); // A stand-in for the AI binding over the REST API, so the eval runs outside a Worker but still goes // through the production client (envelope unwrapping, validation). REST's `result` is what the binding returns. @@ -47,7 +56,7 @@ const jevClient: JevClient = async (request) => { return response; }; -const url = buildAtApiUrl("https://api.crystallize.com", env("CRYSTALLIZE_TENANT"), ""); +const url = buildAtApiUrl("https://api.crystallize.com", process.env.CRYSTALLIZE_TENANT || "furnitut", ""); const introspection = await fetchIntrospection(url, { "X-Crystallize-Access-Token-Id": env("CRYSTALLIZE_TOKEN_ID"), "X-Crystallize-Access-Token-Secret": env("CRYSTALLIZE_TOKEN_SECRET"),