From 18be799177891a1aa79b62dca153747580da71cf Mon Sep 17 00:00:00 2001 From: PredictabilityAtScale <131020168+PredictabilityAtScale@users.noreply.github.com> Date: Mon, 18 May 2026 10:33:55 -0700 Subject: [PATCH] Harden schema_ref module-not-found handling and docs consistency --- README.md | 4 +- docs/index.md | 1 + docs/overrides.md | 2 +- docs/schema-dx.md | 170 ++++++++++++++++++++++++ docs/schema.md | 12 ++ package-lock.json | 12 +- package.json | 3 +- src/parser/loader.ts | 13 +- src/parser/response-schema-ref.ts | 116 +++++++++++++++++ src/schema/schema.ts | 21 ++- src/validation/validate.ts | 45 +++++++ tests/cli.test.ts | 210 ++++++++++++++++++++++++++++++ tests/validation.test.ts | 37 +++++- 13 files changed, 636 insertions(+), 10 deletions(-) create mode 100644 docs/schema-dx.md create mode 100644 src/parser/response-schema-ref.ts diff --git a/README.md b/README.md index c6754d7..ebc7ed4 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ In browser or client-side code, keep provider credentials on the server. Use the ### Provider-specific fields and raw passthrough -Use normalized fields first (`sampling`, `response`, `cache`, `tools`) so prompts stay portable. `response.schema` is the neutral JSON Schema path; adapters emit it as OpenAI/OpenRouter/LLMAsAService `response_format`, OpenAI Responses `text.format`, Anthropic `output_config.format`, and Gemini `generationConfig.responseJsonSchema`. +Use normalized fields first (`sampling`, `response`, `cache`, `tools`) so prompts stay portable. `response.schema` is the neutral JSON Schema path; adapters emit it as OpenAI/OpenRouter/LLMAsAService `response_format`, OpenAI Responses `text.format`, Anthropic `output_config.format`, and Gemini `generationConfig.responseJsonSchema`. You can also provide `response.schema_ref` to load schema from a prompt-relative `.json` file or `.js/.mjs/.cjs` zod module (mutually exclusive with `response.schema`). Use `provider_options` when PromptOpsKit has a known provider-specific mapping, such as Anthropic `top_k`, Gemini's native `response_schema`, OpenRouter routing fields, or LLMAsAService gateway routing/customer metadata. @@ -656,7 +656,7 @@ Prompt files use YAML front matter with these fields: | `fallback_models` | `string[]` | Fallback model list | | `reasoning` | `object` | `{ effort, budget_tokens }` | | `sampling` | `object` | `{ temperature, top_p, frequency_penalty, presence_penalty, stop, max_output_tokens }` | -| `response` | `object` | `{ format, stream, schema, schema_name, schema_description, schema_strict }` | +| `response` | `object` | `{ format, stream, schema, schema_ref, schema_name, schema_description, schema_strict }` | | `cache` | `object` | Provider-specific cache controls (`openai`, `anthropic`, `gemini`/`google`) | | `tools` | `array` | Tool references (string names or inline definitions) | | `provider_options` | `object` | Provider-specific non-portable options (`anthropic`, `gemini`, `openrouter`, `llmasaservice`) | diff --git a/docs/index.md b/docs/index.md index 7c07c07..299a8e8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,6 +17,7 @@ Open-source developer toolkit for managing prompts, system instructions, tools, - [CLI](./cli.md) — Command-line interface: init, validate, compile, render, inspect, skill - [API Reference](./api-reference.md) — TypeScript API: `createPromptOpsKit`, `renderPrompt`, standalone functions - [Schema](./schema.md) — Full YAML front matter schema reference +- [Response Schema DX](./schema-dx.md) — Design proposal for sharing one response schema between prompt files and runtime validation code - [Vendor Schema Gap Analysis](./vendor-schema-gap-analysis.md) — Snapshot comparison against published OpenAI, Anthropic, Gemini, OpenRouter, and LLMAsAService gateway schema capabilities - [Testing](./testing.md) — Test helpers, mock assets, and sidecar test files - [Validation](./validation.md) — Schema validation, "did you mean?" suggestions, variable checks, early regex validation, and YAML regex quoting guidance diff --git a/docs/overrides.md b/docs/overrides.md index fa272fc..459eb10 100644 --- a/docs/overrides.md +++ b/docs/overrides.md @@ -51,7 +51,7 @@ Only these fields can be overridden in `environments` and `tiers`: | `fallback_models` | `string[]` | Fallback model list | | `reasoning` | `object` | `{ effort, budget_tokens }` | | `sampling` | `object` | `{ temperature, top_p, frequency_penalty, presence_penalty, stop, max_output_tokens }` | -| `response` | `object` | `{ format, stream, schema, schema_name, schema_description, schema_strict }` | +| `response` | `object` | `{ format, stream, schema, schema_ref, schema_name, schema_description, schema_strict }` | | `cache` | `object` | Provider-specific cache controls (`openai`, `anthropic`, `gemini`/`google`) | | `raw` | `object` | Provider-specific request-body passthrough blocks | | `tools` | `array` | Tool references | diff --git a/docs/schema-dx.md b/docs/schema-dx.md new file mode 100644 index 0000000..b73436b --- /dev/null +++ b/docs/schema-dx.md @@ -0,0 +1,170 @@ +# Response Schema DX: Author Once, Use in Prompt + Code + +This proposal describes how PromptOpsKit can let teams define a response schema once and use it in: + +1. Prompt front matter (`response.schema`), and +2. Runtime TypeScript validation/parsing. + +## Goals + +- **Single source of truth** for structured output contracts. +- **Low-friction authoring** (no mandatory codegen for simple teams). +- **Strong typing** in application code for teams that want it. +- **Portable behavior** across providers that support JSON Schema. +- **Incremental adoption** without breaking existing prompts. + +## Recommended design + +Use a **three-lane model** so users can choose complexity level. + +### Lane A: Inline schema (existing behavior) + +Keep current `response.schema` behavior unchanged. + +Best for: +- quick experiments +- one-off prompts +- teams without build tooling + +### Lane B: External schema file references (new) + +Allow prompt front matter to reference a schema asset file. + +Example: + +```yaml +response: + format: json + schema_ref: ./schemas/support-reply.schema.json + schema_name: support_reply + schema_strict: true +``` + +Resolution rules: +- `schema_ref` is resolved relative to the prompt file. +- File must be JSON Schema object. +- Compiler inlines resolved schema into compiled prompt output for providers. + +Why this helps DX: +- keeps prompts readable +- enables sharing schema across multiple prompts +- lets app code import the same schema file directly + +### Lane C: Optional codegen for typed validators (new, opt-in) + +Add an optional command: + +```bash +promptopskit schema generate +``` + +This command can emit: +- `*.schema.json` copies (or normalized forms) +- TS validator modules (e.g. Zod/Valibot wrappers) +- inferred TS types for app code + +Why optional: +- some teams only need raw JSON Schema + Ajv +- others want end-to-end typed contracts + +## Authoring/usage patterns + +### Pattern 1: JSON Schema + Ajv (minimal dependencies) + +- Define schema in `*.schema.json`. +- Reference it from prompt via `schema_ref`. +- In app runtime, import same JSON schema into Ajv to validate model output. + +Pros: simplest, ecosystem-standard. + +### Pattern 2: Type-first (Zod) with schema export + +- Define schema in code (`z.object(...)`). +- Export JSON Schema artifact during build (`zod-to-json-schema`). +- Prompt references generated JSON file. + +Pros: strong TS ergonomics. +Cons: build step required. + +### Pattern 3: Schema-first with generated TS types + +- Define `*.schema.json`. +- Generate TS types and/or validators. + +Pros: prompt and runtime both consume same source. + +## Proposed front matter additions + +Under `response`: + +- `schema_ref?: string` — path to external JSON Schema file or zod module. +- Keep existing `schema?: object`. +- Validation rule: exactly one of `schema` or `schema_ref` should be provided. + +Potential future extension: + +- `schema_ref_name?: string` for referencing named schemas from a registry file. + +## Compiler/runtime behavior + +1. Parser reads prompt. +2. If `schema_ref` exists, loader resolves and loads schema source (`.json` or zod module). +3. Validate schema is object-shaped JSON. +4. Normalize to internal `response.schema`. +5. Continue provider mapping unchanged. + +This preserves downstream adapter logic and minimizes invasive changes. + +## CLI UX + +### `compile` + +- Include resolved schema in compiled artifact. +- Include source metadata: + - `response.schema_source.resolved_path` + - `response.schema_source.hash` (optional) + +### `validate` + +- Verify referenced schema file exists. +- Validate JSON parse + object shape. +- Warn on unsupported JSON Schema constructs per provider (best-effort). + +### `inspect` + +- Show whether schema is inline or file-based. +- Display resolved path and size/hash. + +## Error messages (important for DX) + +Make messages actionable: + +- `POK050: response.schema_ref "./schemas/reply.json" not found (resolved from prompts/support/reply.md)` +- `POK050: response.schema_ref "./schemas/reply.json" must resolve to a JSON object schema` +- `POK051: zod schema modules must export a Zod schema as default export or named export "schema"` +- `POK051: response.schema_ref "./schemas/reply.schema.ts" has unsupported extension ".ts". Use .json, .js, .mjs, or .cjs` +- `response.schema and response.schema_ref are mutually exclusive` + +## Migration strategy + +- No breaking changes for existing prompts. +- Teams can migrate prompt-by-prompt: + 1. move inline schema to file + 2. replace `schema` with `schema_ref` + 3. run `promptopskit validate` + +## Recommendation + +`schema_ref` (Lane B) is now implemented. + +Next step: evaluate **Lane C** (optional codegen) once real user needs clarify: +- target validator libs +- desired output layout +- monorepo integration expectations + +This sequencing maximizes immediate DX value with low maintenance cost. + + +## Security note for executable schema modules + +`schema_ref` supports executable JS modules (`.js/.mjs/.cjs`) for zod exports. These modules are imported during load/validate/compile, so only reference trusted repository code. diff --git a/docs/schema.md b/docs/schema.md index 62f9553..3251c33 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -84,6 +84,7 @@ sampling: ## `response` ```yaml +# Inline schema response: format: json # text | json | markdown stream: true @@ -95,6 +96,16 @@ response: schema_name: support_reply schema_description: Support reply payload schema_strict: true + +# External schema reference (mutually exclusive with `schema`) +response: + format: json + schema_ref: ./schemas/support-reply.schema.json + +# External zod module reference (must export default or named `schema`) +response: + format: json + schema_ref: ./schemas/support-reply.schema.mjs ``` | Field | Type | Description | @@ -102,6 +113,7 @@ response: | `format` | `'text' \| 'json' \| 'markdown'` | Response format | | `stream` | `boolean` | Enable streaming | | `schema` | `object` | Portable JSON Schema object for structured output | +| `schema_ref` | `string` | Relative path to external schema (`.json`) or zod module (`.js/.mjs/.cjs`) | | `schema_name` | `string` | Optional schema name (used by OpenAI/OpenAI Responses) | | `schema_description` | `string` | Optional schema description (used by OpenAI/OpenAI Responses/OpenRouter/LLMAsAService structured outputs) | | `schema_strict` | `boolean` | Strict schema enforcement toggle (OpenAI/OpenAI Responses) | diff --git a/package-lock.json b/package-lock.json index 092f4d7..0fb841e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "license": "MIT", "dependencies": { "gray-matter": "^4.0.3", - "zod": "^3.23.0" + "zod": "^3.23.0", + "zod-to-json-schema": "^3.25.2" }, "bin": { "promptopskit": "dist/cli/index.js" @@ -2881,6 +2882,15 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/package.json b/package.json index cc31f11..5c27f21 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,8 @@ }, "dependencies": { "gray-matter": "^4.0.3", - "zod": "^3.23.0" + "zod": "^3.23.0", + "zod-to-json-schema": "^3.25.2" }, "devDependencies": { "@types/node": "^25.6.0", diff --git a/src/parser/loader.ts b/src/parser/loader.ts index e1671b7..dd3ddf9 100644 --- a/src/parser/loader.ts +++ b/src/parser/loader.ts @@ -6,6 +6,7 @@ import type { ParseResult } from './parser.js'; import { extractSections } from './sections.js'; import { PromptDefaultsSchema } from '../schema/index.js'; import type { PromptDefaults } from '../schema/index.js'; +import { resolveResponseSchemaRef } from './response-schema-ref.js'; const DEFAULTS_FILE_NAME = 'defaults.md'; @@ -27,7 +28,17 @@ export async function loadPromptFile(filePath: string, options: LoadPromptOption // walks above the prompt tree when no explicit root is provided. const root = options.defaultsRoot ?? dirname(filePath); const defaults = await loadDefaultsForPath(filePath, root); - const asset = applyDefaults(parsed.asset, defaults); + const withDefaults = applyDefaults(parsed.asset, defaults); + const resolved = await resolveResponseSchemaRef(withDefaults, filePath); + const asset = (resolved.response?.schema !== undefined && !resolved.response?.schema_source) + ? { + ...resolved, + response: { + ...resolved.response, + schema_source: { mode: 'inline' as const }, + }, + } + : resolved; return { ...parsed, diff --git a/src/parser/response-schema-ref.ts b/src/parser/response-schema-ref.ts new file mode 100644 index 0000000..3957340 --- /dev/null +++ b/src/parser/response-schema-ref.ts @@ -0,0 +1,116 @@ +import { readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { dirname, extname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { z } from 'zod'; +import { zodToJsonSchema } from 'zod-to-json-schema'; +import type { PromptAsset } from '../schema/index.js'; + +const SUPPORTED_SCHEMA_REF_EXTENSIONS = new Set(['.json', '.js', '.mjs', '.cjs']); + + +async function resolveJsonSchemaRef(schemaRef: string, promptFilePath: string): Promise<{ schema: Record; resolvedPath: string; hash: string }> { + const resolvedPath = resolve(dirname(promptFilePath), schemaRef); + let raw: string; + + try { + raw = await readFile(resolvedPath, 'utf-8'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw new Error(`POK050: response.schema_ref "${schemaRef}" not found (resolved from ${promptFilePath})`); + } + throw error; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`POK050: response.schema_ref "${schemaRef}" is not valid JSON`); + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`POK050: response.schema_ref "${schemaRef}" must resolve to a JSON object schema`); + } + + return { + schema: parsed as Record, + resolvedPath, + hash: createHash('sha256').update(raw).digest('hex'), + }; +} + +async function resolveZodSchemaRef(schemaRef: string, promptFilePath: string): Promise<{ schema: Record; resolvedPath: string; hash: string }> { + const resolvedPath = resolve(dirname(promptFilePath), schemaRef); + + let moduleSource: string; + try { + moduleSource = await readFile(resolvedPath, 'utf-8'); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + throw new Error(`POK051: response.schema_ref "${schemaRef}" not found (resolved from ${promptFilePath})`); + } + throw error; + } + + let imported: unknown; + try { + imported = await import(pathToFileURL(resolvedPath).href); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`POK051: response.schema_ref "${schemaRef}" could not be imported as a module (${message})`); + } + + const mod = imported as Record; + const candidate = mod.default ?? mod.schema; + + if (!(candidate instanceof z.ZodType)) { + throw new Error('POK051: zod schema modules must export a Zod schema as default export or named export "schema"'); + } + + const jsonSchema = zodToJsonSchema(candidate, { + target: 'jsonSchema7', + $refStrategy: 'none', + }); + + if (!jsonSchema || typeof jsonSchema !== 'object' || Array.isArray(jsonSchema)) { + throw new Error(`POK051: response.schema_ref "${schemaRef}" did not produce a valid JSON schema object`); + } + + return { + schema: jsonSchema as Record, + resolvedPath, + hash: createHash('sha256').update(moduleSource).digest('hex'), + }; +} + +export async function resolveResponseSchemaRef(asset: PromptAsset, promptFilePath: string): Promise { + const schemaRef = asset.response?.schema_ref; + if (!schemaRef) return asset; + + const ext = extname(schemaRef).toLowerCase(); + if (!SUPPORTED_SCHEMA_REF_EXTENSIONS.has(ext)) { + throw new Error(`POK051: response.schema_ref "${schemaRef}" has unsupported extension "${ext}". Use .json, .js, .mjs, or .cjs`); + } + + const resolved = (ext === '.json') + ? await resolveJsonSchemaRef(schemaRef, promptFilePath) + : await resolveZodSchemaRef(schemaRef, promptFilePath); + + return { + ...asset, + response: { + ...asset.response, + schema: resolved.schema, + schema_ref: undefined, + schema_source: { + mode: ext === '.json' ? 'schema_ref_json' : 'schema_ref_zod_module', + ref: schemaRef, + resolved_path: resolved.resolvedPath, + hash: resolved.hash, + }, + }, + }; +} diff --git a/src/schema/schema.ts b/src/schema/schema.ts index 6086a80..7e14c72 100644 --- a/src/schema/schema.ts +++ b/src/schema/schema.ts @@ -48,11 +48,28 @@ export const ResponseSchema = z.object({ format: z.enum(['text', 'json', 'markdown']).optional(), stream: z.boolean().optional(), schema: z.record(z.unknown()).optional(), + schema_ref: z.string().min(1).optional(), + schema_source: z.object({ + mode: z.enum(['inline', 'schema_ref_json', 'schema_ref_zod_module']).optional(), + ref: z.string().optional(), + resolved_path: z.string().optional(), + hash: z.string().optional(), + }).optional(), schema_name: z.string().optional(), schema_description: z.string().optional(), schema_strict: z.boolean().optional(), }); +export const ResponseSchemaWithValidation = ResponseSchema.superRefine((value, ctx) => { + if (value.schema !== undefined && value.schema_ref !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'response.schema and response.schema_ref are mutually exclusive', + path: ['schema_ref'], + }); + } +}); + // --- Provider-specific options --- @@ -215,7 +232,7 @@ export const PromptAssetOverridesSchema = z.object({ fallback_models: z.array(z.string()).optional(), reasoning: ReasoningSchema.optional(), sampling: SamplingSchema.optional(), - response: ResponseSchema.optional(), + response: ResponseSchemaWithValidation.optional(), cache: CacheSchema.optional(), raw: RawProviderBodySchema.optional(), tools: z.array(ToolRefSchema).optional(), @@ -266,7 +283,7 @@ export const PromptAssetSchema = z.object({ reasoning: ReasoningSchema.optional(), sampling: SamplingSchema.optional(), - response: ResponseSchema.optional(), + response: ResponseSchemaWithValidation.optional(), cache: CacheSchema.optional(), raw: RawProviderBodySchema.optional(), diff --git a/src/validation/validate.ts b/src/validation/validate.ts index c9304af..ddf29a2 100644 --- a/src/validation/validate.ts +++ b/src/validation/validate.ts @@ -28,6 +28,22 @@ const KNOWN_FRONT_MATTER_KEYS = new Set([ 'environments', 'tiers', 'metadata', 'cache', 'provider_options', 'raw', ]); + + +function collectSchemaKeywords(value: unknown, acc: Set): void { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) collectSchemaKeywords(item, acc); + return; + } + + const record = value as Record; + for (const [key, child] of Object.entries(record)) { + acc.add(key); + collectSchemaKeywords(child, acc); + } +} + const RISKY_UNBOUNDED_INPUT_NAMES = [ 'message', 'prompt', @@ -202,6 +218,35 @@ export function validateAsset( } } + + if (asset.response?.schema && asset.provider) { + const keywords = new Set(); + collectSchemaKeywords(asset.response.schema, keywords); + + const openAIUnsupported = ['patternProperties', 'unevaluatedProperties', '$dynamicRef']; + const geminiUnsupported = ['patternProperties', '$dynamicRef', 'dependentSchemas']; + const anthropicCaution = ['if', 'then', 'else']; + + const checks: Array<{ provider: string[]; unsupported: string[]; code: string }> = [ + { provider: ['openai', 'openai-responses', 'openrouter', 'llmasaservice'], unsupported: openAIUnsupported, code: 'POK052' }, + { provider: ['gemini', 'google'], unsupported: geminiUnsupported, code: 'POK053' }, + { provider: ['anthropic'], unsupported: anthropicCaution, code: 'POK054' }, + ]; + + for (const check of checks) { + if (!check.provider.includes(asset.provider)) continue; + const hits = check.unsupported.filter((k) => keywords.has(k)); + if (hits.length > 0) { + warnings.push({ + code: check.code, + message: `response.schema includes provider-sensitive JSON Schema keywords for "${asset.provider}": ${hits.join(', ')}`, + filePath, + suggestion: 'Validate schema compatibility with your provider docs or simplify the schema dialect.', + }); + } + } + } + if (asset.provider) { let providerCache: unknown; let cacheSuggestionField: string | undefined; diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 50c7104..c8ef04e 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -443,3 +443,213 @@ Hello. expect(content).toContain('# promptopskit'); }); }); + +describe('response.schema_ref resolution', () => { + + it('inspect shows schema_source metadata for schema_ref prompts', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + await writeFile(join(tmpDir, 'prompts', 'schemas', 'reply.schema.json'), JSON.stringify({ type: 'object' })); + await writeFile(join(tmpDir, 'prompts', 'inspect-schema.md'), `--- +id: schema-ref.inspect +schema_version: 1 +response: + format: json + schema_ref: ./schemas/reply.schema.json +--- + +# Prompt template + +Hello. +`); + + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + await inspect([join(tmpDir, 'prompts', 'inspect-schema.md')]); + const parsed = JSON.parse(String(logSpy.mock.calls[0]?.[0])) as Record; + const response = parsed.response as Record; + const schemaSource = response.schema_source as Record; + expect(schemaSource.mode).toBe('schema_ref_json'); + expect(schemaSource.ref).toBe('./schemas/reply.schema.json'); + expect(typeof schemaSource.hash).toBe('string'); + }); + + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await mkdtemp(join(tmpdir(), 'pok-schema-ref-')); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await rm(tmpDir, { recursive: true, force: true }); + }); + + it('validate resolves response.schema_ref from prompt-relative JSON file', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + + await writeFile(join(tmpDir, 'prompts', 'schemas', 'reply.schema.json'), JSON.stringify({ + type: 'object', + properties: { answer: { type: 'string' } }, + required: ['answer'], + }, null, 2)); + + await writeFile(join(tmpDir, 'prompts', 'reply.md'), `--- +id: schema-ref.validate +schema_version: 1 +response: + format: json + schema_ref: ./schemas/reply.schema.json +--- + +# Prompt template + +Hello. +`); + + await expect(validate([join(tmpDir, 'prompts')])).resolves.toBeUndefined(); + }); + + it('compile outputs resolved response.schema and removes schema_ref', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + const outDir = join(tmpDir, 'compiled'); + + await writeFile(join(tmpDir, 'prompts', 'schemas', 'reply.schema.json'), JSON.stringify({ + type: 'object', + properties: { answer: { type: 'string' } }, + required: ['answer'], + }, null, 2)); + + await writeFile(join(tmpDir, 'prompts', 'reply.md'), `--- +id: schema-ref.compile +schema_version: 1 +response: + format: json + schema_ref: ./schemas/reply.schema.json +--- + +# Prompt template + +Hello. +`); + + await compile([join(tmpDir, 'prompts'), outDir]); + + const compiled = JSON.parse(await readFile(join(outDir, 'reply.json'), 'utf-8')) as Record; + const response = compiled.response as Record; + expect(response.schema).toMatchObject({ type: 'object' }); + expect(response.schema_ref).toBeUndefined(); + }); + + + it('compile resolves response.schema_ref from a zod module export', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + const outDir = join(tmpDir, 'compiled-zod'); + + await writeFile(join(tmpDir, 'prompts', 'schemas', 'reply.schema.mjs'), ` +import { z } from 'zod'; + +export default z.object({ + answer: z.string(), + confidence: z.number().min(0).max(1), +}); +`); + + await writeFile(join(tmpDir, 'prompts', 'reply-zod.md'), `--- +id: schema-ref.compile-zod +schema_version: 1 +response: + format: json + schema_ref: ./schemas/reply.schema.mjs +--- + +# Prompt template + +Hello. +`); + + await compile([join(tmpDir, 'prompts'), outDir]); + + const compiled = JSON.parse(await readFile(join(outDir, 'reply-zod.json'), 'utf-8')) as Record; + const response = compiled.response as Record; + const schema = response.schema as Record; + expect(schema.type).toBe('object'); + expect(response.schema_ref).toBeUndefined(); + }); + + + it('validate fails for invalid JSON schema_ref files', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + await writeFile(join(tmpDir, 'prompts', 'schemas', 'bad.schema.json'), '{ not: json }'); + await writeFile(join(tmpDir, 'prompts', 'bad-json.md'), `--- +id: schema-ref.bad-json +schema_version: 1 +response: + format: json + schema_ref: ./schemas/bad.schema.json +--- + +# Prompt template + +Hello. +`); + + await expect(validate([join(tmpDir, 'prompts')])).rejects.toThrow('process.exit unexpectedly called with "1"'); + }); + + + it('validate fails for missing zod module schema_ref files', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + await writeFile(join(tmpDir, 'prompts', 'missing-zod.md'), `--- +id: schema-ref.missing-zod +schema_version: 1 +response: + format: json + schema_ref: ./schemas/does-not-exist.mjs +--- + +# Prompt template + +Hello. +`); + + await expect(validate([join(tmpDir, 'prompts')])).rejects.toThrow('process.exit unexpectedly called with "1"'); + }); + + it('validate fails for zod module refs without a schema export', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + await writeFile(join(tmpDir, 'prompts', 'schemas', 'not-a-schema.mjs'), 'export default { nope: true };'); + await writeFile(join(tmpDir, 'prompts', 'bad-zod-export.md'), `--- +id: schema-ref.bad-zod-export +schema_version: 1 +response: + format: json + schema_ref: ./schemas/not-a-schema.mjs +--- + +# Prompt template + +Hello. +`); + + await expect(validate([join(tmpDir, 'prompts')])).rejects.toThrow('process.exit unexpectedly called with "1"'); + }); + + it('validate fails for unsupported schema_ref extension', async () => { + await mkdir(join(tmpDir, 'prompts', 'schemas'), { recursive: true }); + await writeFile(join(tmpDir, 'prompts', 'schemas', 'reply.schema.ts'), 'export default {}'); + await writeFile(join(tmpDir, 'prompts', 'bad-schema-ext.md'), `--- +id: schema-ref.bad-ext +schema_version: 1 +response: + format: json + schema_ref: ./schemas/reply.schema.ts +--- + +# Prompt template + +Hello. +`); + + await expect(validate([join(tmpDir, 'prompts')])).rejects.toThrow('process.exit unexpectedly called with "1"'); + }); + +}); diff --git a/tests/validation.test.ts b/tests/validation.test.ts index 1b31e4e..96eb759 100644 --- a/tests/validation.test.ts +++ b/tests/validation.test.ts @@ -266,7 +266,25 @@ describe('validateAsset', () => { expect(result.errors.some((error) => error.code === 'POK001')).toBe(true); }); - it('does not warn when trim is explicitly false without max_size', () => { + + it('warns on provider-sensitive schema keywords', () => { + const result = validateAsset({ + id: 'schema.keywords', + schema_version: 1, + provider: 'openai', + response: { + format: 'json', + schema: { + type: 'object', + patternProperties: { '^x-': { type: 'string' } }, + }, + }, + sections: { prompt_template: 'Hi' }, + }); + + expect(result.warnings.some((warning) => warning.code === 'POK052')).toBe(true); + }); +it('does not warn when trim is explicitly false without max_size', () => { const result = validateAsset({ id: 'test', schema_version: 1, @@ -280,6 +298,22 @@ describe('validateAsset', () => { expect(result.warnings.some((warning) => warning.code === 'POK014')).toBe(false); }); + + it('fails when response.schema and response.schema_ref are both provided', () => { + const result = validateAsset({ + id: 'bad.schema_ref', + schema_version: 1, + response: { + schema: { type: 'object' }, + schema_ref: './schemas/reply.schema.json', + }, + sections: { prompt_template: 'Hi' }, + }); + + expect(result.valid).toBe(false); + expect(result.errors.some((error) => error.message.includes('mutually exclusive'))).toBe(true); + }); + }); describe('levenshtein', () => { @@ -292,7 +326,6 @@ describe('levenshtein', () => { expect(levenshtein('mdoel', 'model')).toBeLessThanOrEqual(2); }); }); - describe('PromptOpsKit.validatePrompt', () => { let tmpDir: string;