diff --git a/tools/typesafe-mock/src/cli.mjs b/tools/typesafe-mock/src/cli.mjs new file mode 100644 index 000000000..5f3433516 --- /dev/null +++ b/tools/typesafe-mock/src/cli.mjs @@ -0,0 +1,7 @@ +import { loadConfig, startMockServer } from './server.mjs'; + +const config = loadConfig(process.env); +const running = await startMockServer(config); +console.log(`TypeSafe mock listening on ${running.url}`); +console.log('POST /v1/systemone'); +console.log('GET /v1/models'); diff --git a/tools/typesafe-mock/src/errors.mjs b/tools/typesafe-mock/src/errors.mjs new file mode 100644 index 000000000..c4111009f --- /dev/null +++ b/tools/typesafe-mock/src/errors.mjs @@ -0,0 +1,329 @@ +/** Auth and 422 bodies observed from api.typesafe.ai, plus the OpenAPI validation shape. */ + +export const MISSING_API_KEY_BODY = { + detail: { + error_type: 'authentication_error', + message: 'Must supply an API key! Check your request and try again.', + }, +}; + +export const INVALID_API_KEY_BODY = { + detail: { + error_type: 'authentication_error', + message: 'Cannot authenticate with the server. Please check your API key and try again.', + }, +}; + +export const INVALID_JSON_BODY = { + detail: [ + { + type: 'json_invalid', + loc: ['body', 0], + msg: 'JSON decode error', + input: {}, + ctx: { error: 'Expecting value' }, + }, + ], +}; + +export const ACCEPTED_MODELS = ['jev-latest', 'jev-preview', 'jev-1.13.0']; + +export const MODEL_LIST = { + models: [ + { + name: 'jev-latest', + description: 'The most recent stable, official release. Points to jev-1.13.0.', + release_date: '2026-09-15', + }, + { + name: 'jev-preview', + description: 'The most recent release, whether or not it is an official one. Currently points to jev-1.13.0.', + release_date: '2026-09-15', + }, + ], +}; + +function isPlainObject(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isJsonContent(value) { + return typeof value === 'string' || Array.isArray(value) || isPlainObject(value); +} + +function missing(loc, input) { + return { type: 'missing', loc, msg: 'Field required', input }; +} + +function invalidContent(loc, input) { + return { + type: 'union_type', + loc, + msg: 'Input should be a string, object, or array', + input, + }; +} + +/** + * Bearer token, or null when the header is absent, non-bearer, or blank. + * The live API treats those cases as "no key" (403), not as a bad key (401). + */ +export function bearerToken(authorization) { + if (typeof authorization !== 'string') { + return null; + } + const match = /^Bearer\s+(\S+)\s*$/i.exec(authorization); + if (!match) { + return null; + } + return match[1]; +} + +export function authenticate({ authorization, apiKey }) { + const token = bearerToken(authorization); + if (token === null) { + return { ok: false, status: 403, body: MISSING_API_KEY_BODY }; + } + if (apiKey !== null && token !== apiKey) { + return { ok: false, status: 401, body: INVALID_API_KEY_BODY }; + } + return { ok: true }; +} + +function validateQuestion({ id, question, errors }) { + const loc = ['body', 'questions', id]; + if (!isPlainObject(question)) { + errors.push({ + type: 'model_attributes_type', + loc, + msg: 'Input should be a valid dictionary or object to extract fields from', + input: question, + }); + return null; + } + + const type = question.type; + if (type !== 'noul' && type !== 'choice' && type !== 'score') { + errors.push( + type === undefined + ? missing([...loc, 'type'], question) + : { + type: 'literal_error', + loc: [...loc, 'type'], + msg: "Input should be 'noul', 'choice', or 'score'", + input: type, + }, + ); + return null; + } + + if (!Object.hasOwn(question, 'instructions')) { + errors.push(missing([...loc, 'instructions'], question)); + return null; + } + if (!isJsonContent(question.instructions)) { + errors.push(invalidContent([...loc, 'instructions'], question.instructions)); + return null; + } + + if (type === 'noul') { + return validateNoul({ id, question, errors, loc }); + } + if (type === 'choice') { + return validateChoice({ id, question, errors, loc }); + } + return validateScore({ id, question, errors, loc }); +} + +function validateNoul({ question, errors, loc }) { + const normalized = { + type: 'noul', + instructions: question.instructions, + }; + if (!Object.hasOwn(question, 'criteria') || question.criteria === undefined) { + return normalized; + } + if (question.criteria === null) { + return { ...normalized, criteria: null }; + } + if (!isPlainObject(question.criteria)) { + errors.push({ + type: 'model_attributes_type', + loc: [...loc, 'criteria'], + msg: 'Input should be an object or null', + input: question.criteria, + }); + return null; + } + const criteria = {}; + for (const key of ['true', 'false']) { + if (!Object.hasOwn(question.criteria, key)) { + continue; + } + const description = question.criteria[key]; + if (description !== null && !isJsonContent(description)) { + errors.push(invalidContent([...loc, 'criteria', key], description)); + return null; + } + criteria[key] = description; + } + return { ...normalized, criteria }; +} + +function validateChoice({ question, errors, loc }) { + if (!Object.hasOwn(question, 'criteria')) { + errors.push(missing([...loc, 'criteria'], question)); + return null; + } + if (!isPlainObject(question.criteria)) { + errors.push({ + type: 'dict_type', + loc: [...loc, 'criteria'], + msg: 'Input should be an object mapping options to descriptions', + input: question.criteria, + }); + return null; + } + const keys = Object.keys(question.criteria); + if (keys.length < 1) { + errors.push({ + type: 'too_short', + loc: [...loc, 'criteria'], + msg: 'At least one option is required', + input: question.criteria, + ctx: { min_length: 1 }, + }); + return null; + } + const criteria = {}; + for (const key of keys) { + const description = question.criteria[key]; + if (description !== null && !isJsonContent(description)) { + errors.push(invalidContent([...loc, 'criteria', key], description)); + return null; + } + criteria[key] = description; + } + return { type: 'choice', instructions: question.instructions, criteria }; +} + +function validateScore({ question, errors, loc }) { + if (!Object.hasOwn(question, 'criteria')) { + errors.push(missing([...loc, 'criteria'], question)); + return null; + } + if (!Array.isArray(question.criteria)) { + errors.push({ + type: 'list_type', + loc: [...loc, 'criteria'], + msg: 'Input should be an array of level descriptions', + input: question.criteria, + }); + return null; + } + if (question.criteria.length < 2) { + errors.push({ + type: 'too_short', + loc: [...loc, 'criteria'], + msg: 'At least two levels are required', + input: question.criteria, + ctx: { min_length: 2, actual_length: question.criteria.length }, + }); + return null; + } + const criteria = []; + for (let index = 0; index < question.criteria.length; index += 1) { + const description = question.criteria[index]; + if (!isJsonContent(description)) { + errors.push(invalidContent([...loc, 'criteria', index], description)); + return null; + } + criteria.push(description); + } + return { type: 'score', instructions: question.instructions, criteria }; +} + +/** + * Validate a parsed JSON value against the systemone request. + * Returns FastAPI-style `detail` entries; does not include the wrapping object. + */ +export function validateSystemOne(value) { + if (!isPlainObject(value)) { + return { + ok: false, + errors: [ + { + type: 'model_attributes_type', + loc: ['body'], + msg: 'Input should be a valid dictionary or object to extract fields from', + input: value, + }, + ], + }; + } + + const errors = []; + let state = null; + let model = null; + + if (!Object.hasOwn(value, 'state')) { + errors.push(missing(['body', 'state'], value)); + } else if (!isJsonContent(value.state)) { + errors.push(invalidContent(['body', 'state'], value.state)); + } else { + state = value.state; + } + + if (!Object.hasOwn(value, 'model')) { + errors.push(missing(['body', 'model'], value)); + } else if (typeof value.model !== 'string' || value.model.length === 0) { + errors.push({ + type: 'string_type', + loc: ['body', 'model'], + msg: 'Input should be a non-empty string', + input: value.model, + }); + } else if (!ACCEPTED_MODELS.includes(value.model)) { + errors.push({ + type: 'enum', + loc: ['body', 'model'], + msg: "Input should be 'jev-latest', 'jev-preview', or 'jev-1.13.0'", + input: value.model, + }); + } else { + model = value.model; + } + + const questions = {}; + if (!Object.hasOwn(value, 'questions')) { + errors.push(missing(['body', 'questions'], value)); + } else if (!isPlainObject(value.questions)) { + errors.push({ + type: 'dict_type', + loc: ['body', 'questions'], + msg: 'Input should be an object mapping question ids to questions', + input: value.questions, + }); + } else if (Object.keys(value.questions).length < 1) { + errors.push({ + type: 'too_short', + loc: ['body', 'questions'], + msg: 'At least one question is required', + input: value.questions, + ctx: { min_length: 1 }, + }); + } else { + for (const [id, question] of Object.entries(value.questions)) { + const normalized = validateQuestion({ id, question, errors }); + if (normalized !== null) { + questions[id] = normalized; + } + } + } + + if (errors.length > 0 || state === null || model === null) { + return { ok: false, errors }; + } + + return { ok: true, request: { state, model, questions } }; +} diff --git a/tools/typesafe-mock/src/evaluate.mjs b/tools/typesafe-mock/src/evaluate.mjs new file mode 100644 index 000000000..4f9bcb1fc --- /dev/null +++ b/tools/typesafe-mock/src/evaluate.mjs @@ -0,0 +1,178 @@ +/** + * Every answer is the published example, not a model call. + * Exact doc requests return that JSON. Any other request reuses the same numbers. + */ + +const USAGE = { input_tokens: 312, output_tokens: 48 }; + +const PAYOUT_STATE = 'Help! My payouts have been failing for 3 days.'; + +export const documentedExamples = [ + { + name: 'noul', + request: { + state: PAYOUT_STATE, + model: 'jev-latest', + questions: { + is_urgent: { + type: 'noul', + instructions: 'Does this convey urgency?', + }, + }, + }, + response: { + model: 'jev-latest', + answers: { + is_urgent: { type: 'noul', noul: 0.92 }, + }, + usage: USAGE, + }, + }, + { + name: 'choice', + request: { + state: PAYOUT_STATE, + model: 'jev-latest', + questions: { + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { + billing: 'Payments, invoicing, refunds', + technical: 'Bugs, outages, integrations', + sales: 'Pricing, upgrades, new accounts', + }, + }, + }, + }, + response: { + model: 'jev-latest', + answers: { + department: { + type: 'choice', + choice: 'technical', + probabilities: { billing: 0.08, technical: 0.85, sales: 0.07 }, + confidence: 0.82, + }, + }, + usage: USAGE, + }, + }, + { + name: 'score', + request: { + state: PAYOUT_STATE, + model: 'jev-latest', + questions: { + frustration: { + type: 'score', + instructions: 'How frustrated is the customer?', + criteria: ['Calm', 'Frustrated', 'Very angry'], + }, + }, + }, + response: { + model: 'jev-latest', + answers: { + frustration: { + type: 'score', + score: 1.6, + legend: { 0: 'Calm', 1: 'Frustrated', 2: 'Very angry' }, + probabilities: { 0: 0.05, 1: 0.3, 2: 0.65 }, + confidence: 0.78, + }, + }, + usage: USAGE, + }, + }, +]; + +function isPlainObject(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function stable(value) { + if (Array.isArray(value)) { + return value.map(item => stable(item)); + } + if (!isPlainObject(value)) { + return value; + } + const sorted = {}; + for (const key of Object.keys(value).sort()) { + sorted[key] = stable(value[key]); + } + return sorted; +} + +function sameRequest(left, right) { + return JSON.stringify(stable(left)) === JSON.stringify(stable(right)); +} + +function fixedChoice(criteria) { + const keys = Object.keys(criteria); + const probabilities = {}; + if (keys.length === 1) { + probabilities[keys[0]] = 1; + return { type: 'choice', choice: keys[0], probabilities, confidence: 0.82 }; + } + if (keys.length === 2) { + probabilities[keys[0]] = 0.15; + probabilities[keys[1]] = 0.85; + return { type: 'choice', choice: keys[1], probabilities, confidence: 0.82 }; + } + const weights = [0.08, 0.85, 0.07]; + keys.forEach((key, index) => { + probabilities[key] = index < weights.length ? weights[index] : 0; + }); + return { type: 'choice', choice: keys[1], probabilities, confidence: 0.82 }; +} + +function fixedScore(criteria) { + const legend = {}; + const probabilities = {}; + criteria.forEach((description, index) => { + legend[String(index)] = description; + probabilities[String(index)] = 0; + }); + if (criteria.length === 2) { + probabilities['0'] = 0.35; + probabilities['1'] = 0.65; + return { type: 'score', score: 0.65, legend, probabilities, confidence: 0.78 }; + } + const weights = [0.05, 0.3, 0.65]; + weights.forEach((weight, index) => { + if (index < criteria.length) { + probabilities[String(index)] = weight; + } + }); + return { type: 'score', score: 1.6, legend, probabilities, confidence: 0.78 }; +} + +function answerQuestion(question) { + if (question.type === 'noul') { + return { type: 'noul', noul: 0.92 }; + } + if (question.type === 'choice') { + return fixedChoice(question.criteria); + } + return fixedScore(question.criteria); +} + +export function evaluate(request) { + for (const example of documentedExamples) { + if (sameRequest(example.request, request)) { + return structuredClone(example.response); + } + } + + const answers = {}; + for (const [id, question] of Object.entries(request.questions)) { + answers[id] = answerQuestion(question); + } + return { + model: request.model, + answers, + usage: USAGE, + }; +} diff --git a/tools/typesafe-mock/src/server.mjs b/tools/typesafe-mock/src/server.mjs new file mode 100644 index 000000000..b35297bbc --- /dev/null +++ b/tools/typesafe-mock/src/server.mjs @@ -0,0 +1,201 @@ +import { randomBytes } from 'node:crypto'; +import { createServer } from 'node:http'; +import { Buffer } from 'node:buffer'; + +import { evaluate } from './evaluate.mjs'; +import { INVALID_JSON_BODY, MODEL_LIST, authenticate, validateSystemOne } from './errors.mjs'; + +const MAX_BODY_BYTES = 8_000_000; + +function requestId() { + return `req_${randomBytes(16).toString('hex')}`; +} + +function headerValue(headers, name) { + const value = headers[name]; + if (typeof value === 'string') { + return value; + } + if (Array.isArray(value)) { + return value[0]; + } + return undefined; +} + +function send(res, status, body, extraHeaders = {}) { + const payload = JSON.stringify(body); + res.writeHead(status, { + 'content-type': 'application/json', + 'x-typesafe-request-id': requestId(), + ...extraHeaders, + }); + res.end(payload); +} + +async function readRawBody(req) { + const chunks = []; + let size = 0; + for await (const chunk of req) { + const buffer = typeof chunk === 'string' ? Buffer.from(chunk) : chunk; + size += buffer.length; + if (size > MAX_BODY_BYTES) { + return { ok: false }; + } + chunks.push(buffer); + } + return { ok: true, raw: Buffer.concat(chunks).toString('utf8') }; +} + +function forcedStatus(config, headers) { + const header = headerValue(headers, 'x-typesafe-mock-status'); + if (header === '429' || header === '529') { + return Number(header); + } + return config.forceStatus; +} + +function sendOverloaded(res, status) { + const message = status === 429 ? 'You have exceeded your rate limit.' : 'TypeSafe is temporarily overloaded.'; + send( + res, + status, + { detail: { error_type: status === 429 ? 'rate_limit_error' : 'overloaded_error', message } }, + { 'retry-after': '1', 'retry-after-ms': '1000' }, + ); +} + +async function handleSystemOne(req, res, config) { + const body = await readRawBody(req); + if (!body.ok) { + send(res, 422, { + detail: [{ type: 'too_long', loc: ['body'], msg: 'Request body exceeds 8000000 bytes', input: null }], + }); + return; + } + + if (body.raw.length > 0) { + try { + JSON.parse(body.raw); + } catch { + send(res, 422, INVALID_JSON_BODY); + return; + } + } + + const auth = authenticate({ authorization: headerValue(req.headers, 'authorization'), apiKey: config.apiKey }); + if (!auth.ok) { + send(res, auth.status, auth.body); + return; + } + + const overload = forcedStatus(config, req.headers); + if (overload !== null) { + sendOverloaded(res, overload); + return; + } + + if (body.raw.length === 0) { + send(res, 422, { detail: [{ type: 'missing', loc: ['body'], msg: 'Field required', input: null }] }); + return; + } + + const parsed = JSON.parse(body.raw); + const validated = validateSystemOne(parsed); + if (!validated.ok) { + send(res, 422, { detail: validated.errors }); + return; + } + + send(res, 200, evaluate(validated.request)); +} + +function handleModels(req, res, config) { + const auth = authenticate({ authorization: headerValue(req.headers, 'authorization'), apiKey: config.apiKey }); + if (!auth.ok) { + send(res, auth.status, auth.body); + return; + } + const overload = forcedStatus(config, req.headers); + if (overload !== null) { + sendOverloaded(res, overload); + return; + } + send(res, 200, MODEL_LIST); +} + +function route(req, res, config) { + const pathname = new URL(req.url ?? '/', 'http://mock.local').pathname; + if (pathname === '/v1/systemone') { + if (req.method !== 'POST') { + send(res, 405, { detail: 'Method Not Allowed' }, { allow: 'POST' }); + return; + } + return handleSystemOne(req, res, config); + } + if (pathname === '/v1/models') { + if (req.method !== 'GET') { + send(res, 405, { detail: 'Method Not Allowed' }, { allow: 'GET' }); + return; + } + return handleModels(req, res, config); + } + send(res, 404, { detail: 'Not Found' }); + return undefined; +} + +export function loadConfig(env) { + const rawKey = env.TYPESAFE_API_KEY; + const apiKey = typeof rawKey === 'string' && rawKey.trim() !== '' ? rawKey.trim() : null; + const rawPort = env.TYPESAFE_MOCK_PORT; + let port = 8787; + if (typeof rawPort === 'string' && rawPort.trim() !== '') { + port = Number(rawPort); + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`TYPESAFE_MOCK_PORT must be an integer from 0 to 65535, received ${rawPort}`); + } + } + const rawForce = env.TYPESAFE_MOCK_FORCE_STATUS; + let forceStatus = null; + if (typeof rawForce === 'string' && rawForce.trim() !== '') { + if (rawForce !== '429' && rawForce !== '529') { + throw new Error(`TYPESAFE_MOCK_FORCE_STATUS must be 429 or 529, received ${rawForce}`); + } + forceStatus = Number(rawForce); + } + const host = typeof env.TYPESAFE_MOCK_HOST === 'string' && env.TYPESAFE_MOCK_HOST.trim() !== '' ? env.TYPESAFE_MOCK_HOST.trim() : '127.0.0.1'; + return { apiKey, port, host, forceStatus }; +} + +export function startMockServer(config) { + const server = createServer((req, res) => { + Promise.resolve(route(req, res, config)).catch(error => { + console.error('typesafe mock failed to handle a request', error); + if (!res.headersSent) { + send(res, 500, { detail: 'Internal Server Error' }); + } + }); + }); + + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(config.port, config.host, () => { + const address = server.address(); + const port = typeof address === 'object' && address !== null ? address.port : config.port; + const hostname = config.host === '0.0.0.0' || config.host === '::' ? '127.0.0.1' : config.host; + resolve({ + url: `http://${hostname}:${port}`, + close() { + return new Promise((closeResolve, closeReject) => { + server.close(error => { + if (error) { + closeReject(error); + return; + } + closeResolve(); + }); + }); + }, + }); + }); + }); +}