diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index eb4d56d..9f735ec 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -1,5 +1,16 @@ # PROJECT_STATUS — Reader +## Free AI gateway source update — 2 October + +Managed AI calls now use the Fleet gateway with canonical `reader` attribution. +Explicit BYOK remains first; production requires the service binding, and +gateway-managed SDK retries are disabled. Existing structured-output parsing, +token limits, and streaming responses are preserved. SDK contract tests and full +local `pnpm quality` pass (152 app tests and 20 extension tests); Cloudflare +build and top-level Wrangler dry-run pass. Pending PR review and +production release; no deployment was performed. Tracked in +[Free AI issue #83](https://github.com/sass-maker/free-ai/issues/83). + This is Reader's durable current and shipped product truth. For the full product, feature, and architecture record, see [`docs/`](docs/index.md). diff --git a/README.md b/README.md index 48d1b15..e30ff6a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ A modern web application for capturing and annotating articles with a distractio | Database | Cloudflare D1 via Drizzle ORM | | Auth | better-auth + Google OAuth | | File storage | Cloudflare R2 (`reader-pdfs`, bound as `PDFS_BUCKET`) | -| AI | free-ai-gateway (Workers AI chokepoint); BYOK providers (OpenAI/Anthropic/Gemini) + local AI | +| AI | free-ai-gateway routes managed requests to available providers; explicit BYOK and local AI | | CI/CD | GitHub Actions checks on push; production deployment is manual | ## Problem diff --git a/src/lib/ai-cloudflare.test.ts b/src/lib/ai-cloudflare.test.ts index 03ce99f..d458b65 100644 --- a/src/lib/ai-cloudflare.test.ts +++ b/src/lib/ai-cloudflare.test.ts @@ -1,4 +1,4 @@ -import { generateText } from 'ai'; +import { generateText, jsonSchema, Output, streamText } from 'ai'; import { describe, expect, it, vi } from 'vitest'; import { getLanguageModel } from './ai-cloudflare'; import { findSharedAiBudgetDenied, SharedAiBudgetDenied } from './shared-ai-budget'; @@ -107,3 +107,66 @@ describe('Workers AI language model budget integration', () => { expect(run).not.toHaveBeenCalled(); }); }); + +describe('Free AI gateway SDK adapter', () => { + it('preserves JSON mode, token bounds, stream passthrough, attribution, and production fail-closed behavior', async () => { + const requests: Request[] = []; + const fetch = vi.fn(async (request: Request) => { + requests.push(request); + const body = JSON.parse(await request.clone().text()) as { stream?: boolean }; + if (body.stream) { + return new Response( + 'data: {"choices":[{"delta":{"content":"streamed"},"finish_reason":null}]}\n\n' + + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n' + + 'data: [DONE]\n\n', + { headers: { 'content-type': 'text/event-stream' } } + ); + } + return Response.json({ + choices: [ + { message: { role: 'assistant', content: '{"ok":true}' }, finish_reason: 'stop' }, + ], + }); + }); + const binding = { fetch } as unknown as Fetcher; + const model = getLanguageModel({ + freeAiBinding: binding, + endpointUrl: '', + apiKey: '', + model: 'ignored', + nodeEnv: 'test', + }); + + await generateText({ + model, + prompt: 'Synthetic JSON request', + output: Output.object({ + schema: jsonSchema({ + type: 'object', + properties: { ok: { type: 'boolean' } }, + required: ['ok'], + }), + }), + maxOutputTokens: 73, + maxRetries: 0, + }); + const stream = streamText({ model, prompt: 'Synthetic stream request', maxRetries: 0 }); + await expect(stream.text).resolves.toBe('streamed'); + + expect(fetch).toHaveBeenCalledTimes(2); + for (const request of requests) { + expect(request.url).toBe('https://fleet-gateway.internal/v1/chat/completions'); + expect(request.headers.get('x-gateway-project-id')).toBe('reader'); + expect(request.headers.get('authorization')).toBe('Bearer service-binding'); + expect(JSON.parse(await request.clone().text()).model).toBe('auto'); + } + const jsonBody = JSON.parse(await requests[0].clone().text()); + expect(jsonBody.response_format).toEqual({ type: 'json_object' }); + expect(jsonBody.max_tokens).toBe(73); + expect(JSON.parse(await requests[1].clone().text()).stream).toBe(true); + + expect(() => + getLanguageModel({ endpointUrl: '', apiKey: '', model: 'ignored', nodeEnv: 'production' }) + ).toThrow(/required in production/); + }); +}); diff --git a/src/lib/ai-cloudflare.ts b/src/lib/ai-cloudflare.ts index f005393..eeb80d0 100644 --- a/src/lib/ai-cloudflare.ts +++ b/src/lib/ai-cloudflare.ts @@ -6,6 +6,7 @@ import type { AIConfig } from './ai-vendor'; import { createBudgetedWorkersAiBinding, type SharedBudgetNamespace } from './shared-ai-budget'; type WorkersAiBinding = Extract['binding']; +type FreeAiBinding = { fetch(request: Request): Promise }; /** * Build a LanguageModel from an AIConfig, talking to any OpenAI-compatible @@ -13,13 +14,22 @@ type WorkersAiBinding = Extract['bindin */ function createAIModel( config: AIConfig, - options?: { headers?: Record; name?: string } + options?: { + headers?: Record; + name?: string; + fetch?: typeof fetch; + supportsStructuredOutputs?: boolean; + } ): LanguageModel { const provider = createOpenAICompatible({ baseURL: config.endpointUrl.trim().replace(/\/+$/, ''), apiKey: config.apiKey, name: options?.name ?? 'reader-direct', headers: options?.headers, + ...(options?.fetch ? { fetch: options.fetch } : {}), + ...(options?.supportsStructuredOutputs !== undefined + ? { supportsStructuredOutputs: options.supportsStructuredOutputs } + : {}), }); return provider.chatModel(config.model); } @@ -29,6 +39,8 @@ const DEFAULT_WORKERS_AI_MODEL = '@cf/meta/llama-3.3-70b-instruct-fp8-fast'; interface CreateLanguageModelArgs { binding?: WorkersAiBinding; + freeAiBinding?: FreeAiBinding; + nodeEnv?: string; endpointUrl: string; apiKey: string; model: string; @@ -48,9 +60,24 @@ function getDirectApiKey(): string { return apiKey; } +function createFreeAiGatewayModel( + freeAiBinding: FreeAiBinding, + headers?: Record +): LanguageModel { + return createAIModel( + { endpointUrl: 'https://fleet-gateway.internal/v1', apiKey: 'service-binding', model: 'auto' }, + { + name: 'free-ai', + headers: { ...headers, 'x-gateway-project-id': 'reader' }, + fetch: (input, init) => freeAiBinding.fetch(new Request(input, init)), + supportsStructuredOutputs: false, + } + ); +} + /** - * Returns a model for an explicit BYOK endpoint or the project's own direct - * free-provider/local endpoint. No shared gateway fallback exists. + * Explicit BYOK endpoints win. Managed requests use the Fleet gateway; local + * direct endpoints are a development-only fallback. */ export function getLanguageModel({ binding, @@ -59,12 +86,21 @@ export function getLanguageModel({ model, headers, budgetNamespace, + freeAiBinding, + nodeEnv, }: CreateLanguageModelArgs): LanguageModel { // Honour explicit BYO config first (settings UI etc.). if (endpointUrl && apiKey) { return createAIModel({ endpointUrl, apiKey, model } as AIConfig, { headers }); } + if (freeAiBinding) { + return createFreeAiGatewayModel(freeAiBinding, headers); + } + if ((nodeEnv ?? process.env.NODE_ENV) === 'production') { + throw new Error('Free AI gateway service binding is required in production'); + } + if (binding) { return createWorkersAI({ binding: createBudgetedWorkersAiBinding(binding, budgetNamespace), diff --git a/src/lib/worker-env.ts b/src/lib/worker-env.ts index bda8e8c..94649c4 100644 --- a/src/lib/worker-env.ts +++ b/src/lib/worker-env.ts @@ -11,6 +11,7 @@ export type WorkerEnv = { AI_API_KEY?: string; AI_BASE_URL?: string; AI?: Ai; + FREE_AI?: { fetch(request: Request): Promise }; NEURON_BUDGET?: DurableObjectNamespace; LOCAL_AI_URL?: string; CLI_BRIDGE_URL?: string; diff --git a/src/worker/routes/ai.ts b/src/worker/routes/ai.ts index 9df7c69..1c429bf 100644 --- a/src/worker/routes/ai.ts +++ b/src/worker/routes/ai.ts @@ -91,6 +91,8 @@ ai.post('/chat', async (c) => { const result = streamText({ model: getLanguageModel({ binding: c.env.AI, + freeAiBinding: c.env.FREE_AI, + nodeEnv: c.env.NODE_ENV, budgetNamespace: c.env.NEURON_BUDGET, endpointUrl, apiKey, @@ -169,6 +171,8 @@ Remember to respond with valid JSON in the exact format specified.`; const result = await generateText({ model: getLanguageModel({ binding: c.env.AI, + freeAiBinding: c.env.FREE_AI, + nodeEnv: c.env.NODE_ENV, budgetNamespace: c.env.NEURON_BUDGET, endpointUrl, apiKey, diff --git a/src/worker/routes/articles.ts b/src/worker/routes/articles.ts index 28102f1..01a7e8b 100644 --- a/src/worker/routes/articles.ts +++ b/src/worker/routes/articles.ts @@ -293,6 +293,8 @@ articles.post('/:id/session-review', async (c) => { const result = await generateText({ model: getLanguageModel({ binding: c.env.AI, + freeAiBinding: c.env.FREE_AI, + nodeEnv: c.env.NODE_ENV, budgetNamespace: c.env.NEURON_BUDGET, endpointUrl, apiKey, @@ -300,7 +302,7 @@ articles.post('/:id/session-review', async (c) => { }), system: SESSION_REVIEW_SYSTEM_PROMPT, prompt: buildSessionReviewPrompt(article.title, noteInputs), - maxRetries: 1, + maxRetries: endpointUrl && apiKey ? 1 : 0, }); let parsed: { diff --git a/src/worker/routes/misc.ts b/src/worker/routes/misc.ts index 1c350ea..228cae8 100644 --- a/src/worker/routes/misc.ts +++ b/src/worker/routes/misc.ts @@ -373,6 +373,8 @@ misc.post('/ext/chat', async (c) => { const result = streamText({ model: getLanguageModel({ binding: c.env.AI, + freeAiBinding: c.env.FREE_AI, + nodeEnv: c.env.NODE_ENV, budgetNamespace: c.env.NEURON_BUDGET, endpointUrl: '', apiKey: '', diff --git a/wrangler.toml b/wrangler.toml index 837ca04..f115bbd 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -23,6 +23,11 @@ cpu_ms = 30000 [ai] binding = "AI" +[[services]] +binding = "FREE_AI" +service = "free-ai-gateway" +entrypoint = "FleetGateway" + [[durable_objects.bindings]] name = "NEURON_BUDGET" class_name = "NeuronBudgetDO"