Route Reader managed AI through Free AI gateway

PROJECT_STATUS.md

@@ -1,5 +1,16 @@
 # PROJECT_STATUS — Reader
 
+## Free AI gateway source update — 2 October
+
+Managed AI calls now use the Fleet gateway with canonical `reader` attribution.
+Explicit BYOK remains first; production requires the service binding, and
+gateway-managed SDK retries are disabled. Existing structured-output parsing,
+token limits, and streaming responses are preserved. SDK contract tests and full
+local `pnpm quality` pass (152 app tests and 20 extension tests); Cloudflare
+build and top-level Wrangler dry-run pass. Pending PR review and
+production release; no deployment was performed. Tracked in
+[Free AI issue #83](https://github.com/sass-maker/free-ai/issues/83).
+
 This is Reader's durable current and shipped product truth. For the full
 product, feature, and architecture record, see [`docs/`](docs/index.md).
 

README.md

@@ -13,7 +13,7 @@ A modern web application for capturing and annotating articles with a distractio
 | Database     | Cloudflare D1 via Drizzle ORM                                                                |
 | Auth         | better-auth + Google OAuth                                                                   |
 | File storage | Cloudflare R2 (`reader-pdfs`, bound as `PDFS_BUCKET`)                                        |
-| AI           | free-ai-gateway (Workers AI chokepoint); BYOK providers (OpenAI/Anthropic/Gemini) + local AI |
+| AI           | free-ai-gateway routes managed requests to available providers; explicit BYOK and local AI |
 | CI/CD        | GitHub Actions checks on push; production deployment is manual                                 |
 
 ## Problem

src/lib/ai-cloudflare.test.ts

@@ -1,4 +1,4 @@
-import { generateText } from 'ai';
+import { generateText, jsonSchema, Output, streamText } from 'ai';
 import { describe, expect, it, vi } from 'vitest';
 import { getLanguageModel } from './ai-cloudflare';
 import { findSharedAiBudgetDenied, SharedAiBudgetDenied } from './shared-ai-budget';
@@ -107,3 +107,66 @@ describe('Workers AI language model budget integration', () => {
     expect(run).not.toHaveBeenCalled();
   });
 });
+
+describe('Free AI gateway SDK adapter', () => {
+  it('preserves JSON mode, token bounds, stream passthrough, attribution, and production fail-closed behavior', async () => {
+    const requests: Request[] = [];
+    const fetch = vi.fn(async (request: Request) => {
+      requests.push(request);
+      const body = JSON.parse(await request.clone().text()) as { stream?: boolean };
+      if (body.stream) {
+        return new Response(
+          'data: {"choices":[{"delta":{"content":"streamed"},"finish_reason":null}]}\n\n' +
+            'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n' +
+            'data: [DONE]\n\n',
+          { headers: { 'content-type': 'text/event-stream' } }
+        );
+      }
+      return Response.json({
+        choices: [
+          { message: { role: 'assistant', content: '{"ok":true}' }, finish_reason: 'stop' },
+        ],
+      });
+    });
+    const binding = { fetch } as unknown as Fetcher;
+    const model = getLanguageModel({
+      freeAiBinding: binding,
+      endpointUrl: '',
+      apiKey: '',
+      model: 'ignored',
+      nodeEnv: 'test',
+    });
+
+    await generateText({
+      model,
+      prompt: 'Synthetic JSON request',
+      output: Output.object({
+        schema: jsonSchema({
+          type: 'object',
+          properties: { ok: { type: 'boolean' } },
+          required: ['ok'],
+        }),
+      }),
+      maxOutputTokens: 73,
+      maxRetries: 0,
+    });
+    const stream = streamText({ model, prompt: 'Synthetic stream request', maxRetries: 0 });
+    await expect(stream.text).resolves.toBe('streamed');
+
+    expect(fetch).toHaveBeenCalledTimes(2);
+    for (const request of requests) {
+      expect(request.url).toBe('https://fleet-gateway.internal/v1/chat/completions');
+      expect(request.headers.get('x-gateway-project-id')).toBe('reader');
+      expect(request.headers.get('authorization')).toBe('Bearer service-binding');
+      expect(JSON.parse(await request.clone().text()).model).toBe('auto');
+    }
+    const jsonBody = JSON.parse(await requests[0].clone().text());
+    expect(jsonBody.response_format).toEqual({ type: 'json_object' });
+    expect(jsonBody.max_tokens).toBe(73);
+    expect(JSON.parse(await requests[1].clone().text()).stream).toBe(true);
+
+    expect(() =>
+      getLanguageModel({ endpointUrl: '', apiKey: '', model: 'ignored', nodeEnv: 'production' })
+    ).toThrow(/required in production/);
+  });
+});

src/lib/ai-cloudflare.ts

@@ -6,20 +6,30 @@ import type { AIConfig } from './ai-vendor';
 import { createBudgetedWorkersAiBinding, type SharedBudgetNamespace } from './shared-ai-budget';
 
 type WorkersAiBinding = Extract<WorkersAISettings, { binding: unknown }>['binding'];
+type FreeAiBinding = { fetch(request: Request): Promise<Response> };
 
 /**
  * Build a LanguageModel from an AIConfig, talking to any OpenAI-compatible
  * endpoint (formerly @saas-maker/ai's createAIModel).
  */
 function createAIModel(
   config: AIConfig,
-  options?: { headers?: Record<string, string>; name?: string }
+  options?: {
+    headers?: Record<string, string>;
+    name?: string;
+    fetch?: typeof fetch;
+    supportsStructuredOutputs?: boolean;
+  }
 ): LanguageModel {
   const provider = createOpenAICompatible({
     baseURL: config.endpointUrl.trim().replace(/\/+$/, ''),
     apiKey: config.apiKey,
     name: options?.name ?? 'reader-direct',
     headers: options?.headers,
+    ...(options?.fetch ? { fetch: options.fetch } : {}),
+    ...(options?.supportsStructuredOutputs !== undefined
+      ? { supportsStructuredOutputs: options.supportsStructuredOutputs }
+      : {}),
   });
   return provider.chatModel(config.model);
 }
@@ -29,6 +39,8 @@ const DEFAULT_WORKERS_AI_MODEL = '@cf/meta/llama-3.3-70b-instruct-fp8-fast';
 
 interface CreateLanguageModelArgs<Id = never> {
   binding?: WorkersAiBinding;
+  freeAiBinding?: FreeAiBinding;
+  nodeEnv?: string;
   endpointUrl: string;
   apiKey: string;
   model: string;
@@ -48,9 +60,24 @@ function getDirectApiKey(): string {
   return apiKey;
 }
 
+function createFreeAiGatewayModel(
+  freeAiBinding: FreeAiBinding,
+  headers?: Record<string, string>
+): LanguageModel {
+  return createAIModel(
+    { endpointUrl: 'https://fleet-gateway.internal/v1', apiKey: 'service-binding', model: 'auto' },
+    {
+      name: 'free-ai',
+      headers: { ...headers, 'x-gateway-project-id': 'reader' },
+      fetch: (input, init) => freeAiBinding.fetch(new Request(input, init)),
+      supportsStructuredOutputs: false,
+    }
+  );
+}
+
 /**
- * Returns a model for an explicit BYOK endpoint or the project's own direct
- * free-provider/local endpoint. No shared gateway fallback exists.
+ * Explicit BYOK endpoints win. Managed requests use the Fleet gateway; local
+ * direct endpoints are a development-only fallback.
  */
 export function getLanguageModel<Id = never>({
   binding,
@@ -59,12 +86,21 @@ export function getLanguageModel<Id = never>({
   model,
   headers,
   budgetNamespace,
+  freeAiBinding,
+  nodeEnv,
 }: CreateLanguageModelArgs<Id>): LanguageModel {
   // Honour explicit BYO config first (settings UI etc.).
   if (endpointUrl && apiKey) {
     return createAIModel({ endpointUrl, apiKey, model } as AIConfig, { headers });
   }
 
+  if (freeAiBinding) {
+    return createFreeAiGatewayModel(freeAiBinding, headers);
+  }
+  if ((nodeEnv ?? process.env.NODE_ENV) === 'production') {
+    throw new Error('Free AI gateway service binding is required in production');
+  }
+
   if (binding) {
     return createWorkersAI({
       binding: createBudgetedWorkersAiBinding(binding, budgetNamespace),

src/lib/worker-env.ts

@@ -11,6 +11,7 @@ export type WorkerEnv = {
   AI_API_KEY?: string;
   AI_BASE_URL?: string;
   AI?: Ai;
+  FREE_AI?: { fetch(request: Request): Promise<Response> };
   NEURON_BUDGET?: DurableObjectNamespace;
   LOCAL_AI_URL?: string;
   CLI_BRIDGE_URL?: string;

src/worker/routes/ai.ts

@@ -91,6 +91,8 @@ ai.post('/chat', async (c) => {
     const result = streamText({
       model: getLanguageModel({
         binding: c.env.AI,
+        freeAiBinding: c.env.FREE_AI,
+        nodeEnv: c.env.NODE_ENV,
         budgetNamespace: c.env.NEURON_BUDGET,
         endpointUrl,
         apiKey,
@@ -169,6 +171,8 @@ Remember to respond with valid JSON in the exact format specified.`;
     const result = await generateText({
       model: getLanguageModel({
         binding: c.env.AI,
+        freeAiBinding: c.env.FREE_AI,
+        nodeEnv: c.env.NODE_ENV,
         budgetNamespace: c.env.NEURON_BUDGET,
         endpointUrl,
         apiKey,

src/worker/routes/articles.ts

@@ -293,14 +293,16 @@ articles.post('/:id/session-review', async (c) => {
     const result = await generateText({
       model: getLanguageModel({
         binding: c.env.AI,
+        freeAiBinding: c.env.FREE_AI,
+        nodeEnv: c.env.NODE_ENV,
         budgetNamespace: c.env.NEURON_BUDGET,
         endpointUrl,
         apiKey,
         model,
       }),
       system: SESSION_REVIEW_SYSTEM_PROMPT,
       prompt: buildSessionReviewPrompt(article.title, noteInputs),
-      maxRetries: 1,
+      maxRetries: endpointUrl && apiKey ? 1 : 0,
     });
 
     let parsed: {

src/worker/routes/misc.ts

@@ -373,6 +373,8 @@ misc.post('/ext/chat', async (c) => {
     const result = streamText({
       model: getLanguageModel({
         binding: c.env.AI,
+        freeAiBinding: c.env.FREE_AI,
+        nodeEnv: c.env.NODE_ENV,
         budgetNamespace: c.env.NEURON_BUDGET,
         endpointUrl: '',
         apiKey: '',

wrangler.toml

@@ -23,6 +23,11 @@ cpu_ms = 30000
 [ai]
 binding = "AI"
 
+[[services]]
+binding = "FREE_AI"
+service = "free-ai-gateway"
+entrypoint = "FleetGateway"
+
 [[durable_objects.bindings]]
 name = "NEURON_BUDGET"
 class_name = "NeuronBudgetDO"