Fix Claude vision requests failing on OpenAI-style image_url blocks.

src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts

@@ -253,6 +253,42 @@ describe('ClaudeProvider.complete request shape', () => {
         expect(args.max_tokens).toBe(0);
     });
 
+    it('converts OpenAI-shaped image_url parts into Anthropic image blocks', async () => {
+        const { provider } = makeProvider();
+        messagesCreateMock.mockResolvedValueOnce(baseResponse);
+
+        await withTestActor(() =>
+            provider.complete({
+                model: 'claude-sonnet-4-6',
+                messages: [
+                    {
+                        role: 'user',
+                        content: [
+                            { type: 'text', text: 'What do you see?' },
+                            {
+                                image_url: {
+                                    url: 'https://assets.puter.site/doge.jpeg',
+                                },
+                            },
+                        ],
+                    },
+                ],
+            }),
+        );
+
+        const [args] = messagesCreateMock.mock.calls[0]!;
+        expect(args.messages[0].content).toEqual([
+            { type: 'text', text: 'What do you see?' },
+            {
+                type: 'image',
+                source: {
+                    type: 'url',
+                    url: 'https://assets.puter.site/doge.jpeg',
+                },
+            },
+        ]);
+    });
+
     it('extracts system messages and forwards them as the top-level `system` field', async () => {
         const { provider } = makeProvider();
         messagesCreateMock.mockResolvedValueOnce(baseResponse);

src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts

@@ -46,6 +46,7 @@ import type {
     AIChatToolUseStream,
 } from '../../utils/Streaming.js';
 import { FILES_API_BETA, processPuterPathUploads } from './fileUpload.js';
+import { coerceImageContentParts } from './imageHandling.js';
 import { CLAUDE_MODELS } from './models.js';
 
 // Anthropic inline-compaction beta. The vendored SDK (0.68.0) doesn't type the
@@ -290,6 +291,8 @@ export class ClaudeProvider implements IChatProvider {
 
         const actor = Context.get('actor');
 
+        coerceImageContentParts(messages);
+
         // Upload any `puter_path` parts to Anthropic's Files API and rewrite
         // them in-place to reference the returned `file_id`. Must happen
         // before sdkParams snapshots `messages`.

src/backend/drivers/ai-chat/providers/claude/imageHandling.test.ts

@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { coerceImageContentParts } from './imageHandling.js';
+
+describe('coerceImageContentParts', () => {
+    it('converts puter.js vision shorthand `{ image_url: { url } }` blocks', () => {
+        const messages = [
+            {
+                role: 'user',
+                content: [
+                    { type: 'text', text: 'What do you see?' },
+                    {
+                        image_url: {
+                            url: 'https://assets.puter.site/doge.jpeg',
+                        },
+                    },
+                ],
+            },
+        ];
+
+        coerceImageContentParts(messages);
+
+        expect(messages[0]!.content).toEqual([
+            { type: 'text', text: 'What do you see?' },
+            {
+                type: 'image',
+                source: {
+                    type: 'url',
+                    url: 'https://assets.puter.site/doge.jpeg',
+                },
+            },
+        ]);
+    });
+
+    it('converts typed OpenAI `image_url` parts with nested url objects', () => {
+        const messages = [
+            {
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: { url: 'https://example.com/a.png' },
+                    },
+                ],
+            },
+        ];
+
+        coerceImageContentParts(messages);
+
+        expect(messages[0]!.content[0]).toEqual({
+            type: 'image',
+            source: { type: 'url', url: 'https://example.com/a.png' },
+        });
+    });
+
+    it('converts flat-string `image_url` parts', () => {
+        const messages = [
+            {
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: 'https://example.com/flat.png',
+                    },
+                ],
+            },
+        ];
+
+        coerceImageContentParts(messages);
+
+        expect(messages[0]!.content[0]).toEqual({
+            type: 'image',
+            source: { type: 'url', url: 'https://example.com/flat.png' },
+        });
+    });
+
+    it('parses data URIs into base64 image sources', () => {
+        const messages = [
+            {
+                content: [
+                    {
+                        image_url: {
+                            url: 'data:image/png;base64,QUJD',
+                        },
+                    },
+                ],
+            },
+        ];
+
+        coerceImageContentParts(messages);
+
+        expect(messages[0]!.content[0]).toEqual({
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: 'image/png',
+                data: 'QUJD',
+            },
+        });
+    });
+
+    it('leaves native Anthropic image blocks unchanged', () => {
+        const native = {
+            type: 'image',
+            source: { type: 'file', file_id: 'file_1' },
+        };
+        const messages = [{ content: [native] }];
+
+        coerceImageContentParts(messages);
+
+        expect(messages[0]!.content[0]).toBe(native);
+    });
+
+    it('leaves non-image content parts unchanged', () => {
+        const messages = [
+            {
+                content: [
+                    { type: 'text', text: 'hello' },
+                    { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} },
+                ],
+            },
+        ];
+
+        coerceImageContentParts(messages);
+
+        expect(messages[0]!.content).toEqual([
+            { type: 'text', text: 'hello' },
+            { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} },
+        ]);
+    });
+});

src/backend/drivers/ai-chat/providers/claude/imageHandling.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+interface ImageUrlPart {
+    type?: string;
+    image_url?: string | { url?: string };
+    source?: unknown;
+    [key: string]: unknown;
+}
+
+type AnthropicImagePart = {
+    type: 'image';
+    source:
+        | { type: 'url'; url: string }
+        | { type: 'base64'; media_type: string; data: string };
+};
+
+const extractImageUrl = (part: ImageUrlPart): string | undefined => {
+    const raw = part.image_url;
+    if (typeof raw === 'string') return raw;
+    if (raw && typeof raw === 'object' && typeof raw.url === 'string') {
+        return raw.url;
+    }
+    return undefined;
+};
+
+const toAnthropicImagePart = (url: string): AnthropicImagePart => {
+    const dataUriMatch = url.match(/^data:([^;,]+);base64,(.+)$/);
+    if (dataUriMatch) {
+        return {
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: dataUriMatch[1]!,
+                data: dataUriMatch[2]!,
+            },
+        };
+    }
+    return {
+        type: 'image',
+        source: { type: 'url', url },
+    };
+};
+
+/**
+ * Rewrite OpenAI-style `image_url` content parts into Anthropic `image`
+ * blocks. The puter.js vision shorthand sends `{ image_url: { url } }`
+ * without a `type`, which Anthropic rejects (`content.N.type: Field required`).
+ */
+export function coerceImageContentParts(
+    messages: Array<{ content?: unknown }>,
+): void {
+    for (const message of messages) {
+        if (!Array.isArray(message.content)) continue;
+        message.content = (message.content as ImageUrlPart[]).map((part) => {
+            if (!part || typeof part !== 'object') return part;
+            if (part.type === 'image' && part.source) return part;
+            const url = extractImageUrl(part);
+            if (url === undefined) return part;
+            return toAnthropicImagePart(url);
+        });
+    }
+}

src/puter-js/test/ai.test.js

@@ -278,10 +278,7 @@ const generateAllTests = function() {
 // per-model tests above, each of these targets a specific known-good model.
 
 const TEST_IMAGE_URL = "https://assets.puter.site/doge.jpeg";
-// Direct-claude models are excluded: the claude provider is the one chat
-// provider that doesn't infer the `type` on the SDK's `{ image_url }`
-// media blocks, so the vision shorthand 400s against Anthropic today.
-const VISION_MODELS = ["gpt-5-nano", "gemini-2.5-pro"];
+const VISION_MODELS = ["gpt-5-nano", "gemini-2.5-pro", "claude-sonnet-4-6"];
 
 // The test image is a Shiba Inu; any vision-capable model should say so.
 const assertMentionsDog = function(result) {