From 9731f7bbadc1494cf76a4c836c50479e465e6797 Mon Sep 17 00:00:00 2001 From: kolaworld Date: Sun, 16 Aug 2026 23:13:52 -0400 Subject: [PATCH 1/3] fix(ai): persist structured output in message history Closes #1072 --- .changeset/persist-structured-output-parts.md | 6 + packages/ai-persistence/src/middleware.ts | 61 +----- .../ai-persistence/tests/reconstruct.test.ts | 31 +++ .../tests/with-persistence.test.ts | 185 +++++++++++++++++- packages/ai/src/activities/chat/index.ts | 98 +++++++++- packages/ai/src/activities/chat/messages.ts | 14 +- packages/ai/src/types.ts | 6 + packages/ai/src/utilities/chat-params.ts | 1 + packages/ai/tests/chat-params.test.ts | 20 ++ packages/ai/tests/chat.test.ts | 48 +++++ packages/ai/tests/message-converters.test.ts | 5 + .../src/routes/api.persistence-durability.ts | 66 ++++++- .../e2e/tests/persistence-durability.spec.ts | 45 +++++ 13 files changed, 509 insertions(+), 77 deletions(-) create mode 100644 .changeset/persist-structured-output-parts.md diff --git a/.changeset/persist-structured-output-parts.md b/.changeset/persist-structured-output-parts.md new file mode 100644 index 0000000000..1245e84b46 --- /dev/null +++ b/.changeset/persist-structured-output-parts.md @@ -0,0 +1,6 @@ +--- +'@tanstack/ai': patch +'@tanstack/ai-persistence': patch +--- + +Persist completed structured outputs as structured-output message parts and restore them during chat hydration. diff --git a/packages/ai-persistence/src/middleware.ts b/packages/ai-persistence/src/middleware.ts index dbeeea2928..de9e97119b 100644 --- a/packages/ai-persistence/src/middleware.ts +++ b/packages/ai-persistence/src/middleware.ts @@ -28,7 +28,6 @@ import type { GenerationFinishInfo, GenerationMiddleware, GenerationMiddlewareContext, - ModelMessage, PersistedArtifactActivity, PersistedArtifactRef, PersistedArtifactRole, @@ -435,43 +434,6 @@ function resumeToolStateFromPending( return { approvals, clientToolResults, cancelledToolCallIds } } -/** - * Build the transcript to persist when a run finishes successfully. - * - * The chat engine appends an assistant message to the middleware message list - * only when that turn carries tool calls (to feed the agent loop); a run's - * terminal *text* reply is never appended. So `ctx.messages` at `onFinish` is - * missing the assistant's final answer. Reattach it from the finish info — - * `info.content` is the last turn's accumulated text (reset each cycle) — so - * the stored thread is the complete conversation a server-authoritative client - * hydrates on load. A guard avoids duplicating a terminal assistant turn should - * the engine ever start appending it itself. - */ -function finishedTranscript( - messages: ReadonlyArray, - info: FinishInfo, - messageId: string | undefined, - createdAt: Date | undefined, -): Array { - const transcript = [...messages] - const last = transcript[transcript.length - 1] - const alreadyPresent = - last?.role === 'assistant' && - last.toolCalls === undefined && - last.content === info.content - if (info.content && !alreadyPresent) { - // Stamp the terminal turn with its stream messageId so a hydrated bubble - // keeps the same identity as the live stream (in-place resume on reload). - transcript.push({ - role: 'assistant', - content: info.content, - ...(messageId ? { id: messageId } : {}), - ...(createdAt ? { createdAt } : {}), - }) - } - return transcript -} - function interruptPayload(interrupt: unknown): Record { return interrupt && typeof interrupt === 'object' ? { ...(interrupt as Record) } @@ -1535,11 +1497,9 @@ export function withPersistence( }, async onChunk(ctx: ChatMiddlewareContext, chunk: StreamChunk) { - // Always capture the current assistant turn's stream messageId (cheap), - // regardless of snapshotStreaming — it's persisted onto the assistant - // message so its identity survives hydrate and a reload resumes the same - // bubble in place. - if (ctx.phase === 'modelStream') { + // Capture the current assistant turn's identity for optional in-progress + // snapshots. Completed messages already live in `ctx.messages`. + if (snapshotStreaming && ctx.phase === 'modelStream') { const s = runState.get(ctx) if (s && chunk.type === 'TEXT_MESSAGE_START') { s.streamingMessageId = chunk.messageId @@ -1559,9 +1519,8 @@ export function withPersistence( // (B) Optional throttled snapshot of the in-progress assistant reply, so // partial output survives a crash/reload before onFinish. Off unless - // `snapshotStreaming` is set. We accumulate the terminal turn's text here - // (the engine only appends assistant turns with tool calls to - // `ctx.messages`, never a streaming text reply), then persist + // `snapshotStreaming` is set. The completed turn enters `ctx.messages` + // only after streaming ends, so accumulate its text here and persist // `ctx.messages` + that partial assistant message (tagged with its id). if ( snapshotStreaming && @@ -1634,15 +1593,7 @@ export function withPersistence( // resumes stay pending so a retry can re-apply them. Completing the run // or consuming approvals before the durable history lands leaves a // "finished" run whose transcript is missing the terminal turn. - await messageStore.saveThread( - ctx.threadId, - finishedTranscript( - ctx.messages, - info, - state?.streamingMessageId, - state?.streamingMessageCreatedAt, - ), - ) + await messageStore.saveThread(ctx.threadId, [...ctx.messages]) await completeRun(runs, ctx.runId, info.usage) await commitPendingResumes(state, persistence.stores.interrupts) }, diff --git a/packages/ai-persistence/tests/reconstruct.test.ts b/packages/ai-persistence/tests/reconstruct.test.ts index 8a5091c108..3e9ae1dd5d 100644 --- a/packages/ai-persistence/tests/reconstruct.test.ts +++ b/packages/ai-persistence/tests/reconstruct.test.ts @@ -34,6 +34,37 @@ describe('reconstructChat', () => { expect(parsed.activeRun).toBeNull() }) + it('restores persisted structured output as a message part', async () => { + const persistence = memoryPersistence() + const structuredOutput = { + type: 'structured-output' as const, + status: 'complete' as const, + raw: '{"name":"Ada"}', + data: { name: 'Ada' }, + partial: { name: 'Ada' }, + } + await persistence.stores.messages!.saveThread('t1', [ + { + id: 'assistant-1', + role: 'assistant', + content: structuredOutput.raw, + structuredOutput, + }, + ]) + + const parsed = await body( + await reconstructChat( + persistence, + new Request('http://example.test/api/chat?threadId=t1'), + ), + ) + expect(parsed.messages[0]).toMatchObject({ + id: 'assistant-1', + role: 'assistant', + parts: [structuredOutput], + }) + }) + it('reports the active run for a thread that is still generating', async () => { const persistence = memoryPersistence() await persistence.stores.messages!.saveThread('t1', [ diff --git a/packages/ai-persistence/tests/with-persistence.test.ts b/packages/ai-persistence/tests/with-persistence.test.ts index 338431c927..48aa12c8c1 100644 --- a/packages/ai-persistence/tests/with-persistence.test.ts +++ b/packages/ai-persistence/tests/with-persistence.test.ts @@ -126,13 +126,12 @@ describe('withPersistence (state-only)', () => { expect(chunks.length).toBeGreaterThan(0) expect(chunks.every((c) => !('cursor' in c))).toBe(true) - // Run is completed and the FULL transcript is saved — including the - // assistant's terminal text reply, which the engine does not append to the - // middleware message list itself (see `finishedTranscript`). + // Run is completed and the full engine transcript is saved, including the + // assistant's terminal text reply. expect((await persistence.stores.runs!.get('r1'))?.status).toBe('completed') expect(await persistence.stores.messages!.loadThread('t1')).toEqual([ { role: 'user', content: 'hi' }, - { role: 'assistant', content: 'hello' }, + expect.objectContaining({ role: 'assistant', content: 'hello' }), ]) }) @@ -330,6 +329,65 @@ describe('withPersistence (state-only)', () => { ) }) + it('does not duplicate a tool-call turn when the loop stops', async () => { + const persistence = memoryPersistence() + const { adapter } = mockAdapter([ + [ + ev.runStarted(), + { + type: EventType.TEXT_MESSAGE_START, + messageId: 'stream-tool', + role: 'assistant', + timestamp: 1, + }, + ev.text('checking'), + { + type: EventType.TOOL_CALL_START, + toolCallId: 'call_1', + toolCallName: 'search', + toolName: 'search', + parentMessageId: 'stream-tool', + timestamp: 1, + }, + { + type: EventType.TOOL_CALL_ARGS, + toolCallId: 'call_1', + delta: '{}', + timestamp: 1, + }, + { + type: EventType.RUN_FINISHED, + runId: 'r1', + threadId: 't1', + finishReason: 'tool_calls', + timestamp: 1, + }, + ], + ]) + + await collect( + chat({ + adapter, + messages: [{ role: 'user', content: 'search' }], + tools: [serverSearchTool()], + agentLoopStrategy: () => false, + runId: 'r1', + threadId: 't1', + middleware: [withPersistence(persistence)], + }) as AsyncIterable, + ) + + const turns = (await persistence.stores.messages!.loadThread('t1')).filter( + (message) => + message.role === 'assistant' && message.content === 'checking', + ) + expect(turns).toHaveLength(1) + expect(turns[0]).toMatchObject({ + id: 'stream-tool', + toolCalls: [expect.objectContaining({ id: 'call_1' })], + }) + }) + it('stamps createdAt at TEXT_MESSAGE_START, not at iteration start', async () => { vi.useFakeTimers() vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')) @@ -412,7 +470,91 @@ describe('withPersistence (state-only)', () => { } }) - it('does not let structured-output TEXT_MESSAGE_START replace the agent-loop id', async () => { + it('persists native-combined output as structured output', async () => { + const persistence = memoryPersistence() + const raw = '{"name":"Ada"}' + const { adapter } = mockAdapter([ + [ + ev.runStarted(), + { + type: EventType.TEXT_MESSAGE_START, + messageId: 'structured-native', + role: 'assistant', + timestamp: 1, + }, + ev.text(raw), + ev.runFinished(), + ], + ]) + adapter.supportsCombinedToolsAndSchema = () => true + + await collect( + chat({ + adapter, + messages: [{ role: 'user', content: 'extract' }], + runId: 'r1', + threadId: 't1', + stream: true, + outputSchema: { + type: 'object', + properties: { name: { type: 'string' } }, + }, + middleware: [withPersistence(persistence)], + }) as AsyncIterable, + ) + + expect(await persistence.stores.messages!.loadThread('t1')).toEqual([ + { role: 'user', content: 'extract' }, + expect.objectContaining({ + id: 'structured-native', + role: 'assistant', + content: raw, + structuredOutput: { + type: 'structured-output', + status: 'complete', + raw, + data: { name: 'Ada' }, + partial: { name: 'Ada' }, + }, + }), + ]) + }) + + it('persists serialized structured data when raw output is empty', async () => { + const persistence = memoryPersistence() + const raw = '{"name":"Ada"}' + const { adapter } = mockAdapter([]) + adapter.structuredOutput = async () => ({ + data: { name: 'Ada' }, + rawText: '', + }) + + await collect( + chat({ + adapter, + messages: [{ role: 'user', content: 'extract' }], + runId: 'r1', + threadId: 't1', + stream: true, + outputSchema: { + type: 'object', + properties: { name: { type: 'string' } }, + }, + middleware: [withPersistence(persistence)], + }) as AsyncIterable, + ) + + expect(await persistence.stores.messages!.loadThread('t1')).toEqual([ + { role: 'user', content: 'extract' }, + expect.objectContaining({ + role: 'assistant', + content: raw, + structuredOutput: expect.objectContaining({ raw }), + }), + ]) + }) + + it('persists separate-finalization output under its structured message id', async () => { const persistence = memoryPersistence() const { adapter } = mockAdapter([ [ @@ -462,7 +604,7 @@ describe('withPersistence (state-only)', () => { rawText: '{"name":"Ada"}', }) - await collect( + const chunks = await collect( chat({ adapter, messages: [{ role: 'user', content: 'extract' }], @@ -479,10 +621,39 @@ describe('withPersistence (state-only)', () => { ) const thread = await persistence.stores.messages!.loadThread('t1') + const start = chunks.find( + (chunk) => + chunk.type === EventType.CUSTOM && + chunk.name === 'structured-output.start', + ) + const messageId = + start?.type === EventType.CUSTOM && + start.value && + typeof start.value === 'object' && + 'messageId' in start.value && + typeof start.value.messageId === 'string' + ? start.value.messageId + : undefined const terminal = thread.find( + (message) => + message.role === 'assistant' && message.content === '{"name":"Ada"}', + ) + const agentFinal = thread.find( (message) => message.role === 'assistant' && message.content === 'hello', ) - expect(terminal?.id).toBe('agent-final') + expect(messageId).toBeDefined() + expect(agentFinal).toMatchObject({ id: 'agent-final' }) + expect(terminal).toMatchObject({ + id: messageId, + structuredOutput: { + type: 'structured-output', + status: 'complete', + raw: '{"name":"Ada"}', + data: { name: 'Ada' }, + partial: { name: 'Ada' }, + }, + }) + expect(thread.indexOf(agentFinal!)).toBeLessThan(thread.indexOf(terminal!)) }) it('records an interrupt and marks the run interrupted', async () => { diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index 15fb215ba0..fb3fea635e 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -42,7 +42,11 @@ import { } from './tools/approval-schema' import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies' import { isCancelRequestedReason } from './cancel' -import { convertMessagesToModelMessages, generateMessageId } from './messages' +import { + convertMessagesToModelMessages, + generateMessageId, + safeJsonStringify, +} from './messages' import { MiddlewareRunner } from './middleware/compose' import { getRunDetached } from './middleware/run-store' import { publishRunDetachedSignal } from '../../delivery-detach' @@ -84,6 +88,7 @@ import type { SchemaInput, StreamChunk, StructuredOutputCompleteEvent, + StructuredOutputPart, StructuredOutputStream, TextMessageContentEvent, TextOptions, @@ -779,8 +784,13 @@ class TextEngine< private readonly logger: InternalLogger // Structured-output finalization state (populated by runStructuredFinalization) - private structuredOutputResult: { data: unknown; rawText: string } | null = - null + private structuredOutputResult: { + data: unknown + rawText: string + reasoning?: string + } | null = null + private structuredOutputMessageId: string | null = null + private structuredOutputMessageCreatedAt: Date | null = null // Native combined mode: tracks whether we've already emitted the synthetic // `structured-output.start` event before the schema-constrained final-turn // text begins streaming. The event must precede the first @@ -1146,6 +1156,7 @@ class TextEngine< duration: Date.now() - this.streamStartTime, }) } else { + this.addTerminalAssistantMessages() this.terminalHookCalled = true await this.middlewareRunner.runOnFinish(this.middlewareCtx, { finishReason: this.lastFinishReason, @@ -1530,6 +1541,11 @@ class TextEngine< } } + private captureStructuredOutputMessageIdentity(messageId: string): void { + this.structuredOutputMessageId = messageId + this.structuredOutputMessageCreatedAt ??= new Date() + } + private handleToolCallStartEvent(chunk: ToolCallStartEvent): void { if ( typeof chunk.parentMessageId === 'string' && @@ -1987,6 +2003,65 @@ class TextEngine< this.middlewareCtx.messages = this.messages } + private addTerminalAssistantMessages(): void { + const structuredResult = this.structuredOutputResult + const raw = structuredResult + ? structuredResult.rawText || safeJsonStringify(structuredResult.data) + : '' + const structuredOutput: StructuredOutputPart | undefined = structuredResult + ? { + type: 'structured-output', + status: 'complete', + data: structuredResult.data, + partial: structuredResult.data, + raw, + ...(structuredResult.reasoning !== undefined + ? { reasoning: structuredResult.reasoning } + : {}), + } + : undefined + const nativeCombined = this.finalStructuredOutput?.nativeCombined === true + const currentTurnAlreadyRecorded = this.messages.some( + (message) => + message.role === 'assistant' && message.id === this.currentMessageId, + ) + const terminalMessages: Array = [] + + if ( + this.accumulatedContent !== '' && + !nativeCombined && + !currentTurnAlreadyRecorded + ) { + terminalMessages.push({ + role: 'assistant', + content: this.accumulatedContent, + id: this.currentMessageId ?? this.createId('msg'), + createdAt: this.currentMessageCreatedAt ?? new Date(), + }) + } + + if (structuredOutput) { + terminalMessages.push({ + role: 'assistant', + content: raw || null, + id: + (nativeCombined + ? this.currentMessageId + : this.structuredOutputMessageId) ?? this.createId('msg'), + createdAt: + (nativeCombined + ? this.currentMessageCreatedAt + : this.structuredOutputMessageCreatedAt) ?? new Date(), + structuredOutput, + }) + } + + if (terminalMessages.length === 0) return + + this.messages = [...this.messages, ...terminalMessages] + this.middlewareCtx.messages = this.messages + } + /** * Extract client state (approvals and client tool results) from original messages. * This is called in the constructor BEFORE converting to ModelMessage format, @@ -2180,6 +2255,9 @@ class TextEngine< id: `snapshot_${this.runIdOverride ?? this.requestId}_${index}`, role: message.role, ...(content !== undefined ? { content } : {}), + ...(message.structuredOutput + ? { parts: [message.structuredOutput] } + : {}), ...('toolCalls' in message && message.toolCalls ? { toolCalls: message.toolCalls } : {}), @@ -2858,6 +2936,7 @@ class TextEngine< const buildSynthesizedStart = (): StreamChunk => { const idForStart = structuredMessageId ?? generateMessageId() structuredMessageId = idForStart + this.captureStructuredOutputMessageIdentity(idForStart) return { type: EventType.CUSTOM, name: 'structured-output.start', @@ -2898,7 +2977,10 @@ class TextEngine< // synthesized start (when needed) uses the SAME id the deltas carry if (!structuredMessageId) { const extracted = extractMessageId(chunk) - if (extracted) structuredMessageId = extracted + if (extracted) { + structuredMessageId = extracted + this.captureStructuredOutputMessageIdentity(extracted) + } } // Synthesis only matters for the streaming client path — the agentic @@ -2965,7 +3047,13 @@ class TextEngine< const object = this.finalStructuredOutput.normalize ? this.finalStructuredOutput.normalize(parsed.object) : parsed.object - this.structuredOutputResult = { data: object, rawText: parsed.raw } + this.structuredOutputResult = { + data: object, + rawText: parsed.raw, + ...(parsed.reasoning !== undefined + ? { reasoning: parsed.reasoning } + : {}), + } // Rewrite the outbound event so the yielded chunk carries the // normalized object (the original `chunk.value` still holds the // widened one). Preserve every other field — `raw`, `reasoning` — diff --git a/packages/ai/src/activities/chat/messages.ts b/packages/ai/src/activities/chat/messages.ts index 41688f73f6..9c1a435f4e 100644 --- a/packages/ai/src/activities/chat/messages.ts +++ b/packages/ai/src/activities/chat/messages.ts @@ -4,6 +4,7 @@ import type { ContentPart, MessagePart, ModelMessage, + StructuredOutputPart, TextPart, ToolCallPart, UIMessage, @@ -26,9 +27,9 @@ function isContentPart(part: MessagePart): part is ContentPart { ) } -function safeJsonStringify(value: unknown): string { +export function safeJsonStringify(value: unknown): string { try { - return JSON.stringify(value) + return JSON.stringify(value) ?? '' } catch { return '' } @@ -196,6 +197,7 @@ function buildUserOrToolMessage(uiMessage: UIMessage): ModelMessage { // Accumulator for building an assistant segment (content + tool calls) interface AssistantSegment { contentParts: Array + structuredOutput?: StructuredOutputPart toolCalls: Array<{ id: string type: 'function' @@ -255,6 +257,9 @@ function buildAssistantMessages(uiMessage: UIMessage): Array { content, ...(hasToolCalls && { toolCalls: current.toolCalls }), ...(pendingThinking.length > 0 && { thinking: pendingThinking }), + ...(current.structuredOutput && { + structuredOutput: current.structuredOutput, + }), ...(uiMessage.createdAt !== undefined && { createdAt: uiMessage.createdAt, }), @@ -333,6 +338,7 @@ function buildAssistantMessages(uiMessage: UIMessage): Array { : '' if (serialized !== '') { current.contentParts.push({ type: 'text', content: serialized }) + current.structuredOutput = part } } break @@ -445,7 +451,9 @@ export function modelMessageToUIMessage( // Handle tool results (when role is "tool") - only produce tool-result part, // not a text part (the content IS the tool result, not display text) - if (modelMessage.role === 'tool' && modelMessage.toolCallId) { + if (modelMessage.role === 'assistant' && modelMessage.structuredOutput) { + parts.push(modelMessage.structuredOutput) + } else if (modelMessage.role === 'tool' && modelMessage.toolCallId) { parts.push({ type: 'tool-result', toolCallId: modelMessage.toolCallId, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 49e7928a99..b6adb0d352 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -367,6 +367,12 @@ export interface ModelMessage< toolCalls?: Array toolCallId?: string thinking?: Array<{ content: string; signature?: string }> + /** + * Completed structured output represented by this assistant message. + * `content` remains the provider-facing JSON text; this field preserves the + * typed UI part across persistence and message conversion. + */ + structuredOutput?: StructuredOutputPart /** * Optional stable message id. Providers ignore it; it exists so a persisted * transcript can retain the streaming `messageId` and survive the diff --git a/packages/ai/src/utilities/chat-params.ts b/packages/ai/src/utilities/chat-params.ts index 36a37e67da..252c5e314c 100644 --- a/packages/ai/src/utilities/chat-params.ts +++ b/packages/ai/src/utilities/chat-params.ts @@ -22,6 +22,7 @@ const KNOWN_PART_TYPES = new Set([ 'tool-call', 'tool-result', 'thinking', + 'structured-output', ]) function isValidParts(value: unknown): value is Array<{ type: string }> { diff --git a/packages/ai/tests/chat-params.test.ts b/packages/ai/tests/chat-params.test.ts index 0c46ea8b87..9384f2a0e0 100644 --- a/packages/ai/tests/chat-params.test.ts +++ b/packages/ai/tests/chat-params.test.ts @@ -229,6 +229,26 @@ describe('chatParamsFromRequestBody — RunAgentInput validation', () => { expect('parts' in result.messages[0]!).toBe(false) }) + it('preserves structured-output parts', async () => { + const structuredOutput = { + type: 'structured-output', + status: 'complete', + raw: '{"name":"Ada"}', + data: { name: 'Ada' }, + } + const result = await chatParamsFromRequestBody( + withMessages([ + { + id: 'm1', + role: 'assistant', + content: structuredOutput.raw, + parts: [structuredOutput], + }, + ]), + ) + expect(result.messages[0]).toMatchObject({ parts: [structuredOutput] }) + }) + it('rejects a malformed tool declaration', async () => { await expect( chatParamsFromRequestBody({ diff --git a/packages/ai/tests/chat.test.ts b/packages/ai/tests/chat.test.ts index 0e362875c9..858be04d7a 100644 --- a/packages/ai/tests/chat.test.ts +++ b/packages/ai/tests/chat.test.ts @@ -678,6 +678,54 @@ describe('chat()', () => { }) }) + it('preserves structured-output parts on the interrupt MESSAGES_SNAPSHOT', async () => { + const structuredOutput = { + type: 'structured-output' as const, + status: 'complete' as const, + raw: '{"name":"Ada"}', + data: { name: 'Ada' }, + partial: { name: 'Ada' }, + } + const { adapter } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'clientSearch'), + ev.toolArgs('call_1', '{"query":"test"}'), + ev.runFinished('tool_calls'), + ], + ], + }) + + const chunks = await collectChunks( + chat({ + adapter, + messages: [ + { + id: 'structured-1', + role: 'assistant', + content: structuredOutput.raw, + structuredOutput, + }, + { id: 'user-1', role: 'user', content: 'Search' }, + ], + tools: [clientTool('clientSearch')], + }) as AsyncIterable, + ) + + const snapshot = chunks.find( + (chunk) => chunk.type === EventType.MESSAGES_SNAPSHOT, + ) + expect(snapshot).toMatchObject({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: 'assistant', + parts: [structuredOutput], + }), + ]), + }) + }) + it('should yield an interrupt outcome for client tools', async () => { const { adapter } = createMockAdapter({ iterations: [ diff --git a/packages/ai/tests/message-converters.test.ts b/packages/ai/tests/message-converters.test.ts index 57f618b82b..a04306f0d7 100644 --- a/packages/ai/tests/message-converters.test.ts +++ b/packages/ai/tests/message-converters.test.ts @@ -1779,6 +1779,10 @@ describe('Message Converters', () => { expect(result).toHaveLength(1) expect(result[0]!.role).toBe('assistant') expect(result[0]!.content).toBe('{"title":"Cheese Toast","servings":2}') + expect(result[0]!.structuredOutput).toEqual(uiMessage.parts[0]) + expect(modelMessagesToUIMessages(result)[0]!.parts).toEqual( + uiMessage.parts, + ) }) it('falls back to JSON.stringify(data) when complete but raw is empty', () => { @@ -1802,6 +1806,7 @@ describe('Message Converters', () => { const result = uiMessageToModelMessages(uiMessage) expect(result).toHaveLength(1) expect(result[0]!.content).toBe(JSON.stringify(data)) + expect(result[0]!.structuredOutput).toEqual(uiMessage.parts[0]) }) it('skips streaming structured-output parts (no in-flight JSON in history)', () => { diff --git a/testing/e2e/src/routes/api.persistence-durability.ts b/testing/e2e/src/routes/api.persistence-durability.ts index 3f5cdeb553..43333dcd69 100644 --- a/testing/e2e/src/routes/api.persistence-durability.ts +++ b/testing/e2e/src/routes/api.persistence-durability.ts @@ -3,21 +3,29 @@ import { INTERRUPT_BINDING_METADATA_KEY, INTERRUPT_BINDING_VERSION, canonicalInterruptJson, + chat, digestInterruptJson, memoryStream, resumeServerSentEventsResponse, + toolDefinition, toServerSentEventsResponse, } from '@tanstack/ai' -import type { StreamChunk } from '@tanstack/ai' +import { + memoryPersistence, + reconstructChat, + withPersistence, +} from '@tanstack/ai-persistence' +import { z } from 'zod' +import type { AnyTextAdapter, StreamChunk } from '@tanstack/ai' /** * Provider-free harness route for the browser-refresh persistence story. It * mirrors the production wiring of `examples/.../api.persistent-chat.ts` — a * `memoryStream(request)` delivery sink plus a GET resume handler that makes the - * connection resumable — but streams a FIXED AG-UI sequence instead of calling - * an LLM, so the e2e is deterministic with nothing to mock. + * connection resumable — but uses fixed AG-UI sequences and a fixed adapter + * instead of calling an LLM, so the e2e is deterministic with nothing to mock. * - * Three scenarios (`?scenario=`): + * Four scenarios (`?scenario=`): * * - `text` (default) — a run that streams one assistant text message and * finishes cleanly (`outcome: success`). The client persists the transcript @@ -33,13 +41,37 @@ import type { StreamChunk } from '@tanstack/ai' * GET below, which returns a `reconstructChat`-shaped JSON carrying a pending * interrupt. Proves a fresh client (empty `localStorage`) re-prompts the * approval from the server alone — the path that was previously broken. + * - `structured-output` — runs separate structured-output finalization through + * `withPersistence`, then hydrates the completed structured-output part from + * the server through `reconstructChat`. * - * Exempt from the aimock policy: this route streams a fixed AG-UI sequence and - * never reaches an LLM provider's HTTP layer, so there is nothing to mock. + * Exempt from the aimock policy: this route never reaches an LLM provider's HTTP + * layer, so there is nothing to mock. */ const REPLY_TEXT = 'PERSIST_OK the lighthouse still turns.' +const structuredOutputPersistence = memoryPersistence() +const structuredOutputSchema = z.object({ name: z.string() }) +const structuredOutputTool = toolDefinition({ + name: 'lookup_programmer', + description: 'Look up a programmer', + inputSchema: z.object({}), +}).server(() => ({ found: true })) +const structuredOutputAdapter: AnyTextAdapter = { + kind: 'text', + name: 'fixed', + model: 'test-model', + '~types': {}, + chatStream: ({ threadId, runId }: { threadId: string; runId: string }) => + textRun(threadId, runId), + structuredOutput: () => + Promise.resolve({ + data: { name: 'Ada Lovelace' }, + rawText: '{"name":"Ada Lovelace"}', + }), +} as unknown as AnyTextAdapter + const confirmSchema = { type: 'object', properties: { confirmed: { type: 'boolean' } }, @@ -135,11 +167,12 @@ function stringField(body: unknown, key: string): string | undefined { function scenarioOf( request: Request, -): 'text' | 'interrupt' | 'server-interrupt' { +): 'text' | 'interrupt' | 'server-interrupt' | 'structured-output' { try { const value = new URL(request.url).searchParams.get('scenario') if (value === 'interrupt') return 'interrupt' if (value === 'server-interrupt') return 'server-interrupt' + if (value === 'structured-output') return 'structured-output' return 'text' } catch { return 'text' @@ -193,6 +226,20 @@ export const Route = createFileRoute('/api/persistence-durability')({ const body: unknown = await request.json() const threadId = stringField(body, 'threadId') ?? 'persistence-thread' const runId = stringField(body, 'runId') ?? crypto.randomUUID() + if (scenarioOf(request) === 'structured-output') { + const stream = chat({ + adapter: structuredOutputAdapter, + messages: [{ role: 'user', content: 'Name the programmer' }], + tools: [structuredOutputTool], + outputSchema: structuredOutputSchema, + stream: true, + threadId, + runId, + middleware: [withPersistence(structuredOutputPersistence)], + }) + for await (const _ of stream) void _ + return Response.json({ runId, threadId }) + } const stream = scenarioOf(request) === 'interrupt' ? interruptRun(threadId, runId) @@ -212,6 +259,11 @@ export const Route = createFileRoute('/api/persistence-durability')({ // `reconstructChat`-shaped JSON; the `server-interrupt` scenario carries // a pending approval so a fresh client re-prompts it from the server. GET: ({ request }) => { + if (scenarioOf(request) === 'structured-output') { + return reconstructChat(structuredOutputPersistence, request, { + authorize: (threadId) => threadId.length > 0, + }) + } const durability = memoryStream(request) if (durability.resumeFrom() !== null) { return resumeServerSentEventsResponse({ adapter: durability }) diff --git a/testing/e2e/tests/persistence-durability.spec.ts b/testing/e2e/tests/persistence-durability.spec.ts index 550debd0e3..e4080e2ff1 100644 --- a/testing/e2e/tests/persistence-durability.spec.ts +++ b/testing/e2e/tests/persistence-durability.spec.ts @@ -140,3 +140,48 @@ test.describe('persistence durability (browser refresh)', () => { expect(stored).toBeNull() }) }) + +test.describe('structured output persistence', () => { + test('restores a completed structured-output part from server persistence', async ({ + request, + }) => { + const threadId = `structured-output-${crypto.randomUUID()}` + const runId = crypto.randomUUID() + const run = await request.post( + '/api/persistence-durability?scenario=structured-output', + { data: { threadId, runId } }, + ) + expect(run.ok()).toBe(true) + + const hydration = await request.get( + `/api/persistence-durability?scenario=structured-output&threadId=${threadId}`, + ) + expect(hydration.ok()).toBe(true) + const body = (await hydration.json()) as { + messages: Array<{ + role: string + parts: Array> + }> + } + const assistants = body.messages.filter( + (message) => message.role === 'assistant', + ) + + expect(assistants).toHaveLength(2) + expect(assistants[0]?.parts).toEqual([ + { + type: 'text', + content: 'PERSIST_OK the lighthouse still turns.', + }, + ]) + expect(assistants[1]?.parts).toEqual([ + { + type: 'structured-output', + status: 'complete', + data: { name: 'Ada Lovelace' }, + partial: { name: 'Ada Lovelace' }, + raw: '{"name":"Ada Lovelace"}', + }, + ]) + }) +}) From 8402b4d3227a9738caaabfd7f92c27fa47c6a311 Mon Sep 17 00:00:00 2001 From: kolaworld Date: Sun, 16 Aug 2026 23:14:05 -0400 Subject: [PATCH 2/3] docs: document structured output lifecycle Refs #1072 --- docs/advanced/middleware.md | 36 ++++--- docs/api/ai.md | 23 +++- docs/chat/structured-outputs.md | 2 +- docs/comparison/vercel-ai-sdk.md | 2 +- docs/persistence/chat-persistence.md | 8 +- docs/persistence/internals.md | 14 ++- docs/reference/interfaces/ModelMessage.md | 58 ++++++++-- docs/structured-outputs/multi-turn.md | 28 ++--- docs/structured-outputs/with-tools.md | 6 +- .../skills/ai-persistence/server/SKILL.md | 26 +++-- .../ai/skills/ai-core/middleware/SKILL.md | 97 +++++++++-------- .../ai-core/structured-outputs/SKILL.md | 101 +++++++++--------- 12 files changed, 252 insertions(+), 149 deletions(-) diff --git a/docs/advanced/middleware.md b/docs/advanced/middleware.md index 80fe5fed46..861d3654dd 100644 --- a/docs/advanced/middleware.md +++ b/docs/advanced/middleware.md @@ -71,9 +71,11 @@ graph TD K --> L{Continue loop?} L -->|Yes| D L -->|No| H - H --> SO{outputSchema?} - SO -->|No| M{Outcome} - SO -->|Yes| SOC[onStructuredOutputConfig] + H --> SO{"Structured output path?"} + SO -->|None| M{Outcome} + SO -->|"Native combined"| SOH["Post-loop structured-output harvest (onChunk)"] + SOH --> M + SO -->|"Separate finalization"| SOC[onStructuredOutputConfig] SOC --> SOM["onConfig (phase: structuredOutput)"] SOM --> SOS["Structured-output finalization (onChunk, onUsage)"] SOS --> M @@ -86,6 +88,7 @@ graph TD style SOC fill:#e1f5ff style SOM fill:#e1f5ff style SOS fill:#e1f5ff + style SOH fill:#e1f5ff style N fill:#e1ffe1 style O fill:#fff4e1 style P fill:#ffe1e1 @@ -102,13 +105,13 @@ The context's `phase` field tracks where you are in the lifecycle: | `modelStream` | While adapter streams chunks | `onChunk`, `onUsage` | | `beforeTools` | Before tool execution | `onBeforeToolCall` | | `afterTools` | After tool execution | `onAfterToolCall` | -| `structuredOutput` | During the final structured-output adapter call (when `outputSchema` is set **and** the adapter does not declare `supportsCombinedToolsAndSchema()`). Chunks from `adapter.structuredOutputStream` (or the synthesized non-streaming fallback) flow through `onChunk` with this phase, and `onUsage` fires for the final call's tokens. **Does not fire** for adapters that natively combine tools + schema in one streaming call (modern OpenAI Chat Completions, OpenAI Responses, Claude 4.5+, Gemini 3.x, Grok 4.x family — see issue #605); on that path middleware observes the run through `beforeModel` / `modelStream` as usual. | `onStructuredOutputConfig`, `onConfig`, `onChunk`, `onUsage` | +| `structuredOutput` | During the final structured-output adapter call (when `outputSchema` is set **and** `supportsCombinedToolsAndSchema()` does not return `true` for the current model/options). Chunks from `adapter.structuredOutputStream` (or the synthesized non-streaming fallback) flow through `onChunk` with this phase, and `onUsage` fires for the final call's tokens. **Does not fire** for adapters that natively combine tools + schema in one streaming call (modern OpenAI Chat Completions, OpenAI Responses, Claude 4.5+, Gemini 3.x, Grok 4.x family — see issue #605); on that path middleware observes the run through `beforeModel` / `modelStream` as usual. | `onStructuredOutputConfig`, `onConfig`, `onChunk`, `onUsage` | ## Hooks Reference ### onConfig -Called once during `init` (startup) and once per iteration during `beforeModel` (before each model call). When `chat()` was invoked with `outputSchema`, `onConfig` additionally re-fires at the structured-output boundary with `ctx.phase === 'structuredOutput'`, receiving the post-`onStructuredOutputConfig` view of the config — so a single-iteration run with `outputSchema` fires `onConfig` three times (`init` + `beforeModel` + `structuredOutput`). Use it to transform the configuration that the model receives. +Called once during `init` (startup) and once per iteration during `beforeModel` (before each model call). On the separate-finalization path, `onConfig` additionally re-fires at the structured-output boundary with `ctx.phase === 'structuredOutput'`, receiving the post-`onStructuredOutputConfig` view of the config. A single-iteration separate-finalization run therefore fires `onConfig` three times (`init` + `beforeModel` + `structuredOutput`). Native-combined output does not add this third call. Use `onConfig` to transform the configuration that the model receives. Return a **partial** config object with only the fields you want to change — they are shallow-merged with the current config automatically. No need to spread the existing config. @@ -164,9 +167,9 @@ When multiple middleware define `onConfig`, the config is **piped** through them ### onStructuredOutputConfig -Called once at the start of the final structured-output adapter call — only when `chat()` was invoked with `outputSchema` **and** the adapter takes the legacy finalization path (i.e. does not declare `supportsCombinedToolsAndSchema()`). Pipes through middleware in order, like `onConfig`, but with access to the **JSON Schema** being sent to the provider. Use this hook when you need to transform the schema (e.g., inject `$defs`, strip vendor-incompatible keywords) or apply structured-output-specific behavior (e.g., suppress system prompts on the final call). +Called once at the start of the final structured-output adapter call — only when `chat()` was invoked with `outputSchema` **and** `supportsCombinedToolsAndSchema()` does not return `true` for the current model/options. Pipes through middleware in order, like `onConfig`, but with access to the **JSON Schema** being sent to the provider. Use this hook when you need to transform the schema (e.g., inject `$defs`, strip vendor-incompatible keywords) or apply structured-output-specific behavior (e.g., suppress system prompts on the final call). -> Native-combined adapters (modern OpenAI, Claude 4.5+, Gemini 3.x, Grok 4.x — see issue #605) skip the separate finalization call and never invoke this hook. If you need to mutate the schema for a native-combined adapter, do it in `onConfig` (the schema is on `config.modelOptions` / the request — adapter-specific). +> Native-combined adapters (modern OpenAI, Claude 4.5+, Gemini 3.x, Grok 4.x — see issue #605) skip the separate finalization call and never invoke this hook. The engine passes the converted schema directly to `chatStream` after `onConfig` runs, so middleware cannot transform the native-combined schema. Return a **partial** `StructuredOutputMiddlewareConfig` with only the fields you want to change — they are shallow-merged with the current config. Return `void` to pass through. @@ -274,7 +277,7 @@ There is **no separate `onStructuredOutputChunk` hook** — and you don't need o How you distinguish them depends on which finalization path the adapter takes: -- **Separate-finalization adapters** (the legacy path — adapters that don't declare `supportsCombinedToolsAndSchema()`): `ctx.phase === 'structuredOutput'` during the finalization call. Discriminate on the phase. +- **Separate-finalization adapters** (`supportsCombinedToolsAndSchema()` does not return `true` for the current model/options): `ctx.phase === 'structuredOutput'` during the finalization call. Discriminate on the phase. - **Native-combined adapters** (modern OpenAI Chat Completions / Responses, Claude 4.5+, Gemini 3.x, Grok 4.x — see issue #605): the schema-constrained JSON is produced on the model's natural final turn, so **`ctx.phase` stays `'modelStream'`** — the `'structuredOutput'` phase never fires. Discriminate on the CUSTOM event name (`structured-output.start` / `structured-output.complete`) instead. ```typescript ignore @@ -297,7 +300,7 @@ const redactStructuredOutput: ChatMiddleware = { }; } - // Both paths: the validated object arrives as a CUSTOM + // Both paths: the completed typed payload arrives as a CUSTOM // `structured-output.complete` event. On the native-combined path this is // your only signal (ctx.phase never flips to 'structuredOutput'), so key // off the event name, not the phase. `chunk.value` carries { object, raw }. @@ -448,15 +451,20 @@ Exactly **one** terminal hook fires per `chat()` invocation. They are mutually e | `onAbort` | Run was aborted (via `ctx.abort()`, an external `AbortSignal`, or a `{ type: 'abort' }` decision from `onBeforeToolCall`) | | `onError` | An unhandled error occurred | -> **Structured-output lifecycle ordering:** When `chat()` is invoked with `outputSchema`, `onFinish` fires **after** the structured-output finalization call completes — not at the end of the agent loop. `onIteration` does **not** fire for the finalization step; it only fires for agent-loop iterations. +> **Separate-finalization path:** Adapters without native-combined support make a separate structured-output provider call after the agent loop. > -> **`onFinish` info fields and structured-output runs:** the `info` object reflects the **agent loop's** terminal state — finalization state is intentionally segregated to keep agent-loop semantics clean. -> -> - `info.content` — the agent loop's accumulated text. Finalization JSON deltas are **not** included here. The structured-output result is delivered via the `structured-output.complete` CUSTOM event, which middleware observes via `onChunk` (with `ctx.phase === 'structuredOutput'`). +> - `onStructuredOutputConfig` fires before the separate provider call, and `ctx.phase` is `'structuredOutput'` for its chunks. +> - `onIteration` does **not** fire for finalization; it only fires for agent-loop iterations. +> - `onFinish` fires after finalization completes. Its `info` object reflects the **agent loop's** terminal state. +> - `info.content` — the agent loop's accumulated text. Separate-finalization JSON deltas are **not** included. Middleware can observe the completed result through the `structured-output.complete` CUSTOM event in `onChunk`. > - `info.usage` — the agent loop's last `RUN_FINISHED.usage`. For a tools-less structured-output run (no agent-loop iteration produces `RUN_FINISHED`), this is `undefined`. To capture finalization tokens, use `onUsage` — that hook fires for **every** `RUN_FINISHED` carrying usage, including the finalization call. > - `info.finishReason` — the agent loop's last `finishReason`. `null` when no agent-loop iteration produced `RUN_FINISHED` (e.g. a tools-less structured-output run). > - `info.duration` — wall-clock duration of the entire `chat()` invocation, including finalization. > +> **Native-combined output:** Adapters with native-combined support produce the schema-constrained JSON in the regular agent-loop stream. `onStructuredOutputConfig` does not fire, `ctx.phase` remains `'modelStream'`, and `onIteration` fires for the iteration that produces the JSON. The JSON is agent-loop text, so `info.content` includes it. Middleware observes the `structured-output.complete` event in `onChunk` during the same phase. +> +> On successful completion in either path, `onFinish` receives the complete canonical transcript in `ctx.messages`. Native-combined output keeps the structured result on its terminal assistant message. The separate-finalization path can preserve the agent loop's plain-text assistant message followed by a distinct structured-output assistant message. This transcript is separate from the path-specific fields on `info`. +> > To aggregate usage across the whole run, accumulate from `onUsage` callbacks rather than relying on `info.usage`. ```typescript @@ -486,7 +494,7 @@ The `info` object for `onFinish` (`FinishInfo`): |-------|------|-------------| | `finishReason` | `string \| null` | The agent loop's last `finishReason`. `null` when no agent-loop iteration produced `RUN_FINISHED` (e.g. a tools-less `chat({ outputSchema })` run). | | `duration` | `number` | Total run duration in milliseconds, including any structured-output finalization. | -| `content` | `string` | The agent loop's accumulated text content. Does **not** include finalization JSON deltas — for that, observe the `structured-output.complete` CUSTOM event via `onChunk`. | +| `content` | `string` | The agent loop's accumulated text content. Includes native-combined structured JSON; excludes separate-finalization JSON. Observe the completed result through the `structured-output.complete` CUSTOM event via `onChunk`. | | `usage` | `{ promptTokens; completionTokens; totalTokens } \| undefined` | **Optional.** The agent loop's last `RUN_FINISHED.usage`. **Does not include finalization tokens** — use `onUsage` to observe those. Always guard with `if (info.usage)` or `info.usage?.`. | ## Context Object diff --git a/docs/api/ai.md b/docs/api/ai.md index c70aeb861b..3cd8cdccb9 100644 --- a/docs/api/ai.md +++ b/docs/api/ai.md @@ -341,10 +341,27 @@ An `AgentLoopStrategy` function. ### `ModelMessage` ```typescript -interface ModelMessage { - role: "user" | "assistant" | "system" | "tool"; - content: string; +import type { + ContentPart, + StructuredOutputPart, + ToolCall, +} from "@tanstack/ai"; + +interface ModelMessage< + TContent extends string | null | ContentPart[] = + | string + | null + | ContentPart[], +> { + role: "user" | "assistant" | "tool"; + content: TContent; + name?: string; + toolCalls?: ToolCall[]; toolCallId?: string; + thinking?: Array<{ content: string; signature?: string }>; + structuredOutput?: StructuredOutputPart; + id?: string; + createdAt?: Date; } ``` diff --git a/docs/chat/structured-outputs.md b/docs/chat/structured-outputs.md index 425dc56abe..4d5ed82533 100644 --- a/docs/chat/structured-outputs.md +++ b/docs/chat/structured-outputs.md @@ -15,7 +15,7 @@ The structured-outputs guide has moved to its own top-level section, split by wh - **[Overview](../structured-outputs/overview)** — what structured output is, schema library options, provider support, and "which page do I read?" - **[One-Shot Extraction](../structured-outputs/one-shot)** — single prompt in, single typed object out. Use this when you don't need streaming or chat history. - **[Streaming UIs](../structured-outputs/streaming)** — `useChat({ outputSchema })` with `partial` and `final` populating a UI field by field. -- **[Multi-Turn Chat](../structured-outputs/multi-turn)** — every assistant turn carries its own typed `StructuredOutputPart`, history stays renderable, and `messages[i].parts.find(p => p.type === "structured-output").data` is typed by your schema. +- **[Multi-Turn Chat](../structured-outputs/multi-turn)** — each successfully completed structured-output run adds a typed response to message history, and `messages[i].parts.find(p => p.type === "structured-output").data` is typed by your schema. - **[With Tools](../structured-outputs/with-tools)** — combining `outputSchema` with the agent loop, including pause/resume for server-tool approvals and client-tool invocations. > **Note:** This URL is kept for backward compatibility. New content lives under `/structured-outputs/*` — update existing bookmarks when you can. diff --git a/docs/comparison/vercel-ai-sdk.md b/docs/comparison/vercel-ai-sdk.md index bc57fcdd68..ef110b9a14 100644 --- a/docs/comparison/vercel-ai-sdk.md +++ b/docs/comparison/vercel-ai-sdk.md @@ -742,7 +742,7 @@ Vercel AI SDK's UI layer has three hooks: `useChat`, `useCompletion`, and `useOb ### Multi-Turn Structured Output -Structured output in TanStack AI is part of the conversation, not a separate call. Pass `outputSchema` to `useChat` and every assistant turn carries its own typed `StructuredOutputPart` - streamed as a `partial`, validated as a `final`, preserved in message history, with the schema generic threading all the way down to `messages[i].parts[j].data`. +TanStack AI preserves structured output in conversation history instead of leaving it only on a call result. Providers may produce it in the agent loop or through separate finalization; both paths create a typed `StructuredOutputPart`, streamed as a `partial` and completed as a `final`, with the schema generic threading all the way down to `messages[i].parts[j].data`. Vercel AI SDK's structured output (`generateObject` / `streamObject` / `Output`) is per-call: the typed object lives on the call result, the message-part union has no structured-output type, and combining `useChat` with typed structured output means manually parsing model text into custom data parts. diff --git a/docs/persistence/chat-persistence.md b/docs/persistence/chat-persistence.md index e366382763..8ed319aa20 100644 --- a/docs/persistence/chat-persistence.md +++ b/docs/persistence/chat-persistence.md @@ -99,7 +99,7 @@ generation hooks. [How persistence works](./internals) has the rest. | --- | --- | --- | | **Start of a run** (`onStart`) | Pending turn (just-submitted user message + prior history) so a reload mid-generation still shows the question | Yes. Failure does not abort the run; finish is authoritative | | **Interrupt boundary** | New interrupt records, run status `interrupted`, and a thread snapshot of current messages | No. Store failures propagate | -| **Finish** (`onFinish`) | Complete transcript (including the terminal assistant reply with its stream `messageId` for in-place reload identity), run status `completed`, and commit of consumed resumes | No. The transcript is saved **before** the run is marked completed | +| **Finish** (`onFinish`) | Complete transcript (including completed assistant messages, their stream identities, and any completed structured-output part), run status `completed`, and commit of consumed resumes | No. The transcript is saved **before** the run is marked completed | | **Optionally while streaming** | Throttled partial assistant text when `snapshotStreaming: true` | Yes | ```ts group=chat-persistence @@ -112,6 +112,12 @@ Streaming snapshots default off (finish is the authoritative save); enable them to trade extra writes for partial-output durability. Tune the interval with `snapshotIntervalMs` (default `1000`). +The chat engine completes the canonical transcript before `onFinish` runs, and +`withPersistence` saves that transcript directly. Native-combined output keeps +the structured result on its terminal assistant message. The +separate-finalization path can preserve a plain-text assistant message followed +by a structured-output assistant message. + On **error**, the run is marked `failed`. On **abort**, the run is marked `aborted` with a `finishedAt`; `interrupted` is written only at an interrupt boundary, and it is not terminal. Resumes accepted in `onConfig` are **not** diff --git a/docs/persistence/internals.md b/docs/persistence/internals.md index 67daea2f38..d91405cec0 100644 --- a/docs/persistence/internals.md +++ b/docs/persistence/internals.md @@ -176,10 +176,16 @@ server event state, not the client's rendered messages. 3. `onChunk` reacts only to a `RUN_FINISHED` interrupt outcome by committing the accepted resumes, storing the new interrupts, marking the run interrupted, and saving messages. -4. `onFinish` and `onError` terminalize the run record. So does `onAbort`, with - one exception: on a run another middleware has declared detachable, a plain - disconnect (no cancel recorded in either band) writes nothing and leaves the - record `'running'` for a later takeover. See +4. Before `onFinish`, the chat engine appends the completed terminal assistant + messages to `ctx.messages`. Native-combined output keeps the structured + result on its terminal assistant message. The separate-finalization path can + append the agent loop's plain-text message followed by the structured-output + message. +5. `onFinish` saves that canonical transcript before marking the run completed. + `onError` terminalizes the run record without replacing the transcript. So + does `onAbort`, with one exception: on a run another middleware has declared + detachable, a plain disconnect (no cancel recorded in either band) writes + nothing and leaves the record `'running'` for a later takeover. See [Takeover & Detached Runs](../sandbox/takeover#detach-vs-cancel). Accepted resumes are committed (interrupts marked resolved/cancelled) only once diff --git a/docs/reference/interfaces/ModelMessage.md b/docs/reference/interfaces/ModelMessage.md index 6d2de2dce4..2bbc1fd1a2 100644 --- a/docs/reference/interfaces/ModelMessage.md +++ b/docs/reference/interfaces/ModelMessage.md @@ -5,7 +5,7 @@ title: ModelMessage # Interface: ModelMessage\ -Defined in: [packages/ai/src/types.ts:347](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L347) +Defined in: [packages/ai/src/types.ts:358](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L358) ## Type Parameters @@ -21,7 +21,37 @@ Defined in: [packages/ai/src/types.ts:347](https://github.com/TanStack/ai/blob/m content: TContent; ``` -Defined in: [packages/ai/src/types.ts:354](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L354) +Defined in: [packages/ai/src/types.ts:365](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L365) + +*** + +### createdAt? + +```ts +optional createdAt: Date; +``` + +Defined in: [packages/ai/src/types.ts:389](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L389) + +Optional message creation timestamp. When present, message converters +preserve it across persist → hydrate round-trips. + +*** + +### id? + +```ts +optional id: string; +``` + +Defined in: [packages/ai/src/types.ts:384](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L384) + +Optional stable message id. Providers ignore it; it exists so a persisted +transcript can retain the streaming `messageId` and survive the +persist → hydrate round-trip. When present, `modelMessagesToUIMessages` +reuses it instead of generating a fresh id, so a hydrated message keeps the +same identity as its live stream — which is what lets a mid-stream reload +resume the SAME message bubble in place (see `@tanstack/ai-persistence`). *** @@ -31,7 +61,7 @@ Defined in: [packages/ai/src/types.ts:354](https://github.com/TanStack/ai/blob/m optional name: string; ``` -Defined in: [packages/ai/src/types.ts:355](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L355) +Defined in: [packages/ai/src/types.ts:366](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L366) *** @@ -41,7 +71,21 @@ Defined in: [packages/ai/src/types.ts:355](https://github.com/TanStack/ai/blob/m role: "user" | "assistant" | "tool"; ``` -Defined in: [packages/ai/src/types.ts:353](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L353) +Defined in: [packages/ai/src/types.ts:364](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L364) + +*** + +### structuredOutput? + +```ts +optional structuredOutput: StructuredOutputPart; +``` + +Defined in: [packages/ai/src/types.ts:375](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L375) + +Completed structured output represented by this assistant message. +`content` remains the provider-facing JSON text; this field preserves the +typed UI part across persistence and message conversion. *** @@ -51,7 +95,7 @@ Defined in: [packages/ai/src/types.ts:353](https://github.com/TanStack/ai/blob/m optional thinking: object[]; ``` -Defined in: [packages/ai/src/types.ts:358](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L358) +Defined in: [packages/ai/src/types.ts:369](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L369) #### content @@ -73,7 +117,7 @@ optional signature: string; optional toolCallId: string; ``` -Defined in: [packages/ai/src/types.ts:357](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L357) +Defined in: [packages/ai/src/types.ts:368](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L368) *** @@ -83,4 +127,4 @@ Defined in: [packages/ai/src/types.ts:357](https://github.com/TanStack/ai/blob/m optional toolCalls: ToolCall[]; ``` -Defined in: [packages/ai/src/types.ts:356](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L356) +Defined in: [packages/ai/src/types.ts:367](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L367) diff --git a/docs/structured-outputs/multi-turn.md b/docs/structured-outputs/multi-turn.md index d681d4ca90..d5e6321798 100644 --- a/docs/structured-outputs/multi-turn.md +++ b/docs/structured-outputs/multi-turn.md @@ -2,7 +2,7 @@ title: Multi-Turn Structured Chat id: structured-outputs-multi-turn order: 4 -description: "Build a chat where users iterate on a typed object across multiple turns — every assistant turn produces its own validated object, history stays renderable, and messages[i].parts.find(p => p.type === 'structured-output') is typed by your schema." +description: "Build a chat where users iterate on a typed object across multiple turns — each successfully completed structured-output run adds a typed response to message history, and messages[i].parts.find(p => p.type === 'structured-output') is typed by your schema." keywords: - tanstack ai - structured outputs @@ -16,9 +16,9 @@ keywords: You want users to iterate on a structured object across turns. "Give me a pasta recipe under $15" → recipe card lands. "Now make it vegan" → a new recipe card lands; the old one stays visible in history. "Add a salad and make it gluten-free" → third card lands; the first two are still there to compare against. -This is the shape of a structured-output chat: every assistant turn produces its own validated object, every old turn stays renderable, and the type of `messages[i].parts.find(p => p.type === 'structured-output').data` is your schema's inferred type — not `unknown`. +This is the shape of a structured-output chat: each successfully completed structured-output run adds a structured-output assistant message, every old response stays renderable, and the type of `messages[i].parts.find(p => p.type === 'structured-output').data` is your schema's inferred type — not `unknown`. -By the end of this guide you'll have a chat UI that walks `messages` directly, renders one typed card per assistant turn, and keeps history across `sendMessage()` calls. +By the end of this guide you'll have a chat UI that walks `messages` directly, renders one typed card per successfully completed structured-output run, and keeps history across `sendMessage()` calls. > **Note:** If you only need a single round-trip (one prompt → one object), use [One-Shot Extraction](./one-shot). If you have one turn that streams progressively but no history, use [Streaming UIs](./streaming) — its `partial` / `final` sugar is the right surface. This page is for the case where history matters. @@ -29,7 +29,7 @@ By the end of this guide you'll have a chat UI that walks `messages` directly, r ## How it lands on the message -When `useChat({ outputSchema })` receives the server's `structured-output.complete` for an assistant turn, the runtime attaches a typed `structured-output` `MessagePart` to that assistant's `UIMessage`. The part looks like this: +When `useChat({ outputSchema })` receives the server's `structured-output.complete` event, the runtime attaches a typed `structured-output` `MessagePart` to the assistant `UIMessage` carrying the structured response. The part looks like this: ```typescript import type { DeepPartial } from "@tanstack/ai"; @@ -39,7 +39,7 @@ type StructuredOutputPart = { status: "streaming" | "complete" | "error"; /** Progressive parse of `raw` — populated while streaming and after complete. */ partial?: DeepPartial; - /** Validated final object — set when `status === "complete"`. */ + /** Completed typed object — set when `status === "complete"`. */ data?: TData; /** Accumulating JSON text. Round-trip source of truth for the next turn. */ raw: string; @@ -54,7 +54,9 @@ type StructuredOutputPart = { > **Note:** The core `@tanstack/ai` package defines `MessagePart` and `UIMessage` with a single generic (no `TTools`) — the tools generic lives in `@tanstack/ai-client` and the framework hook packages. If you're building UI, you almost always want to import from your framework package (`@tanstack/ai-react` / `-vue` / `-solid` / `-svelte`) or from `@tanstack/ai-client` — those carry both generics. The core types come into play only if you're working at the adapter layer below the client. -When the next turn streams in, it lands on a **new** assistant message with its **own** structured-output part. The old turn stays untouched. That's what makes "show history" trivial. +Each new structured response lands on a **new** assistant message with its **own** structured-output part. Earlier responses stay untouched. That's what makes "show history" trivial. + +A run may also contain assistant messages without a structured-output part. Native-combined output keeps the structured JSON and its part on one assistant message. On the separate-finalization path, a plain-text assistant message can precede the structured-output assistant message. Find the part by type instead of assuming that every assistant message contains one. ## Server endpoint @@ -95,7 +97,7 @@ export async function POST(request: Request) { } ``` -Behind the scenes, when the client sends turn N, the previous N-1 assistant turns are serialized back into the request body — each assistant's `structured-output` part is serialized as its `raw` JSON content so the model sees its own prior responses verbatim. Multi-turn coherence is preserved without you doing anything special. +Behind the scenes, when the client sends turn N, prior `UIMessage.parts` remain intact in the request. The client also mirrors each completed structured-output part's `raw` JSON into assistant content. Server conversion preserves the structured-output marker while adapters consume the provider-facing content. Multi-turn coherence is preserved without you doing anything special. ## Client: walk the messages @@ -183,15 +185,15 @@ function RecipeCard({ part }: { part: RecipePart }) { } ``` -That's it. The render loop above produces a card per assistant turn. When the user sends a follow-up, a new assistant message arrives with its own structured-output part — the old card stays exactly as it was. +That's it. The render loop above produces a card per structured response. When the user sends a follow-up, a new structured-output assistant message arrives — the old card stays exactly as it was. > **See the full pattern in code:** the example app at `examples/ts-react-chat/src/routes/generations.structured-chat.tsx` ships a polished version of this exact recipe-builder UI — empty state, streaming placeholder, cuisine-aware hero banner, ingredients grid, numbered method, chef's tips block. Use it as a reference for visual layout; the data wiring matches what's shown above. ## Streaming the latest turn -Every assistant `structured-output` part transitions through `streaming` → `complete` (or `streaming` → `error`). The `data` field only populates on `complete` — while the model is still emitting JSON, only `partial` and `raw` are filled in. Render against `part.data ?? part.partial` and the UI fills in field by field as bytes arrive, then snaps to the validated object on the terminal event. +Every `structured-output` part transitions through `streaming` → `complete` (or `streaming` → `error`). The `data` field only populates on `complete` — while the model is still emitting JSON, only `partial` and `raw` are filled in. Render against `part.data ?? part.partial` and the UI fills in field by field as bytes arrive, then snaps to the completed typed object on the terminal event. -The hook-level `partial` and `final` are still available. They're derived from the latest assistant message's structured-output part — the same part the render loop above already finds. `partial` returns `{}` between `sendMessage()` and the first chunk (because no assistant message exists yet to derive from), and `final` returns `null` until the latest turn lands its `complete` event. Use them for sticky-summary widgets ("Latest recipe title: …"); use the `messages` walk for the full history view. +The hook-level `partial` and `final` are still available. They're derived from the most recent structured-output part after the latest user message — the same part the render loop above already finds. `partial` returns `{}` between `sendMessage()` and the first chunk (because no structured-output part exists yet to derive from), and `final` returns `null` until the latest turn lands its `complete` event. Use them for sticky-summary widgets ("Latest recipe title: …"); use the `messages` walk for the full history view. ## Type-safe access without a named alias @@ -220,8 +222,8 @@ Both forms produce the same typed result. Pick whichever you find more readable. ## What about the round-trip? -When turn N+1 fires, the client sends the previous N turns back to the server. Each assistant message's `structured-output` part is serialized as `{ role: "assistant", content: raw }` — the model receives its own prior recipe as the assistant content of the prior turn. Streaming or errored parts are dropped from the round-trip (you don't want to feed an incomplete JSON fragment back to the LLM). +When turn N+1 fires, completed structured-output parts remain on their UI messages and are mirrored into provider-facing assistant content using `part.raw`. Streaming and errored parts remain UI state but are excluded from model input. -If `raw` is empty (rare — a terminal-only complete event arrived before any deltas, then the runtime couldn't serialize the `data` either), the entire turn is dropped from history rather than shipping an empty assistant turn. This is intentional fail-quiet — better to drop one turn than to confuse the model with a blank assistant message. +If `raw` is empty (rare — a terminal-only complete event arrived before any deltas, then the runtime couldn't serialize the `data` either), the part remains in UI state but is excluded from provider-facing content. This avoids sending a blank assistant turn to the model. -> **Combining with tools?** Multi-turn structured chats compose with the agent loop the same way single-turn streams do — each turn runs tools first, then snaps the structured-output part. See [With Tools](./with-tools) for tool-approval gating and client-tool invocations inside a structured-chat run. +> **Combining with tools?** Multi-turn structured chats compose with the agent loop the same way single-turn streams do — each turn runs tools first, then produces a structured-output message. See [With Tools](./with-tools) for tool-approval gating and client-tool invocations inside a structured-chat run. diff --git a/docs/structured-outputs/with-tools.md b/docs/structured-outputs/with-tools.md index 69e43d7be4..73f2bf2aac 100644 --- a/docs/structured-outputs/with-tools.md +++ b/docs/structured-outputs/with-tools.md @@ -64,11 +64,13 @@ Pass `stream: true` and the wire format changes — the client now sees tool-cal 2. (Agent loop) `TOOL_CALL_START` → `TOOL_CALL_ARGS` → `TOOL_CALL_END` → `TOOL_CALL_RESULT`, possibly repeating for multiple tool calls or iterations 3. `structured-output.start` (once the model begins emitting the JSON response) 4. `TEXT_MESSAGE_CONTENT` deltas (the JSON itself) -5. `structured-output.complete` (validated payload) +5. `structured-output.complete` (completed payload) 6. `RUN_FINISHED` `useChat`'s `partial` stays `{}` and `final` stays `null` while step 2 is running — the structured stream hasn't started yet. Once step 3 fires, `partial` begins filling in; on step 5, `final` snaps. +On the separate-finalization path, the agent loop may also complete a plain-text assistant message before step 3. That message and the structured-output assistant message remain separate. Native-combined output keeps the structured JSON and its part on one assistant message. + The tool-call parts land on the assistant message exactly as they would in a normal streaming chat. Render them however you'd render tool calls outside a structured-output run. ## Server tools that need approval @@ -164,6 +166,6 @@ See [Client Tools](../tools/client-tools) for the full pattern (typed inputs / o ## Multi-turn + tools + structured output -Composes naturally. Every turn runs the agent loop (with any tool gates), then snaps a structured-output part on that turn's assistant message. The next turn sees the prior recipe (or recommendation, or report) as assistant content and can iterate on it. +Composes naturally. Every turn runs the agent loop (with any tool gates), then produces a structured-output assistant message when the run completes successfully. The next turn sees the prior recipe (or recommendation, or report) as assistant content and can iterate on it. The separate-finalization path can also retain the agent loop's plain-text assistant message before that structured response. The only thing to be careful of: between `sendMessage()` and the first structured-output event, the latest turn has no `structured-output` part yet — your render loop's `m.parts.find(p => p.type === "structured-output")` returns `undefined`. Render a "streaming…" placeholder when `isLoading && messages[last]?.role === "user"` to cover that gap. See [Multi-Turn Chat](./multi-turn) for the full pattern. diff --git a/packages/ai-persistence/skills/ai-persistence/server/SKILL.md b/packages/ai-persistence/skills/ai-persistence/server/SKILL.md index 904c3320fc..c9b2933b7c 100644 --- a/packages/ai-persistence/skills/ai-persistence/server/SKILL.md +++ b/packages/ai-persistence/skills/ai-persistence/server/SKILL.md @@ -75,20 +75,26 @@ it because `stores.messages` is possibly `undefined`. ## Authoritative-history contract -- **Non-empty `messages`** → finish **overwrites** the stored thread with that - array. Post the **complete** transcript, never a delta. +- **Non-empty `messages`** seed the authoritative history. On finish, + persistence **overwrites** the stored thread with the engine's completed + canonical transcript. Post the complete history, never a delta. - **Empty `messages`** → middleware **loads** the stored thread and continues. ## When state is written -| Moment | Writes | Best-effort? | -| ------------------ | ----------------------------------------------------------------- | -------------------------------- | -| `onStart` | Pending turn snapshot (user + history) | Yes — failure does not abort | -| Interrupt boundary | New interrupts, run → `interrupted`, message snapshot | No | -| `onFinish` | Full transcript **first**, then run → `completed`, commit resumes | No | -| Stream (optional) | Throttled partial assistant text | Yes if `snapshotStreaming: true` | -| `onError` | Run → `failed` | Resumes stay pending | -| `onAbort` | Run → `aborted` — **but only sometimes** (see below) | Resumes stay pending | +| Moment | Writes | Best-effort? | +| ------------------ | ---------------------------------------------------------------------- | -------------------------------- | +| `onStart` | Pending turn snapshot (user + history) | Yes — failure does not abort | +| Interrupt boundary | New interrupts, run → `interrupted`, message snapshot | No | +| `onFinish` | Canonical transcript **first**, then run → `completed`, commit resumes | No | +| Stream (optional) | Throttled partial assistant text | Yes if `snapshotStreaming: true` | +| `onError` | Run → `failed` | Resumes stay pending | +| `onAbort` | Run → `aborted` — **but only sometimes** (see below) | Resumes stay pending | + +The canonical transcript already contains the completed terminal assistant +messages. Native-combined output keeps the structured result on its terminal +assistant message; the separate-finalization path can preserve plain-text and +structured-output assistant messages separately. ```ts withPersistence(persistence, { diff --git a/packages/ai/skills/ai-core/middleware/SKILL.md b/packages/ai/skills/ai-core/middleware/SKILL.md index 0fdbaa07e1..853fcf471f 100644 --- a/packages/ai/skills/ai-core/middleware/SKILL.md +++ b/packages/ai/skills/ai-core/middleware/SKILL.md @@ -52,21 +52,21 @@ Every hook receives a `ChatMiddlewareContext` as its first argument, which provi `requestId`, `streamId`, `phase`, `iteration`, `chunkIndex`, `model`, `provider`, `signal`, `abort()`, `defer()`, and more. -| Hook | When | Second Argument | -| -------------------------- | -------------------------------------------------------------------------------------------------- | --------------------------------------------------- | -| `onConfig` | Once at startup (`init`) + once per iteration (`beforeModel`) + once at structured-output boundary | `ChatMiddlewareConfig` (return partial to merge) | -| `onStructuredOutputConfig` | Once at the structured-output boundary (only when `chat({ outputSchema })`) | `StructuredOutputMiddlewareConfig` (return partial) | -| `onStart` | Once after initial `onConfig` | none | -| `onIteration` | Start of each agent loop iteration | `IterationInfo` | -| `onShouldContinue` | Whether to start another agent-loop iteration (AND with strategy; `false` stops) | `AgentLoopState` | -| `onChunk` | Every streamed chunk | `StreamChunk` (return void/chunk/chunk[]/null) | -| `onBeforeToolCall` | Before each tool executes | `ToolCallHookContext` (return decision or void) | -| `onAfterToolCall` | After each tool executes | `AfterToolCallInfo` | -| `onToolPhaseComplete` | After all tool calls in an iteration | `ToolPhaseCompleteInfo` | -| `onUsage` | When `RUN_FINISHED` includes usage data | `UsageInfo` | -| `onFinish` | Run completed normally | `FinishInfo` | -| `onAbort` | Run was aborted | `AbortInfo` | -| `onError` | Unhandled error occurred | `ErrorInfo` | +| Hook | When | Second Argument | +| -------------------------- | -------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | +| `onConfig` | Once at startup (`init`) + once per iteration (`beforeModel`) + once at a separate-finalization boundary | `ChatMiddlewareConfig` (return partial to merge) | +| `onStructuredOutputConfig` | Once at the separate-finalization boundary | `StructuredOutputMiddlewareConfig` (return partial) | +| `onStart` | Once after initial `onConfig` | none | +| `onIteration` | Start of each agent loop iteration | `IterationInfo` | +| `onShouldContinue` | Whether to start another agent-loop iteration (AND with strategy; `false` stops) | `AgentLoopState` | +| `onChunk` | Every streamed chunk | `StreamChunk` (return void/chunk/chunk[]/null) | +| `onBeforeToolCall` | Before each tool executes | `ToolCallHookContext` (return decision or void) | +| `onAfterToolCall` | After each tool executes | `AfterToolCallInfo` | +| `onToolPhaseComplete` | After all tool calls in an iteration | `ToolPhaseCompleteInfo` | +| `onUsage` | When `RUN_FINISHED` includes usage data | `UsageInfo` | +| `onFinish` | Run completed normally | `FinishInfo` | +| `onAbort` | Run was aborted | `AbortInfo` | +| `onError` | Unhandled error occurred | `ErrorInfo` | Terminal hooks (`onFinish`, `onAbort`, `onError`) are **mutually exclusive** -- exactly one fires per `chat()` invocation. @@ -82,31 +82,39 @@ one fires per `chat()` invocation. `ctx.phase` is one of: -| Phase | When | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `'init'` | Initial setup (before the first `onConfig` snapshot is built). | -| `'beforeModel'` | Right before each agent-loop adapter call (`onConfig` re-fires here). | -| `'modelStream'` | During model streaming chunks within the agent loop. | -| `'beforeTools'` | Before tool execution phase. | -| `'afterTools'` | After tool execution phase. | -| `'structuredOutput'` | During the final structured-output adapter call (set for all chunks from `adapter.structuredOutputStream` or the synthesized fallback). Triggered only when `chat({ outputSchema })` is invoked; one phase transition per `chat()` invocation. | +| Phase | When | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `'init'` | Initial setup (before the first `onConfig` snapshot is built). | +| `'beforeModel'` | Right before each agent-loop adapter call (`onConfig` re-fires here). | +| `'modelStream'` | During model streaming chunks within the agent loop. | +| `'beforeTools'` | Before tool execution phase. | +| `'afterTools'` | After tool execution phase. | +| `'structuredOutput'` | During the separate-finalization adapter call (set for all chunks from `adapter.structuredOutputStream` or the synthesized fallback). Does not occur for native-combined output. | -**Structured-output lifecycle rules** (when `chat({ outputSchema })` is used): +**Separate-finalization path** (adapters without native-combined support): - `onStructuredOutputConfig` fires **before** `onConfig` at the structured-output boundary. - `onConfig` re-fires at the same boundary with `ctx.phase === 'structuredOutput'`, receiving the post-`onStructuredOutputConfig` view of the config (minus `outputSchema`). - `onChunk` and `onUsage` fire for every chunk and usage event emitted by the structured-output call, with `ctx.phase === 'structuredOutput'`. - `onIteration` does **not** fire for finalization — it is agent-loop-only. -- `onFinish` fires once at the end of the whole `chat()` invocation, **after** the structured-output finalization completes (not after the agent loop). Terminal-hook exclusivity still holds (one of `onFinish` / `onAbort` / `onError`). - **Terminal `info` and structured-output:** `info.usage` / `info.finishReason` / `info.content` reflect the **agent loop's** terminal state, NOT the finalization step. Finalization state is intentionally segregated to keep agent-loop semantics clean. For a tools-less `chat({ outputSchema })` run, `info.usage` is `undefined` and `info.finishReason` is `null` (no agent-loop iteration produced `RUN_FINISHED`). To capture finalization tokens, use `onUsage` — it fires for both agent-loop iterations and the final call. For the structured-output result itself, observe the `structured-output.complete` CUSTOM event in `onChunk`. +**Native-combined output:** + +- The schema-constrained JSON is produced by a normal agent-loop iteration. `onStructuredOutputConfig` does not fire, `ctx.phase` remains `'modelStream'`, and `onIteration` fires for that iteration. +- `info.content` includes the structured JSON because it is agent-loop text. Middleware observes the `structured-output.complete` event in `onChunk` during the same phase. + +**Both paths:** + +- On successful completion, `onFinish` fires once after the structured result completes. Terminal-hook exclusivity still holds. +- By `onFinish`, `ctx.messages` includes the completed terminal assistant messages. Native-combined output keeps the structured result on its terminal assistant message. The separate-finalization path can preserve the agent loop's plain-text message followed by a distinct structured-output message. + ## onStructuredOutputConfig -A dedicated config hook that fires **only** at the structured-output boundary -(when `chat({ outputSchema })` is invoked). Use it to transform the JSON Schema -sent to the provider (inject `$defs`, strip vendor-incompatible keywords) or to -apply structured-output-specific config changes that should not affect the -agent-loop adapter calls. +A dedicated config hook that fires **only** at the separate-finalization +boundary. Use it to transform the JSON Schema sent to the provider (inject +`$defs`, strip vendor-incompatible keywords) or to apply structured-output- +specific config changes that should not affect the agent-loop adapter calls. **Signature:** @@ -259,13 +267,14 @@ const toolGuard: ChatMiddleware = { ### Pattern 3: Structured-Output Middleware -When `chat({ outputSchema })` is used, the final structured-output adapter call -now flows through the same middleware chain as the agent loop (with -`ctx.phase === 'structuredOutput'`). Before this change, the final call bypassed -middleware entirely — `onChunk`, `onUsage`, `onConfig`, and terminal hooks did -not see it. +On the separate-finalization path, the final structured-output adapter call +flows through the same middleware chain as the agent loop with +`ctx.phase === 'structuredOutput'`. Native-combined output has no separate +provider call: middleware observes its chunks during `modelStream`, and +`onStructuredOutputConfig` does not fire. Middleware cannot transform the +native-combined schema. -**Example A — Observability (tracing every chunk, including finalization):** +**Example A — Observability (tracing every chunk, including separate finalization):** ```typescript import type { ChatMiddleware } from '@tanstack/ai' @@ -278,10 +287,10 @@ const tracing: ChatMiddleware = { } ``` -This middleware now observes every chunk from the final structured-output call, -attributed to `ctx.phase === 'structuredOutput'`. Before the fix, the final -adapter call bypassed middleware entirely — `tracing` would only see agent-loop -chunks. +On the separate-finalization path, this middleware observes every chunk from +the final structured-output call with `ctx.phase === 'structuredOutput'`. On +the native-combined path, it observes the structured stream with +`ctx.phase === 'modelStream'`. **Example B — Schema rewriting (inject shared `$defs`):** @@ -298,9 +307,9 @@ const injectDefs: ChatMiddleware = { } ``` -`onStructuredOutputConfig` is the right hook here because it has direct access -to `config.outputSchema` and runs only on the structured-output boundary — -schema rewrites do not leak into the agent-loop adapter calls. +`onStructuredOutputConfig` is the right hook here on the separate-finalization +path because it has direct access to `config.outputSchema`. Native-combined +schema transformation is not exposed through middleware. ### Pattern 4: Multiple Middleware Composition @@ -779,6 +788,6 @@ Source: docs/advanced/middleware.md, `packages/ai/src/activities/chat/middleware ## Cross-References - See also: **ai-core/chat-experience/SKILL.md** -- Middleware hooks into the chat lifecycle -- See also: **ai-core/structured-outputs/SKILL.md** -- Middleware now wraps the final structured-output call; use `onStructuredOutputConfig` for JSON-Schema transforms +- See also: **ai-core/structured-outputs/SKILL.md** -- Separate finalization uses `onStructuredOutputConfig` for JSON-Schema transforms; native-combined schema transformation is not exposed through middleware - See also: **ai-core/ag-ui-protocol/SKILL.md** -- Reading the `sandbox.file` / `sandbox.file.diff` `CUSTOM` chunks the sandbox runtime emits alongside these `sandbox` hooks, via `ChatStream`'s typed `KnownCustomEvent` narrowing - See also: **`@tanstack/ai-persistence` skills** (`skills/ai-persistence/SKILL.md` in that package) -- Full persistence suite (`withPersistence`, client storage, store contracts, adapter recipes, locks). This file only sketches server `withPersistence`. diff --git a/packages/ai/skills/ai-core/structured-outputs/SKILL.md b/packages/ai/skills/ai-core/structured-outputs/SKILL.md index 6126921fcc..32799e4103 100644 --- a/packages/ai/skills/ai-core/structured-outputs/SKILL.md +++ b/packages/ai/skills/ai-core/structured-outputs/SKILL.md @@ -5,12 +5,11 @@ description: > and useChat(). Supports Zod, ArkType, and Valibot schemas. The adapter handles provider-specific strategies transparently — never configure structured output at the provider level. Pass stream:true alongside - outputSchema for incremental JSON deltas + a terminal validated object - via the `structured-output.complete` event. Every assistant turn in - useChat carries its own typed `StructuredOutputPart` on - `messages[i].parts`, so multi-turn structured chats preserve history - automatically — partial/final derive from the latest assistant turn's - part. convertSchemaToJsonSchema() for manual schema conversion. + outputSchema for incremental JSON deltas + a completed typed object + via the `structured-output.complete` event. Each successfully completed + structured-output run adds a typed `StructuredOutputPart` to message + history. partial/final derive from the most recent structured-output part + after the latest user message. convertSchemaToJsonSchema() for manual schema conversion. type: sub-skill library: tanstack-ai library_version: '0.42.0' @@ -48,7 +47,7 @@ person.age // number When `outputSchema` is provided, `chat()` returns `Promise>` instead of `AsyncIterable`. The result is fully typed. -Adding `stream: true` switches the return to `StructuredOutputStream>` — incremental JSON deltas plus a terminal validated object. See **Pattern 3** below for direct iteration, **Pattern 4** for the `useChat` shape on the client, and **Pattern 5** for multi-turn structured chats. +Adding `stream: true` switches the return to `StructuredOutputStream>` — incremental JSON deltas plus a completed typed object. See **Pattern 3** below for direct iteration, **Pattern 4** for the `useChat` shape on the client, and **Pattern 5** for multi-turn structured chats. ## Decision: which pattern fits @@ -144,7 +143,7 @@ console.log(company.financials?.revenue) ### Pattern 3: Direct stream iteration -Pass `stream: true` alongside `outputSchema` to get an async iterable of standard streaming chunks plus a terminal validated object. Use this when you're a single process end-to-end — Node script, CLI, test, or a server endpoint that responds with one JSON blob. For the in-browser progressive-UI case, jump to Pattern 4 instead. +Pass `stream: true` alongside `outputSchema` to get an async iterable of standard streaming chunks plus a completed typed object. Use this when you're a single process end-to-end — Node script, CLI, test, or a server endpoint that responds with one JSON blob. For the in-browser progressive-UI case, jump to Pattern 4 instead. ```typescript import { chat } from '@tanstack/ai' @@ -168,8 +167,8 @@ const stream = chat({ for await (const chunk of stream) { if (chunk.type === 'CUSTOM' && chunk.name === 'structured-output.complete') { - // Terminal event. `chunk.value.object` is fully validated and typed - // against the schema you passed in — no helper or cast required. + // Terminal event. `chunk.value.object` is complete and typed against the + // schema you passed in. Validate it in the consumer when required. chunk.value.object.name // string chunk.value.object.age // number chunk.value.reasoning // string | undefined (thinking models only) @@ -181,17 +180,17 @@ The terminal event is a `CUSTOM` chunk: `{ type: 'CUSTOM', name: 'structured-out **Adapter coverage for streaming:** -| Adapter | `outputSchema` + `stream: true` | -| --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `@tanstack/ai-openai` (Responses + Chat Completions) | **Native combined mode (#605)** — schema wired into the regular `chatStream` call alongside `tools`; engine harvests JSON, no finalization round-trip | -| `@tanstack/ai-anthropic` (Claude 4.5+ only) | **Native combined mode (#605)** — `output_config.format` + `tools` in one beta Messages call. Older Claude models fall back | -| `@tanstack/ai-gemini` (Gemini 3.x only) | **Native combined mode (#605)** — `responseSchema` + `tools` in one `generateContentStream`. Gemini 2.x falls back | -| `@tanstack/ai-grok` (Grok 4 family only) | **Native combined mode (#605)** — `response_format: json_schema` + `tools`. Grok 2 / 3 fall back | -| `@tanstack/ai-openrouter` | Native single-request stream (legacy `structuredOutputStream` path; per-call combined-mode lookup is a follow-up) | -| `@tanstack/ai-groq` | Legacy `structuredOutputStream` only (no tools — Groq's API rejects schema + tools + stream) | -| All other adapters (ollama, older Claude, Gemini 2.x, Grok 2/3) | Fallback: runs non-streaming `structuredOutput`, emits one `structured-output.complete` event | - -**Native combined mode vs fallback** is signaled by the adapter's +| Adapter | `outputSchema` + `stream: true` | +| --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `@tanstack/ai-openai` (Responses + Chat Completions) | **Native-combined output (#605)** — schema wired into the regular `chatStream` call alongside `tools`; engine harvests JSON, no finalization round-trip | +| `@tanstack/ai-anthropic` (Claude 4.5+ only) | **Native-combined output (#605)** — `output_config.format` + `tools` in one beta Messages call. Older Claude models fall back | +| `@tanstack/ai-gemini` (Gemini 3.x only) | **Native-combined output (#605)** — `responseSchema` + `tools` in one `generateContentStream`. Gemini 2.x falls back | +| `@tanstack/ai-grok` (Grok 4 family only) | **Native-combined output (#605)** — `response_format: json_schema` + `tools`. Grok 2 / 3 fall back | +| `@tanstack/ai-openrouter` | Native single-request stream (legacy `structuredOutputStream` path; per-call combined-mode lookup is a follow-up) | +| `@tanstack/ai-groq` | Legacy `structuredOutputStream` only (no tools — Groq's API rejects schema + tools + stream) | +| All other adapters (ollama, older Claude, Gemini 2.x, Grok 2/3) | Fallback: runs non-streaming `structuredOutput`, emits one `structured-output.complete` event | + +**Native-combined output vs separate finalization** is signaled by the adapter's optional `supportsCombinedToolsAndSchema(modelOptions)` method. When it returns `true`, the engine wires the JSON Schema into the regular `chatStream` call and harvests the final-turn text — middleware sees @@ -205,7 +204,7 @@ Consumer code is identical across providers — always read the final object off ### Pattern 4: useChat with outputSchema (progressive UI) -Pass `outputSchema` to `useChat` and you get a `partial` field that fills in as JSON streams in, plus a `final` field that snaps to the validated object on the terminal event. No `onChunk` ceremony, no manual JSON accumulation, no `parsePartialJSON` calls. +Pass `outputSchema` to `useChat` and you get a `partial` field that fills in as JSON streams in, plus a `final` field that snaps to the completed typed object on the terminal event. No `onChunk` ceremony, no manual JSON accumulation, no `parsePartialJSON` calls. **Server** (same as Pattern 3, just behind an SSE endpoint): @@ -263,7 +262,7 @@ function PersonExtractor() {

Name: {partial.name ?? '…'}

Age: {partial.age ?? '…'}

Email: {partial.email ?? '…'}

- {final &&
Validated: {JSON.stringify(final, null, 2)}
} + {final &&
Completed: {JSON.stringify(final, null, 2)}
} ) } @@ -271,12 +270,12 @@ function PersonExtractor() { - `partial` is `DeepPartial>` — every property optional, every nested array element optional. Updated from `TEXT_MESSAGE_CONTENT` deltas. - `final` is `z.infer | null` — populated when `structured-output.complete` arrives. -- `outputSchema` is for client-side type inference only. **Validation runs on the server** against the schema you pass to `chat({ outputSchema })` there. +- `outputSchema` in `useChat` is for client-side type inference. The streaming server path does not run Standard Schema validation; validate the completed object in the consumer when required. - Same shape works for non-streaming adapters: the fallback path emits one whole-JSON `TEXT_MESSAGE_CONTENT` then the terminal event, so `partial` populates and `final` snaps in the same render tick — same consumer code as the native-streaming providers, just without an intermediate field-by-field reveal. ### Pattern 5: Multi-turn structured chat -Every assistant turn produced by `useChat({ outputSchema })` carries its own typed `StructuredOutputPart` on `messages[i].parts`. Old turns stay renderable; new turns produce new parts; history is preserved without manual state plumbing. This is what makes the recipe-builder shape ("now make it vegan") work. +Each successfully completed structured-output run adds a typed `StructuredOutputPart` to an assistant message in `messages`. Old responses stay renderable; new completed runs produce new parts; history is preserved without manual state plumbing. This is what makes the recipe-builder shape ("now make it vegan") work. ```tsx import { useChat, fetchServerSentEvents } from '@tanstack/ai-react' @@ -336,10 +335,10 @@ function RecipeCard({ part }: { part: RecipePart }) { Key behaviors: -- **Per-turn parts.** Each `sendMessage()` produces a new assistant message with its own `StructuredOutputPart`. The previous turn's part is untouched — `messages.map(...)` renders the whole history. +- **Per-turn parts.** Each successfully completed structured-output run adds a structured-output assistant message with its own `StructuredOutputPart`. The separate-finalization path can also produce a plain-text assistant message before it. The previous turn's part is untouched — `messages.map(...)` renders the whole history. - **Typed by schema.** `messages[i].parts.find(p => p.type === 'structured-output').data` is typed as `Recipe` (no cast, no `unknown`). Works because `useChat` threads `InferSchemaType` down through `UIMessage` → `MessagePart` → `StructuredOutputPart`. **In `@tanstack/ai` core** the message types are single-generic (`UIMessage`); the tools generic lives in `@tanstack/ai-client` and the framework hook packages — import from your framework package or `ai-client`, not from `@tanstack/ai`. -- **`partial` / `final` are derived.** The hook-level `partial` and `final` are NOT singleton state — they're derived from the latest assistant message's part (the one after the most recent user message). Between `sendMessage()` and the first chunk, `partial` reads `{}` and `final` reads `null` because no new assistant turn exists yet. -- **Round-trip preserves history.** When the client sends turn N+1, each prior assistant turn's `structured-output` part is serialized back as `{ role: 'assistant', content: }` so the model sees its own prior structured response. Streaming / errored parts are dropped from the round-trip. +- **`partial` / `final` are derived.** The hook-level `partial` and `final` are NOT singleton state — they're derived from the latest structured-output part after the most recent user message. Between `sendMessage()` and the first chunk, `partial` reads `{}` and `final` reads `null` because no new structured-output part exists yet. +- **Round-trip preserves history.** Completed structured-output parts remain on their UI messages and are mirrored into provider-facing assistant content using `part.raw`. Streaming and errored parts remain UI state but are excluded from model input. ## Common Mistakes @@ -377,9 +376,9 @@ Source: PR #577 — structured-output became a typed UIMessage part. ### HIGH: Treating `partial` / `final` as sticky state across turns -`partial` and `final` are **derived from the latest assistant message's `structured-output` part**, not a sticky hook-level slot. In a multi-turn chat: +`partial` and `final` are **derived from the most recent structured-output part after the latest user message**, not a sticky hook-level slot. In a multi-turn chat: -- Between `sendMessage()` and the first chunk, `partial` reads `{}` and `final` reads `null` (no assistant message after the latest user yet). +- Between `sendMessage()` and the first chunk, `partial` reads `{}` and `final` reads `null` (no structured-output part after the latest user message yet). - Once the latest turn completes, `partial === final`. Earlier turns' data is NOT in `partial` / `final` — it lives on the prior assistant messages' parts. To render history, walk `messages` directly (see Pattern 5). Use `partial` / `final` for a sticky summary of the **most recent** turn only. @@ -388,7 +387,7 @@ To render history, walk `messages` directly (see Pattern 5). Use `partial` / `fi // WRONG — `final` only reflects the latest turn; earlier recipes vanish from this view {final && } -// CORRECT for history — walk messages, render every assistant's structured-output part +// CORRECT for history — walk messages, render each structured-output part {messages.map((m) => m.role === 'assistant' ? m.parts.find((p) => p.type === 'structured-output') @@ -398,11 +397,11 @@ To render history, walk `messages` directly (see Pattern 5). Use `partial` / `fi )} ``` -Source: PR #577 — partial/final derive from the latest assistant turn's part. +Source: PR #577 — partial/final derive from the most recent structured-output part after the latest user message. ### HIGH: Parsing streaming JSON deltas yourself -When iterating `chat({ outputSchema, stream: true })` directly (Pattern 3), the `TEXT_MESSAGE_CONTENT` chunks contain _partial_ JSON fragments — they are not valid JSON until the stream completes. Always read the validated object from the terminal `structured-output.complete` event. Validation runs once, on the complete payload. +When iterating `chat({ outputSchema, stream: true })` directly (Pattern 3), the `TEXT_MESSAGE_CONTENT` chunks contain _partial_ JSON fragments — they are not valid JSON until the stream completes. Read the completed typed object from the terminal `structured-output.complete` event. Standard Schema validation remains the consumer's responsibility. ```typescript // WRONG -- partial JSON, throws SyntaxError mid-stream, no schema validation @@ -415,12 +414,12 @@ for await (const chunk of stream) { // CORRECT -- trust the terminal event for await (const chunk of stream) { if (chunk.type === 'CUSTOM' && chunk.name === 'structured-output.complete') { - const result = chunk.value.object // ✅ typed and validated + const result = chunk.value.object // ✅ complete and typed } } ``` -If you need progressive parsed state in a non-React environment, use a partial-JSON parser on the accumulated raw string at render time — but do NOT treat the result as schema-validated; only the terminal event is. In `useChat`, this is already done for you (`partial` field on Pattern 4). +If you need progressive parsed state in a non-React environment, use a partial-JSON parser on the accumulated raw string at render time. Neither that partial state nor the terminal streaming event is Standard Schema validated. In `useChat`, progressive parsing is already done for you through the `partial` field from Pattern 4. Source: maintainer interview @@ -457,7 +456,7 @@ of using the schema validation library already in the project (Zod, ArkType, Valibot). Always check what the project uses and match it. ```typescript -// WRONG -- raw object, no runtime validation, no type inference +// WRONG -- raw schema object, no schema-library type inference chat({ adapter, messages, @@ -485,28 +484,32 @@ chat({ }) ``` -Using the project's schema library gives you runtime validation, TypeScript -type inference on the result, and correct JSON Schema conversion automatically. -Check `package.json` for `zod`, `arktype`, or `valibot` and use whichever is -already installed. +Using the project's schema library gives you TypeScript type inference and +correct JSON Schema conversion automatically. The non-streaming +`await chat({ outputSchema })` path also runs Standard Schema validation; the +streaming path leaves validation to the consumer. Check `package.json` for +`zod`, `arktype`, or `valibot` and use whichever is already installed. Source: maintainer interview ## Middleware coverage -The final structured-output adapter call runs through the same middleware -pipeline as the agent loop. `onChunk` observes chunks attributed to -`ctx.phase === 'structuredOutput'`; `onUsage` fires for the final call's -tokens; `onFinish` fires once at the end of the whole `chat()` invocation, -after the structured-output result is available. +On the separate-finalization path, the final structured-output adapter call +runs through the middleware pipeline with +`ctx.phase === 'structuredOutput'`. Use `onStructuredOutputConfig` to transform +the JSON Schema or finalization config before that provider call. -For schema-aware middleware (e.g., transforming the JSON Schema before the -provider call, stripping system prompts), use the dedicated -`onStructuredOutputConfig` hook. See [middleware skill](../middleware/SKILL.md). +Native-combined output stays in the regular agent loop. Its chunks use +`ctx.phase === 'modelStream'`, and `onStructuredOutputConfig` does not fire. + +On both paths, `onChunk` observes the `structured-output.complete` event, +`onUsage` observes usage from the provider calls that ran, and `onFinish` fires +once after the structured-output result is available. See +[middleware skill](../middleware/SKILL.md). ## Cross-References - See also: **ai-core/chat-experience/SKILL.md** — Base `useChat` surface; the structured-output additions documented here layer on top. - See also: **ai-core/adapter-configuration/SKILL.md** — Adapter handles structured-output strategy transparently. - See also: **ai-core/tool-calling/SKILL.md** — Combine `tools` with `outputSchema` for an agent loop that runs tools first and returns a typed object. Tool-approval and client-tool flows compose with structured runs without extra wiring; see [docs/structured-outputs/with-tools.md](https://github.com/TanStack/ai/blob/main/docs/structured-outputs/with-tools.md). -- See also: **ai-core/middleware/SKILL.md** — `onStructuredOutputConfig` hook and the `structuredOutput` phase for observing/transforming the final structured-output call. +- See also: **ai-core/middleware/SKILL.md** — separate-finalization `onStructuredOutputConfig` / `structuredOutput` behavior and native-combined `modelStream` behavior. From aefff4df3efba6941c1f21ec502b079695823b63 Mon Sep 17 00:00:00 2001 From: kolaworld Date: Sun, 16 Aug 2026 23:45:40 -0400 Subject: [PATCH 3/3] fix(ai): preserve middleware transcript additions Append terminal assistant messages to the middleware-visible transcript so chunk observers do not lose messages they recorded before persistence runs.\n\nRefs #1072 --- packages/ai/src/activities/chat/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index fb3fea635e..729ce274a3 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -2021,7 +2021,8 @@ class TextEngine< } : undefined const nativeCombined = this.finalStructuredOutput?.nativeCombined === true - const currentTurnAlreadyRecorded = this.messages.some( + const messages = this.middlewareCtx.messages + const currentTurnAlreadyRecorded = messages.some( (message) => message.role === 'assistant' && message.id === this.currentMessageId, ) @@ -2058,7 +2059,7 @@ class TextEngine< if (terminalMessages.length === 0) return - this.messages = [...this.messages, ...terminalMessages] + this.messages = [...messages, ...terminalMessages] this.middlewareCtx.messages = this.messages }