From 2ec7b65129b910f507c0c1eeb7b0f2e2dd2ad30e Mon Sep 17 00:00:00 2001 From: Steven Mendez Date: Fri, 24 Jul 2026 10:44:44 -0600 Subject: [PATCH 1/2] feat(google): native Google Search grounding behind OPENCLAW_GOOGLE_GROUNDING When the env flag is set, the Google transport adds the google_search built-in tool to every request (with includeServerSideToolInvocations so it can coexist with function declarations), captures the streamed groundingMetadata onto the assistant message as a normalized webGrounding payload (queries + sources ranked by citation count), and surfaces it as a 'grounding' agent event that the OpenAI-compat endpoint forwards as a content-free delta.dojo_grounding chunk for downstream consumers. Co-Authored-By: Claude Fable 5 --- src/agents/google-transport-stream.ts | 85 +++++++++++++++++++ ...pi-embedded-subscribe.handlers.messages.ts | 20 +++++ src/gateway/openai-http.ts | 28 ++++++ 3 files changed, 133 insertions(+) diff --git a/src/agents/google-transport-stream.ts b/src/agents/google-transport-stream.ts index 5c66b722ba8e4..b5ff8fdb9c022 100644 --- a/src/agents/google-transport-stream.ts +++ b/src/agents/google-transport-stream.ts @@ -81,6 +81,25 @@ type MutableAssistantOutput = { timestamp: number; responseId?: string; errorMessage?: string; + webGrounding?: TransportWebGrounding; +}; + +type GoogleGroundingMetadata = { + webSearchQueries?: string[]; + groundingChunks?: Array<{ + web?: { title?: string; uri?: string; domain?: string }; + }>; + groundingSupports?: Array<{ groundingChunkIndices?: number[] }>; +}; + +/** + * Normalized Google Search grounding payload attached to the assistant + * output message. Consumers (gateway plugins, the OpenAI-compat endpoint) + * read it as `(message as { webGrounding?: TransportWebGrounding })`. + */ +export type TransportWebGrounding = { + queries: string[]; + sources: Array<{ title: string; url: string; cited?: number }>; }; type GoogleSseChunk = { @@ -99,6 +118,7 @@ type GoogleSseChunk = { }>; }; finishReason?: string; + groundingMetadata?: GoogleGroundingMetadata; }>; usageMetadata?: { promptTokenCount?: number; @@ -408,6 +428,59 @@ function convertGoogleMessages(model: GoogleTransportModel, context: Context) { return contents; } +/** + * Server-side Google Search grounding (opt-in via env). When enabled the + * request carries the `google_search` built-in tool; mixing it with + * `functionDeclarations` additionally requires + * `toolConfig.includeServerSideToolInvocations` (the API 400s without it). + * The resulting `groundingMetadata` is normalized onto the assistant output + * as `webGrounding`. + */ +function isGoogleSearchGroundingEnabled(): boolean { + const raw = process.env.OPENCLAW_GOOGLE_GROUNDING?.trim().toLowerCase(); + return raw === "1" || raw === "true" || raw === "on"; +} + +function mergeWebGrounding( + output: MutableAssistantOutput, + metadata: GoogleGroundingMetadata, +): void { + const grounding = (output.webGrounding ??= { queries: [], sources: [] }); + for (const query of metadata.webSearchQueries ?? []) { + if (typeof query === "string" && query && !grounding.queries.includes(query)) { + grounding.queries.push(query); + } + } + // Citation counts per chunk index (`groundingSupports`) let consumers rank + // sources by how often the answer actually cited them. + const citeCounts = new Map(); + for (const support of metadata.groundingSupports ?? []) { + for (const idx of support?.groundingChunkIndices ?? []) { + if (typeof idx === "number") { + citeCounts.set(idx, (citeCounts.get(idx) ?? 0) + 1); + } + } + } + const chunks = metadata.groundingChunks ?? []; + for (let idx = 0; idx < chunks.length; idx++) { + const chunk = chunks[idx]; + const url = chunk?.web?.uri; + if (typeof url !== "string" || !url) { + continue; + } + const existing = grounding.sources.find((source) => source.url === url); + if (existing) { + existing.cited = (existing.cited ?? 0) + (citeCounts.get(idx) ?? 0); + continue; + } + grounding.sources.push({ + title: chunk.web?.title || chunk.web?.domain || url, + url, + cited: citeCounts.get(idx) ?? 0, + }); + } +} + function convertGoogleTools(tools: NonNullable) { if (tools.length === 0) { return undefined; @@ -467,6 +540,15 @@ export function buildGoogleGenerativeAiParams( }; } } + if (isGoogleSearchGroundingEnabled()) { + params.tools = [...(params.tools ?? []), { google_search: {} }]; + if (context.tools?.length) { + params.toolConfig = { + ...(params.toolConfig ?? {}), + includeServerSideToolInvocations: true, + }; + } + } return params; } @@ -625,6 +707,9 @@ export function createGoogleGenerativeAiTransportStreamFn(): StreamFn { output.responseId ||= chunk.responseId; updateUsage(output, model, chunk); const candidate = chunk.candidates?.[0]; + if (candidate?.groundingMetadata) { + mergeWebGrounding(output, candidate.groundingMetadata); + } if (candidate?.content?.parts) { for (const part of candidate.content.parts) { if (typeof part.text === "string") { diff --git a/src/agents/pi-embedded-subscribe.handlers.messages.ts b/src/agents/pi-embedded-subscribe.handlers.messages.ts index 1be884188417e..59e5dd33f7eed 100644 --- a/src/agents/pi-embedded-subscribe.handlers.messages.ts +++ b/src/agents/pi-embedded-subscribe.handlers.messages.ts @@ -528,6 +528,26 @@ export function handleMessageEnd( ctx.state.lastStreamedAssistantCleaned = cleanedText; } + // Server-side Google Search grounding (OPENCLAW_GOOGLE_GROUNDING): the + // transport normalizes `groundingMetadata` onto the assistant message as + // `webGrounding`. Surface it as its own event stream so channel consumers + // (e.g. the OpenAI-compat endpoint) can forward the cited sources. + const webGrounding = ( + assistantMessage as { webGrounding?: { queries: string[]; sources: unknown[] } } + ).webGrounding; + if (webGrounding && webGrounding.sources.length > 0) { + const groundingData = webGrounding as unknown as Record; + emitAgentEvent({ + runId: ctx.params.runId, + stream: "grounding", + data: groundingData, + }); + void ctx.params.onAgentEvent?.({ + stream: "grounding", + data: groundingData, + }); + } + const silentExpectedWithoutSentinel = ctx.params.silentExpected && !isSilentReplyText(trimmedText, SILENT_REPLY_TOKEN); const finalAssistantText = silentExpectedWithoutSentinel ? "" : text; diff --git a/src/gateway/openai-http.ts b/src/gateway/openai-http.ts index 604b27212cba8..94c18ad1402e6 100644 --- a/src/gateway/openai-http.ts +++ b/src/gateway/openai-http.ts @@ -159,6 +159,25 @@ function writeAssistantContentChunk( }); } +/** + * Content-free chunk carrying Google Search grounding sources (queries + + * cited URLs) emitted by the `grounding` agent-event stream. Standard OpenAI + * clients ignore the unknown `dojo_grounding` delta field; the Dojo web-chat + * proxy consumes it to render a cited-sources card. + */ +function writeGroundingChunk( + res: ServerResponse, + params: { runId: string; model: string; grounding: Record }, +) { + writeSse(res, { + id: params.runId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: params.model, + choices: [{ index: 0, delta: { dojo_grounding: params.grounding } }], + }); +} + function asMessages(val: unknown): OpenAiChatMessage[] { return Array.isArray(val) ? (val as OpenAiChatMessage[]) : []; } @@ -564,6 +583,15 @@ export async function handleOpenAiHttpRequest( return; } + if (evt.stream === "grounding") { + if (!wroteRole) { + wroteRole = true; + writeAssistantRoleChunk(res, { runId, model }); + } + writeGroundingChunk(res, { runId, model, grounding: evt.data ?? {} }); + return; + } + if (evt.stream === "assistant") { const content = resolveAssistantStreamDeltaText(evt) ?? ""; if (!content) { From 9e3a1b0214431d5d58e3b45442ad6beec9f16a17 Mon Sep 17 00:00:00 2001 From: Steven Mendez Date: Fri, 24 Jul 2026 11:13:42 -0600 Subject: [PATCH 2/2] style(google): drop empty-object fallback in toolConfig spread (oxlint) Co-Authored-By: Claude Fable 5 --- src/agents/google-transport-stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agents/google-transport-stream.ts b/src/agents/google-transport-stream.ts index b5ff8fdb9c022..5a54a5989c142 100644 --- a/src/agents/google-transport-stream.ts +++ b/src/agents/google-transport-stream.ts @@ -544,7 +544,7 @@ export function buildGoogleGenerativeAiParams( params.tools = [...(params.tools ?? []), { google_search: {} }]; if (context.tools?.length) { params.toolConfig = { - ...(params.toolConfig ?? {}), + ...params.toolConfig, includeServerSideToolInvocations: true, }; }