Skip to content

.NET: fix(hosting): emit url_citation annotation events from streamed AI Search responses #6649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
anneheartrecord wants to merge 1 commit into
base: main
Choose a base branch
from
+284 −9
Open

.NET: fix(hosting): emit url_citation annotation events from streamed AI Search responses #6649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 72 additions & 9 deletions dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
Expand Down Expand Up @@ -45,6 +46,7 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
OutputItemMessageBuilder? currentMessageBuilder = null;
TextContentBuilder? currentTextBuilder = null;
StringBuilder? accumulatedText = null;
List<Annotation>? accumulatedAnnotations = null;
string? previousMessageId = null;
bool hasTerminalEvent = false;
var executorItemIds = new Dictionary<string, string>();
Expand All @@ -60,14 +62,15 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
if (update.RawRepresentation is WorkflowEvent workflowEvent && update.Contents.Count == 0)
{
// Close any open message builder before emitting workflow items
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}

currentTextBuilder = null;
currentMessageBuilder = null;
accumulatedText = null;
accumulatedAnnotations = null;
previousMessageId = null;

foreach (var evt in EmitWorkflowEvent(stream, workflowEvent, executorItemIds))
Expand All @@ -86,14 +89,15 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
{
if (!IsSameMessage(update.MessageId, previousMessageId) && currentMessageBuilder is not null)
{
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}

currentTextBuilder = null;
currentMessageBuilder = null;
accumulatedText = null;
accumulatedAnnotations = null;
}

previousMessageId = update.MessageId;
Expand All @@ -115,6 +119,14 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
yield return currentTextBuilder!.EmitDelta(textContent.Text);
}

if (textContent.Annotations is { Count: > 0 })
{
foreach (var sdkAnnotation in ConvertToSdkAnnotations(textContent.Annotations))
{
(accumulatedAnnotations ??= []).Add(sdkAnnotation);
}
}

break;
}

Expand All @@ -125,14 +137,15 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
break;
}

foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}

currentTextBuilder = null;
currentMessageBuilder = null;
accumulatedText = null;
accumulatedAnnotations = null;
previousMessageId = null;

var arguments = functionCall.Arguments is not null
Expand All @@ -149,14 +162,15 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents

case TextReasoningContent reasoningContent:
{
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}

currentTextBuilder = null;
currentMessageBuilder = null;
accumulatedText = null;
accumulatedAnnotations = null;
previousMessageId = null;

var reasoningBuilder = stream.AddOutputItemReasoningItem();
Expand All @@ -176,14 +190,15 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents

case ToolApprovalRequestContent approvalRequest when approvalRequest.ToolCall is FunctionCallContent approvalFunctionCall:
{
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}

currentTextBuilder = null;
currentMessageBuilder = null;
accumulatedText = null;
accumulatedAnnotations = null;
previousMessageId = null;

// The Responses API only standardizes the MCP-flavored approval primitive.
Expand Down Expand Up @@ -237,14 +252,15 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents

case ErrorContent errorContent:
{
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}

currentTextBuilder = null;
currentMessageBuilder = null;
accumulatedText = null;
accumulatedAnnotations = null;
previousMessageId = null;
hasTerminalEvent = true;

Expand All @@ -269,14 +285,15 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
break;
}

foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}

currentTextBuilder = null;
currentMessageBuilder = null;
accumulatedText = null;
accumulatedAnnotations = null;
previousMessageId = null;

var outputText = EncodeFunctionResultAsJsonStringPayload(functionResult.Result);
Expand Down Expand Up @@ -304,7 +321,7 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
}

// Close any remaining open message
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
{
yield return evt;
}
Expand All @@ -318,7 +335,8 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
private static IEnumerable<ResponseStreamEvent> CloseCurrentMessage(
OutputItemMessageBuilder? messageBuilder,
TextContentBuilder? textBuilder,
StringBuilder? accumulatedText)
StringBuilder? accumulatedText,
List<Annotation>? annotations = null)
{
if (messageBuilder is null)
{
Expand All @@ -329,6 +347,16 @@ private static IEnumerable<ResponseStreamEvent> CloseCurrentMessage(
{
var finalText = accumulatedText?.ToString() ?? string.Empty;
yield return textBuilder.EmitTextDone(finalText);

// Annotations must be emitted after EmitTextDone and before EmitDone.
if (annotations is not null)
{
foreach (var annotation in annotations)
{
yield return textBuilder.EmitAnnotationAdded(annotation);
}
}

yield return textBuilder.EmitDone();
}

Expand All @@ -338,6 +366,41 @@ private static IEnumerable<ResponseStreamEvent> CloseCurrentMessage(
private static bool IsSameMessage(string? currentId, string? previousId) =>
currentId is not { Length: > 0 } || previousId is not { Length: > 0 } || currentId == previousId;

/// <summary>
/// Converts MEAI <see cref="AIAnnotation"/> instances to Responses SDK <see cref="Annotation"/> objects.
/// Only <see cref="CitationAnnotation"/> with a URL and at least one <see cref="TextSpanAnnotatedRegion"/>
/// with explicit start/end indices is converted; all other shapes are skipped.
/// </summary>
private static IEnumerable<Annotation> ConvertToSdkAnnotations(IList<AIAnnotation> annotations)
{
foreach (var ann in annotations)
{
if (ann is not CitationAnnotation citation || citation.Url is null)
{
continue;
}

var regions = citation.AnnotatedRegions?
.OfType<TextSpanAnnotatedRegion>()
.Where(r => r.StartIndex is not null && r.EndIndex is not null)
.ToList();

if (regions is not { Count: > 0 })
{
continue;
}

foreach (var region in regions)
{
yield return new UrlCitationBody(
citation.Url,
(long)region.StartIndex!.Value,
(long)region.EndIndex!.Value,
citation.Title ?? string.Empty);
}
}
}

private static ResponseUsage ConvertUsage(UsageDetails details, ResponseUsage? existing)
{
var inputTokens = details.InputTokenCount ?? 0;
Expand Down
Loading