From 515dad0accf4aa47b1389f46f85ab8e0cac5a23d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 21:39:18 -0500 Subject: [PATCH 01/18] Add Exie conversation tracking and failure diagnostics --- .../Utility/AppDiagnostics.cs | 3 + .../Api/Endpoints/AssistantEndpoints.cs | 46 ++- src/Exceptionless.Web/ApmExtensions.cs | 2 +- .../Assistant/AssistantModels.cs | 4 + .../Assistant/AssistantProviderDiagnostics.cs | 111 ++++++ .../Assistant/AssistantProviderException.cs | 5 +- .../Assistant/AssistantService.cs | 70 +++- .../Assistant/AssistantTurnDiagnostics.cs | 157 ++++++++ .../assistant/assistant-telemetry.test.ts | 102 ++++++ .../features/assistant/assistant-telemetry.ts | 116 ++++++ .../assistant-message-actions.svelte | 19 +- .../assistant-message-actions.svelte.test.ts | 20 +- .../components/assistant-message.svelte | 5 +- .../components/assistant-panel.svelte | 229 ++++++++++-- .../components/assistant-panel.svelte.test.ts | 153 ++++++++ .../src/lib/features/assistant/models.ts | 1 + .../auth/exceptionless-session.test.ts | 51 +++ .../features/auth/exceptionless-session.ts | 22 +- src/Exceptionless.Web/appsettings.yml | 1 + .../Assistant/AssistantDiagnosticsTests.cs | 345 ++++++++++++++++++ .../Assistant/AssistantServiceTests.cs | 72 +++- tests/Exceptionless.Tests/Assistant/README.md | 75 ++++ 22 files changed, 1534 insertions(+), 75 deletions(-) create mode 100644 src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs create mode 100644 src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts create mode 100644 tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs diff --git a/src/Exceptionless.Core/Utility/AppDiagnostics.cs b/src/Exceptionless.Core/Utility/AppDiagnostics.cs index e59dc6bf7a..44ad4422de 100644 --- a/src/Exceptionless.Core/Utility/AppDiagnostics.cs +++ b/src/Exceptionless.Core/Utility/AppDiagnostics.cs @@ -91,6 +91,9 @@ public GaugeInfo(Meter meter, string name) internal static readonly Counter EventsSubmitted = Meter.CreateCounter("ex.events.submitted", description: "Events submitted to the pipeline to be processed"); internal static readonly Counter AssistantTurns = Meter.CreateCounter("ex.assistant.turns", description: "Assistant turns accepted"); internal static readonly Counter AssistantTurnOutcomes = Meter.CreateCounter("ex.assistant.turn.outcomes", description: "Assistant turn outcomes"); + internal static readonly Histogram AssistantTurnDuration = Meter.CreateHistogram("ex.assistant.turn.duration", unit: "ms", description: "Assistant turn duration by outcome and failure reason"); + internal static readonly Histogram AssistantProviderDuration = Meter.CreateHistogram("ex.assistant.provider.duration", unit: "ms", description: "Assistant provider request duration by outcome"); + internal static readonly Histogram AssistantToolDuration = Meter.CreateHistogram("ex.assistant.tool.duration", unit: "ms", description: "Assistant tool duration by tool and outcome"); internal static readonly Counter AssistantTurnsBlocked = Meter.CreateCounter("ex.assistant.turns.blocked", description: "Assistant turns blocked by a usage limit"); internal static readonly Counter AssistantProviderRequests = Meter.CreateCounter("ex.assistant.provider.requests", description: "Assistant provider requests"); internal static readonly Counter AssistantToolCalls = Meter.CreateCounter("ex.assistant.tool.calls", description: "Assistant tool calls"); diff --git a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs index 9a0fbaa0dd..37c109acee 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs @@ -99,17 +99,37 @@ private static async Task StreamChatAsync( using var turnCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(httpContext.RequestAborted); turnCancellationSource.CancelAfter(TimeSpan.FromSeconds(AssistantLimits.MaximumTurnDurationSeconds)); + using var diagnostics = new AssistantTurnDiagnostics(logger, timeProvider, organizationId!, request.ConversationId!, httpContext.TraceIdentifier); + var response = assistantService.StreamAsync(request, userId, planOptions, diagnostics, turnCancellationSource.Token); + await WriteResponseAsync(httpContext, response, assistantUsageService, organizationId!, diagnostics, turnCancellationSource.Token); + + return HttpResults.Empty; + } + + internal static async Task WriteResponseAsync( + HttpContext httpContext, + IAsyncEnumerable response, + AssistantUsageService assistantUsageService, + string organizationId, + AssistantTurnDiagnostics diagnostics, + CancellationToken cancellationToken) + { bool responseFailed = false; try { - await foreach (var item in assistantService.StreamAsync(request, userId, planOptions, turnCancellationSource.Token)) + await foreach (var item in response.WithCancellation(cancellationToken)) { + diagnostics.Observe(item); responseFailed |= item.Type == "error"; - await JsonSerializer.SerializeAsync(httpContext.Response.Body, item, s_jsonOptions, turnCancellationSource.Token); - await httpContext.Response.WriteAsync("\n", turnCancellationSource.Token); - await httpContext.Response.Body.FlushAsync(turnCancellationSource.Token); + string stage = diagnostics.Stage; + diagnostics.Stage = "response_write"; + await JsonSerializer.SerializeAsync(httpContext.Response.Body, item, s_jsonOptions, cancellationToken); + await httpContext.Response.WriteAsync("\n", cancellationToken); + await httpContext.Response.Body.FlushAsync(cancellationToken); + diagnostics.Stage = stage; } + diagnostics.Finish(responseFailed ? "failed" : "completed"); if (responseFailed) await assistantUsageService.RecordTurnFailedAsync(organizationId); else @@ -118,10 +138,14 @@ private static async Task StreamChatAsync( catch (OperationCanceledException) when (httpContext.RequestAborted.IsCancellationRequested) { // The browser closing or stopping the stream is expected. + diagnostics.Finish("cancelled", "client_disconnected"); await assistantUsageService.RecordTurnCancelledAsync(organizationId); } catch (OperationCanceledException) { + string failureCode = cancellationToken.IsCancellationRequested ? "turn_timeout" + : diagnostics.Stage is "provider_request" or "provider_stream" ? "provider_timeout" : "operation_cancelled"; + diagnostics.Finish("failed", failureCode); await assistantUsageService.RecordTurnFailedAsync(organizationId); var error = AssistantStreamEvent.Error("Exie took too long to complete this response. Try narrowing the question."); await JsonSerializer.SerializeAsync(httpContext.Response.Body, error, s_jsonOptions, CancellationToken.None); @@ -129,14 +153,22 @@ private static async Task StreamChatAsync( } catch (Exception ex) { + string failureCode = ex switch + { + AssistantProviderException providerException => providerException.FailureCode, + _ when diagnostics.Stage == "response_write" => "response_write_error", + _ when diagnostics.Stage == "tool_execution" => "tool_execution_error", + HttpRequestException when diagnostics.Stage is "provider_request" or "provider_stream" => "provider_transport_error", + JsonException when diagnostics.Stage == "provider_stream" => "invalid_provider_response", + IOException when diagnostics.Stage == "provider_stream" => "provider_stream_error", + _ => "internal_error" + }; + diagnostics.Finish("failed", failureCode, ex); await assistantUsageService.RecordTurnFailedAsync(organizationId); - logger.LogError(ex, "Unable to stream an in-app assistant response"); var error = AssistantStreamEvent.Error(ex is AssistantProviderException ? ex.Message : "Exie could not complete this request."); await JsonSerializer.SerializeAsync(httpContext.Response.Body, error, s_jsonOptions, CancellationToken.None); await httpContext.Response.WriteAsync("\n", CancellationToken.None); } - - return HttpResults.Empty; } private static async Task GetAccessAsync( diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 0f6af7263f..d37ef0ba1b 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -72,7 +72,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }); b.AddHttpClientInstrumentation(); - b.AddSource("Exceptionless", "Foundatio"); + b.AddSource("Exceptionless", "Exceptionless.Core", "Foundatio"); if (config.EnableRedis) b.AddRedisInstrumentation(c => diff --git a/src/Exceptionless.Web/Assistant/AssistantModels.cs b/src/Exceptionless.Web/Assistant/AssistantModels.cs index 098ccab84c..4a9fc6ad84 100644 --- a/src/Exceptionless.Web/Assistant/AssistantModels.cs +++ b/src/Exceptionless.Web/Assistant/AssistantModels.cs @@ -43,10 +43,14 @@ public sealed record AssistantStreamEvent( string? Message = null, IReadOnlyCollection? SuggestedActions = null) { + [System.Text.Json.Serialization.JsonIgnore] + internal string? FailureCode { get; init; } + public static AssistantStreamEvent TextDelta(string text) => new("text_delta", Text: text); public static AssistantStreamEvent ToolCall(string id, string name, string arguments) => new("tool_call", ToolCallId: id, ToolName: name, Arguments: arguments); public static AssistantStreamEvent ToolResult(string id, string name, string result) => new("tool_result", ToolCallId: id, ToolName: name, Result: result); public static AssistantStreamEvent Suggestions(IReadOnlyCollection actions) => new("suggested_actions", SuggestedActions: actions); public static AssistantStreamEvent Error(string message) => new("error", Message: message); + internal static AssistantStreamEvent Error(string message, string failureCode) => new("error", Message: message) { FailureCode = failureCode }; public static AssistantStreamEvent Done() => new("done"); } diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs new file mode 100644 index 0000000000..9a722c8eac --- /dev/null +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -0,0 +1,111 @@ +using System.Diagnostics; +using System.Text.Json; + +namespace Exceptionless.Web.Assistant; + +internal sealed class AssistantProviderDiagnostics( + ILogger logger, + TimeProvider timeProvider, + AssistantTurnDiagnostics turn, + int inputCharacters, + bool allowTools, + CancellationToken cancellationToken) : IDisposable +{ + private readonly long _started = timeProvider.GetTimestamp(); + private readonly Activity? _activity = AppDiagnostics.StartActivity("assistant.provider"); + private bool _finished; + + public string? GenerationId { get; private set; } + public string? Model { get; private set; } + public string? ProviderName { get; private set; } + public int? StatusCode { get; private set; } + public string? FinishReason { get; private set; } + public bool UsageReceived { get; private set; } + public long? PromptTokens { get; private set; } + public long? CompletionTokens { get; private set; } + public long? ReasoningTokens { get; private set; } + + public void ObserveResponse(HttpResponseMessage response) + { + StatusCode = (int)response.StatusCode; + if (response.Headers.TryGetValues("X-Generation-Id", out var values)) + GenerationId = SafeMetadata(values.FirstOrDefault()); + turn.Stage = "provider_stream"; + } + + public void ObserveChunk(JsonElement chunk) + { + if (chunk.ValueKind != JsonValueKind.Object) + return; + GenerationId = GetMetadata(chunk, "id") ?? GenerationId; + Model = GetMetadata(chunk, "model") ?? Model; + ProviderName = GetMetadata(chunk, "provider") ?? ProviderName; + if (chunk.TryGetProperty("usage", out var usage) && usage.ValueKind == JsonValueKind.Object) + { + UsageReceived = true; + PromptTokens = GetTokenCount(usage, "prompt_tokens") ?? PromptTokens; + CompletionTokens = GetTokenCount(usage, "completion_tokens") ?? CompletionTokens; + if (usage.TryGetProperty("completion_tokens_details", out var details) && details.ValueKind == JsonValueKind.Object + && GetTokenCount(details, "reasoning_tokens") is { } value) + ReasoningTokens = value; + } + if (chunk.TryGetProperty("choices", out var choices) && choices.ValueKind == JsonValueKind.Array && choices.GetArrayLength() > 0) + { + string? reason = GetMetadata(choices[0], "finish_reason"); + if (reason is not null) + FinishReason = reason is "stop" or "length" or "tool_calls" or "content_filter" or "error" ? reason : "unknown"; + } + } + + public void Complete(int outputCharacters, int toolCalls, bool receivedDone) + { + string outcome = FinishReason switch + { + "length" => "output_limit", + "content_filter" => "content_filter", + "error" => "provider_error", + _ when outputCharacters == 0 && toolCalls == 0 => "empty_response", + _ when !receivedDone && FinishReason is null => "incomplete_stream", + _ => "completed" + }; + Finish(outcome, outputCharacters, toolCalls, receivedDone); + } + + private void Finish(string outcome, int? outputCharacters = null, int? toolCalls = null, bool receivedDone = false) + { + if (_finished) + return; + _finished = true; + double duration = timeProvider.GetElapsedTime(_started).TotalMilliseconds; + _activity?.SetTag("assistant.provider.outcome", outcome); + _activity?.SetTag("assistant.provider.generation_id", GenerationId); + _activity?.SetTag("assistant.provider.model", Model ?? turn.Model); + _activity?.SetTag("assistant.provider.name", ProviderName); + _activity?.SetTag("assistant.provider.finish_reason", FinishReason); + _activity?.SetTag("http.response.status_code", StatusCode); + if (outcome is not ("completed" or "cancelled")) + _activity?.SetStatus(ActivityStatusCode.Error, outcome); + AppDiagnostics.AssistantProviderDuration.Record(duration, new KeyValuePair("outcome", outcome)); + logger.Log(outcome is "completed" or "cancelled" ? LogLevel.Information : LogLevel.Warning, + "Assistant provider request {ProviderRequestNumber} {ProviderOutcome} for turn {AssistantTurnId}: duration={DurationMs} ms generation={ProviderGenerationId} model={ProviderModel} provider={ProviderName} status={ProviderStatusCode} finish={ProviderFinishReason} input_characters={InputCharacters} output_characters={OutputCharacters} tools_allowed={ToolsAllowed} tool_calls={ToolCalls} usage_received={UsageReceived} prompt_tokens={PromptTokens} completion_tokens={CompletionTokens} reasoning_tokens={ReasoningTokens} received_done={ReceivedDone}", + turn.ProviderRequests, outcome, turn.TurnId, duration, GenerationId, Model ?? turn.Model, ProviderName, StatusCode, + FinishReason, inputCharacters, outputCharacters, allowTools, toolCalls, UsageReceived, PromptTokens, CompletionTokens, ReasoningTokens, receivedDone); + _activity?.Dispose(); + } + + public void Dispose() => Finish(StatusCode is >= 400 ? "http_error" + : FinishReason == "error" ? "provider_error" + : cancellationToken.IsCancellationRequested ? "cancelled" : "interrupted"); + + private static string? GetMetadata(JsonElement element, string name) + => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String + ? SafeMetadata(property.GetString()) : null; + + private static long? GetTokenCount(JsonElement element, string name) + => element.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number && property.TryGetInt64(out long value) + ? Math.Max(0, value) : null; + + private static string? SafeMetadata(string? value) + => value is { Length: > 0 and <= 128 } && value.All(character => Char.IsAsciiLetterOrDigit(character) || character is '-' or '_' or '/' or '.' or ':' or '~' or ' ') + ? value : null; +} diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderException.cs b/src/Exceptionless.Web/Assistant/AssistantProviderException.cs index da724717e0..501ac03bd2 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderException.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderException.cs @@ -1,3 +1,6 @@ namespace Exceptionless.Web.Assistant; -public sealed class AssistantProviderException(string message) : Exception(message); +public sealed class AssistantProviderException(string message) : Exception(message) +{ + internal string FailureCode { get; init; } = "provider_error"; +} diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index c52de20f52..a25161555a 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -36,14 +36,24 @@ public sealed class AssistantService( private static readonly JsonSerializerOptions s_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web).ConfigureExceptionlessApiDefaults(); private static readonly Regex s_rawDsmlPattern = new(@"<\s*/?\s*[|｜]\s*DSML\s*[|｜]", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)); - public async IAsyncEnumerable StreamAsync( + public IAsyncEnumerable StreamAsync( AssistantChatRequest request, string userId, AssistantPlanOptions planOptions, + CancellationToken cancellationToken = default) + => StreamAsync(request, userId, planOptions, null, cancellationToken); + + internal async IAsyncEnumerable StreamAsync( + AssistantChatRequest request, + string userId, + AssistantPlanOptions planOptions, + AssistantTurnDiagnostics? diagnostics, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var options = appOptions.AssistantOptions; string model = (await assistantModelSettingsService.GetAsync()).Model; + if (diagnostics is not null) + diagnostics.Model = model; AssistantConversationState? conversationState = null; if (!String.IsNullOrWhiteSpace(request.OrganizationId) && !String.IsNullOrWhiteSpace(request.ConversationId)) { @@ -71,10 +81,12 @@ public async IAsyncEnumerable StreamAsync( { if (completedToolRounds > 0) { + if (diagnostics is not null) + diagnostics.Stage = "usage_check"; var usageDecision = await assistantUsageService.TryContinueTurnAsync(request.OrganizationId, planOptions); if (!usageDecision.Allowed) { - yield return AssistantStreamEvent.Error(usageDecision.Message ?? "Exie reached this organization's usage limit."); + yield return AssistantStreamEvent.Error(usageDecision.Message ?? "Exie reached this organization's usage limit.", "usage_limit"); yield return AssistantStreamEvent.Done(); yield break; } @@ -110,14 +122,18 @@ public async IAsyncEnumerable StreamAsync( if (providerInputCharacters > AssistantLimits.MaximumProviderInputCharacters) { throw new AssistantProviderException( - "This conversation contains too much context for one response. Clear the conversation or narrow the question."); + "This conversation contains too much context for one response. Clear the conversation or narrow the question.") { FailureCode = "context_limit" }; } + if (diagnostics is not null) + diagnostics.Stage = "usage_reservation"; await using var providerRequest = await assistantUsageService.StartProviderRequestAsync(request.OrganizationId, providerInputCharacters); - using var response = await SendRequestAsync(messages, options, model, allowTools, request, cancellationToken); + using var providerDiagnostics = diagnostics?.StartProviderRequest(providerInputCharacters, allowTools, cancellationToken); + using var response = await SendRequestAsync(messages, options, model, allowTools, request, providerDiagnostics, cancellationToken); providerRequest.MarkAccepted(); await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); using var reader = new StreamReader(stream); + bool receivedDone = false; while (await reader.ReadLineAsync(cancellationToken) is { } line) { @@ -125,10 +141,16 @@ public async IAsyncEnumerable StreamAsync( continue; string payload = line[5..].Trim(); - if (payload.Length == 0 || payload == "[DONE]") + if (payload == "[DONE]") + { + receivedDone = true; + continue; + } + if (payload.Length == 0) continue; using var document = JsonDocument.Parse(payload); + providerDiagnostics?.ObserveChunk(document.RootElement); if (document.RootElement.TryGetProperty("error", out var error)) throw new AssistantProviderException(GetProviderError(error)); @@ -186,11 +208,17 @@ public async IAsyncEnumerable StreamAsync( } } + providerDiagnostics?.Complete(assistantContent.Length, toolCalls.Count, receivedDone); + if (diagnostics is not null) + diagnostics.Stage = "response_validation"; + if (s_rawDsmlPattern.IsMatch(assistantContent.ToString())) { if (malformedResponseRetries < AssistantLimits.MaximumMalformedResponseRetries) { malformedResponseRetries++; + if (diagnostics is not null) + diagnostics.MalformedResponseRetries = malformedResponseRetries; logger.LogWarning( "Assistant provider returned raw DSML content for organization {OrganizationId}; retrying response", request.OrganizationId); @@ -206,7 +234,7 @@ public async IAsyncEnumerable StreamAsync( logger.LogWarning( "Assistant provider returned raw DSML content again for organization {OrganizationId}", request.OrganizationId); - yield return AssistantStreamEvent.Error("Exie received a malformed response from the AI provider. Please try again."); + yield return AssistantStreamEvent.Error("Exie received a malformed response from the AI provider. Please try again.", "malformed_response"); yield return AssistantStreamEvent.Done(); yield break; } @@ -226,7 +254,14 @@ public async IAsyncEnumerable StreamAsync( { if (assistantContent.Length == 0) { - yield return AssistantStreamEvent.Error("Exie stopped before providing an answer. Please try again."); + string failureCode = providerDiagnostics?.FinishReason switch + { + "length" => "output_limit", + "content_filter" => "content_filter", + "error" => "provider_error", + _ => "empty_response" + }; + yield return AssistantStreamEvent.Error("Exie stopped before providing an answer. Please try again.", failureCode); } else if (pendingSuggestedActions.Count > 0) { @@ -239,7 +274,7 @@ public async IAsyncEnumerable StreamAsync( if (!allowTools) { - yield return AssistantStreamEvent.Error("Exie could not finish using the available tool results. Try narrowing the question."); + yield return AssistantStreamEvent.Error("Exie could not finish using the available tool results. Try narrowing the question.", "tool_round_limit"); yield return AssistantStreamEvent.Done(); yield break; } @@ -293,6 +328,8 @@ public async IAsyncEnumerable StreamAsync( requireFinalAnswer = true; completedToolRounds++; + if (diagnostics is not null) + diagnostics.ToolRounds = completedToolRounds; continue; } @@ -316,8 +353,10 @@ public async IAsyncEnumerable StreamAsync( foreach (var toolCall in executableToolCalls) { string arguments = toolCall.Arguments.ToString(); + diagnostics?.StartTool(toolCall.Name); yield return AssistantStreamEvent.ToolCall(toolCall.Id, toolCall.Name, arguments); + long toolStarted = timeProvider.GetTimestamp(); string result; if (remainingToolCalls <= 0) { @@ -353,6 +392,7 @@ public async IAsyncEnumerable StreamAsync( result = await ExecuteToolAsync(toolCall.Name, arguments, request, cancellationToken); } + diagnostics?.RecordToolResult(result, timeProvider.GetElapsedTime(toolStarted).TotalMilliseconds); if (toolCall.Name == GetProjectSetupTool) configureHref = AssistantSuggestedActionParser.GetProjectSetupHref(result) ?? configureHref; @@ -372,6 +412,8 @@ public async IAsyncEnumerable StreamAsync( && !String.IsNullOrWhiteSpace(request.OrganizationId) && !String.IsNullOrWhiteSpace(request.ConversationId)) { + if (diagnostics is not null) + diagnostics.Stage = "conversation_save"; await assistantConversationService.AppendToolResultsAsync( userId, request.OrganizationId, @@ -381,10 +423,12 @@ await assistantConversationService.AppendToolResultsAsync( } completedToolRounds++; + if (diagnostics is not null) + diagnostics.ToolRounds = completedToolRounds; } } - private async Task SendRequestAsync(List messages, AssistantOptions options, string model, bool allowTools, AssistantChatRequest chatRequest, CancellationToken cancellationToken) + private async Task SendRequestAsync(List messages, AssistantOptions options, string model, bool allowTools, AssistantChatRequest chatRequest, AssistantProviderDiagnostics? diagnostics, CancellationToken cancellationToken) { var client = httpClientFactory.CreateClient(nameof(AssistantService)); using var providerRequest = new HttpRequestMessage(HttpMethod.Post, options.Endpoint); @@ -413,13 +457,15 @@ private async Task SendRequestAsync(List messages, providerRequest.Content = JsonContent.Create(payload); var response = await client.SendAsync(providerRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + diagnostics?.ObserveResponse(response); if (response.IsSuccessStatusCode) return response; - string detail = await response.Content.ReadAsStringAsync(cancellationToken); - logger.LogWarning("Assistant provider returned {StatusCode}: {Detail}", (int)response.StatusCode, detail); + // Provider error bodies can echo prompts or credentials. Status and generation ID + // provide diagnostic context without copying those bodies into application logs. + logger.LogWarning("Assistant provider returned HTTP {StatusCode} with generation {ProviderGenerationId}", (int)response.StatusCode, diagnostics?.GenerationId); response.Dispose(); - throw new AssistantProviderException($"The AI provider returned status {(int)response.StatusCode}."); + throw new AssistantProviderException($"The AI provider returned status {(int)response.StatusCode}.") { FailureCode = "provider_http_error" }; } private async Task ExecuteToolAsync( diff --git a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs new file mode 100644 index 0000000000..e33bd4d062 --- /dev/null +++ b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs @@ -0,0 +1,157 @@ +using System.Diagnostics; +using System.Text.Json; +using Exceptionless.Web.Mcp; + +namespace Exceptionless.Web.Assistant; + +internal sealed class AssistantTurnDiagnostics : IDisposable +{ + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly long _started; + private readonly Activity? _activity; + private readonly IDisposable? _scope; + private bool _finished; + private double? _firstTextDuration; + private string? _failureCode; + + public AssistantTurnDiagnostics(ILogger logger, TimeProvider timeProvider, string organizationId, string conversationId, string requestId) + { + _logger = logger; + _timeProvider = timeProvider; + _started = timeProvider.GetTimestamp(); + _activity = AppDiagnostics.StartActivity("assistant.turn"); + TurnId = Guid.NewGuid().ToString("N"); + OrganizationId = organizationId; + ConversationId = conversationId; + RequestId = requestId; + TraceId = Activity.Current?.TraceId.ToString(); + _activity?.SetTag("assistant.turn.id", TurnId); + _activity?.SetTag("organization.id", organizationId); + _activity?.SetTag("assistant.conversation.id", conversationId); + _scope = logger.BeginScope(new Dictionary + { + ["AssistantTurnId"] = TurnId, + ["OrganizationId"] = organizationId, + ["ConversationId"] = conversationId, + ["RequestId"] = requestId, + ["TraceId"] = TraceId + }); + } + + public string TurnId { get; } + public string OrganizationId { get; } + public string ConversationId { get; } + public string RequestId { get; } + public string? TraceId { get; } + public string? Model { get; set; } + public string Stage { get; set; } = "initializing"; + public int ProviderRequests { get; private set; } + public int ToolCalls { get; private set; } + public int ToolFailures { get; private set; } + public int ToolRounds { get; set; } + public int MalformedResponseRetries { get; set; } + public string? LastTool { get; private set; } + public string? LastToolError { get; private set; } + public AssistantProviderDiagnostics? Provider { get; private set; } + + public AssistantProviderDiagnostics StartProviderRequest(int inputCharacters, bool allowTools, CancellationToken cancellationToken) + { + Stage = "provider_request"; + ProviderRequests++; + Provider = new AssistantProviderDiagnostics(_logger, _timeProvider, this, inputCharacters, allowTools, cancellationToken); + return Provider; + } + + public void Observe(AssistantStreamEvent item) + { + if (item.Type == "error") + _failureCode ??= item.FailureCode ?? "response_error"; + if (item.Type == "text_delta" && !String.IsNullOrEmpty(item.Text)) + _firstTextDuration ??= ElapsedMilliseconds; + } + + public void StartTool(string name) + { + Stage = "tool_execution"; + LastTool = GetToolName(name); + ToolCalls++; + } + + public void RecordToolResult(string result, double durationMilliseconds) + { + using var document = JsonDocument.Parse(result); + var root = document.RootElement; + bool failed = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("ok", out var ok) && ok.ValueKind == JsonValueKind.False; + string? errorCode = null; + if (failed) + { + ToolFailures++; + errorCode = "tool_error"; + if (root.TryGetProperty("error", out var error) && error.ValueKind == JsonValueKind.Object + && error.TryGetProperty("code", out var code) && code.ValueKind == JsonValueKind.String) + errorCode = GetToolErrorCode(code.GetString()); + LastToolError = errorCode; + _logger.LogWarning("Assistant tool {ToolName} failed with {ToolErrorCode} in {DurationMs} ms for turn {AssistantTurnId}", + LastTool, errorCode, durationMilliseconds, TurnId); + } + + AppDiagnostics.AssistantToolDuration.Record(durationMilliseconds, + new("tool", LastTool), new("outcome", failed ? "failed" : "completed"), new("reason", errorCode ?? "none")); + } + + public void Finish(string outcome, string? failureCode = null, Exception? exception = null) + { + if (_finished) + return; + _finished = true; + string reason = failureCode ?? _failureCode ?? "none"; + _activity?.SetTag("assistant.outcome", outcome); + _activity?.SetTag("assistant.failure.reason", reason); + _activity?.SetTag("assistant.stage", Stage); + _activity?.SetTag("assistant.model", Model); + _activity?.SetTag("assistant.provider.requests", ProviderRequests); + _activity?.SetTag("assistant.tool.calls", ToolCalls); + if (outcome == "failed") + _activity?.SetStatus(ActivityStatusCode.Error, reason); + + AppDiagnostics.AssistantTurnDuration.Record(ElapsedMilliseconds, + new("outcome", outcome), new("reason", reason), new("stage", Stage)); + + // Include correlation in the message as well as the scope: the production console + // formatter does not render scope properties, and streaming errors retain HTTP 200. + var level = outcome == "failed" ? exception is null ? LogLevel.Warning : LogLevel.Error : LogLevel.Information; + _logger.Log(level, + "Assistant turn {AssistantTurnId} {Outcome}: reason={FailureReason} stage={Stage} duration={DurationMs} ms first_text={FirstTextDurationMs} ms organization={OrganizationId} conversation={ConversationId} request={RequestId} trace={TraceId} model={Model} provider_requests={ProviderRequests} tool_rounds={ToolRounds} tool_calls={ToolCalls} tool_failures={ToolFailures} last_tool={LastTool} last_tool_error={LastToolError} malformed_retries={MalformedResponseRetries} generation={ProviderGenerationId} provider_model={ProviderModel} provider={ProviderName} status={ProviderStatusCode} finish={ProviderFinishReason} usage_received={ProviderUsageReceived} reasoning_tokens={ReasoningTokens} exception_type={ExceptionType} exception_stack={ExceptionStackTrace}", + TurnId, outcome, reason, Stage, ElapsedMilliseconds, _firstTextDuration, OrganizationId, ConversationId, RequestId, TraceId, + Model, ProviderRequests, ToolRounds, ToolCalls, ToolFailures, LastTool, LastToolError, MalformedResponseRetries, + Provider?.GenerationId, Provider?.Model, Provider?.ProviderName, Provider?.StatusCode, Provider?.FinishReason, + Provider?.UsageReceived, Provider?.ReasoningTokens, exception?.GetType().FullName, exception?.StackTrace); + } + + private double ElapsedMilliseconds => _timeProvider.GetElapsedTime(_started).TotalMilliseconds; + + public void Dispose() + { + _scope?.Dispose(); + _activity?.Dispose(); + } + + private static string GetToolName(string name) => name switch + { + "get_event" or "get_stack" or "get_project_setup" or "get_stack_events" or "list_projects" or "search_stacks" + or "update_stack_status" or "snooze_stack" or "set_stack_critical" or "add_stack_reference_link" or "remove_stack_reference_link" => name, + _ => "unknown" + }; + + private static string GetToolErrorCode(string? code) => code switch + { + McpErrorCodes.ContextMismatch or McpErrorCodes.ContextRequired or McpErrorCodes.Forbidden or McpErrorCodes.InvalidClientPlatform + or McpErrorCodes.InvalidCursor or McpErrorCodes.InvalidDetailSize or McpErrorCodes.InvalidFilter or McpErrorCodes.InvalidGroupBy + or McpErrorCodes.InvalidId or McpErrorCodes.InvalidInterval or McpErrorCodes.InvalidLimit or McpErrorCodes.InvalidReferenceUrl + or McpErrorCodes.InvalidSnooze or McpErrorCodes.InvalidSort or McpErrorCodes.InvalidStatus or McpErrorCodes.InvalidTimeRange + or McpErrorCodes.InvalidVersion or McpErrorCodes.NotAccessible or McpErrorCodes.NotFound or McpErrorCodes.QueryFailed + or McpErrorCodes.UnknownFilterField or "tool_call_limit_reached" or "project_search_limit_reached" => code, + _ => "tool_error" + }; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts new file mode 100644 index 0000000000..d580ddd370 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts @@ -0,0 +1,102 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { submitFeatureUsage, submitLog } = vi.hoisted(() => ({ + submitFeatureUsage: vi.fn(() => Promise.resolve()), + submitLog: vi.fn(() => Promise.resolve()) +})); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); + +import { AssistantTurnTelemetry, trackAssistantEvent } from './assistant-telemetry'; + +const context = { + assistant_message_id: 'response-1', + conversation_id: 'conversation-1', + mode: 'sheet' as const, + organization_id: 'organization-1', + user_message_id: 'prompt-1' +}; + +describe('Exie session telemetry', () => { + beforeEach(() => { + submitFeatureUsage.mockReset(); + submitLog.mockReset(); + }); + + it('records one prompt and one assembled response with the same conversation context', () => { + const turn = new AssistantTurnTelemetry(context, 'What happened to checkout?', 'composer'); + turn.observe({ text: 'The checkout ', type: 'text_delta' }); + turn.observe({ arguments: '{"secret":"tool-argument"}', tool_call_id: 'tool-1', tool_name: 'get_event', type: 'tool_call' }); + turn.observe({ result: '{"ok":true,"data":{"secret":"tool-result"}}', tool_call_id: 'tool-1', type: 'tool_result' }); + turn.observe({ text: 'request timed out.', type: 'text_delta' }); + turn.observe({ type: 'done' }); + + expect(turn.finish()).toBe('completed'); + expect(turn.finish()).toBeUndefined(); + expect(submitLog).toHaveBeenCalledTimes(2); + expect(submitLog).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', 'What happened to checkout?', { + exie: expect.objectContaining({ ...context, prompt_source: 'composer', role: 'user' }) + }); + expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseCompleted', 'The checkout request timed out.', { + exie: expect.objectContaining({ ...context, outcome: 'completed', role: 'assistant', tool_calls: 1, tool_failures: 0 }) + }); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('tool-argument'); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('tool-result'); + }); + + it('records a streamed failure even when a done event follows it', () => { + const turn = new AssistantTurnTelemetry(context, 'Investigate', 'starter'); + turn.observe({ text: 'Partial answer', type: 'text_delta' }); + turn.observe({ message: 'Exie took too long.', type: 'error' }); + turn.observe({ type: 'done' }); + expect(turn.finish()).toBe('failed'); + expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseFailed', 'Partial answer', { + exie: expect.objectContaining({ error_message: 'Exie took too long.', reason: 'stream_error' }) + }); + }); + + it('distinguishes stopped responses and ignores late content after cancellation', () => { + const turn = new AssistantTurnTelemetry(context, 'Investigate', 'composer'); + turn.observe({ text: 'Partial', type: 'text_delta' }); + expect(turn.finish('organization_changed')).toBe('cancelled'); + turn.observe({ text: 'Late content', type: 'text_delta' }); + turn.observe({ type: 'done' }); + turn.finish(); + expect(submitLog).toHaveBeenCalledTimes(2); + expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseCancelled', 'Partial', { + exie: expect.objectContaining({ organization_id: 'organization-1', reason: 'organization_changed' }) + }); + }); + + it('does not count a silently interrupted stream as a successful response', () => { + const turn = new AssistantTurnTelemetry(context, 'Investigate', 'composer'); + turn.observe({ text: 'Partial', type: 'text_delta' }); + expect(turn.finish()).toBe('failed'); + expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseFailed', 'Partial', { + exie: expect.objectContaining({ reason: 'incomplete_stream', received_done: false }) + }); + }); + + it('bounds transcript size and marks truncation without storing each streamed chunk', () => { + const content = 'x'.repeat(20_000); + const turn = new AssistantTurnTelemetry(context, content, 'retry', { previous_conversation_id: 'previous-conversation' }); + turn.observe({ text: content, type: 'text_delta' }); + turn.observe({ type: 'done' }); + turn.finish(); + expect(submitLog).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', content.slice(0, 16_384), { + exie: expect.objectContaining({ message_characters: 20_000, message_truncated: true, previous_conversation_id: 'previous-conversation' }) + }); + expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseCompleted', content.slice(0, 16_384), { + exie: expect.objectContaining({ message_characters: 20_000, message_truncated: true, response_characters: 20_000, response_truncated: true }) + }); + }); + + it('keeps chat interactions working when telemetry cannot be submitted', async () => { + submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); + submitLog.mockRejectedValueOnce(new Error('offline')); + expect(() => trackAssistantEvent('assistant.ResponseHelpful', context)).not.toThrow(); + expect(() => new AssistantTurnTelemetry(context, 'Investigate', 'composer')).not.toThrow(); + await Promise.resolve(); + expect(submitFeatureUsage).toHaveBeenCalledOnce(); + expect(submitLog).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts new file mode 100644 index 0000000000..a615e701fb --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts @@ -0,0 +1,116 @@ +import { submitFeatureUsage, submitLog } from '$features/auth/exceptionless-session'; + +import type { AssistantStreamEvent } from './assistant-stream'; + +import { assistantToolResultFailed } from './assistant-tool-result'; + +const maximumMessageCharacters = 16_384; + +export type AssistantPromptSource = 'composer' | 'queued' | 'regenerate' | 'retry' | 'starter' | 'suggested_action'; + +export type AssistantStopReason = 'access_changed' | 'component_unmounted' | 'conversation_cleared' | 'organization_changed' | 'user_stopped'; +export interface AssistantTelemetryContext { + assistant_message_id?: string; + conversation_id: string; + mode: 'page' | 'sheet'; + organization_id?: string; + path?: string; + project_id?: string; + user_message_id?: string; +} +export type AssistantTurnOutcome = 'cancelled' | 'completed' | 'failed'; + +export class AssistantTurnTelemetry { + private content = ''; + private contentCharacters = 0; + private errorMessage: string | undefined; + private failureReason: string | undefined; + private finished = false; + private firstTextDuration: number | undefined; + private receivedDone = false; + private started = performance.now(); + private toolCalls = 0; + private toolFailures = 0; + + constructor( + readonly context: AssistantTelemetryContext, + prompt: string, + source: AssistantPromptSource, + details: Record = {} + ) { + trackAssistantEvent('assistant.MessageSent', context, { ...details, prompt_source: source, role: 'user' }, prompt); + } + + fail(message: string, reason: string): void { + this.errorMessage ??= message.slice(0, 2048); + this.failureReason ??= reason; + } + + finish(stopReason?: AssistantStopReason, details: Record = {}): AssistantTurnOutcome | undefined { + if (this.finished) { + return; + } + this.finished = true; + const outcome = stopReason ? 'cancelled' : this.failureReason || !this.receivedDone || this.contentCharacters === 0 ? 'failed' : 'completed'; + const reason = + stopReason ?? this.failureReason ?? (!this.receivedDone ? 'incomplete_stream' : this.contentCharacters === 0 ? 'empty_response' : undefined); + const feature = { cancelled: 'assistant.ResponseCancelled', completed: 'assistant.ResponseCompleted', failed: 'assistant.ResponseFailed' }[outcome]; + trackAssistantEvent( + feature, + this.context, + { + ...details, + duration_ms: Math.round(performance.now() - this.started), + error_message: this.errorMessage, + first_text_duration_ms: this.firstTextDuration, + message_characters: this.contentCharacters || this.errorMessage?.length || 0, + message_truncated: this.contentCharacters > maximumMessageCharacters, + outcome, + reason, + received_done: this.receivedDone, + response_characters: this.contentCharacters, + response_truncated: this.contentCharacters > maximumMessageCharacters, + role: 'assistant', + tool_calls: this.toolCalls, + tool_failures: this.toolFailures + }, + this.content || this.errorMessage || '' + ); + return outcome; + } + + observe(event: AssistantStreamEvent): void { + if (this.finished) { + return; + } + if (event.type === 'text_delta' && event.text) { + this.contentCharacters += event.text.length; + this.content += event.text.slice(0, Math.max(0, maximumMessageCharacters - this.content.length)); + this.firstTextDuration ??= Math.round(performance.now() - this.started); + } else if (event.type === 'tool_call') { + this.toolCalls++; + } else if (event.type === 'tool_result' && assistantToolResultFailed(event.result)) { + this.toolFailures++; + } else if (event.type === 'error') { + this.fail(event.message ?? 'Exie could not complete this request.', 'stream_error'); + } else if (event.type === 'done') { + this.receivedDone = true; + } + } +} + +export function trackAssistantEvent(feature: string, context: AssistantTelemetryContext, details: Record = {}, message?: string): void { + const properties = { + exie: { + ...context, + ...(message !== undefined && { message_characters: message.length, message_truncated: message.length > maximumMessageCharacters }), + ...details, + schema_version: 1 + } + }; + // Session/user identity, queueing, and filtering come from the existing SDK. + // A telemetry failure must not interrupt a chat or generate another telemetry event. + const submission = + message === undefined ? submitFeatureUsage(feature, properties) : submitLog(feature, message.slice(0, maximumMessageCharacters), properties); + void submission.catch(() => {}); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte index 25a8111fd1..5713045e9f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte @@ -1,7 +1,6 @@ @@ -66,7 +67,7 @@ {#snippet child({ props })} - {/snippet} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts index 71c594b132..291f7222bd 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts @@ -1,9 +1,7 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -const submitFeatureUsage = vi.hoisted(() => vi.fn(() => Promise.resolve())); const toast = vi.hoisted(() => ({ error: vi.fn(), success: vi.fn() })); -vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage })); vi.mock('svelte-sonner', () => ({ toast })); import AssistantMessageActions from './assistant-message-actions.svelte'; @@ -13,7 +11,6 @@ describe('AssistantMessageActions', () => { beforeEach(() => { writeText.mockClear(); - submitFeatureUsage.mockClear(); toast.error.mockClear(); toast.success.mockClear(); Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }); @@ -21,25 +18,25 @@ describe('AssistantMessageActions', () => { it('copies the complete message and regenerates the response', async () => { const onRegenerate = vi.fn(); - render(AssistantMessageActions, { props: { content: 'The answer', onRegenerate } }); + const onCopy = vi.fn(); + render(AssistantMessageActions, { props: { content: 'The answer', onCopy, onRegenerate } }); await fireEvent.click(screen.getByRole('button', { name: 'Copy message' })); await fireEvent.click(screen.getByRole('button', { name: 'Regenerate response' })); expect(writeText).toHaveBeenCalledWith('The answer'); + await waitFor(() => expect(onCopy).toHaveBeenCalledOnce()); expect(onRegenerate).toHaveBeenCalledOnce(); }); - it('records helpful feedback without including message contents', async () => { + it('reports feedback to the conversation owner for correlated telemetry', async () => { const onFeedback = vi.fn(); render(AssistantMessageActions, { props: { content: 'Sensitive answer', onFeedback, showFeedback: true } }); await fireEvent.click(screen.getByRole('button', { name: 'Good response' })); expect(onFeedback).toHaveBeenCalledWith('helpful'); - await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.ResponseHelpful')); expect(toast.success).toHaveBeenCalledWith('Marked as helpful.'); - expect(submitFeatureUsage).not.toHaveBeenCalledWith(expect.stringContaining('Sensitive answer')); }); it('disables regenerate while the callback is pending', async () => { @@ -54,14 +51,13 @@ describe('AssistantMessageActions', () => { await waitFor(() => expect(screen.getByRole('button', { name: 'Regenerate response' }).hasAttribute('disabled')).toBe(false)); }); - it('reports telemetry failure without reverting the feedback callback', async () => { + it('reports clearing feedback to the conversation owner', async () => { const onFeedback = vi.fn(); - submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); render(AssistantMessageActions, { props: { content: 'The answer', feedback: 'helpful', onFeedback, showFeedback: true } }); - await fireEvent.click(screen.getByRole('button', { name: 'Poor response' })); + await fireEvent.click(screen.getByRole('button', { name: 'Good response' })); - expect(onFeedback).toHaveBeenCalledWith('not-helpful'); - await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Your feedback is selected, but telemetry could not be sent.')); + expect(onFeedback).toHaveBeenCalledWith(undefined); + expect(toast.error).not.toHaveBeenCalled(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte index cca446c1a0..d600140c4b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte @@ -16,6 +16,7 @@ isLast?: boolean; isStreaming?: boolean; message: AssistantChatMessage; + onCopy?: () => void; onFeedback?: (feedback: AssistantFeedback | undefined) => void; onRegenerate?: () => Promise | void; onSuggestedAction?: (action: AssistantSuggestedAction) => void; @@ -27,6 +28,7 @@ isLast = false, isStreaming = false, message, + onCopy, onFeedback, onRegenerate, onSuggestedAction, @@ -40,7 +42,7 @@ {#if message.role === 'user'}
{message.content}
- +
{:else}
@@ -95,6 +97,7 @@ (); let abortController: AbortController | undefined; let handledPromptRequestId: string | undefined; + let activeTurn: AssistantTurnTelemetry | undefined; + let lastOutcome: AssistantTurnOutcome | undefined; + let wasVisible = false; + let previousMode: 'page' | 'sheet' | undefined; let latestAssistantMessage = $derived(messages.filter((message) => message.role === 'assistant').at(-1)); const suggestions = [ @@ -83,20 +95,53 @@ $effect(() => { if (accessState !== 'available') { - untrack(stopStreaming); + untrack(() => stopStreaming('access_changed')); } }); $effect(() => { const currentOrganizationId = organizationId; if (conversationOrganizationId !== currentOrganizationId) { - untrack(stopStreaming); + untrack(() => { + if (messages.length > 0) { + trackConversationEvent('assistant.ConversationLeft', { + reason: 'organization_changed' + }); + } + stopStreaming('organization_changed'); + }); messages = []; errorMessage = undefined; prompt = ''; conversationId = crypto.randomUUID(); conversationOrganizationId = currentOrganizationId; + lastOutcome = undefined; + } + }); + + $effect(() => { + const visible = mode === 'page' || open; + const currentMode = mode; + untrack(() => { + if (visible !== wasVisible) { + trackConversationEvent(visible ? 'assistant.Opened' : 'assistant.Closed'); + } else if (visible && currentMode !== previousMode) { + trackConversationEvent('assistant.ViewChanged', { + previous_mode: previousMode + }); + } + wasVisible = visible; + previousMode = currentMode; + }); + }); + + onDestroy(() => { + if (wasVisible) { + trackConversationEvent('assistant.ConversationLeft', { + reason: 'component_unmounted' + }); } + stopStreaming('component_unmounted'); }); $effect(() => { @@ -113,10 +158,12 @@ } handledPromptRequestId = promptRequest.id; - void submitPrompt(promptRequest.prompt); + void submitPrompt(promptRequest.prompt, { + source: 'queued' + }); }); - async function submitPrompt(value = prompt, isSuggestedAction = false, suggestedActionLabel?: string, suggestedActionPath?: string): Promise { + async function submitPrompt(value = prompt, options: { action?: AssistantSuggestedAction; source?: AssistantPromptSource } = {}): Promise { const content = value.trim(); if (!content || isStreaming) { return; @@ -131,32 +178,42 @@ errorMessage = undefined; const userMessage: AssistantChatMessage = { content, + conversationId, id: crypto.randomUUID(), - isSuggestedAction, + isSuggestedAction: options.source === 'suggested_action', role: 'user', - suggestedActionLabel, - suggestedActionPath, + suggestedActionLabel: options.action?.label, + suggestedActionPath: options.action?.sourcePath, tools: [] }; const assistantMessage: AssistantChatMessage = { content: '', + conversationId, id: crypto.randomUUID(), role: 'assistant', tools: [] }; const history = [...messages, userMessage]; messages = [...history, assistantMessage]; - await streamResponse(history, assistantMessage); + await streamResponse(history, assistantMessage, options.source ?? 'composer'); } - async function handleSuggestedAction(action: AssistantSuggestedAction): Promise { + async function handleSuggestedAction(action: AssistantSuggestedAction, message: AssistantChatMessage): Promise { + trackAssistantEvent('assistant.SuggestedActionSelected', getTelemetryContext(message), { + action_label: action.label, + action_type: action.href ? 'navigation' : 'prompt', + target_path: action.href?.split(/[?#]/)[0] + }); if (action.href) { open = false; await goto(action.href); return; } - await submitPrompt(action.prompt, true, action.label, action.sourcePath); + await submitPrompt(action.prompt, { + action, + source: 'suggested_action' + }); } async function regenerateResponse(assistantMessageId: string): Promise { @@ -174,41 +231,74 @@ return; } + const source = errorMessage ? 'retry' : 'regenerate'; + const previousConversationId = conversationId; + trackAssistantEvent('assistant.ResponseRegenerated', getTelemetryContext(messages[assistantMessageIndex]), { + prompt_source: source + }); errorMessage = undefined; const history = messages.slice(0, userMessageIndex + 1); + conversationId = crypto.randomUUID(); const replacement: AssistantChatMessage = { content: '', + conversationId, id: crypto.randomUUID(), role: 'assistant', tools: [] }; messages = [...history, replacement]; - conversationId = crypto.randomUUID(); - await streamResponse(history, replacement); + await streamResponse(history, replacement, source, { + previous_conversation_id: previousConversationId, + retry_of_message_id: assistantMessageId + }); } - async function streamResponse(history: AssistantChatMessage[], assistantMessage: AssistantChatMessage): Promise { + async function streamResponse( + history: AssistantChatMessage[], + assistantMessage: AssistantChatMessage, + source: AssistantPromptSource, + details: Record = {} + ): Promise { isStreaming = true; - abortController = new AbortController(); - await scrollToLatest('smooth', true); + const controller = new AbortController(); + abortController = controller; const requestPath = path ?? `${page.url.pathname}${page.url.search}`; + const userMessage = history.at(-1)!; + const telemetry = new AssistantTurnTelemetry( + { + ...getTelemetryContext(assistantMessage), + user_message_id: userMessage.id + }, + userMessage.content, + source, + { + ...details, + turn_index: history.filter((message) => message.role === 'user').length + } + ); + activeTurn = telemetry; + lastOutcome = undefined; + const request = createAssistantChatRequest(history, conversationId, organizationId, requestPath, projectId); try { + await scrollToLatest('smooth', true); const response = await fetch('/api/v2/assistant/chat', { - body: JSON.stringify(createAssistantChatRequest(history, conversationId, organizationId, requestPath, projectId)), + body: JSON.stringify(request), headers: { Authorization: `Bearer ${accessToken.current}`, 'Content-Type': 'application/json' }, method: 'POST', - signal: abortController.signal + signal: controller.signal }); if (!response.ok) { const problem = response.headers.get('content-type')?.includes('json') ? ((await response.json()) as { detail?: string; title?: string }) : undefined; - throw new Error(problem?.detail ?? problem?.title ?? `The assistant returned status ${response.status}.`); + const message = problem?.detail ?? problem?.title ?? `The assistant returned status ${response.status}.`; + telemetry.fail(message, `http_${response.status}`); + throw new Error(message); } if (!response.body) { @@ -216,19 +306,31 @@ } await readAssistantStream(response.body, async (event) => { + if (controller.signal.aborted) { + return; + } + telemetry.observe(event); applyStreamEvent(assistantMessage.id, event, requestPath); await scrollToLatest('auto'); }); } catch (error) { - if (error instanceof DOMException && error.name === 'AbortError') { + if (controller.signal.aborted || (error instanceof DOMException && error.name === 'AbortError')) { return; } errorMessage = error instanceof Error ? error.message : 'Exie could not complete this request.'; + telemetry.fail(errorMessage, 'request_error'); } finally { - isStreaming = false; - abortController = undefined; - await scrollToLatest('auto'); + const outcome = telemetry.finish(controller.signal.aborted ? 'user_stopped' : undefined, { + is_visible: mode === 'page' || open + }); + if (abortController === controller) { + lastOutcome = outcome ?? lastOutcome; + activeTurn = undefined; + isStreaming = false; + abortController = undefined; + await scrollToLatest('auto'); + } } } @@ -300,7 +402,13 @@ } } - function stopStreaming(): void { + function stopStreaming(reason: AssistantStopReason = 'user_stopped'): void { + const outcome = activeTurn?.finish(reason, { + is_visible: mode === 'page' || open + }); + if (outcome) { + lastOutcome = outcome; + } abortController?.abort(); if (!messages.some((message) => message.tools.some((tool) => tool.status === 'running'))) { return; @@ -320,13 +428,15 @@ } function clearConversation(): void { - stopStreaming(); + trackConversationEvent('assistant.ConversationCleared'); + stopStreaming('conversation_cleared'); messages = []; conversationId = crypto.randomUUID(); errorMessage = undefined; prompt = ''; isNearBottom = true; showScrollToBottom = false; + lastOutcome = undefined; } function collapseToSidePanel(): void { @@ -344,6 +454,19 @@ } function setMessageFeedback(messageId: string, feedback: AssistantFeedback | undefined): void { + const message = messages.find((message) => message.id === messageId); + if (!message) { + return; + } + const feature = + feedback === 'helpful' + ? 'assistant.ResponseHelpful' + : feedback === 'not-helpful' + ? 'assistant.ResponseNotHelpful' + : 'assistant.ResponseFeedbackCleared'; + trackAssistantEvent(feature, getTelemetryContext(message), { + feedback: feedback ?? 'cleared' + }); messages = messages.map((message) => message.id === messageId ? { @@ -354,6 +477,41 @@ ); } + function getTelemetryContext(message?: AssistantChatMessage): AssistantTelemetryContext { + return { + assistant_message_id: message?.role === 'assistant' ? message.id : undefined, + conversation_id: message?.conversationId ?? conversationId, + mode, + organization_id: conversationOrganizationId ?? organizationId, + path: (path ?? page.url.pathname).split(/[?#]/)[0], + project_id: projectId, + user_message_id: message?.role === 'user' ? message.id : undefined + }; + } + + function trackConversationEvent(feature: string, details: Record = {}): void { + trackAssistantEvent( + feature, + { + ...(activeTurn?.context ?? getTelemetryContext(latestAssistantMessage)), + mode + }, + { + ...details, + is_streaming: isStreaming, + last_feedback: latestAssistantMessage?.feedback, + last_outcome: lastOutcome, + message_count: messages.length + } + ); + } + + function handlePageHide(): void { + if (wasVisible || messages.length > 0) { + trackConversationEvent('assistant.PageLeft'); + } + } + async function scrollToLatest(behavior: 'auto' | 'smooth' = 'smooth', force = false): Promise { if (!force && !isNearBottom) { showScrollToBottom = true; @@ -370,6 +528,8 @@ } + + {#snippet conversation()}
{#if accessState !== 'available'} @@ -399,7 +559,10 @@ {#each suggestions as suggestion (suggestion)}
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index 81a1879ff7..86cebfa265 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -6,7 +6,14 @@ vi.mock('$features/auth/index.svelte', () => ({ accessToken: { current: 'access- vi.mock('$features/billing/stripe.svelte', () => ({ isStripeEnabled: () => true })); vi.mock('katex/dist/katex.min.css', () => ({})); const goto = vi.hoisted(() => vi.fn(() => Promise.resolve())); +const submitFeatureUsage = vi.hoisted(() => + vi.fn<(feature: string, properties?: Record, message?: string) => Promise>().mockResolvedValue(undefined) +); vi.mock('$app/navigation', () => ({ goto })); +const submitLog = vi.hoisted(() => + vi.fn<(source: string, message: string, properties?: Record) => Promise>().mockResolvedValue(undefined) +); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); import AssistantPanel from './assistant-panel.svelte'; @@ -30,6 +37,145 @@ describe('AssistantPanel', () => { expect(screen.getByText('Bring Exie onto your team')).toBeTruthy(); }); + it('records the prompt, response, and feedback under the same conversation and message IDs', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response('{"type":"text_delta","text":"The answer"}\n{"type":"done"}\n')) + ); + render(AssistantPanel, { + props: { + open: true, + organizationId: 'organization-1', + path: '/next/stack/stack-1?filter=private', + promptRequest: { id: 'prompt-request', prompt: 'My question' } + } + }); + await screen.findByText('The answer'); + await screen.findByRole('button', { name: 'Good response' }); + await fireEvent.click(screen.getByRole('button', { name: 'Good response' })); + await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.ResponseHelpful', expect.anything())); + + const prompt = eventData('assistant.MessageSent'); + expect(prompt).toMatchObject({ organization_id: 'organization-1', path: '/next/stack/stack-1', prompt_source: 'queued', role: 'user' }); + expect(eventData('assistant.ResponseCompleted')).toMatchObject({ + assistant_message_id: prompt.assistant_message_id, + conversation_id: prompt.conversation_id, + is_visible: true, + outcome: 'completed' + }); + expect(eventData('assistant.ResponseHelpful')).toMatchObject({ + assistant_message_id: prompt.assistant_message_id, + conversation_id: prompt.conversation_id + }); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseHelpful')).toHaveLength(1); + expect(submitLog).toHaveBeenCalledWith('assistant.MessageSent', 'My question', expect.anything()); + expect(submitLog).toHaveBeenCalledWith('assistant.ResponseCompleted', 'The answer', expect.anything()); + }); + + it('links a retry to the failed response across the new server conversation', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response('{"type":"error","message":"Provider timed out"}\n{"type":"done"}\n')) + .mockResolvedValueOnce(new Response('{"type":"text_delta","text":"Recovered answer"}\n{"type":"done"}\n')); + vi.stubGlobal('fetch', fetchMock); + render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } } + }); + await screen.findByText('Provider timed out'); + await fireEvent.click(await screen.findByRole('button', { name: 'Retry' })); + await screen.findByText('Recovered answer'); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); + + const failed = eventData('assistant.ResponseFailed'); + const retried = eventData('assistant.MessageSent', 1); + expect(retried).toMatchObject({ + previous_conversation_id: failed.conversation_id, + prompt_source: 'retry', + retry_of_message_id: failed.assistant_message_id + }); + expect(retried.conversation_id).not.toBe(failed.conversation_id); + expect(submitLog.mock.calls.filter(([feature]) => feature === 'assistant.ResponseFailed')).toHaveLength(1); + }); + + it('records closing while waiting without cancelling a response that finishes in the background', async () => { + let streamController: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(controller) { + streamController = controller; + } + }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(stream)) + ); + const props = { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } }; + const view = render(AssistantPanel, { props }); + await screen.findByRole('button', { name: 'Stop generating' }); + await view.rerender({ ...props, open: false }); + await waitFor(() => expect(eventData('assistant.Closed').is_streaming).toBe(true)); + streamController!.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Background answer"}\n{"type":"done"}\n')); + streamController!.close(); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').is_visible).toBe(false)); + expect(submitLog.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); + }); + + it('records one cancellation with the original organization when organization context changes', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async (_input: RequestInfo | URL, init?: RequestInit) => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Partial answer"}\n')); + init?.signal?.addEventListener('abort', () => controller.error(new DOMException('Aborted', 'AbortError'))); + } + }) + ) + ) + ); + const view = render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } } + }); + await screen.findByText('Partial answer'); + await view.rerender({ open: true, organizationId: 'organization-2' }); + await waitFor(() => expect(eventData('assistant.ResponseCancelled').reason).toBe('organization_changed')); + expect(eventData('assistant.ResponseCancelled')).toMatchObject({ organization_id: 'organization-1' }); + expect(submitLog.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); + expect(submitLog.mock.calls.some(([feature]) => feature === 'assistant.ResponseCompleted')).toBe(false); + expect(screen.queryByText('Partial answer')).toBeNull(); + }); + + it('records an explicit stop once and distinguishes it from leaving the page', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async (_input: RequestInfo | URL, init?: RequestInit) => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Partial answer"}\n')); + init?.signal?.addEventListener('abort', () => controller.error(new DOMException('Aborted', 'AbortError'))); + } + }) + ) + ) + ); + render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } } + }); + await screen.findByText('Partial answer'); + await fireEvent(window, new Event('pagehide')); + expect(eventData('assistant.PageLeft')).toMatchObject({ is_streaming: true }); + expect(submitLog.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); + + await fireEvent.click(screen.getByRole('button', { name: 'Stop generating' })); + await screen.findByRole('button', { name: 'Send message' }); + expect(eventData('assistant.ResponseCancelled')).toMatchObject({ outcome: 'cancelled', reason: 'user_stopped' }); + expect(submitLog.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); + expect(screen.getByText('Partial answer')).toBeTruthy(); + }); + it('renders as a full-page chat and opens the side panel when collapsed', async () => { const onCollapse = vi.fn(); render(AssistantPanel, { @@ -83,6 +229,7 @@ describe('AssistantPanel', () => { expect(screen.getByText('The conversation is still here.')).toBeTruthy(); expect(screen.getByRole('button', { name: 'Clear conversation' }).hasAttribute('disabled')).toBe(false); expect(fetchMock).toHaveBeenCalledOnce(); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.Closed')).toBe(false); }); afterEach(() => { @@ -236,3 +383,9 @@ describe('AssistantPanel', () => { expect(fetchMock).toHaveBeenCalledOnce(); }); }); + +function eventData(feature: string, index = 0): Record { + const properties = + submitFeatureUsage.mock.calls.filter(([name]) => name === feature)[index]?.[1] ?? submitLog.mock.calls.filter(([name]) => name === feature)[index]?.[2]; + return (properties?.exie ?? {}) as Record; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts index 856a88ef15..b186a40d21 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts @@ -10,6 +10,7 @@ export type AssistantAccessState = 'available' | 'disabled' | 'error' | 'loading export interface AssistantChatMessage { content: string; + conversationId?: string; feedback?: AssistantFeedback; id: string; isSuggestedAction?: boolean; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts new file mode 100644 index 0000000000..0197aeef8b --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts @@ -0,0 +1,51 @@ +import { Exceptionless } from '@exceptionless/browser'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('$app/environment', () => ({ browser: true })); +vi.mock('@exceptionless/browser', async () => { + const { Configuration, ExceptionlessClient } = await import('@exceptionless/core'); + const config = new Configuration(); + config.apiKey = 'local-test-key'; + config.serverUrl = 'https://localhost'; + config.defaultTags.push('UI', 'Svelte'); + config.useSessions(false); + // Exercise real event builders and plugins, without starting timers or sending events. + config.services.queue.enqueue = vi.fn().mockResolvedValue(undefined); + return { Exceptionless: new ExceptionlessClient(config) }; +}); + +import { setUserIdentity, submitFeatureUsage, submitLog } from './exceptionless-session'; + +describe('Exceptionless session events', () => { + beforeEach(() => { + vi.mocked(Exceptionless.config.services.queue.enqueue).mockClear(); + }); + + it('keeps transcript logs and feedback attached to the existing user session', async () => { + await setUserIdentity('exie-test-user'); + const properties = { exie: { conversation_id: 'conversation-1', role: 'user' } }; + await submitLog('assistant.MessageSent', 'Why did checkout fail?', properties); + await submitFeatureUsage('assistant.ResponseHelpful', properties); + + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events).toHaveLength(3); + expect(events[0]).toMatchObject({ type: 'session' }); + expect(events[1]).toMatchObject({ + data: properties, + message: 'Why did checkout fail?', + source: 'assistant.MessageSent', + type: 'log' + }); + expect(events[2]).toMatchObject({ data: properties, source: 'assistant.ResponseHelpful', type: 'usage' }); + for (const event of events) { + expect(event.data?.['@user']).toMatchObject({ identity: 'exie-test-user' }); + expect(event.tags).toEqual(expect.arrayContaining(['UI', 'Svelte'])); + } + }); + + it('preserves ordinary feature usage submissions without extended data', async () => { + await submitFeatureUsage('project.Created'); + expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledOnce(); + expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledWith(expect.objectContaining({ source: 'project.Created', type: 'usage' })); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts index 08ee51f5dd..8bc6f6f88e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts @@ -51,13 +51,31 @@ export async function setUserIdentity(userId: string, userName?: string): Promis * Submits a feature usage event for telemetry tracking. * Mirrors the legacy Angular $ExceptionlessClient.submitFeatureUsage pattern. */ -export async function submitFeatureUsage(feature: string): Promise { +export async function submitFeatureUsage(feature: string, properties?: Record): Promise { const Exceptionless = await getExceptionless(); if (!Exceptionless) { return; } - await Exceptionless.submitFeatureUsage(feature); + const event = Exceptionless.createFeatureUsage(feature); + for (const [name, value] of Object.entries(properties ?? {})) { + event.setProperty(name, value); + } + await event.submit(); +} + +/** Submits a readable log entry in the user's existing Exceptionless session. */ +export async function submitLog(source: string, message: string, properties?: Record): Promise { + const Exceptionless = await getExceptionless(); + if (!Exceptionless) { + return; + } + + const event = Exceptionless.createLog(source, message); + for (const [name, value] of Object.entries(properties ?? {})) { + event.setProperty(name, value); + } + await event.submit(); } async function getExceptionless() { diff --git a/src/Exceptionless.Web/appsettings.yml b/src/Exceptionless.Web/appsettings.yml index 785c089916..1273009d3b 100644 --- a/src/Exceptionless.Web/appsettings.yml +++ b/src/Exceptionless.Web/appsettings.yml @@ -9,6 +9,7 @@ Serilog: #Exceptionless.Core.Repositories.StackRepository: Verbose #Exceptionless.Core.Repositories: Verbose Exceptionless.Web.Program: Information + Exceptionless.Web.Assistant: Information Exceptionless.Web.Security.ApiKeyAuthenticationHandler: Warning Foundatio.Metrics: Warning Foundatio.Utility.ScheduledTimer: Warning diff --git a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs new file mode 100644 index 0000000000..e08de497bc --- /dev/null +++ b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs @@ -0,0 +1,345 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Net; +using System.Text.Json; +using Exceptionless.Core; +using Exceptionless.Web.Api.Endpoints; +using Exceptionless.Web.Assistant; +using Foundatio.Caching; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using OpenTelemetry.Trace; +using Xunit; + +namespace Exceptionless.Tests.Assistant; + +public sealed class AssistantDiagnosticsTests +{ + [Fact] + public void AddApm_ExieActivitySource_CreatesExportableSpans() + { + // The APM source file is linked into both Web and Job. Select the Web copy + // explicitly to exercise its real registration without ambiguous type references. + var assembly = typeof(AssistantService).Assembly; + var configType = assembly.GetType("OpenTelemetry.ApmConfig", throwOnError: true)!; + var config = Activator.CreateInstance(configType, new ConfigurationBuilder().Build(), "test", "1.0", false); + var builder = new HostBuilder(); + assembly.GetType("OpenTelemetry.ApmExtensions", throwOnError: true)! + .GetMethod("AddApm")!.Invoke(null, [builder, config]); + using var host = builder.Build(); + _ = host.Services.GetRequiredService(); + + using var activity = AppDiagnostics.StartActivity("assistant.turn"); + + Assert.NotNull(activity); + Assert.True(activity.IsAllDataRequested); + } + + [Theory] + [InlineData("empty_response")] + [InlineData("output_limit")] + [InlineData("malformed_response")] + [InlineData("tool_round_limit")] + [InlineData("usage_limit")] + public async Task WriteResponseAsync_StreamedError_RecordsCorrelatedFailureWithoutChangingResponse(string reason) + { + var logger = new RecordingAssistantLogger(); + var time = new FakeTimeProvider(); + using var diagnostics = new AssistantTurnDiagnostics(logger, time, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + time.Advance(TimeSpan.FromSeconds(12)); + + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([AssistantStreamEvent.Error("private error detail", reason), AssistantStreamEvent.Done()]), + CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(LogLevel.Warning, entry.Level); + Assert.Equal("failed", entry.Properties["Outcome"]); + Assert.Equal(reason, entry.Properties["FailureReason"]); + Assert.Equal(12_000d, entry.Properties["DurationMs"]); + Assert.Equal("organization-id", entry.Properties["OrganizationId"]); + Assert.Equal("conversation-id", entry.Properties["ConversationId"]); + Assert.Equal("request-id", entry.Properties["RequestId"]); + Assert.Equal(diagnostics.TurnId, entry.Properties["AssistantTurnId"]); + Assert.DoesNotContain("private error detail", entry.Message); + Assert.Equal(1, Assert.Single(recorder.Records).Increment.Failed); + + Assert.Equal(StatusCodes.Status200OK, context.Response.StatusCode); + var events = await ReadEventsAsync(context); + Assert.Equal("private error detail", events[0].GetProperty("message").GetString()); + Assert.False(events[0].TryGetProperty("failure_code", out _)); + Assert.False(events[0].TryGetProperty("FailureCode", out _)); + Assert.Equal("done", events[1].GetProperty("type").GetString()); + } + + [Theory] + [InlineData(false, true, "provider_stream", "failed", "turn_timeout")] + [InlineData(false, false, "provider_stream", "failed", "provider_timeout")] + [InlineData(true, true, "provider_stream", "cancelled", "client_disconnected")] + [InlineData(false, false, "tool_execution", "failed", "operation_cancelled")] + public async Task WriteResponseAsync_Cancellation_DistinguishesClientAndServer( + bool clientDisconnected, bool deadlineExpired, string stage, string outcome, string reason) + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.Stage = stage; + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + using var cancellation = new CancellationTokenSource(); + if (deadlineExpired) + await cancellation.CancelAsync(); + if (clientDisconnected) + context.RequestAborted = cancellation.Token; + + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([], new OperationCanceledException("private provider detail")), + CreateUsageService(cache, recorder), "organization-id", diagnostics, cancellation.Token); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(outcome, entry.Properties["Outcome"]); + Assert.Equal(reason, entry.Properties["FailureReason"]); + Assert.Equal(stage, entry.Properties["Stage"]); + Assert.DoesNotContain("private provider detail", entry.Message); + var usage = Assert.Single(recorder.Records).Increment; + Assert.Equal(clientDisconnected ? 0 : 1, usage.Failed); + Assert.Equal(clientDisconnected ? 1 : 0, usage.Cancelled); + var events = await ReadEventsAsync(context); + if (clientDisconnected) + Assert.Empty(events); + else + Assert.Equal("Exie took too long to complete this response. Try narrowing the question.", Assert.Single(events).GetProperty("message").GetString()); + } + + [Fact] + public async Task WriteResponseAsync_ProviderException_RecordsReasonWithoutLoggingProviderMessage() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([], new AssistantProviderException("private provider detail") { FailureCode = "provider_http_error" }), + CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(LogLevel.Error, entry.Level); + Assert.Equal("provider_http_error", entry.Properties["FailureReason"]); + Assert.Equal(typeof(AssistantProviderException).FullName, entry.Properties["ExceptionType"]); + Assert.DoesNotContain("private provider detail", entry.Message); + Assert.Equal(1, Assert.Single(recorder.Records).Increment.Failed); + } + + [Fact] + public async Task WriteResponseAsync_Success_RecordsCompletionAndFirstTextWithoutLoggingAnswer() + { + var logger = new RecordingAssistantLogger(); + var time = new FakeTimeProvider(); + using var diagnostics = new AssistantTurnDiagnostics(logger, time, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + time.Advance(TimeSpan.FromSeconds(3)); + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([AssistantStreamEvent.TextDelta("private answer"), AssistantStreamEvent.Done()]), + CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken); + + var entry = Assert.Single(logger.Entries); + Assert.Equal("completed", entry.Properties["Outcome"]); + Assert.Equal("none", entry.Properties["FailureReason"]); + Assert.Equal(3000d, entry.Properties["FirstTextDurationMs"]); + Assert.DoesNotContain("private answer", entry.Message); + Assert.Equal(1, Assert.Single(recorder.Records).Increment.Completed); + } + + [Fact] + public void Finish_FailedTurn_RecordsErrorSpanAndBoundedMetricTagsOnce() + { + var activities = new ConcurrentQueue(); + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == AppDiagnostics.ActivitySource.Name, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + SampleUsingParentId = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = activity => activities.Enqueue(activity) + }; + ActivitySource.AddActivityListener(listener); + var measurements = new List>(); + string? turnId = null; + using var meterListener = new MeterListener + { + InstrumentPublished = (instrument, current) => + { + if (instrument.Name == "ex.assistant.turn.duration") + current.EnableMeasurementEvents(instrument); + } + }; + meterListener.SetMeasurementEventCallback((_, _, tags, _) => + { + if (Activity.Current?.GetTagItem("assistant.turn.id") as string == turnId) + measurements.Add(tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value)); + }); + meterListener.Start(); + var logger = new RecordingAssistantLogger(); + using (var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id")) + { + turnId = diagnostics.TurnId; + diagnostics.Stage = "provider_stream"; + diagnostics.Finish("failed", "turn_timeout"); + diagnostics.Finish("completed"); + } + + var measurement = Assert.Single(measurements); + Assert.Equal(3, measurement.Count); + Assert.Equal("failed", measurement["outcome"]); + Assert.Equal("turn_timeout", measurement["reason"]); + Assert.Equal("provider_stream", measurement["stage"]); + var activity = Assert.Single(activities, activity => activity.GetTagItem("assistant.turn.id") as string == turnId); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("turn_timeout", activity.StatusDescription); + Assert.Single(logger.Entries); + } + + [Fact] + public void ObserveChunk_OutputLimit_RecordsGenerationAndReasoningWithoutContent() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.Model = "configured-model"; + using var provider = diagnostics.StartProviderRequest(1000, true, TestContext.Current.CancellationToken); + using var response = new HttpResponseMessage(HttpStatusCode.OK); + response.Headers.Add("X-Generation-Id", "gen-header"); + provider.ObserveResponse(response); + using var document = JsonDocument.Parse(""" + {"id":"gen-stream","model":"resolved-model","provider":"Example Provider", + "choices":[{"delta":{"reasoning":"private reasoning"},"finish_reason":"length"}], + "usage":{"prompt_tokens":100,"completion_tokens":2048,"completion_tokens_details":{"reasoning_tokens":2048}}} + """); + provider.ObserveChunk(document.RootElement); + provider.Complete(0, 0, true); + + Assert.Equal("gen-stream", provider.GenerationId); + Assert.Equal("length", provider.FinishReason); + Assert.Equal(2048, provider.ReasoningTokens); + var entry = Assert.Single(logger.Entries); + Assert.Equal(LogLevel.Warning, entry.Level); + Assert.Equal("output_limit", entry.Properties["ProviderOutcome"]); + Assert.Equal("resolved-model", entry.Properties["ProviderModel"]); + Assert.Equal(100L, entry.Properties["PromptTokens"]); + Assert.Equal(2048L, entry.Properties["CompletionTokens"]); + Assert.DoesNotContain("private reasoning", entry.Message); + } + + [Fact] + public void ObserveChunk_UnexpectedMetadata_DoesNotThrowOrLogUnboundedValues() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + using var provider = diagnostics.StartProviderRequest(1000, true, TestContext.Current.CancellationToken); + using var document = JsonDocument.Parse(""" + {"id":"private\nvalue","model":{"unexpected":true},"provider":null, + "choices":[{"finish_reason":"unexpected-provider-string"}], + "usage":{"completion_tokens_details":{"reasoning_tokens":"not-a-number"}}} + """); + provider.ObserveChunk(document.RootElement); + provider.Complete(10, 0, true); + + Assert.Null(provider.GenerationId); + Assert.Null(provider.Model); + Assert.Null(provider.ReasoningTokens); + Assert.Equal("unknown", provider.FinishReason); + Assert.DoesNotContain("private", Assert.Single(logger.Entries).Message); + } + + [Fact] + public void RecordToolResult_FailedTool_RecordsCodeWithoutArgumentsOrResult() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.StartTool("search_stacks"); + diagnostics.RecordToolResult("""{"ok":false,"error":{"code":"invalid_filter","message":"private filter"}}""", 100); + diagnostics.StartTool("private-hallucinated-tool-name"); + diagnostics.RecordToolResult("""{"ok":false,"error":{"code":"private-error-code","message":"private details"}}""", 100); + diagnostics.Finish("completed"); + + Assert.Equal(2, diagnostics.ToolFailures); + Assert.Equal("unknown", diagnostics.LastTool); + Assert.Equal("tool_error", diagnostics.LastToolError); + Assert.Equal("invalid_filter", logger.Entries[0].Properties["ToolErrorCode"]); + Assert.All(logger.Entries, entry => Assert.DoesNotContain("private", entry.Message)); + Assert.Equal("completed", logger.Entries[^1].Properties["Outcome"]); + } + + [Theory] + [InlineData("null")] + [InlineData("[]")] + [InlineData("\"text\"")] + public void RecordToolResult_NonObjectResult_DoesNotInterruptTheTurn(string result) + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.StartTool("get_event"); + diagnostics.RecordToolResult(result, 100); + + Assert.Equal(0, diagnostics.ToolFailures); + Assert.Empty(logger.Entries); + } + + private static DefaultHttpContext CreateHttpContext() + { + var context = new DefaultHttpContext(); + context.Response.Body = new MemoryStream(); + return context; + } + + private static async Task ReadEventsAsync(HttpContext context) + { + context.Response.Body.Position = 0; + using var reader = new StreamReader(context.Response.Body, leaveOpen: true); + string content = await reader.ReadToEndAsync(TestContext.Current.CancellationToken); + return content.Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(line => + { + using var document = JsonDocument.Parse(line); + return document.RootElement.Clone(); + }).ToArray(); + } + + private static AssistantUsageService CreateUsageService(ICacheClient cache, RecordingAssistantUsageRecorder recorder) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost" }) + .Build()); + return new AssistantUsageService(cache, null!, recorder, options, TimeProvider.System, NullLogger.Instance); + } + + private static async IAsyncEnumerable StreamEvents(AssistantStreamEvent[] events, Exception? exception = null) + { + await Task.Yield(); + if (exception is not null) + throw exception; + foreach (var item in events) + yield return item; + } +} + +internal sealed class RecordingAssistantLogger : ILogger +{ + public List Entries { get; } = []; + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + => Entries.Add(new AssistantLogEntry(logLevel, formatter(state, exception), + ((IEnumerable>)(object)state!).ToDictionary(property => property.Key, property => property.Value))); +} + +internal sealed record AssistantLogEntry(LogLevel Level, string Message, Dictionary Properties); diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 35ce5e8d9b..fbc8768e71 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -16,6 +16,7 @@ using Foundatio.Serializer; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Xunit; @@ -986,6 +987,72 @@ public async Task StreamAsync_EmptyResponse_EmitsClearErrorAndCompletion() item => Assert.Equal("done", item.Type)); } + [Theory] + [InlineData("length", "output_limit")] + [InlineData("content_filter", "content_filter")] + [InlineData("stop", "empty_response")] + public async Task StreamAsync_EmptyProviderAnswer_RecordsProviderReasonAndGeneration(string finishReason, string failureCode) + { + string payload = JsonSerializer.Serialize(new + { + id = "gen-empty-answer", + model = "resolved-model", + choices = new[] { new { delta = new { content = "" }, finish_reason = finishReason } }, + usage = new { prompt_tokens = 100, completion_tokens = 2048, completion_tokens_details = new { reasoning_tokens = 2048 } } + }); + var handler = new StubHttpMessageHandler($"data: {payload}\n\ndata: [DONE]\n\n"); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(handler, options); + var events = new List(); + + await foreach (var item in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + events.Add(item); + + Assert.Equal(failureCode, Assert.Single(events, item => item.Type == "error").FailureCode); + Assert.Equal("gen-empty-answer", diagnostics.Provider?.GenerationId); + Assert.Equal("resolved-model", diagnostics.Provider?.Model); + Assert.Equal(2048, diagnostics.Provider?.ReasoningTokens); + Assert.Equal(1, diagnostics.ProviderRequests); + Assert.DoesNotContain(logger.Entries, entry => entry.Message.Contains("private question", StringComparison.Ordinal)); + } + + [Fact] + public async Task StreamAsync_HttpRejection_RecordsStatusWithoutLoggingProviderErrorBody() + { + var handler = new RejectedHttpMessageHandler(HttpStatusCode.TooManyRequests); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(handler, options, logger: logger); + + var exception = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + { + } + }); + + Assert.Equal("provider_http_error", exception.FailureCode); + Assert.Equal(429, diagnostics.Provider?.StatusCode); + Assert.Contains(logger.Entries, entry => entry.Properties.TryGetValue("StatusCode", out var status) && status is 429); + Assert.All(logger.Entries, entry => + { + Assert.DoesNotContain("Rejected", entry.Message); + Assert.DoesNotContain("private question", entry.Message); + Assert.DoesNotContain("test-key", entry.Message); + }); + } + [Fact] public async Task StreamAsync_RawDsmlResponse_RetriesWithoutEmittingMarkup() { @@ -1229,7 +1296,8 @@ private static AssistantService CreateAssistantService( ICacheClient? cache = null, ILockProvider? lockProvider = null, AssistantUsageService? usageService = null, - AssistantModelSettingsService? modelSettingsService = null) + AssistantModelSettingsService? modelSettingsService = null, + ILogger? logger = null) { cache ??= new InMemoryCacheClient(new InMemoryCacheClientOptions { @@ -1256,7 +1324,7 @@ private static AssistantService CreateAssistantService( modelSettingsService, usageService, TimeProvider.System, - NullLogger.Instance); + logger ?? NullLogger.Instance); } private static AssistantModelSettingsService CreateAssistantModelSettingsService(AppOptions appOptions) diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index fbc6d8e9e2..fb4fac6257 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -19,3 +19,78 @@ dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll \ ``` Set `EX_Assistant__Model` and `EX_Assistant__Endpoint` to evaluate a candidate model or compatible provider. Use a dedicated provider key with a small monthly hard limit; the tests never print the key. + +## Runtime diagnostics + +Every accepted Exie turn emits one structured completion log with `Outcome` (`completed`, `failed`, or `cancelled`), `FailureReason`, `Stage`, duration, time to first answer text, model, provider request count, tool rounds, and tool failures. `AssistantTurnId`, `OrganizationId`, `ConversationId`, `RequestId`, and `TraceId` correlate a turn across logs and traces. These identifiers are also included in the rendered message so Kubernetes console logs remain useful without a structured log viewer. + +The `Exceptionless.Web.Assistant` logging override retains Information-level turn and provider summaries even when the production default is Warning. Failed turns log at Warning; unexpected/provider exceptions log at Error. Client disconnects remain cancellations and log at Information. A `completed` turn means the response finished without a streamed error; it does not establish that the answer was useful or correct. Keep running the quality evaluations above when changing the model, prompt, or tools. + +Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. + +Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events separately capture the user-visible conversation, as described below. + +| Failure reason | Investigation | +| --- | --- | +| `turn_timeout` | The shared 120-second turn deadline expired; inspect the last stage, provider duration, and tool progress. | +| `provider_timeout` | A provider operation was cancelled before the shared turn deadline expired. | +| `operation_cancelled` | A non-provider operation was cancelled before the shared turn deadline expired; inspect the stage. | +| `provider_http_error`, `provider_error`, `provider_transport_error`, `provider_stream_error` | Check status, generation ID, provider, and whether the stream completed. | +| `empty_response` | The provider returned neither answer text nor tool calls. | +| `output_limit` | The provider returned no answer text and reported `finish_reason=length`; inspect reasoning and completion tokens before changing budgets. | +| `content_filter` | An empty answer ended with the provider's content-filter finish reason. | +| `malformed_response` | Internal provider markup remained after the existing recovery retry. | +| `tool_round_limit` | The provider continued requesting tools after the final-answer instruction. | +| `usage_limit`, `context_limit` | A turn reached an organization usage limit or the conversation context bound. | +| `invalid_provider_response`, `response_write_error`, `tool_execution_error`, `internal_error` | Inspect the stage, exception type/stack, and correlated trace. | + +Returned tool errors are logged with the tool name and error code, even when the model recovers and completes the turn. Provider `output_limit` or `incomplete_stream` warnings can also accompany a completed turn when text was returned. Diagnostics preserve the existing response and accounting behavior; they do not automatically retry tools or change output budgets. + +The existing `ex.assistant.turn.outcomes` metric remains unchanged. Additional histograms provide immediate duration and outcome counts: + +| Instrument | Dimensions | +| --- | --- | +| `ex.assistant.turn.duration` (ms) | `outcome`, `reason`, `stage` | +| `ex.assistant.provider.duration` (ms) | `outcome` | +| `ex.assistant.tool.duration` (ms) | `tool`, `outcome`, `reason` | + +Track the completed share of completed + failed turns, failure counts by reason, cancellation rate separately, and turn/provider latency percentiles. Histogram counts can supply the immediate success-rate denominator; durable usage totals may lag. Failed `assistant.turn` spans carry error status even though the enclosing HTTP response is 200. Traces follow the deployment's existing sampling policy, so use logs and metrics for unsampled failures. + +In the hosted Helm deployment, the `ex-prod-app` workload runs `Exceptionless.Web` and serves in-app Exie requests; `ex-prod-api` primarily serves collector API traffic. Start with organization and UTC time, then follow the turn ID and generation IDs. Retained pod logs may not cover replaced pods or old conversations, and counters alone cannot reconstruct a historical failure reason. + +The diagnostics and provider-stream tests run locally without Aspire or billable provider requests: + +```powershell +dotnet build tests/Exceptionless.Tests/Exceptionless.Tests.csproj --maxcpucount:1 +dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll --filter-namespace Exceptionless.Tests.Assistant --filter-not-class Exceptionless.Tests.Assistant.AssistantQualityEvaluationTests --progress off +``` + +## Conversation session events + +The Svelte app submits Exie events through the existing Exceptionless browser client. They share the signed-in user's session, client configuration, queue, tags, and event exclusions. They go to the app's configured telemetry project. Starting a conversation does not create a separate user session. + +Each submitted prompt produces an `assistant.MessageSent` log event. The assembled response produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` log event. The log message contains the prompt, answer, or partial answer; a failure without answer text uses the error displayed to the user. Native log summaries make these messages readable in the existing session timeline. Messages are capped at 16,384 characters, with length and truncation metadata. Drafts, individual streamed chunks, reasoning, and raw tool arguments/results are not submitted. + +Events carry an `exie` extended-data object with `schema_version: 1`, conversation and message IDs, organization/project context, page path without query/fragment, and page/sheet mode. The conversation ID matches the server diagnostics. Turn summaries also include outcome, elapsed time, time to first text, tool counts/failures, and whether the chat was visible when the turn finished. Retries link the new server conversation back through `previous_conversation_id` and `retry_of_message_id`. + +Interactions remain feature usage events: + +| Event source | Meaning | +| --- | --- | +| `assistant.Opened`, `assistant.Closed`, `assistant.ViewChanged` | Open, close, or switch between the panel and full page. Closing the panel does not cancel an ongoing response. | +| `assistant.ResponseHelpful`, `assistant.ResponseNotHelpful`, `assistant.ResponseFeedbackCleared` | Explicit feedback linked to the response. | +| `assistant.ResponseRegenerated` | Retry or regenerate, linked to the previous response. | +| `assistant.MessageCopied`, `assistant.SuggestedActionSelected` | Copy a message or act on an Exie suggestion. | +| `assistant.ConversationCleared`, `assistant.ConversationLeft`, `assistant.PageLeft` | Clear, switch organization, unmount, or leave the browser page, with the last outcome/feedback, message count, and whether a turn was still streaming. | + +Filter the app telemetry project by `source:assistant.*`, then open an event's session timeline and use the `exie` IDs in event details to follow a conversation. Compare explicit positive/negative feedback, repeated retries, response latency, and departures while waiting. A completed response only means text arrived without an error; it does not prove the answer helped. Copying and continuing are useful signals, while closing the chat alone does not establish frustration. + +These browser events are best effort. Page-leave events may be lost during unload, network failure, or a browser crash, and configured client filtering still applies. Use server metrics for operational failure rates; use session events to understand the user journey. A missing terminal event alone is not proof of cancellation or abandonment. + +Focused frontend tests exercise the real SDK builders with the queue intercepted, plus streamed success/failure, retries, feedback, context changes, and panel visibility. They do not submit events to a running collector: + +```powershell +Set-Location src/Exceptionless.Web/ClientApp +npm run test:unit -- src/lib/features/assistant/assistant-telemetry.test.ts src/lib/features/assistant/components/assistant-panel.svelte.test.ts src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts src/lib/features/auth/exceptionless-session.test.ts +npm run check +``` From 5ccd9e6c2654a5cc33ee70cbd8caeaf96bd1bb03 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 21:52:45 -0500 Subject: [PATCH 02/18] Classify streamed provider errors and thrown tool outcomes --- .../Assistant/AssistantProviderDiagnostics.cs | 6 +- .../Assistant/AssistantService.cs | 10 +- .../Assistant/AssistantTurnDiagnostics.cs | 17 +++ .../Assistant/AssistantDiagnosticsTests.cs | 3 +- .../Assistant/AssistantServiceTests.cs | 109 ++++++++++++++++++ tests/Exceptionless.Tests/Assistant/README.md | 2 +- 6 files changed, 142 insertions(+), 5 deletions(-) diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs index 9a722c8eac..41e7456230 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -14,6 +14,7 @@ internal sealed class AssistantProviderDiagnostics( private readonly long _started = timeProvider.GetTimestamp(); private readonly Activity? _activity = AppDiagnostics.StartActivity("assistant.provider"); private bool _finished; + private bool _receivedError; public string? GenerationId { get; private set; } public string? Model { get; private set; } @@ -40,6 +41,7 @@ public void ObserveChunk(JsonElement chunk) GenerationId = GetMetadata(chunk, "id") ?? GenerationId; Model = GetMetadata(chunk, "model") ?? Model; ProviderName = GetMetadata(chunk, "provider") ?? ProviderName; + _receivedError |= chunk.TryGetProperty("error", out _); if (chunk.TryGetProperty("usage", out var usage) && usage.ValueKind == JsonValueKind.Object) { UsageReceived = true; @@ -59,7 +61,7 @@ public void ObserveChunk(JsonElement chunk) public void Complete(int outputCharacters, int toolCalls, bool receivedDone) { - string outcome = FinishReason switch + string outcome = _receivedError ? "provider_error" : FinishReason switch { "length" => "output_limit", "content_filter" => "content_filter", @@ -94,7 +96,7 @@ private void Finish(string outcome, int? outputCharacters = null, int? toolCalls } public void Dispose() => Finish(StatusCode is >= 400 ? "http_error" - : FinishReason == "error" ? "provider_error" + : _receivedError || FinishReason == "error" ? "provider_error" : cancellationToken.IsCancellationRequested ? "cancelled" : "interrupted"); private static string? GetMetadata(JsonElement element, string name) diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index a25161555a..9eca8285c9 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -389,7 +389,15 @@ internal async IAsyncEnumerable StreamAsync( if (toolCall.Name == SearchStacksTool) remainingProjectSearches--; - result = await ExecuteToolAsync(toolCall.Name, arguments, request, cancellationToken); + try + { + result = await ExecuteToolAsync(toolCall.Name, arguments, request, cancellationToken); + } + catch (Exception ex) + { + diagnostics?.RecordToolException(ex, timeProvider.GetElapsedTime(toolStarted).TotalMilliseconds); + throw; + } } diagnostics?.RecordToolResult(result, timeProvider.GetElapsedTime(toolStarted).TotalMilliseconds); diff --git a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs index e33bd4d062..096fa12354 100644 --- a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs @@ -100,6 +100,23 @@ public void RecordToolResult(string result, double durationMilliseconds) new("tool", LastTool), new("outcome", failed ? "failed" : "completed"), new("reason", errorCode ?? "none")); } + public void RecordToolException(Exception exception, double durationMilliseconds) + { + bool cancelled = exception is OperationCanceledException; + string outcome = cancelled ? "cancelled" : "failed"; + LastToolError = cancelled ? "operation_cancelled" : "tool_execution_error"; + if (!cancelled) + { + ToolFailures++; + } + + _logger.Log(cancelled ? LogLevel.Information : LogLevel.Warning, + "Assistant tool {ToolName} {ToolOutcome} with {ToolErrorCode} in {DurationMs} ms for turn {AssistantTurnId}: exception_type={ExceptionType}", + LastTool, outcome, LastToolError, durationMilliseconds, TurnId, exception.GetType().FullName); + AppDiagnostics.AssistantToolDuration.Record(durationMilliseconds, + new("tool", LastTool), new("outcome", outcome), new("reason", LastToolError)); + } + public void Finish(string outcome, string? failureCode = null, Exception? exception = null) { if (_finished) diff --git a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs index e08de497bc..b268b367ae 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs @@ -166,9 +166,10 @@ await AssistantEndpoints.WriteResponseAsync(context, public void Finish_FailedTurn_RecordsErrorSpanAndBoundedMetricTagsOnce() { var activities = new ConcurrentQueue(); + var activitySource = AppDiagnostics.ActivitySource; using var listener = new ActivityListener { - ShouldListenTo = source => source.Name == AppDiagnostics.ActivitySource.Name, + ShouldListenTo = source => source == activitySource, Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, SampleUsingParentId = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, ActivityStopped = activity => activities.Enqueue(activity) diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index fbc8768e71..3222e59ddb 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; using System.Net; using System.Text; using System.Text.Json; @@ -1022,6 +1024,113 @@ public async Task StreamAsync_EmptyProviderAnswer_RecordsProviderReasonAndGenera Assert.DoesNotContain(logger.Entries, entry => entry.Message.Contains("private question", StringComparison.Ordinal)); } + [Fact] + public async Task StreamAsync_ProviderStreamError_RecordsProviderErrorWithoutLoggingBody() + { + var handler = new StubHttpMessageHandler(""" + data: {"id":"gen-stream-error","error":{"message":"private provider error"}} + + """); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(handler, options); + + var exception = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + { + } + }); + + Assert.Equal("provider_error", exception.FailureCode); + var entry = Assert.Single(logger.Entries); + Assert.Equal("provider_error", entry.Properties["ProviderOutcome"]); + Assert.Equal(200, entry.Properties["ProviderStatusCode"]); + Assert.Equal("gen-stream-error", entry.Properties["ProviderGenerationId"]); + Assert.DoesNotContain("private provider error", entry.Message); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(bool cancelled) + { + var activitySource = AppDiagnostics.ActivitySource; + using var activityListener = new ActivityListener + { + ShouldListenTo = source => source == activitySource, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded + }; + ActivitySource.AddActivityListener(activityListener); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var measurements = new List>(); + using var meterListener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Name == "ex.assistant.tool.duration") + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + meterListener.SetMeasurementEventCallback((_, _, tags, _) => + { + if (Activity.Current?.GetTagItem("assistant.turn.id") as string == diagnostics.TurnId) + { + measurements.Add(tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value)); + } + }); + meterListener.Start(); + // An array where a tool argument object is required throws during invocation. + var handler = new StubHttpMessageHandler(""" + data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"tool-1","function":{"name":"search_stacks","arguments":"[]"}}]}}]} + + data: [DONE] + + """); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var service = CreateAssistantService(handler, options); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + var exception = await Record.ExceptionAsync(async () => + { + await foreach (var item in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "Find my errors")]), + "user-id", CreatePlanOptions(), diagnostics, cancellation.Token)) + { + if (cancelled && item.Type == "tool_call") + { + cancellation.Cancel(); + } + } + }); + + if (cancelled) + { + Assert.IsAssignableFrom(exception); + } + else + { + Assert.IsType(exception); + } + Assert.Equal(1, diagnostics.ToolCalls); + Assert.Equal(cancelled ? 0 : 1, diagnostics.ToolFailures); + Assert.Equal(cancelled ? "operation_cancelled" : "tool_execution_error", diagnostics.LastToolError); + var measurement = Assert.Single(measurements); + Assert.Equal("search_stacks", measurement["tool"]); + Assert.Equal(cancelled ? "cancelled" : "failed", measurement["outcome"]); + Assert.Equal(diagnostics.LastToolError, measurement["reason"]); + } + [Fact] public async Task StreamAsync_HttpRejection_RecordsStatusWithoutLoggingProviderErrorBody() { diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index fb4fac6257..4d8d5f8428 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -44,7 +44,7 @@ Server diagnostics deliberately exclude prompts, answer text, reasoning text, to | `usage_limit`, `context_limit` | A turn reached an organization usage limit or the conversation context bound. | | `invalid_provider_response`, `response_write_error`, `tool_execution_error`, `internal_error` | Inspect the stage, exception type/stack, and correlated trace. | -Returned tool errors are logged with the tool name and error code, even when the model recovers and completes the turn. Provider `output_limit` or `incomplete_stream` warnings can also accompany a completed turn when text was returned. Diagnostics preserve the existing response and accounting behavior; they do not automatically retry tools or change output budgets. +Returned tool errors are logged with the tool name and error code, even when the model recovers and completes the turn. Thrown tool exceptions also increment tool failures and record duration; interrupted tool invocations record a separate `cancelled` duration with `operation_cancelled`. Provider error objects inside an HTTP 200 stream record `provider_error` even without a finish reason. Provider `output_limit` or `incomplete_stream` warnings can also accompany a completed turn when text was returned. Diagnostics preserve the existing response and accounting behavior; they do not automatically retry tools or change output budgets. The existing `ex.assistant.turn.outcomes` metric remains unchanged. Additional histograms provide immediate duration and outcome counts: From fe5d3513dadedd3568d587f5c110e68ea0f3c596 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 22:08:55 -0500 Subject: [PATCH 03/18] Isolate Exie tracing and classify provider exceptions --- .../Utility/AppDiagnostics.cs | 1 + src/Exceptionless.Web/ApmExtensions.cs | 2 +- .../Assistant/AssistantProviderDiagnostics.cs | 17 ++- .../Assistant/AssistantService.cs | 122 ++++++++++-------- .../Assistant/AssistantTurnDiagnostics.cs | 2 +- .../Assistant/AssistantDiagnosticsTests.cs | 6 +- .../Assistant/AssistantServiceTests.cs | 49 ++++++- tests/Exceptionless.Tests/Assistant/README.md | 4 +- 8 files changed, 138 insertions(+), 65 deletions(-) diff --git a/src/Exceptionless.Core/Utility/AppDiagnostics.cs b/src/Exceptionless.Core/Utility/AppDiagnostics.cs index 44ad4422de..883cbfd65b 100644 --- a/src/Exceptionless.Core/Utility/AppDiagnostics.cs +++ b/src/Exceptionless.Core/Utility/AppDiagnostics.cs @@ -10,6 +10,7 @@ public static class AppDiagnostics internal static readonly AssemblyName AssemblyName = typeof(AppDiagnostics).Assembly.GetName(); internal static readonly string? AssemblyVersion = typeof(AppDiagnostics).Assembly.GetCustomAttribute()?.InformationalVersion ?? AssemblyName.Version?.ToString(); internal static readonly ActivitySource ActivitySource = new(AssemblyName.Name ?? "Exceptionless", AssemblyVersion); + internal static readonly ActivitySource AssistantActivitySource = new("Exceptionless.Assistant", AssemblyVersion); internal static readonly Meter Meter = new("Exceptionless", AssemblyVersion); private static readonly string _metricsPrefix = "ex."; diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index d37ef0ba1b..be2f81c505 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -72,7 +72,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }); b.AddHttpClientInstrumentation(); - b.AddSource("Exceptionless", "Exceptionless.Core", "Foundatio"); + b.AddSource("Exceptionless", "Exceptionless.Assistant", "Foundatio"); if (config.EnableRedis) b.AddRedisInstrumentation(c => diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs index 41e7456230..2ef63d4a0e 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -12,7 +12,7 @@ internal sealed class AssistantProviderDiagnostics( CancellationToken cancellationToken) : IDisposable { private readonly long _started = timeProvider.GetTimestamp(); - private readonly Activity? _activity = AppDiagnostics.StartActivity("assistant.provider"); + private readonly Activity? _activity = AppDiagnostics.AssistantActivitySource.StartActivity("assistant.provider"); private bool _finished; private bool _receivedError; @@ -73,6 +73,21 @@ public void Complete(int outputCharacters, int toolCalls, bool receivedDone) Finish(outcome, outputCharacters, toolCalls, receivedDone); } + public void RecordException(Exception exception) + { + string outcome = exception switch + { + AssistantProviderException when StatusCode is >= 400 => "http_error", + AssistantProviderException => "provider_error", + OperationCanceledException => cancellationToken.IsCancellationRequested ? "cancelled" : "provider_timeout", + HttpRequestException => "provider_transport_error", + JsonException => "invalid_provider_response", + IOException => "provider_stream_error", + _ => "internal_error" + }; + Finish(outcome); + } + private void Finish(string outcome, int? outputCharacters = null, int? toolCalls = null, bool receivedDone = false) { if (_finished) diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index 9eca8285c9..72cc15925f 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -129,84 +129,92 @@ internal async IAsyncEnumerable StreamAsync( diagnostics.Stage = "usage_reservation"; await using var providerRequest = await assistantUsageService.StartProviderRequestAsync(request.OrganizationId, providerInputCharacters); using var providerDiagnostics = diagnostics?.StartProviderRequest(providerInputCharacters, allowTools, cancellationToken); - using var response = await SendRequestAsync(messages, options, model, allowTools, request, providerDiagnostics, cancellationToken); - providerRequest.MarkAccepted(); - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); - using var reader = new StreamReader(stream); bool receivedDone = false; - - while (await reader.ReadLineAsync(cancellationToken) is { } line) + try { - if (!line.StartsWith("data:", StringComparison.Ordinal)) - continue; + using var response = await SendRequestAsync(messages, options, model, allowTools, request, providerDiagnostics, cancellationToken); + providerRequest.MarkAccepted(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var reader = new StreamReader(stream); - string payload = line[5..].Trim(); - if (payload == "[DONE]") + while (await reader.ReadLineAsync(cancellationToken) is { } line) { - receivedDone = true; - continue; - } - if (payload.Length == 0) - continue; - - using var document = JsonDocument.Parse(payload); - providerDiagnostics?.ObserveChunk(document.RootElement); - if (document.RootElement.TryGetProperty("error", out var error)) - throw new AssistantProviderException(GetProviderError(error)); + if (!line.StartsWith("data:", StringComparison.Ordinal)) + continue; - if (!usageRecorded && TryGetProviderUsage(document.RootElement, out var usage)) - { - usageRecorded = true; - try + string payload = line[5..].Trim(); + if (payload == "[DONE]") { - await providerRequest.ReconcileAsync(usage); + receivedDone = true; + continue; } - catch (Exception ex) + if (payload.Length == 0) + continue; + + using var document = JsonDocument.Parse(payload); + providerDiagnostics?.ObserveChunk(document.RootElement); + if (document.RootElement.TryGetProperty("error", out var error)) + throw new AssistantProviderException(GetProviderError(error)); + + if (!usageRecorded && TryGetProviderUsage(document.RootElement, out var usage)) { - // Disposal records the conservative reservation when detailed provider - // accounting cannot be reconciled. - logger.LogError(ex, "Unable to record assistant provider usage for organization {OrganizationId}", request.OrganizationId); + usageRecorded = true; + try + { + await providerRequest.ReconcileAsync(usage); + } + catch (Exception ex) + { + // Disposal records the conservative reservation when detailed provider + // accounting cannot be reconciled. + logger.LogError(ex, "Unable to record assistant provider usage for organization {OrganizationId}", request.OrganizationId); + } } - } - if (!document.RootElement.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0) - continue; + if (!document.RootElement.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0) + continue; - var delta = choices[0].GetProperty("delta"); - if (delta.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) - { - string? text = content.GetString(); - if (!String.IsNullOrEmpty(text)) + var delta = choices[0].GetProperty("delta"); + if (delta.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) { - assistantContent.Append(text); - assistantContentChunks.Add(text); + string? text = content.GetString(); + if (!String.IsNullOrEmpty(text)) + { + assistantContent.Append(text); + assistantContentChunks.Add(text); + } } - } - if (!delta.TryGetProperty("tool_calls", out var toolCallUpdates)) - continue; + if (!delta.TryGetProperty("tool_calls", out var toolCallUpdates)) + continue; - foreach (var update in toolCallUpdates.EnumerateArray()) - { - int index = update.GetProperty("index").GetInt32(); - if (!toolCalls.TryGetValue(index, out var pending)) + foreach (var update in toolCallUpdates.EnumerateArray()) { - pending = new PendingToolCall(); - toolCalls[index] = pending; - } + int index = update.GetProperty("index").GetInt32(); + if (!toolCalls.TryGetValue(index, out var pending)) + { + pending = new PendingToolCall(); + toolCalls[index] = pending; + } - if (update.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) - pending.Id = id.GetString() ?? pending.Id; + if (update.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) + pending.Id = id.GetString() ?? pending.Id; - if (!update.TryGetProperty("function", out var function)) - continue; + if (!update.TryGetProperty("function", out var function)) + continue; - if (function.TryGetProperty("name", out var name) && name.ValueKind == JsonValueKind.String) - pending.Name += name.GetString(); - if (function.TryGetProperty("arguments", out var arguments) && arguments.ValueKind == JsonValueKind.String) - pending.Arguments.Append(arguments.GetString()); + if (function.TryGetProperty("name", out var name) && name.ValueKind == JsonValueKind.String) + pending.Name += name.GetString(); + if (function.TryGetProperty("arguments", out var arguments) && arguments.ValueKind == JsonValueKind.String) + pending.Arguments.Append(arguments.GetString()); + } } } + catch (Exception ex) + { + providerDiagnostics?.RecordException(ex); + throw; + } providerDiagnostics?.Complete(assistantContent.Length, toolCalls.Count, receivedDone); if (diagnostics is not null) diff --git a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs index 096fa12354..dc1a6d3c51 100644 --- a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs @@ -20,7 +20,7 @@ public AssistantTurnDiagnostics(ILogger logger, TimeProvider timeProvider, strin _logger = logger; _timeProvider = timeProvider; _started = timeProvider.GetTimestamp(); - _activity = AppDiagnostics.StartActivity("assistant.turn"); + _activity = AppDiagnostics.AssistantActivitySource.StartActivity("assistant.turn"); TurnId = Guid.NewGuid().ToString("N"); OrganizationId = organizationId; ConversationId = conversationId; diff --git a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs index b268b367ae..cfd9c79f9a 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs @@ -35,10 +35,12 @@ public void AddApm_ExieActivitySource_CreatesExportableSpans() using var host = builder.Build(); _ = host.Services.GetRequiredService(); - using var activity = AppDiagnostics.StartActivity("assistant.turn"); + using var activity = AppDiagnostics.AssistantActivitySource.StartActivity("assistant.turn"); Assert.NotNull(activity); Assert.True(activity.IsAllDataRequested); + using var pipelineActivity = AppDiagnostics.StartActivity("Event Pipeline"); + Assert.Null(pipelineActivity); } [Theory] @@ -166,7 +168,7 @@ await AssistantEndpoints.WriteResponseAsync(context, public void Finish_FailedTurn_RecordsErrorSpanAndBoundedMetricTagsOnce() { var activities = new ConcurrentQueue(); - var activitySource = AppDiagnostics.ActivitySource; + var activitySource = AppDiagnostics.AssistantActivitySource; using var listener = new ActivityListener { ShouldListenTo = source => source == activitySource, diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 3222e59ddb..20c0941b79 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -1055,12 +1055,41 @@ public async Task StreamAsync_ProviderStreamError_RecordsProviderErrorWithoutLog Assert.DoesNotContain("private provider error", entry.Message); } + [Theory] + [InlineData("transport", "provider_transport_error")] + [InlineData("stream", "provider_stream_error")] + [InlineData("json", "invalid_provider_response")] + [InlineData("timeout", "provider_timeout")] + public async Task StreamAsync_ProviderThrows_RecordsFailureCategory(string failure, string expectedOutcome) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(new FailingProviderHttpMessageHandler(failure), options); + + var exception = await Record.ExceptionAsync(async () => + { + await foreach (var _ in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + { + } + }); + + Assert.NotNull(exception); + var entry = Assert.Single(logger.Entries); + Assert.Equal(expectedOutcome, entry.Properties["ProviderOutcome"]); + Assert.DoesNotContain("private", entry.Message); + } + [Theory] [InlineData(false)] [InlineData(true)] public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(bool cancelled) { - var activitySource = AppDiagnostics.ActivitySource; + var activitySource = AppDiagnostics.AssistantActivitySource; using var activityListener = new ActivityListener { ShouldListenTo = source => source == activitySource, @@ -1491,6 +1520,24 @@ protected override async Task SendAsync(HttpRequestMessage } } + private sealed class FailingProviderHttpMessageHandler(string failure) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => failure switch + { + "transport" => Task.FromException(new HttpRequestException("private transport detail")), + "timeout" => Task.FromException(new TaskCanceledException("private timeout detail")), + "stream" => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(new FailingProviderStream()) }), + _ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("data: invalid-json\n\n") }) + }; + } + + private sealed class FailingProviderStream : MemoryStream + { + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => ValueTask.FromException(new IOException("private stream detail")); + } + private sealed class RejectedHttpMessageHandler(HttpStatusCode statusCode) : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 4d8d5f8428..4c500d9e54 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -24,9 +24,9 @@ Set `EX_Assistant__Model` and `EX_Assistant__Endpoint` to evaluate a candidate m Every accepted Exie turn emits one structured completion log with `Outcome` (`completed`, `failed`, or `cancelled`), `FailureReason`, `Stage`, duration, time to first answer text, model, provider request count, tool rounds, and tool failures. `AssistantTurnId`, `OrganizationId`, `ConversationId`, `RequestId`, and `TraceId` correlate a turn across logs and traces. These identifiers are also included in the rendered message so Kubernetes console logs remain useful without a structured log viewer. -The `Exceptionless.Web.Assistant` logging override retains Information-level turn and provider summaries even when the production default is Warning. Failed turns log at Warning; unexpected/provider exceptions log at Error. Client disconnects remain cancellations and log at Information. A `completed` turn means the response finished without a streamed error; it does not establish that the answer was useful or correct. Keep running the quality evaluations above when changing the model, prompt, or tools. +The `Exceptionless.Web.Assistant` logging override retains Information-level turn and provider summaries even when the production default is Warning. Assistant spans use the dedicated `Exceptionless.Assistant` activity source, leaving the shared Core ingestion activity source unchanged. Failed turns log at Warning; unexpected/provider exceptions log at Error. Client disconnects remain cancellations and log at Information. A `completed` turn means the response finished without a streamed error; it does not establish that the answer was useful or correct. Keep running the quality evaluations above when changing the model, prompt, or tools. -Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. +Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. Thrown transport, parsing, stream, and timeout errors record explicit provider outcomes before propagating to the turn handler. See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events separately capture the user-visible conversation, as described below. From 607b56463c7b1311c38e46c5efefa6a1d5fb314e Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 22:19:56 -0500 Subject: [PATCH 04/18] Distinguish Exie deadlines from client cancellations --- .../Api/Endpoints/AssistantEndpoints.cs | 2 +- .../Assistant/AssistantProviderDiagnostics.cs | 7 +++-- .../Assistant/AssistantTurnDiagnostics.cs | 5 +++- .../Assistant/AssistantDiagnosticsTests.cs | 29 +++++++++++++++++++ tests/Exceptionless.Tests/Assistant/README.md | 2 +- 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs index 37c109acee..8f2acc0ccb 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs @@ -99,7 +99,7 @@ private static async Task StreamChatAsync( using var turnCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(httpContext.RequestAborted); turnCancellationSource.CancelAfter(TimeSpan.FromSeconds(AssistantLimits.MaximumTurnDurationSeconds)); - using var diagnostics = new AssistantTurnDiagnostics(logger, timeProvider, organizationId!, request.ConversationId!, httpContext.TraceIdentifier); + using var diagnostics = new AssistantTurnDiagnostics(logger, timeProvider, organizationId!, request.ConversationId!, httpContext.TraceIdentifier, httpContext.RequestAborted); var response = assistantService.StreamAsync(request, userId, planOptions, diagnostics, turnCancellationSource.Token); await WriteResponseAsync(httpContext, response, assistantUsageService, organizationId!, diagnostics, turnCancellationSource.Token); diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs index 2ef63d4a0e..be336f7879 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -79,7 +79,7 @@ public void RecordException(Exception exception) { AssistantProviderException when StatusCode is >= 400 => "http_error", AssistantProviderException => "provider_error", - OperationCanceledException => cancellationToken.IsCancellationRequested ? "cancelled" : "provider_timeout", + OperationCanceledException => GetCancellationOutcome(), HttpRequestException => "provider_transport_error", JsonException => "invalid_provider_response", IOException => "provider_stream_error", @@ -112,7 +112,10 @@ private void Finish(string outcome, int? outputCharacters = null, int? toolCalls public void Dispose() => Finish(StatusCode is >= 400 ? "http_error" : _receivedError || FinishReason == "error" ? "provider_error" - : cancellationToken.IsCancellationRequested ? "cancelled" : "interrupted"); + : cancellationToken.IsCancellationRequested ? GetCancellationOutcome() : "interrupted"); + + private string GetCancellationOutcome() => turn.IsClientDisconnected ? "cancelled" + : cancellationToken.IsCancellationRequested ? "turn_timeout" : "provider_timeout"; private static string? GetMetadata(JsonElement element, string name) => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String diff --git a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs index dc1a6d3c51..dad952ef33 100644 --- a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs @@ -11,14 +11,16 @@ internal sealed class AssistantTurnDiagnostics : IDisposable private readonly long _started; private readonly Activity? _activity; private readonly IDisposable? _scope; + private readonly CancellationToken _requestAborted; private bool _finished; private double? _firstTextDuration; private string? _failureCode; - public AssistantTurnDiagnostics(ILogger logger, TimeProvider timeProvider, string organizationId, string conversationId, string requestId) + public AssistantTurnDiagnostics(ILogger logger, TimeProvider timeProvider, string organizationId, string conversationId, string requestId, CancellationToken requestAborted = default) { _logger = logger; _timeProvider = timeProvider; + _requestAborted = requestAborted; _started = timeProvider.GetTimestamp(); _activity = AppDiagnostics.AssistantActivitySource.StartActivity("assistant.turn"); TurnId = Guid.NewGuid().ToString("N"); @@ -44,6 +46,7 @@ public AssistantTurnDiagnostics(ILogger logger, TimeProvider timeProvider, strin public string ConversationId { get; } public string RequestId { get; } public string? TraceId { get; } + public bool IsClientDisconnected => _requestAborted.IsCancellationRequested; public string? Model { get; set; } public string Stage { get; set; } = "initializing"; public int ProviderRequests { get; private set; } diff --git a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs index cfd9c79f9a..acaa36139a 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs @@ -213,6 +213,35 @@ public void Finish_FailedTurn_RecordsErrorSpanAndBoundedMetricTagsOnce() Assert.Single(logger.Entries); } + [Theory] + [InlineData("client", "cancelled")] + [InlineData("turn", "turn_timeout")] + [InlineData("provider", "provider_timeout")] + public void RecordException_CancellationSource_RecordsExpectedProviderOutcome(string source, string expectedOutcome) + { + using var requestAborted = new CancellationTokenSource(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(requestAborted.Token); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id", requestAborted.Token); + using (var provider = diagnostics.StartProviderRequest(100, true, deadline.Token)) + { + if (source == "client") + { + requestAborted.Cancel(); + } + else if (source == "turn") + { + deadline.Cancel(); + } + provider.RecordException(new OperationCanceledException("private cancellation detail")); + } + + var entry = Assert.Single(logger.Entries); + Assert.Equal(expectedOutcome, entry.Properties["ProviderOutcome"]); + Assert.Equal(source == "client" ? LogLevel.Information : LogLevel.Warning, entry.Level); + Assert.DoesNotContain("private cancellation detail", entry.Message); + } + [Fact] public void ObserveChunk_OutputLimit_RecordsGenerationAndReasoningWithoutContent() { diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 4c500d9e54..62f543c583 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -26,7 +26,7 @@ Every accepted Exie turn emits one structured completion log with `Outcome` (`co The `Exceptionless.Web.Assistant` logging override retains Information-level turn and provider summaries even when the production default is Warning. Assistant spans use the dedicated `Exceptionless.Assistant` activity source, leaving the shared Core ingestion activity source unchanged. Failed turns log at Warning; unexpected/provider exceptions log at Error. Client disconnects remain cancellations and log at Information. A `completed` turn means the response finished without a streamed error; it does not establish that the answer was useful or correct. Keep running the quality evaluations above when changing the model, prompt, or tools. -Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. Thrown transport, parsing, stream, and timeout errors record explicit provider outcomes before propagating to the turn handler. See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. +Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. Thrown transport, parsing, stream, and timeout errors record explicit provider outcomes before propagating to the turn handler. Provider outcomes distinguish browser disconnects (`cancelled`), the shared turn deadline (`turn_timeout`), and provider-only cancellation (`provider_timeout`). See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events separately capture the user-visible conversation, as described below. From 9f55a0b8505f081ae197c769199f5c78fc665adf Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 22:24:13 -0500 Subject: [PATCH 05/18] Align tool cancellation metrics with turn outcomes --- .../Assistant/AssistantProviderDiagnostics.cs | 7 +++-- .../Assistant/AssistantService.cs | 2 +- .../Assistant/AssistantTurnDiagnostics.cs | 9 ++++--- .../Assistant/AssistantServiceTests.cs | 27 +++++++++++-------- tests/Exceptionless.Tests/Assistant/README.md | 2 +- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs index be336f7879..5d19802946 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -114,8 +114,11 @@ public void Dispose() => Finish(StatusCode is >= 400 ? "http_error" : _receivedError || FinishReason == "error" ? "provider_error" : cancellationToken.IsCancellationRequested ? GetCancellationOutcome() : "interrupted"); - private string GetCancellationOutcome() => turn.IsClientDisconnected ? "cancelled" - : cancellationToken.IsCancellationRequested ? "turn_timeout" : "provider_timeout"; + private string GetCancellationOutcome() + { + string reason = turn.GetCancellationReason(cancellationToken, "provider_timeout"); + return reason == "client_disconnected" ? "cancelled" : reason; + } private static string? GetMetadata(JsonElement element, string name) => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index 72cc15925f..f0c69b22dc 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -403,7 +403,7 @@ internal async IAsyncEnumerable StreamAsync( } catch (Exception ex) { - diagnostics?.RecordToolException(ex, timeProvider.GetElapsedTime(toolStarted).TotalMilliseconds); + diagnostics?.RecordToolException(ex, timeProvider.GetElapsedTime(toolStarted).TotalMilliseconds, cancellationToken); throw; } } diff --git a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs index dad952ef33..6468ccc2ac 100644 --- a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs @@ -58,6 +58,9 @@ public AssistantTurnDiagnostics(ILogger logger, TimeProvider timeProvider, strin public string? LastToolError { get; private set; } public AssistantProviderDiagnostics? Provider { get; private set; } + public string GetCancellationReason(CancellationToken cancellationToken, string operationReason) + => IsClientDisconnected ? "client_disconnected" : cancellationToken.IsCancellationRequested ? "turn_timeout" : operationReason; + public AssistantProviderDiagnostics StartProviderRequest(int inputCharacters, bool allowTools, CancellationToken cancellationToken) { Stage = "provider_request"; @@ -103,11 +106,11 @@ public void RecordToolResult(string result, double durationMilliseconds) new("tool", LastTool), new("outcome", failed ? "failed" : "completed"), new("reason", errorCode ?? "none")); } - public void RecordToolException(Exception exception, double durationMilliseconds) + public void RecordToolException(Exception exception, double durationMilliseconds, CancellationToken cancellationToken) { - bool cancelled = exception is OperationCanceledException; + LastToolError = exception is OperationCanceledException ? GetCancellationReason(cancellationToken, "operation_cancelled") : "tool_execution_error"; + bool cancelled = LastToolError == "client_disconnected"; string outcome = cancelled ? "cancelled" : "failed"; - LastToolError = cancelled ? "operation_cancelled" : "tool_execution_error"; if (!cancelled) { ToolFailures++; diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 20c0941b79..7f19f4ed94 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -1085,9 +1085,10 @@ public async Task StreamAsync_ProviderThrows_RecordsFailureCategory(string failu } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(bool cancelled) + [InlineData("exception", "failed", "tool_execution_error")] + [InlineData("client", "cancelled", "client_disconnected")] + [InlineData("turn", "failed", "turn_timeout")] + public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(string failure, string outcome, string reason) { var activitySource = AppDiagnostics.AssistantActivitySource; using var activityListener = new ActivityListener @@ -1097,7 +1098,9 @@ public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(bool canc }; ActivitySource.AddActivityListener(activityListener); var logger = new RecordingAssistantLogger(); - using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + using var requestAborted = new CancellationTokenSource(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(requestAborted.Token, TestContext.Current.CancellationToken); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id", requestAborted.Token); var measurements = new List>(); using var meterListener = new MeterListener { @@ -1128,22 +1131,24 @@ public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(bool canc .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) .Build()); var service = CreateAssistantService(handler, options); - using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - var exception = await Record.ExceptionAsync(async () => { await foreach (var item in service.StreamAsync( new AssistantChatRequest([new AssistantChatMessage("user", "Find my errors")]), "user-id", CreatePlanOptions(), diagnostics, cancellation.Token)) { - if (cancelled && item.Type == "tool_call") + if (failure == "client" && item.Type == "tool_call") + { + requestAborted.Cancel(); + } + else if (failure == "turn" && item.Type == "tool_call") { cancellation.Cancel(); } } }); - if (cancelled) + if (failure != "exception") { Assert.IsAssignableFrom(exception); } @@ -1152,11 +1157,11 @@ public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(bool canc Assert.IsType(exception); } Assert.Equal(1, diagnostics.ToolCalls); - Assert.Equal(cancelled ? 0 : 1, diagnostics.ToolFailures); - Assert.Equal(cancelled ? "operation_cancelled" : "tool_execution_error", diagnostics.LastToolError); + Assert.Equal(failure == "client" ? 0 : 1, diagnostics.ToolFailures); + Assert.Equal(reason, diagnostics.LastToolError); var measurement = Assert.Single(measurements); Assert.Equal("search_stacks", measurement["tool"]); - Assert.Equal(cancelled ? "cancelled" : "failed", measurement["outcome"]); + Assert.Equal(outcome, measurement["outcome"]); Assert.Equal(diagnostics.LastToolError, measurement["reason"]); } diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 62f543c583..87c00c0081 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -44,7 +44,7 @@ Server diagnostics deliberately exclude prompts, answer text, reasoning text, to | `usage_limit`, `context_limit` | A turn reached an organization usage limit or the conversation context bound. | | `invalid_provider_response`, `response_write_error`, `tool_execution_error`, `internal_error` | Inspect the stage, exception type/stack, and correlated trace. | -Returned tool errors are logged with the tool name and error code, even when the model recovers and completes the turn. Thrown tool exceptions also increment tool failures and record duration; interrupted tool invocations record a separate `cancelled` duration with `operation_cancelled`. Provider error objects inside an HTTP 200 stream record `provider_error` even without a finish reason. Provider `output_limit` or `incomplete_stream` warnings can also accompany a completed turn when text was returned. Diagnostics preserve the existing response and accounting behavior; they do not automatically retry tools or change output budgets. +Returned tool errors are logged with the tool name and error code, even when the model recovers and completes the turn. Thrown tool exceptions and deadlines also increment tool failures and record duration; browser disconnects record a separate `cancelled` duration with `client_disconnected`. Tool cancellation reasons distinguish the shared deadline (`turn_timeout`) from other operation cancellation (`operation_cancelled`). Provider error objects inside an HTTP 200 stream record `provider_error` even without a finish reason. Provider `output_limit` or `incomplete_stream` warnings can also accompany a completed turn when text was returned. Diagnostics preserve the existing response and accounting behavior; they do not automatically retry tools or change output budgets. The existing `ex.assistant.turn.outcomes` metric remains unchanged. Additional histograms provide immediate duration and outcome counts: From 4196ca56dd2a67a2eb9da373ea0adb15098211a8 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 22:37:06 -0500 Subject: [PATCH 06/18] Normalize Exie conversation IDs across telemetry and requests --- .../assistant/components/assistant-panel.svelte | 13 +++++++++---- .../components/assistant-panel.svelte.test.ts | 10 ++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte index 1444d5ffb9..fbb890c8d1 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte @@ -65,7 +65,7 @@ promptRequest }: Props = $props(); let messages = $state([]); - let conversationId = $state(crypto.randomUUID()); + let conversationId = $state(createConversationId()); let conversationOrganizationId = $state(); let prompt = $state(''); let errorMessage = $state(); @@ -113,7 +113,7 @@ messages = []; errorMessage = undefined; prompt = ''; - conversationId = crypto.randomUUID(); + conversationId = createConversationId(); conversationOrganizationId = currentOrganizationId; lastOutcome = undefined; } @@ -238,7 +238,7 @@ }); errorMessage = undefined; const history = messages.slice(0, userMessageIndex + 1); - conversationId = crypto.randomUUID(); + conversationId = createConversationId(); const replacement: AssistantChatMessage = { content: '', conversationId, @@ -431,7 +431,7 @@ trackConversationEvent('assistant.ConversationCleared'); stopStreaming('conversation_cleared'); messages = []; - conversationId = crypto.randomUUID(); + conversationId = createConversationId(); errorMessage = undefined; prompt = ''; isNearBottom = true; @@ -489,6 +489,11 @@ }; } + function createConversationId(): string { + // Match the server's Guid.ToString("N") representation for exact log correlation. + return crypto.randomUUID().replaceAll('-', ''); + } + function trackConversationEvent(feature: string, details: Record = {}): void { trackAssistantEvent( feature, diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index 86cebfa265..d3421c9e6e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -38,10 +38,8 @@ describe('AssistantPanel', () => { }); it('records the prompt, response, and feedback under the same conversation and message IDs', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => new Response('{"type":"text_delta","text":"The answer"}\n{"type":"done"}\n')) - ); + const fetchMock = vi.fn().mockResolvedValue(new Response('{"type":"text_delta","text":"The answer"}\n{"type":"done"}\n')); + vi.stubGlobal('fetch', fetchMock); render(AssistantPanel, { props: { open: true, @@ -56,6 +54,8 @@ describe('AssistantPanel', () => { await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.ResponseHelpful', expect.anything())); const prompt = eventData('assistant.MessageSent'); + expect(prompt.conversation_id).toMatch(/^[0-9a-f]{32}$/); + expect(JSON.parse(fetchMock.mock.calls[0]?.[1]?.body as string).conversation_id).toBe(prompt.conversation_id); expect(prompt).toMatchObject({ organization_id: 'organization-1', path: '/next/stack/stack-1', prompt_source: 'queued', role: 'user' }); expect(eventData('assistant.ResponseCompleted')).toMatchObject({ assistant_message_id: prompt.assistant_message_id, @@ -94,6 +94,8 @@ describe('AssistantPanel', () => { retry_of_message_id: failed.assistant_message_id }); expect(retried.conversation_id).not.toBe(failed.conversation_id); + expect(retried.conversation_id).toMatch(/^[0-9a-f]{32}$/); + expect(JSON.parse(fetchMock.mock.calls[1]?.[1]?.body as string).conversation_id).toBe(retried.conversation_id); expect(submitLog.mock.calls.filter(([feature]) => feature === 'assistant.ResponseFailed')).toHaveLength(1); }); From 77e63e73f0f129a102378209455e9bd38ef985e1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 22:49:23 -0500 Subject: [PATCH 07/18] Record Exie usage without chat content --- .../assistant/assistant-telemetry.test.ts | 67 ++++++++++--------- .../features/assistant/assistant-telemetry.ts | 57 ++++++---------- .../components/assistant-panel.svelte | 10 ++- .../components/assistant-panel.svelte.test.ts | 37 +++++----- .../auth/exceptionless-session.test.ts | 10 +-- .../features/auth/exceptionless-session.ts | 14 ---- tests/Exceptionless.Tests/Assistant/README.md | 8 +-- 7 files changed, 87 insertions(+), 116 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts index d580ddd370..2a85ed525f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts @@ -1,10 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -const { submitFeatureUsage, submitLog } = vi.hoisted(() => ({ - submitFeatureUsage: vi.fn(() => Promise.resolve()), - submitLog: vi.fn(() => Promise.resolve()) +const { submitFeatureUsage } = vi.hoisted(() => ({ + submitFeatureUsage: vi.fn(() => Promise.resolve()) })); -vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage })); import { AssistantTurnTelemetry, trackAssistantEvent } from './assistant-telemetry'; @@ -19,11 +18,10 @@ const context = { describe('Exie session telemetry', () => { beforeEach(() => { submitFeatureUsage.mockReset(); - submitLog.mockReset(); }); - it('records one prompt and one assembled response with the same conversation context', () => { - const turn = new AssistantTurnTelemetry(context, 'What happened to checkout?', 'composer'); + it('records one prompt and one response outcome without conversation or tool content', () => { + const turn = new AssistantTurnTelemetry(context, 25, 'composer'); turn.observe({ text: 'The checkout ', type: 'text_delta' }); turn.observe({ arguments: '{"secret":"tool-argument"}', tool_call_id: 'tool-1', tool_name: 'get_event', type: 'tool_call' }); turn.observe({ result: '{"ok":true,"data":{"secret":"tool-result"}}', tool_call_id: 'tool-1', type: 'tool_result' }); @@ -32,71 +30,76 @@ describe('Exie session telemetry', () => { expect(turn.finish()).toBe('completed'); expect(turn.finish()).toBeUndefined(); - expect(submitLog).toHaveBeenCalledTimes(2); - expect(submitLog).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', 'What happened to checkout?', { - exie: expect.objectContaining({ ...context, prompt_source: 'composer', role: 'user' }) + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(submitFeatureUsage).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', { + exie: expect.objectContaining({ ...context, message_characters: 25, prompt_source: 'composer', role: 'user' }) }); - expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseCompleted', 'The checkout request timed out.', { - exie: expect.objectContaining({ ...context, outcome: 'completed', role: 'assistant', tool_calls: 1, tool_failures: 0 }) + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseCompleted', { + exie: expect.objectContaining({ ...context, outcome: 'completed', response_characters: 31, role: 'assistant', tool_calls: 1, tool_failures: 0 }) }); - expect(JSON.stringify(submitLog.mock.calls)).not.toContain('tool-argument'); - expect(JSON.stringify(submitLog.mock.calls)).not.toContain('tool-result'); + const telemetry = JSON.stringify(submitFeatureUsage.mock.calls); + expect(telemetry).not.toContain('checkout'); + expect(telemetry).not.toContain('tool-argument'); + expect(telemetry).not.toContain('tool-result'); }); it('records a streamed failure even when a done event follows it', () => { - const turn = new AssistantTurnTelemetry(context, 'Investigate', 'starter'); + const turn = new AssistantTurnTelemetry(context, 11, 'starter'); turn.observe({ text: 'Partial answer', type: 'text_delta' }); turn.observe({ message: 'Exie took too long.', type: 'error' }); turn.observe({ type: 'done' }); expect(turn.finish()).toBe('failed'); - expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseFailed', 'Partial answer', { - exie: expect.objectContaining({ error_message: 'Exie took too long.', reason: 'stream_error' }) + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseFailed', { + exie: expect.objectContaining({ reason: 'stream_error' }) }); + expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('Exie took too long.'); + expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('Partial answer'); }); it('distinguishes stopped responses and ignores late content after cancellation', () => { - const turn = new AssistantTurnTelemetry(context, 'Investigate', 'composer'); + const turn = new AssistantTurnTelemetry(context, 11, 'composer'); turn.observe({ text: 'Partial', type: 'text_delta' }); expect(turn.finish('organization_changed')).toBe('cancelled'); turn.observe({ text: 'Late content', type: 'text_delta' }); turn.observe({ type: 'done' }); turn.finish(); - expect(submitLog).toHaveBeenCalledTimes(2); - expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseCancelled', 'Partial', { + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseCancelled', { exie: expect.objectContaining({ organization_id: 'organization-1', reason: 'organization_changed' }) }); }); it('does not count a silently interrupted stream as a successful response', () => { - const turn = new AssistantTurnTelemetry(context, 'Investigate', 'composer'); + const turn = new AssistantTurnTelemetry(context, 11, 'composer'); turn.observe({ text: 'Partial', type: 'text_delta' }); expect(turn.finish()).toBe('failed'); - expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseFailed', 'Partial', { + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseFailed', { exie: expect.objectContaining({ reason: 'incomplete_stream', received_done: false }) }); }); - it('bounds transcript size and marks truncation without storing each streamed chunk', () => { + it('counts long messages without storing text or individual streamed chunks', () => { const content = 'x'.repeat(20_000); - const turn = new AssistantTurnTelemetry(context, content, 'retry', { previous_conversation_id: 'previous-conversation' }); + const turn = new AssistantTurnTelemetry(context, content.length, 'retry', { previous_conversation_id: 'previous-conversation' }); turn.observe({ text: content, type: 'text_delta' }); turn.observe({ type: 'done' }); turn.finish(); - expect(submitLog).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', content.slice(0, 16_384), { - exie: expect.objectContaining({ message_characters: 20_000, message_truncated: true, previous_conversation_id: 'previous-conversation' }) + expect(submitFeatureUsage).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', { + exie: expect.objectContaining({ message_characters: 20_000, previous_conversation_id: 'previous-conversation' }) }); - expect(submitLog).toHaveBeenLastCalledWith('assistant.ResponseCompleted', content.slice(0, 16_384), { - exie: expect.objectContaining({ message_characters: 20_000, message_truncated: true, response_characters: 20_000, response_truncated: true }) + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseCompleted', { + exie: expect.objectContaining({ message_characters: 20_000, response_characters: 20_000 }) }); + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('xxx'); }); it('keeps chat interactions working when telemetry cannot be submitted', async () => { submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); - submitLog.mockRejectedValueOnce(new Error('offline')); + submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); expect(() => trackAssistantEvent('assistant.ResponseHelpful', context)).not.toThrow(); - expect(() => new AssistantTurnTelemetry(context, 'Investigate', 'composer')).not.toThrow(); + expect(() => new AssistantTurnTelemetry(context, 11, 'composer')).not.toThrow(); await Promise.resolve(); - expect(submitFeatureUsage).toHaveBeenCalledOnce(); - expect(submitLog).toHaveBeenCalledOnce(); + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts index a615e701fb..19b04351a6 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts @@ -1,11 +1,9 @@ -import { submitFeatureUsage, submitLog } from '$features/auth/exceptionless-session'; +import { submitFeatureUsage } from '$features/auth/exceptionless-session'; import type { AssistantStreamEvent } from './assistant-stream'; import { assistantToolResultFailed } from './assistant-tool-result'; -const maximumMessageCharacters = 16_384; - export type AssistantPromptSource = 'composer' | 'queued' | 'regenerate' | 'retry' | 'starter' | 'suggested_action'; export type AssistantStopReason = 'access_changed' | 'component_unmounted' | 'conversation_cleared' | 'organization_changed' | 'user_stopped'; @@ -21,9 +19,7 @@ export interface AssistantTelemetryContext { export type AssistantTurnOutcome = 'cancelled' | 'completed' | 'failed'; export class AssistantTurnTelemetry { - private content = ''; private contentCharacters = 0; - private errorMessage: string | undefined; private failureReason: string | undefined; private finished = false; private firstTextDuration: number | undefined; @@ -34,15 +30,14 @@ export class AssistantTurnTelemetry { constructor( readonly context: AssistantTelemetryContext, - prompt: string, + promptCharacters: number, source: AssistantPromptSource, details: Record = {} ) { - trackAssistantEvent('assistant.MessageSent', context, { ...details, prompt_source: source, role: 'user' }, prompt); + trackAssistantEvent('assistant.MessageSent', context, { ...details, message_characters: promptCharacters, prompt_source: source, role: 'user' }); } - fail(message: string, reason: string): void { - this.errorMessage ??= message.slice(0, 2048); + fail(reason: 'request_error' | 'stream_error' | `http_${number}`): void { this.failureReason ??= reason; } @@ -55,27 +50,19 @@ export class AssistantTurnTelemetry { const reason = stopReason ?? this.failureReason ?? (!this.receivedDone ? 'incomplete_stream' : this.contentCharacters === 0 ? 'empty_response' : undefined); const feature = { cancelled: 'assistant.ResponseCancelled', completed: 'assistant.ResponseCompleted', failed: 'assistant.ResponseFailed' }[outcome]; - trackAssistantEvent( - feature, - this.context, - { - ...details, - duration_ms: Math.round(performance.now() - this.started), - error_message: this.errorMessage, - first_text_duration_ms: this.firstTextDuration, - message_characters: this.contentCharacters || this.errorMessage?.length || 0, - message_truncated: this.contentCharacters > maximumMessageCharacters, - outcome, - reason, - received_done: this.receivedDone, - response_characters: this.contentCharacters, - response_truncated: this.contentCharacters > maximumMessageCharacters, - role: 'assistant', - tool_calls: this.toolCalls, - tool_failures: this.toolFailures - }, - this.content || this.errorMessage || '' - ); + trackAssistantEvent(feature, this.context, { + ...details, + duration_ms: Math.round(performance.now() - this.started), + first_text_duration_ms: this.firstTextDuration, + message_characters: this.contentCharacters, + outcome, + reason, + received_done: this.receivedDone, + response_characters: this.contentCharacters, + role: 'assistant', + tool_calls: this.toolCalls, + tool_failures: this.toolFailures + }); return outcome; } @@ -85,32 +72,28 @@ export class AssistantTurnTelemetry { } if (event.type === 'text_delta' && event.text) { this.contentCharacters += event.text.length; - this.content += event.text.slice(0, Math.max(0, maximumMessageCharacters - this.content.length)); this.firstTextDuration ??= Math.round(performance.now() - this.started); } else if (event.type === 'tool_call') { this.toolCalls++; } else if (event.type === 'tool_result' && assistantToolResultFailed(event.result)) { this.toolFailures++; } else if (event.type === 'error') { - this.fail(event.message ?? 'Exie could not complete this request.', 'stream_error'); + this.fail('stream_error'); } else if (event.type === 'done') { this.receivedDone = true; } } } -export function trackAssistantEvent(feature: string, context: AssistantTelemetryContext, details: Record = {}, message?: string): void { +export function trackAssistantEvent(feature: string, context: AssistantTelemetryContext, details: Record = {}): void { const properties = { exie: { ...context, - ...(message !== undefined && { message_characters: message.length, message_truncated: message.length > maximumMessageCharacters }), ...details, schema_version: 1 } }; // Session/user identity, queueing, and filtering come from the existing SDK. // A telemetry failure must not interrupt a chat or generate another telemetry event. - const submission = - message === undefined ? submitFeatureUsage(feature, properties) : submitLog(feature, message.slice(0, maximumMessageCharacters), properties); - void submission.catch(() => {}); + void submitFeatureUsage(feature, properties).catch(() => {}); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte index fbb890c8d1..c99aac3b76 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte @@ -200,9 +200,7 @@ async function handleSuggestedAction(action: AssistantSuggestedAction, message: AssistantChatMessage): Promise { trackAssistantEvent('assistant.SuggestedActionSelected', getTelemetryContext(message), { - action_label: action.label, - action_type: action.href ? 'navigation' : 'prompt', - target_path: action.href?.split(/[?#]/)[0] + action_type: action.href ? 'navigation' : 'prompt' }); if (action.href) { open = false; @@ -269,7 +267,7 @@ ...getTelemetryContext(assistantMessage), user_message_id: userMessage.id }, - userMessage.content, + userMessage.content.length, source, { ...details, @@ -297,7 +295,7 @@ ? ((await response.json()) as { detail?: string; title?: string }) : undefined; const message = problem?.detail ?? problem?.title ?? `The assistant returned status ${response.status}.`; - telemetry.fail(message, `http_${response.status}`); + telemetry.fail(`http_${response.status}`); throw new Error(message); } @@ -319,7 +317,7 @@ } errorMessage = error instanceof Error ? error.message : 'Exie could not complete this request.'; - telemetry.fail(errorMessage, 'request_error'); + telemetry.fail('request_error'); } finally { const outcome = telemetry.finish(controller.signal.aborted ? 'user_stopped' : undefined, { is_visible: mode === 'page' || open diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index d3421c9e6e..95e5f0492e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -6,14 +6,9 @@ vi.mock('$features/auth/index.svelte', () => ({ accessToken: { current: 'access- vi.mock('$features/billing/stripe.svelte', () => ({ isStripeEnabled: () => true })); vi.mock('katex/dist/katex.min.css', () => ({})); const goto = vi.hoisted(() => vi.fn(() => Promise.resolve())); -const submitFeatureUsage = vi.hoisted(() => - vi.fn<(feature: string, properties?: Record, message?: string) => Promise>().mockResolvedValue(undefined) -); +const submitFeatureUsage = vi.hoisted(() => vi.fn<(feature: string, properties?: Record) => Promise>().mockResolvedValue(undefined)); vi.mock('$app/navigation', () => ({ goto })); -const submitLog = vi.hoisted(() => - vi.fn<(source: string, message: string, properties?: Record) => Promise>().mockResolvedValue(undefined) -); -vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage })); import AssistantPanel from './assistant-panel.svelte'; @@ -37,7 +32,7 @@ describe('AssistantPanel', () => { expect(screen.getByText('Bring Exie onto your team')).toBeTruthy(); }); - it('records the prompt, response, and feedback under the same conversation and message IDs', async () => { + it('correlates message outcomes and feedback without recording chat text', async () => { const fetchMock = vi.fn().mockResolvedValue(new Response('{"type":"text_delta","text":"The answer"}\n{"type":"done"}\n')); vi.stubGlobal('fetch', fetchMock); render(AssistantPanel, { @@ -68,8 +63,9 @@ describe('AssistantPanel', () => { conversation_id: prompt.conversation_id }); expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseHelpful')).toHaveLength(1); - expect(submitLog).toHaveBeenCalledWith('assistant.MessageSent', 'My question', expect.anything()); - expect(submitLog).toHaveBeenCalledWith('assistant.ResponseCompleted', 'The answer', expect.anything()); + const telemetry = JSON.stringify(submitFeatureUsage.mock.calls); + expect(telemetry).not.toContain('My question'); + expect(telemetry).not.toContain('The answer'); }); it('links a retry to the failed response across the new server conversation', async () => { @@ -96,7 +92,8 @@ describe('AssistantPanel', () => { expect(retried.conversation_id).not.toBe(failed.conversation_id); expect(retried.conversation_id).toMatch(/^[0-9a-f]{32}$/); expect(JSON.parse(fetchMock.mock.calls[1]?.[1]?.body as string).conversation_id).toBe(retried.conversation_id); - expect(submitLog.mock.calls.filter(([feature]) => feature === 'assistant.ResponseFailed')).toHaveLength(1); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseFailed')).toHaveLength(1); + expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('Provider timed out'); }); it('records closing while waiting without cancelling a response that finishes in the background', async () => { @@ -118,7 +115,7 @@ describe('AssistantPanel', () => { streamController!.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Background answer"}\n{"type":"done"}\n')); streamController!.close(); await waitFor(() => expect(eventData('assistant.ResponseCompleted').is_visible).toBe(false)); - expect(submitLog.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); }); it('records one cancellation with the original organization when organization context changes', async () => { @@ -143,8 +140,8 @@ describe('AssistantPanel', () => { await view.rerender({ open: true, organizationId: 'organization-2' }); await waitFor(() => expect(eventData('assistant.ResponseCancelled').reason).toBe('organization_changed')); expect(eventData('assistant.ResponseCancelled')).toMatchObject({ organization_id: 'organization-1' }); - expect(submitLog.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); - expect(submitLog.mock.calls.some(([feature]) => feature === 'assistant.ResponseCompleted')).toBe(false); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.ResponseCompleted')).toBe(false); expect(screen.queryByText('Partial answer')).toBeNull(); }); @@ -169,12 +166,12 @@ describe('AssistantPanel', () => { await screen.findByText('Partial answer'); await fireEvent(window, new Event('pagehide')); expect(eventData('assistant.PageLeft')).toMatchObject({ is_streaming: true }); - expect(submitLog.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); await fireEvent.click(screen.getByRole('button', { name: 'Stop generating' })); await screen.findByRole('button', { name: 'Send message' }); expect(eventData('assistant.ResponseCancelled')).toMatchObject({ outcome: 'cancelled', reason: 'user_stopped' }); - expect(submitLog.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); expect(screen.getByText('Partial answer')).toBeTruthy(); }); @@ -383,11 +380,15 @@ describe('AssistantPanel', () => { await waitFor(() => expect(goto).toHaveBeenCalledWith(configureHref)); expect(fetchMock).toHaveBeenCalledOnce(); + expect(eventData('assistant.SuggestedActionSelected')).toMatchObject({ action_type: 'navigation' }); + const telemetry = JSON.stringify(submitFeatureUsage.mock.calls); + expect(telemetry).not.toContain('Open Client Setup'); + expect(telemetry).not.toContain('How do I configure'); + expect(telemetry).not.toContain(configureHref); }); }); function eventData(feature: string, index = 0): Record { - const properties = - submitFeatureUsage.mock.calls.filter(([name]) => name === feature)[index]?.[1] ?? submitLog.mock.calls.filter(([name]) => name === feature)[index]?.[2]; + const properties = submitFeatureUsage.mock.calls.filter(([name]) => name === feature)[index]?.[1]; return (properties?.exie ?? {}) as Record; } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts index 0197aeef8b..aee0de6ba7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts @@ -14,17 +14,17 @@ vi.mock('@exceptionless/browser', async () => { return { Exceptionless: new ExceptionlessClient(config) }; }); -import { setUserIdentity, submitFeatureUsage, submitLog } from './exceptionless-session'; +import { setUserIdentity, submitFeatureUsage } from './exceptionless-session'; describe('Exceptionless session events', () => { beforeEach(() => { vi.mocked(Exceptionless.config.services.queue.enqueue).mockClear(); }); - it('keeps transcript logs and feedback attached to the existing user session', async () => { + it('keeps usage metadata and feedback attached to the existing user session', async () => { await setUserIdentity('exie-test-user'); const properties = { exie: { conversation_id: 'conversation-1', role: 'user' } }; - await submitLog('assistant.MessageSent', 'Why did checkout fail?', properties); + await submitFeatureUsage('assistant.MessageSent', properties); await submitFeatureUsage('assistant.ResponseHelpful', properties); const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); @@ -32,12 +32,12 @@ describe('Exceptionless session events', () => { expect(events[0]).toMatchObject({ type: 'session' }); expect(events[1]).toMatchObject({ data: properties, - message: 'Why did checkout fail?', source: 'assistant.MessageSent', - type: 'log' + type: 'usage' }); expect(events[2]).toMatchObject({ data: properties, source: 'assistant.ResponseHelpful', type: 'usage' }); for (const event of events) { + expect(event.message).toBeUndefined(); expect(event.data?.['@user']).toMatchObject({ identity: 'exie-test-user' }); expect(event.tags).toEqual(expect.arrayContaining(['UI', 'Svelte'])); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts index 8bc6f6f88e..04e70c6db6 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts @@ -64,20 +64,6 @@ export async function submitFeatureUsage(feature: string, properties?: Record): Promise { - const Exceptionless = await getExceptionless(); - if (!Exceptionless) { - return; - } - - const event = Exceptionless.createLog(source, message); - for (const [name, value] of Object.entries(properties ?? {})) { - event.setProperty(name, value); - } - await event.submit(); -} - async function getExceptionless() { if (!browser) { return; diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 87c00c0081..fba4ac6730 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -28,7 +28,7 @@ The `Exceptionless.Web.Assistant` logging override retains Information-level tur Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. Thrown transport, parsing, stream, and timeout errors record explicit provider outcomes before propagating to the turn handler. Provider outcomes distinguish browser disconnects (`cancelled`), the shared turn deadline (`turn_timeout`), and provider-only cancellation (`provider_timeout`). See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. -Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events separately capture the user-visible conversation, as described below. +Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events record usage metadata without conversation content, as described below. | Failure reason | Investigation | | --- | --- | @@ -69,11 +69,11 @@ dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll --fil The Svelte app submits Exie events through the existing Exceptionless browser client. They share the signed-in user's session, client configuration, queue, tags, and event exclusions. They go to the app's configured telemetry project. Starting a conversation does not create a separate user session. -Each submitted prompt produces an `assistant.MessageSent` log event. The assembled response produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` log event. The log message contains the prompt, answer, or partial answer; a failure without answer text uses the error displayed to the user. Native log summaries make these messages readable in the existing session timeline. Messages are capped at 16,384 characters, with length and truncation metadata. Drafts, individual streamed chunks, reasoning, and raw tool arguments/results are not submitted. +Each submitted prompt produces an `assistant.MessageSent` feature usage event. A turn produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` feature usage event. These events record character counts and outcomes, with no message text. Prompts, answers, drafts, reasoning, raw errors, tool arguments/results, and generated suggestion labels/destinations are not submitted. Suggestion events record only whether the action navigated or submitted a prompt. Events carry an `exie` extended-data object with `schema_version: 1`, conversation and message IDs, organization/project context, page path without query/fragment, and page/sheet mode. The conversation ID matches the server diagnostics. Turn summaries also include outcome, elapsed time, time to first text, tool counts/failures, and whether the chat was visible when the turn finished. Retries link the new server conversation back through `previous_conversation_id` and `retry_of_message_id`. -Interactions remain feature usage events: +Other feature usage events describe interactions: | Event source | Meaning | | --- | --- | @@ -87,7 +87,7 @@ Filter the app telemetry project by `source:assistant.*`, then open an event's s These browser events are best effort. Page-leave events may be lost during unload, network failure, or a browser crash, and configured client filtering still applies. Use server metrics for operational failure rates; use session events to understand the user journey. A missing terminal event alone is not proof of cancellation or abandonment. -Focused frontend tests exercise the real SDK builders with the queue intercepted, plus streamed success/failure, retries, feedback, context changes, and panel visibility. They do not submit events to a running collector: +Focused frontend tests exercise the real SDK builders with the queue intercepted, verify that chat content is absent, and cover streamed success/failure, retries, feedback, context changes, and panel visibility. They do not submit events to a running collector: ```powershell Set-Location src/Exceptionless.Web/ClientApp From 39588e226201be1efcf47870855569c3a1395f9c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 22:49:23 -0500 Subject: [PATCH 08/18] Align provider HTTP failure diagnostic codes --- .../Assistant/AssistantProviderDiagnostics.cs | 4 ++-- tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs index 5d19802946..3e92487d4d 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -77,7 +77,7 @@ public void RecordException(Exception exception) { string outcome = exception switch { - AssistantProviderException when StatusCode is >= 400 => "http_error", + AssistantProviderException when StatusCode is >= 400 => "provider_http_error", AssistantProviderException => "provider_error", OperationCanceledException => GetCancellationOutcome(), HttpRequestException => "provider_transport_error", @@ -110,7 +110,7 @@ private void Finish(string outcome, int? outputCharacters = null, int? toolCalls _activity?.Dispose(); } - public void Dispose() => Finish(StatusCode is >= 400 ? "http_error" + public void Dispose() => Finish(StatusCode is >= 400 ? "provider_http_error" : _receivedError || FinishReason == "error" ? "provider_error" : cancellationToken.IsCancellationRequested ? GetCancellationOutcome() : "interrupted"); diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 7f19f4ed94..0d6c56123e 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -1186,6 +1186,8 @@ public async Task StreamAsync_HttpRejection_RecordsStatusWithoutLoggingProviderE }); Assert.Equal("provider_http_error", exception.FailureCode); + var providerEntry = Assert.Single(logger.Entries, entry => entry.Properties.ContainsKey("ProviderOutcome")); + Assert.Equal(exception.FailureCode, providerEntry.Properties["ProviderOutcome"]); Assert.Equal(429, diagnostics.Provider?.StatusCode); Assert.Contains(logger.Entries, entry => entry.Properties.TryGetValue("StatusCode", out var status) && status is 429); Assert.All(logger.Entries, entry => From 65c5b30f41e5b6e3da0742b46394b3519584bf85 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 23:01:13 -0500 Subject: [PATCH 09/18] Classify malformed provider response structures --- .../Assistant/AssistantService.cs | 97 ++++++++++--------- .../Assistant/AssistantServiceTests.cs | 39 ++++++++ 2 files changed, 91 insertions(+), 45 deletions(-) diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index f0c69b22dc..2b589c190a 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -151,62 +151,69 @@ internal async IAsyncEnumerable StreamAsync( if (payload.Length == 0) continue; - using var document = JsonDocument.Parse(payload); - providerDiagnostics?.ObserveChunk(document.RootElement); - if (document.RootElement.TryGetProperty("error", out var error)) - throw new AssistantProviderException(GetProviderError(error)); - - if (!usageRecorded && TryGetProviderUsage(document.RootElement, out var usage)) + try { - usageRecorded = true; - try - { - await providerRequest.ReconcileAsync(usage); - } - catch (Exception ex) - { - // Disposal records the conservative reservation when detailed provider - // accounting cannot be reconciled. - logger.LogError(ex, "Unable to record assistant provider usage for organization {OrganizationId}", request.OrganizationId); - } - } + using var document = JsonDocument.Parse(payload); + providerDiagnostics?.ObserveChunk(document.RootElement); + if (document.RootElement.TryGetProperty("error", out var error)) + throw new AssistantProviderException(GetProviderError(error)); - if (!document.RootElement.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0) - continue; - - var delta = choices[0].GetProperty("delta"); - if (delta.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) - { - string? text = content.GetString(); - if (!String.IsNullOrEmpty(text)) + if (!usageRecorded && TryGetProviderUsage(document.RootElement, out var usage)) { - assistantContent.Append(text); - assistantContentChunks.Add(text); + usageRecorded = true; + try + { + await providerRequest.ReconcileAsync(usage); + } + catch (Exception ex) + { + // Disposal records the conservative reservation when detailed provider + // accounting cannot be reconciled. + logger.LogError(ex, "Unable to record assistant provider usage for organization {OrganizationId}", request.OrganizationId); + } } - } - if (!delta.TryGetProperty("tool_calls", out var toolCallUpdates)) - continue; + if (!document.RootElement.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0) + continue; - foreach (var update in toolCallUpdates.EnumerateArray()) - { - int index = update.GetProperty("index").GetInt32(); - if (!toolCalls.TryGetValue(index, out var pending)) + var delta = choices[0].GetProperty("delta"); + if (delta.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) { - pending = new PendingToolCall(); - toolCalls[index] = pending; + string? text = content.GetString(); + if (!String.IsNullOrEmpty(text)) + { + assistantContent.Append(text); + assistantContentChunks.Add(text); + } } - if (update.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) - pending.Id = id.GetString() ?? pending.Id; - - if (!update.TryGetProperty("function", out var function)) + if (!delta.TryGetProperty("tool_calls", out var toolCallUpdates)) continue; - if (function.TryGetProperty("name", out var name) && name.ValueKind == JsonValueKind.String) - pending.Name += name.GetString(); - if (function.TryGetProperty("arguments", out var arguments) && arguments.ValueKind == JsonValueKind.String) - pending.Arguments.Append(arguments.GetString()); + foreach (var update in toolCallUpdates.EnumerateArray()) + { + int index = update.GetProperty("index").GetInt32(); + if (!toolCalls.TryGetValue(index, out var pending)) + { + pending = new PendingToolCall(); + toolCalls[index] = pending; + } + + if (update.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) + pending.Id = id.GetString() ?? pending.Id; + + if (!update.TryGetProperty("function", out var function)) + continue; + + if (function.TryGetProperty("name", out var name) && name.ValueKind == JsonValueKind.String) + pending.Name += name.GetString(); + if (function.TryGetProperty("arguments", out var arguments) && arguments.ValueKind == JsonValueKind.String) + pending.Arguments.Append(arguments.GetString()); + } + } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException or FormatException) + { + throw new JsonException("The AI provider returned an invalid response structure.", ex); } } } diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 0d6c56123e..072baa9cef 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -9,6 +9,7 @@ using Exceptionless.Core.Models.Billing; using Exceptionless.Core.Serialization; using Exceptionless.Core.Services; +using Exceptionless.Web.Api.Endpoints; using Exceptionless.Web.Assistant; using Exceptionless.Web.Mcp; using Foundatio.Caching; @@ -1084,6 +1085,44 @@ public async Task StreamAsync_ProviderThrows_RecordsFailureCategory(string failu Assert.DoesNotContain("private", entry.Message); } + [Theory] + [InlineData("[]")] + [InlineData("null")] + [InlineData("{\"choices\":{}}")] + [InlineData("{\"choices\":[{}]}")] + [InlineData("{\"choices\":[{\"delta\":[]}]}")] + [InlineData("{\"choices\":[{\"delta\":{\"tool_calls\":[{}]}}]}")] + [InlineData("{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":2147483648}]}}]}")] + [InlineData("{\"usage\":{\"prompt_tokens\":\"private invalid token count\"}}")] + public async Task StreamAsync_InvalidProviderShape_RecordsProviderAndTurnFailure(string payload) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var usageService = new AssistantUsageService(cache, CreateLockProvider(cache, TimeProvider.System), recorder, options, + TimeProvider.System, NullLogger.Instance); + var service = CreateAssistantService(new StubHttpMessageHandler($"data: {payload}\n\ndata: [DONE]\n"), options, cache, usageService: usageService); + var context = new DefaultHttpContext(); + using var response = new MemoryStream(); + context.Response.Body = response; + + await AssistantEndpoints.WriteResponseAsync(context, + service.StreamAsync(new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken), + usageService, "organization-id", diagnostics, TestContext.Current.CancellationToken); + + var providerEntry = Assert.Single(logger.Entries, entry => entry.Properties.ContainsKey("ProviderOutcome")); + var turnEntry = Assert.Single(logger.Entries, entry => entry.Properties.ContainsKey("Outcome")); + Assert.Equal("invalid_provider_response", providerEntry.Properties["ProviderOutcome"]); + Assert.Equal("invalid_provider_response", turnEntry.Properties["FailureReason"]); + Assert.Equal("failed", turnEntry.Properties["Outcome"]); + Assert.All(logger.Entries, entry => Assert.DoesNotContain("private", entry.Message)); + } + [Theory] [InlineData("exception", "failed", "tool_execution_error")] [InlineData("client", "cancelled", "client_disconnected")] From a144b70e2c805dc253450030bc3d7345c255ed16 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 23:22:00 -0500 Subject: [PATCH 10/18] Add an admin switch for full Exie conversation logging --- .../Models/SystemSettings.cs | 2 + .../Indexes/SystemSettingsIndex.cs | 1 + .../Services/SystemSettingsService.cs | 6 ++ .../Api/Endpoints/AdminEndpoints.cs | 12 ++++ .../Api/Endpoints/AssistantEndpoints.cs | 11 ++- .../AssistantModelSettingsService.cs | 12 +++- .../src/lib/features/admin/api.svelte.ts | 21 ++++++ .../components/assistant-settings.svelte | 55 ++++++++++++++- .../assistant-settings.svelte.test.ts | 63 +++++++++++++++++ .../src/lib/features/admin/models.ts | 2 + .../assistant/assistant-telemetry.test.ts | 27 +++++-- .../features/assistant/assistant-telemetry.ts | 57 ++++++++++++--- .../components/assistant-panel.svelte | 8 ++- .../components/assistant-panel.svelte.test.ts | 55 ++++++++++++++- .../auth/exceptionless-session.test.ts | 11 ++- .../features/auth/exceptionless-session.ts | 14 ++++ .../ClientApp/src/lib/generated/api.ts | 5 ++ .../ClientApp/src/lib/generated/schemas.ts | 8 +++ .../UpdateAssistantFullLoggingSettings.cs | 6 ++ .../Api/Data/endpoint-manifest.json | 14 ++++ .../Exceptionless.Tests/Api/Data/openapi.json | 70 ++++++++++++++++++- .../Api/Endpoints/AdminEndpointTests.cs | 39 ++++++++++- .../Assistant/AssistantDiagnosticsTests.cs | 9 ++- .../AssistantModelSettingsServiceTests.cs | 44 ++++++++++++ tests/Exceptionless.Tests/Assistant/README.md | 14 ++-- tests/http/admin.http | 18 +++++ tests/http/assistant.http | 2 + 27 files changed, 548 insertions(+), 38 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts create mode 100644 src/Exceptionless.Web/Models/Admin/UpdateAssistantFullLoggingSettings.cs create mode 100644 tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs diff --git a/src/Exceptionless.Core/Models/SystemSettings.cs b/src/Exceptionless.Core/Models/SystemSettings.cs index 607aaa6029..da867cae75 100644 --- a/src/Exceptionless.Core/Models/SystemSettings.cs +++ b/src/Exceptionless.Core/Models/SystemSettings.cs @@ -17,6 +17,8 @@ public sealed class SystemSettings : IIdentity, IHaveDates public bool? AssistantEnabled { get; set; } + public bool AssistantFullLoggingEnabled { get; set; } + public bool? EventSubmissionEnabled { get; set; } public SystemNotification? SystemNotification { get; set; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs index 4f48d72bd7..8e5be9a95b 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs @@ -24,6 +24,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor .SetupDefaults() .Keyword(settings => settings.AssistantModel) .Boolean(settings => settings.AssistantEnabled) + .Boolean(settings => settings.AssistantFullLoggingEnabled) .Boolean(settings => settings.EventSubmissionEnabled) .Object(settings => settings.SystemNotification, notification => notification.Properties(properties => properties .Date("date") diff --git a/src/Exceptionless.Core/Services/SystemSettingsService.cs b/src/Exceptionless.Core/Services/SystemSettingsService.cs index 1899e0e66b..af5281afcf 100644 --- a/src/Exceptionless.Core/Services/SystemSettingsService.cs +++ b/src/Exceptionless.Core/Services/SystemSettingsService.cs @@ -92,4 +92,10 @@ public async Task IsEventSubmissionEnabledAsync() var settings = await _getSettingsAsync(); return settings?.EventSubmissionEnabled ?? !_appOptions.EventSubmissionDisabled; } + + public async Task IsAssistantFullLoggingEnabledAsync() + { + var settings = await _getSettingsAsync(); + return settings?.AssistantFullLoggingEnabled ?? false; + } } diff --git a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs index bc495454ca..b0bbd2fa7b 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs @@ -60,6 +60,18 @@ public static IEndpointRouteBuilder MapAdminEndpoints(this IEndpointRouteBuilder .WithTags(nameof(AdminEndpoints)) .WithSummary("Update Exie assistant availability"); + endpoints.MapPut("api/v2/admin/assistant-settings/full-logging", async (HttpContext httpContext, [FromBody] UpdateAssistantFullLoggingSettings request, AssistantModelSettingsService settingsService) + => HttpResults.Ok(await settingsService.SetFullLoggingEnabledAsync(request.Enabled, httpContext.Request.GetUser().Id))) + .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy) + .AddEndpointFilter() + .Accepts("application/json", "application/*+json") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .ProducesProblem(StatusCodes.Status422UnprocessableEntity) + .WithTags(nameof(AdminEndpoints)) + .WithSummary("Update Exie full conversation logging"); + endpoints.MapGet("api/v2/admin/event-submission-settings", GetEventSubmissionSettingsAsync) .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy) .Produces(StatusCodes.Status200OK) diff --git a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs index 8f2acc0ccb..d3ed44e6a9 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs @@ -4,6 +4,7 @@ using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; using Exceptionless.Core.Serialization; +using Exceptionless.Core.Services; using Exceptionless.Web.Assistant; using Microsoft.AspNetCore.Mvc; using HttpResults = Microsoft.AspNetCore.Http.Results; @@ -12,6 +13,7 @@ namespace Exceptionless.Web.Api.Endpoints; public static class AssistantEndpoints { + internal const string FullLoggingHeaderName = "X-Exie-Full-Logging"; private static readonly JsonSerializerOptions s_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web).ConfigureExceptionlessApiDefaults(); public static IEndpointRouteBuilder MapAssistantEndpoints(this IEndpointRouteBuilder endpoints) @@ -26,6 +28,7 @@ public static IEndpointRouteBuilder MapAssistantEndpoints(this IEndpointRouteBui endpoints.MapPost("api/v2/assistant/chat", StreamChatAsync) .WithName("StreamAssistantChat") + .WithDescription("The X-Exie-Full-Logging response header indicates whether the browser should record prompts and responses for this turn. Missing or false disables conversation logging.") .RequireAuthorization(AuthorizationRoles.UserPolicy) .WithMetadata(new RequestSizeLimitAttribute(256 * 1024)) .Produces(StatusCodes.Status200OK, contentType: "application/x-ndjson") @@ -46,6 +49,7 @@ private static async Task StreamChatAsync( AssistantAccessService assistantAccessService, AssistantUsageService assistantUsageService, AssistantService assistantService, + SystemSettingsService systemSettingsService, TimeProvider timeProvider, ILogger logger) { @@ -96,12 +100,13 @@ private static async Task StreamChatAsync( httpContext.Response.ContentType = "application/x-ndjson"; httpContext.Response.Headers.CacheControl = "no-store"; httpContext.Response.Headers.Append("X-Accel-Buffering", "no"); + bool fullLoggingEnabled = await systemSettingsService.IsAssistantFullLoggingEnabledAsync(); using var turnCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(httpContext.RequestAborted); turnCancellationSource.CancelAfter(TimeSpan.FromSeconds(AssistantLimits.MaximumTurnDurationSeconds)); using var diagnostics = new AssistantTurnDiagnostics(logger, timeProvider, organizationId!, request.ConversationId!, httpContext.TraceIdentifier, httpContext.RequestAborted); var response = assistantService.StreamAsync(request, userId, planOptions, diagnostics, turnCancellationSource.Token); - await WriteResponseAsync(httpContext, response, assistantUsageService, organizationId!, diagnostics, turnCancellationSource.Token); + await WriteResponseAsync(httpContext, response, assistantUsageService, organizationId!, diagnostics, turnCancellationSource.Token, fullLoggingEnabled); return HttpResults.Empty; } @@ -112,8 +117,10 @@ internal static async Task WriteResponseAsync( AssistantUsageService assistantUsageService, string organizationId, AssistantTurnDiagnostics diagnostics, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool fullLoggingEnabled = false) { + httpContext.Response.Headers[FullLoggingHeaderName] = fullLoggingEnabled ? "true" : "false"; bool responseFailed = false; try { diff --git a/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs index 20f2b559a6..51d21904ca 100644 --- a/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs @@ -43,6 +43,12 @@ public async Task SetEnabledAsync(bool? enabled, string return CreateResponse(settings); } + public async Task SetFullLoggingEnabledAsync(bool enabled, string userId) + { + var settings = await _systemSettingsService.UpdateAsync(userId, value => value.AssistantFullLoggingEnabled = enabled); + return CreateResponse(settings); + } + private AssistantModelSettings CreateResponse(SystemSettings? settings) { string configuredModel = _appOptions.AssistantOptions.Model; @@ -58,7 +64,8 @@ private AssistantModelSettings CreateResponse(SystemSettings? settings) enabledOverride ?? configuredEnabled, configuredEnabled, enabledOverride.HasValue, - _appOptions.AssistantOptions.IsConfigured); + _appOptions.AssistantOptions.IsConfigured, + settings?.AssistantFullLoggingEnabled ?? false); } } @@ -69,4 +76,5 @@ public sealed record AssistantModelSettings( bool Enabled, bool ConfiguredEnabled, bool IsEnabledOverridden, - bool IsConfigured); + bool IsConfigured, + bool FullLoggingEnabled = false); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts index 5ef6d164e9..4bed6955d1 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts @@ -17,6 +17,7 @@ import type { OAuthApplicationRequest, PredefinedSavedViewDefinition, UpdateAssistantEnabledSettingsRequest, + UpdateAssistantFullLoggingSettingsRequest, UpdateAssistantSettingsRequest, UpdateEventSubmissionSettingsRequest } from './models'; @@ -359,6 +360,26 @@ export function putAdminAssistantEnabledSettingsMutation() { })); } +export function putAdminAssistantFullLoggingSettingsMutation() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + mutationFn: async (request) => { + const client = useFetchClient(); + const response = await client.putJSON('admin/assistant-settings/full-logging', request); + + if (!response.ok) { + throw response.problem; + } + + return response.data!; + }, + onSuccess: (settings) => { + queryClient.setQueryData(queryKeys.assistantSettings, settings); + } + })); +} + export function putAdminAssistantSettingsMutation() { const queryClient = useQueryClient(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte index 319e367fdb..59f064472d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte @@ -7,7 +7,12 @@ import { Separator } from '$comp/ui/separator'; import { Spinner } from '$comp/ui/spinner'; import { Switch } from '$comp/ui/switch'; - import { getAdminAssistantSettingsQuery, putAdminAssistantEnabledSettingsMutation, putAdminAssistantSettingsMutation } from '$features/admin/api.svelte'; + import { + getAdminAssistantSettingsQuery, + putAdminAssistantEnabledSettingsMutation, + putAdminAssistantFullLoggingSettingsMutation, + putAdminAssistantSettingsMutation + } from '$features/admin/api.svelte'; import { type AssistantSettingsFormData, AssistantSettingsSchema } from '$features/admin/schemas'; import { ariaInvalid, getFormErrorMessages, mapFieldErrors, problemDetailsToFormErrors } from '$features/shared/validation'; import { ProblemDetails } from '@foundatiofx/fetchclient'; @@ -16,9 +21,12 @@ const settingsQuery = getAdminAssistantSettingsQuery(); const updateEnabledSettings = putAdminAssistantEnabledSettingsMutation(); + const updateFullLoggingSettings = putAdminAssistantFullLoggingSettingsMutation(); const updateSettings = putAdminAssistantSettingsMutation(); let assistantEnabled = $state(false); + let fullLoggingEnabled = $state(false); let loadedAvailabilityKey = $state(null); + let loadedFullLoggingEnabled = $state(); let loadedSettingsKey = $state(null); const settings = $derived(settingsQuery.data); const availabilityKey = $derived( @@ -62,6 +70,15 @@ assistantEnabled = settings.enabled; }); + $effect(() => { + if (!settings || loadedFullLoggingEnabled === settings.full_logging_enabled) { + return; + } + + loadedFullLoggingEnabled = settings.full_logging_enabled; + fullLoggingEnabled = settings.full_logging_enabled; + }); + $effect(() => { if (!settings || loadedSettingsKey === settingsKey) { return; @@ -106,6 +123,18 @@ toast.error('Failed to reset Exie availability.'); } } + + async function saveFullLogging() { + try { + const saved = await updateFullLoggingSettings.mutateAsync({ + enabled: fullLoggingEnabled + }); + fullLoggingEnabled = saved.full_logging_enabled; + toast.success(saved.full_logging_enabled ? 'Exie full logging is enabled.' : 'Exie full logging is disabled.'); + } catch { + toast.error('Failed to update Exie full logging.'); + } + } {#if settingsQuery.isPending} @@ -159,6 +188,30 @@ + + + Full logging + + Record submitted prompts and responses in session events for all organizations. Usage and error diagnostics remain available when off. Changes + apply to new turns. + + +
+ + +
+ + + +
{ event.preventDefault(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts new file mode 100644 index 0000000000..c815232ef8 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts @@ -0,0 +1,63 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ + enabled: false, + error: vi.fn(), + success: vi.fn(), + update: vi.fn() +})); +vi.mock('svelte-sonner', () => ({ toast: { error: state.error, success: state.success } })); +vi.mock('$features/admin/api.svelte', () => ({ + getAdminAssistantSettingsQuery: () => ({ + data: { + configured_enabled: true, + configured_model: 'example/model', + enabled: true, + full_logging_enabled: state.enabled, + is_configured: true, + is_enabled_overridden: false, + is_overridden: false, + model: 'example/model' + }, + isError: false, + isPending: false + }), + putAdminAssistantEnabledSettingsMutation: () => ({ isPending: false, mutateAsync: vi.fn() }), + putAdminAssistantFullLoggingSettingsMutation: () => ({ isPending: false, mutateAsync: state.update }), + putAdminAssistantSettingsMutation: () => ({ isPending: false, mutateAsync: vi.fn() }) +})); + +import AssistantSettings from './assistant-settings.svelte'; + +describe('Exie full logging settings', () => { + beforeEach(() => { + state.enabled = false; + state.update.mockReset(); + state.success.mockClear(); + state.error.mockClear(); + state.update.mockImplementation(async ({ enabled }: { enabled: boolean }) => ({ full_logging_enabled: enabled })); + }); + + it.each([false, true])('loads and saves the full logging switch from %s', async (enabled) => { + state.enabled = enabled; + render(AssistantSettings); + const toggle = screen.getByRole('switch', { name: 'Full logging' }); + const save = screen.getByRole('button', { name: 'Save Exie full logging' }); + await waitFor(() => expect(toggle.getAttribute('aria-checked')).toBe(String(enabled))); + expect(save.hasAttribute('disabled')).toBe(true); + await fireEvent.click(toggle); + await fireEvent.click(save); + await waitFor(() => expect(state.update).toHaveBeenCalledWith({ enabled: !enabled })); + await waitFor(() => expect(state.success).toHaveBeenCalledOnce()); + }); + + it('shows a save failure without claiming the logging mode changed', async () => { + state.update.mockRejectedValueOnce(new Error('Unavailable')); + render(AssistantSettings); + await fireEvent.click(screen.getByRole('switch', { name: 'Full logging' })); + await fireEvent.click(screen.getByRole('button', { name: 'Save Exie full logging' })); + await waitFor(() => expect(state.error).toHaveBeenCalledWith('Failed to update Exie full logging.')); + expect(state.success).not.toHaveBeenCalled(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts index b362c30b17..5de273c176 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts @@ -3,6 +3,7 @@ import type { CountResult, EventSubmissionSettings, UpdateAssistantEnabledSettings, + UpdateAssistantFullLoggingSettings, UpdateAssistantSettings, UpdateEventSubmissionSettings } from '$generated/api'; @@ -204,6 +205,7 @@ export type ShardMetric = { }; export type UpdateAssistantEnabledSettingsRequest = UpdateAssistantEnabledSettings; +export type UpdateAssistantFullLoggingSettingsRequest = UpdateAssistantFullLoggingSettings; export type UpdateAssistantSettingsRequest = UpdateAssistantSettings; export type UpdateEventSubmissionSettingsRequest = UpdateEventSubmissionSettings; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts index 2a85ed525f..6eaaa84915 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts @@ -1,9 +1,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -const { submitFeatureUsage } = vi.hoisted(() => ({ - submitFeatureUsage: vi.fn(() => Promise.resolve()) +const { submitFeatureUsage, submitLog } = vi.hoisted(() => ({ + submitFeatureUsage: vi.fn(() => Promise.resolve()), + submitLog: vi.fn(() => Promise.resolve()) })); -vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage })); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); import { AssistantTurnTelemetry, trackAssistantEvent } from './assistant-telemetry'; @@ -18,6 +19,7 @@ const context = { describe('Exie session telemetry', () => { beforeEach(() => { submitFeatureUsage.mockReset(); + submitLog.mockReset(); }); it('records one prompt and one response outcome without conversation or tool content', () => { @@ -31,6 +33,7 @@ describe('Exie session telemetry', () => { expect(turn.finish()).toBe('completed'); expect(turn.finish()).toBeUndefined(); expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(submitLog).not.toHaveBeenCalled(); expect(submitFeatureUsage).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', { exie: expect.objectContaining({ ...context, message_characters: 25, prompt_source: 'composer', role: 'user' }) }); @@ -50,9 +53,8 @@ describe('Exie session telemetry', () => { turn.observe({ type: 'done' }); expect(turn.finish()).toBe('failed'); expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseFailed', { - exie: expect.objectContaining({ reason: 'stream_error' }) + exie: expect.objectContaining({ error_message: 'Exie took too long.', reason: 'stream_error' }) }); - expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('Exie took too long.'); expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('Partial answer'); }); @@ -78,9 +80,11 @@ describe('Exie session telemetry', () => { }); }); - it('counts long messages without storing text or individual streamed chunks', () => { + it('bounds full logging while keeping accurate message counts and emitting one assembled response', () => { const content = 'x'.repeat(20_000); const turn = new AssistantTurnTelemetry(context, content.length, 'retry', { previous_conversation_id: 'previous-conversation' }); + turn.enableFullLogging(content); + turn.enableFullLogging(content); turn.observe({ text: content, type: 'text_delta' }); turn.observe({ type: 'done' }); turn.finish(); @@ -92,14 +96,23 @@ describe('Exie session telemetry', () => { }); expect(submitFeatureUsage).toHaveBeenCalledTimes(2); expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('xxx'); + expect(submitLog).toHaveBeenCalledTimes(2); + expect(submitLog).toHaveBeenNthCalledWith(1, 'assistant.Prompt', content.slice(0, 16_384), { + exie: expect.objectContaining({ ...context, message_characters: 20_000, message_truncated: true, prompt_source: 'retry' }) + }); + expect(submitLog).toHaveBeenLastCalledWith('assistant.Response', content.slice(0, 16_384), { + exie: expect.objectContaining({ ...context, message_characters: 20_000, message_truncated: true, outcome: 'completed' }) + }); }); it('keeps chat interactions working when telemetry cannot be submitted', async () => { submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); + submitLog.mockRejectedValueOnce(new Error('offline')); expect(() => trackAssistantEvent('assistant.ResponseHelpful', context)).not.toThrow(); - expect(() => new AssistantTurnTelemetry(context, 11, 'composer')).not.toThrow(); + expect(() => new AssistantTurnTelemetry(context, 11, 'composer').enableFullLogging('Investigate')).not.toThrow(); await Promise.resolve(); expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(submitLog).toHaveBeenCalledOnce(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts index 19b04351a6..844709ea9a 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts @@ -1,9 +1,11 @@ -import { submitFeatureUsage } from '$features/auth/exceptionless-session'; +import { submitFeatureUsage, submitLog } from '$features/auth/exceptionless-session'; import type { AssistantStreamEvent } from './assistant-stream'; import { assistantToolResultFailed } from './assistant-tool-result'; +const maximumMessageCharacters = 16_384; + export type AssistantPromptSource = 'composer' | 'queued' | 'regenerate' | 'retry' | 'starter' | 'suggested_action'; export type AssistantStopReason = 'access_changed' | 'component_unmounted' | 'conversation_cleared' | 'organization_changed' | 'user_stopped'; @@ -20,10 +22,13 @@ export type AssistantTurnOutcome = 'cancelled' | 'completed' | 'failed'; export class AssistantTurnTelemetry { private contentCharacters = 0; + private errorMessage: string | undefined; private failureReason: string | undefined; private finished = false; private firstTextDuration: number | undefined; + private readonly promptDetails: Record; private receivedDone = false; + private responseContent: string | undefined; private started = performance.now(); private toolCalls = 0; private toolFailures = 0; @@ -34,10 +39,24 @@ export class AssistantTurnTelemetry { source: AssistantPromptSource, details: Record = {} ) { - trackAssistantEvent('assistant.MessageSent', context, { ...details, message_characters: promptCharacters, prompt_source: source, role: 'user' }); + this.promptDetails = { ...details, message_characters: promptCharacters, prompt_source: source, role: 'user' }; + trackAssistantEvent('assistant.MessageSent', context, this.promptDetails); + } + + enableFullLogging(prompt: string): void { + if (this.finished || this.responseContent !== undefined) { + return; + } + + this.responseContent = ''; + trackAssistantLog('assistant.Prompt', prompt, this.context, { + ...this.promptDetails, + message_truncated: prompt.length > maximumMessageCharacters + }); } - fail(reason: 'request_error' | 'stream_error' | `http_${number}`): void { + fail(message: string, reason: 'request_error' | 'stream_error' | `http_${number}`): void { + this.errorMessage ??= message.slice(0, 2048); this.failureReason ??= reason; } @@ -50,9 +69,10 @@ export class AssistantTurnTelemetry { const reason = stopReason ?? this.failureReason ?? (!this.receivedDone ? 'incomplete_stream' : this.contentCharacters === 0 ? 'empty_response' : undefined); const feature = { cancelled: 'assistant.ResponseCancelled', completed: 'assistant.ResponseCompleted', failed: 'assistant.ResponseFailed' }[outcome]; - trackAssistantEvent(feature, this.context, { + const summary = { ...details, duration_ms: Math.round(performance.now() - this.started), + error_message: this.errorMessage, first_text_duration_ms: this.firstTextDuration, message_characters: this.contentCharacters, outcome, @@ -62,7 +82,15 @@ export class AssistantTurnTelemetry { role: 'assistant', tool_calls: this.toolCalls, tool_failures: this.toolFailures - }); + }; + trackAssistantEvent(feature, this.context, summary); + if (this.responseContent !== undefined) { + trackAssistantLog('assistant.Response', this.responseContent || this.errorMessage || '', this.context, { + ...summary, + message_characters: this.contentCharacters || this.errorMessage?.length || 0, + message_truncated: this.contentCharacters > maximumMessageCharacters + }); + } return outcome; } @@ -72,13 +100,16 @@ export class AssistantTurnTelemetry { } if (event.type === 'text_delta' && event.text) { this.contentCharacters += event.text.length; + if (this.responseContent !== undefined) { + this.responseContent += event.text.slice(0, Math.max(0, maximumMessageCharacters - this.responseContent.length)); + } this.firstTextDuration ??= Math.round(performance.now() - this.started); } else if (event.type === 'tool_call') { this.toolCalls++; } else if (event.type === 'tool_result' && assistantToolResultFailed(event.result)) { this.toolFailures++; } else if (event.type === 'error') { - this.fail('stream_error'); + this.fail(event.message ?? 'Exie could not complete this request.', 'stream_error'); } else if (event.type === 'done') { this.receivedDone = true; } @@ -86,14 +117,20 @@ export class AssistantTurnTelemetry { } export function trackAssistantEvent(feature: string, context: AssistantTelemetryContext, details: Record = {}): void { - const properties = { + // A telemetry failure must not interrupt a chat or generate another telemetry event. + void submitFeatureUsage(feature, getProperties(context, details)).catch(() => {}); +} + +function getProperties(context: AssistantTelemetryContext, details: Record) { + return { exie: { ...context, ...details, schema_version: 1 } }; - // Session/user identity, queueing, and filtering come from the existing SDK. - // A telemetry failure must not interrupt a chat or generate another telemetry event. - void submitFeatureUsage(feature, properties).catch(() => {}); +} + +function trackAssistantLog(source: string, message: string, context: AssistantTelemetryContext, details: Record): void { + void submitLog(source, message.slice(0, maximumMessageCharacters), getProperties(context, details)).catch(() => {}); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte index c99aac3b76..c11d17d935 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte @@ -295,7 +295,7 @@ ? ((await response.json()) as { detail?: string; title?: string }) : undefined; const message = problem?.detail ?? problem?.title ?? `The assistant returned status ${response.status}.`; - telemetry.fail(`http_${response.status}`); + telemetry.fail(message, `http_${response.status}`); throw new Error(message); } @@ -303,6 +303,10 @@ throw new Error('The assistant returned an empty response.'); } + if (response.headers.get('X-Exie-Full-Logging') === 'true') { + telemetry.enableFullLogging(userMessage.content); + } + await readAssistantStream(response.body, async (event) => { if (controller.signal.aborted) { return; @@ -317,7 +321,7 @@ } errorMessage = error instanceof Error ? error.message : 'Exie could not complete this request.'; - telemetry.fail('request_error'); + telemetry.fail(errorMessage, 'request_error'); } finally { const outcome = telemetry.finish(controller.signal.aborted ? 'user_stopped' : undefined, { is_visible: mode === 'page' || open diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index 95e5f0492e..59c50cc5ae 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -7,8 +7,11 @@ vi.mock('$features/billing/stripe.svelte', () => ({ isStripeEnabled: () => true vi.mock('katex/dist/katex.min.css', () => ({})); const goto = vi.hoisted(() => vi.fn(() => Promise.resolve())); const submitFeatureUsage = vi.hoisted(() => vi.fn<(feature: string, properties?: Record) => Promise>().mockResolvedValue(undefined)); +const submitLog = vi.hoisted(() => + vi.fn<(source: string, message: string, properties?: Record) => Promise>().mockResolvedValue(undefined) +); vi.mock('$app/navigation', () => ({ goto })); -vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage })); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); import AssistantPanel from './assistant-panel.svelte'; @@ -63,7 +66,7 @@ describe('AssistantPanel', () => { conversation_id: prompt.conversation_id }); expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseHelpful')).toHaveLength(1); - const telemetry = JSON.stringify(submitFeatureUsage.mock.calls); + const telemetry = JSON.stringify([...submitFeatureUsage.mock.calls, ...submitLog.mock.calls]); expect(telemetry).not.toContain('My question'); expect(telemetry).not.toContain('The answer'); }); @@ -93,7 +96,53 @@ describe('AssistantPanel', () => { expect(retried.conversation_id).toMatch(/^[0-9a-f]{32}$/); expect(JSON.parse(fetchMock.mock.calls[1]?.[1]?.body as string).conversation_id).toBe(retried.conversation_id); expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseFailed')).toHaveLength(1); - expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('Provider timed out'); + expect(failed.error_message).toBe('Provider timed out'); + }); + + it.each([undefined, 'false', 'true'])('records full chat text only when the current response enables it (%s)', async (flag) => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response('{"type":"text_delta","text":"Full answer"}\n{"type":"done"}\n', { + headers: flag === undefined ? {} : { 'X-Exie-Full-Logging': flag } + }) + ) + ); + render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'request-1', prompt: 'Full question' } } + }); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); + if (flag === 'true') { + expect(submitLog).toHaveBeenCalledWith('assistant.Prompt', 'Full question', expect.anything()); + expect(submitLog).toHaveBeenCalledWith('assistant.Response', 'Full answer', expect.anything()); + expect(submitLog).toHaveBeenCalledTimes(2); + } else { + expect(submitLog).not.toHaveBeenCalled(); + } + }); + + it('stops recording transcript text when full logging is disabled for the next turn', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response('{"type":"text_delta","text":"First answer"}\n{"type":"done"}\n', { headers: { 'X-Exie-Full-Logging': 'true' } }) + ) + .mockResolvedValueOnce( + new Response('{"type":"text_delta","text":"Second answer"}\n{"type":"done"}\n', { headers: { 'X-Exie-Full-Logging': 'false' } }) + ); + vi.stubGlobal('fetch', fetchMock); + render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'request-1', prompt: 'First question' } } + }); + await screen.findByText('First answer'); + const composer = screen.getByRole('textbox', { name: 'Message Exie' }); + await screen.findByRole('button', { name: 'Send message' }); + await fireEvent.input(composer, { target: { value: 'Second question' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Send message' })); + await waitFor(() => expect(eventData('assistant.ResponseCompleted', 1).outcome).toBe('completed')); + expect(submitLog).toHaveBeenCalledTimes(2); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Second'); }); it('records closing while waiting without cancelling a response that finishes in the background', async () => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts index aee0de6ba7..d92b42c111 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts @@ -14,7 +14,7 @@ vi.mock('@exceptionless/browser', async () => { return { Exceptionless: new ExceptionlessClient(config) }; }); -import { setUserIdentity, submitFeatureUsage } from './exceptionless-session'; +import { setUserIdentity, submitFeatureUsage, submitLog } from './exceptionless-session'; describe('Exceptionless session events', () => { beforeEach(() => { @@ -48,4 +48,13 @@ describe('Exceptionless session events', () => { expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledOnce(); expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledWith(expect.objectContaining({ source: 'project.Created', type: 'usage' })); }); + + it('attaches explicitly enabled transcript logs to the existing session', async () => { + await setUserIdentity('exie-test-user'); + const properties = { exie: { conversation_id: 'conversation-1', role: 'user' } }; + await submitLog('assistant.Prompt', 'Why did checkout fail?', properties); + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events.at(-1)).toMatchObject({ data: properties, message: 'Why did checkout fail?', source: 'assistant.Prompt', type: 'log' }); + expect(events.at(-1)?.data?.['@user']).toMatchObject({ identity: 'exie-test-user' }); + }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts index 04e70c6db6..940789a9a2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts @@ -64,6 +64,20 @@ export async function submitFeatureUsage(feature: string, properties?: Record): Promise { + const Exceptionless = await getExceptionless(); + if (!Exceptionless) { + return; + } + + const event = Exceptionless.createLog(source, message); + for (const [name, value] of Object.entries(properties ?? {})) { + event.setProperty(name, value); + } + await event.submit(); +} + async function getExceptionless() { if (!browser) { return; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts index 8db136b95e..29ab37b988 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts @@ -105,6 +105,7 @@ export interface AssistantModelSettings { configured_enabled: boolean; is_enabled_overridden: boolean; is_configured: boolean; + full_logging_enabled: boolean; } export interface BillingPlan { @@ -629,6 +630,10 @@ export interface UpdateAssistantEnabledSettings { enabled?: null | boolean; } +export interface UpdateAssistantFullLoggingSettings { + enabled: boolean; +} + export interface UpdateAssistantSettings { model?: null | string; } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index ec58c99678..811c397d7a 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -131,6 +131,7 @@ export const AssistantModelSettingsSchema = object({ configured_enabled: boolean(), is_enabled_overridden: boolean(), is_configured: boolean(), + full_logging_enabled: boolean(), }); export type AssistantModelSettingsFormData = Infer< typeof AssistantModelSettingsSchema @@ -737,6 +738,13 @@ export type UpdateAssistantEnabledSettingsFormData = Infer< typeof UpdateAssistantEnabledSettingsSchema >; +export const UpdateAssistantFullLoggingSettingsSchema = object({ + enabled: boolean(), +}); +export type UpdateAssistantFullLoggingSettingsFormData = Infer< + typeof UpdateAssistantFullLoggingSettingsSchema +>; + export const UpdateAssistantSettingsSchema = object({ model: string() .min(1, "Model is required") diff --git a/src/Exceptionless.Web/Models/Admin/UpdateAssistantFullLoggingSettings.cs b/src/Exceptionless.Web/Models/Admin/UpdateAssistantFullLoggingSettings.cs new file mode 100644 index 0000000000..58c0c7f18e --- /dev/null +++ b/src/Exceptionless.Web/Models/Admin/UpdateAssistantFullLoggingSettings.cs @@ -0,0 +1,6 @@ +namespace Exceptionless.Web.Models.Admin; + +public sealed record UpdateAssistantFullLoggingSettings +{ + public bool Enabled { get; init; } +} diff --git a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json index 01cc5636a7..b875ec0caa 100644 --- a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json +++ b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json @@ -301,6 +301,20 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "PUT", + "route": "/api/v2/admin/assistant-settings/full-logging", + "displayName": "HTTP: PUT api/v2/admin/assistant-settings/full-logging", + "tags": [ + "AdminEndpoints" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "GlobalAdminPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "GET", "route": "/api/v2/admin/assistant-usage", diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index 6e88d09679..d3e7414985 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -70,6 +70,7 @@ "tags": [ "AssistantEndpoints" ], + "description": "The X-Exie-Full-Logging response header indicates whether the browser should record prompts and responses for this turn. Missing or false disables conversation logging.", "operationId": "StreamAssistantChat", "requestBody": { "content": { @@ -299,6 +300,57 @@ } } }, + "/api/v2/admin/assistant-settings/full-logging": { + "put": { + "tags": [ + "AdminEndpoints" + ], + "summary": "Update Exie full conversation logging", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateAssistantFullLoggingSettings" + } + }, + "application/*\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/UpdateAssistantFullLoggingSettings" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssistantModelSettings" + } + } + } + }, + "401": { + "description": "Unauthorized" + }, + "403": { + "description": "Forbidden" + }, + "422": { + "description": "Unprocessable Entity", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, "/api/v2/admin/event-submission-settings": { "get": { "tags": [ @@ -11807,7 +11859,8 @@ "enabled", "configured_enabled", "is_enabled_overridden", - "is_configured" + "is_configured", + "full_logging_enabled" ], "type": "object", "properties": { @@ -11831,6 +11884,10 @@ }, "is_configured": { "type": "boolean" + }, + "full_logging_enabled": { + "type": "boolean", + "default": false } } }, @@ -13584,6 +13641,17 @@ } } }, + "UpdateAssistantFullLoggingSettings": { + "required": [ + "enabled" + ], + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, "UpdateAssistantSettings": { "type": "object", "properties": { diff --git a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs index 73fc81999a..0a2dcad3ce 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs @@ -228,6 +228,42 @@ await SendRequestAsAsync(request => request Assert.False(cleared.IsEnabledOverridden); } + [Fact] + public async Task AssistantFullLoggingSettingsAsync_AsGlobalAdmin_PersistsBothStatesWithoutChangingOtherSettings() + { + var initial = await SendRequestAsAsync(request => request + .AsGlobalAdminUser().AppendPaths("admin", "assistant-settings").StatusCodeShouldBeOk()); + Assert.NotNull(initial); + Assert.False(initial.FullLoggingEnabled); + + foreach (bool enabled in new[] { true, false }) + { + var updated = await SendRequestAsAsync(request => request + .Put().AsGlobalAdminUser().AppendPaths("admin", "assistant-settings", "full-logging") + .Content(new UpdateAssistantFullLoggingSettings { Enabled = enabled }).StatusCodeShouldBeOk()); + Assert.NotNull(updated); + Assert.Equal(enabled, updated.FullLoggingEnabled); + + await GetService().RemoveAllAsync(); + var persisted = await SendRequestAsAsync(request => request + .AsGlobalAdminUser().AppendPaths("admin", "assistant-settings").StatusCodeShouldBeOk()); + Assert.NotNull(persisted); + Assert.Equal(enabled, persisted.FullLoggingEnabled); + Assert.Equal(initial.Model, persisted.Model); + Assert.Equal(initial.Enabled, persisted.Enabled); + } + } + + [Fact] + public Task AssistantFullLoggingSettingsAsync_AsOrganizationUser_ReturnsForbidden() => SendRequestAsync(request => request + .Put().AsTestOrganizationUser().AppendPaths("admin", "assistant-settings", "full-logging") + .Content(new UpdateAssistantFullLoggingSettings { Enabled = true }).StatusCodeShouldBeForbidden()); + + [Fact] + public Task AssistantFullLoggingSettingsAsync_AsAnonymous_ReturnsUnauthorized() => SendRequestAsync(request => request + .Put().AsAnonymousUser().AppendPaths("admin", "assistant-settings", "full-logging") + .Content(new UpdateAssistantFullLoggingSettings { Enabled = true }).StatusCodeShouldBeUnauthorized()); + [Fact] public async Task EventSubmissionSettingsAsync_AsGlobalAdmin_UpdatesAndClearsRuntimeOverride() { @@ -1355,6 +1391,7 @@ private sealed record AssistantModelSettingsResponse( bool Enabled, bool ConfiguredEnabled, bool IsEnabledOverridden, - bool IsConfigured); + bool IsConfigured, + bool FullLoggingEnabled); private sealed record RequeueResult([property: JsonPropertyName("enqueued")] int Enqueued); } diff --git a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs index acaa36139a..094276c043 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs @@ -142,8 +142,10 @@ await AssistantEndpoints.WriteResponseAsync(context, Assert.Equal(1, Assert.Single(recorder.Records).Increment.Failed); } - [Fact] - public async Task WriteResponseAsync_Success_RecordsCompletionAndFirstTextWithoutLoggingAnswer() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task WriteResponseAsync_Success_RecordsCompletionAndFirstTextWithoutLoggingAnswer(bool fullLoggingEnabled) { var logger = new RecordingAssistantLogger(); var time = new FakeTimeProvider(); @@ -154,8 +156,9 @@ public async Task WriteResponseAsync_Success_RecordsCompletionAndFirstTextWithou time.Advance(TimeSpan.FromSeconds(3)); await AssistantEndpoints.WriteResponseAsync(context, StreamEvents([AssistantStreamEvent.TextDelta("private answer"), AssistantStreamEvent.Done()]), - CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken); + CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken, fullLoggingEnabled); + Assert.Equal(fullLoggingEnabled ? "true" : "false", context.Response.Headers[AssistantEndpoints.FullLoggingHeaderName]); var entry = Assert.Single(logger.Entries); Assert.Equal("completed", entry.Properties["Outcome"]); Assert.Equal("none", entry.Properties["FailureReason"]); diff --git a/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs new file mode 100644 index 0000000000..ca8c1c2f42 --- /dev/null +++ b/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs @@ -0,0 +1,44 @@ +using System.Text.Json; +using Exceptionless.Core; +using Exceptionless.Core.Models; +using Exceptionless.Core.Services; +using Exceptionless.Web.Assistant; +using Microsoft.Extensions.Configuration; +using Xunit; + +namespace Exceptionless.Tests.Assistant; + +public sealed class AssistantModelSettingsServiceTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SetFullLoggingEnabledAsync_NewAndLegacySettings_DefaultOffAndPreserveOtherSettings(bool hasLegacyRecord) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost" }).Build()); + SystemSettings? persisted = hasLegacyRecord ? JsonSerializer.Deserialize("{}") : null; + var systemSettings = new SystemSettingsService(() => Task.FromResult(persisted), value => + { + persisted = value; + return Task.CompletedTask; + }, options, TimeProvider.System); + var service = new AssistantModelSettingsService(systemSettings, options); + + Assert.False((await service.GetAsync()).FullLoggingEnabled); + Assert.False(await systemSettings.IsAssistantFullLoggingEnabledAsync()); + + Assert.True((await service.SetFullLoggingEnabledAsync(true, "admin-user")).FullLoggingEnabled); + Assert.True(await systemSettings.IsAssistantFullLoggingEnabledAsync()); + await service.SetModelAsync("example/model", "admin-user"); + await service.SetEnabledAsync(true, "admin-user"); + Assert.True((await service.GetAsync()).FullLoggingEnabled); + + Assert.False((await service.SetFullLoggingEnabledAsync(false, "admin-user")).FullLoggingEnabled); + Assert.False(await systemSettings.IsAssistantFullLoggingEnabledAsync()); + Assert.NotNull(persisted); + Assert.Equal("example/model", persisted.AssistantModel); + Assert.True(persisted.AssistantEnabled); + Assert.Equal("admin-user", persisted.UpdatedByUserId); + } +} diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index fba4ac6730..2846fa386f 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -28,7 +28,7 @@ The `Exceptionless.Web.Assistant` logging override retains Information-level tur Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. Thrown transport, parsing, stream, and timeout errors record explicit provider outcomes before propagating to the turn handler. Provider outcomes distinguish browser disconnects (`cancelled`), the shared turn deadline (`turn_timeout`), and provider-only cancellation (`provider_timeout`). See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. -Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events record usage metadata without conversation content, as described below. +Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events record usage and error details, with optional conversation logging controlled by the global admin setting below. | Failure reason | Investigation | | --- | --- | @@ -69,7 +69,11 @@ dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll --fil The Svelte app submits Exie events through the existing Exceptionless browser client. They share the signed-in user's session, client configuration, queue, tags, and event exclusions. They go to the app's configured telemetry project. Starting a conversation does not create a separate user session. -Each submitted prompt produces an `assistant.MessageSent` feature usage event. A turn produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` feature usage event. These events record character counts and outcomes, with no message text. Prompts, answers, drafts, reasoning, raw errors, tool arguments/results, and generated suggestion labels/destinations are not submitted. Suggestion events record only whether the action navigated or submitted a prompt. +Each submitted prompt produces an `assistant.MessageSent` feature usage event. A turn produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` feature usage event. These events record character counts and outcomes. Failure events retain the error displayed to the user in `error_message`, capped at 2,048 characters for diagnosis. Existing application error collection is unchanged. + +Global admins can enable **Full logging** beside the Exie availability and model settings. It defaults to off, including for existing settings records with no logging field. The setting persists across restarts and can be changed through `PUT /api/v2/admin/assistant-settings/full-logging` with `{ "enabled": true }` or `{ "enabled": false }`. Each accepted turn reads the setting and returns `X-Exie-Full-Logging: true` or `false`; the browser records transcript text only for an explicit `true`. Changes apply to new turns, including subsequent turns in an existing conversation. An in-flight turn retains the logging mode selected when it started. + +When enabled, full logging adds an `assistant.Prompt` log event and one assembled `assistant.Response` log event in the existing session. Each message is capped at 16,384 characters, with original character count and truncation metadata. When disabled, prompt and response text is not copied into telemetry. Drafts, individual streamed chunks, reasoning, raw tool arguments/results, and generated suggestion labels/destinations are never submitted. Suggestion events record only whether the action navigated or submitted a prompt. Events carry an `exie` extended-data object with `schema_version: 1`, conversation and message IDs, organization/project context, page path without query/fragment, and page/sheet mode. The conversation ID matches the server diagnostics. Turn summaries also include outcome, elapsed time, time to first text, tool counts/failures, and whether the chat was visible when the turn finished. Retries link the new server conversation back through `previous_conversation_id` and `retry_of_message_id`. @@ -83,14 +87,14 @@ Other feature usage events describe interactions: | `assistant.MessageCopied`, `assistant.SuggestedActionSelected` | Copy a message or act on an Exie suggestion. | | `assistant.ConversationCleared`, `assistant.ConversationLeft`, `assistant.PageLeft` | Clear, switch organization, unmount, or leave the browser page, with the last outcome/feedback, message count, and whether a turn was still streaming. | -Filter the app telemetry project by `source:assistant.*`, then open an event's session timeline and use the `exie` IDs in event details to follow a conversation. Compare explicit positive/negative feedback, repeated retries, response latency, and departures while waiting. A completed response only means text arrived without an error; it does not prove the answer helped. Copying and continuing are useful signals, while closing the chat alone does not establish frustration. +Filter the app telemetry project by `source:assistant.*`, then open an event's session timeline and use the `exie` IDs in event details to follow a conversation. Use `type:usage` when counting turns or interactions so optional transcript logs do not inflate the counts. Compare explicit positive/negative feedback, repeated retries, response latency, and departures while waiting. A completed response only means text arrived without an error; it does not prove the answer helped. Copying and continuing are useful signals, while closing the chat alone does not establish frustration. These browser events are best effort. Page-leave events may be lost during unload, network failure, or a browser crash, and configured client filtering still applies. Use server metrics for operational failure rates; use session events to understand the user journey. A missing terminal event alone is not proof of cancellation or abandonment. -Focused frontend tests exercise the real SDK builders with the queue intercepted, verify that chat content is absent, and cover streamed success/failure, retries, feedback, context changes, and panel visibility. They do not submit events to a running collector: +Focused frontend tests exercise the real SDK builders with the queue intercepted, verify transcript capture is controlled by the server flag, and cover settings changes, streamed success/failure, retries, feedback, context changes, and panel visibility. They do not submit events to a running collector: ```powershell Set-Location src/Exceptionless.Web/ClientApp -npm run test:unit -- src/lib/features/assistant/assistant-telemetry.test.ts src/lib/features/assistant/components/assistant-panel.svelte.test.ts src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts src/lib/features/auth/exceptionless-session.test.ts +npm run test:unit -- src/lib/features/assistant/assistant-telemetry.test.ts src/lib/features/assistant/components/assistant-panel.svelte.test.ts src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts src/lib/features/auth/exceptionless-session.test.ts src/lib/features/admin/components/assistant-settings.svelte.test.ts npm run check ``` diff --git a/tests/http/admin.http b/tests/http/admin.http index a349739e29..9e954661ae 100644 --- a/tests/http/admin.http +++ b/tests/http/admin.http @@ -58,6 +58,24 @@ Authorization: Bearer {{token}} GET {{apiUrl}}/admin/assistant-settings Authorization: Bearer {{token}} +### Enable Full Exie Conversation Logging +PUT {{apiUrl}}/admin/assistant-settings/full-logging +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "enabled": true +} + +### Disable Full Exie Conversation Logging (Usage and Error Diagnostics Continue) +PUT {{apiUrl}}/admin/assistant-settings/full-logging +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "enabled": false +} + ### Set Exie Model Override PUT {{apiUrl}}/admin/assistant-settings Authorization: Bearer {{token}} diff --git a/tests/http/assistant.http b/tests/http/assistant.http index e080819e49..d76e842485 100644 --- a/tests/http/assistant.http +++ b/tests/http/assistant.http @@ -20,6 +20,8 @@ GET {{apiUrl}}/assistant/access?organization_id={{organizationId}} Authorization: Bearer {{login.response.body.$.token}} ### Stream an assistant response +# X-Exie-Full-Logging: true enables prompt/response session logs for this turn. +# A missing or false header keeps transcript logging off. Usage and error events continue. POST {{apiUrl}}/assistant/chat Authorization: Bearer {{login.response.body.$.token}} Content-Type: application/json From c1dcc9694e443d5ac4b634f9d4604f4759cb7cb2 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 23:22:00 -0500 Subject: [PATCH 11/18] Keep provider HTTP outcomes consistent for redirects --- .../Assistant/AssistantProviderDiagnostics.cs | 5 ++--- .../Assistant/AssistantServiceTests.cs | 13 ++++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs index 3e92487d4d..f875b0af6c 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -77,8 +77,7 @@ public void RecordException(Exception exception) { string outcome = exception switch { - AssistantProviderException when StatusCode is >= 400 => "provider_http_error", - AssistantProviderException => "provider_error", + AssistantProviderException providerException => providerException.FailureCode, OperationCanceledException => GetCancellationOutcome(), HttpRequestException => "provider_transport_error", JsonException => "invalid_provider_response", @@ -110,7 +109,7 @@ private void Finish(string outcome, int? outputCharacters = null, int? toolCalls _activity?.Dispose(); } - public void Dispose() => Finish(StatusCode is >= 400 ? "provider_http_error" + public void Dispose() => Finish(StatusCode is < 200 or >= 300 ? "provider_http_error" : _receivedError || FinishReason == "error" ? "provider_error" : cancellationToken.IsCancellationRequested ? GetCancellationOutcome() : "interrupted"); diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 072baa9cef..207429f77f 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -1204,10 +1204,13 @@ public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(string fa Assert.Equal(diagnostics.LastToolError, measurement["reason"]); } - [Fact] - public async Task StreamAsync_HttpRejection_RecordsStatusWithoutLoggingProviderErrorBody() + [Theory] + [InlineData(HttpStatusCode.TemporaryRedirect)] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.InternalServerError)] + public async Task StreamAsync_HttpRejection_RecordsStatusWithoutLoggingProviderErrorBody(HttpStatusCode responseStatus) { - var handler = new RejectedHttpMessageHandler(HttpStatusCode.TooManyRequests); + var handler = new RejectedHttpMessageHandler(responseStatus); var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) .Build()); @@ -1227,8 +1230,8 @@ public async Task StreamAsync_HttpRejection_RecordsStatusWithoutLoggingProviderE Assert.Equal("provider_http_error", exception.FailureCode); var providerEntry = Assert.Single(logger.Entries, entry => entry.Properties.ContainsKey("ProviderOutcome")); Assert.Equal(exception.FailureCode, providerEntry.Properties["ProviderOutcome"]); - Assert.Equal(429, diagnostics.Provider?.StatusCode); - Assert.Contains(logger.Entries, entry => entry.Properties.TryGetValue("StatusCode", out var status) && status is 429); + Assert.Equal((int)responseStatus, diagnostics.Provider?.StatusCode); + Assert.Contains(logger.Entries, entry => entry.Properties.TryGetValue("StatusCode", out var status) && status is int code && code == (int)responseStatus); Assert.All(logger.Entries, entry => { Assert.DoesNotContain("Rejected", entry.Message); From 2350fb341e34c2a1223dac62d8e7bed9aa1ae081 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 23:47:21 -0500 Subject: [PATCH 12/18] Let users override the Exie conversation sharing default --- .../Models/SystemSettings.cs | 2 +- src/Exceptionless.Core/Models/User.cs | 1 + .../Indexes/SystemSettingsIndex.cs | 2 +- .../Configuration/Indexes/UserIndex.cs | 1 + .../Services/SystemSettingsService.cs | 4 +- .../Api/Endpoints/AdminEndpoints.cs | 6 +- .../Api/Endpoints/AssistantEndpoints.cs | 41 +++++- .../AssistantConversationSharingService.cs | 41 ++++++ .../AssistantModelSettingsService.cs | 8 +- .../Assistant/AssistantProviderDiagnostics.cs | 2 + .../Assistant/AssistantService.cs | 7 +- .../src/lib/features/admin/api.svelte.ts | 17 +-- .../components/assistant-settings.svelte | 48 +++---- .../assistant-settings.svelte.test.ts | 18 +-- .../src/lib/features/admin/models.ts | 4 +- .../src/lib/features/assistant/api.svelte.ts | 42 ++++++- .../features/assistant/assistant-telemetry.ts | 4 + .../assistant-conversation-sharing.svelte | 41 ++++++ .../components/assistant-panel.svelte | 59 ++++++++- .../components/assistant-panel.svelte.test.ts | 94 +++++++++++++- .../src/lib/features/assistant/models.ts | 2 + .../ClientApp/src/lib/generated/api.ts | 17 ++- .../ClientApp/src/lib/generated/schemas.ts | 33 +++-- .../ClientApp/src/routes/(app)/+layout.svelte | 18 ++- ...teAssistantConversationSharingSettings.cs} | 2 +- src/Exceptionless.Web/Program.cs | 1 + .../Api/Data/endpoint-manifest.json | 32 ++++- .../Exceptionless.Tests/Api/Data/openapi.json | 117 ++++++++++++++++-- .../Api/Endpoints/AdminEndpointTests.cs | 26 ++-- .../Api/Endpoints/AssistantEndpointTests.cs | 60 +++++++++ .../AssistantModelSettingsServiceTests.cs | 26 ++-- .../Assistant/AssistantServiceTests.cs | 31 ++++- tests/Exceptionless.Tests/Assistant/README.md | 6 +- tests/http/admin.http | 8 +- tests/http/assistant.http | 16 ++- 35 files changed, 718 insertions(+), 119 deletions(-) create mode 100644 src/Exceptionless.Web/Assistant/AssistantConversationSharingService.cs create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte rename src/Exceptionless.Web/Models/Admin/{UpdateAssistantFullLoggingSettings.cs => UpdateAssistantConversationSharingSettings.cs} (57%) diff --git a/src/Exceptionless.Core/Models/SystemSettings.cs b/src/Exceptionless.Core/Models/SystemSettings.cs index da867cae75..01edad4a4d 100644 --- a/src/Exceptionless.Core/Models/SystemSettings.cs +++ b/src/Exceptionless.Core/Models/SystemSettings.cs @@ -17,7 +17,7 @@ public sealed class SystemSettings : IIdentity, IHaveDates public bool? AssistantEnabled { get; set; } - public bool AssistantFullLoggingEnabled { get; set; } + public bool AssistantConversationSharingDefaultEnabled { get; set; } public bool? EventSubmissionEnabled { get; set; } diff --git a/src/Exceptionless.Core/Models/User.cs b/src/Exceptionless.Core/Models/User.cs index 8168c0302e..82bb84ec3a 100644 --- a/src/Exceptionless.Core/Models/User.cs +++ b/src/Exceptionless.Core/Models/User.cs @@ -40,6 +40,7 @@ public record User : IIdentity, IHaveDates, IValidatableObject public string? AvatarFileName { get; set; } public bool EmailNotificationsEnabled { get; set; } = true; + public bool? AssistantConversationSharingEnabled { get; set; } public bool IsEmailAddressVerified { get; set; } public string? VerifyEmailAddressToken { get; set; } public DateTime VerifyEmailAddressTokenExpiration { get; set; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs index 8e5be9a95b..3c4f65b0d0 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs @@ -24,7 +24,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor .SetupDefaults() .Keyword(settings => settings.AssistantModel) .Boolean(settings => settings.AssistantEnabled) - .Boolean(settings => settings.AssistantFullLoggingEnabled) + .Boolean(settings => settings.AssistantConversationSharingDefaultEnabled) .Boolean(settings => settings.EventSubmissionEnabled) .Object(settings => settings.SystemNotification, notification => notification.Properties(properties => properties .Date("date") diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs index d777ac2013..0e85e7a7fb 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs @@ -27,6 +27,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor map) .Text(e => e.FullName, t => t.AddKeywordField()) .Text(e => e.EmailAddress, t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) .Boolean(e => e.IsEmailAddressVerified) + .Boolean(e => e.AssistantConversationSharingEnabled) .Keyword(e => e.VerifyEmailAddressToken) .Date(e => e.VerifyEmailAddressTokenExpiration) .Keyword(e => e.PasswordResetToken) diff --git a/src/Exceptionless.Core/Services/SystemSettingsService.cs b/src/Exceptionless.Core/Services/SystemSettingsService.cs index af5281afcf..232b4a5e35 100644 --- a/src/Exceptionless.Core/Services/SystemSettingsService.cs +++ b/src/Exceptionless.Core/Services/SystemSettingsService.cs @@ -93,9 +93,9 @@ public async Task IsEventSubmissionEnabledAsync() return settings?.EventSubmissionEnabled ?? !_appOptions.EventSubmissionDisabled; } - public async Task IsAssistantFullLoggingEnabledAsync() + public async Task IsAssistantConversationSharingDefaultEnabledAsync() { var settings = await _getSettingsAsync(); - return settings?.AssistantFullLoggingEnabled ?? false; + return settings?.AssistantConversationSharingDefaultEnabled ?? false; } } diff --git a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs index b0bbd2fa7b..5ebac61909 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs @@ -60,11 +60,11 @@ public static IEndpointRouteBuilder MapAdminEndpoints(this IEndpointRouteBuilder .WithTags(nameof(AdminEndpoints)) .WithSummary("Update Exie assistant availability"); - endpoints.MapPut("api/v2/admin/assistant-settings/full-logging", async (HttpContext httpContext, [FromBody] UpdateAssistantFullLoggingSettings request, AssistantModelSettingsService settingsService) - => HttpResults.Ok(await settingsService.SetFullLoggingEnabledAsync(request.Enabled, httpContext.Request.GetUser().Id))) + endpoints.MapPut("api/v2/admin/assistant-settings/conversation-sharing", async (HttpContext httpContext, [FromBody] UpdateAssistantConversationSharingSettings request, AssistantModelSettingsService settingsService) + => HttpResults.Ok(await settingsService.SetConversationSharingDefaultEnabledAsync(request.Enabled, httpContext.Request.GetUser().Id))) .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy) .AddEndpointFilter() - .Accepts("application/json", "application/*+json") + .Accepts("application/json", "application/*+json") .Produces(StatusCodes.Status200OK) .Produces(StatusCodes.Status401Unauthorized) .Produces(StatusCodes.Status403Forbidden) diff --git a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs index d3ed44e6a9..b0def02bbd 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs @@ -4,7 +4,6 @@ using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; using Exceptionless.Core.Serialization; -using Exceptionless.Core.Services; using Exceptionless.Web.Assistant; using Microsoft.AspNetCore.Mvc; using HttpResults = Microsoft.AspNetCore.Http.Results; @@ -18,6 +17,22 @@ public static class AssistantEndpoints public static IEndpointRouteBuilder MapAssistantEndpoints(this IEndpointRouteBuilder endpoints) { + endpoints.MapGet("api/v2/assistant/conversation-sharing", GetConversationSharingAsync) + .WithName("GetAssistantConversationSharing") + .RequireAuthorization(AuthorizationRoles.UserPolicy) + .Produces() + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status404NotFound); + + endpoints.MapPut("api/v2/assistant/conversation-sharing", SetConversationSharingAsync) + .WithName("SetAssistantConversationSharing") + .WithDescription("Saves the current user's choice. Null follows the admin default; true and false remain explicit choices when the default changes.") + .RequireAuthorization(AuthorizationRoles.UserPolicy) + .Produces() + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status404NotFound); + endpoints.MapGet("api/v2/assistant/access", GetAccessAsync) .WithName("GetAssistantAccess") .RequireAuthorization(AuthorizationRoles.UserPolicy) @@ -49,7 +64,7 @@ private static async Task StreamChatAsync( AssistantAccessService assistantAccessService, AssistantUsageService assistantUsageService, AssistantService assistantService, - SystemSettingsService systemSettingsService, + AssistantConversationSharingService conversationSharingService, TimeProvider timeProvider, ILogger logger) { @@ -100,7 +115,7 @@ private static async Task StreamChatAsync( httpContext.Response.ContentType = "application/x-ndjson"; httpContext.Response.Headers.CacheControl = "no-store"; httpContext.Response.Headers.Append("X-Accel-Buffering", "no"); - bool fullLoggingEnabled = await systemSettingsService.IsAssistantFullLoggingEnabledAsync(); + bool fullLoggingEnabled = (await conversationSharingService.GetAsync(userId))?.Enabled == true; using var turnCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(httpContext.RequestAborted); turnCancellationSource.CancelAfter(TimeSpan.FromSeconds(AssistantLimits.MaximumTurnDurationSeconds)); @@ -187,6 +202,26 @@ private static async Task GetAccessAsync( return HttpResults.Ok(access.ToResponse()); } + private static async Task GetConversationSharingAsync(HttpContext httpContext, AssistantConversationSharingService service) + { + string? userId = httpContext.User.GetUserId(); + if (String.IsNullOrWhiteSpace(userId)) + return HttpResults.Unauthorized(); + + var settings = await service.GetAsync(userId); + return settings is null ? HttpResults.NotFound() : HttpResults.Ok(settings); + } + + private static async Task SetConversationSharingAsync(UpdateAssistantConversationSharing request, HttpContext httpContext, AssistantConversationSharingService service) + { + string? userId = httpContext.User.GetUserId(); + if (String.IsNullOrWhiteSpace(userId)) + return HttpResults.Unauthorized(); + + var settings = await service.SetAsync(userId, request.Enabled); + return settings is null ? HttpResults.NotFound() : HttpResults.Ok(settings); + } + internal static IResult? MapAccessFailure(AssistantAccessDecision access) => access.Reason switch { AssistantAccessReason.Available => null, diff --git a/src/Exceptionless.Web/Assistant/AssistantConversationSharingService.cs b/src/Exceptionless.Web/Assistant/AssistantConversationSharingService.cs new file mode 100644 index 0000000000..60d4754519 --- /dev/null +++ b/src/Exceptionless.Web/Assistant/AssistantConversationSharingService.cs @@ -0,0 +1,41 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Web.Assistant; + +public sealed class AssistantConversationSharingService(IUserRepository userRepository, SystemSettingsService systemSettingsService) +{ + public async Task GetAsync(string userId) + { + // Read the saved choice for every turn, including choices made on another device. + var user = await userRepository.GetByIdAsync(userId); + if (user is null) + return null; + + bool defaultEnabled = await systemSettingsService.IsAssistantConversationSharingDefaultEnabledAsync(); + return Resolve(user.AssistantConversationSharingEnabled, defaultEnabled); + } + + public async Task SetAsync(string userId, bool? enabled) + { + // Preserve an explicit choice even when it matches today's default. + bool updated = await userRepository.PatchAsync(userId, + new ActionPatch(user => user.AssistantConversationSharingEnabled = enabled), + options => options.Cache().ImmediateConsistency()); + return updated ? await GetAsync(userId) : null; + } + + internal static AssistantConversationSharingSettings Resolve(bool? enabled, bool defaultEnabled) => + new(enabled ?? defaultEnabled, defaultEnabled, enabled.HasValue); +} + +public sealed record AssistantConversationSharingSettings(bool Enabled, bool DefaultEnabled, bool IsOverridden); + +public sealed record UpdateAssistantConversationSharing +{ + [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.Never)] + public required bool? Enabled { get; init; } +} diff --git a/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs index 51d21904ca..2af954beac 100644 --- a/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs @@ -43,9 +43,9 @@ public async Task SetEnabledAsync(bool? enabled, string return CreateResponse(settings); } - public async Task SetFullLoggingEnabledAsync(bool enabled, string userId) + public async Task SetConversationSharingDefaultEnabledAsync(bool enabled, string userId) { - var settings = await _systemSettingsService.UpdateAsync(userId, value => value.AssistantFullLoggingEnabled = enabled); + var settings = await _systemSettingsService.UpdateAsync(userId, value => value.AssistantConversationSharingDefaultEnabled = enabled); return CreateResponse(settings); } @@ -65,7 +65,7 @@ private AssistantModelSettings CreateResponse(SystemSettings? settings) configuredEnabled, enabledOverride.HasValue, _appOptions.AssistantOptions.IsConfigured, - settings?.AssistantFullLoggingEnabled ?? false); + settings?.AssistantConversationSharingDefaultEnabled ?? false); } } @@ -77,4 +77,4 @@ public sealed record AssistantModelSettings( bool ConfiguredEnabled, bool IsEnabledOverridden, bool IsConfigured, - bool FullLoggingEnabled = false); + bool ConversationSharingDefaultEnabled = false); diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs index f875b0af6c..279f80f6a8 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -87,6 +87,8 @@ public void RecordException(Exception exception) Finish(outcome); } + public void Reject(string reason, int outputCharacters, int toolCalls, bool receivedDone) => Finish(reason, outputCharacters, toolCalls, receivedDone); + private void Finish(string outcome, int? outputCharacters = null, int? toolCalls = null, bool receivedDone = false) { if (_finished) diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index 2b589c190a..d8e855cfb4 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -223,12 +223,12 @@ internal async IAsyncEnumerable StreamAsync( throw; } - providerDiagnostics?.Complete(assistantContent.Length, toolCalls.Count, receivedDone); if (diagnostics is not null) diagnostics.Stage = "response_validation"; if (s_rawDsmlPattern.IsMatch(assistantContent.ToString())) { + providerDiagnostics?.Reject("malformed_response", assistantContent.Length, toolCalls.Count, receivedDone); if (malformedResponseRetries < AssistantLimits.MaximumMalformedResponseRetries) { malformedResponseRetries++; @@ -260,6 +260,11 @@ internal async IAsyncEnumerable StreamAsync( malformedResponseCorrection = null; } + if (!allowTools && toolCalls.Count > 0) + providerDiagnostics?.Reject("tool_round_limit", assistantContent.Length, toolCalls.Count, receivedDone); + else + providerDiagnostics?.Complete(assistantContent.Length, toolCalls.Count, receivedDone); + foreach (string text in assistantContentChunks) { yield return AssistantStreamEvent.TextDelta(text); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts index 4bed6955d1..ebc575cad3 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts @@ -16,8 +16,8 @@ import type { OAuthApplication, OAuthApplicationRequest, PredefinedSavedViewDefinition, + UpdateAssistantConversationSharingSettingsRequest, UpdateAssistantEnabledSettingsRequest, - UpdateAssistantFullLoggingSettingsRequest, UpdateAssistantSettingsRequest, UpdateEventSubmissionSettingsRequest } from './models'; @@ -339,13 +339,13 @@ export function postOAuthApplicationMutation() { })); } -export function putAdminAssistantEnabledSettingsMutation() { +export function putAdminAssistantConversationSharingSettingsMutation() { const queryClient = useQueryClient(); - return createMutation(() => ({ + return createMutation(() => ({ mutationFn: async (request) => { const client = useFetchClient(); - const response = await client.putJSON('admin/assistant-settings/enabled', request); + const response = await client.putJSON('admin/assistant-settings/conversation-sharing', request); if (!response.ok) { throw response.problem; @@ -360,13 +360,13 @@ export function putAdminAssistantEnabledSettingsMutation() { })); } -export function putAdminAssistantFullLoggingSettingsMutation() { +export function putAdminAssistantEnabledSettingsMutation() { const queryClient = useQueryClient(); - return createMutation(() => ({ + return createMutation(() => ({ mutationFn: async (request) => { const client = useFetchClient(); - const response = await client.putJSON('admin/assistant-settings/full-logging', request); + const response = await client.putJSON('admin/assistant-settings/enabled', request); if (!response.ok) { throw response.problem; @@ -374,8 +374,9 @@ export function putAdminAssistantFullLoggingSettingsMutation() { return response.data!; }, - onSuccess: (settings) => { + onSuccess: async (settings) => { queryClient.setQueryData(queryKeys.assistantSettings, settings); + await invalidateAssistantAccessQueries(queryClient); } })); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte index 59f064472d..0a4a95b4fd 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte @@ -9,8 +9,8 @@ import { Switch } from '$comp/ui/switch'; import { getAdminAssistantSettingsQuery, + putAdminAssistantConversationSharingSettingsMutation, putAdminAssistantEnabledSettingsMutation, - putAdminAssistantFullLoggingSettingsMutation, putAdminAssistantSettingsMutation } from '$features/admin/api.svelte'; import { type AssistantSettingsFormData, AssistantSettingsSchema } from '$features/admin/schemas'; @@ -21,12 +21,12 @@ const settingsQuery = getAdminAssistantSettingsQuery(); const updateEnabledSettings = putAdminAssistantEnabledSettingsMutation(); - const updateFullLoggingSettings = putAdminAssistantFullLoggingSettingsMutation(); + const updateConversationSharingSettings = putAdminAssistantConversationSharingSettingsMutation(); const updateSettings = putAdminAssistantSettingsMutation(); let assistantEnabled = $state(false); - let fullLoggingEnabled = $state(false); + let conversationSharingDefaultEnabled = $state(false); let loadedAvailabilityKey = $state(null); - let loadedFullLoggingEnabled = $state(); + let loadedConversationSharingDefaultEnabled = $state(); let loadedSettingsKey = $state(null); const settings = $derived(settingsQuery.data); const availabilityKey = $derived( @@ -71,12 +71,12 @@ }); $effect(() => { - if (!settings || loadedFullLoggingEnabled === settings.full_logging_enabled) { + if (!settings || loadedConversationSharingDefaultEnabled === settings.conversation_sharing_default_enabled) { return; } - loadedFullLoggingEnabled = settings.full_logging_enabled; - fullLoggingEnabled = settings.full_logging_enabled; + loadedConversationSharingDefaultEnabled = settings.conversation_sharing_default_enabled; + conversationSharingDefaultEnabled = settings.conversation_sharing_default_enabled; }); $effect(() => { @@ -124,15 +124,17 @@ } } - async function saveFullLogging() { + async function saveConversationSharingDefault() { try { - const saved = await updateFullLoggingSettings.mutateAsync({ - enabled: fullLoggingEnabled + const saved = await updateConversationSharingSettings.mutateAsync({ + enabled: conversationSharingDefaultEnabled }); - fullLoggingEnabled = saved.full_logging_enabled; - toast.success(saved.full_logging_enabled ? 'Exie full logging is enabled.' : 'Exie full logging is disabled.'); + conversationSharingDefaultEnabled = saved.conversation_sharing_default_enabled; + toast.success( + saved.conversation_sharing_default_enabled ? 'Exie conversation sharing default is enabled.' : 'Exie conversation sharing default is disabled.' + ); } catch { - toast.error('Failed to update Exie full logging.'); + toast.error('Failed to update Exie conversation sharing default.'); } } @@ -190,22 +192,26 @@ - Full logging + Conversation sharing default - Record submitted prompts and responses in session events for all organizations. Usage and error diagnostics remain available when off. Changes - apply to new turns. + Choose whether users share Exie messages and replies by default to help improve the feature. Users can change this in Exie; their saved choice + always takes precedence. Usage and error diagnostics remain available either way.
- +
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts index c815232ef8..ccf29bfd9f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts @@ -13,8 +13,8 @@ vi.mock('$features/admin/api.svelte', () => ({ data: { configured_enabled: true, configured_model: 'example/model', + conversation_sharing_default_enabled: state.enabled, enabled: true, - full_logging_enabled: state.enabled, is_configured: true, is_enabled_overridden: false, is_overridden: false, @@ -23,27 +23,27 @@ vi.mock('$features/admin/api.svelte', () => ({ isError: false, isPending: false }), + putAdminAssistantConversationSharingSettingsMutation: () => ({ isPending: false, mutateAsync: state.update }), putAdminAssistantEnabledSettingsMutation: () => ({ isPending: false, mutateAsync: vi.fn() }), - putAdminAssistantFullLoggingSettingsMutation: () => ({ isPending: false, mutateAsync: state.update }), putAdminAssistantSettingsMutation: () => ({ isPending: false, mutateAsync: vi.fn() }) })); import AssistantSettings from './assistant-settings.svelte'; -describe('Exie full logging settings', () => { +describe('Exie conversation sharing default settings', () => { beforeEach(() => { state.enabled = false; state.update.mockReset(); state.success.mockClear(); state.error.mockClear(); - state.update.mockImplementation(async ({ enabled }: { enabled: boolean }) => ({ full_logging_enabled: enabled })); + state.update.mockImplementation(async ({ enabled }: { enabled: boolean }) => ({ conversation_sharing_default_enabled: enabled })); }); it.each([false, true])('loads and saves the full logging switch from %s', async (enabled) => { state.enabled = enabled; render(AssistantSettings); - const toggle = screen.getByRole('switch', { name: 'Full logging' }); - const save = screen.getByRole('button', { name: 'Save Exie full logging' }); + const toggle = screen.getByRole('switch', { name: 'Conversation sharing default' }); + const save = screen.getByRole('button', { name: 'Save Exie conversation sharing default' }); await waitFor(() => expect(toggle.getAttribute('aria-checked')).toBe(String(enabled))); expect(save.hasAttribute('disabled')).toBe(true); await fireEvent.click(toggle); @@ -55,9 +55,9 @@ describe('Exie full logging settings', () => { it('shows a save failure without claiming the logging mode changed', async () => { state.update.mockRejectedValueOnce(new Error('Unavailable')); render(AssistantSettings); - await fireEvent.click(screen.getByRole('switch', { name: 'Full logging' })); - await fireEvent.click(screen.getByRole('button', { name: 'Save Exie full logging' })); - await waitFor(() => expect(state.error).toHaveBeenCalledWith('Failed to update Exie full logging.')); + await fireEvent.click(screen.getByRole('switch', { name: 'Conversation sharing default' })); + await fireEvent.click(screen.getByRole('button', { name: 'Save Exie conversation sharing default' })); + await waitFor(() => expect(state.error).toHaveBeenCalledWith('Failed to update Exie conversation sharing default.')); expect(state.success).not.toHaveBeenCalled(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts index 5de273c176..09666db567 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts @@ -2,8 +2,8 @@ import type { AssistantModelSettings, CountResult, EventSubmissionSettings, + UpdateAssistantConversationSharingSettings, UpdateAssistantEnabledSettings, - UpdateAssistantFullLoggingSettings, UpdateAssistantSettings, UpdateEventSubmissionSettings } from '$generated/api'; @@ -204,8 +204,8 @@ export type ShardMetric = { value: number; }; +export type UpdateAssistantConversationSharingSettingsRequest = UpdateAssistantConversationSharingSettings; export type UpdateAssistantEnabledSettingsRequest = UpdateAssistantEnabledSettings; -export type UpdateAssistantFullLoggingSettingsRequest = UpdateAssistantFullLoggingSettings; export type UpdateAssistantSettingsRequest = UpdateAssistantSettings; export type UpdateEventSubmissionSettingsRequest = UpdateEventSubmissionSettings; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts index 23a3767371..f089573b35 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts @@ -2,12 +2,13 @@ import type { QueryClient } from '@tanstack/svelte-query'; import { accessToken } from '$features/auth/index.svelte'; import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; -import { createQuery } from '@tanstack/svelte-query'; +import { createMutation, createQuery, useQueryClient } from '@tanstack/svelte-query'; -import type { AssistantAccess } from './models'; +import type { AssistantAccess, AssistantConversationSharingSettings } from './models'; export const queryKeys = { access: (organizationId: string | undefined) => [...queryKeys.type, 'access', organizationId] as const, + conversationSharing: ['Assistant', 'conversation-sharing'] as const, type: ['Assistant'] as const }; @@ -36,8 +37,45 @@ export function getAssistantAccessQuery(request: GetAssistantAccessRequest) { })); } +export function getAssistantConversationSharingQuery(request: { enabled: boolean }) { + return createQuery(() => ({ + enabled: () => !!accessToken.current && request.enabled, + queryFn: async ({ signal }) => { + const response = await useFetchClient().getJSON('assistant/conversation-sharing', { + signal + }); + if (!response.ok) { + throw response.problem; + } + return response.data!; + }, + queryKey: queryKeys.conversationSharing + })); +} + export async function invalidateAssistantAccessQueries(queryClient: QueryClient): Promise { await queryClient.invalidateQueries({ queryKey: queryKeys.type }); } + +export function putAssistantConversationSharingMutation() { + const queryClient = useQueryClient(); + return createMutation(() => ({ + mutationFn: async (request) => { + const response = await useFetchClient().putJSON('assistant/conversation-sharing', request); + if (!response.ok) { + throw response.problem; + } + return response.data!; + }, + onMutate: async () => { + await queryClient.cancelQueries({ + queryKey: queryKeys.conversationSharing + }); + }, + onSuccess: (settings) => { + queryClient.setQueryData(queryKeys.conversationSharing, settings); + } + })); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts index 844709ea9a..cb06ead740 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts @@ -43,6 +43,10 @@ export class AssistantTurnTelemetry { trackAssistantEvent('assistant.MessageSent', context, this.promptDetails); } + disableFullLogging(): void { + this.responseContent = undefined; + } + enableFullLogging(prompt: string): void { if (this.finished || this.responseContent !== undefined) { return; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte new file mode 100644 index 0000000000..2d2cb2eb28 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte @@ -0,0 +1,41 @@ + + +
+ + void onChange(value)} /> + + Share conversations to improve Exie + + Allow Exceptionless to review your submitted messages and replies. + {#if settings && !settings.is_overridden}Default: {settings.default_enabled ? 'on' : 'off'}. You can change this anytime.{/if} + + + {#if settings?.is_overridden} + + {/if} + + {#if error} + + + {/if} +
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte index c11d17d935..9808560b73 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte @@ -14,7 +14,14 @@ import Minimize2 from '@lucide/svelte/icons/minimize-2'; import { onDestroy, tick, untrack } from 'svelte'; - import type { AssistantAccessState, AssistantChatMessage, AssistantFeedback, AssistantPromptRequest, AssistantSuggestedAction } from '../models'; + import type { + AssistantAccessState, + AssistantChatMessage, + AssistantConversationSharingSettings, + AssistantFeedback, + AssistantPromptRequest, + AssistantSuggestedAction + } from '../models'; import { createAssistantChatRequest } from '../assistant-request'; import { type AssistantStreamEvent, readAssistantStream } from '../assistant-stream'; @@ -28,6 +35,7 @@ } from '../assistant-telemetry'; import { assistantToolResultFailed } from '../assistant-tool-result'; import AssistantComposer from './assistant-composer.svelte'; + import AssistantConversationSharing from './assistant-conversation-sharing.svelte'; import AssistantMessage from './assistant-message.svelte'; import AssistantUpgradeRequired from './assistant-upgrade-required.svelte'; @@ -35,11 +43,13 @@ accessMessage?: string; accessState?: AssistantAccessState; collapseHref?: string; + conversationSharing?: AssistantConversationSharingSettings; expandHref?: string; minimumPlanId?: string; mode?: 'page' | 'sheet'; onAccessChanged?: () => Promise | void; onCollapse?: () => void; + onConversationSharingChange?: (enabled: boolean | null) => Promise; onRetryAccess?: () => Promise | void; open?: boolean; organizationId?: string; @@ -52,11 +62,13 @@ accessMessage, accessState = 'available', collapseHref, + conversationSharing, expandHref, minimumPlanId, mode = 'sheet', onAccessChanged, onCollapse, + onConversationSharingChange, onRetryAccess, open = $bindable(false), organizationId, @@ -72,6 +84,12 @@ let isStreaming = $state(false); let isNearBottom = $state(true); let showToolCalls = $state(false); + let sharingSettings = $state(); + let sharingSuppressed = $state(false); + let sharingError = $state(); + let isSavingSharing = $state(false); + let requestedSharing = $state(null); + const isSharingEnabled = $derived(sharingSettings?.enabled === true && !sharingSuppressed); let showScrollToBottom = $state(false); let conversationElement = $state(); let abortController: AbortController | undefined; @@ -87,6 +105,35 @@ 'Which open stacks occurred most recently?', 'Explain what I can investigate on this page.' ]; + $effect(() => { + sharingSettings = conversationSharing; + if (conversationSharing?.enabled !== true) { + activeTurn?.disableFullLogging(); + } + }); + + async function changeConversationSharing(enabled: boolean | null) { + if (!onConversationSharingChange || isSavingSharing) { + return; + } + sharingError = undefined; + requestedSharing = enabled; + isSavingSharing = true; + if (enabled !== true) { + sharingSuppressed = true; + activeTurn?.disableFullLogging(); + } + try { + sharingSettings = await onConversationSharingChange(enabled); + sharingSuppressed = !sharingSettings.enabled; + } catch { + sharingError = sharingSuppressed + ? 'Could not save your choice. Sharing is paused on this page. Try again to save it for all devices.' + : 'Could not save your sharing choice. Please try again.'; + } finally { + isSavingSharing = false; + } + } $effect(() => { if (open && conversationElement) { void scrollToLatest('auto', true); @@ -303,7 +350,7 @@ throw new Error('The assistant returned an empty response.'); } - if (response.headers.get('X-Exie-Full-Logging') === 'true') { + if (isSharingEnabled && response.headers.get('X-Exie-Full-Logging') === 'true') { telemetry.enableFullLogging(userMessage.content); } @@ -637,6 +684,14 @@ {showToolCalls} /> AI can make mistakes. Check important changes. + changeConversationSharing(requestedSharing)} + settings={onConversationSharingChange ? sharingSettings : undefined} + /> {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index 59c50cc5ae..60470aaf22 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -99,7 +99,13 @@ describe('AssistantPanel', () => { expect(failed.error_message).toBe('Provider timed out'); }); - it.each([undefined, 'false', 'true'])('records full chat text only when the current response enables it (%s)', async (flag) => { + it.each([ + [undefined, true], + ['false', true], + ['true', true], + ['true', false], + ['true', undefined] + ] as const)('requires both server permission (%s) and the visible sharing choice (%s)', async (flag, enabled) => { vi.stubGlobal( 'fetch', vi.fn( @@ -110,10 +116,15 @@ describe('AssistantPanel', () => { ) ); render(AssistantPanel, { - props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'request-1', prompt: 'Full question' } } + props: { + conversationSharing: enabled === undefined ? undefined : { default_enabled: true, enabled, is_overridden: !enabled }, + open: true, + organizationId: 'organization-1', + promptRequest: { id: 'request-1', prompt: 'Full question' } + } }); await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); - if (flag === 'true') { + if (flag === 'true' && enabled) { expect(submitLog).toHaveBeenCalledWith('assistant.Prompt', 'Full question', expect.anything()); expect(submitLog).toHaveBeenCalledWith('assistant.Response', 'Full answer', expect.anything()); expect(submitLog).toHaveBeenCalledTimes(2); @@ -133,7 +144,12 @@ describe('AssistantPanel', () => { ); vi.stubGlobal('fetch', fetchMock); render(AssistantPanel, { - props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'request-1', prompt: 'First question' } } + props: { + conversationSharing: { default_enabled: true, enabled: true, is_overridden: false }, + open: true, + organizationId: 'organization-1', + promptRequest: { id: 'request-1', prompt: 'First question' } + } }); await screen.findByText('First answer'); const composer = screen.getByRole('textbox', { name: 'Message Exie' }); @@ -145,6 +161,76 @@ describe('AssistantPanel', () => { expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Second'); }); + it('shows the inherited default and saves an explicit choice or a reset', async () => { + const save = vi + .fn() + .mockResolvedValueOnce({ default_enabled: false, enabled: true, is_overridden: true }) + .mockResolvedValueOnce({ default_enabled: false, enabled: false, is_overridden: false }); + render(AssistantPanel, { + props: { + conversationSharing: { default_enabled: false, enabled: false, is_overridden: false }, + onConversationSharingChange: save, + open: true, + organizationId: 'organization-1' + } + }); + expect(screen.getByText(/Default: off/)).toBeTruthy(); + await fireEvent.click(screen.getByRole('switch', { name: 'Share conversations to improve Exie' })); + await waitFor(() => expect(save).toHaveBeenCalledWith(true)); + await fireEvent.click(await screen.findByRole('button', { name: 'Use default (off)' })); + await waitFor(() => expect(save).toHaveBeenLastCalledWith(null)); + await waitFor(() => expect(screen.getByRole('switch', { name: 'Share conversations to improve Exie' }).getAttribute('aria-checked')).toBe('false')); + }); + + it.each([false, true])('stops collecting the active reply and subsequent prompts when opting out (save fails: %s)', async (saveFails) => { + let controller!: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(value) { + controller = value; + } + }); + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce(new Response(stream, { headers: { 'X-Exie-Full-Logging': 'true' } })) + .mockResolvedValueOnce( + new Response('{"type":"text_delta","text":"Later answer"}\n{"type":"done"}\n', { headers: { 'X-Exie-Full-Logging': 'true' } }) + ) + ); + const saved = { default_enabled: true, enabled: false, is_overridden: true }; + const save = saveFails ? vi.fn().mockRejectedValueOnce(new Error('Offline')).mockResolvedValue(saved) : vi.fn().mockResolvedValue(saved); + render(AssistantPanel, { + props: { + conversationSharing: { default_enabled: true, enabled: true, is_overridden: false }, + onConversationSharingChange: save, + open: true, + organizationId: 'organization-1', + promptRequest: { id: 'sharing-request', prompt: 'Initial question' } + } + }); + await waitFor(() => expect(submitLog).toHaveBeenCalledWith('assistant.Prompt', 'Initial question', expect.anything())); + controller.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Active reply"}\n')); + await screen.findByText('Active reply'); + await fireEvent.click(screen.getByRole('switch', { name: 'Share conversations to improve Exie' })); + await waitFor(() => expect(save).toHaveBeenCalledWith(false)); + if (saveFails) await screen.findByText(/Sharing is paused on this page/); + controller.enqueue(new TextEncoder().encode('{"type":"done"}\n')); + controller.close(); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); + await fireEvent.input(screen.getByRole('textbox', { name: 'Message Exie' }), { target: { value: 'Later question' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Send message' })); + await waitFor(() => expect(eventData('assistant.ResponseCompleted', 1).outcome).toBe('completed')); + expect(submitLog).toHaveBeenCalledTimes(1); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Active reply'); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Later'); + if (saveFails) { + await fireEvent.click(screen.getByRole('button', { name: 'Retry saving' })); + await waitFor(() => expect(save).toHaveBeenCalledTimes(2)); + expect(save).toHaveBeenLastCalledWith(false); + } + }); + it('records closing while waiting without cancelling a response that finishes in the background', async () => { let streamController: ReadableStreamDefaultController; const stream = new ReadableStream({ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts index b186a40d21..52606bbc2e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts @@ -1,3 +1,5 @@ +export type { AssistantConversationSharingSettings } from '$generated/api'; + export interface AssistantAccess { enabled: boolean; has_access: boolean; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts index 29ab37b988..12132452d9 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts @@ -97,6 +97,12 @@ export interface AssistantChatRequest { conversation_id?: null | string; } +export interface AssistantConversationSharingSettings { + enabled: boolean; + default_enabled: boolean; + is_overridden: boolean; +} + export interface AssistantModelSettings { model: string; configured_model: string; @@ -105,7 +111,7 @@ export interface AssistantModelSettings { configured_enabled: boolean; is_enabled_overridden: boolean; is_configured: boolean; - full_logging_enabled: boolean; + conversation_sharing_default_enabled: boolean; } export interface BillingPlan { @@ -626,14 +632,18 @@ export interface TokenResult { token: string; } -export interface UpdateAssistantEnabledSettings { +export interface UpdateAssistantConversationSharing { enabled?: null | boolean; } -export interface UpdateAssistantFullLoggingSettings { +export interface UpdateAssistantConversationSharingSettings { enabled: boolean; } +export interface UpdateAssistantEnabledSettings { + enabled?: null | boolean; +} + export interface UpdateAssistantSettings { model?: null | string; } @@ -748,6 +758,7 @@ export interface User { email_address: string; avatar_file_name?: null | string; email_notifications_enabled: boolean; + assistant_conversation_sharing_enabled?: null | boolean; is_email_address_verified: boolean; verify_email_address_token?: null | string; /** @format date-time */ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index 811c397d7a..b3ec21926f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -123,6 +123,15 @@ export type AssistantChatRequestFormData = Infer< typeof AssistantChatRequestSchema >; +export const AssistantConversationSharingSettingsSchema = object({ + enabled: boolean(), + default_enabled: boolean(), + is_overridden: boolean(), +}); +export type AssistantConversationSharingSettingsFormData = Infer< + typeof AssistantConversationSharingSettingsSchema +>; + export const AssistantModelSettingsSchema = object({ model: string().min(1, "Model is required"), configured_model: string().min(1, "Configured model is required"), @@ -131,7 +140,7 @@ export const AssistantModelSettingsSchema = object({ configured_enabled: boolean(), is_enabled_overridden: boolean(), is_configured: boolean(), - full_logging_enabled: boolean(), + conversation_sharing_default_enabled: boolean(), }); export type AssistantModelSettingsFormData = Infer< typeof AssistantModelSettingsSchema @@ -731,18 +740,25 @@ export const TokenResultSchema = object({ }); export type TokenResultFormData = Infer; -export const UpdateAssistantEnabledSettingsSchema = object({ - enabled: boolean().nullable().optional(), +export const UpdateAssistantConversationSharingSchema = object({ + enabled: boolean().nullable(), }); -export type UpdateAssistantEnabledSettingsFormData = Infer< - typeof UpdateAssistantEnabledSettingsSchema +export type UpdateAssistantConversationSharingFormData = Infer< + typeof UpdateAssistantConversationSharingSchema >; -export const UpdateAssistantFullLoggingSettingsSchema = object({ +export const UpdateAssistantConversationSharingSettingsSchema = object({ enabled: boolean(), }); -export type UpdateAssistantFullLoggingSettingsFormData = Infer< - typeof UpdateAssistantFullLoggingSettingsSchema +export type UpdateAssistantConversationSharingSettingsFormData = Infer< + typeof UpdateAssistantConversationSharingSettingsSchema +>; + +export const UpdateAssistantEnabledSettingsSchema = object({ + enabled: boolean().nullable().optional(), +}); +export type UpdateAssistantEnabledSettingsFormData = Infer< + typeof UpdateAssistantEnabledSettingsSchema >; export const UpdateAssistantSettingsSchema = object({ @@ -878,6 +894,7 @@ export const UserSchema = object({ .nullable() .optional(), email_notifications_enabled: boolean(), + assistant_conversation_sharing_enabled: boolean().nullable().optional(), is_email_address_verified: boolean(), verify_email_address_token: string() .min(1, "Verify email address token is required") diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 6994b60722..4d34630f90 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -9,7 +9,12 @@ import { useSidebar } from '$comp/ui/sidebar'; import { env } from '$env/dynamic/public'; import { resolveAssistantAccessState } from '$features/assistant/access-state'; - import { getAssistantAccessQuery, invalidateAssistantAccessQueries } from '$features/assistant/api.svelte'; + import { + getAssistantAccessQuery, + getAssistantConversationSharingQuery, + invalidateAssistantAccessQueries, + putAssistantConversationSharingMutation + } from '$features/assistant/api.svelte'; import { setAssistantControls } from '$features/assistant/controls.svelte'; import { assistantPageContext, type AssistantResourceContext } from '$features/assistant/page-context.svelte'; import { getIntercomTokenQuery } from '$features/auth/api.svelte'; @@ -268,6 +273,12 @@ ) ); let isAssistantEnabled = $derived(assistantAccessState !== 'disabled'); + const assistantConversationSharingQuery = getAssistantConversationSharingQuery({ + get enabled() { + return assistantAccessState === 'available' && (isAssistantOpen || isAssistantPage); + } + }); + const updateAssistantConversationSharing = putAssistantConversationSharingMutation(); setAssistantControls({ ask: (prompt) => void askAssistant(prompt), @@ -765,6 +776,11 @@ accessMessage={assistantAccess?.message} accessState={assistantAccessState} collapseHref={isAssistantPage ? assistantReturnHref : undefined} + conversationSharing={assistantConversationSharingQuery.data} + onConversationSharingChange={(enabled) => + updateAssistantConversationSharing.mutateAsync({ + enabled + })} expandHref={!isAssistantPage ? assistantExpandHref : undefined} bind:open={isAssistantOpen} minimumPlanId={assistantAccess?.minimum_plan_id} diff --git a/src/Exceptionless.Web/Models/Admin/UpdateAssistantFullLoggingSettings.cs b/src/Exceptionless.Web/Models/Admin/UpdateAssistantConversationSharingSettings.cs similarity index 57% rename from src/Exceptionless.Web/Models/Admin/UpdateAssistantFullLoggingSettings.cs rename to src/Exceptionless.Web/Models/Admin/UpdateAssistantConversationSharingSettings.cs index 58c0c7f18e..3cc8cf73eb 100644 --- a/src/Exceptionless.Web/Models/Admin/UpdateAssistantFullLoggingSettings.cs +++ b/src/Exceptionless.Web/Models/Admin/UpdateAssistantConversationSharingSettings.cs @@ -1,6 +1,6 @@ namespace Exceptionless.Web.Models.Admin; -public sealed record UpdateAssistantFullLoggingSettings +public sealed record UpdateAssistantConversationSharingSettings { public bool Enabled { get; init; } } diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index 5522853172..f5de1088da 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -193,6 +193,7 @@ public static async Task Main(string[] args) builder.Services.AddHttpClient(nameof(AssistantService), client => client.Timeout = TimeSpan.FromMinutes(2)); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); diff --git a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json index b875ec0caa..ae89ccb00b 100644 --- a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json +++ b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json @@ -289,8 +289,8 @@ }, { "method": "PUT", - "route": "/api/v2/admin/assistant-settings/enabled", - "displayName": "HTTP: PUT api/v2/admin/assistant-settings/enabled", + "route": "/api/v2/admin/assistant-settings/conversation-sharing", + "displayName": "HTTP: PUT api/v2/admin/assistant-settings/conversation-sharing", "tags": [ "AdminEndpoints" ], @@ -303,8 +303,8 @@ }, { "method": "PUT", - "route": "/api/v2/admin/assistant-settings/full-logging", - "displayName": "HTTP: PUT api/v2/admin/assistant-settings/full-logging", + "route": "/api/v2/admin/assistant-settings/enabled", + "displayName": "HTTP: PUT api/v2/admin/assistant-settings/enabled", "tags": [ "AdminEndpoints" ], @@ -639,6 +639,30 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "GET", + "route": "/api/v2/assistant/conversation-sharing", + "displayName": "HTTP: GET api/v2/assistant/conversation-sharing =\u003E GetConversationSharingAsync", + "tags": [], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, + { + "method": "PUT", + "route": "/api/v2/assistant/conversation-sharing", + "displayName": "HTTP: PUT api/v2/assistant/conversation-sharing =\u003E SetConversationSharingAsync", + "tags": [], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "POST", "route": "/api/v2/auth/cancel-reset-password/{token:minlength(1)}", diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index d3e7414985..d329226969 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -20,6 +20,70 @@ } ], "paths": { + "/api/v2/assistant/conversation-sharing": { + "get": { + "tags": [ + "AssistantEndpoints" + ], + "operationId": "GetAssistantConversationSharing", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssistantConversationSharingSettings" + } + } + } + }, + "401": { + "description": "Unauthorized" + }, + "404": { + "description": "Not Found" + } + } + }, + "put": { + "tags": [ + "AssistantEndpoints" + ], + "description": "Saves the current user\u0027s choice. Null follows the admin default; true and false remain explicit choices when the default changes.", + "operationId": "SetAssistantConversationSharing", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateAssistantConversationSharing" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssistantConversationSharingSettings" + } + } + } + }, + "400": { + "description": "Bad Request" + }, + "401": { + "description": "Unauthorized" + }, + "404": { + "description": "Not Found" + } + } + } + }, "/api/v2/assistant/access": { "get": { "tags": [ @@ -300,7 +364,7 @@ } } }, - "/api/v2/admin/assistant-settings/full-logging": { + "/api/v2/admin/assistant-settings/conversation-sharing": { "put": { "tags": [ "AdminEndpoints" @@ -310,12 +374,12 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateAssistantFullLoggingSettings" + "$ref": "#/components/schemas/UpdateAssistantConversationSharingSettings" } }, "application/*\u002Bjson": { "schema": { - "$ref": "#/components/schemas/UpdateAssistantFullLoggingSettings" + "$ref": "#/components/schemas/UpdateAssistantConversationSharingSettings" } } }, @@ -11851,6 +11915,25 @@ } } }, + "AssistantConversationSharingSettings": { + "required": [ + "enabled", + "default_enabled", + "is_overridden" + ], + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "default_enabled": { + "type": "boolean" + }, + "is_overridden": { + "type": "boolean" + } + } + }, "AssistantModelSettings": { "required": [ "model", @@ -11860,7 +11943,7 @@ "configured_enabled", "is_enabled_overridden", "is_configured", - "full_logging_enabled" + "conversation_sharing_default_enabled" ], "type": "object", "properties": { @@ -11885,7 +11968,7 @@ "is_configured": { "type": "boolean" }, - "full_logging_enabled": { + "conversation_sharing_default_enabled": { "type": "boolean", "default": false } @@ -13630,7 +13713,10 @@ } } }, - "UpdateAssistantEnabledSettings": { + "UpdateAssistantConversationSharing": { + "required": [ + "enabled" + ], "type": "object", "properties": { "enabled": { @@ -13641,7 +13727,7 @@ } } }, - "UpdateAssistantFullLoggingSettings": { + "UpdateAssistantConversationSharingSettings": { "required": [ "enabled" ], @@ -13652,6 +13738,17 @@ } } }, + "UpdateAssistantEnabledSettings": { + "type": "object", + "properties": { + "enabled": { + "type": [ + "null", + "boolean" + ] + } + } + }, "UpdateAssistantSettings": { "type": "object", "properties": { @@ -14018,6 +14115,12 @@ "email_notifications_enabled": { "type": "boolean" }, + "assistant_conversation_sharing_enabled": { + "type": [ + "null", + "boolean" + ] + }, "is_email_address_verified": { "type": "boolean" }, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs index 0a2dcad3ce..6598740024 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs @@ -229,40 +229,40 @@ await SendRequestAsAsync(request => request } [Fact] - public async Task AssistantFullLoggingSettingsAsync_AsGlobalAdmin_PersistsBothStatesWithoutChangingOtherSettings() + public async Task AssistantConversationSharingSettingsAsync_AsGlobalAdmin_PersistsBothStatesWithoutChangingOtherSettings() { var initial = await SendRequestAsAsync(request => request .AsGlobalAdminUser().AppendPaths("admin", "assistant-settings").StatusCodeShouldBeOk()); Assert.NotNull(initial); - Assert.False(initial.FullLoggingEnabled); + Assert.False(initial.ConversationSharingDefaultEnabled); foreach (bool enabled in new[] { true, false }) { var updated = await SendRequestAsAsync(request => request - .Put().AsGlobalAdminUser().AppendPaths("admin", "assistant-settings", "full-logging") - .Content(new UpdateAssistantFullLoggingSettings { Enabled = enabled }).StatusCodeShouldBeOk()); + .Put().AsGlobalAdminUser().AppendPaths("admin", "assistant-settings", "conversation-sharing") + .Content(new UpdateAssistantConversationSharingSettings { Enabled = enabled }).StatusCodeShouldBeOk()); Assert.NotNull(updated); - Assert.Equal(enabled, updated.FullLoggingEnabled); + Assert.Equal(enabled, updated.ConversationSharingDefaultEnabled); await GetService().RemoveAllAsync(); var persisted = await SendRequestAsAsync(request => request .AsGlobalAdminUser().AppendPaths("admin", "assistant-settings").StatusCodeShouldBeOk()); Assert.NotNull(persisted); - Assert.Equal(enabled, persisted.FullLoggingEnabled); + Assert.Equal(enabled, persisted.ConversationSharingDefaultEnabled); Assert.Equal(initial.Model, persisted.Model); Assert.Equal(initial.Enabled, persisted.Enabled); } } [Fact] - public Task AssistantFullLoggingSettingsAsync_AsOrganizationUser_ReturnsForbidden() => SendRequestAsync(request => request - .Put().AsTestOrganizationUser().AppendPaths("admin", "assistant-settings", "full-logging") - .Content(new UpdateAssistantFullLoggingSettings { Enabled = true }).StatusCodeShouldBeForbidden()); + public Task AssistantConversationSharingSettingsAsync_AsOrganizationUser_ReturnsForbidden() => SendRequestAsync(request => request + .Put().AsTestOrganizationUser().AppendPaths("admin", "assistant-settings", "conversation-sharing") + .Content(new UpdateAssistantConversationSharingSettings { Enabled = true }).StatusCodeShouldBeForbidden()); [Fact] - public Task AssistantFullLoggingSettingsAsync_AsAnonymous_ReturnsUnauthorized() => SendRequestAsync(request => request - .Put().AsAnonymousUser().AppendPaths("admin", "assistant-settings", "full-logging") - .Content(new UpdateAssistantFullLoggingSettings { Enabled = true }).StatusCodeShouldBeUnauthorized()); + public Task AssistantConversationSharingSettingsAsync_AsAnonymous_ReturnsUnauthorized() => SendRequestAsync(request => request + .Put().AsAnonymousUser().AppendPaths("admin", "assistant-settings", "conversation-sharing") + .Content(new UpdateAssistantConversationSharingSettings { Enabled = true }).StatusCodeShouldBeUnauthorized()); [Fact] public async Task EventSubmissionSettingsAsync_AsGlobalAdmin_UpdatesAndClearsRuntimeOverride() @@ -1392,6 +1392,6 @@ private sealed record AssistantModelSettingsResponse( bool ConfiguredEnabled, bool IsEnabledOverridden, bool IsConfigured, - bool FullLoggingEnabled); + bool ConversationSharingDefaultEnabled); private sealed record RequeueResult([property: JsonPropertyName("enqueued")] int Enqueued); } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs index a458915a10..9626c227a5 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs @@ -1,5 +1,7 @@ using System.Net; using System.Text.Json; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; using Exceptionless.Web.Assistant; @@ -19,6 +21,64 @@ protected override async Task ResetDataAsync() await GetService().CreateDataAsync(); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ConversationSharingAsync_PreservesExplicitChoicesAcrossDefaultChanges(bool choice) + { + var users = GetService(); + var user = await users.GetByEmailAddressAsync(SampleDataService.TEST_ORG_USER_EMAIL); + var otherUser = await users.GetByEmailAddressAsync(SampleDataService.TEST_USER_EMAIL); + Assert.NotNull(user); + Assert.NotNull(otherUser); + var systemSettings = GetService(); + var sharingService = GetService(); + + Assert.Null(user.AssistantConversationSharingEnabled); + Assert.Equal(new(false, false, false), await ReadAsync()); + await systemSettings.UpdateAsync(otherUser.Id, settings => settings.AssistantConversationSharingDefaultEnabled = choice); + Assert.Equal(new(choice, choice, false), await ReadAsync()); + + // An explicit choice equal to the current default must survive later default changes. + Assert.Equal(new(choice, choice, true), await SaveAsync(choice)); + await systemSettings.UpdateAsync(otherUser.Id, settings => settings.AssistantConversationSharingDefaultEnabled = !choice); + Assert.Equal(new(choice, !choice, true), await ReadAsync()); + Assert.Equal(new(choice, !choice, true), await sharingService.GetAsync(user.Id)); + + var savedUser = await users.GetByIdAsync(user.Id); + Assert.NotNull(savedUser); + Assert.Equal(choice, savedUser.AssistantConversationSharingEnabled); + Assert.Equal(user.FullName, savedUser.FullName); + Assert.Equal(user.EmailAddress, savedUser.EmailAddress); + Assert.Equal(user.OrganizationIds, savedUser.OrganizationIds); + Assert.Equal(user.EmailNotificationsEnabled, savedUser.EmailNotificationsEnabled); + Assert.Null((await users.GetByIdAsync(otherUser.Id))!.AssistantConversationSharingEnabled); + + Assert.Equal(new(!choice, !choice, false), await SaveAsync(null)); + Assert.Null((await users.GetByIdAsync(user.Id))!.AssistantConversationSharingEnabled); + Assert.Equal(new(!choice, !choice, false), await ReadAsync()); + + Task ReadAsync() => SendRequestAsAsync(request => request + .AsTestOrganizationUser().AppendPath("assistant/conversation-sharing").StatusCodeShouldBeOk()); + Task SaveAsync(bool? enabled) => SendRequestAsAsync(request => request + .Put().AsTestOrganizationUser().AppendPath("assistant/conversation-sharing") + .Content(new UpdateAssistantConversationSharing { Enabled = enabled }).StatusCodeShouldBeOk()); + } + + [Fact] + public Task SetConversationSharingAsync_Anonymous_ReturnsUnauthorized() => SendRequestAsync(request => request + .Put().AsAnonymousUser().AppendPath("assistant/conversation-sharing") + .Content(new UpdateAssistantConversationSharing { Enabled = true }).StatusCodeShouldBeUnauthorized()); + + [Fact] + public Task GetConversationSharingAsync_Anonymous_ReturnsUnauthorized() => SendRequestAsync(request => request + .AsAnonymousUser().AppendPath("assistant/conversation-sharing").StatusCodeShouldBeUnauthorized()); + + [Fact] + public Task SetConversationSharingAsync_MissingChoice_ReturnsBadRequest() => SendRequestAsync(request => request + .Put().AsTestOrganizationUser().AppendPath("assistant/conversation-sharing") + .Content(new { }).ExpectedStatus(HttpStatusCode.BadRequest)); + [Fact] public Task StreamAssistantChatAsync_Anonymous_ReturnsUnauthorized() { diff --git a/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs index ca8c1c2f42..88e46c6cb6 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs @@ -10,10 +10,20 @@ namespace Exceptionless.Tests.Assistant; public sealed class AssistantModelSettingsServiceTests { + [Fact] + public void LegacyUser_FollowsConversationSharingDefault() + { + var user = JsonSerializer.Deserialize("{}"); + Assert.NotNull(user); + Assert.Null(user.AssistantConversationSharingEnabled); + Assert.False(AssistantConversationSharingService.Resolve(user.AssistantConversationSharingEnabled, false).Enabled); + Assert.True(AssistantConversationSharingService.Resolve(user.AssistantConversationSharingEnabled, true).Enabled); + } + [Theory] [InlineData(false)] [InlineData(true)] - public async Task SetFullLoggingEnabledAsync_NewAndLegacySettings_DefaultOffAndPreserveOtherSettings(bool hasLegacyRecord) + public async Task SetConversationSharingDefaultEnabledAsync_NewAndLegacySettings_DefaultOffAndPreserveOtherSettings(bool hasLegacyRecord) { var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost" }).Build()); @@ -25,17 +35,17 @@ public async Task SetFullLoggingEnabledAsync_NewAndLegacySettings_DefaultOffAndP }, options, TimeProvider.System); var service = new AssistantModelSettingsService(systemSettings, options); - Assert.False((await service.GetAsync()).FullLoggingEnabled); - Assert.False(await systemSettings.IsAssistantFullLoggingEnabledAsync()); + Assert.False((await service.GetAsync()).ConversationSharingDefaultEnabled); + Assert.False(await systemSettings.IsAssistantConversationSharingDefaultEnabledAsync()); - Assert.True((await service.SetFullLoggingEnabledAsync(true, "admin-user")).FullLoggingEnabled); - Assert.True(await systemSettings.IsAssistantFullLoggingEnabledAsync()); + Assert.True((await service.SetConversationSharingDefaultEnabledAsync(true, "admin-user")).ConversationSharingDefaultEnabled); + Assert.True(await systemSettings.IsAssistantConversationSharingDefaultEnabledAsync()); await service.SetModelAsync("example/model", "admin-user"); await service.SetEnabledAsync(true, "admin-user"); - Assert.True((await service.GetAsync()).FullLoggingEnabled); + Assert.True((await service.GetAsync()).ConversationSharingDefaultEnabled); - Assert.False((await service.SetFullLoggingEnabledAsync(false, "admin-user")).FullLoggingEnabled); - Assert.False(await systemSettings.IsAssistantFullLoggingEnabledAsync()); + Assert.False((await service.SetConversationSharingDefaultEnabledAsync(false, "admin-user")).ConversationSharingDefaultEnabled); + Assert.False(await systemSettings.IsAssistantConversationSharingDefaultEnabledAsync()); Assert.NotNull(persisted); Assert.Equal("example/model", persisted.AssistantModel); Assert.True(persisted.AssistantEnabled); diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 207429f77f..060b96464c 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -1339,11 +1339,14 @@ public async Task StreamAsync_RepeatedRawDsmlResponse_EmitsClearErrorAndCompleti .Build()); var service = CreateAssistantService(handler, appOptions); var events = new List(); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); await foreach (var item in service.StreamAsync( new AssistantChatRequest([new AssistantChatMessage("user", "Find recent errors")]), "user-id", CreatePlanOptions(), + diagnostics, TestContext.Current.CancellationToken)) { events.Add(item); @@ -1358,10 +1361,15 @@ public async Task StreamAsync_RepeatedRawDsmlResponse_EmitsClearErrorAndCompleti }, item => Assert.Equal("done", item.Type)); Assert.DoesNotContain(events, item => item.Type == "text_delta"); + var providerEntries = logger.Entries.Where(entry => entry.Properties.ContainsKey("ProviderOutcome")).ToArray(); + Assert.Equal(2, providerEntries.Length); + Assert.All(providerEntries, entry => Assert.Equal("malformed_response", entry.Properties["ProviderOutcome"])); } - [Fact] - public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutTools() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutTools(bool providerIgnoresToolLimit) { const string toolCallResponse = """ data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"unknown_tool","arguments":"{}"}}]}}]} @@ -1373,7 +1381,7 @@ public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutT toolCallResponse, toolCallResponse.Replace("call-1", "call-2"), toolCallResponse.Replace("call-1", "call-3"), - """ + providerIgnoresToolLimit ? toolCallResponse.Replace("call-1", "call-4") : """ data: {"choices":[{"delta":{"content":"Here is the available result."}}]} data: [DONE] @@ -1388,6 +1396,8 @@ public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutT .Build()); var service = CreateAssistantService(handler, appOptions); var events = new List(); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); await foreach (var item in service.StreamAsync( new AssistantChatRequest( @@ -1395,6 +1405,7 @@ [new AssistantChatMessage("user", "Investigate the errors")], OrganizationId: "organization-id"), "user-id", CreatePlanOptions(), + diagnostics, TestContext.Current.CancellationToken)) { events.Add(item); @@ -1404,9 +1415,19 @@ [new AssistantChatMessage("user", "Investigate the errors")], Assert.All(handler.RequestBodies.Take(3), body => Assert.Contains("\"tools\":", body)); Assert.DoesNotContain("\"tools\":", handler.RequestBodies[3]); Assert.Contains("The tool budget is exhausted", handler.RequestBodies[3]); - Assert.Contains(events, item => item.Text == "Here is the available result."); Assert.Equal("done", events[^1].Type); - Assert.DoesNotContain(events, item => item.Type == "error"); + var finalProviderEntry = logger.Entries.Last(entry => entry.Properties.ContainsKey("ProviderOutcome")); + if (providerIgnoresToolLimit) + { + Assert.Equal("tool_round_limit", Assert.Single(events, item => item.Type == "error").FailureCode); + Assert.Equal("tool_round_limit", finalProviderEntry.Properties["ProviderOutcome"]); + } + else + { + Assert.Contains(events, item => item.Text == "Here is the available result."); + Assert.DoesNotContain(events, item => item.Type == "error"); + Assert.Equal("completed", finalProviderEntry.Properties["ProviderOutcome"]); + } } [Fact] diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 2846fa386f..8511776aeb 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -71,7 +71,11 @@ The Svelte app submits Exie events through the existing Exceptionless browser cl Each submitted prompt produces an `assistant.MessageSent` feature usage event. A turn produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` feature usage event. These events record character counts and outcomes. Failure events retain the error displayed to the user in `error_message`, capped at 2,048 characters for diagnosis. Existing application error collection is unchanged. -Global admins can enable **Full logging** beside the Exie availability and model settings. It defaults to off, including for existing settings records with no logging field. The setting persists across restarts and can be changed through `PUT /api/v2/admin/assistant-settings/full-logging` with `{ "enabled": true }` or `{ "enabled": false }`. Each accepted turn reads the setting and returns `X-Exie-Full-Logging: true` or `false`; the browser records transcript text only for an explicit `true`. Changes apply to new turns, including subsequent turns in an existing conversation. An in-flight turn retains the logging mode selected when it started. +Global admins control **Conversation sharing default** beside Exie availability and model settings. It defaults to off for new and legacy settings records. Enable it during the early rollout through the settings page or `PUT /api/v2/admin/assistant-settings/conversation-sharing` with `{ "enabled": true }`; set it to false to stop sharing by default later. + +Users see **Share conversations to improve Exie** beside the composer and can turn it on or off for their account across conversations and devices. `GET /api/v2/assistant/conversation-sharing` returns the effective setting, default, and whether the user has chosen. The authenticated user's `PUT` at the same route saves `{ "enabled": true }` or `{ "enabled": false }`; `{ "enabled": null }` restores the default. Existing users have no override and follow the admin default. Explicit choices always win, even when originally saved equal to the default. Enabling the default therefore uses opt-out sharing; the visible control identifies inherited defaults and allows an immediate change. + +Each accepted turn reads the saved user choice and returns `X-Exie-Full-Logging`. Transcript capture requires both an explicit true header and an enabled, loaded sharing control. Turning sharing off discards the active reply buffer and prevents subsequent transcript capture; already submitted events are retained. If saving an opt-out fails, capture stays paused on the current page and the UI asks the user to retry saving for other devices. Changes elsewhere are read for the next turn; a reply already in progress on another device retains the choice selected at its start. Usage, feedback, and error diagnostics continue regardless of sharing. When enabled, full logging adds an `assistant.Prompt` log event and one assembled `assistant.Response` log event in the existing session. Each message is capped at 16,384 characters, with original character count and truncation metadata. When disabled, prompt and response text is not copied into telemetry. Drafts, individual streamed chunks, reasoning, raw tool arguments/results, and generated suggestion labels/destinations are never submitted. Suggestion events record only whether the action navigated or submitted a prompt. diff --git a/tests/http/admin.http b/tests/http/admin.http index 9e954661ae..7521348fe9 100644 --- a/tests/http/admin.http +++ b/tests/http/admin.http @@ -58,8 +58,8 @@ Authorization: Bearer {{token}} GET {{apiUrl}}/admin/assistant-settings Authorization: Bearer {{token}} -### Enable Full Exie Conversation Logging -PUT {{apiUrl}}/admin/assistant-settings/full-logging +### Enable Exie Conversation Sharing by Default (Saved User Choices Take Precedence) +PUT {{apiUrl}}/admin/assistant-settings/conversation-sharing Authorization: Bearer {{token}} Content-Type: application/json @@ -67,8 +67,8 @@ Content-Type: application/json "enabled": true } -### Disable Full Exie Conversation Logging (Usage and Error Diagnostics Continue) -PUT {{apiUrl}}/admin/assistant-settings/full-logging +### Disable Exie Conversation Sharing by Default (Usage and Error Diagnostics Continue) +PUT {{apiUrl}}/admin/assistant-settings/conversation-sharing Authorization: Bearer {{token}} Content-Type: application/json diff --git a/tests/http/assistant.http b/tests/http/assistant.http index d76e842485..41e85e2919 100644 --- a/tests/http/assistant.http +++ b/tests/http/assistant.http @@ -19,8 +19,22 @@ Content-Type: application/json GET {{apiUrl}}/assistant/access?organization_id={{organizationId}} Authorization: Bearer {{login.response.body.$.token}} +### Get your effective conversation sharing choice +GET {{apiUrl}}/assistant/conversation-sharing +Authorization: Bearer {{login.response.body.$.token}} + +### Turn off conversation sharing for your account (true enables; null follows the admin default) +PUT {{apiUrl}}/assistant/conversation-sharing +Authorization: Bearer {{login.response.body.$.token}} +Content-Type: application/json + +{ + "enabled": false +} + ### Stream an assistant response -# X-Exie-Full-Logging: true enables prompt/response session logs for this turn. +# X-Exie-Full-Logging reflects the saved user choice, falling back to the admin default. +# The browser also requires the visible sharing control to be enabled. # A missing or false header keeps transcript logging off. Usage and error events continue. POST {{apiUrl}}/assistant/chat Authorization: Bearer {{login.response.body.$.token}} From 174fc41837b46356e06f05960a9738b67a06303c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 8 Sep 2026 23:58:37 -0500 Subject: [PATCH 13/18] Resolve Exie sharing settings before reserving a turn --- src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs | 2 +- .../assistant/components/assistant-conversation-sharing.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs index b0def02bbd..807d51df11 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs @@ -93,6 +93,7 @@ private static async Task StreamChatAsync( OrganizationId = organizationId, ConversationId = conversationId ?? Guid.NewGuid().ToString("N") }; + bool fullLoggingEnabled = (await conversationSharingService.GetAsync(userId))?.Enabled == true; var planOptions = access.PlanOptions!; await using var turnReservation = await assistantUsageService.TryStartTurnAsync(organizationId, planOptions); if (!turnReservation.Allowed) @@ -115,7 +116,6 @@ private static async Task StreamChatAsync( httpContext.Response.ContentType = "application/x-ndjson"; httpContext.Response.Headers.CacheControl = "no-store"; httpContext.Response.Headers.Append("X-Accel-Buffering", "no"); - bool fullLoggingEnabled = (await conversationSharingService.GetAsync(userId))?.Enabled == true; using var turnCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(httpContext.RequestAborted); turnCancellationSource.CancelAfter(TimeSpan.FromSeconds(AssistantLimits.MaximumTurnDurationSeconds)); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte index 2d2cb2eb28..e77a98f7de 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte @@ -24,7 +24,7 @@ Share conversations to improve Exie - Allow Exceptionless to review your submitted messages and replies. + Allow Exceptionless to review your submitted messages and replies. Usage and error diagnostics remain enabled when sharing is off. {#if settings && !settings.is_overridden}Default: {settings.default_enabled ? 'on' : 'off'}. You can change this anytime.{/if} From c5002626486de4bd9b0c3337589899ab74d48c00 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 00:16:06 -0500 Subject: [PATCH 14/18] Explain Exie outcome counts with tooltips --- .../src/routes/(app)/system/exie/+page.svelte | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte index 861fdf04b5..5635abcb88 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte @@ -10,6 +10,7 @@ import { Input } from '$comp/ui/input'; import { Skeleton } from '$comp/ui/skeleton'; import * as Table from '$comp/ui/table'; + import * as Tooltip from '$comp/ui/tooltip'; import { getAdminAssistantUsageQuery } from '$features/admin/api.svelte'; import { getBlockedCount, getTotalTokens, getUsageRisk, getUtcMonthKey, type UsageRisk } from '$features/admin/assistant-usage'; import Bot from '@lucide/svelte/icons/bot'; @@ -204,13 +205,40 @@ {/if} - - / - 0 ? 'text-destructive' : 'text-muted-foreground'} - > - / - completed / failed / cancelled + + + completed turns + + + Completed: Turns that finished without a reported error. This does not indicate whether the user found the + answer helpful. + + + + + 0 ? 'text-destructive' : 'text-muted-foreground' + ]} + > + failed turns + + Failed: Turns that ended with an error or timed out. + + + + + cancelled turns + + + Cancelled: Turns stopped by the user or interrupted by a browser disconnect. + + {#if blockedCount > 0} From 25e45c0c17873e28ec31d692f0387e818e134bd6 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 13:10:01 -0500 Subject: [PATCH 15/18] Compact the Exie conversation sharing control --- .../assistant-conversation-sharing.svelte | 43 +++++++++++-------- .../components/assistant-panel.svelte | 20 +++++---- .../components/assistant-panel.svelte.test.ts | 13 +++++- tests/Exceptionless.Tests/Assistant/README.md | 2 +- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte index e77a98f7de..f9a639a9f2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte @@ -1,6 +1,7 @@ -
- - void onChange(value)} /> - - Share conversations to improve Exie - - Allow Exceptionless to review your submitted messages and replies. Usage and error diagnostics remain enabled when sharing is off. - {#if settings && !settings.is_overridden}Default: {settings.default_enabled ? 'on' : 'off'}. You can change this anytime.{/if} - - + + + Chat sharing: {!settings ? 'Loading…' : checked ? 'On' : 'Off'}{error ? ' · Not saved' : ''} + + + + Share conversations to improve Exie + void onChange(value)} /> + +

Allow Exceptionless to review your messages and replies. Usage and error diagnostics stay enabled.

+ {#if settings && !settings.is_overridden} +

Using the default: {settings.default_enabled ? 'on' : 'off'}. You can change this anytime.

+ {/if} {#if settings?.is_overridden} - {/if} - - {#if error} - - - {/if} -
+ {#if error} + + + {/if} + + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte index 9808560b73..a8622f05c2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte @@ -683,15 +683,17 @@ onSubmit={(value) => void submitPrompt(value)} {showToolCalls} /> - AI can make mistakes. Check important changes. - changeConversationSharing(requestedSharing)} - settings={onConversationSharingChange ? sharingSettings : undefined} - /> +
+ AI can make mistakes. Check important changes. + changeConversationSharing(requestedSharing)} + settings={onConversationSharingChange ? sharingSettings : undefined} + /> +
{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index 60470aaf22..d682d79b63 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -174,12 +174,16 @@ describe('AssistantPanel', () => { organizationId: 'organization-1' } }); - expect(screen.getByText(/Default: off/)).toBeTruthy(); + expect(screen.queryByRole('switch', { name: 'Share conversations to improve Exie' })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off' })); + expect(screen.getByText(/Using the default: off/)).toBeTruthy(); await fireEvent.click(screen.getByRole('switch', { name: 'Share conversations to improve Exie' })); await waitFor(() => expect(save).toHaveBeenCalledWith(true)); await fireEvent.click(await screen.findByRole('button', { name: 'Use default (off)' })); await waitFor(() => expect(save).toHaveBeenLastCalledWith(null)); await waitFor(() => expect(screen.getByRole('switch', { name: 'Share conversations to improve Exie' }).getAttribute('aria-checked')).toBe('false')); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off' })); + await waitFor(() => expect(screen.queryByRole('switch', { name: 'Share conversations to improve Exie' })).toBeNull()); }); it.each([false, true])('stops collecting the active reply and subsequent prompts when opting out (save fails: %s)', async (saveFails) => { @@ -212,9 +216,13 @@ describe('AssistantPanel', () => { await waitFor(() => expect(submitLog).toHaveBeenCalledWith('assistant.Prompt', 'Initial question', expect.anything())); controller.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Active reply"}\n')); await screen.findByText('Active reply'); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: On' })); await fireEvent.click(screen.getByRole('switch', { name: 'Share conversations to improve Exie' })); await waitFor(() => expect(save).toHaveBeenCalledWith(false)); - if (saveFails) await screen.findByText(/Sharing is paused on this page/); + if (saveFails) { + await screen.findByText(/Sharing is paused on this page/); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off · Not saved' })); + } controller.enqueue(new TextEncoder().encode('{"type":"done"}\n')); controller.close(); await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); @@ -225,6 +233,7 @@ describe('AssistantPanel', () => { expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Active reply'); expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Later'); if (saveFails) { + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off · Not saved' })); await fireEvent.click(screen.getByRole('button', { name: 'Retry saving' })); await waitFor(() => expect(save).toHaveBeenCalledTimes(2)); expect(save).toHaveBeenLastCalledWith(false); diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 8511776aeb..8d621723f6 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -73,7 +73,7 @@ Each submitted prompt produces an `assistant.MessageSent` feature usage event. A Global admins control **Conversation sharing default** beside Exie availability and model settings. It defaults to off for new and legacy settings records. Enable it during the early rollout through the settings page or `PUT /api/v2/admin/assistant-settings/conversation-sharing` with `{ "enabled": true }`; set it to false to stop sharing by default later. -Users see **Share conversations to improve Exie** beside the composer and can turn it on or off for their account across conversations and devices. `GET /api/v2/assistant/conversation-sharing` returns the effective setting, default, and whether the user has chosen. The authenticated user's `PUT` at the same route saves `{ "enabled": true }` or `{ "enabled": false }`; `{ "enabled": null }` restores the default. Existing users have no override and follow the admin default. Explicit choices always win, even when originally saved equal to the default. Enabling the default therefore uses opt-out sharing; the visible control identifies inherited defaults and allows an immediate change. +Users see a compact **Chat sharing: On/Off** control beside the composer disclaimer. It opens a popover where they can change sharing for their account across conversations and devices. `GET /api/v2/assistant/conversation-sharing` returns the effective setting, default, and whether the user has chosen. The authenticated user's `PUT` at the same route saves `{ "enabled": true }` or `{ "enabled": false }`; `{ "enabled": null }` restores the default. Existing users have no override and follow the admin default. Explicit choices always win, even when originally saved equal to the default. Enabling the default therefore uses opt-out sharing; the visible control identifies inherited defaults and allows an immediate change. Each accepted turn reads the saved user choice and returns `X-Exie-Full-Logging`. Transcript capture requires both an explicit true header and an enabled, loaded sharing control. Turning sharing off discards the active reply buffer and prevents subsequent transcript capture; already submitted events are retained. If saving an opt-out fails, capture stays paused on the current page and the UI asks the user to retry saving for other devices. Changes elsewhere are read for the next turn; a reply already in progress on another device retains the choice selected at its start. Usage, feedback, and error diagnostics continue regardless of sharing. From 021d07ff5385c6e02f9d86012393cf45a3c001dd Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 13:48:30 -0500 Subject: [PATCH 16/18] Enable local browser self-reporting in development --- README.md | 2 ++ src/Exceptionless.AppHost/Program.cs | 3 +++ src/Exceptionless.AppHost/appsettings.Development.json | 1 + src/Exceptionless.Core/Utility/SampleDataService.cs | 2 +- src/Exceptionless.Web/ClientApp/src/hooks.client.ts | 3 ++- tests/Exceptionless.Tests/Assistant/README.md | 2 ++ 6 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f49f665313..891a10891c 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ After startup: 1. Open `https://localhost:7121/` if a browser does not open automatically. 2. In `Development` mode, a global administrator user `admin@exceptionless.test` with password `tester` is created automatically. + The Aspire Svelte app reports its browser errors, usage, and sessions to the seeded **Exceptionless → Exceptionless** project by default (named **API** in older development data). Telemetry uses the current browser origin, so forwarded localhost ports work without extra configuration. Exie conversation text still follows the user's chat-sharing preference. To override the telemetry destination, set `PUBLIC_EXCEPTIONLESS_API_KEY` and `PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL` in the AppHost environment; an empty key disables automatic browser reporting. + Notes: 1. Running `Exceptionless.AppHost` starts the app and required infrastructure together. diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs index b12b48aefe..fc5915374e 100644 --- a/src/Exceptionless.AppHost/Program.cs +++ b/src/Exceptionless.AppHost/Program.cs @@ -192,7 +192,9 @@ .WithBrowserLogs() .WithReference(api) .WithReference(oldApp) + .WithEnvironment("PUBLIC_EXCEPTIONLESS_API_KEY", builder.Configuration["PUBLIC_EXCEPTIONLESS_API_KEY"]) .WithEnvironment("PUBLIC_EXCEPTIONLESS_SERVER_URL", exceptionlessServerUrl) + .WithEnvironment("PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL", builder.Configuration["PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL"] ?? String.Empty) .WithEnvironment("PORT", appPort.ToString()) .WithEndpoint("http", e => { @@ -203,6 +205,7 @@ e.IsProxied = false; }) .WithHttpsDeveloperCertificate() + .WaitFor(api) .WithUrlForEndpoint("http", u => { u.DisplayText = "Open App"; diff --git a/src/Exceptionless.AppHost/appsettings.Development.json b/src/Exceptionless.AppHost/appsettings.Development.json index 0c208ae918..a0748efc92 100644 --- a/src/Exceptionless.AppHost/appsettings.Development.json +++ b/src/Exceptionless.AppHost/appsettings.Development.json @@ -1,4 +1,5 @@ { + "PUBLIC_EXCEPTIONLESS_API_KEY": "Bx7JgglstPG544R34Tw9T7RlCed3OIwtYXVeyhT2", "Logging": { "LogLevel": { "Default": "Information", diff --git a/src/Exceptionless.Core/Utility/SampleDataService.cs b/src/Exceptionless.Core/Utility/SampleDataService.cs index ce6e1a6faa..49b1d8feba 100644 --- a/src/Exceptionless.Core/Utility/SampleDataService.cs +++ b/src/Exceptionless.Core/Utility/SampleDataService.cs @@ -256,7 +256,7 @@ public async Task CreateInternalOrganizationAndProjectAsync(string userId) var project = new Project { Id = INTERNAL_PROJECT_ID, - Name = "API", + Name = "Exceptionless", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = _timeProvider.GetUtcNow().UtcDateTime.Date.AddDays(1).AddHours(1).Ticks }; diff --git a/src/Exceptionless.Web/ClientApp/src/hooks.client.ts b/src/Exceptionless.Web/ClientApp/src/hooks.client.ts index 5ddf0f4807..5cbd599a13 100644 --- a/src/Exceptionless.Web/ClientApp/src/hooks.client.ts +++ b/src/Exceptionless.Web/ClientApp/src/hooks.client.ts @@ -34,7 +34,8 @@ export const init: ClientInit = async () => { await Exceptionless.startup((c) => { c.apiKey = env.PUBLIC_EXCEPTIONLESS_API_KEY; - c.serverUrl = env.PUBLIC_EXCEPTIONLESS_SERVER_URL || window.location.origin; + c.serverUrl = + PUBLIC_EXCEPTIONLESS_SERVER_URL || (env.PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL ?? env.PUBLIC_EXCEPTIONLESS_SERVER_URL) || window.location.origin; c.defaultTags.push('UI', 'Svelte'); if (env.PUBLIC_APP_VERSION) { diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 8d621723f6..1fa4df6161 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -69,6 +69,8 @@ dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll --fil The Svelte app submits Exie events through the existing Exceptionless browser client. They share the signed-in user's session, client configuration, queue, tags, and event exclusions. They go to the app's configured telemetry project. Starting a conversation does not create a separate user session. +The Aspire development app automatically reports to the seeded **Exceptionless → Exceptionless** project (named **API** in older development data). It waits for the API to be ready and uses the browser's current origin through Vite's API proxy, including forwarded localhost ports. No `.env.local` setup is required. Override `PUBLIC_EXCEPTIONLESS_API_KEY` and `PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL` in the AppHost environment to use another telemetry destination; an empty key disables automatic browser reporting. For a standalone frontend, set those values in `ClientApp/.env.local`, with an empty telemetry URL to use the current origin. The advertised client setup URL stays unchanged. Omitting the telemetry URL override preserves the existing server URL behavior; a browser local-storage server URL override still takes precedence. Keep local keys out of source control. + Each submitted prompt produces an `assistant.MessageSent` feature usage event. A turn produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` feature usage event. These events record character counts and outcomes. Failure events retain the error displayed to the user in `error_message`, capped at 2,048 characters for diagnosis. Existing application error collection is unchanged. Global admins control **Conversation sharing default** beside Exie availability and model settings. It defaults to off for new and legacy settings records. Enable it during the early rollout through the settings page or `PUT /api/v2/admin/assistant-settings/conversation-sharing` with `{ "enabled": true }`; set it to false to stop sharing by default later. From 6ac309e9eefff614cd71bc94127b25b31c60feda Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 13:59:48 -0500 Subject: [PATCH 17/18] Avoid unidentified browser sessions and label anonymous summaries --- .../ClientApp/src/hooks.client.ts | 3 +- .../components/assistant-panel.svelte.test.ts | 17 +++++ .../auth/exceptionless-session.test.ts | 71 +++++++++++++++++-- .../features/auth/exceptionless-session.ts | 39 +++++++--- .../summary/event-session-summary.svelte | 8 +-- .../components/summary/summary.svelte.test.ts | 15 ++++ .../src/lib/features/users/api.svelte.ts | 4 +- .../src/lib/telemetry/Telemetry.svelte | 7 +- tests/Exceptionless.Tests/Assistant/README.md | 4 +- 9 files changed, 142 insertions(+), 26 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/hooks.client.ts b/src/Exceptionless.Web/ClientApp/src/hooks.client.ts index 5cbd599a13..7dc20473c0 100644 --- a/src/Exceptionless.Web/ClientApp/src/hooks.client.ts +++ b/src/Exceptionless.Web/ClientApp/src/hooks.client.ts @@ -3,6 +3,7 @@ import type { ClientInit, HandleClientError } from '@sveltejs/kit'; import { dev } from '$app/environment'; import { page } from '$app/state'; import { env } from '$env/dynamic/public'; +import { configureSessions } from '$features/auth/exceptionless-session'; import { normalizePath, normalizeRouteId } from '$lib/telemetry'; import { installSvelteEffectDepthDiagnostics } from '$lib/telemetry/svelte-effect-depth-diagnostics'; import { Exceptionless, guid, toError } from '@exceptionless/browser'; @@ -47,7 +48,7 @@ export const init: ClientInit = async () => { c.settings['@@log:*'] = 'debug'; } - c.useSessions(); + configureSessions(c); c.addPlugin('route-context', 10, async (ctx) => { if (ctx.event.type !== 'usage') { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index d682d79b63..2c016c44ee 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -35,6 +35,23 @@ describe('AssistantPanel', () => { expect(screen.getByText('Bring Exie onto your team')).toBeTruthy(); }); + it('records opens only when becoming visible, not when typing or changing views', async () => { + const props = { open: true, organizationId: 'organization-1', path: '/next/stack' }; + const view = render(AssistantPanel, { props }); + await screen.findByRole('textbox', { name: 'Message Exie' }); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.Opened')).toHaveLength(1); + + await fireEvent.input(screen.getByRole('textbox', { name: 'Message Exie' }), { target: { value: 'Unsent draft' } }); + await view.rerender({ ...props, mode: 'page', path: '/next/event' }); + await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.ViewChanged', expect.anything())); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.Opened')).toHaveLength(1); + + await view.rerender({ ...props, mode: 'sheet', open: false }); + await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.Closed', expect.anything())); + await view.rerender(props); + await waitFor(() => expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.Opened')).toHaveLength(2)); + }); + it('correlates message outcomes and feedback without recording chat text', async () => { const fetchMock = vi.fn().mockResolvedValue(new Response('{"type":"text_delta","text":"The answer"}\n{"type":"done"}\n')); vi.stubGlobal('fetch', fetchMock); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts index d92b42c111..755ff87ae7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts @@ -1,5 +1,5 @@ import { Exceptionless } from '@exceptionless/browser'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('$app/environment', () => ({ browser: true })); vi.mock('@exceptionless/browser', async () => { @@ -8,19 +8,82 @@ vi.mock('@exceptionless/browser', async () => { config.apiKey = 'local-test-key'; config.serverUrl = 'https://localhost'; config.defaultTags.push('UI', 'Svelte'); - config.useSessions(false); + config.updateSettingsWhenIdleInterval = 0; // Exercise real event builders and plugins, without starting timers or sending events. config.services.queue.enqueue = vi.fn().mockResolvedValue(undefined); + config.services.queue.startup = vi.fn().mockResolvedValue(undefined); + config.services.queue.process = vi.fn().mockResolvedValue(undefined); return { Exceptionless: new ExceptionlessClient(config) }; }); -import { setUserIdentity, submitFeatureUsage, submitLog } from './exceptionless-session'; +import { configureSessions, endSession, setUserIdentity, submitFeatureUsage, submitLog } from './exceptionless-session'; describe('Exceptionless session events', () => { - beforeEach(() => { + beforeEach(async () => { + vi.useFakeTimers(); + vi.spyOn(Exceptionless, 'submitSessionEnd').mockResolvedValue(undefined); + await endSession(); + configureSessions(Exceptionless.config); vi.mocked(Exceptionless.config.services.queue.enqueue).mockClear(); }); + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('skips anonymous startup and resume sessions while retaining ordinary diagnostics', async () => { + await Exceptionless.startup(); + await Exceptionless.startup(); + await submitLog('api-failure', 'HTTP 500'); + await submitFeatureUsage('login'); + + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events.map((event) => event.type)).toEqual(['log', 'usage']); + }); + + it('starts an identified session once when the user loads and keeps identity on resume', async () => { + await setUserIdentity('session-user', 'Session User'); + await setUserIdentity('session-user', 'Updated Name'); + expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledOnce(); + + await Exceptionless.startup(); + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events).toHaveLength(2); + expect(events.every((event) => event.type === 'session' && event.data?.['@user']?.identity === 'session-user')).toBe(true); + expect(Exceptionless.config.currentSessionIdentifier).toBe('session-user'); + }); + + it('does not clear a newer identity when an earlier logout finishes', async () => { + await setUserIdentity('previous-user'); + let finishSessionEnd: () => void = () => {}; + vi.mocked(Exceptionless.submitSessionEnd).mockImplementationOnce( + () => + new Promise((resolve) => { + finishSessionEnd = resolve; + }) + ); + const ending = endSession(); + await vi.waitFor(() => expect(Exceptionless.submitSessionEnd).toHaveBeenCalledWith('previous-user')); + await setUserIdentity('next-user'); + finishSessionEnd(); + await ending; + + expect(Exceptionless.config.defaultData['@user']).toMatchObject({ identity: 'next-user' }); + expect(Exceptionless.config.currentSessionIdentifier).toBe('next-user'); + }); + + it('clears the heartbeat identity on logout and suppresses anonymous resume sessions', async () => { + await setUserIdentity('logging-out-user'); + await endSession(); + vi.mocked(Exceptionless.config.services.queue.enqueue).mockClear(); + await Exceptionless.startup(); + + expect(Exceptionless.config.currentSessionIdentifier).toBeNull(); + expect(Exceptionless.config.services.queue.enqueue).not.toHaveBeenCalled(); + }); + it('keeps usage metadata and feedback attached to the existing user session', async () => { await setUserIdentity('exie-test-user'); const properties = { exie: { conversation_id: 'conversation-1', role: 'user' } }; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts index 940789a9a2..d4a744ab8d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts @@ -1,12 +1,25 @@ +import type { Configuration } from '@exceptionless/browser'; + import { browser } from '$app/environment'; let _activeUserId: null | string = null; +/** Keep SDK startup and resume events from creating sessions before authentication resolves. */ +export function configureSessions(config: Configuration): void { + config.useSessions(); + config.addPlugin('authenticated-sessions', 20, async (context) => { + if (context.event.type === 'session' && !context.event.data?.['@user']?.identity) { + context.cancelled = true; + } + }); +} + /** * Ends the current Exceptionless session and clears user identity. - * Call on logout. Clears local state unconditionally even if submitSessionEnd fails. + * Call on logout. A delayed session end must not clear a newer user's identity. */ export async function endSession(): Promise { + const endingUserId = _activeUserId; const Exceptionless = await getExceptionless(); if (!Exceptionless) { _activeUserId = null; @@ -14,16 +27,21 @@ export async function endSession(): Promise { } try { - await Exceptionless.submitSessionEnd(); + if (endingUserId) { + await Exceptionless.submitSessionEnd(endingUserId); + } } finally { - Exceptionless.config.setUserIdentity('', ''); - _activeUserId = null; + if (_activeUserId === endingUserId) { + Exceptionless.config.setUserIdentity('', ''); + Exceptionless.config.currentSessionIdentifier = null; + _activeUserId = null; + } } } /** * Sets the current user identity for Exceptionless error tracking. - * Starts a new session only when the identity changes (guards against repeated onSuccess calls from query refetches). + * Starts a new session only when the identity changes, including across profile refetches. */ export async function setUserIdentity(userId: string, userName?: string): Promise { if (!userId) { @@ -35,15 +53,14 @@ export async function setUserIdentity(userId: string, userName?: string): Promis return; } - if (userName) { - Exceptionless.config.setUserIdentity(userId, userName); - } else { - Exceptionless.config.setUserIdentity(userId); - } + Exceptionless.config.setUserIdentity(userId, userName ?? ''); + Exceptionless.config.currentSessionIdentifier = userId; if (_activeUserId !== userId) { _activeUserId = userId; - await Exceptionless.submitSessionStart(); + await Exceptionless.createSessionStart() + .setUserIdentity(userId, userName ?? '') + .submit(); } } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte index 318e11ca9a..8e1a35e39b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte @@ -27,11 +27,9 @@ {/if} - {#if source.data.Name || source.data.Identity || source.data.SessionId} - {source.data.Name || source.data.Identity || source.data.SessionId} - {#if source.data.Name && source.data.Identity} - ({source.data.Identity}) - {/if} + {source.data.Name || source.data.Identity || source.data.SessionId || 'Anonymous session'} + {#if source.data.Name && source.data.Identity} + ({source.data.Identity}) {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts index ea86d1aa14..5054a68281 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts @@ -7,6 +7,21 @@ import type { EventSummaryModel, StackSummaryModel } from './index'; import Summary from './summary.svelte'; describe('Summary', () => { + it('keeps sessions without identity or session metadata readable and clickable', () => { + const summary: EventSummaryModel<'event-session-summary'> = { + data: { Type: 'session' }, + date: '2026-09-09T00:00:00Z', + id: 'anonymous-session-event', + project_id: 'project-id', + tags: [], + template_key: 'event-session-summary' + }; + + render(Summary, { showStatus: false, showType: false, summary }); + + expect(screen.getByRole('link', { name: 'Anonymous session' }).getAttribute('href')).toBe('/next/event/anonymous-session-event'); + }); + it('links an event summary to that event details page', () => { const summary: EventSummaryModel<'event-error-summary'> = { data: { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts index ff97db0e6d..78b76c1301 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts @@ -1,7 +1,6 @@ import type { WebSocketMessageValue } from '$features/websockets/models'; import type { WorkInProgressResult } from '$shared/models'; -import { setUserIdentity } from '$features/auth/exceptionless-session'; import { accessToken } from '$features/auth/index.svelte'; import { fetchApiJson } from '$features/shared/api/api.svelte'; import { type FetchClientResponse, ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; @@ -135,9 +134,8 @@ export function getMeQuery() { return createQuery(() => ({ enabled: () => !!accessToken.current, - onSuccess: async (data: ViewCurrentUser) => { + onSuccess: (data: ViewCurrentUser) => { queryClient.setQueryData(queryKeys.id(data.id!), data); - await setUserIdentity(data.id, data.full_name); }, queryClient, queryFn: async ({ signal }: { signal: AbortSignal }) => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte b/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte index e0a8ff09ef..7ab0af27fe 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte @@ -1,6 +1,7 @@ diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index 1fa4df6161..3bfadfa56d 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -69,6 +69,8 @@ dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll --fil The Svelte app submits Exie events through the existing Exceptionless browser client. They share the signed-in user's session, client configuration, queue, tags, and event exclusions. They go to the app's configured telemetry project. Starting a conversation does not create a separate user session. +Session starts wait for an identified user: anonymous SDK startup/resume session events are discarded, while ordinary errors, logs, and usage events remain enabled. The telemetry component owns the identity update so profile loading cannot race a second identity writer. Older session rows without identity or session metadata display **Anonymous session**. + The Aspire development app automatically reports to the seeded **Exceptionless → Exceptionless** project (named **API** in older development data). It waits for the API to be ready and uses the browser's current origin through Vite's API proxy, including forwarded localhost ports. No `.env.local` setup is required. Override `PUBLIC_EXCEPTIONLESS_API_KEY` and `PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL` in the AppHost environment to use another telemetry destination; an empty key disables automatic browser reporting. For a standalone frontend, set those values in `ClientApp/.env.local`, with an empty telemetry URL to use the current origin. The advertised client setup URL stays unchanged. Omitting the telemetry URL override preserves the existing server URL behavior; a browser local-storage server URL override still takes precedence. Keep local keys out of source control. Each submitted prompt produces an `assistant.MessageSent` feature usage event. A turn produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` feature usage event. These events record character counts and outcomes. Failure events retain the error displayed to the user in `error_message`, capped at 2,048 characters for diagnosis. Existing application error collection is unchanged. @@ -87,7 +89,7 @@ Other feature usage events describe interactions: | Event source | Meaning | | --- | --- | -| `assistant.Opened`, `assistant.Closed`, `assistant.ViewChanged` | Open, close, or switch between the panel and full page. Closing the panel does not cancel an ongoing response. | +| `assistant.Opened`, `assistant.Closed`, `assistant.ViewChanged` | One open event when Exie becomes visible, including loading or reloading the full Exie page; one close when hidden. Switching panel/page mode emits a view change. Typing, other state updates, and opening the sharing menu do not emit opens. Closing the panel does not cancel an ongoing response. | | `assistant.ResponseHelpful`, `assistant.ResponseNotHelpful`, `assistant.ResponseFeedbackCleared` | Explicit feedback linked to the response. | | `assistant.ResponseRegenerated` | Retry or regenerate, linked to the previous response. | | `assistant.MessageCopied`, `assistant.SuggestedActionSelected` | Copy a message or act on an Exie suggestion. | From 91fb92b14820a395465845b18a2f280894f9d740 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 14:19:08 -0500 Subject: [PATCH 18/18] Enable local server reporting in development Aspire runs --- README.md | 2 +- src/Exceptionless.AppHost/Program.cs | 6 ++++++ src/Exceptionless.AppHost/appsettings.Development.json | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 891a10891c..168a64861e 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ After startup: 1. Open `https://localhost:7121/` if a browser does not open automatically. 2. In `Development` mode, a global administrator user `admin@exceptionless.test` with password `tester` is created automatically. - The Aspire Svelte app reports its browser errors, usage, and sessions to the seeded **Exceptionless → Exceptionless** project by default (named **API** in older development data). Telemetry uses the current browser origin, so forwarded localhost ports work without extra configuration. Exie conversation text still follows the user's chat-sharing preference. To override the telemetry destination, set `PUBLIC_EXCEPTIONLESS_API_KEY` and `PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL` in the AppHost environment; an empty key disables automatic browser reporting. + Development Aspire runs report API and background job warnings and errors, plus Svelte browser errors, usage, and sessions, to the seeded **Exceptionless → Exceptionless** project by default (named **API** in older development data). Server reporting uses the local API endpoint, and browser telemetry uses the current browser origin, so worktrees and forwarded localhost ports work without extra configuration. Exie conversation text still follows the user's chat-sharing preference. Set `ExceptionlessApiKey` to an empty value in the AppHost environment to disable server reporting. To override the browser telemetry destination, set `PUBLIC_EXCEPTIONLESS_API_KEY` and `PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL`; an empty browser key disables automatic browser reporting. Notes: diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs index fc5915374e..9ed7285588 100644 --- a/src/Exceptionless.AppHost/Program.cs +++ b/src/Exceptionless.AppHost/Program.cs @@ -119,6 +119,9 @@ .WithUrlForEndpoint("http", u => u.DisplayLocation = UrlDisplayLocation.DetailsOnly) .WithHttpHealthCheck("/health"); + api.WithEnvironment("EX_ExceptionlessApiKey", builder.Configuration["ExceptionlessApiKey"]) + .WithEnvironment("EX_ExceptionlessServerUrl", api.GetEndpoint("http")); + if (assistantApiKey is not null) { api.WithEnvironment("EX_Assistant__ApiKey", assistantApiKey); @@ -138,6 +141,9 @@ .WithReference(storageBlobs, "AzureStorage") .WithReference(storageQueues, "AzureQueues") .WithEnvironment("ConnectionStrings:Email", SharedEmailConnectionString) + .WithEnvironment("EX_ExceptionlessApiKey", builder.Configuration["ExceptionlessApiKey"]) + .WithEnvironment("EX_ExceptionlessServerUrl", api.GetEndpoint("http")) + .WaitFor(api) .WaitFor(elastic) .WaitFor(cache) .WaitFor(mail) diff --git a/src/Exceptionless.AppHost/appsettings.Development.json b/src/Exceptionless.AppHost/appsettings.Development.json index a0748efc92..7b7eb9e19c 100644 --- a/src/Exceptionless.AppHost/appsettings.Development.json +++ b/src/Exceptionless.AppHost/appsettings.Development.json @@ -1,4 +1,5 @@ { + "ExceptionlessApiKey": "Bx7JgglstPG544R34Tw9T7RlCed3OIwtYXVeyhT2", "PUBLIC_EXCEPTIONLESS_API_KEY": "Bx7JgglstPG544R34Tw9T7RlCed3OIwtYXVeyhT2", "Logging": { "LogLevel": {