diff --git a/README.md b/README.md index f49f665313..168a64861e 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ After startup: 1. Open `https://localhost:7121/` if a browser does not open automatically. 2. In `Development` mode, a global administrator user `admin@exceptionless.test` with password `tester` is created automatically. + Development Aspire runs report API and background job warnings and errors, plus Svelte browser errors, usage, and sessions, to the seeded **Exceptionless → Exceptionless** project by default (named **API** in older development data). Server reporting uses the local API endpoint, and browser telemetry uses the current browser origin, so worktrees and forwarded localhost ports work without extra configuration. Exie conversation text still follows the user's chat-sharing preference. Set `ExceptionlessApiKey` to an empty value in the AppHost environment to disable server reporting. To override the browser telemetry destination, set `PUBLIC_EXCEPTIONLESS_API_KEY` and `PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL`; an empty browser key disables automatic browser reporting. + Notes: 1. Running `Exceptionless.AppHost` starts the app and required infrastructure together. diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs index b12b48aefe..9ed7285588 100644 --- a/src/Exceptionless.AppHost/Program.cs +++ b/src/Exceptionless.AppHost/Program.cs @@ -119,6 +119,9 @@ .WithUrlForEndpoint("http", u => u.DisplayLocation = UrlDisplayLocation.DetailsOnly) .WithHttpHealthCheck("/health"); + api.WithEnvironment("EX_ExceptionlessApiKey", builder.Configuration["ExceptionlessApiKey"]) + .WithEnvironment("EX_ExceptionlessServerUrl", api.GetEndpoint("http")); + if (assistantApiKey is not null) { api.WithEnvironment("EX_Assistant__ApiKey", assistantApiKey); @@ -138,6 +141,9 @@ .WithReference(storageBlobs, "AzureStorage") .WithReference(storageQueues, "AzureQueues") .WithEnvironment("ConnectionStrings:Email", SharedEmailConnectionString) + .WithEnvironment("EX_ExceptionlessApiKey", builder.Configuration["ExceptionlessApiKey"]) + .WithEnvironment("EX_ExceptionlessServerUrl", api.GetEndpoint("http")) + .WaitFor(api) .WaitFor(elastic) .WaitFor(cache) .WaitFor(mail) @@ -192,7 +198,9 @@ .WithBrowserLogs() .WithReference(api) .WithReference(oldApp) + .WithEnvironment("PUBLIC_EXCEPTIONLESS_API_KEY", builder.Configuration["PUBLIC_EXCEPTIONLESS_API_KEY"]) .WithEnvironment("PUBLIC_EXCEPTIONLESS_SERVER_URL", exceptionlessServerUrl) + .WithEnvironment("PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL", builder.Configuration["PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL"] ?? String.Empty) .WithEnvironment("PORT", appPort.ToString()) .WithEndpoint("http", e => { @@ -203,6 +211,7 @@ e.IsProxied = false; }) .WithHttpsDeveloperCertificate() + .WaitFor(api) .WithUrlForEndpoint("http", u => { u.DisplayText = "Open App"; diff --git a/src/Exceptionless.AppHost/appsettings.Development.json b/src/Exceptionless.AppHost/appsettings.Development.json index 0c208ae918..7b7eb9e19c 100644 --- a/src/Exceptionless.AppHost/appsettings.Development.json +++ b/src/Exceptionless.AppHost/appsettings.Development.json @@ -1,4 +1,6 @@ { + "ExceptionlessApiKey": "Bx7JgglstPG544R34Tw9T7RlCed3OIwtYXVeyhT2", + "PUBLIC_EXCEPTIONLESS_API_KEY": "Bx7JgglstPG544R34Tw9T7RlCed3OIwtYXVeyhT2", "Logging": { "LogLevel": { "Default": "Information", diff --git a/src/Exceptionless.Core/Models/SystemSettings.cs b/src/Exceptionless.Core/Models/SystemSettings.cs index 607aaa6029..01edad4a4d 100644 --- a/src/Exceptionless.Core/Models/SystemSettings.cs +++ b/src/Exceptionless.Core/Models/SystemSettings.cs @@ -17,6 +17,8 @@ public sealed class SystemSettings : IIdentity, IHaveDates public bool? AssistantEnabled { get; set; } + public bool AssistantConversationSharingDefaultEnabled { get; set; } + public bool? EventSubmissionEnabled { get; set; } public SystemNotification? SystemNotification { get; set; } diff --git a/src/Exceptionless.Core/Models/User.cs b/src/Exceptionless.Core/Models/User.cs index 8168c0302e..82bb84ec3a 100644 --- a/src/Exceptionless.Core/Models/User.cs +++ b/src/Exceptionless.Core/Models/User.cs @@ -40,6 +40,7 @@ public record User : IIdentity, IHaveDates, IValidatableObject public string? AvatarFileName { get; set; } public bool EmailNotificationsEnabled { get; set; } = true; + public bool? AssistantConversationSharingEnabled { get; set; } public bool IsEmailAddressVerified { get; set; } public string? VerifyEmailAddressToken { get; set; } public DateTime VerifyEmailAddressTokenExpiration { get; set; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs index 4f48d72bd7..3c4f65b0d0 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SystemSettingsIndex.cs @@ -24,6 +24,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor .SetupDefaults() .Keyword(settings => settings.AssistantModel) .Boolean(settings => settings.AssistantEnabled) + .Boolean(settings => settings.AssistantConversationSharingDefaultEnabled) .Boolean(settings => settings.EventSubmissionEnabled) .Object(settings => settings.SystemNotification, notification => notification.Properties(properties => properties .Date("date") diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs index d777ac2013..0e85e7a7fb 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs @@ -27,6 +27,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor map) .Text(e => e.FullName, t => t.AddKeywordField()) .Text(e => e.EmailAddress, t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) .Boolean(e => e.IsEmailAddressVerified) + .Boolean(e => e.AssistantConversationSharingEnabled) .Keyword(e => e.VerifyEmailAddressToken) .Date(e => e.VerifyEmailAddressTokenExpiration) .Keyword(e => e.PasswordResetToken) diff --git a/src/Exceptionless.Core/Services/SystemSettingsService.cs b/src/Exceptionless.Core/Services/SystemSettingsService.cs index 1899e0e66b..232b4a5e35 100644 --- a/src/Exceptionless.Core/Services/SystemSettingsService.cs +++ b/src/Exceptionless.Core/Services/SystemSettingsService.cs @@ -92,4 +92,10 @@ public async Task IsEventSubmissionEnabledAsync() var settings = await _getSettingsAsync(); return settings?.EventSubmissionEnabled ?? !_appOptions.EventSubmissionDisabled; } + + public async Task IsAssistantConversationSharingDefaultEnabledAsync() + { + var settings = await _getSettingsAsync(); + return settings?.AssistantConversationSharingDefaultEnabled ?? false; + } } diff --git a/src/Exceptionless.Core/Utility/AppDiagnostics.cs b/src/Exceptionless.Core/Utility/AppDiagnostics.cs index e59dc6bf7a..883cbfd65b 100644 --- a/src/Exceptionless.Core/Utility/AppDiagnostics.cs +++ b/src/Exceptionless.Core/Utility/AppDiagnostics.cs @@ -10,6 +10,7 @@ public static class AppDiagnostics internal static readonly AssemblyName AssemblyName = typeof(AppDiagnostics).Assembly.GetName(); internal static readonly string? AssemblyVersion = typeof(AppDiagnostics).Assembly.GetCustomAttribute()?.InformationalVersion ?? AssemblyName.Version?.ToString(); internal static readonly ActivitySource ActivitySource = new(AssemblyName.Name ?? "Exceptionless", AssemblyVersion); + internal static readonly ActivitySource AssistantActivitySource = new("Exceptionless.Assistant", AssemblyVersion); internal static readonly Meter Meter = new("Exceptionless", AssemblyVersion); private static readonly string _metricsPrefix = "ex."; @@ -91,6 +92,9 @@ public GaugeInfo(Meter meter, string name) internal static readonly Counter EventsSubmitted = Meter.CreateCounter("ex.events.submitted", description: "Events submitted to the pipeline to be processed"); internal static readonly Counter AssistantTurns = Meter.CreateCounter("ex.assistant.turns", description: "Assistant turns accepted"); internal static readonly Counter AssistantTurnOutcomes = Meter.CreateCounter("ex.assistant.turn.outcomes", description: "Assistant turn outcomes"); + internal static readonly Histogram AssistantTurnDuration = Meter.CreateHistogram("ex.assistant.turn.duration", unit: "ms", description: "Assistant turn duration by outcome and failure reason"); + internal static readonly Histogram AssistantProviderDuration = Meter.CreateHistogram("ex.assistant.provider.duration", unit: "ms", description: "Assistant provider request duration by outcome"); + internal static readonly Histogram AssistantToolDuration = Meter.CreateHistogram("ex.assistant.tool.duration", unit: "ms", description: "Assistant tool duration by tool and outcome"); internal static readonly Counter AssistantTurnsBlocked = Meter.CreateCounter("ex.assistant.turns.blocked", description: "Assistant turns blocked by a usage limit"); internal static readonly Counter AssistantProviderRequests = Meter.CreateCounter("ex.assistant.provider.requests", description: "Assistant provider requests"); internal static readonly Counter AssistantToolCalls = Meter.CreateCounter("ex.assistant.tool.calls", description: "Assistant tool calls"); diff --git a/src/Exceptionless.Core/Utility/SampleDataService.cs b/src/Exceptionless.Core/Utility/SampleDataService.cs index ce6e1a6faa..49b1d8feba 100644 --- a/src/Exceptionless.Core/Utility/SampleDataService.cs +++ b/src/Exceptionless.Core/Utility/SampleDataService.cs @@ -256,7 +256,7 @@ public async Task CreateInternalOrganizationAndProjectAsync(string userId) var project = new Project { Id = INTERNAL_PROJECT_ID, - Name = "API", + Name = "Exceptionless", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = _timeProvider.GetUtcNow().UtcDateTime.Date.AddDays(1).AddHours(1).Ticks }; diff --git a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs index bc495454ca..5ebac61909 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AdminEndpoints.cs @@ -60,6 +60,18 @@ public static IEndpointRouteBuilder MapAdminEndpoints(this IEndpointRouteBuilder .WithTags(nameof(AdminEndpoints)) .WithSummary("Update Exie assistant availability"); + endpoints.MapPut("api/v2/admin/assistant-settings/conversation-sharing", async (HttpContext httpContext, [FromBody] UpdateAssistantConversationSharingSettings request, AssistantModelSettingsService settingsService) + => HttpResults.Ok(await settingsService.SetConversationSharingDefaultEnabledAsync(request.Enabled, httpContext.Request.GetUser().Id))) + .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy) + .AddEndpointFilter() + .Accepts("application/json", "application/*+json") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .ProducesProblem(StatusCodes.Status422UnprocessableEntity) + .WithTags(nameof(AdminEndpoints)) + .WithSummary("Update Exie full conversation logging"); + endpoints.MapGet("api/v2/admin/event-submission-settings", GetEventSubmissionSettingsAsync) .RequireAuthorization(AuthorizationRoles.GlobalAdminPolicy) .Produces(StatusCodes.Status200OK) diff --git a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs index 9a0fbaa0dd..807d51df11 100644 --- a/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/AssistantEndpoints.cs @@ -12,10 +12,27 @@ namespace Exceptionless.Web.Api.Endpoints; public static class AssistantEndpoints { + internal const string FullLoggingHeaderName = "X-Exie-Full-Logging"; private static readonly JsonSerializerOptions s_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web).ConfigureExceptionlessApiDefaults(); public static IEndpointRouteBuilder MapAssistantEndpoints(this IEndpointRouteBuilder endpoints) { + endpoints.MapGet("api/v2/assistant/conversation-sharing", GetConversationSharingAsync) + .WithName("GetAssistantConversationSharing") + .RequireAuthorization(AuthorizationRoles.UserPolicy) + .Produces() + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status404NotFound); + + endpoints.MapPut("api/v2/assistant/conversation-sharing", SetConversationSharingAsync) + .WithName("SetAssistantConversationSharing") + .WithDescription("Saves the current user's choice. Null follows the admin default; true and false remain explicit choices when the default changes.") + .RequireAuthorization(AuthorizationRoles.UserPolicy) + .Produces() + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status404NotFound); + endpoints.MapGet("api/v2/assistant/access", GetAccessAsync) .WithName("GetAssistantAccess") .RequireAuthorization(AuthorizationRoles.UserPolicy) @@ -26,6 +43,7 @@ public static IEndpointRouteBuilder MapAssistantEndpoints(this IEndpointRouteBui endpoints.MapPost("api/v2/assistant/chat", StreamChatAsync) .WithName("StreamAssistantChat") + .WithDescription("The X-Exie-Full-Logging response header indicates whether the browser should record prompts and responses for this turn. Missing or false disables conversation logging.") .RequireAuthorization(AuthorizationRoles.UserPolicy) .WithMetadata(new RequestSizeLimitAttribute(256 * 1024)) .Produces(StatusCodes.Status200OK, contentType: "application/x-ndjson") @@ -46,6 +64,7 @@ private static async Task StreamChatAsync( AssistantAccessService assistantAccessService, AssistantUsageService assistantUsageService, AssistantService assistantService, + AssistantConversationSharingService conversationSharingService, TimeProvider timeProvider, ILogger logger) { @@ -74,6 +93,7 @@ private static async Task StreamChatAsync( OrganizationId = organizationId, ConversationId = conversationId ?? Guid.NewGuid().ToString("N") }; + bool fullLoggingEnabled = (await conversationSharingService.GetAsync(userId))?.Enabled == true; var planOptions = access.PlanOptions!; await using var turnReservation = await assistantUsageService.TryStartTurnAsync(organizationId, planOptions); if (!turnReservation.Allowed) @@ -99,17 +119,39 @@ private static async Task StreamChatAsync( using var turnCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(httpContext.RequestAborted); turnCancellationSource.CancelAfter(TimeSpan.FromSeconds(AssistantLimits.MaximumTurnDurationSeconds)); + using var diagnostics = new AssistantTurnDiagnostics(logger, timeProvider, organizationId!, request.ConversationId!, httpContext.TraceIdentifier, httpContext.RequestAborted); + var response = assistantService.StreamAsync(request, userId, planOptions, diagnostics, turnCancellationSource.Token); + await WriteResponseAsync(httpContext, response, assistantUsageService, organizationId!, diagnostics, turnCancellationSource.Token, fullLoggingEnabled); + + return HttpResults.Empty; + } + + internal static async Task WriteResponseAsync( + HttpContext httpContext, + IAsyncEnumerable response, + AssistantUsageService assistantUsageService, + string organizationId, + AssistantTurnDiagnostics diagnostics, + CancellationToken cancellationToken, + bool fullLoggingEnabled = false) + { + httpContext.Response.Headers[FullLoggingHeaderName] = fullLoggingEnabled ? "true" : "false"; bool responseFailed = false; try { - await foreach (var item in assistantService.StreamAsync(request, userId, planOptions, turnCancellationSource.Token)) + await foreach (var item in response.WithCancellation(cancellationToken)) { + diagnostics.Observe(item); responseFailed |= item.Type == "error"; - await JsonSerializer.SerializeAsync(httpContext.Response.Body, item, s_jsonOptions, turnCancellationSource.Token); - await httpContext.Response.WriteAsync("\n", turnCancellationSource.Token); - await httpContext.Response.Body.FlushAsync(turnCancellationSource.Token); + string stage = diagnostics.Stage; + diagnostics.Stage = "response_write"; + await JsonSerializer.SerializeAsync(httpContext.Response.Body, item, s_jsonOptions, cancellationToken); + await httpContext.Response.WriteAsync("\n", cancellationToken); + await httpContext.Response.Body.FlushAsync(cancellationToken); + diagnostics.Stage = stage; } + diagnostics.Finish(responseFailed ? "failed" : "completed"); if (responseFailed) await assistantUsageService.RecordTurnFailedAsync(organizationId); else @@ -118,10 +160,14 @@ private static async Task StreamChatAsync( catch (OperationCanceledException) when (httpContext.RequestAborted.IsCancellationRequested) { // The browser closing or stopping the stream is expected. + diagnostics.Finish("cancelled", "client_disconnected"); await assistantUsageService.RecordTurnCancelledAsync(organizationId); } catch (OperationCanceledException) { + string failureCode = cancellationToken.IsCancellationRequested ? "turn_timeout" + : diagnostics.Stage is "provider_request" or "provider_stream" ? "provider_timeout" : "operation_cancelled"; + diagnostics.Finish("failed", failureCode); await assistantUsageService.RecordTurnFailedAsync(organizationId); var error = AssistantStreamEvent.Error("Exie took too long to complete this response. Try narrowing the question."); await JsonSerializer.SerializeAsync(httpContext.Response.Body, error, s_jsonOptions, CancellationToken.None); @@ -129,14 +175,22 @@ private static async Task StreamChatAsync( } catch (Exception ex) { + string failureCode = ex switch + { + AssistantProviderException providerException => providerException.FailureCode, + _ when diagnostics.Stage == "response_write" => "response_write_error", + _ when diagnostics.Stage == "tool_execution" => "tool_execution_error", + HttpRequestException when diagnostics.Stage is "provider_request" or "provider_stream" => "provider_transport_error", + JsonException when diagnostics.Stage == "provider_stream" => "invalid_provider_response", + IOException when diagnostics.Stage == "provider_stream" => "provider_stream_error", + _ => "internal_error" + }; + diagnostics.Finish("failed", failureCode, ex); await assistantUsageService.RecordTurnFailedAsync(organizationId); - logger.LogError(ex, "Unable to stream an in-app assistant response"); var error = AssistantStreamEvent.Error(ex is AssistantProviderException ? ex.Message : "Exie could not complete this request."); await JsonSerializer.SerializeAsync(httpContext.Response.Body, error, s_jsonOptions, CancellationToken.None); await httpContext.Response.WriteAsync("\n", CancellationToken.None); } - - return HttpResults.Empty; } private static async Task GetAccessAsync( @@ -148,6 +202,26 @@ private static async Task GetAccessAsync( return HttpResults.Ok(access.ToResponse()); } + private static async Task GetConversationSharingAsync(HttpContext httpContext, AssistantConversationSharingService service) + { + string? userId = httpContext.User.GetUserId(); + if (String.IsNullOrWhiteSpace(userId)) + return HttpResults.Unauthorized(); + + var settings = await service.GetAsync(userId); + return settings is null ? HttpResults.NotFound() : HttpResults.Ok(settings); + } + + private static async Task SetConversationSharingAsync(UpdateAssistantConversationSharing request, HttpContext httpContext, AssistantConversationSharingService service) + { + string? userId = httpContext.User.GetUserId(); + if (String.IsNullOrWhiteSpace(userId)) + return HttpResults.Unauthorized(); + + var settings = await service.SetAsync(userId, request.Enabled); + return settings is null ? HttpResults.NotFound() : HttpResults.Ok(settings); + } + internal static IResult? MapAccessFailure(AssistantAccessDecision access) => access.Reason switch { AssistantAccessReason.Available => null, diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 0f6af7263f..be2f81c505 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -72,7 +72,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }); b.AddHttpClientInstrumentation(); - b.AddSource("Exceptionless", "Foundatio"); + b.AddSource("Exceptionless", "Exceptionless.Assistant", "Foundatio"); if (config.EnableRedis) b.AddRedisInstrumentation(c => diff --git a/src/Exceptionless.Web/Assistant/AssistantConversationSharingService.cs b/src/Exceptionless.Web/Assistant/AssistantConversationSharingService.cs new file mode 100644 index 0000000000..60d4754519 --- /dev/null +++ b/src/Exceptionless.Web/Assistant/AssistantConversationSharingService.cs @@ -0,0 +1,41 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Web.Assistant; + +public sealed class AssistantConversationSharingService(IUserRepository userRepository, SystemSettingsService systemSettingsService) +{ + public async Task GetAsync(string userId) + { + // Read the saved choice for every turn, including choices made on another device. + var user = await userRepository.GetByIdAsync(userId); + if (user is null) + return null; + + bool defaultEnabled = await systemSettingsService.IsAssistantConversationSharingDefaultEnabledAsync(); + return Resolve(user.AssistantConversationSharingEnabled, defaultEnabled); + } + + public async Task SetAsync(string userId, bool? enabled) + { + // Preserve an explicit choice even when it matches today's default. + bool updated = await userRepository.PatchAsync(userId, + new ActionPatch(user => user.AssistantConversationSharingEnabled = enabled), + options => options.Cache().ImmediateConsistency()); + return updated ? await GetAsync(userId) : null; + } + + internal static AssistantConversationSharingSettings Resolve(bool? enabled, bool defaultEnabled) => + new(enabled ?? defaultEnabled, defaultEnabled, enabled.HasValue); +} + +public sealed record AssistantConversationSharingSettings(bool Enabled, bool DefaultEnabled, bool IsOverridden); + +public sealed record UpdateAssistantConversationSharing +{ + [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.Never)] + public required bool? Enabled { get; init; } +} diff --git a/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs index 20f2b559a6..2af954beac 100644 --- a/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantModelSettingsService.cs @@ -43,6 +43,12 @@ public async Task SetEnabledAsync(bool? enabled, string return CreateResponse(settings); } + public async Task SetConversationSharingDefaultEnabledAsync(bool enabled, string userId) + { + var settings = await _systemSettingsService.UpdateAsync(userId, value => value.AssistantConversationSharingDefaultEnabled = enabled); + return CreateResponse(settings); + } + private AssistantModelSettings CreateResponse(SystemSettings? settings) { string configuredModel = _appOptions.AssistantOptions.Model; @@ -58,7 +64,8 @@ private AssistantModelSettings CreateResponse(SystemSettings? settings) enabledOverride ?? configuredEnabled, configuredEnabled, enabledOverride.HasValue, - _appOptions.AssistantOptions.IsConfigured); + _appOptions.AssistantOptions.IsConfigured, + settings?.AssistantConversationSharingDefaultEnabled ?? false); } } @@ -69,4 +76,5 @@ public sealed record AssistantModelSettings( bool Enabled, bool ConfiguredEnabled, bool IsEnabledOverridden, - bool IsConfigured); + bool IsConfigured, + bool ConversationSharingDefaultEnabled = false); diff --git a/src/Exceptionless.Web/Assistant/AssistantModels.cs b/src/Exceptionless.Web/Assistant/AssistantModels.cs index 098ccab84c..4a9fc6ad84 100644 --- a/src/Exceptionless.Web/Assistant/AssistantModels.cs +++ b/src/Exceptionless.Web/Assistant/AssistantModels.cs @@ -43,10 +43,14 @@ public sealed record AssistantStreamEvent( string? Message = null, IReadOnlyCollection? SuggestedActions = null) { + [System.Text.Json.Serialization.JsonIgnore] + internal string? FailureCode { get; init; } + public static AssistantStreamEvent TextDelta(string text) => new("text_delta", Text: text); public static AssistantStreamEvent ToolCall(string id, string name, string arguments) => new("tool_call", ToolCallId: id, ToolName: name, Arguments: arguments); public static AssistantStreamEvent ToolResult(string id, string name, string result) => new("tool_result", ToolCallId: id, ToolName: name, Result: result); public static AssistantStreamEvent Suggestions(IReadOnlyCollection actions) => new("suggested_actions", SuggestedActions: actions); public static AssistantStreamEvent Error(string message) => new("error", Message: message); + internal static AssistantStreamEvent Error(string message, string failureCode) => new("error", Message: message) { FailureCode = failureCode }; public static AssistantStreamEvent Done() => new("done"); } diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs new file mode 100644 index 0000000000..279f80f6a8 --- /dev/null +++ b/src/Exceptionless.Web/Assistant/AssistantProviderDiagnostics.cs @@ -0,0 +1,135 @@ +using System.Diagnostics; +using System.Text.Json; + +namespace Exceptionless.Web.Assistant; + +internal sealed class AssistantProviderDiagnostics( + ILogger logger, + TimeProvider timeProvider, + AssistantTurnDiagnostics turn, + int inputCharacters, + bool allowTools, + CancellationToken cancellationToken) : IDisposable +{ + private readonly long _started = timeProvider.GetTimestamp(); + private readonly Activity? _activity = AppDiagnostics.AssistantActivitySource.StartActivity("assistant.provider"); + private bool _finished; + private bool _receivedError; + + public string? GenerationId { get; private set; } + public string? Model { get; private set; } + public string? ProviderName { get; private set; } + public int? StatusCode { get; private set; } + public string? FinishReason { get; private set; } + public bool UsageReceived { get; private set; } + public long? PromptTokens { get; private set; } + public long? CompletionTokens { get; private set; } + public long? ReasoningTokens { get; private set; } + + public void ObserveResponse(HttpResponseMessage response) + { + StatusCode = (int)response.StatusCode; + if (response.Headers.TryGetValues("X-Generation-Id", out var values)) + GenerationId = SafeMetadata(values.FirstOrDefault()); + turn.Stage = "provider_stream"; + } + + public void ObserveChunk(JsonElement chunk) + { + if (chunk.ValueKind != JsonValueKind.Object) + return; + GenerationId = GetMetadata(chunk, "id") ?? GenerationId; + Model = GetMetadata(chunk, "model") ?? Model; + ProviderName = GetMetadata(chunk, "provider") ?? ProviderName; + _receivedError |= chunk.TryGetProperty("error", out _); + if (chunk.TryGetProperty("usage", out var usage) && usage.ValueKind == JsonValueKind.Object) + { + UsageReceived = true; + PromptTokens = GetTokenCount(usage, "prompt_tokens") ?? PromptTokens; + CompletionTokens = GetTokenCount(usage, "completion_tokens") ?? CompletionTokens; + if (usage.TryGetProperty("completion_tokens_details", out var details) && details.ValueKind == JsonValueKind.Object + && GetTokenCount(details, "reasoning_tokens") is { } value) + ReasoningTokens = value; + } + if (chunk.TryGetProperty("choices", out var choices) && choices.ValueKind == JsonValueKind.Array && choices.GetArrayLength() > 0) + { + string? reason = GetMetadata(choices[0], "finish_reason"); + if (reason is not null) + FinishReason = reason is "stop" or "length" or "tool_calls" or "content_filter" or "error" ? reason : "unknown"; + } + } + + public void Complete(int outputCharacters, int toolCalls, bool receivedDone) + { + string outcome = _receivedError ? "provider_error" : FinishReason switch + { + "length" => "output_limit", + "content_filter" => "content_filter", + "error" => "provider_error", + _ when outputCharacters == 0 && toolCalls == 0 => "empty_response", + _ when !receivedDone && FinishReason is null => "incomplete_stream", + _ => "completed" + }; + Finish(outcome, outputCharacters, toolCalls, receivedDone); + } + + public void RecordException(Exception exception) + { + string outcome = exception switch + { + AssistantProviderException providerException => providerException.FailureCode, + OperationCanceledException => GetCancellationOutcome(), + HttpRequestException => "provider_transport_error", + JsonException => "invalid_provider_response", + IOException => "provider_stream_error", + _ => "internal_error" + }; + Finish(outcome); + } + + public void Reject(string reason, int outputCharacters, int toolCalls, bool receivedDone) => Finish(reason, outputCharacters, toolCalls, receivedDone); + + private void Finish(string outcome, int? outputCharacters = null, int? toolCalls = null, bool receivedDone = false) + { + if (_finished) + return; + _finished = true; + double duration = timeProvider.GetElapsedTime(_started).TotalMilliseconds; + _activity?.SetTag("assistant.provider.outcome", outcome); + _activity?.SetTag("assistant.provider.generation_id", GenerationId); + _activity?.SetTag("assistant.provider.model", Model ?? turn.Model); + _activity?.SetTag("assistant.provider.name", ProviderName); + _activity?.SetTag("assistant.provider.finish_reason", FinishReason); + _activity?.SetTag("http.response.status_code", StatusCode); + if (outcome is not ("completed" or "cancelled")) + _activity?.SetStatus(ActivityStatusCode.Error, outcome); + AppDiagnostics.AssistantProviderDuration.Record(duration, new KeyValuePair("outcome", outcome)); + logger.Log(outcome is "completed" or "cancelled" ? LogLevel.Information : LogLevel.Warning, + "Assistant provider request {ProviderRequestNumber} {ProviderOutcome} for turn {AssistantTurnId}: duration={DurationMs} ms generation={ProviderGenerationId} model={ProviderModel} provider={ProviderName} status={ProviderStatusCode} finish={ProviderFinishReason} input_characters={InputCharacters} output_characters={OutputCharacters} tools_allowed={ToolsAllowed} tool_calls={ToolCalls} usage_received={UsageReceived} prompt_tokens={PromptTokens} completion_tokens={CompletionTokens} reasoning_tokens={ReasoningTokens} received_done={ReceivedDone}", + turn.ProviderRequests, outcome, turn.TurnId, duration, GenerationId, Model ?? turn.Model, ProviderName, StatusCode, + FinishReason, inputCharacters, outputCharacters, allowTools, toolCalls, UsageReceived, PromptTokens, CompletionTokens, ReasoningTokens, receivedDone); + _activity?.Dispose(); + } + + public void Dispose() => Finish(StatusCode is < 200 or >= 300 ? "provider_http_error" + : _receivedError || FinishReason == "error" ? "provider_error" + : cancellationToken.IsCancellationRequested ? GetCancellationOutcome() : "interrupted"); + + private string GetCancellationOutcome() + { + string reason = turn.GetCancellationReason(cancellationToken, "provider_timeout"); + return reason == "client_disconnected" ? "cancelled" : reason; + } + + private static string? GetMetadata(JsonElement element, string name) + => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String + ? SafeMetadata(property.GetString()) : null; + + private static long? GetTokenCount(JsonElement element, string name) + => element.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.Number && property.TryGetInt64(out long value) + ? Math.Max(0, value) : null; + + private static string? SafeMetadata(string? value) + => value is { Length: > 0 and <= 128 } && value.All(character => Char.IsAsciiLetterOrDigit(character) || character is '-' or '_' or '/' or '.' or ':' or '~' or ' ') + ? value : null; +} diff --git a/src/Exceptionless.Web/Assistant/AssistantProviderException.cs b/src/Exceptionless.Web/Assistant/AssistantProviderException.cs index da724717e0..501ac03bd2 100644 --- a/src/Exceptionless.Web/Assistant/AssistantProviderException.cs +++ b/src/Exceptionless.Web/Assistant/AssistantProviderException.cs @@ -1,3 +1,6 @@ namespace Exceptionless.Web.Assistant; -public sealed class AssistantProviderException(string message) : Exception(message); +public sealed class AssistantProviderException(string message) : Exception(message) +{ + internal string FailureCode { get; init; } = "provider_error"; +} diff --git a/src/Exceptionless.Web/Assistant/AssistantService.cs b/src/Exceptionless.Web/Assistant/AssistantService.cs index c52de20f52..d8e855cfb4 100644 --- a/src/Exceptionless.Web/Assistant/AssistantService.cs +++ b/src/Exceptionless.Web/Assistant/AssistantService.cs @@ -36,14 +36,24 @@ public sealed class AssistantService( private static readonly JsonSerializerOptions s_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web).ConfigureExceptionlessApiDefaults(); private static readonly Regex s_rawDsmlPattern = new(@"<\s*/?\s*[|｜]\s*DSML\s*[|｜]", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)); - public async IAsyncEnumerable StreamAsync( + public IAsyncEnumerable StreamAsync( AssistantChatRequest request, string userId, AssistantPlanOptions planOptions, + CancellationToken cancellationToken = default) + => StreamAsync(request, userId, planOptions, null, cancellationToken); + + internal async IAsyncEnumerable StreamAsync( + AssistantChatRequest request, + string userId, + AssistantPlanOptions planOptions, + AssistantTurnDiagnostics? diagnostics, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var options = appOptions.AssistantOptions; string model = (await assistantModelSettingsService.GetAsync()).Model; + if (diagnostics is not null) + diagnostics.Model = model; AssistantConversationState? conversationState = null; if (!String.IsNullOrWhiteSpace(request.OrganizationId) && !String.IsNullOrWhiteSpace(request.ConversationId)) { @@ -71,10 +81,12 @@ public async IAsyncEnumerable StreamAsync( { if (completedToolRounds > 0) { + if (diagnostics is not null) + diagnostics.Stage = "usage_check"; var usageDecision = await assistantUsageService.TryContinueTurnAsync(request.OrganizationId, planOptions); if (!usageDecision.Allowed) { - yield return AssistantStreamEvent.Error(usageDecision.Message ?? "Exie reached this organization's usage limit."); + yield return AssistantStreamEvent.Error(usageDecision.Message ?? "Exie reached this organization's usage limit.", "usage_limit"); yield return AssistantStreamEvent.Done(); yield break; } @@ -110,87 +122,118 @@ public async IAsyncEnumerable StreamAsync( if (providerInputCharacters > AssistantLimits.MaximumProviderInputCharacters) { throw new AssistantProviderException( - "This conversation contains too much context for one response. Clear the conversation or narrow the question."); + "This conversation contains too much context for one response. Clear the conversation or narrow the question.") { FailureCode = "context_limit" }; } + if (diagnostics is not null) + diagnostics.Stage = "usage_reservation"; await using var providerRequest = await assistantUsageService.StartProviderRequestAsync(request.OrganizationId, providerInputCharacters); - using var response = await SendRequestAsync(messages, options, model, allowTools, request, cancellationToken); - providerRequest.MarkAccepted(); - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); - using var reader = new StreamReader(stream); - - while (await reader.ReadLineAsync(cancellationToken) is { } line) + using var providerDiagnostics = diagnostics?.StartProviderRequest(providerInputCharacters, allowTools, cancellationToken); + bool receivedDone = false; + try { - if (!line.StartsWith("data:", StringComparison.Ordinal)) - continue; + using var response = await SendRequestAsync(messages, options, model, allowTools, request, providerDiagnostics, cancellationToken); + providerRequest.MarkAccepted(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var reader = new StreamReader(stream); - string payload = line[5..].Trim(); - if (payload.Length == 0 || payload == "[DONE]") - continue; - - using var document = JsonDocument.Parse(payload); - if (document.RootElement.TryGetProperty("error", out var error)) - throw new AssistantProviderException(GetProviderError(error)); - - if (!usageRecorded && TryGetProviderUsage(document.RootElement, out var usage)) + while (await reader.ReadLineAsync(cancellationToken) is { } line) { - usageRecorded = true; - try - { - await providerRequest.ReconcileAsync(usage); - } - catch (Exception ex) + if (!line.StartsWith("data:", StringComparison.Ordinal)) + continue; + + string payload = line[5..].Trim(); + if (payload == "[DONE]") { - // Disposal records the conservative reservation when detailed provider - // accounting cannot be reconciled. - logger.LogError(ex, "Unable to record assistant provider usage for organization {OrganizationId}", request.OrganizationId); + receivedDone = true; + continue; } - } - - if (!document.RootElement.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0) - continue; + if (payload.Length == 0) + continue; - var delta = choices[0].GetProperty("delta"); - if (delta.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) - { - string? text = content.GetString(); - if (!String.IsNullOrEmpty(text)) + try { - assistantContent.Append(text); - assistantContentChunks.Add(text); - } - } + using var document = JsonDocument.Parse(payload); + providerDiagnostics?.ObserveChunk(document.RootElement); + if (document.RootElement.TryGetProperty("error", out var error)) + throw new AssistantProviderException(GetProviderError(error)); - if (!delta.TryGetProperty("tool_calls", out var toolCallUpdates)) - continue; + if (!usageRecorded && TryGetProviderUsage(document.RootElement, out var usage)) + { + usageRecorded = true; + try + { + await providerRequest.ReconcileAsync(usage); + } + catch (Exception ex) + { + // Disposal records the conservative reservation when detailed provider + // accounting cannot be reconciled. + logger.LogError(ex, "Unable to record assistant provider usage for organization {OrganizationId}", request.OrganizationId); + } + } - foreach (var update in toolCallUpdates.EnumerateArray()) - { - int index = update.GetProperty("index").GetInt32(); - if (!toolCalls.TryGetValue(index, out var pending)) - { - pending = new PendingToolCall(); - toolCalls[index] = pending; - } + if (!document.RootElement.TryGetProperty("choices", out var choices) || choices.GetArrayLength() == 0) + continue; - if (update.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) - pending.Id = id.GetString() ?? pending.Id; + var delta = choices[0].GetProperty("delta"); + if (delta.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.String) + { + string? text = content.GetString(); + if (!String.IsNullOrEmpty(text)) + { + assistantContent.Append(text); + assistantContentChunks.Add(text); + } + } - if (!update.TryGetProperty("function", out var function)) - continue; + if (!delta.TryGetProperty("tool_calls", out var toolCallUpdates)) + continue; - if (function.TryGetProperty("name", out var name) && name.ValueKind == JsonValueKind.String) - pending.Name += name.GetString(); - if (function.TryGetProperty("arguments", out var arguments) && arguments.ValueKind == JsonValueKind.String) - pending.Arguments.Append(arguments.GetString()); + foreach (var update in toolCallUpdates.EnumerateArray()) + { + int index = update.GetProperty("index").GetInt32(); + if (!toolCalls.TryGetValue(index, out var pending)) + { + pending = new PendingToolCall(); + toolCalls[index] = pending; + } + + if (update.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) + pending.Id = id.GetString() ?? pending.Id; + + if (!update.TryGetProperty("function", out var function)) + continue; + + if (function.TryGetProperty("name", out var name) && name.ValueKind == JsonValueKind.String) + pending.Name += name.GetString(); + if (function.TryGetProperty("arguments", out var arguments) && arguments.ValueKind == JsonValueKind.String) + pending.Arguments.Append(arguments.GetString()); + } + } + catch (Exception ex) when (ex is InvalidOperationException or KeyNotFoundException or FormatException) + { + throw new JsonException("The AI provider returned an invalid response structure.", ex); + } } } + catch (Exception ex) + { + providerDiagnostics?.RecordException(ex); + throw; + } + + if (diagnostics is not null) + diagnostics.Stage = "response_validation"; if (s_rawDsmlPattern.IsMatch(assistantContent.ToString())) { + providerDiagnostics?.Reject("malformed_response", assistantContent.Length, toolCalls.Count, receivedDone); if (malformedResponseRetries < AssistantLimits.MaximumMalformedResponseRetries) { malformedResponseRetries++; + if (diagnostics is not null) + diagnostics.MalformedResponseRetries = malformedResponseRetries; logger.LogWarning( "Assistant provider returned raw DSML content for organization {OrganizationId}; retrying response", request.OrganizationId); @@ -206,7 +249,7 @@ public async IAsyncEnumerable StreamAsync( logger.LogWarning( "Assistant provider returned raw DSML content again for organization {OrganizationId}", request.OrganizationId); - yield return AssistantStreamEvent.Error("Exie received a malformed response from the AI provider. Please try again."); + yield return AssistantStreamEvent.Error("Exie received a malformed response from the AI provider. Please try again.", "malformed_response"); yield return AssistantStreamEvent.Done(); yield break; } @@ -217,6 +260,11 @@ public async IAsyncEnumerable StreamAsync( malformedResponseCorrection = null; } + if (!allowTools && toolCalls.Count > 0) + providerDiagnostics?.Reject("tool_round_limit", assistantContent.Length, toolCalls.Count, receivedDone); + else + providerDiagnostics?.Complete(assistantContent.Length, toolCalls.Count, receivedDone); + foreach (string text in assistantContentChunks) { yield return AssistantStreamEvent.TextDelta(text); @@ -226,7 +274,14 @@ public async IAsyncEnumerable StreamAsync( { if (assistantContent.Length == 0) { - yield return AssistantStreamEvent.Error("Exie stopped before providing an answer. Please try again."); + string failureCode = providerDiagnostics?.FinishReason switch + { + "length" => "output_limit", + "content_filter" => "content_filter", + "error" => "provider_error", + _ => "empty_response" + }; + yield return AssistantStreamEvent.Error("Exie stopped before providing an answer. Please try again.", failureCode); } else if (pendingSuggestedActions.Count > 0) { @@ -239,7 +294,7 @@ public async IAsyncEnumerable StreamAsync( if (!allowTools) { - yield return AssistantStreamEvent.Error("Exie could not finish using the available tool results. Try narrowing the question."); + yield return AssistantStreamEvent.Error("Exie could not finish using the available tool results. Try narrowing the question.", "tool_round_limit"); yield return AssistantStreamEvent.Done(); yield break; } @@ -293,6 +348,8 @@ public async IAsyncEnumerable StreamAsync( requireFinalAnswer = true; completedToolRounds++; + if (diagnostics is not null) + diagnostics.ToolRounds = completedToolRounds; continue; } @@ -316,8 +373,10 @@ public async IAsyncEnumerable StreamAsync( foreach (var toolCall in executableToolCalls) { string arguments = toolCall.Arguments.ToString(); + diagnostics?.StartTool(toolCall.Name); yield return AssistantStreamEvent.ToolCall(toolCall.Id, toolCall.Name, arguments); + long toolStarted = timeProvider.GetTimestamp(); string result; if (remainingToolCalls <= 0) { @@ -350,9 +409,18 @@ public async IAsyncEnumerable StreamAsync( if (toolCall.Name == SearchStacksTool) remainingProjectSearches--; - result = await ExecuteToolAsync(toolCall.Name, arguments, request, cancellationToken); + try + { + result = await ExecuteToolAsync(toolCall.Name, arguments, request, cancellationToken); + } + catch (Exception ex) + { + diagnostics?.RecordToolException(ex, timeProvider.GetElapsedTime(toolStarted).TotalMilliseconds, cancellationToken); + throw; + } } + diagnostics?.RecordToolResult(result, timeProvider.GetElapsedTime(toolStarted).TotalMilliseconds); if (toolCall.Name == GetProjectSetupTool) configureHref = AssistantSuggestedActionParser.GetProjectSetupHref(result) ?? configureHref; @@ -372,6 +440,8 @@ public async IAsyncEnumerable StreamAsync( && !String.IsNullOrWhiteSpace(request.OrganizationId) && !String.IsNullOrWhiteSpace(request.ConversationId)) { + if (diagnostics is not null) + diagnostics.Stage = "conversation_save"; await assistantConversationService.AppendToolResultsAsync( userId, request.OrganizationId, @@ -381,10 +451,12 @@ await assistantConversationService.AppendToolResultsAsync( } completedToolRounds++; + if (diagnostics is not null) + diagnostics.ToolRounds = completedToolRounds; } } - private async Task SendRequestAsync(List messages, AssistantOptions options, string model, bool allowTools, AssistantChatRequest chatRequest, CancellationToken cancellationToken) + private async Task SendRequestAsync(List messages, AssistantOptions options, string model, bool allowTools, AssistantChatRequest chatRequest, AssistantProviderDiagnostics? diagnostics, CancellationToken cancellationToken) { var client = httpClientFactory.CreateClient(nameof(AssistantService)); using var providerRequest = new HttpRequestMessage(HttpMethod.Post, options.Endpoint); @@ -413,13 +485,15 @@ private async Task SendRequestAsync(List messages, providerRequest.Content = JsonContent.Create(payload); var response = await client.SendAsync(providerRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + diagnostics?.ObserveResponse(response); if (response.IsSuccessStatusCode) return response; - string detail = await response.Content.ReadAsStringAsync(cancellationToken); - logger.LogWarning("Assistant provider returned {StatusCode}: {Detail}", (int)response.StatusCode, detail); + // Provider error bodies can echo prompts or credentials. Status and generation ID + // provide diagnostic context without copying those bodies into application logs. + logger.LogWarning("Assistant provider returned HTTP {StatusCode} with generation {ProviderGenerationId}", (int)response.StatusCode, diagnostics?.GenerationId); response.Dispose(); - throw new AssistantProviderException($"The AI provider returned status {(int)response.StatusCode}."); + throw new AssistantProviderException($"The AI provider returned status {(int)response.StatusCode}.") { FailureCode = "provider_http_error" }; } private async Task ExecuteToolAsync( diff --git a/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs new file mode 100644 index 0000000000..6468ccc2ac --- /dev/null +++ b/src/Exceptionless.Web/Assistant/AssistantTurnDiagnostics.cs @@ -0,0 +1,180 @@ +using System.Diagnostics; +using System.Text.Json; +using Exceptionless.Web.Mcp; + +namespace Exceptionless.Web.Assistant; + +internal sealed class AssistantTurnDiagnostics : IDisposable +{ + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly long _started; + private readonly Activity? _activity; + private readonly IDisposable? _scope; + private readonly CancellationToken _requestAborted; + private bool _finished; + private double? _firstTextDuration; + private string? _failureCode; + + public AssistantTurnDiagnostics(ILogger logger, TimeProvider timeProvider, string organizationId, string conversationId, string requestId, CancellationToken requestAborted = default) + { + _logger = logger; + _timeProvider = timeProvider; + _requestAborted = requestAborted; + _started = timeProvider.GetTimestamp(); + _activity = AppDiagnostics.AssistantActivitySource.StartActivity("assistant.turn"); + TurnId = Guid.NewGuid().ToString("N"); + OrganizationId = organizationId; + ConversationId = conversationId; + RequestId = requestId; + TraceId = Activity.Current?.TraceId.ToString(); + _activity?.SetTag("assistant.turn.id", TurnId); + _activity?.SetTag("organization.id", organizationId); + _activity?.SetTag("assistant.conversation.id", conversationId); + _scope = logger.BeginScope(new Dictionary + { + ["AssistantTurnId"] = TurnId, + ["OrganizationId"] = organizationId, + ["ConversationId"] = conversationId, + ["RequestId"] = requestId, + ["TraceId"] = TraceId + }); + } + + public string TurnId { get; } + public string OrganizationId { get; } + public string ConversationId { get; } + public string RequestId { get; } + public string? TraceId { get; } + public bool IsClientDisconnected => _requestAborted.IsCancellationRequested; + public string? Model { get; set; } + public string Stage { get; set; } = "initializing"; + public int ProviderRequests { get; private set; } + public int ToolCalls { get; private set; } + public int ToolFailures { get; private set; } + public int ToolRounds { get; set; } + public int MalformedResponseRetries { get; set; } + public string? LastTool { get; private set; } + public string? LastToolError { get; private set; } + public AssistantProviderDiagnostics? Provider { get; private set; } + + public string GetCancellationReason(CancellationToken cancellationToken, string operationReason) + => IsClientDisconnected ? "client_disconnected" : cancellationToken.IsCancellationRequested ? "turn_timeout" : operationReason; + + public AssistantProviderDiagnostics StartProviderRequest(int inputCharacters, bool allowTools, CancellationToken cancellationToken) + { + Stage = "provider_request"; + ProviderRequests++; + Provider = new AssistantProviderDiagnostics(_logger, _timeProvider, this, inputCharacters, allowTools, cancellationToken); + return Provider; + } + + public void Observe(AssistantStreamEvent item) + { + if (item.Type == "error") + _failureCode ??= item.FailureCode ?? "response_error"; + if (item.Type == "text_delta" && !String.IsNullOrEmpty(item.Text)) + _firstTextDuration ??= ElapsedMilliseconds; + } + + public void StartTool(string name) + { + Stage = "tool_execution"; + LastTool = GetToolName(name); + ToolCalls++; + } + + public void RecordToolResult(string result, double durationMilliseconds) + { + using var document = JsonDocument.Parse(result); + var root = document.RootElement; + bool failed = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("ok", out var ok) && ok.ValueKind == JsonValueKind.False; + string? errorCode = null; + if (failed) + { + ToolFailures++; + errorCode = "tool_error"; + if (root.TryGetProperty("error", out var error) && error.ValueKind == JsonValueKind.Object + && error.TryGetProperty("code", out var code) && code.ValueKind == JsonValueKind.String) + errorCode = GetToolErrorCode(code.GetString()); + LastToolError = errorCode; + _logger.LogWarning("Assistant tool {ToolName} failed with {ToolErrorCode} in {DurationMs} ms for turn {AssistantTurnId}", + LastTool, errorCode, durationMilliseconds, TurnId); + } + + AppDiagnostics.AssistantToolDuration.Record(durationMilliseconds, + new("tool", LastTool), new("outcome", failed ? "failed" : "completed"), new("reason", errorCode ?? "none")); + } + + public void RecordToolException(Exception exception, double durationMilliseconds, CancellationToken cancellationToken) + { + LastToolError = exception is OperationCanceledException ? GetCancellationReason(cancellationToken, "operation_cancelled") : "tool_execution_error"; + bool cancelled = LastToolError == "client_disconnected"; + string outcome = cancelled ? "cancelled" : "failed"; + if (!cancelled) + { + ToolFailures++; + } + + _logger.Log(cancelled ? LogLevel.Information : LogLevel.Warning, + "Assistant tool {ToolName} {ToolOutcome} with {ToolErrorCode} in {DurationMs} ms for turn {AssistantTurnId}: exception_type={ExceptionType}", + LastTool, outcome, LastToolError, durationMilliseconds, TurnId, exception.GetType().FullName); + AppDiagnostics.AssistantToolDuration.Record(durationMilliseconds, + new("tool", LastTool), new("outcome", outcome), new("reason", LastToolError)); + } + + public void Finish(string outcome, string? failureCode = null, Exception? exception = null) + { + if (_finished) + return; + _finished = true; + string reason = failureCode ?? _failureCode ?? "none"; + _activity?.SetTag("assistant.outcome", outcome); + _activity?.SetTag("assistant.failure.reason", reason); + _activity?.SetTag("assistant.stage", Stage); + _activity?.SetTag("assistant.model", Model); + _activity?.SetTag("assistant.provider.requests", ProviderRequests); + _activity?.SetTag("assistant.tool.calls", ToolCalls); + if (outcome == "failed") + _activity?.SetStatus(ActivityStatusCode.Error, reason); + + AppDiagnostics.AssistantTurnDuration.Record(ElapsedMilliseconds, + new("outcome", outcome), new("reason", reason), new("stage", Stage)); + + // Include correlation in the message as well as the scope: the production console + // formatter does not render scope properties, and streaming errors retain HTTP 200. + var level = outcome == "failed" ? exception is null ? LogLevel.Warning : LogLevel.Error : LogLevel.Information; + _logger.Log(level, + "Assistant turn {AssistantTurnId} {Outcome}: reason={FailureReason} stage={Stage} duration={DurationMs} ms first_text={FirstTextDurationMs} ms organization={OrganizationId} conversation={ConversationId} request={RequestId} trace={TraceId} model={Model} provider_requests={ProviderRequests} tool_rounds={ToolRounds} tool_calls={ToolCalls} tool_failures={ToolFailures} last_tool={LastTool} last_tool_error={LastToolError} malformed_retries={MalformedResponseRetries} generation={ProviderGenerationId} provider_model={ProviderModel} provider={ProviderName} status={ProviderStatusCode} finish={ProviderFinishReason} usage_received={ProviderUsageReceived} reasoning_tokens={ReasoningTokens} exception_type={ExceptionType} exception_stack={ExceptionStackTrace}", + TurnId, outcome, reason, Stage, ElapsedMilliseconds, _firstTextDuration, OrganizationId, ConversationId, RequestId, TraceId, + Model, ProviderRequests, ToolRounds, ToolCalls, ToolFailures, LastTool, LastToolError, MalformedResponseRetries, + Provider?.GenerationId, Provider?.Model, Provider?.ProviderName, Provider?.StatusCode, Provider?.FinishReason, + Provider?.UsageReceived, Provider?.ReasoningTokens, exception?.GetType().FullName, exception?.StackTrace); + } + + private double ElapsedMilliseconds => _timeProvider.GetElapsedTime(_started).TotalMilliseconds; + + public void Dispose() + { + _scope?.Dispose(); + _activity?.Dispose(); + } + + private static string GetToolName(string name) => name switch + { + "get_event" or "get_stack" or "get_project_setup" or "get_stack_events" or "list_projects" or "search_stacks" + or "update_stack_status" or "snooze_stack" or "set_stack_critical" or "add_stack_reference_link" or "remove_stack_reference_link" => name, + _ => "unknown" + }; + + private static string GetToolErrorCode(string? code) => code switch + { + McpErrorCodes.ContextMismatch or McpErrorCodes.ContextRequired or McpErrorCodes.Forbidden or McpErrorCodes.InvalidClientPlatform + or McpErrorCodes.InvalidCursor or McpErrorCodes.InvalidDetailSize or McpErrorCodes.InvalidFilter or McpErrorCodes.InvalidGroupBy + or McpErrorCodes.InvalidId or McpErrorCodes.InvalidInterval or McpErrorCodes.InvalidLimit or McpErrorCodes.InvalidReferenceUrl + or McpErrorCodes.InvalidSnooze or McpErrorCodes.InvalidSort or McpErrorCodes.InvalidStatus or McpErrorCodes.InvalidTimeRange + or McpErrorCodes.InvalidVersion or McpErrorCodes.NotAccessible or McpErrorCodes.NotFound or McpErrorCodes.QueryFailed + or McpErrorCodes.UnknownFilterField or "tool_call_limit_reached" or "project_search_limit_reached" => code, + _ => "tool_error" + }; +} diff --git a/src/Exceptionless.Web/ClientApp/src/hooks.client.ts b/src/Exceptionless.Web/ClientApp/src/hooks.client.ts index 5ddf0f4807..7dc20473c0 100644 --- a/src/Exceptionless.Web/ClientApp/src/hooks.client.ts +++ b/src/Exceptionless.Web/ClientApp/src/hooks.client.ts @@ -3,6 +3,7 @@ import type { ClientInit, HandleClientError } from '@sveltejs/kit'; import { dev } from '$app/environment'; import { page } from '$app/state'; import { env } from '$env/dynamic/public'; +import { configureSessions } from '$features/auth/exceptionless-session'; import { normalizePath, normalizeRouteId } from '$lib/telemetry'; import { installSvelteEffectDepthDiagnostics } from '$lib/telemetry/svelte-effect-depth-diagnostics'; import { Exceptionless, guid, toError } from '@exceptionless/browser'; @@ -34,7 +35,8 @@ export const init: ClientInit = async () => { await Exceptionless.startup((c) => { c.apiKey = env.PUBLIC_EXCEPTIONLESS_API_KEY; - c.serverUrl = env.PUBLIC_EXCEPTIONLESS_SERVER_URL || window.location.origin; + c.serverUrl = + PUBLIC_EXCEPTIONLESS_SERVER_URL || (env.PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL ?? env.PUBLIC_EXCEPTIONLESS_SERVER_URL) || window.location.origin; c.defaultTags.push('UI', 'Svelte'); if (env.PUBLIC_APP_VERSION) { @@ -46,7 +48,7 @@ export const init: ClientInit = async () => { c.settings['@@log:*'] = 'debug'; } - c.useSessions(); + configureSessions(c); c.addPlugin('route-context', 10, async (ctx) => { if (ctx.event.type !== 'usage') { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts index 5ef6d164e9..ebc575cad3 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/api.svelte.ts @@ -16,6 +16,7 @@ import type { OAuthApplication, OAuthApplicationRequest, PredefinedSavedViewDefinition, + UpdateAssistantConversationSharingSettingsRequest, UpdateAssistantEnabledSettingsRequest, UpdateAssistantSettingsRequest, UpdateEventSubmissionSettingsRequest @@ -338,6 +339,27 @@ export function postOAuthApplicationMutation() { })); } +export function putAdminAssistantConversationSharingSettingsMutation() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + mutationFn: async (request) => { + const client = useFetchClient(); + const response = await client.putJSON('admin/assistant-settings/conversation-sharing', request); + + if (!response.ok) { + throw response.problem; + } + + return response.data!; + }, + onSuccess: async (settings) => { + queryClient.setQueryData(queryKeys.assistantSettings, settings); + await invalidateAssistantAccessQueries(queryClient); + } + })); +} + export function putAdminAssistantEnabledSettingsMutation() { const queryClient = useQueryClient(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte index 319e367fdb..0a4a95b4fd 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte @@ -7,7 +7,12 @@ import { Separator } from '$comp/ui/separator'; import { Spinner } from '$comp/ui/spinner'; import { Switch } from '$comp/ui/switch'; - import { getAdminAssistantSettingsQuery, putAdminAssistantEnabledSettingsMutation, putAdminAssistantSettingsMutation } from '$features/admin/api.svelte'; + import { + getAdminAssistantSettingsQuery, + putAdminAssistantConversationSharingSettingsMutation, + putAdminAssistantEnabledSettingsMutation, + putAdminAssistantSettingsMutation + } from '$features/admin/api.svelte'; import { type AssistantSettingsFormData, AssistantSettingsSchema } from '$features/admin/schemas'; import { ariaInvalid, getFormErrorMessages, mapFieldErrors, problemDetailsToFormErrors } from '$features/shared/validation'; import { ProblemDetails } from '@foundatiofx/fetchclient'; @@ -16,9 +21,12 @@ const settingsQuery = getAdminAssistantSettingsQuery(); const updateEnabledSettings = putAdminAssistantEnabledSettingsMutation(); + const updateConversationSharingSettings = putAdminAssistantConversationSharingSettingsMutation(); const updateSettings = putAdminAssistantSettingsMutation(); let assistantEnabled = $state(false); + let conversationSharingDefaultEnabled = $state(false); let loadedAvailabilityKey = $state(null); + let loadedConversationSharingDefaultEnabled = $state(); let loadedSettingsKey = $state(null); const settings = $derived(settingsQuery.data); const availabilityKey = $derived( @@ -62,6 +70,15 @@ assistantEnabled = settings.enabled; }); + $effect(() => { + if (!settings || loadedConversationSharingDefaultEnabled === settings.conversation_sharing_default_enabled) { + return; + } + + loadedConversationSharingDefaultEnabled = settings.conversation_sharing_default_enabled; + conversationSharingDefaultEnabled = settings.conversation_sharing_default_enabled; + }); + $effect(() => { if (!settings || loadedSettingsKey === settingsKey) { return; @@ -106,6 +123,20 @@ toast.error('Failed to reset Exie availability.'); } } + + async function saveConversationSharingDefault() { + try { + const saved = await updateConversationSharingSettings.mutateAsync({ + enabled: conversationSharingDefaultEnabled + }); + conversationSharingDefaultEnabled = saved.conversation_sharing_default_enabled; + toast.success( + saved.conversation_sharing_default_enabled ? 'Exie conversation sharing default is enabled.' : 'Exie conversation sharing default is disabled.' + ); + } catch { + toast.error('Failed to update Exie conversation sharing default.'); + } + } {#if settingsQuery.isPending} @@ -159,6 +190,34 @@ + + + Conversation sharing default + + Choose whether users share Exie messages and replies by default to help improve the feature. Users can change this in Exie; their saved choice + always takes precedence. Usage and error diagnostics remain available either way. + + +
+ + +
+ + + +
{ event.preventDefault(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts new file mode 100644 index 0000000000..ccf29bfd9f --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/components/assistant-settings.svelte.test.ts @@ -0,0 +1,63 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ + enabled: false, + error: vi.fn(), + success: vi.fn(), + update: vi.fn() +})); +vi.mock('svelte-sonner', () => ({ toast: { error: state.error, success: state.success } })); +vi.mock('$features/admin/api.svelte', () => ({ + getAdminAssistantSettingsQuery: () => ({ + data: { + configured_enabled: true, + configured_model: 'example/model', + conversation_sharing_default_enabled: state.enabled, + enabled: true, + is_configured: true, + is_enabled_overridden: false, + is_overridden: false, + model: 'example/model' + }, + isError: false, + isPending: false + }), + putAdminAssistantConversationSharingSettingsMutation: () => ({ isPending: false, mutateAsync: state.update }), + putAdminAssistantEnabledSettingsMutation: () => ({ isPending: false, mutateAsync: vi.fn() }), + putAdminAssistantSettingsMutation: () => ({ isPending: false, mutateAsync: vi.fn() }) +})); + +import AssistantSettings from './assistant-settings.svelte'; + +describe('Exie conversation sharing default settings', () => { + beforeEach(() => { + state.enabled = false; + state.update.mockReset(); + state.success.mockClear(); + state.error.mockClear(); + state.update.mockImplementation(async ({ enabled }: { enabled: boolean }) => ({ conversation_sharing_default_enabled: enabled })); + }); + + it.each([false, true])('loads and saves the full logging switch from %s', async (enabled) => { + state.enabled = enabled; + render(AssistantSettings); + const toggle = screen.getByRole('switch', { name: 'Conversation sharing default' }); + const save = screen.getByRole('button', { name: 'Save Exie conversation sharing default' }); + await waitFor(() => expect(toggle.getAttribute('aria-checked')).toBe(String(enabled))); + expect(save.hasAttribute('disabled')).toBe(true); + await fireEvent.click(toggle); + await fireEvent.click(save); + await waitFor(() => expect(state.update).toHaveBeenCalledWith({ enabled: !enabled })); + await waitFor(() => expect(state.success).toHaveBeenCalledOnce()); + }); + + it('shows a save failure without claiming the logging mode changed', async () => { + state.update.mockRejectedValueOnce(new Error('Unavailable')); + render(AssistantSettings); + await fireEvent.click(screen.getByRole('switch', { name: 'Conversation sharing default' })); + await fireEvent.click(screen.getByRole('button', { name: 'Save Exie conversation sharing default' })); + await waitFor(() => expect(state.error).toHaveBeenCalledWith('Failed to update Exie conversation sharing default.')); + expect(state.success).not.toHaveBeenCalled(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts index b362c30b17..09666db567 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/admin/models.ts @@ -2,6 +2,7 @@ import type { AssistantModelSettings, CountResult, EventSubmissionSettings, + UpdateAssistantConversationSharingSettings, UpdateAssistantEnabledSettings, UpdateAssistantSettings, UpdateEventSubmissionSettings @@ -203,6 +204,7 @@ export type ShardMetric = { value: number; }; +export type UpdateAssistantConversationSharingSettingsRequest = UpdateAssistantConversationSharingSettings; export type UpdateAssistantEnabledSettingsRequest = UpdateAssistantEnabledSettings; export type UpdateAssistantSettingsRequest = UpdateAssistantSettings; export type UpdateEventSubmissionSettingsRequest = UpdateEventSubmissionSettings; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts index 23a3767371..f089573b35 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/api.svelte.ts @@ -2,12 +2,13 @@ import type { QueryClient } from '@tanstack/svelte-query'; import { accessToken } from '$features/auth/index.svelte'; import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; -import { createQuery } from '@tanstack/svelte-query'; +import { createMutation, createQuery, useQueryClient } from '@tanstack/svelte-query'; -import type { AssistantAccess } from './models'; +import type { AssistantAccess, AssistantConversationSharingSettings } from './models'; export const queryKeys = { access: (organizationId: string | undefined) => [...queryKeys.type, 'access', organizationId] as const, + conversationSharing: ['Assistant', 'conversation-sharing'] as const, type: ['Assistant'] as const }; @@ -36,8 +37,45 @@ export function getAssistantAccessQuery(request: GetAssistantAccessRequest) { })); } +export function getAssistantConversationSharingQuery(request: { enabled: boolean }) { + return createQuery(() => ({ + enabled: () => !!accessToken.current && request.enabled, + queryFn: async ({ signal }) => { + const response = await useFetchClient().getJSON('assistant/conversation-sharing', { + signal + }); + if (!response.ok) { + throw response.problem; + } + return response.data!; + }, + queryKey: queryKeys.conversationSharing + })); +} + export async function invalidateAssistantAccessQueries(queryClient: QueryClient): Promise { await queryClient.invalidateQueries({ queryKey: queryKeys.type }); } + +export function putAssistantConversationSharingMutation() { + const queryClient = useQueryClient(); + return createMutation(() => ({ + mutationFn: async (request) => { + const response = await useFetchClient().putJSON('assistant/conversation-sharing', request); + if (!response.ok) { + throw response.problem; + } + return response.data!; + }, + onMutate: async () => { + await queryClient.cancelQueries({ + queryKey: queryKeys.conversationSharing + }); + }, + onSuccess: (settings) => { + queryClient.setQueryData(queryKeys.conversationSharing, settings); + } + })); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts new file mode 100644 index 0000000000..6eaaa84915 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.test.ts @@ -0,0 +1,118 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { submitFeatureUsage, submitLog } = vi.hoisted(() => ({ + submitFeatureUsage: vi.fn(() => Promise.resolve()), + submitLog: vi.fn(() => Promise.resolve()) +})); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); + +import { AssistantTurnTelemetry, trackAssistantEvent } from './assistant-telemetry'; + +const context = { + assistant_message_id: 'response-1', + conversation_id: 'conversation-1', + mode: 'sheet' as const, + organization_id: 'organization-1', + user_message_id: 'prompt-1' +}; + +describe('Exie session telemetry', () => { + beforeEach(() => { + submitFeatureUsage.mockReset(); + submitLog.mockReset(); + }); + + it('records one prompt and one response outcome without conversation or tool content', () => { + const turn = new AssistantTurnTelemetry(context, 25, 'composer'); + turn.observe({ text: 'The checkout ', type: 'text_delta' }); + turn.observe({ arguments: '{"secret":"tool-argument"}', tool_call_id: 'tool-1', tool_name: 'get_event', type: 'tool_call' }); + turn.observe({ result: '{"ok":true,"data":{"secret":"tool-result"}}', tool_call_id: 'tool-1', type: 'tool_result' }); + turn.observe({ text: 'request timed out.', type: 'text_delta' }); + turn.observe({ type: 'done' }); + + expect(turn.finish()).toBe('completed'); + expect(turn.finish()).toBeUndefined(); + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(submitLog).not.toHaveBeenCalled(); + expect(submitFeatureUsage).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', { + exie: expect.objectContaining({ ...context, message_characters: 25, prompt_source: 'composer', role: 'user' }) + }); + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseCompleted', { + exie: expect.objectContaining({ ...context, outcome: 'completed', response_characters: 31, role: 'assistant', tool_calls: 1, tool_failures: 0 }) + }); + const telemetry = JSON.stringify(submitFeatureUsage.mock.calls); + expect(telemetry).not.toContain('checkout'); + expect(telemetry).not.toContain('tool-argument'); + expect(telemetry).not.toContain('tool-result'); + }); + + it('records a streamed failure even when a done event follows it', () => { + const turn = new AssistantTurnTelemetry(context, 11, 'starter'); + turn.observe({ text: 'Partial answer', type: 'text_delta' }); + turn.observe({ message: 'Exie took too long.', type: 'error' }); + turn.observe({ type: 'done' }); + expect(turn.finish()).toBe('failed'); + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseFailed', { + exie: expect.objectContaining({ error_message: 'Exie took too long.', reason: 'stream_error' }) + }); + expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('Partial answer'); + }); + + it('distinguishes stopped responses and ignores late content after cancellation', () => { + const turn = new AssistantTurnTelemetry(context, 11, 'composer'); + turn.observe({ text: 'Partial', type: 'text_delta' }); + expect(turn.finish('organization_changed')).toBe('cancelled'); + turn.observe({ text: 'Late content', type: 'text_delta' }); + turn.observe({ type: 'done' }); + turn.finish(); + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseCancelled', { + exie: expect.objectContaining({ organization_id: 'organization-1', reason: 'organization_changed' }) + }); + }); + + it('does not count a silently interrupted stream as a successful response', () => { + const turn = new AssistantTurnTelemetry(context, 11, 'composer'); + turn.observe({ text: 'Partial', type: 'text_delta' }); + expect(turn.finish()).toBe('failed'); + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseFailed', { + exie: expect.objectContaining({ reason: 'incomplete_stream', received_done: false }) + }); + }); + + it('bounds full logging while keeping accurate message counts and emitting one assembled response', () => { + const content = 'x'.repeat(20_000); + const turn = new AssistantTurnTelemetry(context, content.length, 'retry', { previous_conversation_id: 'previous-conversation' }); + turn.enableFullLogging(content); + turn.enableFullLogging(content); + turn.observe({ text: content, type: 'text_delta' }); + turn.observe({ type: 'done' }); + turn.finish(); + expect(submitFeatureUsage).toHaveBeenNthCalledWith(1, 'assistant.MessageSent', { + exie: expect.objectContaining({ message_characters: 20_000, previous_conversation_id: 'previous-conversation' }) + }); + expect(submitFeatureUsage).toHaveBeenLastCalledWith('assistant.ResponseCompleted', { + exie: expect.objectContaining({ message_characters: 20_000, response_characters: 20_000 }) + }); + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(JSON.stringify(submitFeatureUsage.mock.calls)).not.toContain('xxx'); + expect(submitLog).toHaveBeenCalledTimes(2); + expect(submitLog).toHaveBeenNthCalledWith(1, 'assistant.Prompt', content.slice(0, 16_384), { + exie: expect.objectContaining({ ...context, message_characters: 20_000, message_truncated: true, prompt_source: 'retry' }) + }); + expect(submitLog).toHaveBeenLastCalledWith('assistant.Response', content.slice(0, 16_384), { + exie: expect.objectContaining({ ...context, message_characters: 20_000, message_truncated: true, outcome: 'completed' }) + }); + }); + + it('keeps chat interactions working when telemetry cannot be submitted', async () => { + submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); + submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); + submitLog.mockRejectedValueOnce(new Error('offline')); + expect(() => trackAssistantEvent('assistant.ResponseHelpful', context)).not.toThrow(); + expect(() => new AssistantTurnTelemetry(context, 11, 'composer').enableFullLogging('Investigate')).not.toThrow(); + await Promise.resolve(); + expect(submitFeatureUsage).toHaveBeenCalledTimes(2); + expect(submitLog).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts new file mode 100644 index 0000000000..cb06ead740 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/assistant-telemetry.ts @@ -0,0 +1,140 @@ +import { submitFeatureUsage, submitLog } from '$features/auth/exceptionless-session'; + +import type { AssistantStreamEvent } from './assistant-stream'; + +import { assistantToolResultFailed } from './assistant-tool-result'; + +const maximumMessageCharacters = 16_384; + +export type AssistantPromptSource = 'composer' | 'queued' | 'regenerate' | 'retry' | 'starter' | 'suggested_action'; + +export type AssistantStopReason = 'access_changed' | 'component_unmounted' | 'conversation_cleared' | 'organization_changed' | 'user_stopped'; +export interface AssistantTelemetryContext { + assistant_message_id?: string; + conversation_id: string; + mode: 'page' | 'sheet'; + organization_id?: string; + path?: string; + project_id?: string; + user_message_id?: string; +} +export type AssistantTurnOutcome = 'cancelled' | 'completed' | 'failed'; + +export class AssistantTurnTelemetry { + private contentCharacters = 0; + private errorMessage: string | undefined; + private failureReason: string | undefined; + private finished = false; + private firstTextDuration: number | undefined; + private readonly promptDetails: Record; + private receivedDone = false; + private responseContent: string | undefined; + private started = performance.now(); + private toolCalls = 0; + private toolFailures = 0; + + constructor( + readonly context: AssistantTelemetryContext, + promptCharacters: number, + source: AssistantPromptSource, + details: Record = {} + ) { + this.promptDetails = { ...details, message_characters: promptCharacters, prompt_source: source, role: 'user' }; + trackAssistantEvent('assistant.MessageSent', context, this.promptDetails); + } + + disableFullLogging(): void { + this.responseContent = undefined; + } + + enableFullLogging(prompt: string): void { + if (this.finished || this.responseContent !== undefined) { + return; + } + + this.responseContent = ''; + trackAssistantLog('assistant.Prompt', prompt, this.context, { + ...this.promptDetails, + message_truncated: prompt.length > maximumMessageCharacters + }); + } + + fail(message: string, reason: 'request_error' | 'stream_error' | `http_${number}`): void { + this.errorMessage ??= message.slice(0, 2048); + this.failureReason ??= reason; + } + + finish(stopReason?: AssistantStopReason, details: Record = {}): AssistantTurnOutcome | undefined { + if (this.finished) { + return; + } + this.finished = true; + const outcome = stopReason ? 'cancelled' : this.failureReason || !this.receivedDone || this.contentCharacters === 0 ? 'failed' : 'completed'; + const reason = + stopReason ?? this.failureReason ?? (!this.receivedDone ? 'incomplete_stream' : this.contentCharacters === 0 ? 'empty_response' : undefined); + const feature = { cancelled: 'assistant.ResponseCancelled', completed: 'assistant.ResponseCompleted', failed: 'assistant.ResponseFailed' }[outcome]; + const summary = { + ...details, + duration_ms: Math.round(performance.now() - this.started), + error_message: this.errorMessage, + first_text_duration_ms: this.firstTextDuration, + message_characters: this.contentCharacters, + outcome, + reason, + received_done: this.receivedDone, + response_characters: this.contentCharacters, + role: 'assistant', + tool_calls: this.toolCalls, + tool_failures: this.toolFailures + }; + trackAssistantEvent(feature, this.context, summary); + if (this.responseContent !== undefined) { + trackAssistantLog('assistant.Response', this.responseContent || this.errorMessage || '', this.context, { + ...summary, + message_characters: this.contentCharacters || this.errorMessage?.length || 0, + message_truncated: this.contentCharacters > maximumMessageCharacters + }); + } + return outcome; + } + + observe(event: AssistantStreamEvent): void { + if (this.finished) { + return; + } + if (event.type === 'text_delta' && event.text) { + this.contentCharacters += event.text.length; + if (this.responseContent !== undefined) { + this.responseContent += event.text.slice(0, Math.max(0, maximumMessageCharacters - this.responseContent.length)); + } + this.firstTextDuration ??= Math.round(performance.now() - this.started); + } else if (event.type === 'tool_call') { + this.toolCalls++; + } else if (event.type === 'tool_result' && assistantToolResultFailed(event.result)) { + this.toolFailures++; + } else if (event.type === 'error') { + this.fail(event.message ?? 'Exie could not complete this request.', 'stream_error'); + } else if (event.type === 'done') { + this.receivedDone = true; + } + } +} + +export function trackAssistantEvent(feature: string, context: AssistantTelemetryContext, details: Record = {}): void { + // A telemetry failure must not interrupt a chat or generate another telemetry event. + void submitFeatureUsage(feature, getProperties(context, details)).catch(() => {}); +} + +function getProperties(context: AssistantTelemetryContext, details: Record) { + return { + exie: { + ...context, + ...details, + schema_version: 1 + } + }; +} + +function trackAssistantLog(source: string, message: string, context: AssistantTelemetryContext, details: Record): void { + void submitLog(source, message.slice(0, maximumMessageCharacters), getProperties(context, details)).catch(() => {}); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte new file mode 100644 index 0000000000..f9a639a9f2 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-conversation-sharing.svelte @@ -0,0 +1,50 @@ + + + + + Chat sharing: {!settings ? 'Loading…' : checked ? 'On' : 'Off'}{error ? ' · Not saved' : ''} + + + + Share conversations to improve Exie + void onChange(value)} /> + +

Allow Exceptionless to review your messages and replies. Usage and error diagnostics stay enabled.

+ {#if settings && !settings.is_overridden} +

Using the default: {settings.default_enabled ? 'on' : 'off'}. You can change this anytime.

+ {/if} + {#if settings?.is_overridden} + + {/if} + {#if error} + + + {/if} + + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte index 25a8111fd1..5713045e9f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte @@ -1,7 +1,6 @@ @@ -66,7 +67,7 @@ {#snippet child({ props })} - {/snippet} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts index 71c594b132..291f7222bd 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts @@ -1,9 +1,7 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -const submitFeatureUsage = vi.hoisted(() => vi.fn(() => Promise.resolve())); const toast = vi.hoisted(() => ({ error: vi.fn(), success: vi.fn() })); -vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage })); vi.mock('svelte-sonner', () => ({ toast })); import AssistantMessageActions from './assistant-message-actions.svelte'; @@ -13,7 +11,6 @@ describe('AssistantMessageActions', () => { beforeEach(() => { writeText.mockClear(); - submitFeatureUsage.mockClear(); toast.error.mockClear(); toast.success.mockClear(); Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }); @@ -21,25 +18,25 @@ describe('AssistantMessageActions', () => { it('copies the complete message and regenerates the response', async () => { const onRegenerate = vi.fn(); - render(AssistantMessageActions, { props: { content: 'The answer', onRegenerate } }); + const onCopy = vi.fn(); + render(AssistantMessageActions, { props: { content: 'The answer', onCopy, onRegenerate } }); await fireEvent.click(screen.getByRole('button', { name: 'Copy message' })); await fireEvent.click(screen.getByRole('button', { name: 'Regenerate response' })); expect(writeText).toHaveBeenCalledWith('The answer'); + await waitFor(() => expect(onCopy).toHaveBeenCalledOnce()); expect(onRegenerate).toHaveBeenCalledOnce(); }); - it('records helpful feedback without including message contents', async () => { + it('reports feedback to the conversation owner for correlated telemetry', async () => { const onFeedback = vi.fn(); render(AssistantMessageActions, { props: { content: 'Sensitive answer', onFeedback, showFeedback: true } }); await fireEvent.click(screen.getByRole('button', { name: 'Good response' })); expect(onFeedback).toHaveBeenCalledWith('helpful'); - await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.ResponseHelpful')); expect(toast.success).toHaveBeenCalledWith('Marked as helpful.'); - expect(submitFeatureUsage).not.toHaveBeenCalledWith(expect.stringContaining('Sensitive answer')); }); it('disables regenerate while the callback is pending', async () => { @@ -54,14 +51,13 @@ describe('AssistantMessageActions', () => { await waitFor(() => expect(screen.getByRole('button', { name: 'Regenerate response' }).hasAttribute('disabled')).toBe(false)); }); - it('reports telemetry failure without reverting the feedback callback', async () => { + it('reports clearing feedback to the conversation owner', async () => { const onFeedback = vi.fn(); - submitFeatureUsage.mockRejectedValueOnce(new Error('offline')); render(AssistantMessageActions, { props: { content: 'The answer', feedback: 'helpful', onFeedback, showFeedback: true } }); - await fireEvent.click(screen.getByRole('button', { name: 'Poor response' })); + await fireEvent.click(screen.getByRole('button', { name: 'Good response' })); - expect(onFeedback).toHaveBeenCalledWith('not-helpful'); - await waitFor(() => expect(toast.error).toHaveBeenCalledWith('Your feedback is selected, but telemetry could not be sent.')); + expect(onFeedback).toHaveBeenCalledWith(undefined); + expect(toast.error).not.toHaveBeenCalled(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte index cca446c1a0..d600140c4b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-message.svelte @@ -16,6 +16,7 @@ isLast?: boolean; isStreaming?: boolean; message: AssistantChatMessage; + onCopy?: () => void; onFeedback?: (feedback: AssistantFeedback | undefined) => void; onRegenerate?: () => Promise | void; onSuggestedAction?: (action: AssistantSuggestedAction) => void; @@ -27,6 +28,7 @@ isLast = false, isStreaming = false, message, + onCopy, onFeedback, onRegenerate, onSuggestedAction, @@ -40,7 +42,7 @@ {#if message.role === 'user'}
{message.content}
- +
{:else}
@@ -95,6 +97,7 @@ Promise | void; onCollapse?: () => void; + onConversationSharingChange?: (enabled: boolean | null) => Promise; onRetryAccess?: () => Promise | void; open?: boolean; organizationId?: string; @@ -44,11 +62,13 @@ accessMessage, accessState = 'available', collapseHref, + conversationSharing, expandHref, minimumPlanId, mode = 'sheet', onAccessChanged, onCollapse, + onConversationSharingChange, onRetryAccess, open = $bindable(false), organizationId, @@ -57,17 +77,27 @@ promptRequest }: Props = $props(); let messages = $state([]); - let conversationId = $state(crypto.randomUUID()); + let conversationId = $state(createConversationId()); let conversationOrganizationId = $state(); let prompt = $state(''); let errorMessage = $state(); let isStreaming = $state(false); let isNearBottom = $state(true); let showToolCalls = $state(false); + let sharingSettings = $state(); + let sharingSuppressed = $state(false); + let sharingError = $state(); + let isSavingSharing = $state(false); + let requestedSharing = $state(null); + const isSharingEnabled = $derived(sharingSettings?.enabled === true && !sharingSuppressed); let showScrollToBottom = $state(false); let conversationElement = $state(); let abortController: AbortController | undefined; let handledPromptRequestId: string | undefined; + let activeTurn: AssistantTurnTelemetry | undefined; + let lastOutcome: AssistantTurnOutcome | undefined; + let wasVisible = false; + let previousMode: 'page' | 'sheet' | undefined; let latestAssistantMessage = $derived(messages.filter((message) => message.role === 'assistant').at(-1)); const suggestions = [ @@ -75,6 +105,35 @@ 'Which open stacks occurred most recently?', 'Explain what I can investigate on this page.' ]; + $effect(() => { + sharingSettings = conversationSharing; + if (conversationSharing?.enabled !== true) { + activeTurn?.disableFullLogging(); + } + }); + + async function changeConversationSharing(enabled: boolean | null) { + if (!onConversationSharingChange || isSavingSharing) { + return; + } + sharingError = undefined; + requestedSharing = enabled; + isSavingSharing = true; + if (enabled !== true) { + sharingSuppressed = true; + activeTurn?.disableFullLogging(); + } + try { + sharingSettings = await onConversationSharingChange(enabled); + sharingSuppressed = !sharingSettings.enabled; + } catch { + sharingError = sharingSuppressed + ? 'Could not save your choice. Sharing is paused on this page. Try again to save it for all devices.' + : 'Could not save your sharing choice. Please try again.'; + } finally { + isSavingSharing = false; + } + } $effect(() => { if (open && conversationElement) { void scrollToLatest('auto', true); @@ -83,20 +142,53 @@ $effect(() => { if (accessState !== 'available') { - untrack(stopStreaming); + untrack(() => stopStreaming('access_changed')); } }); $effect(() => { const currentOrganizationId = organizationId; if (conversationOrganizationId !== currentOrganizationId) { - untrack(stopStreaming); + untrack(() => { + if (messages.length > 0) { + trackConversationEvent('assistant.ConversationLeft', { + reason: 'organization_changed' + }); + } + stopStreaming('organization_changed'); + }); messages = []; errorMessage = undefined; prompt = ''; - conversationId = crypto.randomUUID(); + conversationId = createConversationId(); conversationOrganizationId = currentOrganizationId; + lastOutcome = undefined; + } + }); + + $effect(() => { + const visible = mode === 'page' || open; + const currentMode = mode; + untrack(() => { + if (visible !== wasVisible) { + trackConversationEvent(visible ? 'assistant.Opened' : 'assistant.Closed'); + } else if (visible && currentMode !== previousMode) { + trackConversationEvent('assistant.ViewChanged', { + previous_mode: previousMode + }); + } + wasVisible = visible; + previousMode = currentMode; + }); + }); + + onDestroy(() => { + if (wasVisible) { + trackConversationEvent('assistant.ConversationLeft', { + reason: 'component_unmounted' + }); } + stopStreaming('component_unmounted'); }); $effect(() => { @@ -113,10 +205,12 @@ } handledPromptRequestId = promptRequest.id; - void submitPrompt(promptRequest.prompt); + void submitPrompt(promptRequest.prompt, { + source: 'queued' + }); }); - async function submitPrompt(value = prompt, isSuggestedAction = false, suggestedActionLabel?: string, suggestedActionPath?: string): Promise { + async function submitPrompt(value = prompt, options: { action?: AssistantSuggestedAction; source?: AssistantPromptSource } = {}): Promise { const content = value.trim(); if (!content || isStreaming) { return; @@ -131,32 +225,40 @@ errorMessage = undefined; const userMessage: AssistantChatMessage = { content, + conversationId, id: crypto.randomUUID(), - isSuggestedAction, + isSuggestedAction: options.source === 'suggested_action', role: 'user', - suggestedActionLabel, - suggestedActionPath, + suggestedActionLabel: options.action?.label, + suggestedActionPath: options.action?.sourcePath, tools: [] }; const assistantMessage: AssistantChatMessage = { content: '', + conversationId, id: crypto.randomUUID(), role: 'assistant', tools: [] }; const history = [...messages, userMessage]; messages = [...history, assistantMessage]; - await streamResponse(history, assistantMessage); + await streamResponse(history, assistantMessage, options.source ?? 'composer'); } - async function handleSuggestedAction(action: AssistantSuggestedAction): Promise { + async function handleSuggestedAction(action: AssistantSuggestedAction, message: AssistantChatMessage): Promise { + trackAssistantEvent('assistant.SuggestedActionSelected', getTelemetryContext(message), { + action_type: action.href ? 'navigation' : 'prompt' + }); if (action.href) { open = false; await goto(action.href); return; } - await submitPrompt(action.prompt, true, action.label, action.sourcePath); + await submitPrompt(action.prompt, { + action, + source: 'suggested_action' + }); } async function regenerateResponse(assistantMessageId: string): Promise { @@ -174,61 +276,110 @@ return; } + const source = errorMessage ? 'retry' : 'regenerate'; + const previousConversationId = conversationId; + trackAssistantEvent('assistant.ResponseRegenerated', getTelemetryContext(messages[assistantMessageIndex]), { + prompt_source: source + }); errorMessage = undefined; const history = messages.slice(0, userMessageIndex + 1); + conversationId = createConversationId(); const replacement: AssistantChatMessage = { content: '', + conversationId, id: crypto.randomUUID(), role: 'assistant', tools: [] }; messages = [...history, replacement]; - conversationId = crypto.randomUUID(); - await streamResponse(history, replacement); + await streamResponse(history, replacement, source, { + previous_conversation_id: previousConversationId, + retry_of_message_id: assistantMessageId + }); } - async function streamResponse(history: AssistantChatMessage[], assistantMessage: AssistantChatMessage): Promise { + async function streamResponse( + history: AssistantChatMessage[], + assistantMessage: AssistantChatMessage, + source: AssistantPromptSource, + details: Record = {} + ): Promise { isStreaming = true; - abortController = new AbortController(); - await scrollToLatest('smooth', true); + const controller = new AbortController(); + abortController = controller; const requestPath = path ?? `${page.url.pathname}${page.url.search}`; + const userMessage = history.at(-1)!; + const telemetry = new AssistantTurnTelemetry( + { + ...getTelemetryContext(assistantMessage), + user_message_id: userMessage.id + }, + userMessage.content.length, + source, + { + ...details, + turn_index: history.filter((message) => message.role === 'user').length + } + ); + activeTurn = telemetry; + lastOutcome = undefined; + const request = createAssistantChatRequest(history, conversationId, organizationId, requestPath, projectId); try { + await scrollToLatest('smooth', true); const response = await fetch('/api/v2/assistant/chat', { - body: JSON.stringify(createAssistantChatRequest(history, conversationId, organizationId, requestPath, projectId)), + body: JSON.stringify(request), headers: { Authorization: `Bearer ${accessToken.current}`, 'Content-Type': 'application/json' }, method: 'POST', - signal: abortController.signal + signal: controller.signal }); if (!response.ok) { const problem = response.headers.get('content-type')?.includes('json') ? ((await response.json()) as { detail?: string; title?: string }) : undefined; - throw new Error(problem?.detail ?? problem?.title ?? `The assistant returned status ${response.status}.`); + const message = problem?.detail ?? problem?.title ?? `The assistant returned status ${response.status}.`; + telemetry.fail(message, `http_${response.status}`); + throw new Error(message); } if (!response.body) { throw new Error('The assistant returned an empty response.'); } + if (isSharingEnabled && response.headers.get('X-Exie-Full-Logging') === 'true') { + telemetry.enableFullLogging(userMessage.content); + } + await readAssistantStream(response.body, async (event) => { + if (controller.signal.aborted) { + return; + } + telemetry.observe(event); applyStreamEvent(assistantMessage.id, event, requestPath); await scrollToLatest('auto'); }); } catch (error) { - if (error instanceof DOMException && error.name === 'AbortError') { + if (controller.signal.aborted || (error instanceof DOMException && error.name === 'AbortError')) { return; } errorMessage = error instanceof Error ? error.message : 'Exie could not complete this request.'; + telemetry.fail(errorMessage, 'request_error'); } finally { - isStreaming = false; - abortController = undefined; - await scrollToLatest('auto'); + const outcome = telemetry.finish(controller.signal.aborted ? 'user_stopped' : undefined, { + is_visible: mode === 'page' || open + }); + if (abortController === controller) { + lastOutcome = outcome ?? lastOutcome; + activeTurn = undefined; + isStreaming = false; + abortController = undefined; + await scrollToLatest('auto'); + } } } @@ -300,7 +451,13 @@ } } - function stopStreaming(): void { + function stopStreaming(reason: AssistantStopReason = 'user_stopped'): void { + const outcome = activeTurn?.finish(reason, { + is_visible: mode === 'page' || open + }); + if (outcome) { + lastOutcome = outcome; + } abortController?.abort(); if (!messages.some((message) => message.tools.some((tool) => tool.status === 'running'))) { return; @@ -320,13 +477,15 @@ } function clearConversation(): void { - stopStreaming(); + trackConversationEvent('assistant.ConversationCleared'); + stopStreaming('conversation_cleared'); messages = []; - conversationId = crypto.randomUUID(); + conversationId = createConversationId(); errorMessage = undefined; prompt = ''; isNearBottom = true; showScrollToBottom = false; + lastOutcome = undefined; } function collapseToSidePanel(): void { @@ -344,6 +503,19 @@ } function setMessageFeedback(messageId: string, feedback: AssistantFeedback | undefined): void { + const message = messages.find((message) => message.id === messageId); + if (!message) { + return; + } + const feature = + feedback === 'helpful' + ? 'assistant.ResponseHelpful' + : feedback === 'not-helpful' + ? 'assistant.ResponseNotHelpful' + : 'assistant.ResponseFeedbackCleared'; + trackAssistantEvent(feature, getTelemetryContext(message), { + feedback: feedback ?? 'cleared' + }); messages = messages.map((message) => message.id === messageId ? { @@ -354,6 +526,46 @@ ); } + function getTelemetryContext(message?: AssistantChatMessage): AssistantTelemetryContext { + return { + assistant_message_id: message?.role === 'assistant' ? message.id : undefined, + conversation_id: message?.conversationId ?? conversationId, + mode, + organization_id: conversationOrganizationId ?? organizationId, + path: (path ?? page.url.pathname).split(/[?#]/)[0], + project_id: projectId, + user_message_id: message?.role === 'user' ? message.id : undefined + }; + } + + function createConversationId(): string { + // Match the server's Guid.ToString("N") representation for exact log correlation. + return crypto.randomUUID().replaceAll('-', ''); + } + + function trackConversationEvent(feature: string, details: Record = {}): void { + trackAssistantEvent( + feature, + { + ...(activeTurn?.context ?? getTelemetryContext(latestAssistantMessage)), + mode + }, + { + ...details, + is_streaming: isStreaming, + last_feedback: latestAssistantMessage?.feedback, + last_outcome: lastOutcome, + message_count: messages.length + } + ); + } + + function handlePageHide(): void { + if (wasVisible || messages.length > 0) { + trackConversationEvent('assistant.PageLeft'); + } + } + async function scrollToLatest(behavior: 'auto' | 'smooth' = 'smooth', force = false): Promise { if (!force && !isNearBottom) { showScrollToBottom = true; @@ -370,6 +582,8 @@ } + + {#snippet conversation()}
{#if accessState !== 'available'} @@ -399,7 +613,10 @@ {#each suggestions as suggestion (suggestion)}
{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts index 81a1879ff7..2c016c44ee 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/components/assistant-panel.svelte.test.ts @@ -6,7 +6,12 @@ vi.mock('$features/auth/index.svelte', () => ({ accessToken: { current: 'access- vi.mock('$features/billing/stripe.svelte', () => ({ isStripeEnabled: () => true })); vi.mock('katex/dist/katex.min.css', () => ({})); const goto = vi.hoisted(() => vi.fn(() => Promise.resolve())); +const submitFeatureUsage = vi.hoisted(() => vi.fn<(feature: string, properties?: Record) => Promise>().mockResolvedValue(undefined)); +const submitLog = vi.hoisted(() => + vi.fn<(source: string, message: string, properties?: Record) => Promise>().mockResolvedValue(undefined) +); vi.mock('$app/navigation', () => ({ goto })); +vi.mock('$features/auth/exceptionless-session', () => ({ submitFeatureUsage, submitLog })); import AssistantPanel from './assistant-panel.svelte'; @@ -30,6 +35,307 @@ describe('AssistantPanel', () => { expect(screen.getByText('Bring Exie onto your team')).toBeTruthy(); }); + it('records opens only when becoming visible, not when typing or changing views', async () => { + const props = { open: true, organizationId: 'organization-1', path: '/next/stack' }; + const view = render(AssistantPanel, { props }); + await screen.findByRole('textbox', { name: 'Message Exie' }); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.Opened')).toHaveLength(1); + + await fireEvent.input(screen.getByRole('textbox', { name: 'Message Exie' }), { target: { value: 'Unsent draft' } }); + await view.rerender({ ...props, mode: 'page', path: '/next/event' }); + await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.ViewChanged', expect.anything())); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.Opened')).toHaveLength(1); + + await view.rerender({ ...props, mode: 'sheet', open: false }); + await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.Closed', expect.anything())); + await view.rerender(props); + await waitFor(() => expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.Opened')).toHaveLength(2)); + }); + + it('correlates message outcomes and feedback without recording chat text', async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response('{"type":"text_delta","text":"The answer"}\n{"type":"done"}\n')); + vi.stubGlobal('fetch', fetchMock); + render(AssistantPanel, { + props: { + open: true, + organizationId: 'organization-1', + path: '/next/stack/stack-1?filter=private', + promptRequest: { id: 'prompt-request', prompt: 'My question' } + } + }); + await screen.findByText('The answer'); + await screen.findByRole('button', { name: 'Good response' }); + await fireEvent.click(screen.getByRole('button', { name: 'Good response' })); + await waitFor(() => expect(submitFeatureUsage).toHaveBeenCalledWith('assistant.ResponseHelpful', expect.anything())); + + const prompt = eventData('assistant.MessageSent'); + expect(prompt.conversation_id).toMatch(/^[0-9a-f]{32}$/); + expect(JSON.parse(fetchMock.mock.calls[0]?.[1]?.body as string).conversation_id).toBe(prompt.conversation_id); + expect(prompt).toMatchObject({ organization_id: 'organization-1', path: '/next/stack/stack-1', prompt_source: 'queued', role: 'user' }); + expect(eventData('assistant.ResponseCompleted')).toMatchObject({ + assistant_message_id: prompt.assistant_message_id, + conversation_id: prompt.conversation_id, + is_visible: true, + outcome: 'completed' + }); + expect(eventData('assistant.ResponseHelpful')).toMatchObject({ + assistant_message_id: prompt.assistant_message_id, + conversation_id: prompt.conversation_id + }); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseHelpful')).toHaveLength(1); + const telemetry = JSON.stringify([...submitFeatureUsage.mock.calls, ...submitLog.mock.calls]); + expect(telemetry).not.toContain('My question'); + expect(telemetry).not.toContain('The answer'); + }); + + it('links a retry to the failed response across the new server conversation', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response('{"type":"error","message":"Provider timed out"}\n{"type":"done"}\n')) + .mockResolvedValueOnce(new Response('{"type":"text_delta","text":"Recovered answer"}\n{"type":"done"}\n')); + vi.stubGlobal('fetch', fetchMock); + render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } } + }); + await screen.findByText('Provider timed out'); + await fireEvent.click(await screen.findByRole('button', { name: 'Retry' })); + await screen.findByText('Recovered answer'); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); + + const failed = eventData('assistant.ResponseFailed'); + const retried = eventData('assistant.MessageSent', 1); + expect(retried).toMatchObject({ + previous_conversation_id: failed.conversation_id, + prompt_source: 'retry', + retry_of_message_id: failed.assistant_message_id + }); + expect(retried.conversation_id).not.toBe(failed.conversation_id); + expect(retried.conversation_id).toMatch(/^[0-9a-f]{32}$/); + expect(JSON.parse(fetchMock.mock.calls[1]?.[1]?.body as string).conversation_id).toBe(retried.conversation_id); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseFailed')).toHaveLength(1); + expect(failed.error_message).toBe('Provider timed out'); + }); + + it.each([ + [undefined, true], + ['false', true], + ['true', true], + ['true', false], + ['true', undefined] + ] as const)('requires both server permission (%s) and the visible sharing choice (%s)', async (flag, enabled) => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response('{"type":"text_delta","text":"Full answer"}\n{"type":"done"}\n', { + headers: flag === undefined ? {} : { 'X-Exie-Full-Logging': flag } + }) + ) + ); + render(AssistantPanel, { + props: { + conversationSharing: enabled === undefined ? undefined : { default_enabled: true, enabled, is_overridden: !enabled }, + open: true, + organizationId: 'organization-1', + promptRequest: { id: 'request-1', prompt: 'Full question' } + } + }); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); + if (flag === 'true' && enabled) { + expect(submitLog).toHaveBeenCalledWith('assistant.Prompt', 'Full question', expect.anything()); + expect(submitLog).toHaveBeenCalledWith('assistant.Response', 'Full answer', expect.anything()); + expect(submitLog).toHaveBeenCalledTimes(2); + } else { + expect(submitLog).not.toHaveBeenCalled(); + } + }); + + it('stops recording transcript text when full logging is disabled for the next turn', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response('{"type":"text_delta","text":"First answer"}\n{"type":"done"}\n', { headers: { 'X-Exie-Full-Logging': 'true' } }) + ) + .mockResolvedValueOnce( + new Response('{"type":"text_delta","text":"Second answer"}\n{"type":"done"}\n', { headers: { 'X-Exie-Full-Logging': 'false' } }) + ); + vi.stubGlobal('fetch', fetchMock); + render(AssistantPanel, { + props: { + conversationSharing: { default_enabled: true, enabled: true, is_overridden: false }, + open: true, + organizationId: 'organization-1', + promptRequest: { id: 'request-1', prompt: 'First question' } + } + }); + await screen.findByText('First answer'); + const composer = screen.getByRole('textbox', { name: 'Message Exie' }); + await screen.findByRole('button', { name: 'Send message' }); + await fireEvent.input(composer, { target: { value: 'Second question' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Send message' })); + await waitFor(() => expect(eventData('assistant.ResponseCompleted', 1).outcome).toBe('completed')); + expect(submitLog).toHaveBeenCalledTimes(2); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Second'); + }); + + it('shows the inherited default and saves an explicit choice or a reset', async () => { + const save = vi + .fn() + .mockResolvedValueOnce({ default_enabled: false, enabled: true, is_overridden: true }) + .mockResolvedValueOnce({ default_enabled: false, enabled: false, is_overridden: false }); + render(AssistantPanel, { + props: { + conversationSharing: { default_enabled: false, enabled: false, is_overridden: false }, + onConversationSharingChange: save, + open: true, + organizationId: 'organization-1' + } + }); + expect(screen.queryByRole('switch', { name: 'Share conversations to improve Exie' })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off' })); + expect(screen.getByText(/Using the default: off/)).toBeTruthy(); + await fireEvent.click(screen.getByRole('switch', { name: 'Share conversations to improve Exie' })); + await waitFor(() => expect(save).toHaveBeenCalledWith(true)); + await fireEvent.click(await screen.findByRole('button', { name: 'Use default (off)' })); + await waitFor(() => expect(save).toHaveBeenLastCalledWith(null)); + await waitFor(() => expect(screen.getByRole('switch', { name: 'Share conversations to improve Exie' }).getAttribute('aria-checked')).toBe('false')); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off' })); + await waitFor(() => expect(screen.queryByRole('switch', { name: 'Share conversations to improve Exie' })).toBeNull()); + }); + + it.each([false, true])('stops collecting the active reply and subsequent prompts when opting out (save fails: %s)', async (saveFails) => { + let controller!: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(value) { + controller = value; + } + }); + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce(new Response(stream, { headers: { 'X-Exie-Full-Logging': 'true' } })) + .mockResolvedValueOnce( + new Response('{"type":"text_delta","text":"Later answer"}\n{"type":"done"}\n', { headers: { 'X-Exie-Full-Logging': 'true' } }) + ) + ); + const saved = { default_enabled: true, enabled: false, is_overridden: true }; + const save = saveFails ? vi.fn().mockRejectedValueOnce(new Error('Offline')).mockResolvedValue(saved) : vi.fn().mockResolvedValue(saved); + render(AssistantPanel, { + props: { + conversationSharing: { default_enabled: true, enabled: true, is_overridden: false }, + onConversationSharingChange: save, + open: true, + organizationId: 'organization-1', + promptRequest: { id: 'sharing-request', prompt: 'Initial question' } + } + }); + await waitFor(() => expect(submitLog).toHaveBeenCalledWith('assistant.Prompt', 'Initial question', expect.anything())); + controller.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Active reply"}\n')); + await screen.findByText('Active reply'); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: On' })); + await fireEvent.click(screen.getByRole('switch', { name: 'Share conversations to improve Exie' })); + await waitFor(() => expect(save).toHaveBeenCalledWith(false)); + if (saveFails) { + await screen.findByText(/Sharing is paused on this page/); + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off · Not saved' })); + } + controller.enqueue(new TextEncoder().encode('{"type":"done"}\n')); + controller.close(); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').outcome).toBe('completed')); + await fireEvent.input(screen.getByRole('textbox', { name: 'Message Exie' }), { target: { value: 'Later question' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Send message' })); + await waitFor(() => expect(eventData('assistant.ResponseCompleted', 1).outcome).toBe('completed')); + expect(submitLog).toHaveBeenCalledTimes(1); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Active reply'); + expect(JSON.stringify(submitLog.mock.calls)).not.toContain('Later'); + if (saveFails) { + await fireEvent.click(screen.getByRole('button', { name: 'Chat sharing: Off · Not saved' })); + await fireEvent.click(screen.getByRole('button', { name: 'Retry saving' })); + await waitFor(() => expect(save).toHaveBeenCalledTimes(2)); + expect(save).toHaveBeenLastCalledWith(false); + } + }); + + it('records closing while waiting without cancelling a response that finishes in the background', async () => { + let streamController: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(controller) { + streamController = controller; + } + }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(stream)) + ); + const props = { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } }; + const view = render(AssistantPanel, { props }); + await screen.findByRole('button', { name: 'Stop generating' }); + await view.rerender({ ...props, open: false }); + await waitFor(() => expect(eventData('assistant.Closed').is_streaming).toBe(true)); + streamController!.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Background answer"}\n{"type":"done"}\n')); + streamController!.close(); + await waitFor(() => expect(eventData('assistant.ResponseCompleted').is_visible).toBe(false)); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); + }); + + it('records one cancellation with the original organization when organization context changes', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async (_input: RequestInfo | URL, init?: RequestInit) => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Partial answer"}\n')); + init?.signal?.addEventListener('abort', () => controller.error(new DOMException('Aborted', 'AbortError'))); + } + }) + ) + ) + ); + const view = render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } } + }); + await screen.findByText('Partial answer'); + await view.rerender({ open: true, organizationId: 'organization-2' }); + await waitFor(() => expect(eventData('assistant.ResponseCancelled').reason).toBe('organization_changed')); + expect(eventData('assistant.ResponseCancelled')).toMatchObject({ organization_id: 'organization-1' }); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.ResponseCompleted')).toBe(false); + expect(screen.queryByText('Partial answer')).toBeNull(); + }); + + it('records an explicit stop once and distinguishes it from leaving the page', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async (_input: RequestInfo | URL, init?: RequestInit) => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"type":"text_delta","text":"Partial answer"}\n')); + init?.signal?.addEventListener('abort', () => controller.error(new DOMException('Aborted', 'AbortError'))); + } + }) + ) + ) + ); + render(AssistantPanel, { + props: { open: true, organizationId: 'organization-1', promptRequest: { id: 'prompt-request', prompt: 'My question' } } + }); + await screen.findByText('Partial answer'); + await fireEvent(window, new Event('pagehide')); + expect(eventData('assistant.PageLeft')).toMatchObject({ is_streaming: true }); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.ResponseCancelled')).toBe(false); + + await fireEvent.click(screen.getByRole('button', { name: 'Stop generating' })); + await screen.findByRole('button', { name: 'Send message' }); + expect(eventData('assistant.ResponseCancelled')).toMatchObject({ outcome: 'cancelled', reason: 'user_stopped' }); + expect(submitFeatureUsage.mock.calls.filter(([feature]) => feature === 'assistant.ResponseCancelled')).toHaveLength(1); + expect(screen.getByText('Partial answer')).toBeTruthy(); + }); + it('renders as a full-page chat and opens the side panel when collapsed', async () => { const onCollapse = vi.fn(); render(AssistantPanel, { @@ -83,6 +389,7 @@ describe('AssistantPanel', () => { expect(screen.getByText('The conversation is still here.')).toBeTruthy(); expect(screen.getByRole('button', { name: 'Clear conversation' }).hasAttribute('disabled')).toBe(false); expect(fetchMock).toHaveBeenCalledOnce(); + expect(submitFeatureUsage.mock.calls.some(([feature]) => feature === 'assistant.Closed')).toBe(false); }); afterEach(() => { @@ -234,5 +541,15 @@ describe('AssistantPanel', () => { await waitFor(() => expect(goto).toHaveBeenCalledWith(configureHref)); expect(fetchMock).toHaveBeenCalledOnce(); + expect(eventData('assistant.SuggestedActionSelected')).toMatchObject({ action_type: 'navigation' }); + const telemetry = JSON.stringify(submitFeatureUsage.mock.calls); + expect(telemetry).not.toContain('Open Client Setup'); + expect(telemetry).not.toContain('How do I configure'); + expect(telemetry).not.toContain(configureHref); }); }); + +function eventData(feature: string, index = 0): Record { + const properties = submitFeatureUsage.mock.calls.filter(([name]) => name === feature)[index]?.[1]; + return (properties?.exie ?? {}) as Record; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts index 856a88ef15..52606bbc2e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/assistant/models.ts @@ -1,3 +1,5 @@ +export type { AssistantConversationSharingSettings } from '$generated/api'; + export interface AssistantAccess { enabled: boolean; has_access: boolean; @@ -10,6 +12,7 @@ export type AssistantAccessState = 'available' | 'disabled' | 'error' | 'loading export interface AssistantChatMessage { content: string; + conversationId?: string; feedback?: AssistantFeedback; id: string; isSuggestedAction?: boolean; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts new file mode 100644 index 0000000000..755ff87ae7 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.test.ts @@ -0,0 +1,123 @@ +import { Exceptionless } from '@exceptionless/browser'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('$app/environment', () => ({ browser: true })); +vi.mock('@exceptionless/browser', async () => { + const { Configuration, ExceptionlessClient } = await import('@exceptionless/core'); + const config = new Configuration(); + config.apiKey = 'local-test-key'; + config.serverUrl = 'https://localhost'; + config.defaultTags.push('UI', 'Svelte'); + config.updateSettingsWhenIdleInterval = 0; + // Exercise real event builders and plugins, without starting timers or sending events. + config.services.queue.enqueue = vi.fn().mockResolvedValue(undefined); + config.services.queue.startup = vi.fn().mockResolvedValue(undefined); + config.services.queue.process = vi.fn().mockResolvedValue(undefined); + return { Exceptionless: new ExceptionlessClient(config) }; +}); + +import { configureSessions, endSession, setUserIdentity, submitFeatureUsage, submitLog } from './exceptionless-session'; + +describe('Exceptionless session events', () => { + beforeEach(async () => { + vi.useFakeTimers(); + vi.spyOn(Exceptionless, 'submitSessionEnd').mockResolvedValue(undefined); + await endSession(); + configureSessions(Exceptionless.config); + vi.mocked(Exceptionless.config.services.queue.enqueue).mockClear(); + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('skips anonymous startup and resume sessions while retaining ordinary diagnostics', async () => { + await Exceptionless.startup(); + await Exceptionless.startup(); + await submitLog('api-failure', 'HTTP 500'); + await submitFeatureUsage('login'); + + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events.map((event) => event.type)).toEqual(['log', 'usage']); + }); + + it('starts an identified session once when the user loads and keeps identity on resume', async () => { + await setUserIdentity('session-user', 'Session User'); + await setUserIdentity('session-user', 'Updated Name'); + expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledOnce(); + + await Exceptionless.startup(); + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events).toHaveLength(2); + expect(events.every((event) => event.type === 'session' && event.data?.['@user']?.identity === 'session-user')).toBe(true); + expect(Exceptionless.config.currentSessionIdentifier).toBe('session-user'); + }); + + it('does not clear a newer identity when an earlier logout finishes', async () => { + await setUserIdentity('previous-user'); + let finishSessionEnd: () => void = () => {}; + vi.mocked(Exceptionless.submitSessionEnd).mockImplementationOnce( + () => + new Promise((resolve) => { + finishSessionEnd = resolve; + }) + ); + const ending = endSession(); + await vi.waitFor(() => expect(Exceptionless.submitSessionEnd).toHaveBeenCalledWith('previous-user')); + await setUserIdentity('next-user'); + finishSessionEnd(); + await ending; + + expect(Exceptionless.config.defaultData['@user']).toMatchObject({ identity: 'next-user' }); + expect(Exceptionless.config.currentSessionIdentifier).toBe('next-user'); + }); + + it('clears the heartbeat identity on logout and suppresses anonymous resume sessions', async () => { + await setUserIdentity('logging-out-user'); + await endSession(); + vi.mocked(Exceptionless.config.services.queue.enqueue).mockClear(); + await Exceptionless.startup(); + + expect(Exceptionless.config.currentSessionIdentifier).toBeNull(); + expect(Exceptionless.config.services.queue.enqueue).not.toHaveBeenCalled(); + }); + + it('keeps usage metadata and feedback attached to the existing user session', async () => { + await setUserIdentity('exie-test-user'); + const properties = { exie: { conversation_id: 'conversation-1', role: 'user' } }; + await submitFeatureUsage('assistant.MessageSent', properties); + await submitFeatureUsage('assistant.ResponseHelpful', properties); + + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events).toHaveLength(3); + expect(events[0]).toMatchObject({ type: 'session' }); + expect(events[1]).toMatchObject({ + data: properties, + source: 'assistant.MessageSent', + type: 'usage' + }); + expect(events[2]).toMatchObject({ data: properties, source: 'assistant.ResponseHelpful', type: 'usage' }); + for (const event of events) { + expect(event.message).toBeUndefined(); + expect(event.data?.['@user']).toMatchObject({ identity: 'exie-test-user' }); + expect(event.tags).toEqual(expect.arrayContaining(['UI', 'Svelte'])); + } + }); + + it('preserves ordinary feature usage submissions without extended data', async () => { + await submitFeatureUsage('project.Created'); + expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledOnce(); + expect(Exceptionless.config.services.queue.enqueue).toHaveBeenCalledWith(expect.objectContaining({ source: 'project.Created', type: 'usage' })); + }); + + it('attaches explicitly enabled transcript logs to the existing session', async () => { + await setUserIdentity('exie-test-user'); + const properties = { exie: { conversation_id: 'conversation-1', role: 'user' } }; + await submitLog('assistant.Prompt', 'Why did checkout fail?', properties); + const events = vi.mocked(Exceptionless.config.services.queue.enqueue).mock.calls.map(([event]) => event); + expect(events.at(-1)).toMatchObject({ data: properties, message: 'Why did checkout fail?', source: 'assistant.Prompt', type: 'log' }); + expect(events.at(-1)?.data?.['@user']).toMatchObject({ identity: 'exie-test-user' }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts index 08ee51f5dd..d4a744ab8d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts @@ -1,12 +1,25 @@ +import type { Configuration } from '@exceptionless/browser'; + import { browser } from '$app/environment'; let _activeUserId: null | string = null; +/** Keep SDK startup and resume events from creating sessions before authentication resolves. */ +export function configureSessions(config: Configuration): void { + config.useSessions(); + config.addPlugin('authenticated-sessions', 20, async (context) => { + if (context.event.type === 'session' && !context.event.data?.['@user']?.identity) { + context.cancelled = true; + } + }); +} + /** * Ends the current Exceptionless session and clears user identity. - * Call on logout. Clears local state unconditionally even if submitSessionEnd fails. + * Call on logout. A delayed session end must not clear a newer user's identity. */ export async function endSession(): Promise { + const endingUserId = _activeUserId; const Exceptionless = await getExceptionless(); if (!Exceptionless) { _activeUserId = null; @@ -14,16 +27,21 @@ export async function endSession(): Promise { } try { - await Exceptionless.submitSessionEnd(); + if (endingUserId) { + await Exceptionless.submitSessionEnd(endingUserId); + } } finally { - Exceptionless.config.setUserIdentity('', ''); - _activeUserId = null; + if (_activeUserId === endingUserId) { + Exceptionless.config.setUserIdentity('', ''); + Exceptionless.config.currentSessionIdentifier = null; + _activeUserId = null; + } } } /** * Sets the current user identity for Exceptionless error tracking. - * Starts a new session only when the identity changes (guards against repeated onSuccess calls from query refetches). + * Starts a new session only when the identity changes, including across profile refetches. */ export async function setUserIdentity(userId: string, userName?: string): Promise { if (!userId) { @@ -35,15 +53,14 @@ export async function setUserIdentity(userId: string, userName?: string): Promis return; } - if (userName) { - Exceptionless.config.setUserIdentity(userId, userName); - } else { - Exceptionless.config.setUserIdentity(userId); - } + Exceptionless.config.setUserIdentity(userId, userName ?? ''); + Exceptionless.config.currentSessionIdentifier = userId; if (_activeUserId !== userId) { _activeUserId = userId; - await Exceptionless.submitSessionStart(); + await Exceptionless.createSessionStart() + .setUserIdentity(userId, userName ?? '') + .submit(); } } @@ -51,13 +68,31 @@ export async function setUserIdentity(userId: string, userName?: string): Promis * Submits a feature usage event for telemetry tracking. * Mirrors the legacy Angular $ExceptionlessClient.submitFeatureUsage pattern. */ -export async function submitFeatureUsage(feature: string): Promise { +export async function submitFeatureUsage(feature: string, properties?: Record): Promise { + const Exceptionless = await getExceptionless(); + if (!Exceptionless) { + return; + } + + const event = Exceptionless.createFeatureUsage(feature); + for (const [name, value] of Object.entries(properties ?? {})) { + event.setProperty(name, value); + } + await event.submit(); +} + +/** Submits a log entry through the existing session, identity, and client settings. */ +export async function submitLog(source: string, message: string, properties?: Record): Promise { const Exceptionless = await getExceptionless(); if (!Exceptionless) { return; } - await Exceptionless.submitFeatureUsage(feature); + const event = Exceptionless.createLog(source, message); + for (const [name, value] of Object.entries(properties ?? {})) { + event.setProperty(name, value); + } + await event.submit(); } async function getExceptionless() { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte index 318e11ca9a..8e1a35e39b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/event-session-summary.svelte @@ -27,11 +27,9 @@ {/if} - {#if source.data.Name || source.data.Identity || source.data.SessionId} - {source.data.Name || source.data.Identity || source.data.SessionId} - {#if source.data.Name && source.data.Identity} - ({source.data.Identity}) - {/if} + {source.data.Name || source.data.Identity || source.data.SessionId || 'Anonymous session'} + {#if source.data.Name && source.data.Identity} + ({source.data.Identity}) {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts index ea86d1aa14..5054a68281 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/summary.svelte.test.ts @@ -7,6 +7,21 @@ import type { EventSummaryModel, StackSummaryModel } from './index'; import Summary from './summary.svelte'; describe('Summary', () => { + it('keeps sessions without identity or session metadata readable and clickable', () => { + const summary: EventSummaryModel<'event-session-summary'> = { + data: { Type: 'session' }, + date: '2026-09-09T00:00:00Z', + id: 'anonymous-session-event', + project_id: 'project-id', + tags: [], + template_key: 'event-session-summary' + }; + + render(Summary, { showStatus: false, showType: false, summary }); + + expect(screen.getByRole('link', { name: 'Anonymous session' }).getAttribute('href')).toBe('/next/event/anonymous-session-event'); + }); + it('links an event summary to that event details page', () => { const summary: EventSummaryModel<'event-error-summary'> = { data: { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts index ff97db0e6d..78b76c1301 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts @@ -1,7 +1,6 @@ import type { WebSocketMessageValue } from '$features/websockets/models'; import type { WorkInProgressResult } from '$shared/models'; -import { setUserIdentity } from '$features/auth/exceptionless-session'; import { accessToken } from '$features/auth/index.svelte'; import { fetchApiJson } from '$features/shared/api/api.svelte'; import { type FetchClientResponse, ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; @@ -135,9 +134,8 @@ export function getMeQuery() { return createQuery(() => ({ enabled: () => !!accessToken.current, - onSuccess: async (data: ViewCurrentUser) => { + onSuccess: (data: ViewCurrentUser) => { queryClient.setQueryData(queryKeys.id(data.id!), data); - await setUserIdentity(data.id, data.full_name); }, queryClient, queryFn: async ({ signal }: { signal: AbortSignal }) => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts index 8db136b95e..12132452d9 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts @@ -97,6 +97,12 @@ export interface AssistantChatRequest { conversation_id?: null | string; } +export interface AssistantConversationSharingSettings { + enabled: boolean; + default_enabled: boolean; + is_overridden: boolean; +} + export interface AssistantModelSettings { model: string; configured_model: string; @@ -105,6 +111,7 @@ export interface AssistantModelSettings { configured_enabled: boolean; is_enabled_overridden: boolean; is_configured: boolean; + conversation_sharing_default_enabled: boolean; } export interface BillingPlan { @@ -625,6 +632,14 @@ export interface TokenResult { token: string; } +export interface UpdateAssistantConversationSharing { + enabled?: null | boolean; +} + +export interface UpdateAssistantConversationSharingSettings { + enabled: boolean; +} + export interface UpdateAssistantEnabledSettings { enabled?: null | boolean; } @@ -743,6 +758,7 @@ export interface User { email_address: string; avatar_file_name?: null | string; email_notifications_enabled: boolean; + assistant_conversation_sharing_enabled?: null | boolean; is_email_address_verified: boolean; verify_email_address_token?: null | string; /** @format date-time */ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index ec58c99678..b3ec21926f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -123,6 +123,15 @@ export type AssistantChatRequestFormData = Infer< typeof AssistantChatRequestSchema >; +export const AssistantConversationSharingSettingsSchema = object({ + enabled: boolean(), + default_enabled: boolean(), + is_overridden: boolean(), +}); +export type AssistantConversationSharingSettingsFormData = Infer< + typeof AssistantConversationSharingSettingsSchema +>; + export const AssistantModelSettingsSchema = object({ model: string().min(1, "Model is required"), configured_model: string().min(1, "Configured model is required"), @@ -131,6 +140,7 @@ export const AssistantModelSettingsSchema = object({ configured_enabled: boolean(), is_enabled_overridden: boolean(), is_configured: boolean(), + conversation_sharing_default_enabled: boolean(), }); export type AssistantModelSettingsFormData = Infer< typeof AssistantModelSettingsSchema @@ -730,6 +740,20 @@ export const TokenResultSchema = object({ }); export type TokenResultFormData = Infer; +export const UpdateAssistantConversationSharingSchema = object({ + enabled: boolean().nullable(), +}); +export type UpdateAssistantConversationSharingFormData = Infer< + typeof UpdateAssistantConversationSharingSchema +>; + +export const UpdateAssistantConversationSharingSettingsSchema = object({ + enabled: boolean(), +}); +export type UpdateAssistantConversationSharingSettingsFormData = Infer< + typeof UpdateAssistantConversationSharingSettingsSchema +>; + export const UpdateAssistantEnabledSettingsSchema = object({ enabled: boolean().nullable().optional(), }); @@ -870,6 +894,7 @@ export const UserSchema = object({ .nullable() .optional(), email_notifications_enabled: boolean(), + assistant_conversation_sharing_enabled: boolean().nullable().optional(), is_email_address_verified: boolean(), verify_email_address_token: string() .min(1, "Verify email address token is required") diff --git a/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte b/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte index e0a8ff09ef..7ab0af27fe 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/telemetry/Telemetry.svelte @@ -1,6 +1,7 @@ diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index e6c4e7f16b..59beb6f6e7 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -9,7 +9,12 @@ import { useSidebar } from '$comp/ui/sidebar'; import { env } from '$env/dynamic/public'; import { resolveAssistantAccessState } from '$features/assistant/access-state'; - import { getAssistantAccessQuery, invalidateAssistantAccessQueries } from '$features/assistant/api.svelte'; + import { + getAssistantAccessQuery, + getAssistantConversationSharingQuery, + invalidateAssistantAccessQueries, + putAssistantConversationSharingMutation + } from '$features/assistant/api.svelte'; import { setAssistantControls } from '$features/assistant/controls.svelte'; import { assistantPageContext, type AssistantResourceContext } from '$features/assistant/page-context.svelte'; import { getIntercomTokenQuery } from '$features/auth/api.svelte'; @@ -281,6 +286,12 @@ ) ); let isAssistantEnabled = $derived(assistantAccessState !== 'disabled'); + const assistantConversationSharingQuery = getAssistantConversationSharingQuery({ + get enabled() { + return assistantAccessState === 'available' && (isAssistantOpen || isAssistantPage); + } + }); + const updateAssistantConversationSharing = putAssistantConversationSharingMutation(); setAssistantControls({ ask: (prompt) => void askAssistant(prompt), @@ -778,6 +789,11 @@ accessMessage={assistantAccess?.message} accessState={assistantAccessState} collapseHref={isAssistantPage ? assistantReturnHref : undefined} + conversationSharing={assistantConversationSharingQuery.data} + onConversationSharingChange={(enabled) => + updateAssistantConversationSharing.mutateAsync({ + enabled + })} expandHref={!isAssistantPage ? assistantExpandHref : undefined} bind:open={isAssistantOpen} minimumPlanId={assistantAccess?.minimum_plan_id} diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte index 861fdf04b5..5635abcb88 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/system/exie/+page.svelte @@ -10,6 +10,7 @@ import { Input } from '$comp/ui/input'; import { Skeleton } from '$comp/ui/skeleton'; import * as Table from '$comp/ui/table'; + import * as Tooltip from '$comp/ui/tooltip'; import { getAdminAssistantUsageQuery } from '$features/admin/api.svelte'; import { getBlockedCount, getTotalTokens, getUsageRisk, getUtcMonthKey, type UsageRisk } from '$features/admin/assistant-usage'; import Bot from '@lucide/svelte/icons/bot'; @@ -204,13 +205,40 @@ {/if} - - / - 0 ? 'text-destructive' : 'text-muted-foreground'} - > - / - completed / failed / cancelled + + + completed turns + + + Completed: Turns that finished without a reported error. This does not indicate whether the user found the + answer helpful. + + + + + 0 ? 'text-destructive' : 'text-muted-foreground' + ]} + > + failed turns + + Failed: Turns that ended with an error or timed out. + + + + + cancelled turns + + + Cancelled: Turns stopped by the user or interrupted by a browser disconnect. + + {#if blockedCount > 0} diff --git a/src/Exceptionless.Web/Models/Admin/UpdateAssistantConversationSharingSettings.cs b/src/Exceptionless.Web/Models/Admin/UpdateAssistantConversationSharingSettings.cs new file mode 100644 index 0000000000..3cc8cf73eb --- /dev/null +++ b/src/Exceptionless.Web/Models/Admin/UpdateAssistantConversationSharingSettings.cs @@ -0,0 +1,6 @@ +namespace Exceptionless.Web.Models.Admin; + +public sealed record UpdateAssistantConversationSharingSettings +{ + public bool Enabled { get; init; } +} diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index 5522853172..f5de1088da 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -193,6 +193,7 @@ public static async Task Main(string[] args) builder.Services.AddHttpClient(nameof(AssistantService), client => client.Timeout = TimeSpan.FromMinutes(2)); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); diff --git a/src/Exceptionless.Web/appsettings.yml b/src/Exceptionless.Web/appsettings.yml index 785c089916..1273009d3b 100644 --- a/src/Exceptionless.Web/appsettings.yml +++ b/src/Exceptionless.Web/appsettings.yml @@ -9,6 +9,7 @@ Serilog: #Exceptionless.Core.Repositories.StackRepository: Verbose #Exceptionless.Core.Repositories: Verbose Exceptionless.Web.Program: Information + Exceptionless.Web.Assistant: Information Exceptionless.Web.Security.ApiKeyAuthenticationHandler: Warning Foundatio.Metrics: Warning Foundatio.Utility.ScheduledTimer: Warning diff --git a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json index 01cc5636a7..ae89ccb00b 100644 --- a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json +++ b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json @@ -287,6 +287,20 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "PUT", + "route": "/api/v2/admin/assistant-settings/conversation-sharing", + "displayName": "HTTP: PUT api/v2/admin/assistant-settings/conversation-sharing", + "tags": [ + "AdminEndpoints" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "GlobalAdminPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "PUT", "route": "/api/v2/admin/assistant-settings/enabled", @@ -625,6 +639,30 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "GET", + "route": "/api/v2/assistant/conversation-sharing", + "displayName": "HTTP: GET api/v2/assistant/conversation-sharing =\u003E GetConversationSharingAsync", + "tags": [], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, + { + "method": "PUT", + "route": "/api/v2/assistant/conversation-sharing", + "displayName": "HTTP: PUT api/v2/assistant/conversation-sharing =\u003E SetConversationSharingAsync", + "tags": [], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "POST", "route": "/api/v2/auth/cancel-reset-password/{token:minlength(1)}", diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index 6e88d09679..d329226969 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -20,6 +20,70 @@ } ], "paths": { + "/api/v2/assistant/conversation-sharing": { + "get": { + "tags": [ + "AssistantEndpoints" + ], + "operationId": "GetAssistantConversationSharing", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssistantConversationSharingSettings" + } + } + } + }, + "401": { + "description": "Unauthorized" + }, + "404": { + "description": "Not Found" + } + } + }, + "put": { + "tags": [ + "AssistantEndpoints" + ], + "description": "Saves the current user\u0027s choice. Null follows the admin default; true and false remain explicit choices when the default changes.", + "operationId": "SetAssistantConversationSharing", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateAssistantConversationSharing" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssistantConversationSharingSettings" + } + } + } + }, + "400": { + "description": "Bad Request" + }, + "401": { + "description": "Unauthorized" + }, + "404": { + "description": "Not Found" + } + } + } + }, "/api/v2/assistant/access": { "get": { "tags": [ @@ -70,6 +134,7 @@ "tags": [ "AssistantEndpoints" ], + "description": "The X-Exie-Full-Logging response header indicates whether the browser should record prompts and responses for this turn. Missing or false disables conversation logging.", "operationId": "StreamAssistantChat", "requestBody": { "content": { @@ -299,6 +364,57 @@ } } }, + "/api/v2/admin/assistant-settings/conversation-sharing": { + "put": { + "tags": [ + "AdminEndpoints" + ], + "summary": "Update Exie full conversation logging", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateAssistantConversationSharingSettings" + } + }, + "application/*\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/UpdateAssistantConversationSharingSettings" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssistantModelSettings" + } + } + } + }, + "401": { + "description": "Unauthorized" + }, + "403": { + "description": "Forbidden" + }, + "422": { + "description": "Unprocessable Entity", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, "/api/v2/admin/event-submission-settings": { "get": { "tags": [ @@ -11799,6 +11915,25 @@ } } }, + "AssistantConversationSharingSettings": { + "required": [ + "enabled", + "default_enabled", + "is_overridden" + ], + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "default_enabled": { + "type": "boolean" + }, + "is_overridden": { + "type": "boolean" + } + } + }, "AssistantModelSettings": { "required": [ "model", @@ -11807,7 +11942,8 @@ "enabled", "configured_enabled", "is_enabled_overridden", - "is_configured" + "is_configured", + "conversation_sharing_default_enabled" ], "type": "object", "properties": { @@ -11831,6 +11967,10 @@ }, "is_configured": { "type": "boolean" + }, + "conversation_sharing_default_enabled": { + "type": "boolean", + "default": false } } }, @@ -13573,6 +13713,31 @@ } } }, + "UpdateAssistantConversationSharing": { + "required": [ + "enabled" + ], + "type": "object", + "properties": { + "enabled": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "UpdateAssistantConversationSharingSettings": { + "required": [ + "enabled" + ], + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, "UpdateAssistantEnabledSettings": { "type": "object", "properties": { @@ -13950,6 +14115,12 @@ "email_notifications_enabled": { "type": "boolean" }, + "assistant_conversation_sharing_enabled": { + "type": [ + "null", + "boolean" + ] + }, "is_email_address_verified": { "type": "boolean" }, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs index 73fc81999a..6598740024 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/AdminEndpointTests.cs @@ -228,6 +228,42 @@ await SendRequestAsAsync(request => request Assert.False(cleared.IsEnabledOverridden); } + [Fact] + public async Task AssistantConversationSharingSettingsAsync_AsGlobalAdmin_PersistsBothStatesWithoutChangingOtherSettings() + { + var initial = await SendRequestAsAsync(request => request + .AsGlobalAdminUser().AppendPaths("admin", "assistant-settings").StatusCodeShouldBeOk()); + Assert.NotNull(initial); + Assert.False(initial.ConversationSharingDefaultEnabled); + + foreach (bool enabled in new[] { true, false }) + { + var updated = await SendRequestAsAsync(request => request + .Put().AsGlobalAdminUser().AppendPaths("admin", "assistant-settings", "conversation-sharing") + .Content(new UpdateAssistantConversationSharingSettings { Enabled = enabled }).StatusCodeShouldBeOk()); + Assert.NotNull(updated); + Assert.Equal(enabled, updated.ConversationSharingDefaultEnabled); + + await GetService().RemoveAllAsync(); + var persisted = await SendRequestAsAsync(request => request + .AsGlobalAdminUser().AppendPaths("admin", "assistant-settings").StatusCodeShouldBeOk()); + Assert.NotNull(persisted); + Assert.Equal(enabled, persisted.ConversationSharingDefaultEnabled); + Assert.Equal(initial.Model, persisted.Model); + Assert.Equal(initial.Enabled, persisted.Enabled); + } + } + + [Fact] + public Task AssistantConversationSharingSettingsAsync_AsOrganizationUser_ReturnsForbidden() => SendRequestAsync(request => request + .Put().AsTestOrganizationUser().AppendPaths("admin", "assistant-settings", "conversation-sharing") + .Content(new UpdateAssistantConversationSharingSettings { Enabled = true }).StatusCodeShouldBeForbidden()); + + [Fact] + public Task AssistantConversationSharingSettingsAsync_AsAnonymous_ReturnsUnauthorized() => SendRequestAsync(request => request + .Put().AsAnonymousUser().AppendPaths("admin", "assistant-settings", "conversation-sharing") + .Content(new UpdateAssistantConversationSharingSettings { Enabled = true }).StatusCodeShouldBeUnauthorized()); + [Fact] public async Task EventSubmissionSettingsAsync_AsGlobalAdmin_UpdatesAndClearsRuntimeOverride() { @@ -1355,6 +1391,7 @@ private sealed record AssistantModelSettingsResponse( bool Enabled, bool ConfiguredEnabled, bool IsEnabledOverridden, - bool IsConfigured); + bool IsConfigured, + bool ConversationSharingDefaultEnabled); private sealed record RequeueResult([property: JsonPropertyName("enqueued")] int Enqueued); } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs index a458915a10..9626c227a5 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/AssistantEndpointTests.cs @@ -1,5 +1,7 @@ using System.Net; using System.Text.Json; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; using Exceptionless.Web.Assistant; @@ -19,6 +21,64 @@ protected override async Task ResetDataAsync() await GetService().CreateDataAsync(); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ConversationSharingAsync_PreservesExplicitChoicesAcrossDefaultChanges(bool choice) + { + var users = GetService(); + var user = await users.GetByEmailAddressAsync(SampleDataService.TEST_ORG_USER_EMAIL); + var otherUser = await users.GetByEmailAddressAsync(SampleDataService.TEST_USER_EMAIL); + Assert.NotNull(user); + Assert.NotNull(otherUser); + var systemSettings = GetService(); + var sharingService = GetService(); + + Assert.Null(user.AssistantConversationSharingEnabled); + Assert.Equal(new(false, false, false), await ReadAsync()); + await systemSettings.UpdateAsync(otherUser.Id, settings => settings.AssistantConversationSharingDefaultEnabled = choice); + Assert.Equal(new(choice, choice, false), await ReadAsync()); + + // An explicit choice equal to the current default must survive later default changes. + Assert.Equal(new(choice, choice, true), await SaveAsync(choice)); + await systemSettings.UpdateAsync(otherUser.Id, settings => settings.AssistantConversationSharingDefaultEnabled = !choice); + Assert.Equal(new(choice, !choice, true), await ReadAsync()); + Assert.Equal(new(choice, !choice, true), await sharingService.GetAsync(user.Id)); + + var savedUser = await users.GetByIdAsync(user.Id); + Assert.NotNull(savedUser); + Assert.Equal(choice, savedUser.AssistantConversationSharingEnabled); + Assert.Equal(user.FullName, savedUser.FullName); + Assert.Equal(user.EmailAddress, savedUser.EmailAddress); + Assert.Equal(user.OrganizationIds, savedUser.OrganizationIds); + Assert.Equal(user.EmailNotificationsEnabled, savedUser.EmailNotificationsEnabled); + Assert.Null((await users.GetByIdAsync(otherUser.Id))!.AssistantConversationSharingEnabled); + + Assert.Equal(new(!choice, !choice, false), await SaveAsync(null)); + Assert.Null((await users.GetByIdAsync(user.Id))!.AssistantConversationSharingEnabled); + Assert.Equal(new(!choice, !choice, false), await ReadAsync()); + + Task ReadAsync() => SendRequestAsAsync(request => request + .AsTestOrganizationUser().AppendPath("assistant/conversation-sharing").StatusCodeShouldBeOk()); + Task SaveAsync(bool? enabled) => SendRequestAsAsync(request => request + .Put().AsTestOrganizationUser().AppendPath("assistant/conversation-sharing") + .Content(new UpdateAssistantConversationSharing { Enabled = enabled }).StatusCodeShouldBeOk()); + } + + [Fact] + public Task SetConversationSharingAsync_Anonymous_ReturnsUnauthorized() => SendRequestAsync(request => request + .Put().AsAnonymousUser().AppendPath("assistant/conversation-sharing") + .Content(new UpdateAssistantConversationSharing { Enabled = true }).StatusCodeShouldBeUnauthorized()); + + [Fact] + public Task GetConversationSharingAsync_Anonymous_ReturnsUnauthorized() => SendRequestAsync(request => request + .AsAnonymousUser().AppendPath("assistant/conversation-sharing").StatusCodeShouldBeUnauthorized()); + + [Fact] + public Task SetConversationSharingAsync_MissingChoice_ReturnsBadRequest() => SendRequestAsync(request => request + .Put().AsTestOrganizationUser().AppendPath("assistant/conversation-sharing") + .Content(new { }).ExpectedStatus(HttpStatusCode.BadRequest)); + [Fact] public Task StreamAssistantChatAsync_Anonymous_ReturnsUnauthorized() { diff --git a/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs new file mode 100644 index 0000000000..094276c043 --- /dev/null +++ b/tests/Exceptionless.Tests/Assistant/AssistantDiagnosticsTests.cs @@ -0,0 +1,380 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Net; +using System.Text.Json; +using Exceptionless.Core; +using Exceptionless.Web.Api.Endpoints; +using Exceptionless.Web.Assistant; +using Foundatio.Caching; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using OpenTelemetry.Trace; +using Xunit; + +namespace Exceptionless.Tests.Assistant; + +public sealed class AssistantDiagnosticsTests +{ + [Fact] + public void AddApm_ExieActivitySource_CreatesExportableSpans() + { + // The APM source file is linked into both Web and Job. Select the Web copy + // explicitly to exercise its real registration without ambiguous type references. + var assembly = typeof(AssistantService).Assembly; + var configType = assembly.GetType("OpenTelemetry.ApmConfig", throwOnError: true)!; + var config = Activator.CreateInstance(configType, new ConfigurationBuilder().Build(), "test", "1.0", false); + var builder = new HostBuilder(); + assembly.GetType("OpenTelemetry.ApmExtensions", throwOnError: true)! + .GetMethod("AddApm")!.Invoke(null, [builder, config]); + using var host = builder.Build(); + _ = host.Services.GetRequiredService(); + + using var activity = AppDiagnostics.AssistantActivitySource.StartActivity("assistant.turn"); + + Assert.NotNull(activity); + Assert.True(activity.IsAllDataRequested); + using var pipelineActivity = AppDiagnostics.StartActivity("Event Pipeline"); + Assert.Null(pipelineActivity); + } + + [Theory] + [InlineData("empty_response")] + [InlineData("output_limit")] + [InlineData("malformed_response")] + [InlineData("tool_round_limit")] + [InlineData("usage_limit")] + public async Task WriteResponseAsync_StreamedError_RecordsCorrelatedFailureWithoutChangingResponse(string reason) + { + var logger = new RecordingAssistantLogger(); + var time = new FakeTimeProvider(); + using var diagnostics = new AssistantTurnDiagnostics(logger, time, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + time.Advance(TimeSpan.FromSeconds(12)); + + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([AssistantStreamEvent.Error("private error detail", reason), AssistantStreamEvent.Done()]), + CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(LogLevel.Warning, entry.Level); + Assert.Equal("failed", entry.Properties["Outcome"]); + Assert.Equal(reason, entry.Properties["FailureReason"]); + Assert.Equal(12_000d, entry.Properties["DurationMs"]); + Assert.Equal("organization-id", entry.Properties["OrganizationId"]); + Assert.Equal("conversation-id", entry.Properties["ConversationId"]); + Assert.Equal("request-id", entry.Properties["RequestId"]); + Assert.Equal(diagnostics.TurnId, entry.Properties["AssistantTurnId"]); + Assert.DoesNotContain("private error detail", entry.Message); + Assert.Equal(1, Assert.Single(recorder.Records).Increment.Failed); + + Assert.Equal(StatusCodes.Status200OK, context.Response.StatusCode); + var events = await ReadEventsAsync(context); + Assert.Equal("private error detail", events[0].GetProperty("message").GetString()); + Assert.False(events[0].TryGetProperty("failure_code", out _)); + Assert.False(events[0].TryGetProperty("FailureCode", out _)); + Assert.Equal("done", events[1].GetProperty("type").GetString()); + } + + [Theory] + [InlineData(false, true, "provider_stream", "failed", "turn_timeout")] + [InlineData(false, false, "provider_stream", "failed", "provider_timeout")] + [InlineData(true, true, "provider_stream", "cancelled", "client_disconnected")] + [InlineData(false, false, "tool_execution", "failed", "operation_cancelled")] + public async Task WriteResponseAsync_Cancellation_DistinguishesClientAndServer( + bool clientDisconnected, bool deadlineExpired, string stage, string outcome, string reason) + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.Stage = stage; + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + using var cancellation = new CancellationTokenSource(); + if (deadlineExpired) + await cancellation.CancelAsync(); + if (clientDisconnected) + context.RequestAborted = cancellation.Token; + + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([], new OperationCanceledException("private provider detail")), + CreateUsageService(cache, recorder), "organization-id", diagnostics, cancellation.Token); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(outcome, entry.Properties["Outcome"]); + Assert.Equal(reason, entry.Properties["FailureReason"]); + Assert.Equal(stage, entry.Properties["Stage"]); + Assert.DoesNotContain("private provider detail", entry.Message); + var usage = Assert.Single(recorder.Records).Increment; + Assert.Equal(clientDisconnected ? 0 : 1, usage.Failed); + Assert.Equal(clientDisconnected ? 1 : 0, usage.Cancelled); + var events = await ReadEventsAsync(context); + if (clientDisconnected) + Assert.Empty(events); + else + Assert.Equal("Exie took too long to complete this response. Try narrowing the question.", Assert.Single(events).GetProperty("message").GetString()); + } + + [Fact] + public async Task WriteResponseAsync_ProviderException_RecordsReasonWithoutLoggingProviderMessage() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([], new AssistantProviderException("private provider detail") { FailureCode = "provider_http_error" }), + CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(LogLevel.Error, entry.Level); + Assert.Equal("provider_http_error", entry.Properties["FailureReason"]); + Assert.Equal(typeof(AssistantProviderException).FullName, entry.Properties["ExceptionType"]); + Assert.DoesNotContain("private provider detail", entry.Message); + Assert.Equal(1, Assert.Single(recorder.Records).Increment.Failed); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task WriteResponseAsync_Success_RecordsCompletionAndFirstTextWithoutLoggingAnswer(bool fullLoggingEnabled) + { + var logger = new RecordingAssistantLogger(); + var time = new FakeTimeProvider(); + using var diagnostics = new AssistantTurnDiagnostics(logger, time, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var context = CreateHttpContext(); + time.Advance(TimeSpan.FromSeconds(3)); + await AssistantEndpoints.WriteResponseAsync(context, + StreamEvents([AssistantStreamEvent.TextDelta("private answer"), AssistantStreamEvent.Done()]), + CreateUsageService(cache, recorder), "organization-id", diagnostics, TestContext.Current.CancellationToken, fullLoggingEnabled); + + Assert.Equal(fullLoggingEnabled ? "true" : "false", context.Response.Headers[AssistantEndpoints.FullLoggingHeaderName]); + var entry = Assert.Single(logger.Entries); + Assert.Equal("completed", entry.Properties["Outcome"]); + Assert.Equal("none", entry.Properties["FailureReason"]); + Assert.Equal(3000d, entry.Properties["FirstTextDurationMs"]); + Assert.DoesNotContain("private answer", entry.Message); + Assert.Equal(1, Assert.Single(recorder.Records).Increment.Completed); + } + + [Fact] + public void Finish_FailedTurn_RecordsErrorSpanAndBoundedMetricTagsOnce() + { + var activities = new ConcurrentQueue(); + var activitySource = AppDiagnostics.AssistantActivitySource; + using var listener = new ActivityListener + { + ShouldListenTo = source => source == activitySource, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + SampleUsingParentId = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = activity => activities.Enqueue(activity) + }; + ActivitySource.AddActivityListener(listener); + var measurements = new List>(); + string? turnId = null; + using var meterListener = new MeterListener + { + InstrumentPublished = (instrument, current) => + { + if (instrument.Name == "ex.assistant.turn.duration") + current.EnableMeasurementEvents(instrument); + } + }; + meterListener.SetMeasurementEventCallback((_, _, tags, _) => + { + if (Activity.Current?.GetTagItem("assistant.turn.id") as string == turnId) + measurements.Add(tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value)); + }); + meterListener.Start(); + var logger = new RecordingAssistantLogger(); + using (var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id")) + { + turnId = diagnostics.TurnId; + diagnostics.Stage = "provider_stream"; + diagnostics.Finish("failed", "turn_timeout"); + diagnostics.Finish("completed"); + } + + var measurement = Assert.Single(measurements); + Assert.Equal(3, measurement.Count); + Assert.Equal("failed", measurement["outcome"]); + Assert.Equal("turn_timeout", measurement["reason"]); + Assert.Equal("provider_stream", measurement["stage"]); + var activity = Assert.Single(activities, activity => activity.GetTagItem("assistant.turn.id") as string == turnId); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("turn_timeout", activity.StatusDescription); + Assert.Single(logger.Entries); + } + + [Theory] + [InlineData("client", "cancelled")] + [InlineData("turn", "turn_timeout")] + [InlineData("provider", "provider_timeout")] + public void RecordException_CancellationSource_RecordsExpectedProviderOutcome(string source, string expectedOutcome) + { + using var requestAborted = new CancellationTokenSource(); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(requestAborted.Token); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id", requestAborted.Token); + using (var provider = diagnostics.StartProviderRequest(100, true, deadline.Token)) + { + if (source == "client") + { + requestAborted.Cancel(); + } + else if (source == "turn") + { + deadline.Cancel(); + } + provider.RecordException(new OperationCanceledException("private cancellation detail")); + } + + var entry = Assert.Single(logger.Entries); + Assert.Equal(expectedOutcome, entry.Properties["ProviderOutcome"]); + Assert.Equal(source == "client" ? LogLevel.Information : LogLevel.Warning, entry.Level); + Assert.DoesNotContain("private cancellation detail", entry.Message); + } + + [Fact] + public void ObserveChunk_OutputLimit_RecordsGenerationAndReasoningWithoutContent() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.Model = "configured-model"; + using var provider = diagnostics.StartProviderRequest(1000, true, TestContext.Current.CancellationToken); + using var response = new HttpResponseMessage(HttpStatusCode.OK); + response.Headers.Add("X-Generation-Id", "gen-header"); + provider.ObserveResponse(response); + using var document = JsonDocument.Parse(""" + {"id":"gen-stream","model":"resolved-model","provider":"Example Provider", + "choices":[{"delta":{"reasoning":"private reasoning"},"finish_reason":"length"}], + "usage":{"prompt_tokens":100,"completion_tokens":2048,"completion_tokens_details":{"reasoning_tokens":2048}}} + """); + provider.ObserveChunk(document.RootElement); + provider.Complete(0, 0, true); + + Assert.Equal("gen-stream", provider.GenerationId); + Assert.Equal("length", provider.FinishReason); + Assert.Equal(2048, provider.ReasoningTokens); + var entry = Assert.Single(logger.Entries); + Assert.Equal(LogLevel.Warning, entry.Level); + Assert.Equal("output_limit", entry.Properties["ProviderOutcome"]); + Assert.Equal("resolved-model", entry.Properties["ProviderModel"]); + Assert.Equal(100L, entry.Properties["PromptTokens"]); + Assert.Equal(2048L, entry.Properties["CompletionTokens"]); + Assert.DoesNotContain("private reasoning", entry.Message); + } + + [Fact] + public void ObserveChunk_UnexpectedMetadata_DoesNotThrowOrLogUnboundedValues() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + using var provider = diagnostics.StartProviderRequest(1000, true, TestContext.Current.CancellationToken); + using var document = JsonDocument.Parse(""" + {"id":"private\nvalue","model":{"unexpected":true},"provider":null, + "choices":[{"finish_reason":"unexpected-provider-string"}], + "usage":{"completion_tokens_details":{"reasoning_tokens":"not-a-number"}}} + """); + provider.ObserveChunk(document.RootElement); + provider.Complete(10, 0, true); + + Assert.Null(provider.GenerationId); + Assert.Null(provider.Model); + Assert.Null(provider.ReasoningTokens); + Assert.Equal("unknown", provider.FinishReason); + Assert.DoesNotContain("private", Assert.Single(logger.Entries).Message); + } + + [Fact] + public void RecordToolResult_FailedTool_RecordsCodeWithoutArgumentsOrResult() + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.StartTool("search_stacks"); + diagnostics.RecordToolResult("""{"ok":false,"error":{"code":"invalid_filter","message":"private filter"}}""", 100); + diagnostics.StartTool("private-hallucinated-tool-name"); + diagnostics.RecordToolResult("""{"ok":false,"error":{"code":"private-error-code","message":"private details"}}""", 100); + diagnostics.Finish("completed"); + + Assert.Equal(2, diagnostics.ToolFailures); + Assert.Equal("unknown", diagnostics.LastTool); + Assert.Equal("tool_error", diagnostics.LastToolError); + Assert.Equal("invalid_filter", logger.Entries[0].Properties["ToolErrorCode"]); + Assert.All(logger.Entries, entry => Assert.DoesNotContain("private", entry.Message)); + Assert.Equal("completed", logger.Entries[^1].Properties["Outcome"]); + } + + [Theory] + [InlineData("null")] + [InlineData("[]")] + [InlineData("\"text\"")] + public void RecordToolResult_NonObjectResult_DoesNotInterruptTheTurn(string result) + { + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + diagnostics.StartTool("get_event"); + diagnostics.RecordToolResult(result, 100); + + Assert.Equal(0, diagnostics.ToolFailures); + Assert.Empty(logger.Entries); + } + + private static DefaultHttpContext CreateHttpContext() + { + var context = new DefaultHttpContext(); + context.Response.Body = new MemoryStream(); + return context; + } + + private static async Task ReadEventsAsync(HttpContext context) + { + context.Response.Body.Position = 0; + using var reader = new StreamReader(context.Response.Body, leaveOpen: true); + string content = await reader.ReadToEndAsync(TestContext.Current.CancellationToken); + return content.Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(line => + { + using var document = JsonDocument.Parse(line); + return document.RootElement.Clone(); + }).ToArray(); + } + + private static AssistantUsageService CreateUsageService(ICacheClient cache, RecordingAssistantUsageRecorder recorder) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost" }) + .Build()); + return new AssistantUsageService(cache, null!, recorder, options, TimeProvider.System, NullLogger.Instance); + } + + private static async IAsyncEnumerable StreamEvents(AssistantStreamEvent[] events, Exception? exception = null) + { + await Task.Yield(); + if (exception is not null) + throw exception; + foreach (var item in events) + yield return item; + } +} + +internal sealed class RecordingAssistantLogger : ILogger +{ + public List Entries { get; } = []; + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + => Entries.Add(new AssistantLogEntry(logLevel, formatter(state, exception), + ((IEnumerable>)(object)state!).ToDictionary(property => property.Key, property => property.Value))); +} + +internal sealed record AssistantLogEntry(LogLevel Level, string Message, Dictionary Properties); diff --git a/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs new file mode 100644 index 0000000000..88e46c6cb6 --- /dev/null +++ b/tests/Exceptionless.Tests/Assistant/AssistantModelSettingsServiceTests.cs @@ -0,0 +1,54 @@ +using System.Text.Json; +using Exceptionless.Core; +using Exceptionless.Core.Models; +using Exceptionless.Core.Services; +using Exceptionless.Web.Assistant; +using Microsoft.Extensions.Configuration; +using Xunit; + +namespace Exceptionless.Tests.Assistant; + +public sealed class AssistantModelSettingsServiceTests +{ + [Fact] + public void LegacyUser_FollowsConversationSharingDefault() + { + var user = JsonSerializer.Deserialize("{}"); + Assert.NotNull(user); + Assert.Null(user.AssistantConversationSharingEnabled); + Assert.False(AssistantConversationSharingService.Resolve(user.AssistantConversationSharingEnabled, false).Enabled); + Assert.True(AssistantConversationSharingService.Resolve(user.AssistantConversationSharingEnabled, true).Enabled); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SetConversationSharingDefaultEnabledAsync_NewAndLegacySettings_DefaultOffAndPreserveOtherSettings(bool hasLegacyRecord) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost" }).Build()); + SystemSettings? persisted = hasLegacyRecord ? JsonSerializer.Deserialize("{}") : null; + var systemSettings = new SystemSettingsService(() => Task.FromResult(persisted), value => + { + persisted = value; + return Task.CompletedTask; + }, options, TimeProvider.System); + var service = new AssistantModelSettingsService(systemSettings, options); + + Assert.False((await service.GetAsync()).ConversationSharingDefaultEnabled); + Assert.False(await systemSettings.IsAssistantConversationSharingDefaultEnabledAsync()); + + Assert.True((await service.SetConversationSharingDefaultEnabledAsync(true, "admin-user")).ConversationSharingDefaultEnabled); + Assert.True(await systemSettings.IsAssistantConversationSharingDefaultEnabledAsync()); + await service.SetModelAsync("example/model", "admin-user"); + await service.SetEnabledAsync(true, "admin-user"); + Assert.True((await service.GetAsync()).ConversationSharingDefaultEnabled); + + Assert.False((await service.SetConversationSharingDefaultEnabledAsync(false, "admin-user")).ConversationSharingDefaultEnabled); + Assert.False(await systemSettings.IsAssistantConversationSharingDefaultEnabledAsync()); + Assert.NotNull(persisted); + Assert.Equal("example/model", persisted.AssistantModel); + Assert.True(persisted.AssistantEnabled); + Assert.Equal("admin-user", persisted.UpdatedByUserId); + } +} diff --git a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs index 35ce5e8d9b..060b96464c 100644 --- a/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs +++ b/tests/Exceptionless.Tests/Assistant/AssistantServiceTests.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; using System.Net; using System.Text; using System.Text.Json; @@ -7,6 +9,7 @@ using Exceptionless.Core.Models.Billing; using Exceptionless.Core.Serialization; using Exceptionless.Core.Services; +using Exceptionless.Web.Api.Endpoints; using Exceptionless.Web.Assistant; using Exceptionless.Web.Mcp; using Foundatio.Caching; @@ -16,6 +19,7 @@ using Foundatio.Serializer; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Xunit; @@ -986,6 +990,256 @@ public async Task StreamAsync_EmptyResponse_EmitsClearErrorAndCompletion() item => Assert.Equal("done", item.Type)); } + [Theory] + [InlineData("length", "output_limit")] + [InlineData("content_filter", "content_filter")] + [InlineData("stop", "empty_response")] + public async Task StreamAsync_EmptyProviderAnswer_RecordsProviderReasonAndGeneration(string finishReason, string failureCode) + { + string payload = JsonSerializer.Serialize(new + { + id = "gen-empty-answer", + model = "resolved-model", + choices = new[] { new { delta = new { content = "" }, finish_reason = finishReason } }, + usage = new { prompt_tokens = 100, completion_tokens = 2048, completion_tokens_details = new { reasoning_tokens = 2048 } } + }); + var handler = new StubHttpMessageHandler($"data: {payload}\n\ndata: [DONE]\n\n"); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(handler, options); + var events = new List(); + + await foreach (var item in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + events.Add(item); + + Assert.Equal(failureCode, Assert.Single(events, item => item.Type == "error").FailureCode); + Assert.Equal("gen-empty-answer", diagnostics.Provider?.GenerationId); + Assert.Equal("resolved-model", diagnostics.Provider?.Model); + Assert.Equal(2048, diagnostics.Provider?.ReasoningTokens); + Assert.Equal(1, diagnostics.ProviderRequests); + Assert.DoesNotContain(logger.Entries, entry => entry.Message.Contains("private question", StringComparison.Ordinal)); + } + + [Fact] + public async Task StreamAsync_ProviderStreamError_RecordsProviderErrorWithoutLoggingBody() + { + var handler = new StubHttpMessageHandler(""" + data: {"id":"gen-stream-error","error":{"message":"private provider error"}} + + """); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(handler, options); + + var exception = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + { + } + }); + + Assert.Equal("provider_error", exception.FailureCode); + var entry = Assert.Single(logger.Entries); + Assert.Equal("provider_error", entry.Properties["ProviderOutcome"]); + Assert.Equal(200, entry.Properties["ProviderStatusCode"]); + Assert.Equal("gen-stream-error", entry.Properties["ProviderGenerationId"]); + Assert.DoesNotContain("private provider error", entry.Message); + } + + [Theory] + [InlineData("transport", "provider_transport_error")] + [InlineData("stream", "provider_stream_error")] + [InlineData("json", "invalid_provider_response")] + [InlineData("timeout", "provider_timeout")] + public async Task StreamAsync_ProviderThrows_RecordsFailureCategory(string failure, string expectedOutcome) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(new FailingProviderHttpMessageHandler(failure), options); + + var exception = await Record.ExceptionAsync(async () => + { + await foreach (var _ in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + { + } + }); + + Assert.NotNull(exception); + var entry = Assert.Single(logger.Entries); + Assert.Equal(expectedOutcome, entry.Properties["ProviderOutcome"]); + Assert.DoesNotContain("private", entry.Message); + } + + [Theory] + [InlineData("[]")] + [InlineData("null")] + [InlineData("{\"choices\":{}}")] + [InlineData("{\"choices\":[{}]}")] + [InlineData("{\"choices\":[{\"delta\":[]}]}")] + [InlineData("{\"choices\":[{\"delta\":{\"tool_calls\":[{}]}}]}")] + [InlineData("{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":2147483648}]}}]}")] + [InlineData("{\"usage\":{\"prompt_tokens\":\"private invalid token count\"}}")] + public async Task StreamAsync_InvalidProviderShape_RecordsProviderAndTurnFailure(string payload) + { + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + using var cache = new InMemoryCacheClient(); + var recorder = new RecordingAssistantUsageRecorder(); + var usageService = new AssistantUsageService(cache, CreateLockProvider(cache, TimeProvider.System), recorder, options, + TimeProvider.System, NullLogger.Instance); + var service = CreateAssistantService(new StubHttpMessageHandler($"data: {payload}\n\ndata: [DONE]\n"), options, cache, usageService: usageService); + var context = new DefaultHttpContext(); + using var response = new MemoryStream(); + context.Response.Body = response; + + await AssistantEndpoints.WriteResponseAsync(context, + service.StreamAsync(new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken), + usageService, "organization-id", diagnostics, TestContext.Current.CancellationToken); + + var providerEntry = Assert.Single(logger.Entries, entry => entry.Properties.ContainsKey("ProviderOutcome")); + var turnEntry = Assert.Single(logger.Entries, entry => entry.Properties.ContainsKey("Outcome")); + Assert.Equal("invalid_provider_response", providerEntry.Properties["ProviderOutcome"]); + Assert.Equal("invalid_provider_response", turnEntry.Properties["FailureReason"]); + Assert.Equal("failed", turnEntry.Properties["Outcome"]); + Assert.All(logger.Entries, entry => Assert.DoesNotContain("private", entry.Message)); + } + + [Theory] + [InlineData("exception", "failed", "tool_execution_error")] + [InlineData("client", "cancelled", "client_disconnected")] + [InlineData("turn", "failed", "turn_timeout")] + public async Task StreamAsync_ToolThrows_RecordsToolOutcomeAndDuration(string failure, string outcome, string reason) + { + var activitySource = AppDiagnostics.AssistantActivitySource; + using var activityListener = new ActivityListener + { + ShouldListenTo = source => source == activitySource, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded + }; + ActivitySource.AddActivityListener(activityListener); + var logger = new RecordingAssistantLogger(); + using var requestAborted = new CancellationTokenSource(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(requestAborted.Token, TestContext.Current.CancellationToken); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id", requestAborted.Token); + var measurements = new List>(); + using var meterListener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Name == "ex.assistant.tool.duration") + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + meterListener.SetMeasurementEventCallback((_, _, tags, _) => + { + if (Activity.Current?.GetTagItem("assistant.turn.id") as string == diagnostics.TurnId) + { + measurements.Add(tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value)); + } + }); + meterListener.Start(); + // An array where a tool argument object is required throws during invocation. + var handler = new StubHttpMessageHandler(""" + data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"tool-1","function":{"name":"search_stacks","arguments":"[]"}}]}}]} + + data: [DONE] + + """); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var service = CreateAssistantService(handler, options); + var exception = await Record.ExceptionAsync(async () => + { + await foreach (var item in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "Find my errors")]), + "user-id", CreatePlanOptions(), diagnostics, cancellation.Token)) + { + if (failure == "client" && item.Type == "tool_call") + { + requestAborted.Cancel(); + } + else if (failure == "turn" && item.Type == "tool_call") + { + cancellation.Cancel(); + } + } + }); + + if (failure != "exception") + { + Assert.IsAssignableFrom(exception); + } + else + { + Assert.IsType(exception); + } + Assert.Equal(1, diagnostics.ToolCalls); + Assert.Equal(failure == "client" ? 0 : 1, diagnostics.ToolFailures); + Assert.Equal(reason, diagnostics.LastToolError); + var measurement = Assert.Single(measurements); + Assert.Equal("search_stacks", measurement["tool"]); + Assert.Equal(outcome, measurement["outcome"]); + Assert.Equal(diagnostics.LastToolError, measurement["reason"]); + } + + [Theory] + [InlineData(HttpStatusCode.TemporaryRedirect)] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.InternalServerError)] + public async Task StreamAsync_HttpRejection_RecordsStatusWithoutLoggingProviderErrorBody(HttpStatusCode responseStatus) + { + var handler = new RejectedHttpMessageHandler(responseStatus); + var options = AppOptions.ReadFromConfiguration(new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["BaseURL"] = "https://localhost", ["Assistant:ApiKey"] = "test-key" }) + .Build()); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); + var service = CreateAssistantService(handler, options, logger: logger); + + var exception = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in service.StreamAsync( + new AssistantChatRequest([new AssistantChatMessage("user", "private question")]), + "user-id", CreatePlanOptions(), diagnostics, TestContext.Current.CancellationToken)) + { + } + }); + + Assert.Equal("provider_http_error", exception.FailureCode); + var providerEntry = Assert.Single(logger.Entries, entry => entry.Properties.ContainsKey("ProviderOutcome")); + Assert.Equal(exception.FailureCode, providerEntry.Properties["ProviderOutcome"]); + Assert.Equal((int)responseStatus, diagnostics.Provider?.StatusCode); + Assert.Contains(logger.Entries, entry => entry.Properties.TryGetValue("StatusCode", out var status) && status is int code && code == (int)responseStatus); + Assert.All(logger.Entries, entry => + { + Assert.DoesNotContain("Rejected", entry.Message); + Assert.DoesNotContain("private question", entry.Message); + Assert.DoesNotContain("test-key", entry.Message); + }); + } + [Fact] public async Task StreamAsync_RawDsmlResponse_RetriesWithoutEmittingMarkup() { @@ -1085,11 +1339,14 @@ public async Task StreamAsync_RepeatedRawDsmlResponse_EmitsClearErrorAndCompleti .Build()); var service = CreateAssistantService(handler, appOptions); var events = new List(); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); await foreach (var item in service.StreamAsync( new AssistantChatRequest([new AssistantChatMessage("user", "Find recent errors")]), "user-id", CreatePlanOptions(), + diagnostics, TestContext.Current.CancellationToken)) { events.Add(item); @@ -1104,10 +1361,15 @@ public async Task StreamAsync_RepeatedRawDsmlResponse_EmitsClearErrorAndCompleti }, item => Assert.Equal("done", item.Type)); Assert.DoesNotContain(events, item => item.Type == "text_delta"); + var providerEntries = logger.Entries.Where(entry => entry.Properties.ContainsKey("ProviderOutcome")).ToArray(); + Assert.Equal(2, providerEntries.Length); + Assert.All(providerEntries, entry => Assert.Equal("malformed_response", entry.Properties["ProviderOutcome"])); } - [Fact] - public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutTools() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutTools(bool providerIgnoresToolLimit) { const string toolCallResponse = """ data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"unknown_tool","arguments":"{}"}}]}}]} @@ -1119,7 +1381,7 @@ public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutT toolCallResponse, toolCallResponse.Replace("call-1", "call-2"), toolCallResponse.Replace("call-1", "call-3"), - """ + providerIgnoresToolLimit ? toolCallResponse.Replace("call-1", "call-4") : """ data: {"choices":[{"delta":{"content":"Here is the available result."}}]} data: [DONE] @@ -1134,6 +1396,8 @@ public async Task StreamAsync_ToolBudgetExhausted_RequestsFinalSynthesisWithoutT .Build()); var service = CreateAssistantService(handler, appOptions); var events = new List(); + var logger = new RecordingAssistantLogger(); + using var diagnostics = new AssistantTurnDiagnostics(logger, TimeProvider.System, "organization-id", "conversation-id", "request-id"); await foreach (var item in service.StreamAsync( new AssistantChatRequest( @@ -1141,6 +1405,7 @@ [new AssistantChatMessage("user", "Investigate the errors")], OrganizationId: "organization-id"), "user-id", CreatePlanOptions(), + diagnostics, TestContext.Current.CancellationToken)) { events.Add(item); @@ -1150,9 +1415,19 @@ [new AssistantChatMessage("user", "Investigate the errors")], Assert.All(handler.RequestBodies.Take(3), body => Assert.Contains("\"tools\":", body)); Assert.DoesNotContain("\"tools\":", handler.RequestBodies[3]); Assert.Contains("The tool budget is exhausted", handler.RequestBodies[3]); - Assert.Contains(events, item => item.Text == "Here is the available result."); Assert.Equal("done", events[^1].Type); - Assert.DoesNotContain(events, item => item.Type == "error"); + var finalProviderEntry = logger.Entries.Last(entry => entry.Properties.ContainsKey("ProviderOutcome")); + if (providerIgnoresToolLimit) + { + Assert.Equal("tool_round_limit", Assert.Single(events, item => item.Type == "error").FailureCode); + Assert.Equal("tool_round_limit", finalProviderEntry.Properties["ProviderOutcome"]); + } + else + { + Assert.Contains(events, item => item.Text == "Here is the available result."); + Assert.DoesNotContain(events, item => item.Type == "error"); + Assert.Equal("completed", finalProviderEntry.Properties["ProviderOutcome"]); + } } [Fact] @@ -1229,7 +1504,8 @@ private static AssistantService CreateAssistantService( ICacheClient? cache = null, ILockProvider? lockProvider = null, AssistantUsageService? usageService = null, - AssistantModelSettingsService? modelSettingsService = null) + AssistantModelSettingsService? modelSettingsService = null, + ILogger? logger = null) { cache ??= new InMemoryCacheClient(new InMemoryCacheClientOptions { @@ -1256,7 +1532,7 @@ private static AssistantService CreateAssistantService( modelSettingsService, usageService, TimeProvider.System, - NullLogger.Instance); + logger ?? NullLogger.Instance); } private static AssistantModelSettingsService CreateAssistantModelSettingsService(AppOptions appOptions) @@ -1314,6 +1590,24 @@ protected override async Task SendAsync(HttpRequestMessage } } + private sealed class FailingProviderHttpMessageHandler(string failure) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => failure switch + { + "transport" => Task.FromException(new HttpRequestException("private transport detail")), + "timeout" => Task.FromException(new TaskCanceledException("private timeout detail")), + "stream" => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(new FailingProviderStream()) }), + _ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("data: invalid-json\n\n") }) + }; + } + + private sealed class FailingProviderStream : MemoryStream + { + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => ValueTask.FromException(new IOException("private stream detail")); + } + private sealed class RejectedHttpMessageHandler(HttpStatusCode statusCode) : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/tests/Exceptionless.Tests/Assistant/README.md b/tests/Exceptionless.Tests/Assistant/README.md index fbc6d8e9e2..3bfadfa56d 100644 --- a/tests/Exceptionless.Tests/Assistant/README.md +++ b/tests/Exceptionless.Tests/Assistant/README.md @@ -19,3 +19,90 @@ dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll \ ``` Set `EX_Assistant__Model` and `EX_Assistant__Endpoint` to evaluate a candidate model or compatible provider. Use a dedicated provider key with a small monthly hard limit; the tests never print the key. + +## Runtime diagnostics + +Every accepted Exie turn emits one structured completion log with `Outcome` (`completed`, `failed`, or `cancelled`), `FailureReason`, `Stage`, duration, time to first answer text, model, provider request count, tool rounds, and tool failures. `AssistantTurnId`, `OrganizationId`, `ConversationId`, `RequestId`, and `TraceId` correlate a turn across logs and traces. These identifiers are also included in the rendered message so Kubernetes console logs remain useful without a structured log viewer. + +The `Exceptionless.Web.Assistant` logging override retains Information-level turn and provider summaries even when the production default is Warning. Assistant spans use the dedicated `Exceptionless.Assistant` activity source, leaving the shared Core ingestion activity source unchanged. Failed turns log at Warning; unexpected/provider exceptions log at Error. Client disconnects remain cancellations and log at Information. A `completed` turn means the response finished without a streamed error; it does not establish that the answer was useful or correct. Keep running the quality evaluations above when changing the model, prompt, or tools. + +Provider summaries include the generation ID (from `X-Generation-Id` or the stream), resolved model, provider, HTTP status, normalized finish reason, token counts including reasoning tokens when supplied, and whether final usage and `[DONE]` arrived. Thrown transport, parsing, stream, and timeout errors record explicit provider outcomes before propagating to the turn handler. Provider outcomes distinguish browser disconnects (`cancelled`), the shared turn deadline (`turn_timeout`), and provider-only cancellation (`provider_timeout`). See the [OpenRouter streaming contract](https://openrouter.ai/docs/api/reference/streaming). A request can return HTTP 200 and subsequently fail inside the stream, so HTTP error rates alone do not measure Exie reliability. Generation IDs can be used for provider-side investigation without logging the conversation. + +Server diagnostics deliberately exclude prompts, answer text, reasoning text, tool arguments/results, raw provider error bodies, and exception messages. Exception type and stack trace remain available. Tool names and error codes used as metric dimensions come from a fixed allowlist; organization, conversation, generation, and model identifiers are restricted to logs/traces. Browser session events record usage and error details, with optional conversation logging controlled by the global admin setting below. + +| Failure reason | Investigation | +| --- | --- | +| `turn_timeout` | The shared 120-second turn deadline expired; inspect the last stage, provider duration, and tool progress. | +| `provider_timeout` | A provider operation was cancelled before the shared turn deadline expired. | +| `operation_cancelled` | A non-provider operation was cancelled before the shared turn deadline expired; inspect the stage. | +| `provider_http_error`, `provider_error`, `provider_transport_error`, `provider_stream_error` | Check status, generation ID, provider, and whether the stream completed. | +| `empty_response` | The provider returned neither answer text nor tool calls. | +| `output_limit` | The provider returned no answer text and reported `finish_reason=length`; inspect reasoning and completion tokens before changing budgets. | +| `content_filter` | An empty answer ended with the provider's content-filter finish reason. | +| `malformed_response` | Internal provider markup remained after the existing recovery retry. | +| `tool_round_limit` | The provider continued requesting tools after the final-answer instruction. | +| `usage_limit`, `context_limit` | A turn reached an organization usage limit or the conversation context bound. | +| `invalid_provider_response`, `response_write_error`, `tool_execution_error`, `internal_error` | Inspect the stage, exception type/stack, and correlated trace. | + +Returned tool errors are logged with the tool name and error code, even when the model recovers and completes the turn. Thrown tool exceptions and deadlines also increment tool failures and record duration; browser disconnects record a separate `cancelled` duration with `client_disconnected`. Tool cancellation reasons distinguish the shared deadline (`turn_timeout`) from other operation cancellation (`operation_cancelled`). Provider error objects inside an HTTP 200 stream record `provider_error` even without a finish reason. Provider `output_limit` or `incomplete_stream` warnings can also accompany a completed turn when text was returned. Diagnostics preserve the existing response and accounting behavior; they do not automatically retry tools or change output budgets. + +The existing `ex.assistant.turn.outcomes` metric remains unchanged. Additional histograms provide immediate duration and outcome counts: + +| Instrument | Dimensions | +| --- | --- | +| `ex.assistant.turn.duration` (ms) | `outcome`, `reason`, `stage` | +| `ex.assistant.provider.duration` (ms) | `outcome` | +| `ex.assistant.tool.duration` (ms) | `tool`, `outcome`, `reason` | + +Track the completed share of completed + failed turns, failure counts by reason, cancellation rate separately, and turn/provider latency percentiles. Histogram counts can supply the immediate success-rate denominator; durable usage totals may lag. Failed `assistant.turn` spans carry error status even though the enclosing HTTP response is 200. Traces follow the deployment's existing sampling policy, so use logs and metrics for unsampled failures. + +In the hosted Helm deployment, the `ex-prod-app` workload runs `Exceptionless.Web` and serves in-app Exie requests; `ex-prod-api` primarily serves collector API traffic. Start with organization and UTC time, then follow the turn ID and generation IDs. Retained pod logs may not cover replaced pods or old conversations, and counters alone cannot reconstruct a historical failure reason. + +The diagnostics and provider-stream tests run locally without Aspire or billable provider requests: + +```powershell +dotnet build tests/Exceptionless.Tests/Exceptionless.Tests.csproj --maxcpucount:1 +dotnet tests/Exceptionless.Tests/bin/Debug/net10.0/Exceptionless.Tests.dll --filter-namespace Exceptionless.Tests.Assistant --filter-not-class Exceptionless.Tests.Assistant.AssistantQualityEvaluationTests --progress off +``` + +## Conversation session events + +The Svelte app submits Exie events through the existing Exceptionless browser client. They share the signed-in user's session, client configuration, queue, tags, and event exclusions. They go to the app's configured telemetry project. Starting a conversation does not create a separate user session. + +Session starts wait for an identified user: anonymous SDK startup/resume session events are discarded, while ordinary errors, logs, and usage events remain enabled. The telemetry component owns the identity update so profile loading cannot race a second identity writer. Older session rows without identity or session metadata display **Anonymous session**. + +The Aspire development app automatically reports to the seeded **Exceptionless → Exceptionless** project (named **API** in older development data). It waits for the API to be ready and uses the browser's current origin through Vite's API proxy, including forwarded localhost ports. No `.env.local` setup is required. Override `PUBLIC_EXCEPTIONLESS_API_KEY` and `PUBLIC_EXCEPTIONLESS_TELEMETRY_SERVER_URL` in the AppHost environment to use another telemetry destination; an empty key disables automatic browser reporting. For a standalone frontend, set those values in `ClientApp/.env.local`, with an empty telemetry URL to use the current origin. The advertised client setup URL stays unchanged. Omitting the telemetry URL override preserves the existing server URL behavior; a browser local-storage server URL override still takes precedence. Keep local keys out of source control. + +Each submitted prompt produces an `assistant.MessageSent` feature usage event. A turn produces one `assistant.ResponseCompleted`, `assistant.ResponseFailed`, or `assistant.ResponseCancelled` feature usage event. These events record character counts and outcomes. Failure events retain the error displayed to the user in `error_message`, capped at 2,048 characters for diagnosis. Existing application error collection is unchanged. + +Global admins control **Conversation sharing default** beside Exie availability and model settings. It defaults to off for new and legacy settings records. Enable it during the early rollout through the settings page or `PUT /api/v2/admin/assistant-settings/conversation-sharing` with `{ "enabled": true }`; set it to false to stop sharing by default later. + +Users see a compact **Chat sharing: On/Off** control beside the composer disclaimer. It opens a popover where they can change sharing for their account across conversations and devices. `GET /api/v2/assistant/conversation-sharing` returns the effective setting, default, and whether the user has chosen. The authenticated user's `PUT` at the same route saves `{ "enabled": true }` or `{ "enabled": false }`; `{ "enabled": null }` restores the default. Existing users have no override and follow the admin default. Explicit choices always win, even when originally saved equal to the default. Enabling the default therefore uses opt-out sharing; the visible control identifies inherited defaults and allows an immediate change. + +Each accepted turn reads the saved user choice and returns `X-Exie-Full-Logging`. Transcript capture requires both an explicit true header and an enabled, loaded sharing control. Turning sharing off discards the active reply buffer and prevents subsequent transcript capture; already submitted events are retained. If saving an opt-out fails, capture stays paused on the current page and the UI asks the user to retry saving for other devices. Changes elsewhere are read for the next turn; a reply already in progress on another device retains the choice selected at its start. Usage, feedback, and error diagnostics continue regardless of sharing. + +When enabled, full logging adds an `assistant.Prompt` log event and one assembled `assistant.Response` log event in the existing session. Each message is capped at 16,384 characters, with original character count and truncation metadata. When disabled, prompt and response text is not copied into telemetry. Drafts, individual streamed chunks, reasoning, raw tool arguments/results, and generated suggestion labels/destinations are never submitted. Suggestion events record only whether the action navigated or submitted a prompt. + +Events carry an `exie` extended-data object with `schema_version: 1`, conversation and message IDs, organization/project context, page path without query/fragment, and page/sheet mode. The conversation ID matches the server diagnostics. Turn summaries also include outcome, elapsed time, time to first text, tool counts/failures, and whether the chat was visible when the turn finished. Retries link the new server conversation back through `previous_conversation_id` and `retry_of_message_id`. + +Other feature usage events describe interactions: + +| Event source | Meaning | +| --- | --- | +| `assistant.Opened`, `assistant.Closed`, `assistant.ViewChanged` | One open event when Exie becomes visible, including loading or reloading the full Exie page; one close when hidden. Switching panel/page mode emits a view change. Typing, other state updates, and opening the sharing menu do not emit opens. Closing the panel does not cancel an ongoing response. | +| `assistant.ResponseHelpful`, `assistant.ResponseNotHelpful`, `assistant.ResponseFeedbackCleared` | Explicit feedback linked to the response. | +| `assistant.ResponseRegenerated` | Retry or regenerate, linked to the previous response. | +| `assistant.MessageCopied`, `assistant.SuggestedActionSelected` | Copy a message or act on an Exie suggestion. | +| `assistant.ConversationCleared`, `assistant.ConversationLeft`, `assistant.PageLeft` | Clear, switch organization, unmount, or leave the browser page, with the last outcome/feedback, message count, and whether a turn was still streaming. | + +Filter the app telemetry project by `source:assistant.*`, then open an event's session timeline and use the `exie` IDs in event details to follow a conversation. Use `type:usage` when counting turns or interactions so optional transcript logs do not inflate the counts. Compare explicit positive/negative feedback, repeated retries, response latency, and departures while waiting. A completed response only means text arrived without an error; it does not prove the answer helped. Copying and continuing are useful signals, while closing the chat alone does not establish frustration. + +These browser events are best effort. Page-leave events may be lost during unload, network failure, or a browser crash, and configured client filtering still applies. Use server metrics for operational failure rates; use session events to understand the user journey. A missing terminal event alone is not proof of cancellation or abandonment. + +Focused frontend tests exercise the real SDK builders with the queue intercepted, verify transcript capture is controlled by the server flag, and cover settings changes, streamed success/failure, retries, feedback, context changes, and panel visibility. They do not submit events to a running collector: + +```powershell +Set-Location src/Exceptionless.Web/ClientApp +npm run test:unit -- src/lib/features/assistant/assistant-telemetry.test.ts src/lib/features/assistant/components/assistant-panel.svelte.test.ts src/lib/features/assistant/components/assistant-message-actions.svelte.test.ts src/lib/features/auth/exceptionless-session.test.ts src/lib/features/admin/components/assistant-settings.svelte.test.ts +npm run check +``` diff --git a/tests/http/admin.http b/tests/http/admin.http index a349739e29..7521348fe9 100644 --- a/tests/http/admin.http +++ b/tests/http/admin.http @@ -58,6 +58,24 @@ Authorization: Bearer {{token}} GET {{apiUrl}}/admin/assistant-settings Authorization: Bearer {{token}} +### Enable Exie Conversation Sharing by Default (Saved User Choices Take Precedence) +PUT {{apiUrl}}/admin/assistant-settings/conversation-sharing +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "enabled": true +} + +### Disable Exie Conversation Sharing by Default (Usage and Error Diagnostics Continue) +PUT {{apiUrl}}/admin/assistant-settings/conversation-sharing +Authorization: Bearer {{token}} +Content-Type: application/json + +{ + "enabled": false +} + ### Set Exie Model Override PUT {{apiUrl}}/admin/assistant-settings Authorization: Bearer {{token}} diff --git a/tests/http/assistant.http b/tests/http/assistant.http index e080819e49..41e85e2919 100644 --- a/tests/http/assistant.http +++ b/tests/http/assistant.http @@ -19,7 +19,23 @@ Content-Type: application/json GET {{apiUrl}}/assistant/access?organization_id={{organizationId}} Authorization: Bearer {{login.response.body.$.token}} +### Get your effective conversation sharing choice +GET {{apiUrl}}/assistant/conversation-sharing +Authorization: Bearer {{login.response.body.$.token}} + +### Turn off conversation sharing for your account (true enables; null follows the admin default) +PUT {{apiUrl}}/assistant/conversation-sharing +Authorization: Bearer {{login.response.body.$.token}} +Content-Type: application/json + +{ + "enabled": false +} + ### Stream an assistant response +# X-Exie-Full-Logging reflects the saved user choice, falling back to the admin default. +# The browser also requires the visible sharing control to be enabled. +# A missing or false header keeps transcript logging off. Usage and error events continue. POST {{apiUrl}}/assistant/chat Authorization: Bearer {{login.response.body.$.token}} Content-Type: application/json