diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs index 1e184836b1..bf8132a921 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs @@ -81,7 +81,10 @@ public async Task UpdateEntry_CanUpdateExampleSentenceTranslations_WhenNoTransla var before = entry.Copy(); var exampleSentence = entry.Senses[0].ExampleSentences[0]; - exampleSentence.Translations = [new() { Text = { { "en", new RichString("updated") } } }]; + exampleSentence.Translations = + [ + new() { Id = Guid.NewGuid(), Text = { { "en", new RichString("updated") } } } + ]; // Act var updatedEntry = await Api.UpdateEntry(before, entry); diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index c760dffb27..588803caba 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -4,7 +4,6 @@ using LexCore.Sync; using Microsoft.Extensions.Logging; using MiniLcm; -using MiniLcm.Normalization; using MiniLcm.SyncHelpers; using MiniLcm.Validators; @@ -12,8 +11,7 @@ namespace FwLiteProjectSync; public class CrdtFwdataProjectSyncService(MiniLcmImport miniLcmImport, ILogger logger, - MiniLcmApiValidationWrapperFactory validationWrapperFactory, - MiniLcmApiStringNormalizationWrapperFactory normalizationWrapperFactory) + MiniLcmApiValidationWrapperFactory validationWrapperFactory) { public record DryRunSyncResult( int CrdtChanges, @@ -63,8 +61,10 @@ private async Task SyncOrImportInternal(IMiniLcmApi crdtApi, IMiniLc throw new InvalidOperationException("Project sync state does not match presence of snapshot."); } - crdtApi = normalizationWrapperFactory.Create(validationWrapperFactory.Create(crdtApi)); - fwdataApi = normalizationWrapperFactory.Create(validationWrapperFactory.Create(fwdataApi)); + // No write normalization: Data is already normalised on both sides. + // No query normalization: The sync doesn't do any querying. + crdtApi = validationWrapperFactory.Create(crdtApi); + fwdataApi = validationWrapperFactory.Create(fwdataApi); if (dryRun) { diff --git a/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs b/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs index 10f58081ba..a339270bbb 100644 --- a/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs +++ b/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs @@ -5,8 +5,6 @@ using MiniLcm; using MiniLcm.Media; using MiniLcm.Models; -using MiniLcm.Normalization; -using MiniLcm.Validators; using MiniLcm.Wrappers; using Reinforced.Typings.Attributes; @@ -18,11 +16,10 @@ public class MiniLcmJsInvokable( IProjectIdentifier project, ILogger logger, MiniLcmApiNotifyWrapperFactory notificationWrapperFactory, - MiniLcmApiValidationWrapperFactory validationWrapperFactory, - MiniLcmApiStringNormalizationWrapperFactory normalizationWrapperFactory + MiniLcmApiUserFacingWrappers userFacingWrappers ) : IDisposable { - private readonly IMiniLcmApi _wrappedApi = api.WrapWith([normalizationWrapperFactory, validationWrapperFactory, notificationWrapperFactory], project); + private readonly IMiniLcmApi _wrappedApi = userFacingWrappers.Apply(api, project, notificationWrapperFactory); public record MiniLcmFeatures(bool? History, bool? Write, bool? OpenWithFlex, bool? Feedback, bool? Sync, bool? Audio, bool? CustomViews); private bool SupportsSync => project.DataFormat == ProjectDataFormat.Harmony && api is CrdtMiniLcmApi; diff --git a/backend/FwLite/FwLiteWeb/Hubs/CrdtMiniLcmApiHub.cs b/backend/FwLite/FwLiteWeb/Hubs/CrdtMiniLcmApiHub.cs index f21ecd36cc..f8559f6ee0 100644 --- a/backend/FwLite/FwLiteWeb/Hubs/CrdtMiniLcmApiHub.cs +++ b/backend/FwLite/FwLiteWeb/Hubs/CrdtMiniLcmApiHub.cs @@ -1,15 +1,13 @@ -using FwLiteShared; using FwLiteShared.Events; using FwLiteShared.Projects; using FwLiteShared.Sync; using LcmCrdt; using LcmCrdt.Data; -using FwLiteWeb.Services; using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Caching.Memory; using MiniLcm; using MiniLcm.Models; -using MiniLcm.Validators; +using MiniLcm.Wrappers; using SystemTextJsonPatch; namespace FwLiteWeb.Hubs; @@ -23,8 +21,8 @@ public class CrdtMiniLcmApiHub( LexboxProjectService lexboxProjectService, IMemoryCache memoryCache, IHubContext hubContext, - MiniLcmApiValidationWrapperFactory validationWrapperFactory -) : MiniLcmApiHubBase(miniLcmApi, validationWrapperFactory) + MiniLcmApiUserFacingWrappers userFacingWrappers +) : MiniLcmApiHubBase(miniLcmApi, userFacingWrappers, projectContext.Project) { public const string ProjectRouteKey = "project"; public static string ProjectGroup(string projectName) => "crdt-" + projectName; diff --git a/backend/FwLite/FwLiteWeb/Hubs/FwDataMiniLcmHub.cs b/backend/FwLite/FwLiteWeb/Hubs/FwDataMiniLcmHub.cs index c392f77017..d71735f7fd 100644 --- a/backend/FwLite/FwLiteWeb/Hubs/FwDataMiniLcmHub.cs +++ b/backend/FwLite/FwLiteWeb/Hubs/FwDataMiniLcmHub.cs @@ -1,10 +1,7 @@ using FwDataMiniLcmBridge; -using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Options; using MiniLcm; -using MiniLcm.Validators; +using MiniLcm.Wrappers; using SIL.LCModel; -using SystemTextJsonPatch; namespace FwLiteWeb.Hubs; @@ -13,8 +10,9 @@ public class FwDataMiniLcmHub( IMiniLcmApi miniLcmApi, FwDataFactory fwDataFactory, FwDataProjectContext context, - MiniLcmApiValidationWrapperFactory validationWrapperFactory -) : MiniLcmApiHubBase(miniLcmApi, validationWrapperFactory) + MiniLcmApiUserFacingWrappers userFacingWrappers +) +: MiniLcmApiHubBase(miniLcmApi, userFacingWrappers, context.Project ?? throw new InvalidOperationException("No project is set in the context.")) { public const string ProjectRouteKey = "fwdata"; public override async Task OnConnectedAsync() diff --git a/backend/FwLite/FwLiteWeb/Hubs/MiniLcmApiHubBase.cs b/backend/FwLite/FwLiteWeb/Hubs/MiniLcmApiHubBase.cs index 5c3b962423..071a1fc338 100644 --- a/backend/FwLite/FwLiteWeb/Hubs/MiniLcmApiHubBase.cs +++ b/backend/FwLite/FwLiteWeb/Hubs/MiniLcmApiHubBase.cs @@ -1,16 +1,17 @@ using Microsoft.AspNetCore.SignalR; -using Microsoft.Extensions.Options; using MiniLcm; using MiniLcm.Models; -using MiniLcm.Validators; +using MiniLcm.Wrappers; using SystemTextJsonPatch; namespace FwLiteWeb.Hubs; -public abstract class MiniLcmApiHubBase(IMiniLcmApi miniLcmApi, - MiniLcmApiValidationWrapperFactory validationWrapperFactory) : Hub +public abstract class MiniLcmApiHubBase( + IMiniLcmApi miniLcmApi, + MiniLcmApiUserFacingWrappers userFacingWrappers, + IProjectIdentifier projectIdentifier) : Hub { - private readonly IMiniLcmApi _miniLcmApi = validationWrapperFactory.Create(miniLcmApi); + private readonly IMiniLcmApi _miniLcmApi = userFacingWrappers.Apply(miniLcmApi, projectIdentifier); public async Task GetWritingSystems() { diff --git a/backend/FwLite/FwLiteWeb/Routes/MiniLcmRoutes.cs b/backend/FwLite/FwLiteWeb/Routes/MiniLcmRoutes.cs index 8e84c19d4f..6161cc51bc 100644 --- a/backend/FwLite/FwLiteWeb/Routes/MiniLcmRoutes.cs +++ b/backend/FwLite/FwLiteWeb/Routes/MiniLcmRoutes.cs @@ -7,7 +7,7 @@ using MiniLcm.Filtering; using MiniLcm.Models; using MiniLcm.Project; -using MiniLcm.Validators; +using MiniLcm.Wrappers; namespace FwLiteWeb.Routes; @@ -86,12 +86,11 @@ public static IEndpointConventionBuilder MapMiniLcmRoutes(this IEndpointRouteBui return Results.Problem($"Project {projectCode} not found"); } - var validationWrapperFactory = context.HttpContext.RequestServices - .GetRequiredService(); + var services = context.HttpContext.RequestServices; + var userFacingWrappers = services.GetRequiredService(); - miniLcmHolder.MiniLcmApi = validationWrapperFactory.Create( - await projectProvider.OpenProject(project, context.HttpContext.RequestServices) - ); + miniLcmHolder.MiniLcmApi = userFacingWrappers.Apply( + await projectProvider.OpenProject(project, services), project); return await next(context); }); diff --git a/backend/FwLite/MiniLcm.Tests/CreateEntryTestsBase.cs b/backend/FwLite/MiniLcm.Tests/CreateEntryTestsBase.cs index 8ba898e353..8e309871d5 100644 --- a/backend/FwLite/MiniLcm.Tests/CreateEntryTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/CreateEntryTestsBase.cs @@ -20,7 +20,8 @@ public async Task CanCreateEntry_AutoFaker() var createdEntry = await Api.CreateEntry(entry); createdEntry.Should().BeEquivalentTo(entry, options => options .For(e => e.Components).Exclude(e => e.Id) - .For(e => e.ComplexForms).Exclude(e => e.Id)); + .For(e => e.ComplexForms).Exclude(e => e.Id) + .Excluding(member => member.Name == nameof(IOrderable.Order))); } [Fact] diff --git a/backend/FwLite/MiniLcm.Tests/ExampleSentenceTestsBase.cs b/backend/FwLite/MiniLcm.Tests/ExampleSentenceTestsBase.cs index 8c0c50e87d..f48e787dd0 100644 --- a/backend/FwLite/MiniLcm.Tests/ExampleSentenceTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/ExampleSentenceTestsBase.cs @@ -49,13 +49,21 @@ public async Task CanCreateExampleSentence() { var expectedExampleSentence = new ExampleSentence() { + Id = Guid.NewGuid(), SenseId = _senseId, Reference = new RichString("This is a reference", "en"), Sentence = { { "en", new RichString("test", "en") } }, - Translations = { new Translation() { Text = { { "en", new RichString("test", "en") } } } } + Translations = { + new Translation() + { + Id = Guid.NewGuid(), + Text = { { "en", new RichString("test", "en") } }, + } + } }; var actualSentence = await Api.CreateExampleSentence(_entryId, _senseId, expectedExampleSentence); - actualSentence.Should().BeEquivalentTo(expectedExampleSentence); + actualSentence.Should().BeEquivalentTo(expectedExampleSentence, + options => options.Excluding(s => s.Order)); } [Fact] @@ -63,10 +71,12 @@ public async Task CanCreateEmptyExampleSentence() { var expectedExampleSentence = new ExampleSentence() { + Id = Guid.NewGuid(), SenseId = _senseId }; var actualSentence = await Api.CreateExampleSentence(_entryId, _senseId, expectedExampleSentence); - actualSentence.Should().BeEquivalentTo(expectedExampleSentence); + actualSentence.Should().BeEquivalentTo(expectedExampleSentence, + options => options.Excluding(s => s.Order)); } [Fact] diff --git a/backend/FwLite/MiniLcm.Tests/Helpers/NfcTestData.cs b/backend/FwLite/MiniLcm.Tests/Helpers/NfcTestData.cs new file mode 100644 index 0000000000..61144e6a4f --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/Helpers/NfcTestData.cs @@ -0,0 +1,202 @@ +namespace MiniLcm.Tests.Helpers; + +public static class NfcTestData +{ + // U+00EF LATIN SMALL LETTER I WITH DIAERESIS (composed) + public const string Nfc = "naïve"; + // U+0069 LATIN SMALL LETTER I + U+0308 COMBINING DIAERESIS (decomposed) + public const string Nfd = "naïve"; + + // D: ï C: ï + + public static MultiString CreateNfcMultiString() + { + return new() { Values = { { "en", Nfc }, { "fr", Nfc } } }; + } + + public static RichString CreateNfcRichString() + { + return new([ + new RichSpan { Text = Nfc, Ws = "en" }, + new RichSpan { Text = Nfc, Ws = "en", Bold = RichTextToggle.On } + ]); + } + + public static RichMultiString CreateNfcRichMultiString() + { + return new() + { + { "en", CreateNfcRichString() }, + { "fr", CreateNfcRichString() } + }; + } + + public static WritingSystem CreateNfcWritingSystem() + { + return new() + { + Id = Guid.NewGuid(), + WsId = "en", + Type = WritingSystemType.Analysis, + Name = Nfc, + Abbreviation = Nfc, + Font = Nfc, + Exemplars = [Nfc, Nfc] + }; + } + + public static PartOfSpeech CreateNfcPartOfSpeech() + { + return new() + { + Id = Guid.NewGuid(), + Name = CreateNfcMultiString() + }; + } + + public static Publication CreateNfcPublication() + { + return new() + { + Id = Guid.NewGuid(), + Name = CreateNfcMultiString() + }; + } + + public static SemanticDomain CreateNfcSemanticDomain() + { + return new() + { + Id = Guid.NewGuid(), + Code = Nfc, + Name = CreateNfcMultiString() + }; + } + + public static ComplexFormType CreateNfcComplexFormType() + { + return new() + { + Id = Guid.NewGuid(), + Name = CreateNfcMultiString() + }; + } + + public static MorphType CreateNfcMorphType() + { + return new() + { + Id = Guid.NewGuid(), + Kind = MorphTypeKind.Stem, + Name = CreateNfcMultiString(), + Abbreviation = CreateNfcMultiString(), + Description = CreateNfcRichMultiString(), + Prefix = Nfc, + Postfix = Nfc + }; + } + + public static Translation CreateNfcTranslation() + { + return new() + { + Id = Guid.NewGuid(), + Text = CreateNfcRichMultiString() + }; + } + + public static ExampleSentence CreateNfcExampleSentence() + { + return new() + { + Id = Guid.NewGuid(), + Sentence = CreateNfcRichMultiString(), + Reference = CreateNfcRichString() + }; + } + + public static ExampleSentence CreateNfcExampleSentenceWithTranslations() + { + return new() + { + Id = Guid.NewGuid(), + Sentence = CreateNfcRichMultiString(), + Reference = CreateNfcRichString(), + Translations = [CreateNfcTranslation(), CreateNfcTranslation()] + }; + } + + public static Sense CreateNfcSense() + { + return new() + { + Id = Guid.NewGuid(), + Gloss = CreateNfcMultiString(), + Definition = CreateNfcRichMultiString() + }; + } + + public static Sense CreateNfcSenseWithExamples() + { + return new() + { + Id = Guid.NewGuid(), + Gloss = CreateNfcMultiString(), + Definition = CreateNfcRichMultiString(), + SemanticDomains = [CreateNfcSemanticDomain()], + PartOfSpeech = CreateNfcPartOfSpeech(), + ExampleSentences = [CreateNfcExampleSentenceWithTranslations()] + }; + } + + public static ComplexFormComponent CreateNfcComplexFormComponent() + { + return new() + { + Id = Guid.NewGuid(), + ComplexFormEntryId = Guid.NewGuid(), + ComponentEntryId = Guid.NewGuid(), + ComplexFormHeadword = Nfc, + ComponentHeadword = Nfc + }; + } + + public static Entry CreateNfcEntry() + { + return new() + { + Id = Guid.NewGuid(), + LexemeForm = CreateNfcMultiString(), + CitationForm = CreateNfcMultiString(), + LiteralMeaning = CreateNfcRichMultiString(), + Note = CreateNfcRichMultiString() + }; + } + + public static Entry CreateNfcEntryWithSenses() + { + return new() + { + Id = Guid.NewGuid(), + LexemeForm = CreateNfcMultiString(), + CitationForm = CreateNfcMultiString(), + LiteralMeaning = CreateNfcRichMultiString(), + Note = CreateNfcRichMultiString(), + Senses = [CreateNfcSenseWithExamples()] + }; + } + + public static Entry CreateNfcEntryWithComponents() + { + return new() + { + Id = Guid.NewGuid(), + LexemeForm = CreateNfcMultiString(), + CitationForm = CreateNfcMultiString(), + LiteralMeaning = CreateNfcRichMultiString(), + Note = CreateNfcRichMultiString(), + Components = [CreateNfcComplexFormComponent()], + ComplexForms = [CreateNfcComplexFormComponent()] + }; + } +} diff --git a/backend/FwLite/MiniLcm.Tests/Helpers/NormalizationAssert.cs b/backend/FwLite/MiniLcm.Tests/Helpers/NormalizationAssert.cs new file mode 100644 index 0000000000..a0b048464b --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/Helpers/NormalizationAssert.cs @@ -0,0 +1,137 @@ +using System.Collections; +using System.Reflection; +using System.Text; + +namespace MiniLcm.Tests.Helpers; + +/// +/// For verifying that every string-bearing property of an object is normalized (NFC or NFD). +/// +public static class NormalizationAssert +{ + private static readonly Dictionary> SkippedProperties = new() + { + // WritingSystem fields are generally LDML-managed and seem to not be normalized by FieldWorks + [typeof(WritingSystem)] = + [ + nameof(WritingSystem.WsId), + nameof(WritingSystem.Name), + nameof(WritingSystem.Abbreviation), + nameof(WritingSystem.Font), + nameof(WritingSystem.Exemplars), + ], + }; + + public static void AssertAllNfc(object? obj) + { + Assert(obj, NormalizationForm.FormC); + } + + public static void AssertAllNfd(object? obj) + { + Assert(obj, NormalizationForm.FormD); + } + + public static bool IsAllNfd(object obj) + { + return FindIssues(obj, NormalizationForm.FormD).Count == 0; + } + + private static void Assert(object? obj, NormalizationForm form) + { + if (obj is null) throw new Xunit.Sdk.XunitException("Expected object to be non-null but was null"); + var issues = FindIssues(obj, form); + if (issues.Count == 0) return; + var name = FormName(form); + throw new Xunit.Sdk.XunitException( + $"Expected all normalizable properties to contain {name} strings, but found issues:\n" + + string.Join("\n", issues.Select(i => " - " + i)) + ); + } + + private static List FindIssues(object obj, NormalizationForm form) + { + var issues = new List(); + Visit(obj, "", form, issues); + return issues; + } + + private static void Visit(object? obj, string path, NormalizationForm form, List issues) + { + switch (obj) + { + case null: + return; + case string s: + CheckString(s, path, form, issues); + return; + case MultiString ms: + if (ms.Values.Count == 0) issues.Add($"{path}: MultiString has no values (must have at least one for testing)"); + foreach (var (key, value) in ms.Values) CheckString(value, $"{path}.Values[{key}]", form, issues); + return; + case RichString rs: + if (rs.Spans.Count == 0) issues.Add($"{path}: RichString has no spans (must have at least one for testing)"); + for (var i = 0; i < rs.Spans.Count; i++) CheckString(rs.Spans[i].Text, $"{path}.Spans[{i}].Text", form, issues); + return; + case RichMultiString rms: + if (rms.Count == 0) issues.Add($"{path}: RichMultiString has no values (must have at least one for testing)"); + foreach (var (key, value) in rms) Visit(value, $"{path}[{key}]", form, issues); + return; + case IEnumerable seq: + var i2 = 0; + foreach (var item in seq) Visit(item, $"{path}[{i2++}]", form, issues); + return; + default: + break; + } + + VisitModelProperties(obj, path, form, issues); + } + + private static void VisitModelProperties(object obj, string path, NormalizationForm form, List issues) + { + var type = obj.GetType(); + if (type.Namespace?.StartsWith("MiniLcm.Models", StringComparison.Ordinal) != true) + throw new Xunit.Sdk.XunitException($"Unexpected type {type.FullName} at {(string.IsNullOrEmpty(path) ? "" : path)}"); + + var skipped = SkippedProperties.GetValueOrDefault(type); + foreach (var prop in type.GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (!prop.CanRead || skipped?.Contains(prop.Name) == true) continue; + if (IsIgnoredType(prop.PropertyType)) continue; + + var value = prop.GetValue(obj); + if (value is null) continue; + + var propPath = string.IsNullOrEmpty(path) ? prop.Name : $"{path}.{prop.Name}"; + Visit(value, propPath, form, issues); + } + } + + private static bool IsIgnoredType(Type type) + { + var underlying = Nullable.GetUnderlyingType(type) ?? type; + return underlying.IsPrimitive || underlying.IsEnum || + underlying == typeof(Guid) || underlying == typeof(DateTime) || + underlying == typeof(DateTimeOffset) || underlying == typeof(decimal); + } + + private static void CheckString(string? value, string path, NormalizationForm form, List issues) + { + if (string.IsNullOrEmpty(value)) + { + issues.Add($"{path}: string is null or empty (must have a value for testing)"); + return; + } + if (!value.IsNormalized(form)) + { + var name = FormName(form); + issues.Add($"{path}: expected {name} but \"{value}\" is not {name}-normalized"); + } + } + + private static string FormName(NormalizationForm form) + { + return form == NormalizationForm.FormC ? "NFC" : "NFD"; + } +} diff --git a/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs b/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs index 1b3d255600..09b0dd50c9 100644 --- a/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs +++ b/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs @@ -17,7 +17,10 @@ public virtual async Task InitializeAsync() { BaseApi = await NewApi(); BaseApi.Should().NotBeNull(); - Api = BaseApi.WrapWith([new MiniLcmApiStringNormalizationWrapperFactory()], null!); + Api = BaseApi.WrapWith([ + new MiniLcmApiQueryNormalizationWrapperFactory(), + new MiniLcmApiWriteNormalizationWrapperFactory(), + ], null!); Api.Should().NotBeNull(); } diff --git a/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs b/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs index 4feff444c8..8038352c71 100644 --- a/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs +++ b/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs @@ -33,14 +33,14 @@ public NormalizationTests() { MockApi = Mock.Of(); // Mock.Get(MockApi).Setup(api => api.SearchEntries(It.IsAny(), null)).Returns(new List().ToAsyncEnumerable()); - var factory = new MiniLcmApiStringNormalizationWrapperFactory(); + var factory = new MiniLcmApiQueryNormalizationWrapperFactory(); NormalizingApi = factory.Create(MockApi); } [Fact] public void SearchEntriesIsNormalized() { - NormalizingApi.Should().BeOfType(); + NormalizingApi.Should().BeOfType(); var results = NormalizingApi.SearchEntries(NFCString, null); Mock.Get(MockApi).Verify(api => api.SearchEntries(NFDString, null)); } @@ -48,7 +48,7 @@ public void SearchEntriesIsNormalized() [Fact] public void SearchEntriesWithQueryOptionsAreNormalized() { - NormalizingApi.Should().BeOfType(); + NormalizingApi.Should().BeOfType(); var results = NormalizingApi.SearchEntries(NFCString, NFCQueryOptions); Mock.Get(MockApi).Verify(api => api.SearchEntries(NFDString, It.Is( opt => opt.Exemplar!.Value == NFDOptions.Exemplar!.Value && @@ -58,7 +58,7 @@ public void SearchEntriesWithQueryOptionsAreNormalized() [Fact] public void CountEntriesIsNormalized() { - NormalizingApi.Should().BeOfType(); + NormalizingApi.Should().BeOfType(); var results = NormalizingApi.CountEntries(NFCString, null); Mock.Get(MockApi).Verify(api => api.CountEntries(NFDString, null)); } @@ -66,7 +66,7 @@ public void CountEntriesIsNormalized() [Fact] public void CountEntriesWithFilterQueryOptionsIsNormalized() { - NormalizingApi.Should().BeOfType(); + NormalizingApi.Should().BeOfType(); var results = NormalizingApi.CountEntries(NFCString, NFCOptions); Mock.Get(MockApi).Verify(api => api.CountEntries(NFDString, It.Is( opt => opt.Exemplar!.Value == NFDOptions.Exemplar!.Value && @@ -76,7 +76,7 @@ public void CountEntriesWithFilterQueryOptionsIsNormalized() [Fact] public void GetEntriesIsNormalized() { - NormalizingApi.Should().BeOfType(); + NormalizingApi.Should().BeOfType(); var results = NormalizingApi.GetEntries(NFCQueryOptions); Mock.Get(MockApi).Verify(api => api.GetEntries(It.Is( opt => opt.Exemplar!.Value == NFDOptions.Exemplar!.Value && diff --git a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs index bc37968262..022e92eb65 100644 --- a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs @@ -410,12 +410,10 @@ public async Task SuccessfulMatches(string searchTerm, string word, bool identic { // identical is to make the test cases more readable when they only differ in their normalization (searchTerm == word).Should().Be(identical); - // remove next line in https://github.com/sillsdev/languageforge-lexbox/issues/2065 - word = word.Normalize(NormalizationForm.FormD); await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } }); var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync(); words.Should().NotBeEmpty(); - // Like LicLCM the MiniLcm API should normalize to NFD + // The API normalizes to NFD on write, so results should be NFD regardless of input words.Should().Contain(word.Normalize(NormalizationForm.FormD)); } @@ -427,12 +425,10 @@ public async Task SuccessfulMatches(string searchTerm, string word, bool identic [InlineData("É", "È")] // Different accents public async Task NegativeMatches(string searchTerm, string word) { - word = word.Normalize(NormalizationForm.FormD); - //should we be normalizing the search term internally? - searchTerm = searchTerm.Normalize(NormalizationForm.FormD); await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } }); var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync(); words.Should().NotContain(word); + words.Should().NotContain(word.Normalize(NormalizationForm.FormD)); } [Theory] diff --git a/backend/FwLite/MiniLcm.Tests/SortingTestsBase.cs b/backend/FwLite/MiniLcm.Tests/SortingTestsBase.cs index 48ab0f8f97..c4f920af6c 100644 --- a/backend/FwLite/MiniLcm.Tests/SortingTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/SortingTestsBase.cs @@ -143,7 +143,7 @@ static Entry[] CreateSortedEntrySet(string headword) var lastLongestContainsMatches = CreateSortedEntrySet("ccaaaa"); var entryId = Guid.NewGuid(); - Entry nonHeadwordMatch = new() { Id = entryId, Senses = [new() { EntryId = entryId, Gloss = { ["en"] = "aaaa" } }] }; + Entry nonHeadwordMatch = new() { Id = entryId, Senses = [new() { Id = Guid.NewGuid(), EntryId = entryId, Gloss = { ["en"] = "aaaa" } }] }; Entry[] expected = [ .. exactMatches, @@ -167,10 +167,9 @@ static Entry[] CreateSortedEntrySet(string headword) .Where(e => ids.Contains(e.Id)) .ToList(); - results.Should().BeEquivalentTo(expected, - options => options); - results.Should().BeEquivalentTo(expected, - options => options.WithStrictOrdering()); + results.Should().BeEquivalentTo(expected, options => options + .WithStrictOrdering() + .For(e => e.Senses).Exclude(s => s.Order)); } [Theory] @@ -201,7 +200,7 @@ static Entry[] CreateSortedEntrySet(string headword) var lastLongestContainsMatches = CreateSortedEntrySet("ccaaaa"); var entryId = Guid.NewGuid(); - Entry nonHeadwordMatch = new() { Id = entryId, Senses = [new() { EntryId = entryId, Gloss = { ["en"] = "aaaa" } }] }; + Entry nonHeadwordMatch = new() { Id = entryId, Senses = [new() { Id = Guid.NewGuid(), EntryId = entryId, Gloss = { ["en"] = "aaaa" } }] }; Entry[] expected = [ .. exactMatches, @@ -225,10 +224,9 @@ static Entry[] CreateSortedEntrySet(string headword) .Where(e => ids.Contains(e.Id)) .ToList(); - results.Should().BeEquivalentTo(expected, - options => options); - results.Should().BeEquivalentTo(expected, - options => options.WithStrictOrdering()); + results.Should().BeEquivalentTo(expected, options => options + .WithStrictOrdering() + .For(e => e.Senses).Exclude(s => s.Order)); } [Theory] @@ -260,8 +258,6 @@ public async Task SecondaryOrder_Headword_LexemeForm(string searchTerm) .Where(e => ids.Contains(e.Id)) .ToList(); - results.Should().BeEquivalentTo(expected, - options => options); results.Should().BeEquivalentTo(expected, options => options.WithStrictOrdering()); } @@ -295,8 +291,6 @@ public async Task SecondaryOrder_Headword_CitationForm(string searchTerm) .Where(e => ids.Contains(e.Id)) .ToList(); - results.Should().BeEquivalentTo(expected, - options => options); results.Should().BeEquivalentTo(expected, options => options.WithStrictOrdering()); } diff --git a/backend/FwLite/MiniLcm.Tests/UpdateEntryTestsBase.cs b/backend/FwLite/MiniLcm.Tests/UpdateEntryTestsBase.cs index de2f12728f..f7d8663f63 100644 --- a/backend/FwLite/MiniLcm.Tests/UpdateEntryTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/UpdateEntryTestsBase.cs @@ -169,7 +169,9 @@ public async Task UpdateEntry_CanReorderSenses(string before, string after, stri afterEntry.Senses = afterSenses; // sanity checks - beforeEntry.Senses.Should().BeEquivalentTo(beforeSenses, options => options.WithStrictOrdering()); + beforeEntry.Senses.Should().BeEquivalentTo(beforeSenses, options => options + .WithStrictOrdering() + .Excluding(s => s.Order)); if (!ApiUsesImplicitOrdering) { beforeEntry.Senses.Select(s => s.Order).Should() @@ -231,7 +233,9 @@ public async Task UpdateEntry_CanReorderExampleSentence(string before, string af afterSense.ExampleSentences = afterExamples; // sanity checks - beforeSense.ExampleSentences.Should().BeEquivalentTo(beforeExamples, options => options.WithStrictOrdering()); + beforeSense.ExampleSentences.Should().BeEquivalentTo(beforeExamples, options => options + .WithStrictOrdering() + .Excluding(s => s.Order)); if (!ApiUsesImplicitOrdering) { beforeSense.ExampleSentences.Select(s => s.Order).Should() @@ -309,7 +313,8 @@ public async Task UpdateEntry_CanReorderComponents(string before, string after, // sanity checks beforeEntry.Components.Should().BeEquivalentTo(beforeComponents, options => options .WithStrictOrdering() - .Excluding(c => c.Id)); + .Excluding(c => c.Id) + .Excluding(c => c.Order)); if (!ApiUsesImplicitOrdering) { beforeEntry.Components.Select(s => s.Order).Should() diff --git a/backend/FwLite/MiniLcm.Tests/WriteNormalizationTests.cs b/backend/FwLite/MiniLcm.Tests/WriteNormalizationTests.cs new file mode 100644 index 0000000000..78e7ed066d --- /dev/null +++ b/backend/FwLite/MiniLcm.Tests/WriteNormalizationTests.cs @@ -0,0 +1,783 @@ +using System.Text.Json; +using MiniLcm.Normalization; +using MiniLcm.SyncHelpers; +using Moq; +using SystemTextJsonPatch.Operations; +using MiniLcm.Tests.Helpers; +using static MiniLcm.Tests.Helpers.NormalizationAssert; + +namespace MiniLcm.Tests; + +public class WriteNormalizationTests +{ + private readonly IMiniLcmApi _mockApi; + private readonly IMiniLcmApi _normalizingApi; + + public WriteNormalizationTests() + { + _mockApi = Mock.Of(); + var factory = new MiniLcmApiWriteNormalizationWrapperFactory(); + _normalizingApi = factory.Create(_mockApi); + } + + #region WritingSystem Tests + + // WritingSystem.{Name, Abbreviation, Font, Exemplars} are plain strings; in liblcm they are + // LDML-managed by WritingSystemManager rather than stored as TsString, so the wrapper passes + // them through unchanged (no NFD normalization). + + [Fact] + public async Task CreateWritingSystem_DoesNotNormalize() + { + var ws = NfcTestData.CreateNfcWritingSystem(); + + WritingSystem? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateWritingSystem(It.IsAny(), It.IsAny?>())) + .Callback?>((w, _) => captured = w) + .ReturnsAsync(ws); + + await _normalizingApi.CreateWritingSystem(ws); + + captured.Should().NotBeNull(); + captured.Name.Should().Be(NfcTestData.Nfc); + captured.Abbreviation.Should().Be(NfcTestData.Nfc); + captured.Font.Should().Be(NfcTestData.Nfc); + captured.Exemplars.Should().Equal(NfcTestData.Nfc, NfcTestData.Nfc); + } + + [Fact] + public async Task UpdateWritingSystem_BeforeAfter_DoesNotNormalize() + { + var before = NfcTestData.CreateNfcWritingSystem(); + var after = NfcTestData.CreateNfcWritingSystem(); + + WritingSystem? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateWritingSystem(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, a, _) => captured = a) + .ReturnsAsync(after); + + await _normalizingApi.UpdateWritingSystem(before, after); + + captured.Should().NotBeNull(); + captured.Name.Should().Be(NfcTestData.Nfc); + captured.Abbreviation.Should().Be(NfcTestData.Nfc); + captured.Font.Should().Be(NfcTestData.Nfc); + captured.Exemplars.Should().Equal(NfcTestData.Nfc, NfcTestData.Nfc); + } + + #endregion + + #region PartOfSpeech Tests + + [Fact] + public async Task CreatePartOfSpeech_NormalizesToNfd() + { + var pos = NfcTestData.CreateNfcPartOfSpeech(); + + PartOfSpeech? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreatePartOfSpeech(It.IsAny())) + .Callback(p => captured = p) + .ReturnsAsync(pos); + + await _normalizingApi.CreatePartOfSpeech(pos); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task UpdatePartOfSpeech_BeforeAfter_NormalizesBothToNfd() + { + var before = NfcTestData.CreateNfcPartOfSpeech(); + var after = NfcTestData.CreateNfcPartOfSpeech(); + + PartOfSpeech? capturedBefore = null; + PartOfSpeech? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdatePartOfSpeech(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdatePartOfSpeech(before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + #endregion + + #region Publication Tests + + [Fact] + public async Task CreatePublication_NormalizesToNfd() + { + var pub = NfcTestData.CreateNfcPublication(); + + Publication? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreatePublication(It.IsAny())) + .Callback(p => captured = p) + .ReturnsAsync(pub); + + await _normalizingApi.CreatePublication(pub); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task UpdatePublication_BeforeAfter_NormalizesBothToNfd() + { + var before = NfcTestData.CreateNfcPublication(); + var after = NfcTestData.CreateNfcPublication(); + + Publication? capturedBefore = null; + Publication? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdatePublication(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdatePublication(before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + #endregion + + #region SemanticDomain Tests + + [Fact] + public async Task CreateSemanticDomain_NormalizesToNfd() + { + var sd = NfcTestData.CreateNfcSemanticDomain(); + + SemanticDomain? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateSemanticDomain(It.IsAny())) + .Callback(s => captured = s) + .ReturnsAsync(sd); + + await _normalizingApi.CreateSemanticDomain(sd); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task UpdateSemanticDomain_BeforeAfter_NormalizesBothToNfd() + { + var before = NfcTestData.CreateNfcSemanticDomain(); + var after = NfcTestData.CreateNfcSemanticDomain(); + + SemanticDomain? capturedBefore = null; + SemanticDomain? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateSemanticDomain(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdateSemanticDomain(before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + [Fact] + public async Task AddSemanticDomainToSense_NormalizesToNfd() + { + var sd = NfcTestData.CreateNfcSemanticDomain(); + + SemanticDomain? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.AddSemanticDomainToSense(It.IsAny(), It.IsAny())) + .Callback((_, s) => captured = s) + .Returns(Task.CompletedTask); + + await _normalizingApi.AddSemanticDomainToSense(Guid.NewGuid(), sd); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task BulkImportSemanticDomains_NormalizesToNfd() + { + var domains = new[] { NfcTestData.CreateNfcSemanticDomain(), NfcTestData.CreateNfcSemanticDomain() }; + + var captured = new List(); + Mock.Get(_mockApi) + .Setup(api => api.BulkImportSemanticDomains(It.IsAny>())) + .Returns(async (IAsyncEnumerable stream) => + { + await foreach (var sd in stream) captured.Add(sd); + }); + + await _normalizingApi.BulkImportSemanticDomains(domains.ToAsyncEnumerable()); + + captured.Should().HaveCount(2); + AssertAllNfd(captured); + } + + #endregion + + #region ComplexFormType Tests + + [Fact] + public async Task CreateComplexFormType_NormalizesToNfd() + { + var cft = NfcTestData.CreateNfcComplexFormType(); + + ComplexFormType? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateComplexFormType(It.IsAny())) + .Callback(c => captured = c) + .ReturnsAsync(cft); + + await _normalizingApi.CreateComplexFormType(cft); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task UpdateComplexFormType_BeforeAfter_NormalizesBothToNfd() + { + var before = NfcTestData.CreateNfcComplexFormType(); + var after = NfcTestData.CreateNfcComplexFormType(); + + ComplexFormType? capturedBefore = null; + ComplexFormType? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateComplexFormType(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdateComplexFormType(before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + #endregion + + #region MorphType Tests + + [Fact] + public async Task UpdateMorphType_BeforeAfter_NormalizesBothToNfd() + { + var before = NfcTestData.CreateNfcMorphType(); + var after = NfcTestData.CreateNfcMorphType(); + + MorphType? capturedBefore = null; + MorphType? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateMorphType(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdateMorphType(before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + [Fact] + public async Task UpdateMorphType_JsonPatch_NormalizesToNfd() + { + var update = new UpdateObjectInput() + .Set(m => m.Name, NfcTestData.CreateNfcMultiString()) + .Set(m => m.Prefix, NfcTestData.Nfc) + .Set(m => m.Postfix, NfcTestData.Nfc); + + UpdateObjectInput? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateMorphType(It.IsAny(), It.IsAny>())) + .Callback>((_, patch) => captured = patch) + .ReturnsAsync(NfcTestData.CreateNfcMorphType()); + + await _normalizingApi.UpdateMorphType(Guid.NewGuid(), update); + + captured.Should().NotBeNull(); + var byPath = captured.Patch.Operations.ToDictionary(o => o.Path!, o => o.Value); + AssertAllNfd(byPath["/Name"].Should().BeOfType().Subject); + byPath["/Prefix"].Should().Be(NfcTestData.Nfd); + byPath["/Postfix"].Should().Be(NfcTestData.Nfd); + } + + #endregion + + #region Entry Tests + + [Fact] + public async Task CreateEntry_NormalizesToNfd() + { + var entry = NfcTestData.CreateNfcEntry(); + + Entry? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateEntry(It.IsAny(), It.IsAny())) + .Callback((e, _) => captured = e) + .ReturnsAsync(entry); + + await _normalizingApi.CreateEntry(entry); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task UpdateEntry_BeforeAfter_NormalizesBothToNfd() + { + var before = NfcTestData.CreateNfcEntry(); + var after = NfcTestData.CreateNfcEntry(); + + Entry? capturedBefore = null; + Entry? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateEntry(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdateEntry(before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + [Fact] + public async Task CreateEntry_WithNestedSenses_NormalizesToNfd() + { + var entry = NfcTestData.CreateNfcEntryWithSenses(); + + Entry? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateEntry(It.IsAny(), It.IsAny())) + .Callback((e, _) => captured = e) + .ReturnsAsync(entry); + + await _normalizingApi.CreateEntry(entry); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task CreateEntry_WithComplexFormComponents_NormalizesToNfd() + { + var entry = NfcTestData.CreateNfcEntryWithComponents(); + + Entry? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateEntry(It.IsAny(), It.IsAny())) + .Callback((e, _) => captured = e) + .ReturnsAsync(entry); + + await _normalizingApi.CreateEntry(entry); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task BulkCreateEntries_NormalizesToNfd() + { + var entries = new[] { NfcTestData.CreateNfcEntry(), NfcTestData.CreateNfcEntryWithSenses() }; + + var captured = new List(); + Mock.Get(_mockApi) + .Setup(api => api.BulkCreateEntries(It.IsAny>())) + .Returns(async (IAsyncEnumerable stream) => + { + await foreach (var e in stream) captured.Add(e); + }); + + await _normalizingApi.BulkCreateEntries(entries.ToAsyncEnumerable()); + + captured.Should().HaveCount(2); + AssertAllNfd(captured); + } + + #endregion + + #region JsonPatch Tests + + [Fact] + public async Task UpdateEntry_JsonPatch_NormalizesValuesToNfd() + { + var update = new UpdateObjectInput() + .Set(e => e.LexemeForm["en"], NfcTestData.Nfc) + .Set(e => e.CitationForm, NfcTestData.CreateNfcMultiString()) + .Set(e => e.Note["en"], NfcTestData.CreateNfcRichString()) + .Set(e => e.LiteralMeaning, NfcTestData.CreateNfcRichMultiString()); + + UpdateObjectInput? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateEntry(It.IsAny(), It.IsAny>())) + .Callback>((_, patch) => captured = patch) + .ReturnsAsync(NfcTestData.CreateNfcEntry()); + + await _normalizingApi.UpdateEntry(Guid.NewGuid(), update); + + captured.Should().NotBeNull(); + captured.Should().NotBeSameAs(update); // rebuilt because Entry path normalizes + AssertAllPatchValuesNfd(captured); + } + + [Fact] + public async Task UpdateEntry_JsonPatch_NormalizesJsonElementString() + { + using var document = JsonDocument.Parse($"\"{NfcTestData.Nfc}\""); + var update = new UpdateObjectInput(); + update.Patch.Operations.Add(new Operation("replace", "/LexemeForm/en", null, document.RootElement)); + + UpdateObjectInput? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateEntry(It.IsAny(), It.IsAny>())) + .Callback>((_, patch) => captured = patch) + .ReturnsAsync(NfcTestData.CreateNfcEntry()); + + await _normalizingApi.UpdateEntry(Guid.NewGuid(), update); + + captured.Should().NotBeNull(); + captured.Patch.Operations.Single().Value + .Should().BeOfType() + .Which.Should().Be(NfcTestData.Nfd); + } + + [Fact] + public async Task UpdateWritingSystem_JsonPatch_DoesNotNormalize() + { + var update = new UpdateObjectInput() + .Set(ws => ws.Name, NfcTestData.Nfc) + .Set(ws => ws.Exemplars, [NfcTestData.Nfc, NfcTestData.Nfc]); + + UpdateObjectInput? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateWritingSystem(It.IsAny(), It.IsAny(), + It.IsAny>())) + .Callback>((_, _, patch) => + captured = patch) + .ReturnsAsync(NfcTestData.CreateNfcWritingSystem()); + + await _normalizingApi.UpdateWritingSystem("en", WritingSystemType.Analysis, update); + + captured.Should().BeSameAs(update); // pass-through, not rebuilt + var byPath = captured.Patch.Operations.ToDictionary(o => o.Path!, o => o.Value); + byPath.Should().HaveCount(2); + byPath["/Name"].Should().Be(NfcTestData.Nfc); + byPath["/Exemplars"].Should().BeOfType() + .Which.Should().Equal(NfcTestData.Nfc, NfcTestData.Nfc); + } + + /// + /// Asserts every non-null operation value in the patch is NFD. Delegates to + /// , which already understands string, + /// string[], MultiString, RichString, and RichMultiString — so failures report a property path. + /// + private static void AssertAllPatchValuesNfd(UpdateObjectInput update) where T : class + { + update.Patch.Operations.Should().NotBeEmpty(); + foreach (var op in update.Patch.Operations) + { + if (op.Value is not null) AssertAllNfd(op.Value); + } + } + + #endregion + + #region ComplexFormComponent Tests + + [Fact] + public async Task CreateComplexFormComponent_NormalizesToNfd() + { + var cfc = NfcTestData.CreateNfcComplexFormComponent(); + + ComplexFormComponent? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateComplexFormComponent( + It.IsAny(), It.IsAny?>())) + .Callback?>((c, _) => captured = c) + .ReturnsAsync(cfc); + + await _normalizingApi.CreateComplexFormComponent(cfc); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + #endregion + + #region Sense Tests + + [Fact] + public async Task CreateSense_NormalizesToNfd() + { + var sense = NfcTestData.CreateNfcSense(); + + Sense? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateSense(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, s, _) => captured = s) + .ReturnsAsync(sense); + + await _normalizingApi.CreateSense(Guid.NewGuid(), sense); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task UpdateSense_BeforeAfter_NormalizesBothToNfd() + { + var entryId = Guid.NewGuid(); + var before = NfcTestData.CreateNfcSense(); + var after = NfcTestData.CreateNfcSense(); + + Sense? capturedBefore = null; + Sense? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateSense(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdateSense(entryId, before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + [Fact] + public async Task CreateSense_WithNestedExampleSentences_NormalizesToNfd() + { + var sense = NfcTestData.CreateNfcSenseWithExamples(); + + Sense? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateSense(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, s, _) => captured = s) + .ReturnsAsync(sense); + + await _normalizingApi.CreateSense(Guid.NewGuid(), sense); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + #endregion + + #region ExampleSentence Tests + + [Fact] + public async Task CreateExampleSentence_NormalizesToNfd() + { + var example = NfcTestData.CreateNfcExampleSentence(); + + ExampleSentence? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateExampleSentence( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, _, e, _) => captured = e) + .ReturnsAsync(example); + + await _normalizingApi.CreateExampleSentence(Guid.NewGuid(), Guid.NewGuid(), example); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + [Fact] + public async Task UpdateExampleSentence_BeforeAfter_NormalizesBothToNfd() + { + var entryId = Guid.NewGuid(); + var senseId = Guid.NewGuid(); + var before = NfcTestData.CreateNfcExampleSentence(); + var after = NfcTestData.CreateNfcExampleSentence(); + + ExampleSentence? capturedBefore = null; + ExampleSentence? capturedAfter = null; + Mock.Get(_mockApi) + .Setup(api => api.UpdateExampleSentence( + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), + It.IsAny())) + .Callback((_, _, b, a, _) => { capturedBefore = b; capturedAfter = a; }) + .ReturnsAsync(after); + + await _normalizingApi.UpdateExampleSentence(entryId, senseId, before, after); + + AssertAllNfd(capturedBefore); + AssertAllNfd(capturedAfter); + } + + [Fact] + public async Task CreateExampleSentence_WithTranslations_NormalizesToNfd() + { + var example = NfcTestData.CreateNfcExampleSentenceWithTranslations(); + + ExampleSentence? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.CreateExampleSentence( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, _, e, _) => captured = e) + .ReturnsAsync(example); + + await _normalizingApi.CreateExampleSentence(Guid.NewGuid(), Guid.NewGuid(), example); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + #endregion + + #region Translation Tests + + [Fact] + public async Task AddTranslation_NormalizesToNfd() + { + var translation = NfcTestData.CreateNfcTranslation(); + + Translation? captured = null; + Mock.Get(_mockApi) + .Setup(api => api.AddTranslation( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, _, _, t) => captured = t) + .Returns(Task.CompletedTask); + + await _normalizingApi.AddTranslation(Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), translation); + + captured.Should().NotBeNull(); + AssertAllNfd(captured); + } + + #endregion +} + + +public class NormalizationAssertTests +{ + [Fact] + public void AllNfcFactories_ProduceNfcData() + { + AssertAllNfc(NfcTestData.CreateNfcWritingSystem()); + AssertAllNfc(NfcTestData.CreateNfcPartOfSpeech()); + AssertAllNfc(NfcTestData.CreateNfcPublication()); + AssertAllNfc(NfcTestData.CreateNfcSemanticDomain()); + AssertAllNfc(NfcTestData.CreateNfcComplexFormType()); + AssertAllNfc(NfcTestData.CreateNfcMorphType()); + AssertAllNfc(NfcTestData.CreateNfcTranslation()); + AssertAllNfc(NfcTestData.CreateNfcExampleSentence()); + AssertAllNfc(NfcTestData.CreateNfcExampleSentenceWithTranslations()); + AssertAllNfc(NfcTestData.CreateNfcSense()); + AssertAllNfc(NfcTestData.CreateNfcSenseWithExamples()); + AssertAllNfc(NfcTestData.CreateNfcComplexFormComponent()); + AssertAllNfc(NfcTestData.CreateNfcEntry()); + AssertAllNfc(NfcTestData.CreateNfcEntryWithSenses()); + AssertAllNfc(NfcTestData.CreateNfcEntryWithComponents()); + } + + [Fact] + public void AssertAllNfc_WithNfcData_DoesNotThrow() + { + AssertAllNfc(NfcTestData.CreateNfcEntry()); + } + + [Fact] + public void AssertAllNfc_WithNfdData_Throws() + { + var entry = new Entry + { + Id = Guid.NewGuid(), + LexemeForm = new MultiString { Values = { { "en", NfcTestData.Nfd } } }, // NFD should fail + CitationForm = new MultiString { Values = { { "en", NfcTestData.Nfc } } }, + LiteralMeaning = new RichMultiString { { "en", new RichString(NfcTestData.Nfc) } }, + Note = new RichMultiString { { "en", new RichString(NfcTestData.Nfc) } } + }; + + var act = () => AssertAllNfc(entry); + + act.Should().Throw().WithMessage("*NFC*"); + } + + [Fact] + public void AssertAllNfd_WithNfdData_DoesNotThrow() + { + var entry = new Entry + { + Id = Guid.NewGuid(), + LexemeForm = new MultiString { Values = { { "en", NfcTestData.Nfd } } }, + CitationForm = new MultiString { Values = { { "en", NfcTestData.Nfd } } }, + LiteralMeaning = new RichMultiString { { "en", new RichString(NfcTestData.Nfd) } }, + Note = new RichMultiString { { "en", new RichString(NfcTestData.Nfd) } } + }; + + AssertAllNfd(entry); + } + + [Fact] + public void AssertAllNfd_WithNfcData_Throws() + { + var act = () => AssertAllNfd(NfcTestData.CreateNfcEntry()); + + act.Should().Throw().WithMessage("*NFD*"); + } + + [Fact] + public void AssertAllNfc_WithEmptyMultiString_Throws() + { + var entry = new Entry + { + Id = Guid.NewGuid(), + LexemeForm = [], // Empty - should fail + CitationForm = NfcTestData.CreateNfcMultiString() + }; + + var act = () => AssertAllNfc(entry); + + act.Should().Throw().WithMessage("*no values*"); + } + + [Fact] + public void AssertAllNfc_WithNestedNfdData_Throws() + { + var entry = new Entry + { + Id = Guid.NewGuid(), + LexemeForm = NfcTestData.CreateNfcMultiString(), + CitationForm = NfcTestData.CreateNfcMultiString(), + LiteralMeaning = NfcTestData.CreateNfcRichMultiString(), + Note = NfcTestData.CreateNfcRichMultiString(), + Senses = + [ + new Sense + { + Id = Guid.NewGuid(), + Gloss = new MultiString { Values = { { "en", NfcTestData.Nfd } } }, // NFD nested in sense + Definition = NfcTestData.CreateNfcRichMultiString() + } + ] + }; + + var act = () => AssertAllNfc(entry); + + act.Should().Throw().WithMessage("*Senses*Gloss*"); + } + + [Fact] + public void IsAllNfd_WithNfdData_ReturnsTrue() + { + var multiString = new MultiString { Values = { { "en", NfcTestData.Nfd } } }; + + IsAllNfd(multiString).Should().BeTrue(); + } + + [Fact] + public void IsAllNfd_WithNfcData_ReturnsFalse() + { + IsAllNfd(NfcTestData.CreateNfcMultiString()).Should().BeFalse(); + } +} diff --git a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiQueryNormalizationWrapper.cs similarity index 86% rename from backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs rename to backend/FwLite/MiniLcm/Normalization/MiniLcmApiQueryNormalizationWrapper.cs index ea756ef69d..a0bd5ec2fb 100644 --- a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs +++ b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiQueryNormalizationWrapper.cs @@ -4,17 +4,17 @@ namespace MiniLcm.Normalization; -public class MiniLcmApiStringNormalizationWrapperFactory() : IMiniLcmWrapperFactory +public class MiniLcmApiQueryNormalizationWrapperFactory() : IMiniLcmWrapperFactory { public IMiniLcmApi Create(IMiniLcmApi api, IProjectIdentifier _unused) => Create(api); public IMiniLcmApi Create(IMiniLcmApi api) { - return new MiniLcmApiStringNormalizationWrapper(api); + return new MiniLcmApiQueryNormalizationWrapper(api); } } -public partial class MiniLcmApiStringNormalizationWrapper( +public partial class MiniLcmApiQueryNormalizationWrapper( IMiniLcmApi api) : IMiniLcmApi { public const NormalizationForm Form = NormalizationForm.FormD; diff --git a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiWriteNormalizationWrapper.cs b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiWriteNormalizationWrapper.cs new file mode 100644 index 0000000000..c182aca5e2 --- /dev/null +++ b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiWriteNormalizationWrapper.cs @@ -0,0 +1,578 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using MiniLcm.Media; +using MiniLcm.Models; +using MiniLcm.SyncHelpers; +using MiniLcm.Wrappers; + +namespace MiniLcm.Normalization; + +public class MiniLcmApiWriteNormalizationWrapperFactory : IMiniLcmWrapperFactory +{ + public IMiniLcmApi Create(IMiniLcmApi api, IProjectIdentifier _unused) + { + return Create(api); + } + + + public IMiniLcmApi Create(IMiniLcmApi api) + { + return new MiniLcmApiWriteNormalizationWrapper(api); + } +} + +/// +/// Normalizes user-entered linguistic text to NFD on write operations. +/// +/// Design notes: +/// - Read operations are forwarded automatically via BeaKona.AutoInterface. +/// - Write operations need to be manually implemented so nothing is missed (compile-time enforced by IMiniLcmApi). +/// - Should mirror what LibLcm/FieldWorks normalizes, which seems to be EVERYTHING in the "standard editor" UI +/// (so entry fields, but also list fields e.g. Semantic Domains, Morph Types, etc. - not only multi-strings) +/// - For update methods that take both "before" and "after" objects, BOTH are normalized to ensure that any string comparisons done by the API are normalized. +/// This prevents potential diff-noise caused by "before" being bad-data even though we expect it to always already be normalized (because we presumably served it). +/// Non-normalized data that is already persisted (before this wrapper was introduced) will be normalized by LibLcm. That's not something we want to fix here. +/// - Properties that liblcm/FieldWorks does not NFD-normalize are passed through unchanged. +/// Currently only WritingSystem properties. +/// - JsonPatch overloads normalize string-ish values best-effort (string, RichString, MultiString, RichMultiString). +/// JsonElement values are only normalized when they are simple strings; complex JSON values are left as-is +/// to avoid guessing the target type. +/// +public partial class MiniLcmApiWriteNormalizationWrapper(IMiniLcmApi api) : IMiniLcmApi +{ + private readonly IMiniLcmApi _api = api; + + // BeaKona.AutoInterface only forwards IMiniLcmReadApi methods. + // IMiniLcmWriteApi methods are NOT auto-forwarded, ensuring compile-time + // enforcement that all write methods are manually implemented below. + [BeaKona.AutoInterface] + private IMiniLcmReadApi ReadApi => _api; + + #region WritingSystem + + // Intentionally NOT normalized, because FieldWorks/LibLcm doesn't seem to either + public Task CreateWritingSystem(WritingSystem writingSystem, BetweenPosition? between = null) + { + return _api.CreateWritingSystem(writingSystem, between); + } + + public Task UpdateWritingSystem(WritingSystemId id, WritingSystemType type, UpdateObjectInput update) + { + return _api.UpdateWritingSystem(id, type, update); + } + + + public Task UpdateWritingSystem(WritingSystem before, WritingSystem after, IMiniLcmApi? api = null) + { + return _api.UpdateWritingSystem(before, after, api); + } + + public Task MoveWritingSystem(WritingSystemId id, WritingSystemType type, BetweenPosition between) + { + return _api.MoveWritingSystem(id, type, between); + } + + #endregion + + #region PartOfSpeech + + public async Task CreatePartOfSpeech(PartOfSpeech partOfSpeech) + { + return await _api.CreatePartOfSpeech(NormalizePartOfSpeech(partOfSpeech)); + } + + public Task UpdatePartOfSpeech(Guid id, UpdateObjectInput update) + { + return _api.UpdatePartOfSpeech(id, NormalizePatch(update)); + } + + + public async Task UpdatePartOfSpeech(PartOfSpeech before, PartOfSpeech after, IMiniLcmApi? api = null) + { + return await _api.UpdatePartOfSpeech(NormalizePartOfSpeech(before), NormalizePartOfSpeech(after), api); + } + + public Task DeletePartOfSpeech(Guid id) + { + return _api.DeletePartOfSpeech(id); + } + + private static PartOfSpeech NormalizePartOfSpeech(PartOfSpeech pos) + { + return new PartOfSpeech + { + Id = pos.Id, + Name = StringNormalizer.Normalize(pos.Name), + DeletedAt = pos.DeletedAt, + Predefined = pos.Predefined + }; + } + + #endregion + + #region Publication + + public async Task CreatePublication(Publication pub) + { + return await _api.CreatePublication(NormalizePublication(pub)); + } + + public Task UpdatePublication(Guid id, UpdateObjectInput update) + { + return _api.UpdatePublication(id, NormalizePatch(update)); + } + + + public async Task UpdatePublication(Publication before, Publication after, IMiniLcmApi? api = null) + { + return await _api.UpdatePublication(NormalizePublication(before), NormalizePublication(after), api); + } + + public Task DeletePublication(Guid id) + { + return _api.DeletePublication(id); + } + + private static Publication NormalizePublication(Publication pub) + { + return new Publication + { + Id = pub.Id, + Name = StringNormalizer.Normalize(pub.Name), + DeletedAt = pub.DeletedAt + }; + } + + #endregion + + #region SemanticDomain + + public async Task CreateSemanticDomain(SemanticDomain semanticDomain) + { + return await _api.CreateSemanticDomain(NormalizeSemanticDomain(semanticDomain)); + } + + public Task UpdateSemanticDomain(Guid id, UpdateObjectInput update) + { + return _api.UpdateSemanticDomain(id, NormalizePatch(update)); + } + + + public async Task UpdateSemanticDomain(SemanticDomain before, SemanticDomain after, IMiniLcmApi? api = null) + { + return await _api.UpdateSemanticDomain(NormalizeSemanticDomain(before), NormalizeSemanticDomain(after), api); + } + + public Task DeleteSemanticDomain(Guid id) + { + return _api.DeleteSemanticDomain(id); + } + + private static SemanticDomain NormalizeSemanticDomain(SemanticDomain sd) + { + return new SemanticDomain + { + Id = sd.Id, + Name = StringNormalizer.Normalize(sd.Name), + Code = StringNormalizer.Normalize(sd.Code), // yes, LibLcm normalizes this too + DeletedAt = sd.DeletedAt, + Predefined = sd.Predefined + }; + } + + #endregion + + #region ComplexFormType + + public async Task CreateComplexFormType(ComplexFormType complexFormType) + { + return await _api.CreateComplexFormType(NormalizeComplexFormType(complexFormType)); + } + + public Task UpdateComplexFormType(Guid id, UpdateObjectInput update) + { + return _api.UpdateComplexFormType(id, NormalizePatch(update)); + } + + + public async Task UpdateComplexFormType(ComplexFormType before, ComplexFormType after, IMiniLcmApi? api = null) + { + return await _api.UpdateComplexFormType(NormalizeComplexFormType(before), NormalizeComplexFormType(after), api); + } + + public Task DeleteComplexFormType(Guid id) + { + return _api.DeleteComplexFormType(id); + } + + private static ComplexFormType NormalizeComplexFormType(ComplexFormType cft) + { + return cft with + { + Name = StringNormalizer.Normalize(cft.Name) + }; + } + + #endregion + + #region MorphType + + public Task UpdateMorphType(Guid id, UpdateObjectInput update) + { + return _api.UpdateMorphType(id, NormalizePatch(update)); + } + + + public async Task UpdateMorphType(MorphType before, MorphType after, IMiniLcmApi? api = null) + { + return await _api.UpdateMorphType(NormalizeMorphType(before), NormalizeMorphType(after), api); + } + + private static MorphType NormalizeMorphType(MorphType mt) + { + return new MorphType + { + Id = mt.Id, + Kind = mt.Kind, + Name = StringNormalizer.Normalize(mt.Name), + Abbreviation = StringNormalizer.Normalize(mt.Abbreviation), + Description = StringNormalizer.Normalize(mt.Description), + Prefix = StringNormalizer.Normalize(mt.Prefix), + Postfix = StringNormalizer.Normalize(mt.Postfix), + SecondaryOrder = mt.SecondaryOrder, + DeletedAt = mt.DeletedAt + }; + } + + #endregion + + #region Entry + + public async Task CreateEntry(Entry entry, CreateEntryOptions? options = null) + { + return await _api.CreateEntry(NormalizeEntry(entry), options); + } + + public Task UpdateEntry(Guid id, UpdateObjectInput update) + { + return _api.UpdateEntry(id, NormalizePatch(update)); + } + + + public async Task UpdateEntry(Entry before, Entry after, IMiniLcmApi? api = null) + { + return await _api.UpdateEntry(NormalizeEntry(before), NormalizeEntry(after), api); + } + + public Task DeleteEntry(Guid id) + { + return _api.DeleteEntry(id); + } + + public async Task CreateComplexFormComponent(ComplexFormComponent complexFormComponent, BetweenPosition? position = null) + { + return await _api.CreateComplexFormComponent(NormalizeComplexFormComponent(complexFormComponent), position); + } + + public Task MoveComplexFormComponent(ComplexFormComponent complexFormComponent, BetweenPosition between) + { + return _api.MoveComplexFormComponent(complexFormComponent, between); + } + + public Task DeleteComplexFormComponent(ComplexFormComponent complexFormComponent) + { + return _api.DeleteComplexFormComponent(complexFormComponent); + } + + public Task AddComplexFormType(Guid entryId, Guid complexFormTypeId) + { + return _api.AddComplexFormType(entryId, complexFormTypeId); + } + + public Task RemoveComplexFormType(Guid entryId, Guid complexFormTypeId) + { + return _api.RemoveComplexFormType(entryId, complexFormTypeId); + } + + public Task AddPublication(Guid entryId, Guid publicationId) + { + return _api.AddPublication(entryId, publicationId); + } + + public Task RemovePublication(Guid entryId, Guid publicationId) + { + return _api.RemovePublication(entryId, publicationId); + } + + private static Entry NormalizeEntry(Entry entry) + { + return new Entry + { + Id = entry.Id, + DeletedAt = entry.DeletedAt, + LexemeForm = StringNormalizer.Normalize(entry.LexemeForm), + CitationForm = StringNormalizer.Normalize(entry.CitationForm), + LiteralMeaning = StringNormalizer.Normalize(entry.LiteralMeaning), + Note = StringNormalizer.Normalize(entry.Note), + MorphType = entry.MorphType, + Senses = [.. entry.Senses.Select(NormalizeSense)], + Components = [.. entry.Components.Select(NormalizeComplexFormComponent)], + ComplexForms = [.. entry.ComplexForms.Select(NormalizeComplexFormComponent)], + ComplexFormTypes = entry.ComplexFormTypes, + PublishIn = entry.PublishIn + }; + } + + private static ComplexFormComponent NormalizeComplexFormComponent(ComplexFormComponent cfc) + { + return cfc with + { + ComplexFormHeadword = StringNormalizer.Normalize(cfc.ComplexFormHeadword), + ComponentHeadword = StringNormalizer.Normalize(cfc.ComponentHeadword) + }; + } + + #endregion + + #region Sense + + public async Task CreateSense(Guid entryId, Sense sense, BetweenPosition? position = null) + { + return await _api.CreateSense(entryId, NormalizeSense(sense), position); + } + + public Task UpdateSense(Guid entryId, Guid senseId, UpdateObjectInput update) + { + return _api.UpdateSense(entryId, senseId, NormalizePatch(update)); + } + + + public async Task UpdateSense(Guid entryId, Sense before, Sense after, IMiniLcmApi? api = null) + { + return await _api.UpdateSense(entryId, NormalizeSense(before), NormalizeSense(after), api); + } + + public Task MoveSense(Guid entryId, Guid senseId, BetweenPosition position) + { + return _api.MoveSense(entryId, senseId, position); + } + + public Task DeleteSense(Guid entryId, Guid senseId) + { + return _api.DeleteSense(entryId, senseId); + } + + public Task AddSemanticDomainToSense(Guid senseId, SemanticDomain semanticDomain) + { + return _api.AddSemanticDomainToSense(senseId, NormalizeSemanticDomain(semanticDomain)); + } + + public Task RemoveSemanticDomainFromSense(Guid senseId, Guid semanticDomainId) + { + return _api.RemoveSemanticDomainFromSense(senseId, semanticDomainId); + } + + public Task SetSensePartOfSpeech(Guid senseId, Guid? partOfSpeechId) + { + return _api.SetSensePartOfSpeech(senseId, partOfSpeechId); + } + + private static Sense NormalizeSense(Sense sense) + { + return new Sense + { + Id = sense.Id, + Order = sense.Order, + DeletedAt = sense.DeletedAt, + EntryId = sense.EntryId, + Definition = StringNormalizer.Normalize(sense.Definition), + Gloss = StringNormalizer.Normalize(sense.Gloss), + PartOfSpeech = sense.PartOfSpeech is not null ? NormalizePartOfSpeech(sense.PartOfSpeech) : null, + PartOfSpeechId = sense.PartOfSpeechId, + SemanticDomains = [.. sense.SemanticDomains.Select(NormalizeSemanticDomain)], + ExampleSentences = [.. sense.ExampleSentences.Select(NormalizeExampleSentence)] + }; + } + + #endregion + + #region ExampleSentence + + public async Task CreateExampleSentence(Guid entryId, Guid senseId, ExampleSentence exampleSentence, BetweenPosition? position = null) + { + return await _api.CreateExampleSentence(entryId, senseId, NormalizeExampleSentence(exampleSentence), position); + } + + public Task UpdateExampleSentence(Guid entryId, Guid senseId, Guid exampleSentenceId, UpdateObjectInput update) + { + return _api.UpdateExampleSentence(entryId, senseId, exampleSentenceId, NormalizePatch(update)); + } + + + public async Task UpdateExampleSentence(Guid entryId, Guid senseId, ExampleSentence before, ExampleSentence after, IMiniLcmApi? api = null) + { + return await _api.UpdateExampleSentence(entryId, senseId, NormalizeExampleSentence(before), NormalizeExampleSentence(after), api); + } + + public Task MoveExampleSentence(Guid entryId, Guid senseId, Guid exampleSentenceId, BetweenPosition position) + { + return _api.MoveExampleSentence(entryId, senseId, exampleSentenceId, position); + } + + public Task DeleteExampleSentence(Guid entryId, Guid senseId, Guid exampleSentenceId) + { + return _api.DeleteExampleSentence(entryId, senseId, exampleSentenceId); + } + + public Task AddTranslation(Guid entryId, Guid senseId, Guid exampleSentenceId, Translation translation) + { + return _api.AddTranslation(entryId, senseId, exampleSentenceId, NormalizeTranslation(translation)); + } + + public Task RemoveTranslation(Guid entryId, Guid senseId, Guid exampleSentenceId, Guid translationId) + { + return _api.RemoveTranslation(entryId, senseId, exampleSentenceId, translationId); + } + + public Task UpdateTranslation(Guid entryId, Guid senseId, Guid exampleSentenceId, Guid translationId, UpdateObjectInput update) + { + return _api.UpdateTranslation(entryId, senseId, exampleSentenceId, translationId, NormalizePatch(update)); + } + + + private static ExampleSentence NormalizeExampleSentence(ExampleSentence example) + { + return new ExampleSentence + { + Id = example.Id, + Order = example.Order, + DeletedAt = example.DeletedAt, + SenseId = example.SenseId, + Sentence = StringNormalizer.Normalize(example.Sentence), + Translations = [.. example.Translations.Select(NormalizeTranslation)], + Reference = StringNormalizer.Normalize(example.Reference) + }; + } + + private static Translation NormalizeTranslation(Translation translation) + { + return new Translation + { + Id = translation.Id, + Text = StringNormalizer.Normalize(translation.Text) + }; + } + + #endregion + + #region Bulk Import + + // Normalizing the bulk import methods may seem unintuitive: + // We currently only use them for importing from LibLcm, which should NOT be normalized. + // However, we don't use this normalization wrapper in that context and ít's likely that we'll + // start importing from other sources (Paratext 9, The Combine, Language Forge) in which case we DO want to normalize on import. + + public Task BulkImportSemanticDomains(IAsyncEnumerable semanticDomains) + { + return _api.BulkImportSemanticDomains(NormalizeStream(semanticDomains, NormalizeSemanticDomain)); + } + + public Task BulkCreateEntries(IAsyncEnumerable entries) + { + return _api.BulkCreateEntries(NormalizeStream(entries, NormalizeEntry)); + } + + private static async IAsyncEnumerable NormalizeStream( + IAsyncEnumerable source, + Func normalize) + { + await foreach (var item in source) + { + yield return normalize(item); + } + } + + #endregion + + #region CustomView + + // CustomView data is view configuration, not user-entered linguistic text, so no normalization is applied. + public async Task CreateCustomView(CustomView customView) + { + return await _api.CreateCustomView(customView); + } + + public async Task UpdateCustomView(CustomView customView) + { + return await _api.UpdateCustomView(customView); + } + + public Task DeleteCustomView(Guid id) + { + return _api.DeleteCustomView(id); + } + + #endregion + + #region File Operations + + public Task SaveFile(Stream stream, LcmFileMetadata metadata) + { + // File metadata is not user-entered text, so don't normalize + return _api.SaveFile(stream, metadata); + } + + #endregion + + #region Patch Normalization + + private static UpdateObjectInput NormalizePatch(UpdateObjectInput update) where T : class + { + if (update.Patch.Operations.Count == 0) return update; + + var normalizedPatch = new SystemTextJsonPatch.JsonPatchDocument(); + foreach (var op in update.Patch.Operations) + { + var normalizedValue = NormalizePatchValue(op.Value); + var normalizedOp = new SystemTextJsonPatch.Operations.Operation + { + Op = op.Op, + Path = op.Path, + From = op.From, + Value = normalizedValue, + }; + normalizedPatch.Operations.Add(normalizedOp); + } + return new UpdateObjectInput(normalizedPatch); + } + + [return: NotNullIfNotNull(nameof(value))] + private static object? NormalizePatchValue(object? value) + { + return value switch + { + null => null, + string s => s.Normalize(StringNormalizer.Form), + RichString richString => StringNormalizer.Normalize(richString), + MultiString multiString => StringNormalizer.Normalize(multiString), + RichMultiString richMultiString => StringNormalizer.Normalize(richMultiString), + JsonElement jsonElement => NormalizeJsonElement(jsonElement), + _ => value + }; + } + + private static object? NormalizeJsonElement(JsonElement element) + { + if (element.ValueKind != JsonValueKind.String) return element; + var value = element.GetString(); + return value?.Normalize(StringNormalizer.Form); + } + + #endregion + + void IDisposable.Dispose() + { + // No resources to dispose + } + +} diff --git a/backend/FwLite/MiniLcm/Normalization/StringNormalizer.cs b/backend/FwLite/MiniLcm/Normalization/StringNormalizer.cs new file mode 100644 index 0000000000..d213c5a05b --- /dev/null +++ b/backend/FwLite/MiniLcm/Normalization/StringNormalizer.cs @@ -0,0 +1,46 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text; +using MiniLcm.Models; + +namespace MiniLcm.Normalization; + +public static class StringNormalizer +{ + public const NormalizationForm Form = NormalizationForm.FormD; + + [return: NotNullIfNotNull(nameof(value))] + public static string? Normalize(string? value) + { + return value?.Normalize(Form); + } + + public static MultiString Normalize(MultiString multiString) + { + var normalized = new MultiString(multiString.Values.Count); + foreach (var (key, value) in multiString.Values) + { + normalized.Values[key] = string.IsNullOrEmpty(value) ? value : value.Normalize(Form); + } + return normalized; + } + + [return: NotNullIfNotNull(nameof(richString))] + public static RichString? Normalize(RichString? richString) + { + if (richString is null) return null; + return new RichString([.. richString.Spans.Select(span => span with + { + Text = span.Text.Normalize(Form) + })]); + } + + public static RichMultiString Normalize(RichMultiString richMultiString) + { + var normalized = new RichMultiString(richMultiString.Count); + foreach (var (key, value) in richMultiString) + { + normalized[key] = Normalize(value); + } + return normalized; + } +} diff --git a/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs b/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs index 1a325290f3..ae5468250a 100644 --- a/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs +++ b/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.DependencyInjection; using MiniLcm.Models; using MiniLcm.Normalization; +using MiniLcm.Wrappers; namespace MiniLcm.Validators; @@ -91,7 +92,9 @@ public static IServiceCollection AddMiniLcmValidators(this IServiceCollection se services.AddTransient, PublicationValidator>(); services.AddTransient>, MorphTypeUpdateValidator>(); services.AddTransient>, WritingSystemUpdateValidator>(); - services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); return services; } } diff --git a/backend/FwLite/MiniLcm/Wrappers/MiniLcmApiUserFacingWrappers.cs b/backend/FwLite/MiniLcm/Wrappers/MiniLcmApiUserFacingWrappers.cs new file mode 100644 index 0000000000..43ddef6913 --- /dev/null +++ b/backend/FwLite/MiniLcm/Wrappers/MiniLcmApiUserFacingWrappers.cs @@ -0,0 +1,26 @@ +using MiniLcm.Models; +using MiniLcm.Normalization; +using MiniLcm.Validators; + +namespace MiniLcm.Wrappers; + +/// +/// The standard wrapper stack applied at every user-facing API entry point: +/// MiniLcmJsInvokable, MiniLcmApiHubBase, and MiniLcmRoutes. +/// +public class MiniLcmApiUserFacingWrappers( + MiniLcmApiQueryNormalizationWrapperFactory readNormalization, + MiniLcmApiWriteNormalizationWrapperFactory writeNormalization, + MiniLcmApiValidationWrapperFactory validation) +{ + public IMiniLcmApi Apply(IMiniLcmApi api, IProjectIdentifier project, params IMiniLcmWrapperFactory[] innerWrappers) + { + // Validation before write normalisation: bad input is rejected with a clear error + // before the normaliser sees it, so the normaliser can assume structurally valid data.Write normalisation is applied uniformly to both CRDT and FwData: + // It's redundant for FwData (because LibLCM also normalises internally), + // but the uniformity is simpler and the normaliser creates new instances when it normalises. + // We want that behaviour to always be in place for simplicity. + // FwData is desktop-only, so the redundant pass is inconsequential cost-wise. + return api.WrapWith([readNormalization, validation, writeNormalization, .. innerWrappers], project); + } +} diff --git a/backend/FwLite/MiniLcm/Wrappers/MiniLcmWrappers.cs b/backend/FwLite/MiniLcm/Wrappers/MiniLcmWrappers.cs index 578fbf1820..d2e33b52b1 100644 --- a/backend/FwLite/MiniLcm/Wrappers/MiniLcmWrappers.cs +++ b/backend/FwLite/MiniLcm/Wrappers/MiniLcmWrappers.cs @@ -9,6 +9,10 @@ public interface IMiniLcmWrapperFactory public static class MiniLcmWrapperExtensions { + /// + /// Wraps with in runtime call order + /// (outermost to innermost). The first factory becomes the outermost wrapper. + /// public static IMiniLcmApi WrapWith(this IMiniLcmApi api, IEnumerable factories, IProjectIdentifier project) { var wrappedApi = api;