From 3381a2d7fb74de4c4022d9c71776c6ef50cd464b Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 9 Jun 2026 13:25:21 +0700 Subject: [PATCH 01/16] add a new debug helper to diagnose issue cloning a project --- backend/FwLite/LcmDebugger/FakeSyncSource.cs | 13 +++++++++++++ backend/FwLite/LcmDebugger/Utils.cs | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/backend/FwLite/LcmDebugger/FakeSyncSource.cs b/backend/FwLite/LcmDebugger/FakeSyncSource.cs index 39574212d9..aa697ef6b8 100644 --- a/backend/FwLite/LcmDebugger/FakeSyncSource.cs +++ b/backend/FwLite/LcmDebugger/FakeSyncSource.cs @@ -1,5 +1,6 @@ using System.Diagnostics.CodeAnalysis; using System.Text.Json; +using LcmCrdt; using SIL.Harmony; using SIL.Harmony.Changes; using SIL.Harmony.Core; @@ -39,8 +40,20 @@ public static FakeSyncSource FromSingleChangeJson( public static FakeSyncSource FromJsonFile(string path, JsonSerializerOptions? options = null) { + if (options is null) + { + var config = new CrdtConfig(); + LcmCrdtKernel.ConfigureCrdt(config); + options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + TypeInfoResolver = config.MakeLcmCrdtExternalJsonTypeResolver(), + }; + } + var changes = JsonSerializer.Deserialize>(File.OpenRead(path), options); ArgumentNullException.ThrowIfNull(changes); + ArgumentNullException.ThrowIfNull(changes.MissingFromClient); + ArgumentNullException.ThrowIfNull(changes.ServerSyncState); return new FakeSyncSource(changes.MissingFromClient, changes.ServerSyncState); } diff --git a/backend/FwLite/LcmDebugger/Utils.cs b/backend/FwLite/LcmDebugger/Utils.cs index 7ed135a3fa..3430e9244e 100644 --- a/backend/FwLite/LcmDebugger/Utils.cs +++ b/backend/FwLite/LcmDebugger/Utils.cs @@ -5,6 +5,7 @@ using FwLiteProjectSync; using LcmCrdt; using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony; using SIL.LCModel; namespace LcmDebugger; @@ -46,6 +47,15 @@ public static async Task PrintAllEntries(this IServiceProvider services, string } } + public static async Task NewProjectFromSyncable(this IServiceProvider services, ISyncable syncable, Guid? projectId = null) + { + var crdtProjectsService = services.GetRequiredService(); + var crdtProject = await crdtProjectsService.CreateProject(new CrdtProjectsService.CreateProjectRequest("test-project", $"test-{Guid.NewGuid().ToString().Split('-')[0]}", projectId)); + var crdtMiniLcmApi = (CrdtMiniLcmApi)await crdtProjectsService.OpenProject(crdtProject, services); + await services.GetRequiredService().SyncWith(syncable); + return crdtMiniLcmApi; + } + public static async Task OpenDownloadedProject(this IServiceProvider services, string relativePath, bool openCopy = false, string? downloadsRoot = null) { // Default to a path relative to the executing assembly, pointing to the deployment/_downloads folder From 7c096193e6a9ce445e57de5195e66b3624c92873 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 9 Jun 2026 13:29:19 +0700 Subject: [PATCH 02/16] simple attempt to fix issue of always adding predefined morph types at time of project creation --- backend/FwLite/LcmCrdt/CrdtProjectsService.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs index f6ab9cf490..3d5e1f69d1 100644 --- a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs +++ b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs @@ -166,10 +166,7 @@ public virtual async Task CreateProject(CreateProjectRequest reques crdtProject.Data = projectData; await InitProjectDb(db, projectData); await currentProjectService.RefreshProjectData(); - // Morph types are predefined system data that must always exist — seed them - // unconditionally so they're available before AfterCreate (e.g. import) runs. var dataModel = serviceScope.ServiceProvider.GetRequiredService(); - await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData); if (request.SeedNewProjectData) await SeedSystemData(dataModel, projectData); await (request.AfterCreate?.Invoke(serviceScope.ServiceProvider, crdtProject) ?? Task.CompletedTask); @@ -245,7 +242,7 @@ internal static async Task InitProjectDb(LcmCrdtDbContext db, ProjectData data) internal static async Task SeedSystemData(DataModel dataModel, ProjectData projectData) { - // Note: AddPredefinedMorphTypes is seeded unconditionally in CreateProject, not here. + await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData); await PreDefinedData.AddPredefinedComplexFormTypes(dataModel, projectData); await PreDefinedData.AddPredefinedPartsOfSpeech(dataModel, projectData); await PreDefinedData.AddPredefinedSemanticDomains(dataModel, projectData); From c48f964d648cfd91ac83c70946e8a1e3c61e9e6c Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 9 Jun 2026 14:34:07 +0700 Subject: [PATCH 03/16] tweak the download projects test to fail due to morph type migrations. --- .../Data/DownloadProjectTests.cs | 23 +++++++++++++++++-- .../Data/RegressionTestHelper.cs | 5 ++-- .../FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs | 14 ++++++----- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs b/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs index a0ee0bf4d2..bb65d29688 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs @@ -1,16 +1,24 @@ +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using SIL.Harmony.Core; namespace LcmCrdt.Tests.Data; public class DownloadProjectTests : IAsyncLifetime { private readonly RegressionTestHelper _helper = new("DownloadProject"); - private readonly MiniLcmApiFixture _apiFixture = MiniLcmApiFixture.Create(false); + private static readonly Guid _projectId = new("B467051E-A492-4E5B-9C17-858D7797292C");//internal project Id of v2 project + private readonly MiniLcmApiFixture _apiFixture = MiniLcmApiFixture.Create(false, _projectId); public async Task InitializeAsync() { - await _helper.InitializeAsync(); + await _helper.InitializeAsync(RegressionTestHelper.RegressionVersion.v2); + //add a change after migration which creates MorphTypes + await _helper.Services.GetRequiredService().CreateEntry(new Entry() + { + LexemeForm = {{"en", "test"}} + }); await _apiFixture.InitializeAsync(); } @@ -24,6 +32,17 @@ public async Task DisposeAsync() public async Task CanCreateANewProjectViaSync() { var remoteModel = _helper.Services.GetRequiredService(); + var remoteDb = await _helper.Services.GetRequiredService>().CreateDbContextAsync(); + await _apiFixture.DataModel.SyncWith(remoteModel); + var localCommits = await _apiFixture.DbContext.Set().DefaultOrder().ToListAsync(); + var remoteCommits = await remoteDb.Set().DefaultOrder().ToListAsync(); + localCommits.Count.Should().Be(remoteCommits.Count); + for (var i = localCommits.Count - 1; i >= 0; i--) + { + var localCommit = localCommits[i]; + var remoteCommit = remoteCommits[i]; + localCommit.Should().BeEquivalentTo(remoteCommit, "commit index {0} should be the same", i); + } } } diff --git a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs index 82cf03e878..7197664603 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs @@ -38,9 +38,8 @@ private async Task InitDbFromScripts(RegressionVersion version) //need to close the connection, otherwise the collations won't get created, they would normally be created on open or save, so we're closing so they get created when EF opens the connection. await dbConnection.CloseAsync(); - await lcmCrdtDbContext.Database.MigrateAsync(); - - await projectsService.RefreshProjectData(); + //setup again to trigger migrations + await projectsService.SetupProjectContext(crdtProject); } public Task InitializeAsync() diff --git a/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs b/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs index 3218031489..87d24d2325 100644 --- a/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs +++ b/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs @@ -14,6 +14,7 @@ namespace LcmCrdt.Tests; public class MiniLcmApiFixture : IAsyncLifetime, IAsyncDisposable { private readonly bool _seedWs = true; + private readonly Guid? _projectId; private AsyncServiceScope _services; private LcmCrdtDbContext? _crdtDbContext; public CrdtMiniLcmApi Api => (CrdtMiniLcmApi)_services.ServiceProvider.GetRequiredService(); @@ -31,22 +32,23 @@ public MiniLcmApiFixture() { } - public static MiniLcmApiFixture Create(bool seedWs = true) + public static MiniLcmApiFixture Create(bool seedWs = true, Guid? projectId = null) { - return new MiniLcmApiFixture(seedWs); + return new MiniLcmApiFixture(seedWs, projectId); } - private MiniLcmApiFixture(bool seedWs = true) + private MiniLcmApiFixture(bool seedWs = true, Guid? projectId = null) { _seedWs = seedWs; + _projectId = projectId; } public async Task InitializeAsync() { - await InitializeAsync("sena-3"); + await InitializeAsync("sena-3", _projectId); } - public async Task InitializeAsync(string projectName) + public async Task InitializeAsync(string projectName, Guid? projectId = null) { var db = $"file:{Guid.NewGuid():N}?mode=memory&cache=shared"; if (Debugger.IsAttached) @@ -72,7 +74,7 @@ public async Task InitializeAsync(string projectName) _crdtDbContext = await _services.ServiceProvider.GetRequiredService>().CreateDbContextAsync(); await _crdtDbContext.Database.OpenConnectionAsync(); //can't use ProjectsService.CreateProject because it opens and closes the db context, this would wipe out the in memory db. - var projectData = new ProjectData("Sena 3", projectName, Guid.NewGuid(), null, Guid.NewGuid()); + var projectData = new ProjectData("Sena 3", projectName, projectId ?? Guid.NewGuid(), null, Guid.NewGuid()); await CrdtProjectsService.InitProjectDb(_crdtDbContext, projectData); await currentProjectService.RefreshProjectData(); From 6b1fe5333df8244b9a7c5f1f70f4b87415a62419 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 9 Jun 2026 14:43:58 +0700 Subject: [PATCH 04/16] remove always adding morph types in api fixture as well --- backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs b/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs index 87d24d2325..422e983a80 100644 --- a/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs +++ b/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs @@ -78,8 +78,6 @@ public async Task InitializeAsync(string projectName, Guid? projectId = null) await CrdtProjectsService.InitProjectDb(_crdtDbContext, projectData); await currentProjectService.RefreshProjectData(); - // CreateProject would also seed morph types — so we need to do it manually here - await PreDefinedData.AddPredefinedMorphTypes(_services.ServiceProvider.GetRequiredService(), projectData); if (_seedWs) { await Api.CreateWritingSystem(new WritingSystem() From 3f97ec3a8f5450672780d61d7f806fc7dcd97346 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 9 Jun 2026 14:45:34 +0700 Subject: [PATCH 05/16] only seed morphtypes when needed, only use a predefined commit id when seeding. Modify CreateMorphTypeChange to handle creating the same morph type multiple times due to migrations. --- backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs | 8 +++++++- backend/FwLite/LcmCrdt/CrdtProjectsService.cs | 2 +- backend/FwLite/LcmCrdt/CurrentProjectService.cs | 5 ++++- backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs | 4 ++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs b/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs index 5661cb2de6..e3f8c32794 100644 --- a/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs @@ -8,7 +8,7 @@ namespace LcmCrdt.Changes; -public class CreateMorphTypeChange : CreateChange, ISelfNamedType +public class CreateMorphTypeChange : Change, ISelfNamedType { [SetsRequiredMembers] public CreateMorphTypeChange(MorphType morphType) : base(morphType.Id) @@ -58,4 +58,10 @@ public override async ValueTask NewEntity(Commit commit, IChangeConte DeletedAt = alreadyExists ? commit.DateTime : null }; } + + public override ValueTask ApplyChange(MorphType entity, IChangeContext context) + { + //don't do anything here, CreateMorphTypeChange is used in a migration, and a change to create the same morph type may happen multiple times + return ValueTask.CompletedTask; + } } diff --git a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs index 3d5e1f69d1..01e2d81acd 100644 --- a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs +++ b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs @@ -242,7 +242,7 @@ internal static async Task InitProjectDb(LcmCrdtDbContext db, ProjectData data) internal static async Task SeedSystemData(DataModel dataModel, ProjectData projectData) { - await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData); + await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData, false); await PreDefinedData.AddPredefinedComplexFormTypes(dataModel, projectData); await PreDefinedData.AddPredefinedPartsOfSpeech(dataModel, projectData); await PreDefinedData.AddPredefinedSemanticDomains(dataModel, projectData); diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index 885e0856fe..ba51f155d0 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -112,7 +112,10 @@ async Task Execute() // Must happen BEFORE FTS regeneration so headwords include morph-type tokens. var dataModel = services.GetRequiredService(); var projectData = await dbContext.ProjectData.AsNoTracking().FirstAsync(); - await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData); + if (!await dbContext.MorphTypes.AnyAsync()) + { + await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData, true); + } if (EntrySearchServiceFactory is not null) { diff --git a/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs b/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs index f319de8921..5c1454530c 100644 --- a/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs +++ b/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs @@ -104,10 +104,10 @@ await dataModel.AddChanges(projectData.ClientId, CustomViewsSeedCommitId(projectData.Id)); } - internal static async Task AddPredefinedMorphTypes(DataModel dataModel, ProjectData projectData) + internal static async Task AddPredefinedMorphTypes(DataModel dataModel, ProjectData projectData, bool isMigration) { await dataModel.AddChanges(projectData.ClientId, [.. CanonicalMorphTypes.All.Values.Select(mt => new CreateMorphTypeChange(mt))], - MorphTypesSeedCommitId(projectData.Id)); + isMigration ? Guid.NewGuid() : MorphTypesSeedCommitId(projectData.Id)); } } From 7d05131e9cba291ecc2db5602f596f767bb3ad4f Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 9 Jun 2026 12:23:56 +0200 Subject: [PATCH 06/16] Change CreateMorphType back to CreateChange --- backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs b/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs index e3f8c32794..5661cb2de6 100644 --- a/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs +++ b/backend/FwLite/LcmCrdt/Changes/CreateMorphTypeChange.cs @@ -8,7 +8,7 @@ namespace LcmCrdt.Changes; -public class CreateMorphTypeChange : Change, ISelfNamedType +public class CreateMorphTypeChange : CreateChange, ISelfNamedType { [SetsRequiredMembers] public CreateMorphTypeChange(MorphType morphType) : base(morphType.Id) @@ -58,10 +58,4 @@ public override async ValueTask NewEntity(Commit commit, IChangeConte DeletedAt = alreadyExists ? commit.DateTime : null }; } - - public override ValueTask ApplyChange(MorphType entity, IChangeContext context) - { - //don't do anything here, CreateMorphTypeChange is used in a migration, and a change to create the same morph type may happen multiple times - return ValueTask.CompletedTask; - } } From 27ccb6ce826dc341f059f3880807b67ac5c0a7ef Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 9 Jun 2026 15:10:10 +0200 Subject: [PATCH 07/16] Reintroduce morph-type seeding to minimize fix-diff --- backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs | 6 +++--- backend/FwLite/LcmCrdt/CrdtProjectsService.cs | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs b/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs index 422e983a80..0926263cd0 100644 --- a/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs +++ b/backend/FwLite/LcmCrdt.Tests/MiniLcmApiFixture.cs @@ -75,9 +75,9 @@ public async Task InitializeAsync(string projectName, Guid? projectId = null) await _crdtDbContext.Database.OpenConnectionAsync(); //can't use ProjectsService.CreateProject because it opens and closes the db context, this would wipe out the in memory db. var projectData = new ProjectData("Sena 3", projectName, projectId ?? Guid.NewGuid(), null, Guid.NewGuid()); - await CrdtProjectsService.InitProjectDb(_crdtDbContext, - projectData); - await currentProjectService.RefreshProjectData(); + await CrdtProjectsService.InitProjectDb(_crdtDbContext, projectData); + // Also trigger "data migrations" that CreateProject runs + await currentProjectService.SetupProjectContext(crdtProject); if (_seedWs) { await Api.CreateWritingSystem(new WritingSystem() diff --git a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs index 01e2d81acd..e67ba70d73 100644 --- a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs +++ b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs @@ -170,6 +170,10 @@ public virtual async Task CreateProject(CreateProjectRequest reques if (request.SeedNewProjectData) await SeedSystemData(dataModel, projectData); await (request.AfterCreate?.Invoke(serviceScope.ServiceProvider, crdtProject) ?? Task.CompletedTask); + // Ensure "data migrations" are executed on project creation (e.g. seeding morph types) + // These should happen AFTER the initial download, so they can be run conditionally based on + // the current state of the project. + await currentProjectService.SetupProjectContext(crdtProject); } catch (Exception e) { From 81903ebf2b2d9f1bfc47f036be6434c9eb94ff51 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 9 Jun 2026 16:12:42 +0200 Subject: [PATCH 08/16] Fix tests --- .../FwLite/FwLiteProjectSync/MiniLcmImport.cs | 3 +++ .../FwLite/LcmCrdt.Tests/Data/MigrationTests.cs | 16 ++++------------ .../LcmCrdt.Tests/Data/RegressionTestHelper.cs | 4 ++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs b/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs index 30dcd664a9..6e3590755a 100644 --- a/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs +++ b/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs @@ -31,6 +31,9 @@ public async Task Import(IProjectIdentifier project) FwProjectId: fwDataApi.ProjectId, AfterCreate: async (provider, _) => { + var currentProjectService = provider.GetRequiredService(); + // force-trigger "data migrations" + await currentProjectService.SetupProjectContext(currentProjectService.Project); var crdtApi = provider.GetRequiredService(); await ImportProject(crdtApi, fwDataApi, fwDataApi.EntryCount); })); diff --git a/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs b/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs index b8c985e6c4..37c68a6dd7 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs @@ -10,9 +10,8 @@ namespace LcmCrdt.Tests.Data; [Collection("MigrationTests")] -public class MigrationTests : IAsyncLifetime +public class MigrationTests { - private readonly RegressionTestHelper _helper = new("MigrationTest"); private static readonly JsonSerializerOptions IndentedHarmonyJsonOptions = new(TestJsonOptions.Harmony()) { WriteIndented = true @@ -25,21 +24,12 @@ internal static void Init() VerifierSettings.OmitContentFromException(); } - public Task InitializeAsync() - { - return Task.CompletedTask; - } - - public async Task DisposeAsync() - { - await _helper.DisposeAsync(); - } - [Theory] [InlineData(RegressionTestHelper.RegressionVersion.v1)] [InlineData(RegressionTestHelper.RegressionVersion.v2)] public async Task GetEntries_WorksAfterMigrationFromScriptedDb(RegressionTestHelper.RegressionVersion regressionVersion) { + await using RegressionTestHelper _helper = new($"{nameof(GetEntries_WorksAfterMigrationFromScriptedDb)}-{regressionVersion}"); await _helper.InitializeAsync(regressionVersion); var api = _helper.Services.GetRequiredService(); var hasEntries = false; @@ -58,6 +48,7 @@ public async Task GetEntries_WorksAfterMigrationFromScriptedDb(RegressionTestHel [Trait("Category", "Verified")] public async Task VerifyAfterMigrationFromScriptedDb(RegressionTestHelper.RegressionVersion regressionVersion) { + await using RegressionTestHelper _helper = new($"{nameof(VerifyAfterMigrationFromScriptedDb)}-{regressionVersion}"); await _helper.InitializeAsync(regressionVersion); var api = _helper.Services.GetRequiredService(); var crdtConfig = _helper.Services.GetRequiredService>().Value; @@ -110,6 +101,7 @@ await Task.WhenAll( [Trait("Category", "Verified")] public async Task VerifyRegeneratedSnapshotsAfterMigrationFromScriptedDb(RegressionTestHelper.RegressionVersion regressionVersion) { + await using RegressionTestHelper _helper = new($"{nameof(VerifyRegeneratedSnapshotsAfterMigrationFromScriptedDb)}-{regressionVersion}"); await _helper.InitializeAsync(regressionVersion); var api = _helper.Services.GetRequiredService(); var crdtConfig = _helper.Services.GetRequiredService>().Value; diff --git a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs index 7197664603..c61d1f651c 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs @@ -6,7 +6,7 @@ namespace LcmCrdt.Tests.Data; -public class RegressionTestHelper(string dbName): IAsyncLifetime +public class RegressionTestHelper(string dbName): IAsyncDisposable { private IHost _host = null!; private AsyncServiceScope _asyncScope; @@ -57,7 +57,7 @@ public async Task InitializeAsync(RegressionVersion version) await InitDbFromScripts(version); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await _asyncScope.DisposeAsync(); if (_host is IAsyncDisposable asyncDisposable) await asyncDisposable.DisposeAsync(); From 58399f1eb70af196be6c8d595a1b3b5df372996f Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 10 Jun 2026 09:32:06 +0700 Subject: [PATCH 09/16] apply some suggested minor fixes around debugger helpers --- backend/FwLite/LcmDebugger/FakeSyncSource.cs | 3 ++- backend/FwLite/LcmDebugger/Utils.cs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/FwLite/LcmDebugger/FakeSyncSource.cs b/backend/FwLite/LcmDebugger/FakeSyncSource.cs index aa697ef6b8..f488e2a263 100644 --- a/backend/FwLite/LcmDebugger/FakeSyncSource.cs +++ b/backend/FwLite/LcmDebugger/FakeSyncSource.cs @@ -50,7 +50,8 @@ public static FakeSyncSource FromJsonFile(string path, JsonSerializerOptions? op }; } - var changes = JsonSerializer.Deserialize>(File.OpenRead(path), options); + using var file = File.OpenRead(path); + var changes = JsonSerializer.Deserialize>(file, options); ArgumentNullException.ThrowIfNull(changes); ArgumentNullException.ThrowIfNull(changes.MissingFromClient); ArgumentNullException.ThrowIfNull(changes.ServerSyncState); diff --git a/backend/FwLite/LcmDebugger/Utils.cs b/backend/FwLite/LcmDebugger/Utils.cs index 3430e9244e..f6d0f9f75a 100644 --- a/backend/FwLite/LcmDebugger/Utils.cs +++ b/backend/FwLite/LcmDebugger/Utils.cs @@ -52,7 +52,9 @@ public static async Task NewProjectFromSyncable(this IServicePro var crdtProjectsService = services.GetRequiredService(); var crdtProject = await crdtProjectsService.CreateProject(new CrdtProjectsService.CreateProjectRequest("test-project", $"test-{Guid.NewGuid().ToString().Split('-')[0]}", projectId)); var crdtMiniLcmApi = (CrdtMiniLcmApi)await crdtProjectsService.OpenProject(crdtProject, services); - await services.GetRequiredService().SyncWith(syncable); + var syncResult = await services.GetRequiredService().SyncWith(syncable); + if (!syncResult.IsSynced) + throw new InvalidOperationException("New project sync failed."); return crdtMiniLcmApi; } From 25cffd94aab4d58ef711ac46279b265a3e67e0f8 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 10 Jun 2026 11:07:48 +0200 Subject: [PATCH 10/16] Stop specifying commit IDs --- backend/FwLite/LcmCrdt/CrdtProjectsService.cs | 2 +- .../FwLite/LcmCrdt/CurrentProjectService.cs | 2 +- .../FwLite/LcmCrdt/Objects/PreDefinedData.cs | 36 ++++--------------- 3 files changed, 8 insertions(+), 32 deletions(-) diff --git a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs index e67ba70d73..f7d7a4ead8 100644 --- a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs +++ b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs @@ -246,7 +246,7 @@ internal static async Task InitProjectDb(LcmCrdtDbContext db, ProjectData data) internal static async Task SeedSystemData(DataModel dataModel, ProjectData projectData) { - await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData, false); + await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData); await PreDefinedData.AddPredefinedComplexFormTypes(dataModel, projectData); await PreDefinedData.AddPredefinedPartsOfSpeech(dataModel, projectData); await PreDefinedData.AddPredefinedSemanticDomains(dataModel, projectData); diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index ba51f155d0..fd36eed952 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -114,7 +114,7 @@ async Task Execute() var projectData = await dbContext.ProjectData.AsNoTracking().FirstAsync(); if (!await dbContext.MorphTypes.AnyAsync()) { - await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData, true); + await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData); } if (EntrySearchServiceFactory is not null) diff --git a/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs b/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs index 5c1454530c..5f27c4f010 100644 --- a/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs +++ b/backend/FwLite/LcmCrdt/Objects/PreDefinedData.cs @@ -1,6 +1,5 @@ using LcmCrdt.Changes; using SIL.Harmony; -using UUIDNext; namespace LcmCrdt.Objects; @@ -16,32 +15,13 @@ public static class PreDefinedData public static readonly Guid AdjectivePartOfSpeechId = new("30d07580-5052-4d91-bc24-469b8b2d7df9"); public static readonly Guid AdverbPartOfSpeechId = new("46e4fe08-ffa0-4c8b-bf98-2c56f38904d9"); - // Seed commit-ids are derived per-project (UUIDv5 namespaced on projectId) so each project - // owns its own row in LexBox's CrdtCommits table — a shared constant id would collide on the - // primary key and the seed would get attributed to whichever project pushed first. - public static Guid ComplexFormTypesSeedCommitId(Guid projectId) => - Uuid.NewNameBased(projectId, "complex-form-types-seed"); - - public static Guid SemanticDomainsSeedCommitId(Guid projectId) => - Uuid.NewNameBased(projectId, "semantic-domains-seed"); - - public static Guid PartsOfSpeechSeedCommitId(Guid projectId) => - Uuid.NewNameBased(projectId, "parts-of-speech-seed"); - - public static Guid CustomViewsSeedCommitId(Guid projectId) => - Uuid.NewNameBased(projectId, "custom-views-seed"); - - public static Guid MorphTypesSeedCommitId(Guid projectId) => - Uuid.NewNameBased(projectId, "morph-types-seed"); - internal static async Task AddPredefinedComplexFormTypes(DataModel dataModel, ProjectData projectData) { await dataModel.AddChanges(projectData.ClientId, [ new CreateComplexFormType(CompoundComplexFormTypeId, new MultiString() { { "en", "Compound" } } ), new CreateComplexFormType(UnspecifiedComplexFormTypeId, new MultiString() { { "en", "Unspecified" } }) - ], - ComplexFormTypesSeedCommitId(projectData.Id)); + ]); } internal static async Task AddPredefinedSemanticDomains(DataModel dataModel, ProjectData projectData) @@ -56,8 +36,7 @@ await dataModel.AddChanges(projectData.ClientId, new CreateSemanticDomainChange(new Guid("46e4fe08-ffa0-4c8b-bf88-2c56f38904d4"), new MultiString() { { "en", "Body" } }, "2.1", false), new CreateSemanticDomainChange(new Guid("46e4fe08-ffa0-4c8b-bf88-2c56f38904d5"), new MultiString() { { "en", "Head" } }, "2.1.1", false), new CreateSemanticDomainChange(new Guid("46e4fe08-ffa0-4c8b-bf88-2c56f38904d6"), new MultiString() { { "en", "Eye" } }, "2.1.1.1", false), - ], - SemanticDomainsSeedCommitId(projectData.Id)); + ]); } public static async Task AddPredefinedPartsOfSpeech(DataModel dataModel, ProjectData projectData) @@ -69,8 +48,7 @@ await dataModel.AddChanges(projectData.ClientId, new CreatePartOfSpeechChange(VerbPartOfSpeechId, new MultiString() { { "en", "Verb" } }, true), new CreatePartOfSpeechChange(AdjectivePartOfSpeechId, new MultiString() { { "en", "Adjective" } }, true), new CreatePartOfSpeechChange(AdverbPartOfSpeechId, new MultiString() { { "en", "Adverb" } }, true), - ], - PartsOfSpeechSeedCommitId(projectData.Id)); + ]); } internal static async Task AddPredefinedCustomViews(DataModel dataModel, ProjectData projectData) @@ -100,14 +78,12 @@ await dataModel.AddChanges(projectData.ClientId, Vernacular = [new ViewWritingSystem { WsId = "de" }, new ViewWritingSystem { WsId = "de-Zxxx-x-audio" }], Analysis = [new ViewWritingSystem { WsId = "en" }] }) - ], - CustomViewsSeedCommitId(projectData.Id)); + ]); } - internal static async Task AddPredefinedMorphTypes(DataModel dataModel, ProjectData projectData, bool isMigration) + internal static async Task AddPredefinedMorphTypes(DataModel dataModel, ProjectData projectData) { await dataModel.AddChanges(projectData.ClientId, - [.. CanonicalMorphTypes.All.Values.Select(mt => new CreateMorphTypeChange(mt))], - isMigration ? Guid.NewGuid() : MorphTypesSeedCommitId(projectData.Id)); + [.. CanonicalMorphTypes.All.Values.Select(mt => new CreateMorphTypeChange(mt))]); } } From dc8a637dae70286a186c6165bb45a0afb97b7f94 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 10 Jun 2026 11:45:26 +0200 Subject: [PATCH 11/16] Change RegressionTestHelper fix --- .../LcmCrdt.Tests/Data/MigrationTests.cs | 16 +++++++++---- .../Data/RegressionTestHelper.cs | 24 ++++++++++--------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs b/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs index 37c68a6dd7..b8c985e6c4 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/MigrationTests.cs @@ -10,8 +10,9 @@ namespace LcmCrdt.Tests.Data; [Collection("MigrationTests")] -public class MigrationTests +public class MigrationTests : IAsyncLifetime { + private readonly RegressionTestHelper _helper = new("MigrationTest"); private static readonly JsonSerializerOptions IndentedHarmonyJsonOptions = new(TestJsonOptions.Harmony()) { WriteIndented = true @@ -24,12 +25,21 @@ internal static void Init() VerifierSettings.OmitContentFromException(); } + public Task InitializeAsync() + { + return Task.CompletedTask; + } + + public async Task DisposeAsync() + { + await _helper.DisposeAsync(); + } + [Theory] [InlineData(RegressionTestHelper.RegressionVersion.v1)] [InlineData(RegressionTestHelper.RegressionVersion.v2)] public async Task GetEntries_WorksAfterMigrationFromScriptedDb(RegressionTestHelper.RegressionVersion regressionVersion) { - await using RegressionTestHelper _helper = new($"{nameof(GetEntries_WorksAfterMigrationFromScriptedDb)}-{regressionVersion}"); await _helper.InitializeAsync(regressionVersion); var api = _helper.Services.GetRequiredService(); var hasEntries = false; @@ -48,7 +58,6 @@ public async Task GetEntries_WorksAfterMigrationFromScriptedDb(RegressionTestHel [Trait("Category", "Verified")] public async Task VerifyAfterMigrationFromScriptedDb(RegressionTestHelper.RegressionVersion regressionVersion) { - await using RegressionTestHelper _helper = new($"{nameof(VerifyAfterMigrationFromScriptedDb)}-{regressionVersion}"); await _helper.InitializeAsync(regressionVersion); var api = _helper.Services.GetRequiredService(); var crdtConfig = _helper.Services.GetRequiredService>().Value; @@ -101,7 +110,6 @@ await Task.WhenAll( [Trait("Category", "Verified")] public async Task VerifyRegeneratedSnapshotsAfterMigrationFromScriptedDb(RegressionTestHelper.RegressionVersion regressionVersion) { - await using RegressionTestHelper _helper = new($"{nameof(VerifyRegeneratedSnapshotsAfterMigrationFromScriptedDb)}-{regressionVersion}"); await _helper.InitializeAsync(regressionVersion); var api = _helper.Services.GetRequiredService(); var crdtConfig = _helper.Services.GetRequiredService>().Value; diff --git a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs index c61d1f651c..eeba9fc4ea 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs @@ -6,24 +6,19 @@ namespace LcmCrdt.Tests.Data; -public class RegressionTestHelper(string dbName): IAsyncDisposable +public class RegressionTestHelper(string projectName) : IAsyncLifetime { private IHost _host = null!; private AsyncServiceScope _asyncScope; + //unique db path per instance, so CurrentProjectService doesn't think a db has already run migrations + private readonly CrdtProject _crdtProject = new(projectName, $"{projectName}-{Guid.NewGuid():N}.sqlite"); public IServiceProvider Services => _asyncScope.ServiceProvider; private async Task InitDbFromScripts(RegressionVersion version) { var initialSqlFile = GetFilePath($"Scripts/{version}.sql"); var projectsService = _asyncScope.ServiceProvider.GetRequiredService(); - var crdtProject = new CrdtProject(dbName, $"{dbName}.sqlite"); - if (File.Exists(crdtProject.DbPath)) - { - using var clearConn = new SqliteConnection($"Data Source={crdtProject.DbPath}"); - SqliteConnection.ClearPool(clearConn); - File.Delete(crdtProject.DbPath); - } - projectsService.SetupProjectContextForNewDb(crdtProject); + projectsService.SetupProjectContextForNewDb(_crdtProject); await using var lcmCrdtDbContext = await _asyncScope.ServiceProvider.GetRequiredService>().CreateDbContextAsync(); var sql = await File.ReadAllTextAsync(initialSqlFile); var dbConnection = lcmCrdtDbContext.Database.GetDbConnection(); @@ -39,7 +34,7 @@ private async Task InitDbFromScripts(RegressionVersion version) await dbConnection.CloseAsync(); //setup again to trigger migrations - await projectsService.SetupProjectContext(crdtProject); + await projectsService.SetupProjectContext(_crdtProject); } public Task InitializeAsync() @@ -57,7 +52,7 @@ public async Task InitializeAsync(RegressionVersion version) await InitDbFromScripts(version); } - public async ValueTask DisposeAsync() + public async Task DisposeAsync() { await _asyncScope.DisposeAsync(); if (_host is IAsyncDisposable asyncDisposable) await asyncDisposable.DisposeAsync(); @@ -65,6 +60,13 @@ public async ValueTask DisposeAsync() { _host.Dispose(); } + + if (File.Exists(_crdtProject.DbPath)) + { + using var connection = new SqliteConnection($"Data Source={_crdtProject.DbPath}"); + SqliteConnection.ClearPool(connection); + File.Delete(_crdtProject.DbPath); + } } private static string GetFilePath(string name, [CallerFilePath] string sourceFile = "") From d4db943e1896f00d8f18c4fcade10d3da59cccd2 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 10 Jun 2026 13:53:00 +0200 Subject: [PATCH 12/16] Revert running "deprecated" data migrations in all regression tests --- .../LcmCrdt.Tests/Data/DownloadProjectTests.cs | 3 +-- .../LcmCrdt.Tests/Data/RegressionTestHelper.cs | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs b/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs index bb65d29688..d5f423ee83 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/DownloadProjectTests.cs @@ -1,6 +1,5 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using SIL.Harmony.Core; namespace LcmCrdt.Tests.Data; @@ -13,7 +12,7 @@ public class DownloadProjectTests : IAsyncLifetime public async Task InitializeAsync() { - await _helper.InitializeAsync(RegressionTestHelper.RegressionVersion.v2); + await _helper.InitializeAsync(RegressionTestHelper.RegressionVersion.v2, withDataMigrations: true); //add a change after migration which creates MorphTypes await _helper.Services.GetRequiredService().CreateEntry(new Entry() { diff --git a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs index eeba9fc4ea..2285d58adb 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs @@ -33,8 +33,9 @@ private async Task InitDbFromScripts(RegressionVersion version) //need to close the connection, otherwise the collations won't get created, they would normally be created on open or save, so we're closing so they get created when EF opens the connection. await dbConnection.CloseAsync(); - //setup again to trigger migrations - await projectsService.SetupProjectContext(_crdtProject); + await lcmCrdtDbContext.Database.MigrateAsync(); + + await projectsService.RefreshProjectData(); } public Task InitializeAsync() @@ -42,7 +43,7 @@ public Task InitializeAsync() return InitializeAsync(RegressionVersion.v2); } - public async Task InitializeAsync(RegressionVersion version) + public async Task InitializeAsync(RegressionVersion version, bool withDataMigrations = false) { var builder = Host.CreateEmptyApplicationBuilder(null); builder.Services.AddTestLcmCrdtClient(); @@ -50,6 +51,16 @@ public async Task InitializeAsync(RegressionVersion version) var services = _host.Services; _asyncScope = services.CreateAsyncScope(); await InitDbFromScripts(version); + + // Data migrations are already on their way out. It doesn't really make sense for all tests to run them, + // which would change a bunch of verified project snapshots. + // When data migrations are removed, we should run this code unconditionally + // at the end of InitDbFromScripts (instead of the current db-migration and refresh-project-data approach) + // Because that's closer to the prod code. + if (withDataMigrations) + { + await Services.GetRequiredService().SetupProjectContext(_crdtProject); + } } public async Task DisposeAsync() From ab4c2653ef7f650b74c7220ce5dfdffc83398859 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 10 Jun 2026 14:09:27 +0200 Subject: [PATCH 13/16] Implement CreateMorphType in MiniLcm --- backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs | 5 +++++ backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs | 6 ++++++ .../FwLite/FwLiteProjectSync/Import/ResumableImportApi.cs | 4 ++++ backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs | 3 --- backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 6 ++++++ backend/FwLite/MiniLcm/IMiniLcmWriteApi.cs | 1 + .../Normalization/MiniLcmApiWriteNormalizationWrapper.cs | 5 +++++ backend/FwLite/MiniLcm/SyncHelpers/MorphTypeSync.cs | 6 +++--- 8 files changed, 30 insertions(+), 6 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs index 4fd3f1447a..b9536c03bf 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs @@ -592,6 +592,11 @@ internal MorphType FromLcmMorphType(IMoMorphType morphType) }; } + public Task CreateMorphType(MorphType morphType) + { + throw new NotSupportedException("Morph types cannot be created in fwdata; they are predefined"); + } + public Task UpdateMorphType(Guid id, UpdateObjectInput update) { var lcmMorphType = MorphTypeRepository.GetObject(id); diff --git a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs index 698e98bb04..a4351c5366 100644 --- a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs +++ b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs @@ -126,6 +126,12 @@ public Task DeleteComplexFormType(Guid id) return Task.CompletedTask; } + public Task CreateMorphType(MorphType morphType) + { + DryRunRecords.Add(new DryRunRecord(nameof(CreateMorphType), $"Create morph type {morphType.Kind} ({morphType.Id})")); + return Task.FromResult(morphType); + } + public async Task UpdateMorphType(Guid id, UpdateObjectInput update) { DryRunRecords.Add(new DryRunRecord(nameof(UpdateMorphType), $"Update morph type {id}")); diff --git a/backend/FwLite/FwLiteProjectSync/Import/ResumableImportApi.cs b/backend/FwLite/FwLiteProjectSync/Import/ResumableImportApi.cs index 3e562d8b3c..942b81f900 100644 --- a/backend/FwLite/FwLiteProjectSync/Import/ResumableImportApi.cs +++ b/backend/FwLite/FwLiteProjectSync/Import/ResumableImportApi.cs @@ -62,6 +62,10 @@ async Task IMiniLcmWriteApi.CreateSemanticDomain(SemanticDomain { return await HasCreated(semanticDomain, _api.GetSemanticDomains(), () => _api.CreateSemanticDomain(semanticDomain)); } + async Task IMiniLcmWriteApi.CreateMorphType(MorphType morphType) + { + return await HasCreated(morphType, _api.GetMorphTypes(), () => _api.CreateMorphType(morphType)); + } async Task IMiniLcmWriteApi.CreatePublication(Publication publication) { return await HasCreated(publication, _api.GetPublications(), () => _api.CreatePublication(publication)); diff --git a/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs b/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs index 6e3590755a..30dcd664a9 100644 --- a/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs +++ b/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs @@ -31,9 +31,6 @@ public async Task Import(IProjectIdentifier project) FwProjectId: fwDataApi.ProjectId, AfterCreate: async (provider, _) => { - var currentProjectService = provider.GetRequiredService(); - // force-trigger "data migrations" - await currentProjectService.SetupProjectContext(currentProjectService.Project); var crdtApi = provider.GetRequiredService(); await ImportProject(crdtApi, fwDataApi, fwDataApi.EntryCount); })); diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 5a4053ec35..58e963df61 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -373,6 +373,12 @@ public async IAsyncEnumerable GetMorphTypes() return await repo.MorphTypes.SingleOrDefaultAsync(m => m.Kind == kind); } + public async Task CreateMorphType(MorphType morphType) + { + await AddChange(new CreateMorphTypeChange(morphType)); + return await GetMorphType(morphType.Id) ?? throw NotFoundException.ForType(morphType.Id); + } + public async Task UpdateMorphType(Guid id, UpdateObjectInput update) { await AddChange(new JsonPatchChange(id, update.Patch)); diff --git a/backend/FwLite/MiniLcm/IMiniLcmWriteApi.cs b/backend/FwLite/MiniLcm/IMiniLcmWriteApi.cs index 6d59b3e10c..7bcf31255a 100644 --- a/backend/FwLite/MiniLcm/IMiniLcmWriteApi.cs +++ b/backend/FwLite/MiniLcm/IMiniLcmWriteApi.cs @@ -45,6 +45,7 @@ Task UpdateWritingSystem(WritingSystemId id, #endregion #region MorphType + Task CreateMorphType(MorphType morphType); Task UpdateMorphType(Guid id, UpdateObjectInput update); Task UpdateMorphType(MorphType before, MorphType after, IMiniLcmApi? api = null); #endregion diff --git a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiWriteNormalizationWrapper.cs b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiWriteNormalizationWrapper.cs index aa9f030172..bf2d901f8f 100644 --- a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiWriteNormalizationWrapper.cs +++ b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiWriteNormalizationWrapper.cs @@ -209,6 +209,11 @@ private static ComplexFormType NormalizeComplexFormType(ComplexFormType cft) #region MorphType + public async Task CreateMorphType(MorphType morphType) + { + return await _api.CreateMorphType(NormalizeMorphType(morphType)); + } + public Task UpdateMorphType(Guid id, UpdateObjectInput update) { return _api.UpdateMorphType(id, NormalizePatch(update)); diff --git a/backend/FwLite/MiniLcm/SyncHelpers/MorphTypeSync.cs b/backend/FwLite/MiniLcm/SyncHelpers/MorphTypeSync.cs index 2a89dcadc8..9da1655d40 100644 --- a/backend/FwLite/MiniLcm/SyncHelpers/MorphTypeSync.cs +++ b/backend/FwLite/MiniLcm/SyncHelpers/MorphTypeSync.cs @@ -56,10 +56,10 @@ public static async Task Sync(MorphType before, private class MorphTypeDiffApi(IMiniLcmApi api) : ObjectWithIdCollectionDiffApi { - public override Task Add(MorphType currentMorphType) + public override async Task Add(MorphType currentMorphType) { - throw new InvalidOperationException( - $"MorphTypes are predefined and cannot be created. Unexpected morph type: {currentMorphType.Kind} ({currentMorphType.Id}). This indicates a data inconsistency."); + await api.CreateMorphType(currentMorphType); + return 1; } public override Task Remove(MorphType beforeMorphType) From 05efac30b3032786a21aaaa4e6fffa26b0035588 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 10 Jun 2026 14:38:40 +0200 Subject: [PATCH 14/16] Add #2350 comments for follow up work --- backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs | 1 + backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs | 2 +- backend/FwLite/LcmCrdt/CrdtProjectsService.cs | 1 + backend/FwLite/LcmCrdt/CurrentProjectService.cs | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs b/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs index 30dcd664a9..a7cb184ace 100644 --- a/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs +++ b/backend/FwLite/FwLiteProjectSync/MiniLcmImport.cs @@ -72,6 +72,7 @@ public async Task ImportProject(IMiniLcmApi importTo, IMiniLcmApi importFrom, in } // Morph types are created automatically for CRDT projects, so we update them instead of creating them + // Optimize this to a simple foreach like above in #2350 var importFromMorphTypes = await importFrom.GetMorphTypes().ToArrayAsync(); var existingMorphTypes = await importTo.GetMorphTypes().ToArrayAsync(); await MorphTypeSync.Sync(existingMorphTypes, importFromMorphTypes, importTo); diff --git a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs index 2285d58adb..6238cd3157 100644 --- a/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs +++ b/backend/FwLite/LcmCrdt.Tests/Data/RegressionTestHelper.cs @@ -54,7 +54,7 @@ public async Task InitializeAsync(RegressionVersion version, bool withDataMigrat // Data migrations are already on their way out. It doesn't really make sense for all tests to run them, // which would change a bunch of verified project snapshots. - // When data migrations are removed, we should run this code unconditionally + // When data migrations are removed (#2350), we should run this code unconditionally // at the end of InitDbFromScripts (instead of the current db-migration and refresh-project-data approach) // Because that's closer to the prod code. if (withDataMigrations) diff --git a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs index f7d7a4ead8..e6bf67364d 100644 --- a/backend/FwLite/LcmCrdt/CrdtProjectsService.cs +++ b/backend/FwLite/LcmCrdt/CrdtProjectsService.cs @@ -173,6 +173,7 @@ public virtual async Task CreateProject(CreateProjectRequest reques // Ensure "data migrations" are executed on project creation (e.g. seeding morph types) // These should happen AFTER the initial download, so they can be run conditionally based on // the current state of the project. + // probably just remove this in #2350 await currentProjectService.SetupProjectContext(crdtProject); } catch (Exception e) diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index fd36eed952..830cd213dc 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -112,6 +112,7 @@ async Task Execute() // Must happen BEFORE FTS regeneration so headwords include morph-type tokens. var dataModel = services.GetRequiredService(); var projectData = await dbContext.ProjectData.AsNoTracking().FirstAsync(); + // Remove in #2350 if (!await dbContext.MorphTypes.AnyAsync()) { await PreDefinedData.AddPredefinedMorphTypes(dataModel, projectData); From 15e187173914569089a16b97b7f55467ee88b3c6 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 10 Jun 2026 14:39:49 +0200 Subject: [PATCH 15/16] Replace deprecated test with new test --- .../MiniLcmTests/MorphTypeTests.cs | 10 ++++++++++ .../FwLite/MiniLcm.Tests/MorphTypeSyncTests.cs | 18 ------------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/MorphTypeTests.cs b/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/MorphTypeTests.cs index d248f215dc..ab6c2554d0 100644 --- a/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/MorphTypeTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/MiniLcmTests/MorphTypeTests.cs @@ -16,4 +16,14 @@ public override async Task DisposeAsync() await base.DisposeAsync(); await _fixture.DisposeAsync(); } + + // CRDT only, because fwdata does not support creating morph-types + [Fact] + public async Task CreateMorphType_Works() + { + var morphType = CanonicalMorphTypes.All.First().Value.Copy(); + var createdMorphType = await Api.CreateMorphType(morphType); + createdMorphType.Should().NotBeNull(); + createdMorphType.Should().BeEquivalentTo(morphType); + } } diff --git a/backend/FwLite/MiniLcm.Tests/MorphTypeSyncTests.cs b/backend/FwLite/MiniLcm.Tests/MorphTypeSyncTests.cs index d4b7df19ce..a657fb11be 100644 --- a/backend/FwLite/MiniLcm.Tests/MorphTypeSyncTests.cs +++ b/backend/FwLite/MiniLcm.Tests/MorphTypeSyncTests.cs @@ -4,24 +4,6 @@ namespace MiniLcm.Tests; public class MorphTypeSyncTests { - [Fact] - public async Task Sync_ThrowsOnAdd_WhenAfterContainsExtraMorphType() - { - var before = CanonicalMorphTypes.All.Values.ToArray(); - var extraMorphType = new MorphType - { - Id = Guid.NewGuid(), - Kind = MorphTypeKind.Unknown, - Name = new MultiString { { "en", "bogus" } }, - }; - var after = before.Append(extraMorphType).ToArray(); - - var act = () => MorphTypeSync.Sync(before, after, null!); - - await act.Should().ThrowAsync() - .WithMessage("*cannot be created*"); - } - [Fact] public async Task Sync_ThrowsOnRemove_WhenAfterIsMissingMorphType() { From 8de8c07e71bafaf86f5581d2d8aae85e8b1dfd9e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 11 Jun 2026 09:38:36 +0700 Subject: [PATCH 16/16] Return existing morph type if found in `CreateMorphType` to avoid duplicates. --- backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 58e963df61..77635100e3 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -375,6 +375,8 @@ public async IAsyncEnumerable GetMorphTypes() public async Task CreateMorphType(MorphType morphType) { + //I don't like returning a different object than what the user requested, it feels very unexpected, however this is pretty much what happens in the change anyway and that can't be avoided + if (await GetMorphType(morphType.Kind) is {} actualMorphType) return actualMorphType; await AddChange(new CreateMorphTypeChange(morphType)); return await GetMorphType(morphType.Id) ?? throw NotFoundException.ForType(morphType.Id); }