diff --git a/src/SIL.Harmony.Core/IRemoteResourceService.cs b/src/SIL.Harmony.Core/IRemoteResourceService.cs index 3e4eefa..8eb712f 100644 --- a/src/SIL.Harmony.Core/IRemoteResourceService.cs +++ b/src/SIL.Harmony.Core/IRemoteResourceService.cs @@ -5,7 +5,7 @@ /// the remote Id is opaque to the CRDT lib and could be a URL or some other identifier provided by the backend /// the local path returned for the application code to use as required, it could be a URL if needed also. /// -public interface IRemoteResourceService +public interface IRemoteResourceService where TMetadata : class { /// /// instructs application code to download a resource from the remote server @@ -22,8 +22,8 @@ public interface IRemoteResourceService /// id of the resource in the CRDT /// full path to the resource on the local machine /// an upload result with the remote id, the id will be stored and transmitted to other clients so they can also download the resource - Task UploadResource(Guid resourceId, string localPath); + Task> UploadResource(Guid resourceId, string localPath); } public record DownloadResult(string LocalPath); -public record UploadResult(string RemoteId); +public record UploadResult(string RemoteId, TMetadata? Metadata = null) where TMetadata : class; diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index a89454e..40491b5 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -36,7 +36,6 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi services.AddCrdtData(config => { config.EnableProjectedTables = true; - config.AddRemoteResourceEntity(); config.ChangeTypeListBuilder .Add() .Add() @@ -84,6 +83,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi builder.HasIndex(wt => new { wt.WordId, wt.TagId }).IsUnique(); }); }); + services.AddCrdtRemoteResources(); return services; } } diff --git a/src/SIL.Harmony.Sample/MediaMetadata.cs b/src/SIL.Harmony.Sample/MediaMetadata.cs new file mode 100644 index 0000000..71d4183 --- /dev/null +++ b/src/SIL.Harmony.Sample/MediaMetadata.cs @@ -0,0 +1,3 @@ +namespace SIL.Harmony.Sample; + +public record MediaMetadata(string FileName, string MimeType, long SizeBytes); diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index d02fa1a..3a780f3 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -93,16 +93,19 @@ Relational:TableName: LocalResource Relational:ViewName: Relational:ViewSchema: - EntityType: RemoteResource + EntityType: RemoteResource Properties: Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd DeletedAt (DateTimeOffset?) + Metadata (MediaMetadata) + Annotations: + Relational:ColumnType: jsonb RemoteId (string) SnapshotId (no field, Guid?) Shadow FK Index Keys: Id PK Foreign keys: - RemoteResource {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull + RemoteResource {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull Indexes: SnapshotId Unique Annotations: diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesMetadataTests.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesMetadataTests.cs new file mode 100644 index 0000000..1d52a7e --- /dev/null +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesMetadataTests.cs @@ -0,0 +1,103 @@ +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Resource; +using SIL.Harmony.Sample; + +namespace SIL.Harmony.Tests.ResourceTests; + +public class RemoteResourcesMetadataTests : DataModelTestBase +{ + private RemoteServiceMock _remoteServiceMock = new(); + private ResourceService _resourceService => + _services.GetRequiredService>(); + + private string CreateFile(string contents, [CallerMemberName] string fileName = "") + { + var filePath = Path.GetFullPath(fileName + ".txt"); + File.WriteAllText(filePath, contents); + return filePath; + } + + private static MediaMetadata SampleMetadata(string fileName = "photo.jpg") => + new(fileName, "image/jpeg", 102400); + + [Fact] + public async Task CreateWithUpload_UsesPassedMetadataWhenUploadReturnsNone() + { + var metadata = SampleMetadata(); + var localFile = CreateFile("image data"); + var resource = await _resourceService.AddLocalResource(localFile, _localClientId, metadata, + resourceService: _remoteServiceMock); + resource.Metadata.Should().BeEquivalentTo(metadata); + var stored = await DataModel.GetLatest>(resource.Id); + stored!.Metadata.Should().BeEquivalentTo(metadata); + (await _resourceService.GetResource(resource.Id))!.Metadata.Should().BeEquivalentTo(metadata); + } + + [Fact] + public async Task CreateWithUpload_UploadMetadataOverridesPassedMetadata() + { + var passedMetadata = SampleMetadata("passed.jpg"); + var uploadMetadata = new MediaMetadata("from-upload.jpg", "image/png", 204800); + var localFile = CreateFile("image data"); + _remoteServiceMock.SetUploadMetadata(localFile, uploadMetadata); + var resource = await _resourceService.AddLocalResource(localFile, _localClientId, passedMetadata, + resourceService: _remoteServiceMock); + resource.Metadata.Should().BeEquivalentTo(uploadMetadata); + resource.Metadata.Should().NotBeEquivalentTo(passedMetadata); + var stored = await DataModel.GetLatest>(resource.Id); + stored!.Metadata.Should().BeEquivalentTo(uploadMetadata); + (await _resourceService.GetResource(resource.Id))!.Metadata.Should().BeEquivalentTo(uploadMetadata); + } + + [Fact] + public async Task CreatePendingUpload_IncludesMetadata() + { + var metadata = SampleMetadata("pending.mp4"); + var localFile = CreateFile("video data"); + var resource = await _resourceService.AddLocalResource(localFile, _localClientId, metadata, + resourceService: null); + resource.Metadata.Should().BeEquivalentTo(metadata); + var stored = await DataModel.GetLatest>(resource.Id); + stored!.Metadata.Should().BeEquivalentTo(metadata); + } + + [Fact] + public async Task AllResources_IncludesMetadata() + { + var metadata = SampleMetadata(); + var localFile = CreateFile("list test"); + await _resourceService.AddLocalResource(localFile, _localClientId, metadata, resourceService: _remoteServiceMock); + var all = await _resourceService.AllResources(); + all.Should().ContainSingle().Which.Metadata.Should().BeEquivalentTo(metadata); + } + + [Fact] + public async Task SetResourceMetadata_UpdatesAndSyncs() + { + var metadata = SampleMetadata(); + var localFile = CreateFile("sync test"); + var resource = await _resourceService.AddLocalResource(localFile, _localClientId, metadata, + resourceService: _remoteServiceMock); + var remoteClient = ForkDatabase(); + var updated = metadata with { FileName = "renamed.jpg" }; + await _resourceService.SetResourceMetadata(resource.Id, _localClientId, updated); + await DataModel.SyncWith(remoteClient.DataModel); + (await _resourceService.GetResource(resource.Id))!.Metadata.Should().BeEquivalentTo(updated); + (await remoteClient.DataModel.GetLatest>(resource.Id))!.Metadata + .Should().BeEquivalentTo(updated); + } + + [Fact] + public async Task CreateWithoutMetadata_DeserializesWithNullMetadata() + { + var resourceId = Guid.NewGuid(); + var remoteId = _remoteServiceMock.CreateRemoteResource("legacy"); + await DataModel.AddChange(_localClientId, + new CreateRemoteResourceChange(resourceId, remoteId)); + var stored = await DataModel.GetLatest>(resourceId); + stored!.Metadata.Should().BeNull(); + stored.RemoteId.Should().Be(remoteId); + } +} + diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs index 214761d..27fd4ac 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs @@ -1,33 +1,28 @@ using System.Runtime.CompilerServices; using Microsoft.Extensions.DependencyInjection; using SIL.Harmony.Resource; - +using SIL.Harmony.Sample; namespace SIL.Harmony.Tests.ResourceTests; - public class RemoteResourcesTests : DataModelTestBase { private RemoteServiceMock _remoteServiceMock = new(); - private ResourceService _resourceService => _services.GetRequiredService(); - + private ResourceService _resourceService => _services.GetRequiredService>(); public RemoteResourcesTests() { } - private string CreateFile(string contents, [CallerMemberName] string fileName = "") { var filePath = Path.GetFullPath(fileName + ".txt"); File.WriteAllText(filePath, contents); return filePath; } - private async Task<(Guid resourceId, string remoteId)> SetupRemoteResource(string fileContents) { var remoteId = _remoteServiceMock.CreateRemoteResource(fileContents); var resourceId = Guid.NewGuid(); - await DataModel.AddChange(_localClientId, new CreateRemoteResourceChange(resourceId, remoteId)); + await DataModel.AddChange(_localClientId, new CreateRemoteResourceChange(resourceId, remoteId)); return (resourceId, remoteId); } - private async Task<(Guid resourceId, string localPath)> SetupLocalFile(string contents, [CallerMemberName] string fileName = "") { var file = CreateFile(contents, fileName); @@ -35,63 +30,50 @@ private string CreateFile(string contents, [CallerMemberName] string fileName = var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); return (crdtResource.Id, file); } - [Fact] public async Task CreatingAResourceResultsInPendingLocalResources() { var (_, file) = await SetupLocalFile("contents"); - //act var pending = await _resourceService.ListResourcesPendingUpload(); - pending.Should().ContainSingle().Which.LocalPath.Should().Be(file); } - [Fact] public async Task ResourcesNotLocalShouldShowUpAsNotDownloaded() { var (resourceId, remoteId) = await SetupRemoteResource("test"); - //act var pending = await _resourceService.ListResourcesPendingDownload(); - var remoteResource = pending.Should().ContainSingle().Subject; remoteResource.RemoteId.Should().Be(remoteId); remoteResource.Id.Should().Be(resourceId); } - [Fact] public async Task CanUploadFileToRemote() { var fileContents = "resource"; var localFile = CreateFile(fileContents); - //act var crdtResource = await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); - - var resource = await DataModel.GetLatest(crdtResource.Id); + var resource = await DataModel.GetLatest>(crdtResource.Id); ArgumentNullException.ThrowIfNull(resource); ArgumentNullException.ThrowIfNull(resource.RemoteId); _remoteServiceMock.ReadFile(resource.RemoteId).Should().Be(fileContents); var pendingUpload = await _resourceService.ListResourcesPendingUpload(); pendingUpload.Should().BeEmpty(); } - [Fact] public async Task IfUploadingFailsTheResourceIsStillAddedAsPendingUpload() { var fileContents = "resource"; var localFile = CreateFile(fileContents); - //todo setup a mock that throws an exception when uploading _remoteServiceMock.ThrowOnUpload(localFile); - //act var crdtResource = await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); - var harmonyResource = await _resourceService.GetResource(crdtResource.Id); harmonyResource.Should().NotBeNull(); harmonyResource.Id.Should().Be(crdtResource.Id); @@ -100,55 +82,45 @@ public async Task IfUploadingFailsTheResourceIsStillAddedAsPendingUpload() var pendingUpload = await _resourceService.ListResourcesPendingUpload(); pendingUpload.Should().ContainSingle().Which.Id.Should().Be(crdtResource.Id); } - [Fact] public async Task WillUploadMultiplePendingLocalFilesAtOnce() { await SetupLocalFile("file1", "file1"); await SetupLocalFile("file2", "file2"); - //act await _resourceService.UploadPendingResources(_localClientId, _remoteServiceMock); - _remoteServiceMock.ListRemoteFiles() .Select(Path.GetFileName) .Should() .Contain(["file1.txt", "file2.txt"]); } - [Fact] public async Task CanDownloadFileFromRemote() { var fileContents = "resource"; var (resourceId, _) = await SetupRemoteResource(fileContents); - //act var localResource = await _resourceService.DownloadResource(resourceId, _remoteServiceMock); - localResource.Id.Should().Be(resourceId); var actualFileContents = await File.ReadAllTextAsync(localResource.LocalPath, TestContext.Current.CancellationToken); actualFileContents.Should().Be(fileContents); var pendingDownloads = await _resourceService.ListResourcesPendingDownload(); pendingDownloads.Should().BeEmpty(); } - [Fact] public async Task CanGetALocalResourceGivenAnId() { var file = CreateFile("resource"); //because resource service is null the file is not uploaded var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); - //act var localResource = await _resourceService.GetLocalResource(crdtResource.Id); - localResource.Should().NotBeNull(); localResource!.LocalPath.Should().Be(file); } - [Fact] public async Task LocalResourceIsNullIfNotDownloaded() { @@ -156,24 +128,22 @@ public async Task LocalResourceIsNullIfNotDownloaded() var localResource = await _resourceService.GetLocalResource(resourceId); localResource.Should().BeNull(); } - [Fact] public async Task CanListAllResources() { var (localResourceId, localResourcePath) = await SetupLocalFile("localOnly", "localOnly.txt"); var (remoteResourceId, remoteId) = await SetupRemoteResource("remoteOnly"); var localAndRemoteResource = await _resourceService.AddLocalResource(CreateFile("localAndRemove"), _localClientId, resourceService: _remoteServiceMock); - var crdtResources = await _resourceService.AllResources(); crdtResources.Should().BeEquivalentTo( [ - new HarmonyResource + new HarmonyResource { Id = localResourceId, LocalPath = localResourcePath, RemoteId = null }, - new HarmonyResource + new HarmonyResource { Id = remoteResourceId, LocalPath = null, @@ -182,7 +152,6 @@ public async Task CanListAllResources() localAndRemoteResource ]); } - [Fact] public async Task CanGetAResourceGivenAnId() { @@ -191,14 +160,13 @@ public async Task CanGetAResourceGivenAnId() var localAndRemoteResource = await _resourceService.AddLocalResource(CreateFile("localAndRemove"), _localClientId, resourceService: _remoteServiceMock); - - (await _resourceService.GetResource(localResourceId)).Should().BeEquivalentTo(new HarmonyResource + (await _resourceService.GetResource(localResourceId)).Should().BeEquivalentTo(new HarmonyResource { Id = localResourceId, LocalPath = localResourcePath, RemoteId = null }); - (await _resourceService.GetResource(remoteResourceId)).Should().BeEquivalentTo(new HarmonyResource + (await _resourceService.GetResource(remoteResourceId)).Should().BeEquivalentTo(new HarmonyResource { Id = remoteResourceId, LocalPath = null, @@ -207,7 +175,6 @@ public async Task CanGetAResourceGivenAnId() (await _resourceService.GetResource(localAndRemoteResource.Id)).Should().BeEquivalentTo(localAndRemoteResource); (await _resourceService.GetResource(Guid.NewGuid())).Should().BeNull(); } - [Fact] public async Task DeleteResource_RemovesLocalResource() { @@ -215,16 +182,13 @@ public async Task DeleteResource_RemovesLocalResource() var (resourceId, localPath) = await SetupLocalFile("delete-local"); (await _resourceService.GetResource(resourceId)).Should().NotBeNull(); (await _resourceService.GetLocalResource(resourceId)).Should().NotBeNull(); - // Act: delete the resource await _resourceService.DeleteResource(_localClientId, resourceId); - // Assert: resource is gone from all APIs (await _resourceService.GetResource(resourceId)).Should().BeNull(); (await _resourceService.GetLocalResource(resourceId)).Should().BeNull(); (await _resourceService.AllResources()).Should().NotContain(r => r.Id == resourceId); } - [Fact] public async Task DeleteResource_RemovesRemoteResource() { @@ -232,13 +196,12 @@ public async Task DeleteResource_RemovesRemoteResource() var (resourceId, remoteId) = await SetupRemoteResource("delete-remote"); (await _resourceService.GetResource(resourceId)).Should().NotBeNull(); (await _resourceService.GetLocalResource(resourceId)).Should().BeNull(); - // Act: delete the resource await _resourceService.DeleteResource(_localClientId, resourceId); - // Assert: resource is gone from all APIs (await _resourceService.GetResource(resourceId)).Should().BeNull(); (await _resourceService.GetLocalResource(resourceId)).Should().BeNull(); (await _resourceService.AllResources()).Should().NotContain(r => r.Id == resourceId); } } + diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs index 33a4f3b..264535c 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs @@ -1,17 +1,22 @@ +using SIL.Harmony.Sample; + namespace SIL.Harmony.Tests.ResourceTests; -public class RemoteServiceMock : IRemoteResourceService +public class RemoteServiceMock : IRemoteResourceService { public static readonly string RemotePath = Directory.CreateTempSubdirectory("RemoteServiceMock").FullName; + private readonly Dictionary _metadata = new(); /// /// directly creates a remote resource /// /// the remote id - public string CreateRemoteResource(string contents) + public string CreateRemoteResource(string contents, MediaMetadata? metadata = null) { var filePath = Path.Combine(RemotePath, Guid.NewGuid().ToString("N") + ".txt"); File.WriteAllText(filePath, contents); + if (metadata is not null) _metadata.Add(filePath, metadata); + return filePath; } @@ -26,7 +31,7 @@ public Task DownloadResource(string remoteId, string localResour private readonly Queue _throwOnUpload = new(); - public async Task UploadResource(Guid resourceId, string localPath) + public async Task> UploadResource(Guid resourceId, string localPath) { await Task.Yield();//yield back to the scheduler to emulate how exceptions are thrown if (_throwOnUpload.TryPeek(out var throwOnUpload)) @@ -39,9 +44,13 @@ public async Task UploadResource(Guid resourceId, string localPath } var remoteId = Path.Combine(RemotePath, Path.GetFileName(localPath)); File.Copy(localPath, remoteId); - return new UploadResult(remoteId); + _metadata.TryGetValue(localPath, out var metadata); + return new UploadResult(remoteId, metadata); } + public void SetUploadMetadata(string localPath, MediaMetadata metadata) => + _metadata[Path.GetFullPath(localPath)] = metadata; + public void ThrowOnUpload(string localPath) { _throwOnUpload.Enqueue(localPath); diff --git a/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs index a52e101..15fc149 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs @@ -1,23 +1,20 @@ -using System.Runtime.CompilerServices; +using System.Runtime.CompilerServices; using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Sample; using SIL.Harmony.Sample.Changes; using SIL.Harmony.Sample.Models; - namespace SIL.Harmony.Tests.ResourceTests; - public class WordResourceTests: DataModelTestBase { private RemoteServiceMock _remoteServiceMock = new(); - private ResourceService _resourceService => _services.GetRequiredService(); + private ResourceService _resourceService => _services.GetRequiredService>(); private readonly Guid _entity1Id = Guid.NewGuid(); - private string CreateFile(string contents, [CallerMemberName] string fileName = "") { var filePath = Path.GetFullPath(fileName + ".txt"); File.WriteAllText(filePath, contents); return filePath; } - [Fact] public async Task CanReferenceAResourceFromAWord() { @@ -27,15 +24,13 @@ public async Task CanReferenceAResourceFromAWord() MockTimeProvider.SetNextDateTime(NextDate()); var resource = await _resourceService.AddLocalResource(imageFile, Guid.NewGuid(), resourceService: _remoteServiceMock); await WriteNextChange(new AddWordImageChange(_entity1Id, resource.Id)); - var word = await DataModel.GetLatest(_entity1Id); word.Should().NotBeNull(); word!.ImageResourceId.Should().Be(resource.Id); - - + var localResource = await _resourceService.GetLocalResource(word.ImageResourceId!.Value); localResource.Should().NotBeNull(); localResource!.LocalPath.Should().Be(imageFile); (await File.ReadAllTextAsync(localResource.LocalPath, TestContext.Current.CancellationToken)).Should().Be("not image data"); } -} \ No newline at end of file +} diff --git a/src/SIL.Harmony/CrdtConfig.cs b/src/SIL.Harmony/CrdtConfig.cs index 2ae0895..1d6dc8f 100644 --- a/src/SIL.Harmony/CrdtConfig.cs +++ b/src/SIL.Harmony/CrdtConfig.cs @@ -1,4 +1,4 @@ -using System.Text.Json; +using System.Text.Json; using System.Text.Json.Serialization.Metadata; using Microsoft.EntityFrameworkCore; using SIL.Harmony.Adapters; @@ -8,7 +8,9 @@ using SIL.Harmony.Resource; namespace SIL.Harmony; + public delegate ValueTask BeforeSaveObjectDelegate(object obj, ObjectSnapshot snapshot); + public class CrdtConfig { /// @@ -35,7 +37,7 @@ public CrdtConfig() TypeInfoResolver = MakeJsonTypeResolver() }); } - + public Action MakeJsonTypeModifier() { return JsonTypeModifier; @@ -72,18 +74,32 @@ private void JsonTypeModifier(JsonTypeInfo typeInfo) } public bool RemoteResourcesEnabled { get; private set; } + public Type? RemoteResourceMetadataType { get; private set; } public string LocalResourceCachePath { get; set; } = Path.GetFullPath("./localResourceCache"); public string FailedSyncOutputPath { get; set; } = Path.GetFullPath("./failedSyncs"); - - public void AddRemoteResourceEntity(string? cachePath = null) + public void AddRemoteResourceEntity(string? cachePath = null) + where TMetadata : class { RemoteResourcesEnabled = true; + RemoteResourceMetadataType = typeof(TMetadata); LocalResourceCachePath = cachePath ?? LocalResourceCachePath; - ObjectTypeListBuilder.DefaultAdapter().Add(); - ChangeTypeListBuilder.Add(); - ChangeTypeListBuilder.Add(); - ChangeTypeListBuilder.Add(); - ChangeTypeListBuilder.Add>(); + ObjectTypeListBuilder.DefaultAdapter().Add>(builder => + { + builder.ToTable("RemoteResource"); + builder.Property(r => r.Metadata) + .HasColumnType("jsonb") + .HasConversion( + m => JsonSerializer.Serialize(m, (JsonSerializerOptions?)null), + json => string.IsNullOrEmpty(json) + ? null + : JsonSerializer.Deserialize(json, (JsonSerializerOptions?)null) + ); + }); + ChangeTypeListBuilder.Add>(); + ChangeTypeListBuilder.Add>(); + ChangeTypeListBuilder.Add>(); + ChangeTypeListBuilder.Add>(); + ChangeTypeListBuilder.Add>(); ObjectTypeListBuilder.ModelConfigurations.Add((builder, config) => { var entity = builder.Entity(); @@ -151,9 +167,7 @@ internal void CheckFrozen() } internal Dictionary> JsonTypes { get; } = []; - internal List> ModelConfigurations { get; } = []; - internal List AdapterProviders { get; } = []; public DefaultAdapterProvider DefaultAdapter() @@ -164,7 +178,7 @@ public DefaultAdapterProvider DefaultAdapter() AdapterProviders.Add(adapter); return adapter; } - + /// /// add a custom adapter for a common interface /// this is required as CRDT objects must express their references and have an Id property @@ -207,3 +221,4 @@ internal IObjectBase Adapt(object obj) throw new ArgumentException($"Unable to adapt object of type {obj.GetType()}"); } } + diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index 2d1c95d..aaa43f7 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -12,7 +12,6 @@ public static class CrdtKernel public static IServiceCollection AddCrdtDataDbFactory(this IServiceCollection services, Action configureCrdt) where TContext : DbContext, ICrdtDbContext { - services.AddCrdtDataCore(configureCrdt); services.AddScoped>(); return services; @@ -25,6 +24,25 @@ public static IServiceCollection AddCrdtData(this IServiceCollection s services.AddScoped>(); return services; } + + public static IServiceCollection AddCrdtRemoteResources(this IServiceCollection services, + Action? configureCrdt = null, string? cachePath = null) + where TMetadata : class + { + services.Configure(config => + { + config.AddRemoteResourceEntity(cachePath); + configureCrdt?.Invoke(config); + }); + services.AddScoped>(provider => new ResourceService( + provider.GetRequiredService(), + provider.GetRequiredService>(), + provider.GetRequiredService(), + provider.GetRequiredService>>() + )); + return services; + } + public static IServiceCollection AddCrdtDataCore(this IServiceCollection services, Action configureCrdt) { services.AddLogging(); @@ -42,13 +60,6 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service provider.GetRequiredService>(), provider.GetRequiredService>() )); - //must use factory method because ResourceService constructor is internal - services.AddScoped(provider => new ResourceService( - provider.GetRequiredService(), - provider.GetRequiredService>(), - provider.GetRequiredService(), - provider.GetRequiredService>() - )); return services; } @@ -63,5 +74,3 @@ public static HybridDateTimeProvider NewTimeProvider(IServiceProvider servicePro return ActivatorUtilities.CreateInstance(serviceProvider, hybridDateTime); } } - - diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs index 6840198..6338523 100644 --- a/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs +++ b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs @@ -1,19 +1,27 @@ -using SIL.Harmony.Changes; +using SIL.Harmony.Changes; using SIL.Harmony.Entities; namespace SIL.Harmony.Resource; -public class CreateRemoteResourceChange(Guid entityId, string remoteId) : CreateChange(entityId), IPolyType +public class CreateRemoteResourceChange(Guid entityId, string remoteId, TMetadata? metadata = null) + : CreateChange>(entityId), IPolyType + where TMetadata : class { public string RemoteId { get; set; } = remoteId; - public override ValueTask NewEntity(Commit commit, IChangeContext context) + public TMetadata? Metadata { get; set; } = metadata; + + public override ValueTask> NewEntity(Commit commit, IChangeContext context) { - return ValueTask.FromResult(new RemoteResource + return ValueTask.FromResult(new RemoteResource { Id = EntityId, - RemoteId = RemoteId + RemoteId = RemoteId, + Metadata = Metadata }); } public static string TypeName => "create:remote-resource"; } + + + diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs index 73f2a2d..a4aaf5b 100644 --- a/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs +++ b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs @@ -3,14 +3,18 @@ namespace SIL.Harmony.Resource; -public class CreateRemoteResourcePendingUploadChange(Guid entityId) - : CreateChange(entityId), IPolyType +public class CreateRemoteResourcePendingUploadChange(Guid entityId, TMetadata? metadata = null) + : CreateChange>(entityId), IPolyType + where TMetadata : class { - public override ValueTask NewEntity(Commit commit, IChangeContext context) + public TMetadata? Metadata { get; set; } = metadata; + + public override ValueTask> NewEntity(Commit commit, IChangeContext context) { - return ValueTask.FromResult(new RemoteResource + return ValueTask.FromResult(new RemoteResource { - Id = EntityId + Id = EntityId, + Metadata = Metadata }); } diff --git a/src/SIL.Harmony/Resource/DeleteRemoteResourceChange.cs b/src/SIL.Harmony/Resource/DeleteRemoteResourceChange.cs new file mode 100644 index 0000000..56d937d --- /dev/null +++ b/src/SIL.Harmony/Resource/DeleteRemoteResourceChange.cs @@ -0,0 +1,16 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +public class DeleteRemoteResourceChange(Guid entityId) : EditChange>(entityId), IPolyType + where TMetadata : class +{ + public static string TypeName => "delete:RemoteResource"; + + public override ValueTask ApplyChange(RemoteResource entity, IChangeContext context) + { + context.Adapt(entity).DeletedAt = context.Commit.DateTime; + return ValueTask.CompletedTask; + } +} diff --git a/src/SIL.Harmony/Resource/HarmonyResource.cs b/src/SIL.Harmony/Resource/HarmonyResource.cs index cfe455e..4a3f0c6 100644 --- a/src/SIL.Harmony/Resource/HarmonyResource.cs +++ b/src/SIL.Harmony/Resource/HarmonyResource.cs @@ -1,12 +1,13 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.CodeAnalysis; namespace SIL.Harmony.Resource; -public class HarmonyResource +public class HarmonyResource where TMetadata : class { public required Guid Id { get; init; } public string? RemoteId { get; init; } public string? LocalPath { get; init; } + public TMetadata? Metadata { get; init; } [MemberNotNullWhen(true, nameof(LocalPath))] public bool Local => !string.IsNullOrEmpty(LocalPath); [MemberNotNullWhen(true, nameof(RemoteId))] diff --git a/src/SIL.Harmony/Resource/RemoteResource.cs b/src/SIL.Harmony/Resource/RemoteResource.cs index 7d10cc1..da86401 100644 --- a/src/SIL.Harmony/Resource/RemoteResource.cs +++ b/src/SIL.Harmony/Resource/RemoteResource.cs @@ -1,18 +1,29 @@ -using SIL.Harmony.Entities; +using System.Text.Json; +using SIL.Harmony.Entities; namespace SIL.Harmony.Resource; +/// +/// Marker type for apps that do not need synced resource metadata. +/// +public sealed class NoMetadata; + /// /// represents a remote binary resource (e.g. image, video, audio, etc.) /// -public class RemoteResource: IObjectBase +public class RemoteResource : IObjectBase> + where TMetadata : class { + public static string TypeName => "RemoteResource"; + public Guid Id { get; init; } public DateTimeOffset? DeletedAt { get; set; } /// /// will be null when the resource has not been uploaded yet /// public string? RemoteId { get; set; } + public TMetadata? Metadata { get; set; } + public Guid[] GetReferences() { return []; @@ -24,11 +35,21 @@ public void RemoveReference(Guid id, CommitBase commit) public IObjectBase Copy() { - return new RemoteResource + return new RemoteResource { Id = Id, RemoteId = RemoteId, - DeletedAt = DeletedAt + DeletedAt = DeletedAt, + Metadata = CloneMetadata(Metadata) }; } + + private static TMetadata? CloneMetadata(TMetadata? metadata) + { + if (metadata is null) return null; + return JsonSerializer.Deserialize(JsonSerializer.Serialize(metadata)); + } } + + + diff --git a/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs index aaa37a0..01b641d 100644 --- a/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs +++ b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs @@ -1,4 +1,4 @@ namespace SIL.Harmony.Resource; public class RemoteResourceNotEnabledException() - : Exception("remote resources were not enabled, to enable them call CrdtConfig.AddRemoteResourceEntity when adding the CRDT library"); + : Exception("remote resources were not enabled, to enable them call AddCrdtRemoteResources when adding the CRDT library"); diff --git a/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs b/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs index 40a0cba..8316464 100644 --- a/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs +++ b/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs @@ -6,16 +6,19 @@ namespace SIL.Harmony.Resource; /// /// used when a resource is uploaded to the remote server, stores the remote url in the resource entity /// -/// -/// -public class RemoteResourceUploadedChange(Guid entityId, string remoteId) : EditChange(entityId), IPolyType +public class RemoteResourceUploadedChange(Guid entityId, string remoteId) + : EditChange>(entityId), IPolyType + where TMetadata : class { public string RemoteId { get; set; } = remoteId; public static string TypeName => "uploaded:RemoteResource"; - public override ValueTask ApplyChange(RemoteResource entity, IChangeContext context) + public override ValueTask ApplyChange(RemoteResource entity, IChangeContext context) { entity.RemoteId = RemoteId; return ValueTask.CompletedTask; } } + + + diff --git a/src/SIL.Harmony/Resource/SetRemoteResourceMetadataChange.cs b/src/SIL.Harmony/Resource/SetRemoteResourceMetadataChange.cs new file mode 100644 index 0000000..e813765 --- /dev/null +++ b/src/SIL.Harmony/Resource/SetRemoteResourceMetadataChange.cs @@ -0,0 +1,18 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +public class SetRemoteResourceMetadataChange(Guid entityId, TMetadata metadata) + : EditChange>(entityId), IPolyType + where TMetadata : class +{ + public TMetadata Metadata { get; } = metadata; + public static string TypeName => "set:remote-resource-metadata"; + + public override ValueTask ApplyChange(RemoteResource entity, IChangeContext context) + { + entity.Metadata = Metadata; + return ValueTask.CompletedTask; + } +} diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs index 344765c..433c038 100644 --- a/src/SIL.Harmony/ResourceService.cs +++ b/src/SIL.Harmony/ResourceService.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using SIL.Harmony.Changes; @@ -8,14 +8,15 @@ namespace SIL.Harmony; -public class ResourceService +public class ResourceService where TMetadata : class { private readonly CrdtRepositoryFactory _crdtRepositoryFactory; private readonly IOptions _crdtConfig; private readonly DataModel _dataModel; - private readonly ILogger _logger; + private readonly ILogger> _logger; - internal ResourceService(CrdtRepositoryFactory crdtRepositoryFactory, IOptions crdtConfig, DataModel dataModel, ILogger logger) + internal ResourceService(CrdtRepositoryFactory crdtRepositoryFactory, IOptions crdtConfig, + DataModel dataModel, ILogger> logger) { _crdtRepositoryFactory = crdtRepositoryFactory; _crdtConfig = crdtConfig; @@ -30,7 +31,9 @@ private void ValidateResourcesSetup() public async Task AddExistingRemoteResource(string resourcePath, Guid clientId, - Guid resourceId, string remoteId) + Guid resourceId, + string remoteId, + TMetadata? metadata = null) { ValidateResourcesSetup(); var localResource = new LocalResource @@ -40,15 +43,26 @@ public async Task AddExistingRemoteResource(string resourcePath, }; if (!localResource.FileExists()) throw new FileNotFoundException(localResource.LocalPath); - await _dataModel.AddChange(clientId, new CreateRemoteResourceChange(localResource.Id, remoteId)); + await _dataModel.AddChange(clientId, + new CreateRemoteResourceChange(localResource.Id, remoteId, metadata)); await using var repo = await _crdtRepositoryFactory.CreateRepository(); await repo.AddLocalResource(localResource); } - public async Task AddLocalResource(string resourcePath, + /// + /// add and upload a local resource + /// + /// path to the resource on the local machine + /// id of the client + /// metadata for the resource, this metadata will be overridden by the remote service if returned by + /// id of the resource + /// service to upload the resource to the remote server + /// the HarmonyResource created + public async Task> AddLocalResource(string resourcePath, Guid clientId, + TMetadata? metadata = null, Guid id = default, - IRemoteResourceService? resourceService = null) + IRemoteResourceService? resourceService = null) { ValidateResourcesSetup(); var localResource = new LocalResource @@ -57,48 +71,59 @@ public async Task AddLocalResource(string resourcePath, LocalPath = Path.GetFullPath(resourcePath) }; if (!localResource.FileExists()) throw new FileNotFoundException(localResource.LocalPath); - UploadResult? uploadResult = null; + UploadResult? uploadResult = null; if (resourceService is not null) { try { - uploadResult = await resourceService.UploadResource(localResource.Id, localResource.LocalPath); + metadata = uploadResult.Metadata ?? metadata; } catch (Exception e) { - _logger.LogError(e, "Error uploading resource {resourcePath}, resource will be marked as pending upload", localResource.LocalPath); + _logger.LogError(e, "Error uploading resource {resourcePath}, resource will be marked as pending upload", + localResource.LocalPath); } } if (uploadResult is not null) { - await _dataModel.AddChange(clientId, new CreateRemoteResourceChange(localResource.Id, uploadResult.RemoteId)); + await _dataModel.AddChange(clientId, + new CreateRemoteResourceChange(localResource.Id, uploadResult.RemoteId, metadata)); } else { - await _dataModel.AddChange(clientId, new CreateRemoteResourcePendingUploadChange(localResource.Id)); + await _dataModel.AddChange(clientId, + new CreateRemoteResourcePendingUploadChange(localResource.Id, metadata)); } await _crdtRepositoryFactory.Execute(repo => repo.AddLocalResource(localResource)); - return new HarmonyResource + return new HarmonyResource { Id = localResource.Id, RemoteId = uploadResult?.RemoteId, - LocalPath = localResource.LocalPath + LocalPath = localResource.LocalPath, + Metadata = metadata }; } + public async Task SetResourceMetadata(Guid resourceId, Guid clientId, TMetadata metadata) + { + ValidateResourcesSetup(); + await _dataModel.AddChange(clientId, new SetRemoteResourceMetadataChange(resourceId, metadata)); + } + public async Task ListResourcesPendingUpload() { ValidateResourcesSetup(); await using var repo = await _crdtRepositoryFactory.CreateRepository(); - var remoteResources = await repo.GetCurrentObjects().Where(r => r.RemoteId == null).ToArrayAsync(); + var remoteResources = await repo.GetCurrentObjects>() + .Where(r => r.RemoteId == null).ToArrayAsync(); var localResource = repo.LocalResourcesByIds(remoteResources.Select(r => r.Id)); return await localResource.ToArrayAsync(); } - public async Task UploadPendingResources(Guid clientId, IRemoteResourceService remoteResourceService) + public async Task UploadPendingResources(Guid clientId, IRemoteResourceService remoteResourceService) { ValidateResourcesSetup(); var pendingUploads = await ListResourcesPendingUpload(); @@ -108,8 +133,9 @@ public async Task UploadPendingResources(Guid clientId, IRemoteResourceService r { foreach (var localResource in pendingUploads) { - var uploadResult = await remoteResourceService.UploadResource(localResource.Id, localResource.LocalPath); - changes.Add(new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + var uploadResult = + await remoteResourceService.UploadResource(localResource.Id, localResource.LocalPath); + changes.Add(new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); } } finally @@ -119,7 +145,7 @@ public async Task UploadPendingResources(Guid clientId, IRemoteResourceService r } } - public async Task UploadPendingResource(Guid resourceId, Guid clientId, IRemoteResourceService remoteResourceService) + public async Task UploadPendingResource(Guid resourceId, Guid clientId, IRemoteResourceService remoteResourceService) { await using var repo = await _crdtRepositoryFactory.CreateRepository(); var localResource = await repo.GetLocalResource(resourceId) ?? @@ -128,49 +154,52 @@ public async Task UploadPendingResource(Guid resourceId, Guid clientId, IRemoteR await UploadPendingResource(localResource, clientId, remoteResourceService); } - public async Task UploadPendingResource(LocalResource localResource, Guid clientId, IRemoteResourceService remoteResourceService) + public async Task UploadPendingResource(LocalResource localResource, Guid clientId, + IRemoteResourceService remoteResourceService) { ValidateResourcesSetup(); var uploadResult = await remoteResourceService.UploadResource(localResource.Id, localResource.LocalPath); - await _dataModel.AddChange(clientId, new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + await _dataModel.AddChange(clientId, + new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); } - public async Task ListResourcesPendingDownload() + public async Task[]> ListResourcesPendingDownload() { ValidateResourcesSetup(); await using var repo = await _crdtRepositoryFactory.CreateRepository(); var localResourceIds = repo.LocalResourceIds(); - var remoteResources = await repo.GetCurrentObjects() + var remoteResources = await repo.GetCurrentObjects>() .Where(r => r.RemoteId != null && !localResourceIds.Contains(r.Id)) .ToArrayAsync(); return remoteResources; } - public async Task DownloadResource(Guid resourceId, IRemoteResourceService remoteResourceService) + public async Task DownloadResource(Guid resourceId, IRemoteResourceService remoteResourceService) { ValidateResourcesSetup(); await using var repo = await _crdtRepositoryFactory.CreateRepository(); return await DownloadResourceInternal(repo, - await repo.GetCurrent(resourceId) ?? + await repo.GetCurrent>(resourceId) ?? throw new EntityNotFoundException("Unable to find remote resource"), remoteResourceService ); } - public async Task DownloadResource(RemoteResource remoteResource, - IRemoteResourceService remoteResourceService) + public async Task DownloadResource(RemoteResource remoteResource, + IRemoteResourceService remoteResourceService) { await using var repo = await _crdtRepositoryFactory.CreateRepository(); return await DownloadResourceInternal(repo, remoteResource, remoteResourceService); } private async Task DownloadResourceInternal(CrdtRepository repo, - RemoteResource remoteResource, - IRemoteResourceService remoteResourceService) + RemoteResource remoteResource, + IRemoteResourceService remoteResourceService) { ValidateResourcesSetup(); ArgumentNullException.ThrowIfNull(remoteResource.RemoteId); - var downloadResult = await remoteResourceService.DownloadResource(remoteResource.RemoteId, _crdtConfig.Value.LocalResourceCachePath); + var downloadResult = await remoteResourceService.DownloadResource(remoteResource.RemoteId, + _crdtConfig.Value.LocalResourceCachePath); var localResource = new LocalResource { Id = remoteResource.Id, @@ -185,28 +214,30 @@ private async Task DownloadResourceInternal(CrdtRepository repo, return await _crdtRepositoryFactory.Execute(repo => repo.GetLocalResource(resourceId)); } - public async Task AllResources() + public async Task[]> AllResources() { return (await AllResourcesInternal()).ToArray(); } - private async Task> AllResourcesInternal() + private async Task>> AllResourcesInternal() { await using var repo = await _crdtRepositoryFactory.CreateRepository(); - var remoteResources = await repo.GetCurrentObjects().ToArrayAsync(); + var remoteResources = await repo.GetCurrentObjects>().ToArrayAsync(); var localResources = await repo.LocalResources().ToArrayAsync(); - return remoteResources.FullOuterJoin(localResources, + return remoteResources.FullOuterJoin, LocalResource, Guid, HarmonyResource>( + localResources, r => r.Id, l => l.Id, - (r, l, id) => new HarmonyResource + (r, l, id) => new HarmonyResource { Id = id, RemoteId = r?.RemoteId, - LocalPath = l?.LocalPath + LocalPath = l?.LocalPath, + Metadata = r?.Metadata }); } - public async Task GetResource(Guid resourceId) + public async Task?> GetResource(Guid resourceId) { var resources = await AllResourcesInternal(); return resources.FirstOrDefault(r => r.Id == resourceId); @@ -214,7 +245,7 @@ private async Task> AllResourcesInternal() public async Task DeleteResource(Guid clientId, Guid resourceId) { - await _dataModel.AddChange(clientId, new DeleteChange(resourceId)); + await _dataModel.AddChange(clientId, new DeleteRemoteResourceChange(resourceId)); await using var repo = await _crdtRepositoryFactory.CreateRepository(); await repo.DeleteLocalResource(resourceId); } diff --git a/src/SIL.Harmony/SyncHelper.cs b/src/SIL.Harmony/SyncHelper.cs index f9ca2c6..afd6e93 100644 --- a/src/SIL.Harmony/SyncHelper.cs +++ b/src/SIL.Harmony/SyncHelper.cs @@ -1,19 +1,16 @@ -using System.Text.Json; - +using System.Text.Json; namespace SIL.Harmony; - internal static class SyncHelper { - public static async Task SyncWithResourceUpload(this DataModel localModel, + public static async Task SyncWithResourceUpload(this DataModel localModel, ISyncable remoteModel, - ResourceService resourceService, - IRemoteResourceService remoteResourceService, - Guid localClientId) + ResourceService resourceService, + IRemoteResourceService remoteResourceService, + Guid localClientId) where TMetadata : class { await resourceService.UploadPendingResources(localClientId, remoteResourceService); return await localModel.SyncWith(remoteModel); } - /// /// simple sync example, each ISyncable could be over the wire or in memory /// prefer that remote is over the wire for the best performance, however they could both be remote @@ -27,7 +24,6 @@ internal static async Task SyncWith(ISyncable localModel, { if (!await localModel.ShouldSync() || !await remoteModel.ShouldSync()) return new SyncResults([], [], false); var localSyncState = await localModel.GetSyncState(); - var (missingFromLocal, remoteSyncState) = await remoteModel.GetChanges(localSyncState); //todo abort if local and remote heads are the same var (missingFromRemote, _) = await localModel.GetChanges(remoteSyncState); @@ -37,14 +33,12 @@ internal static async Task SyncWith(ISyncable localModel, missingFromLocal = Clone(missingFromLocal, serializerOptions); missingFromRemote = Clone(missingFromRemote, serializerOptions); } - if (missingFromLocal.Length > 0) await localModel.AddRangeFromSync(missingFromLocal); if (missingFromRemote.Length > 0) await remoteModel.AddRangeFromSync(missingFromRemote); return new SyncResults(missingFromLocal, missingFromRemote, true); } - internal static async Task SyncMany(ISyncable localModel, ISyncable[] remotes, JsonSerializerOptions serializerOptions) { var localSyncState = await localModel.GetSyncState(); @@ -61,7 +55,6 @@ internal static async Task SyncMany(ISyncable localModel, ISyncable[] remotes, J remoteSyncStates[i] = remoteSyncState; await localModel.AddRangeFromSync(missingFromLocal); } - // Now the localModel has all the changes from all remotes, so all remotes will get the changes from the localModel as well as all other remotes for (var i = 0; i < remotes.Length; i++) { @@ -76,7 +69,6 @@ internal static async Task SyncMany(ISyncable localModel, ISyncable[] remotes, J await remote.AddRangeFromSync(missingFromRemote); } } - private static T Clone(this T source, JsonSerializerOptions options) { ArgumentNullException.ThrowIfNull(source); @@ -85,3 +77,4 @@ private static T Clone(this T source, JsonSerializerOptions options) return clone ?? throw new NullReferenceException("unable to clone object type " + typeof(T)); } } +