From ea1675c60dfa809c194883cf4986c75086e34c4e Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Sun, 18 Oct 2026 21:44:44 +1300 Subject: [PATCH 1/4] refactor: Derive the capture date and its flashback parts in one place (#283) MediaAsset.SetCaptureDate sets DateCaptured with Month, Day and Year, and CaptureDate.Resolve holds the last-write fallback. NewContentWriter and PairedMovResolver both use them, so #250's in-place update can too. Co-Authored-By: Claude Opus 5.5 --- .../Domain/MediaAssetTests.cs | 30 +++++++++++++++++++ src/Anichron.Core/Domain/MediaAsset.cs | 8 +++++ .../Ingestion/CaptureDateTests.cs | 29 ++++++++++++++++++ src/Anichron.Worker/Ingestion/CaptureDate.cs | 10 +++++++ .../Classification/NewContentWriter.cs | 16 ++-------- .../Ingestion/PairedMovResolver.cs | 8 +---- 6 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 src/Anichron.Core.Tests.Unit/Domain/MediaAssetTests.cs create mode 100644 src/Anichron.Worker.Tests.Unit/Ingestion/CaptureDateTests.cs create mode 100644 src/Anichron.Worker/Ingestion/CaptureDate.cs diff --git a/src/Anichron.Core.Tests.Unit/Domain/MediaAssetTests.cs b/src/Anichron.Core.Tests.Unit/Domain/MediaAssetTests.cs new file mode 100644 index 0000000..85d2945 --- /dev/null +++ b/src/Anichron.Core.Tests.Unit/Domain/MediaAssetTests.cs @@ -0,0 +1,30 @@ +using Anichron.Core.Domain; +using NodaTime; + +namespace Anichron.Core.Tests.Unit.Domain; + +public sealed class MediaAssetTests +{ + [Fact] + public void SetCaptureDate_SetsTheDateAndItsFlashbackParts() + { + var asset = new MediaAsset(); + + asset.SetCaptureDate(new LocalDateTime(2021, 11, 7, 18, 30)); + + asset.DateCaptured.Should().Be(new LocalDateTime(2021, 11, 7, 18, 30)); + (asset.Year, asset.Month, asset.Day).Should().Be((2021, 11, 7)); + } + + [Fact] + public void SetCaptureDate_CalledAgain_ReplacesEveryPart() + { + var asset = new MediaAsset(); + asset.SetCaptureDate(new LocalDateTime(2021, 11, 7, 18, 30)); + + asset.SetCaptureDate(new LocalDateTime(2024, 2, 29, 6, 0)); + + asset.DateCaptured.Should().Be(new LocalDateTime(2024, 2, 29, 6, 0)); + (asset.Year, asset.Month, asset.Day).Should().Be((2024, 2, 29)); + } +} diff --git a/src/Anichron.Core/Domain/MediaAsset.cs b/src/Anichron.Core/Domain/MediaAsset.cs index a02d793..e1f1851 100644 --- a/src/Anichron.Core/Domain/MediaAsset.cs +++ b/src/Anichron.Core/Domain/MediaAsset.cs @@ -46,4 +46,12 @@ public class MediaAsset public virtual ICollection ProxyFiles { get; set; } = []; public virtual ICollection Interactions { get; set; } = []; + + public void SetCaptureDate(LocalDateTime dateCaptured) + { + DateCaptured = dateCaptured; + Month = dateCaptured.Month; + Day = dateCaptured.Day; + Year = dateCaptured.Year; + } } diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/CaptureDateTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/CaptureDateTests.cs new file mode 100644 index 0000000..9ff9d68 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/CaptureDateTests.cs @@ -0,0 +1,29 @@ +using Anichron.Worker.Ingestion; +using NodaTime; +using System.IO.Abstractions.TestingHelpers; + +namespace Anichron.Worker.Tests.Unit.Ingestion; + +public sealed class CaptureDateTests +{ + private static MockFileSystem FileWrittenAt(DateTime lastWrite) => new(new Dictionary + { + ["/abs/photo.jpg"] = new MockFileData([]) { LastWriteTime = new DateTimeOffset(lastWrite) }, + }); + + [Fact] + public void Resolve_WithAnExifDate_ReturnsItWithoutReadingTheFile() + { + var exifDate = new LocalDateTime(2020, 1, 2, 3, 4, 5); + + CaptureDate.Resolve(exifDate, new MockFileSystem(), "/abs/missing.jpg").Should().Be(exifDate); + } + + [Fact] + public void Resolve_WithoutAnExifDate_FallsBackToTheFilesLastWriteTime() + { + var fileSystem = FileWrittenAt(new DateTime(2023, 6, 15, 12, 0, 0, DateTimeKind.Local)); + + CaptureDate.Resolve(null, fileSystem, "/abs/photo.jpg").Should().Be(new LocalDateTime(2023, 6, 15, 12, 0)); + } +} diff --git a/src/Anichron.Worker/Ingestion/CaptureDate.cs b/src/Anichron.Worker/Ingestion/CaptureDate.cs new file mode 100644 index 0000000..8cd1d88 --- /dev/null +++ b/src/Anichron.Worker/Ingestion/CaptureDate.cs @@ -0,0 +1,10 @@ +using NodaTime; +using System.IO.Abstractions; + +namespace Anichron.Worker.Ingestion; + +internal static class CaptureDate +{ + public static LocalDateTime Resolve(LocalDateTime? exifDate, IFileSystem fileSystem, string absolutePath) + => exifDate ?? LocalDateTime.FromDateTime(fileSystem.FileInfo.New(absolutePath).LastWriteTime); +} diff --git a/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs b/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs index be6f368..54dc364 100644 --- a/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs +++ b/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs @@ -42,19 +42,13 @@ private MediaAsset BuildAsset(IngestionContext context) { var relativePath = context.Item.RelativePath; var exif = context.Exif!; - var dateCaptured = exif.DateCaptured ?? FallbackDate(context.Item.AbsolutePath); - - return new MediaAsset + var asset = new MediaAsset { Id = context.AssetId, StorageConfigId = context.Config.Id, FilePath = relativePath, FileName = Path.GetFileName(relativePath), ContentHash = context.ContentHash!, - DateCaptured = dateCaptured, - Month = dateCaptured.Month, - Day = dateCaptured.Day, - Year = dateCaptured.Year, MediaType = context.Item.PrimaryMediaType, IsSoftDeleted = false, LastSeenOnNas = clock.GetCurrentInstant(), @@ -73,12 +67,8 @@ private MediaAsset BuildAsset(IngestionContext context) Blurhash = context.Blurhash, }, }; - } - - private LocalDateTime FallbackDate(string absolutePath) - { - var lastWrite = fileSystem.FileInfo.New(absolutePath).LastWriteTime; - return LocalDateTime.FromDateTime(lastWrite); + asset.SetCaptureDate(CaptureDate.Resolve(exif.DateCaptured, fileSystem, context.Item.AbsolutePath)); + return asset; } private static partial class Log diff --git a/src/Anichron.Worker/Ingestion/PairedMovResolver.cs b/src/Anichron.Worker/Ingestion/PairedMovResolver.cs index 928c239..da61ee5 100644 --- a/src/Anichron.Worker/Ingestion/PairedMovResolver.cs +++ b/src/Anichron.Worker/Ingestion/PairedMovResolver.cs @@ -20,9 +20,6 @@ public MediaAsset Resolve(IngestionContext context, LocalDateTime? dateCaptured) return stored; var mov = context.Item.SecondaryFile!; - var date = dateCaptured - ?? LocalDateTime.FromDateTime(fileSystem.FileInfo.New(mov.AbsolutePath).LastWriteTime); - var asset = new MediaAsset { Id = guidFactory.NewGuid(), @@ -30,14 +27,11 @@ public MediaAsset Resolve(IngestionContext context, LocalDateTime? dateCaptured) FilePath = mov.RelativePath, FileName = Path.GetFileName(mov.RelativePath), ContentHash = context.SecondaryHash!, - DateCaptured = date, - Month = date.Month, - Day = date.Day, - Year = date.Year, MediaType = mov.MediaType, IsSoftDeleted = false, LastSeenOnNas = clock.GetCurrentInstant(), }; + asset.SetCaptureDate(CaptureDate.Resolve(dateCaptured, fileSystem, mov.AbsolutePath)); repository.Add(asset); return asset; } From 417d5d0371001a5296802cfaf2cdbb5ad5afa13a Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Sun, 18 Oct 2026 21:49:25 +1300 Subject: [PATCH 2/4] refactor: Proxy rows reach the context one way, through the repository (#283) NewContentWriter now adds the primary's proxies with AddProxyFile, as PairedMovProxyWriter does for the MOV, instead of through the ProxyFiles navigation. Both pick an owner's proxies with IngestionContext.ProxyFilesOf. Co-Authored-By: Claude Opus 5.5 --- .../Classification/NewContentWriterTests.cs | 14 ++++++++------ .../Pipeline/ImageIngestionBlurhashTests.cs | 4 +++- .../Ingestion/Pipeline/IngestionContextTests.cs | 17 +++++++++++++++++ .../Classification/NewContentWriter.cs | 4 +++- .../Ingestion/Pipeline/IngestionContext.cs | 3 +++ .../Ingestion/Proxy/PairedMovProxyWriter.cs | 2 +- 6 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs index 172fec4..7c2f991 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs @@ -308,16 +308,16 @@ public async Task TryWriteAsync_Always_UsesContextAssetIdForPrimaryAssetAsync() } [Fact] - public async Task TryWriteAsync_Always_CopiesProxyFilesFromContextToAssetAsync() + public async Task TryWriteAsync_Always_AddsTheProxyFilesFromContextToTheRepositoryAsync() { var fx = new TestFixture(); var context = MakeContext(); var proxyFile = new ProxyFile { Id = Guid.NewGuid(), AssetId = context.AssetId, ProxyType = ProxyType.Thumbnail }; context.AddProxyFile(proxyFile); - var asset = await fx.Build().TryWriteAsync(context, CancellationToken.None); + await fx.Build().TryWriteAsync(context, CancellationToken.None); - asset!.ProxyFiles.Should().ContainSingle(p => p.Id == proxyFile.Id); + fx.Repository.Received(1).AddProxyFile(proxyFile); } [Fact] @@ -382,17 +382,19 @@ public async Task TryWriteAsync_LivePhotoPairItem_TranscodesTheMovAsTheMovsOwnPr } [Fact] - public async Task TryWriteAsync_Always_CopiesOnlyTheAssetsOwnProxyFilesToItAsync() + public async Task TryWriteAsync_Always_AddsOnlyTheAssetsOwnProxyFilesAsync() { var fx = new TestFixture(); + var added = new List(); + fx.Repository.When(r => r.AddProxyFile(Arg.Any())).Do(call => added.Add(call.Arg())); var context = MakeContext(); var own = new ProxyFile { Id = Guid.NewGuid(), AssetId = context.AssetId, ProxyType = ProxyType.Thumbnail }; context.AddProxyFile(own); context.AddProxyFile(new ProxyFile { Id = Guid.NewGuid(), AssetId = Guid.NewGuid(), ProxyType = ProxyType.WebVideo }); - var asset = await fx.Build().TryWriteAsync(context, CancellationToken.None); + await fx.Build().TryWriteAsync(context, CancellationToken.None); - asset!.ProxyFiles.Should().ContainSingle().Which.Should().BeSameAs(own); + added.Should().ContainSingle().Which.Should().BeSameAs(own); } // ========================================================================== diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs index 1222e45..006bb07 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs @@ -40,6 +40,8 @@ public async Task NewImage_IsPersistedWithItsBlurhashAndNoBlurhashProxyAsync() var repository = Substitute.For(); MediaAsset? persisted = null; repository.Add(Arg.Do(asset => persisted = asset)); + var proxies = new List(); + repository.AddProxyFile(Arg.Do(proxies.Add)); var options = Options.Create(new WorkerSettings()); var processor = new ImageSharpProcessor(options); var stagingWriter = new ProxyStagingWriter(fileSystem, clock, guids, new FakeLogger()); @@ -81,7 +83,7 @@ public async Task NewImage_IsPersistedWithItsBlurhashAndNoBlurhashProxyAsync() persisted.Should().NotBeNull(); persisted.Metadata!.Blurhash.Should().HaveLength(28); - persisted.ProxyFiles.Select(p => p.ProxyType).Should().BeEquivalentTo([ProxyType.Thumbnail, ProxyType.FullPreview]); + proxies.Select(p => p.ProxyType).Should().BeEquivalentTo([ProxyType.Thumbnail, ProxyType.FullPreview]); fileSystem.AllFiles.Where(path => path.StartsWith(StoragePath.ProxiesRoot + "/", StringComparison.Ordinal)) .Select(Path.GetFileName) .Should().BeEquivalentTo("thumbnail.jpg", "preview.jpg"); diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs index 635b6e6..904eed5 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs @@ -50,4 +50,21 @@ public void ReleaseProxyFiles_DropsOnlyThatOwnersRegistrations() context.ProxyFiles.Should().ContainSingle().Which.Should().BeSameAs(own); } + + [Fact] + public void ProxyFilesOf_ReturnsOnlyThatOwnersRegistrations() + { + var context = new IngestionContext + { + Item = new SingleFileItem("/abs/file.jpg", "file.jpg", MediaType.Image), + Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, + AssetId = Guid.NewGuid(), + }; + var movId = Guid.NewGuid(); + var mov = new ProxyFile { AssetId = movId, ProxyType = ProxyType.WebVideo }; + context.AddProxyFile(new ProxyFile { AssetId = context.AssetId, ProxyType = ProxyType.Thumbnail }); + context.AddProxyFile(mov); + + context.ProxyFilesOf(movId).Should().ContainSingle().Which.Should().BeSameAs(mov); + } } diff --git a/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs b/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs index 54dc364..2322972 100644 --- a/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs +++ b/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs @@ -31,6 +31,9 @@ internal sealed partial class NewContentWriter( } repository.Add(asset); + foreach (var proxy in context.ProxyFilesOf(asset.Id)) + repository.AddProxyFile(proxy); + if (await saver.TrySaveNewAsync(context, asset, mov?.Id, ct)) return asset; @@ -52,7 +55,6 @@ private MediaAsset BuildAsset(IngestionContext context) MediaType = context.Item.PrimaryMediaType, IsSoftDeleted = false, LastSeenOnNas = clock.GetCurrentInstant(), - ProxyFiles = [.. context.ProxyFiles.Where(proxy => proxy.AssetId == context.AssetId)], Metadata = new Metadata { Width = exif.Width, diff --git a/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs b/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs index e5e2f45..d9f057e 100644 --- a/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs +++ b/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs @@ -30,6 +30,9 @@ public IReadOnlyList ProxyFiles } } + public IReadOnlyList ProxyFilesOf(Guid ownerAssetId) + => [.. ProxyFiles.Where(proxy => proxy.AssetId == ownerAssetId)]; + public void AddProxyFile(ProxyFile proxyFile) { lock (proxyFilesLock) diff --git a/src/Anichron.Worker/Ingestion/Proxy/PairedMovProxyWriter.cs b/src/Anichron.Worker/Ingestion/Proxy/PairedMovProxyWriter.cs index 9e30bd5..fbc25ce 100644 --- a/src/Anichron.Worker/Ingestion/Proxy/PairedMovProxyWriter.cs +++ b/src/Anichron.Worker/Ingestion/Proxy/PairedMovProxyWriter.cs @@ -32,7 +32,7 @@ await stagingWriter.WriteAsync( ct); } - foreach (var proxy in context.ProxyFiles.Where(proxy => proxy.AssetId == mov.Id)) + foreach (var proxy in context.ProxyFilesOf(mov.Id)) repository.AddProxyFile(proxy); Log.ProxiesGenerated(logger, missing.Length, context.Item.SecondaryFile.RelativePath); From 5024564c961882f58b6b8b35e439f89a1739c1dd Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Sun, 18 Oct 2026 22:09:01 +1300 Subject: [PATCH 3/4] feat(worker): Update the row in place when a file's content changes at its path (#250) A file whose bytes change at a stored path is classified ContentChanged and processed like new content; ContentChangedWriter then updates that row instead of inserting one beside it, so it no longer fails on every crawl. The row keeps its Id, interactions, BurstId and PairedAssetId; its hash, capture date, media type, Metadata and proxy rows (updated by type) are new. IngestionContext separates AttemptId, which names temporary files and scopes compensation, from AssetId, the owner of the proxies, which classification sets to the stored row for ContentChanged. A failed attempt therefore deletes only its own proxies, never the row's previous ones. The old content directory is left to the orphan sweeper (#174). An edited Live Photo MOV is #285. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 +- .../Repository/MediaAssetRepositoryTests.cs | 30 ++ .../Data/Repository/MediaAssetRepository.cs | 9 + .../WorkerServiceCollectionExtensionsTests.cs | 13 +- .../ContentChangedWriterTests.cs | 351 ++++++++++++++++++ .../IngestionClassifierTests.cs | 75 +++- .../Classification/NewContentWriterTests.cs | 1 + .../Ingestion/IngestionSaverTests.cs | 1 + .../Middlewares/ClassificationFixture.cs | 4 +- .../ClassificationMiddlewareTests.cs | 50 +++ .../ContentHashingMiddlewareTests.cs | 12 +- .../ExifExtractionMiddlewareTests.cs | 2 +- .../Middlewares/ImageProxyMiddlewareTests.cs | 3 + .../Middlewares/LoggingMiddlewareTests.cs | 2 +- .../Middlewares/PersistenceMiddlewareTests.cs | 1 + .../Middlewares/VideoProxyMiddlewareTests.cs | 4 +- .../Pipeline/ImageIngestionBlurhashTests.cs | 1 + .../Pipeline/IngestionContextTests.cs | 4 + .../Pipeline/IngestionPipelineBuilderTests.cs | 2 +- .../Pipeline/IngestionPipelineRunnerTests.cs | 29 +- .../Proxy/PairedMovProxyWriterTests.cs | 1 + .../Proxy/ProxyStagingWriterTests.cs | 2 +- .../Crawling/FileIngestionPipeline.cs | 2 +- .../WorkerServiceCollectionExtensions.cs | 1 + .../Classification/ContentChangedWriter.cs | 58 +++ .../Classification/IngestionClassification.cs | 3 +- .../Classification/IngestionClassifier.cs | 20 +- .../Classification/NewContentWriter.cs | 63 +--- .../Classification/ProcessedContentWriter.cs | 65 ++++ .../Middlewares/ClassificationMiddleware.cs | 1 + .../Ingestion/Pipeline/IngestionContext.cs | 4 +- .../Pipeline/IngestionPipelineRunner.cs | 2 +- .../Ingestion/Proxy/ProxyStagingWriter.cs | 2 +- 33 files changed, 741 insertions(+), 83 deletions(-) create mode 100644 src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs create mode 100644 src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs create mode 100644 src/Anichron.Worker/Ingestion/Classification/ProcessedContentWriter.cs diff --git a/CLAUDE.md b/CLAUDE.md index 79f1963..44d016b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,8 +156,8 @@ The API reads files only through two narrow stores in `Anichron.Infrastructure/S - Each `IImageProxyGenerator`, and the `BlurhashEncoder`, declares a `ProxyBound`; `ImageProxyMiddleware` decodes once to `Rgb24`, shrinks the shared image to each bound largest-first, and hands every step **its own copy** at that size. The encoder's string goes on the `IngestionContext` and `NewContentWriter` writes it to `Metadata`. Generators may crop/recolour/resize their copy freely; only the middleware resizes the shared image - FFmpeg transcodes video with runtime GPU detection: QuickSync (`h264_qsv`) → NVENC (`h264_nvenc`) → AMF (`h264_amf`) → software (`libx264`) - Burst detection (**planned, not yet implemented**): will group rapid-fire sequences and assign a `PrimaryAssetId` cover. The `Burst` entity and schema exist; no detection code is wired into the Worker yet. -- Reconciliation (**planned, Epic 4**): a periodic scan soft-deletes assets whose file is gone (interactions survive) and restores them when it reappears (#33, #34, #37). Moves are already handled at ingest (#35): when the hash matches an active row, `KnownFileReconciler` refreshes `LastSeenOnNas` if the path is unchanged (`Unchanged`), moves the row to the new path if its stored file is gone (`Moved`), and leaves it alone if the stored file still exists (`DuplicateCopy`, reported by #36). A Live Photo's MOV row gets the same treatment by its own hash. A move onto a path another row already holds, soft-deleted rows included (`PathTaken`), is logged as a warning and skipped; if a concurrent consumer takes the path after classification, the save fails and the item is retried on the next crawl -- Failure ownership: `ProxyStagingWriter` is the single place a proxy reaches its final name (architecture rule `proxies-reach-final-name-only-in-staging-writer`) — it writes under a per-attempt `..tmp` suffix (so concurrent attempts on the same content never share a temporary file), renames into place once complete, and registers the proxy on the `IngestionContext` at that moment; the proxy middlewares only supply the bytes. If a step throws, `IngestionPipelineRunner` deletes exactly those registered proxies plus its own temporary files, and removes the directory only if it is left empty — never recursively, since a concurrent attempt on the same content shares it until one of them persists. An attempt that loses a uniqueness race (`LostIngestionRace`: the active content-hash index or the one-proxy-per-asset-and-type index) first releases its paired MOV's registrations, because the winner owns those content-derived files. A clean shutdown is treated differently: the in-flight temporary file is dropped, but proxies already completed in that run are kept, and the consumer loop re-throws the cancellation instead of recording an item failure and draining the queue. A hard kill or the storage-config cascade delete is out of reach in-process and belongs to the orphan sweeper (#174). +- Reconciliation (**planned, Epic 4**): a periodic scan soft-deletes assets whose file is gone (interactions survive) and restores them when it reappears (#33, #34, #37). Moves are already handled at ingest (#35): when the hash matches an active row, `KnownFileReconciler` refreshes `LastSeenOnNas` if the path is unchanged (`Unchanged`), moves the row to the new path if its stored file is gone (`Moved`), and leaves it alone if the stored file still exists (`DuplicateCopy`, reported by #36). A Live Photo's MOV row gets the same treatment by its own hash. A move onto a path another row already holds, soft-deleted rows included (`PathTaken`), is logged as a warning and skipped; if a concurrent consumer takes the path after classification, the save fails and the item is retried on the next crawl. A file **edited in place** (hash unknown, but an active row holds the path: `ContentChanged`) is processed like new content and `ContentChangedWriter` updates that row: new hash, capture date, media type, `Metadata` and proxy rows (updated by type), while `Id`, interactions, `BurstId` and `PairedAssetId` are kept. A soft-deleted row at the path still blocks it (restore, #34). An edited Live Photo **MOV** keeps its stored row unchanged for now +- Failure ownership: `ProxyStagingWriter` is the single place a proxy reaches its final name (architecture rule `proxies-reach-final-name-only-in-staging-writer`) — it writes under a per-attempt `..tmp` suffix (so concurrent attempts on the same content never share a temporary file), renames into place once complete, and registers the proxy on the `IngestionContext` at that moment; the proxy middlewares only supply the bytes. If a step throws, `IngestionPipelineRunner` deletes exactly those registered proxies plus its own temporary files, and removes the directory only if it is left empty — never recursively, since a concurrent attempt on the same content shares it until one of them persists. An attempt that loses a uniqueness race (`LostIngestionRace`: the active content-hash index or the one-proxy-per-asset-and-type index) first releases its paired MOV's registrations, because the winner owns those content-derived files. A clean shutdown is treated differently: the in-flight temporary file is dropped, but proxies already completed in that run are kept, and the consumer loop re-throws the cancellation instead of recording an item failure and draining the queue. Each attempt has its own `IngestionContext.AttemptId`, which names temporary files and scopes compensation; `IngestionContext.AssetId` is the row that owns the proxies, set by classification (the stored row for `ContentChanged`, the attempt id for new content). A hard kill, the storage-config cascade delete, and the old content directory of a file edited in place are out of reach in-process and belong to the orphan sweeper (#174); the old directory is deliberately not deleted after commit, because a copy of the old content elsewhere can be ingested into it in the same crawl. ### Flashback Interaction Rules @@ -181,7 +181,7 @@ Each user has a private set of `UserStorageConfig` entries pointing to NAS roots ### Worker Processing Model -Processing uses a bounded-concurrency `Channel` pipeline. The crawler produces file paths; N consumer tasks process them concurrently. Default concurrency: `Worker:MaxConcurrentFiles = 4`. Processing is always idempotent — files already in the DB (matched by `(StorageConfigId, ContentHash)`) skip extraction and proxy generation; only the row's `FilePath`/`LastSeenOnNas` are updated (see Reconciliation above). +Processing uses a bounded-concurrency `Channel` pipeline. The crawler produces file paths; N consumer tasks process them concurrently. Default concurrency: `Worker:MaxConcurrentFiles = 4`. Processing is always idempotent — files already in the DB (matched by `(StorageConfigId, ContentHash)`) skip extraction and proxy generation; only the row's `FilePath`/`LastSeenOnNas` are updated, and a file whose bytes changed at a stored path updates that row in place (see Reconciliation above). **Classify once, then one handler per outcome (#254).** Right after hashing, `ClassificationMiddleware` asks `IngestionClassifier`, which loads every relevant row in one query (`FindByHashesOrPathsAsync`: active rows by either half's hash, any row at either half's path), and gives each half a `FileOutcome` (`New`, `Unchanged`, `Moved`, `DuplicateCopy`, `PathTaken`) plus a `PairingAction` for the item (`None`, `AttachMov`, `BackfillMovProxy`). No other step asks the database whether content is known, except `IngestionSaver` checking whether a concurrent consumer's row landed after a lost insert race. A `New` primary continues through EXIF, proxies and `PersistenceMiddleware`, which hands the write to the `IOutcomeWriter` for that outcome (`NewContentWriter` inserts). Every other outcome stops the item after `LivePhotoPairer` and `KnownFileReconciler` have updated the stored rows. All saves go through `IngestionSaver`, and unique violations are recognised by index name with `IsUniqueViolation` (`Anichron.Core/Data/UniqueViolations.cs`). A new outcome, such as content changed at a known path (#250), is a classifier branch, a `FileOutcome` value and its handler or writer; no other middleware changes. When a Live Photo's HEIC is already stored without its MOV (the halves arrived in different crawls), `LivePhotoPairer` attaches the MOV to the stored row and commits before the halves are reconciled; `PairedMovResolver` reuses the MOV row the classifier found rather than inserting a duplicate. The paired MOV is its own asset and owns its `WebVideo` proxy, written into the MOV's own content-hash directory by `PairedMovProxyWriter` only when that MOV has none yet — on first ingest, on late pairing, and on any later crawl of an already-paired HEIC whose MOV still lacks it (self-heal). diff --git a/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs b/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs index 2079f8c..1df19df 100644 --- a/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs +++ b/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs @@ -142,4 +142,34 @@ public void AddProxyFile_TracksTheRowForInsertEvenWithItsIdPreset() db.Entry(proxy).State.Should().Be(EntityState.Added); } + + [Fact] + public void RemoveProxyFile_TracksTheRowForDelete() + { + using var db = AnichronDbContexts.InMemory(); + var proxy = new ProxyFile { Id = Guid.NewGuid(), AssetId = Guid.NewGuid(), ProxyType = ProxyType.WebVideo }; + db.ProxyFiles.Attach(proxy); + + new EfMediaAssetRepository(db).RemoveProxyFile(proxy); + + db.Entry(proxy).State.Should().Be(EntityState.Deleted); + } + + [Fact] + public async Task LoadMetadataAsync_LoadsTheAssetsMetadataRow() + { + var ct = TestContext.Current.CancellationToken; + var asset = AssetAt(ConfigA, "cafe", "a.jpg"); + asset.Metadata = new Metadata { AssetId = asset.Id, Width = 640, Height = 480 }; + await using var seed = CreateDb(out var name); + seed.MediaAssets.Add(asset); + await seed.SaveChangesAsync(ct); + await using var db = AnichronDbContexts.InMemory(name); + var tracked = await db.MediaAssets.SingleAsync(a => a.Id == asset.Id, ct); + + await new EfMediaAssetRepository(db).LoadMetadataAsync(tracked, ct); + + tracked.Metadata.Should().NotBeNull(); + tracked.Metadata.Width.Should().Be(640); + } } diff --git a/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs b/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs index d3bb355..3bb6e3d 100644 --- a/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs +++ b/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs @@ -18,8 +18,11 @@ Task> FindByHashesOrPathsAsync( IReadOnlyCollection filePaths, CancellationToken ct); + Task LoadMetadataAsync(MediaAsset asset, CancellationToken ct); + void Add(MediaAsset asset); void AddProxyFile(ProxyFile proxyFile); + void RemoveProxyFile(ProxyFile proxyFile); } public sealed class EfMediaAssetRepository(AnichronDbContext db) : IMediaAssetRepository @@ -40,9 +43,15 @@ public async Task> FindByHashesOrPathsAsync( && ((!a.IsSoftDeleted && contentHashes.Contains(a.ContentHash)) || filePaths.Contains(a.FilePath))) .ToListAsync(ct); + public Task LoadMetadataAsync(MediaAsset asset, CancellationToken ct) + => db.Entry(asset).Reference(a => a.Metadata).LoadAsync(ct); + public void Add(MediaAsset asset) => db.MediaAssets.Add(asset); public void AddProxyFile(ProxyFile proxyFile) => db.ProxyFiles.Add(proxyFile); + + public void RemoveProxyFile(ProxyFile proxyFile) + => db.ProxyFiles.Remove(proxyFile); } diff --git a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs index d38f010..ed479f3 100644 --- a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs @@ -218,7 +218,6 @@ public void AddWorkerHostedServices_RegistersFourHostedServices() { typeof(IngestionClassifier), typeof(IngestionClassifier), ServiceLifetime.Scoped }, { typeof(LivePhotoPairer), typeof(LivePhotoPairer), ServiceLifetime.Scoped }, { typeof(KnownFileReconciler), typeof(KnownFileReconciler), ServiceLifetime.Scoped }, - { typeof(IOutcomeWriter), typeof(NewContentWriter), ServiceLifetime.Scoped }, { typeof(IProcessLauncher), typeof(SystemProcessLauncher), ServiceLifetime.Singleton }, { typeof(IVideoProcessor), typeof(FfmpegVideoProcessor), ServiceLifetime.Singleton }, { typeof(IImageProcessor), typeof(ImageSharpProcessor), ServiceLifetime.Singleton }, @@ -236,6 +235,18 @@ public void AddIngestionServices_RegistersTheImplementationWithItsLifetime(Type .Which.Should().Match(d => d.ImplementationType == implementation && d.Lifetime == lifetime); } + [Fact] + public void AddIngestionServices_RegistersAScopedWriterForEveryOutcomeThatNeedsProcessing() + { + var services = new ServiceCollection(); + + services.AddIngestionServices(); + + var writers = services.Where(d => d.ServiceType == typeof(IOutcomeWriter)).ToList(); + writers.Select(d => d.ImplementationType).Should().BeEquivalentTo([typeof(NewContentWriter), typeof(ContentChangedWriter)]); + writers.Should().OnlyContain(d => d.Lifetime == ServiceLifetime.Scoped); + } + [Fact] public void AddWorkerCoreServices_RegistersTheSystemTimeProvider() { diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs new file mode 100644 index 0000000..56131fe --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs @@ -0,0 +1,351 @@ +using Anichron.Core; +using Anichron.Core.Data; +using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; +using Anichron.Testing.Data; +using Anichron.Worker.Ingestion; +using Anichron.Worker.Ingestion.Classification; +using Anichron.Worker.Ingestion.Pipeline; +using Anichron.Worker.Ingestion.Proxy; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Testing; +using NodaTime; +using NSubstitute.ExceptionExtensions; +using System.IO.Abstractions.TestingHelpers; + +namespace Anichron.Worker.Tests.Unit.Ingestion.Classification; + +public sealed class ContentChangedWriterTests +{ + private static readonly Guid StoredId = Guid.Parse("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee"); + private static readonly Guid MovId = Guid.Parse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"); + private static readonly Guid BurstId = Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"); + private static readonly Instant FirstSeen = Instant.FromUtc(2024, 1, 1, 0, 0, 0); + private static readonly Instant Now = Instant.FromUtc(2026, 5, 19, 10, 0, 0); + + private static readonly ExifData EditedExif = new( + Width: 4032, + Height: 3024, + Orientation: 6, + DateCaptured: new LocalDateTime(2022, 8, 9, 7, 30), + Latitude: 35.68, + Longitude: 139.69, + CameraMake: "Fujifilm", + CameraModel: "X100V", + LensModel: "23mm F2", + DurationInSeconds: 3); + + private sealed class TestFixture + { + public IMediaAssetRepository Repository { get; } = Substitute.For(); + public IUnitOfWork UnitOfWork { get; } = Substitute.For(); + public IGuidFactory GuidFactory { get; } = Substitute.For(); + public IVideoProxyGenerator MovGenerator { get; } = Substitute.For(); + public MockFileSystem FileSystem { get; } = new(); + public List Added { get; } = []; + public List Removed { get; } = []; + public Metadata StoredMetadata { get; } = new() { AssetId = StoredId, Width = 10, Height = 10, CameraMake = "Old" }; + public MediaAsset Stored { get; } + + public TestFixture() + { + Stored = new MediaAsset + { + Id = StoredId, + StorageConfigId = Guid.NewGuid(), + BurstId = BurstId, + FilePath = "2022/photo.jpg", + FileName = "photo.jpg", + ContentHash = "0ldc0ntent", + MediaType = MediaType.Image, + LastSeenOnNas = FirstSeen, + ProxyFiles = + [ + OldProxy(ProxyType.Thumbnail, "0l/old/thumbnail.jpg"), + OldProxy(ProxyType.FullPreview, "0l/old/preview.jpg"), + ], + Interactions = [new AssetInteraction { AssetId = StoredId, UserId = Guid.NewGuid(), IsStarred = true }], + }; + Stored.SetCaptureDate(new LocalDateTime(2019, 3, 4, 5, 6)); + Repository.LoadMetadataAsync(Stored, Arg.Any()) + .Returns(_ => + { + Stored.Metadata = StoredMetadata; + return Task.CompletedTask; + }); + Repository.When(r => r.AddProxyFile(Arg.Any())).Do(call => Added.Add(call.Arg())); + Repository.When(r => r.RemoveProxyFile(Arg.Any())).Do(call => Removed.Add(call.Arg())); + } + + public ContentChangedWriter Build() + { + var clock = Substitute.For(); + clock.GetCurrentInstant().Returns(Now); + MovGenerator.FileName.Returns("video_720p.mp4"); + MovGenerator.ProxyType.Returns(ProxyType.WebVideo); + MovGenerator.TranscodeAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => FileSystem.File.WriteAllBytesAsync(call.ArgAt(1), [0x01])); + var stagingWriter = new ProxyStagingWriter(FileSystem, clock, GuidFactory, new FakeLogger()); + return new(Repository, new IngestionSaver(UnitOfWork, Repository), FileSystem, clock, + new PairedMovResolver(Repository, FileSystem, clock, GuidFactory), + new PairedMovProxyWriter( + [MovGenerator], new TwoLevelHexShardStrategy(), stagingWriter, Repository, + new FakeLogger()), + new FakeLogger()); + } + + public IngestionContext Context(IngestionItem? item = null, MediaAsset? storedMov = null) + { + var resolvedItem = item ?? new SingleFileItem("/abs/2022/photo.jpg", "2022/photo.jpg", MediaType.Image); + var isPair = resolvedItem.SecondaryFile is not null; + return new IngestionContext + { + Item = resolvedItem, + Config = new UserStorageConfig { Id = Stored.StorageConfigId, UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), + AssetId = StoredId, + ContentHash = "n3wc0ntent", + SecondaryHash = isPair ? "cafebabe" : null, + Classification = new IngestionClassification( + new FileClassification(FileOutcome.ContentChanged, Stored), + isPair ? new FileClassification(FileOutcome.Unchanged, storedMov) : null, + PairingAction.None), + Exif = EditedExif, + Blurhash = "LEHV6nWB2yk8pyo0adR*.7kCMdnj", + }; + } + + public Task WriteAsync(IngestionContext context) + => Build().TryWriteAsync(context, CancellationToken.None); + + private static ProxyFile OldProxy(ProxyType type, string path) => new() + { + Id = Guid.NewGuid(), + AssetId = StoredId, + ProxyType = type, + ProxyPath = path, + SizeBytes = 1, + CreatedAt = FirstSeen, + }; + } + + private static ProxyFile NewProxy(ProxyType type, string path, Guid? owner = null) => new() + { + Id = Guid.NewGuid(), + AssetId = owner ?? StoredId, + ProxyType = type, + ProxyPath = path, + SizeBytes = 2048, + CreatedAt = Now, + }; + + [Fact] + public void Outcome_IsContentChanged() + => new TestFixture().Build().Outcome.Should().Be(FileOutcome.ContentChanged); + + // ========================================================================== + // The row + // ========================================================================== + + [Fact] + public async Task TryWriteAsync_KeepsTheStoredRowAndItsIdentityAsync() + { + var fx = new TestFixture(); + var interactions = fx.Stored.Interactions.ToList(); + + var asset = await fx.WriteAsync(fx.Context()); + + asset.Should().BeSameAs(fx.Stored); + asset.Id.Should().Be(StoredId); + asset.BurstId.Should().Be(BurstId); + asset.FilePath.Should().Be("2022/photo.jpg"); + asset.Interactions.Should().Equal(interactions); + fx.Repository.DidNotReceive().Add(Arg.Any()); + } + + [Fact] + public async Task TryWriteAsync_DescribesTheNewContentAsync() + { + var fx = new TestFixture(); + + var asset = await fx.WriteAsync(fx.Context(new SingleFileItem("/abs/2022/photo.jpg", "2022/photo.jpg", MediaType.Video))); + + asset!.ContentHash.Should().Be("n3wc0ntent"); + asset.DateCaptured.Should().Be(new LocalDateTime(2022, 8, 9, 7, 30)); + (asset.Year, asset.Month, asset.Day).Should().Be((2022, 8, 9)); + asset.MediaType.Should().Be(MediaType.Video); + asset.LastSeenOnNas.Should().Be(Now); + } + + [Fact] + public async Task TryWriteAsync_RewritesEveryMetadataFieldOnTheLoadedRowAsync() + { + var fx = new TestFixture(); + + var asset = await fx.WriteAsync(fx.Context()); + + asset!.Metadata.Should().BeSameAs(fx.StoredMetadata); + asset.Metadata.Should().BeEquivalentTo(new + { + AssetId = StoredId, + Width = 4032, + Height = 3024, + Orientation = 6, + Latitude = 35.68, + Longitude = 139.69, + CameraMake = "Fujifilm", + CameraModel = "X100V", + LensModel = "23mm F2", + DurationInSeconds = 3, + Blurhash = "LEHV6nWB2yk8pyo0adR*.7kCMdnj", + }); + } + + [Fact] + public async Task TryWriteAsync_RowWithoutMetadata_GivesItAMetadataRowAsync() + { + var fx = new TestFixture(); + fx.Repository.LoadMetadataAsync(fx.Stored, Arg.Any()).Returns(Task.CompletedTask); + + var asset = await fx.WriteAsync(fx.Context()); + + asset!.Metadata.Should().NotBeNull(); + asset.Metadata.AssetId.Should().Be(StoredId); + asset.Metadata.Width.Should().Be(4032); + } + + // ========================================================================== + // Proxy rows + // ========================================================================== + + [Fact] + public async Task TryWriteAsync_RegeneratedType_UpdatesTheExistingRowInPlaceAsync() + { + var fx = new TestFixture(); + var context = fx.Context(); + var thumbnailRow = fx.Stored.ProxyFiles.Single(p => p.ProxyType == ProxyType.Thumbnail); + var previewRow = fx.Stored.ProxyFiles.Single(p => p.ProxyType == ProxyType.FullPreview); + var thumbnailId = thumbnailRow.Id; + context.AddProxyFile(NewProxy(ProxyType.Thumbnail, "n3/new/thumbnail.jpg")); + context.AddProxyFile(NewProxy(ProxyType.FullPreview, "n3/new/preview.jpg")); + + await fx.WriteAsync(context); + + thumbnailRow.Id.Should().Be(thumbnailId); + thumbnailRow.Should().BeEquivalentTo(new { ProxyPath = "n3/new/thumbnail.jpg", SizeBytes = 2048L, CreatedAt = Now }); + previewRow.ProxyPath.Should().Be("n3/new/preview.jpg"); + fx.Added.Should().BeEmpty(); + fx.Removed.Should().BeEmpty(); + } + + [Fact] + public async Task TryWriteAsync_TypeTheRowLacked_IsAddedAsync() + { + var fx = new TestFixture(); + var context = fx.Context(); + var webVideo = NewProxy(ProxyType.WebVideo, "n3/new/video_720p.mp4"); + context.AddProxyFile(NewProxy(ProxyType.Thumbnail, "n3/new/thumbnail.jpg")); + context.AddProxyFile(NewProxy(ProxyType.FullPreview, "n3/new/preview.jpg")); + context.AddProxyFile(webVideo); + + await fx.WriteAsync(context); + + fx.Added.Should().ContainSingle().Which.Should().BeSameAs(webVideo); + } + + [Fact] + public async Task TryWriteAsync_TypeNoLongerGenerated_IsRemovedAsync() + { + var fx = new TestFixture(); + var context = fx.Context(); + var previewRow = fx.Stored.ProxyFiles.Single(p => p.ProxyType == ProxyType.FullPreview); + context.AddProxyFile(NewProxy(ProxyType.Thumbnail, "n3/new/thumbnail.jpg")); + + await fx.WriteAsync(context); + + fx.Removed.Should().ContainSingle().Which.Should().BeSameAs(previewRow); + } + + [Fact] + public async Task TryWriteAsync_ProxiesOfAnotherOwner_AreNotAttachedToTheRowAsync() + { + var fx = new TestFixture(); + var context = fx.Context(); + var thumbnailRow = fx.Stored.ProxyFiles.Single(p => p.ProxyType == ProxyType.Thumbnail); + context.AddProxyFile(NewProxy(ProxyType.Thumbnail, "n3/new/thumbnail.jpg")); + context.AddProxyFile(NewProxy(ProxyType.FullPreview, "n3/new/preview.jpg")); + context.AddProxyFile(NewProxy(ProxyType.Thumbnail, "ca/mov/thumbnail.jpg", owner: MovId)); + + await fx.WriteAsync(context); + + thumbnailRow.ProxyPath.Should().Be("n3/new/thumbnail.jpg"); + fx.Added.Should().BeEmpty(); + } + + // ========================================================================== + // Live Photo + // ========================================================================== + + [Fact] + public async Task TryWriteAsync_EditedHeic_StaysPairedToItsStoredMovAsync() + { + var fx = new TestFixture(); + fx.Stored.PairedAssetId = MovId; + var mov = new MediaAsset + { + Id = MovId, + ContentHash = "cafebabe", + ProxyFiles = [new ProxyFile { ProxyType = ProxyType.WebVideo }], + }; + var context = fx.Context( + new LivePhotoPairItem("/abs/2022/live.heic", "2022/live.heic", "/abs/2022/live.mov", "2022/live.mov"), mov); + + var asset = await fx.WriteAsync(context); + + asset!.PairedAssetId.Should().Be(MovId); + asset.MediaType.Should().Be(MediaType.LivePhoto); + fx.Repository.DidNotReceive().Add(Arg.Any()); + await fx.MovGenerator.DidNotReceive().TranscodeAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + + // ========================================================================== + // Saving + // ========================================================================== + + [Fact] + public async Task TryWriteAsync_SavesOnceAsync() + { + var fx = new TestFixture(); + + await fx.WriteAsync(fx.Context()); + + await fx.UnitOfWork.Received(1).SaveChangesAsync(Arg.Any()); + } + + [Fact] + public async Task TryWriteAsync_AnotherConsumerCommittedTheNewContentFirst_SkipsWithoutThrowingAsync() + { + var fx = new TestFixture(); + var context = fx.Context(); + fx.UnitOfWork.SaveChangesAsync(Arg.Any()) + .ThrowsAsync(DatabaseErrors.UniqueViolation(MediaAssetIndexNames.ActiveContentHashUnique)); + fx.Repository.FindByHashAsync("n3wc0ntent", context.Config.Id, Arg.Any()) + .Returns(new MediaAsset { ContentHash = "n3wc0ntent" }); + + var asset = await fx.WriteAsync(context); + + asset.Should().BeNull(); + } + + [Fact] + public async Task TryWriteAsync_SaveFailsForAnotherReason_RethrowsAsync() + { + var fx = new TestFixture(); + fx.UnitOfWork.SaveChangesAsync(Arg.Any()) + .ThrowsAsync(DatabaseErrors.UniqueViolation("IX_MediaAssets_StorageConfigId_FilePath")); + + var act = () => fx.WriteAsync(fx.Context()); + + await act.Should().ThrowAsync(); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/IngestionClassifierTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/IngestionClassifierTests.cs index 8c041b8..9e8a430 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/IngestionClassifierTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/IngestionClassifierTests.cs @@ -36,7 +36,7 @@ public Task ClassifyAsync(IngestionContext context) { Item = new SingleFileItem($"/abs/{relativePath}", relativePath, MediaType.Image), Config = Config, - AssetId = Guid.NewGuid(), + AttemptId = Guid.NewGuid(), ContentHash = "abc123", }; @@ -44,7 +44,7 @@ public Task ClassifyAsync(IngestionContext context) { Item = new LivePhotoPairItem($"/abs/{heicPath}", heicPath, $"/abs/{movPath}", movPath), Config = Config, - AssetId = Guid.NewGuid(), + AttemptId = Guid.NewGuid(), ContentHash = "abc123", SecondaryHash = "cafebabe", }; @@ -139,6 +139,41 @@ public async Task ClassifyAsync_OnlyASoftDeletedRowHoldsTheContent_IsNew() result.Primary.Outcome.Should().Be(FileOutcome.New); } + [Fact] + public async Task ClassifyAsync_UnknownHashAtAPathAnActiveRowHolds_IsContentChangedAndNeedsProcessing() + { + var fx = new TestFixture(); + var stored = Row("f00d", "2023/photo.jpg"); + fx.Store(stored); + + var result = await fx.ClassifyAsync(Single("2023/photo.jpg")); + + result.Primary.Should().Be(new FileClassification(FileOutcome.ContentChanged, stored)); + result.NeedsProcessing.Should().BeTrue(); + } + + [Fact] + public async Task ClassifyAsync_UnknownHashAtAPathOnlyASoftDeletedRowHolds_IsNew() + { + var fx = new TestFixture(); + fx.Store(Row("f00d", "2023/photo.jpg", softDeleted: true)); + + var result = await fx.ClassifyAsync(Single("2023/photo.jpg")); + + result.Primary.Should().Be(new FileClassification(FileOutcome.New, null)); + } + + [Fact] + public async Task ClassifyAsync_UnknownHashWhileAnActiveRowHoldsAnotherPath_IsNew() + { + var fx = new TestFixture(); + fx.Store(Row("f00d", "2023/other.jpg")); + + var result = await fx.ClassifyAsync(Single("2023/photo.jpg")); + + result.Primary.Should().Be(new FileClassification(FileOutcome.New, null)); + } + [Fact] public async Task ClassifyAsync_SingleFile_HasNoSecondaryAndNoPairing() { @@ -248,4 +283,40 @@ public async Task ClassifyAsync_HeicPairedButThisMovNotStored_LeavesThePairingAl (await fx.ClassifyAsync(Pair())).Pairing.Should().Be(PairingAction.None); } + + [Fact] + public async Task ClassifyAsync_EditedHeicPairedToThisStoredMov_LeavesThePairingToPersistence() + { + var fx = new TestFixture(); + var heic = Row("f00d", "live.heic", pairedAssetId: MovId); + fx.Store(heic, Row("cafebabe", "live.mov", id: MovId)); + + var result = await fx.ClassifyAsync(Pair()); + + result.Primary.Should().Be(new FileClassification(FileOutcome.ContentChanged, heic)); + result.Pairing.Should().Be(PairingAction.None); + } + + [Fact] + public async Task ClassifyAsync_EditedUnpairedHeic_LeavesThePairingToPersistence() + { + var fx = new TestFixture(); + fx.Store(Row("f00d", "live.heic")); + + (await fx.ClassifyAsync(Pair())).Pairing.Should().Be(PairingAction.None); + } + + [Fact] + public async Task ClassifyAsync_EditedMov_IsContentChangedOnItsOwn() + { + var fx = new TestFixture(); + var heic = Row("abc123", "live.heic", pairedAssetId: MovId); + var mov = Row("beef", "live.mov", id: MovId); + fx.Store(heic, mov); + + var result = await fx.ClassifyAsync(Pair()); + + result.Primary.Should().Be(new FileClassification(FileOutcome.Unchanged, heic)); + result.Secondary.Should().Be(new FileClassification(FileOutcome.ContentChanged, mov)); + } } diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs index 7c2f991..6a67f32 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/NewContentWriterTests.cs @@ -81,6 +81,7 @@ private static IngestionContext MakeContext( { Item = resolvedItem, Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), ContentHash = contentHash, SecondaryHash = isPair ? "cafebabe" : null, diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/IngestionSaverTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/IngestionSaverTests.cs index 89422e7..fe8d5bc 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/IngestionSaverTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/IngestionSaverTests.cs @@ -22,6 +22,7 @@ private sealed class TestFixture { Item = new SingleFileItem("/abs/photo.heic", "photo.heic", MediaType.Image), Config = new UserStorageConfig { Id = ConfigId, UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), }; diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationFixture.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationFixture.cs index fe7c9c9..38f3f34 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationFixture.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationFixture.cs @@ -66,7 +66,7 @@ public Task DidNotTranscodeAsync() { Item = new SingleFileItem($"/abs/{relativePath}", relativePath, MediaType.Image), Config = new UserStorageConfig { Id = ConfigId, UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.NewGuid(), + AttemptId = Guid.NewGuid(), ContentHash = "abc123", }; @@ -74,7 +74,7 @@ public Task DidNotTranscodeAsync() { Item = new LivePhotoPairItem($"/abs/{heicPath}", heicPath, $"/abs/{movPath}", movPath), Config = new UserStorageConfig { Id = ConfigId, UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.NewGuid(), + AttemptId = Guid.NewGuid(), ContentHash = "abc123", SecondaryHash = "cafebabe", }; diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationMiddlewareTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationMiddlewareTests.cs index 1a6af07..a59b13c 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationMiddlewareTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ClassificationMiddlewareTests.cs @@ -71,6 +71,56 @@ public async Task InvokeAsync_KnownHashInDifferentConfig_CallsNextAsync() (await NextCalledAsync(fx, Single())).Should().BeTrue(); } + [Fact] + public async Task InvokeAsync_UnknownHash_OwnsTheProxiesUnderTheAttemptIdAsync() + { + var fx = new ClassificationFixture(); + var context = Single(); + + await fx.Build().InvokeAsync(context, NoOpNextAsync, CancellationToken.None); + + context.AssetId.Should().Be(context.AttemptId); + } + + // ========================================================================== + // Content changed at the same path + // ========================================================================== + + [Fact] + public async Task InvokeAsync_EditedFile_CallsNextWithTheStoredRowAsTheOwnerAsync() + { + var fx = new ClassificationFixture(); + var stored = Stored("photo.jpg", contentHash: "f00d"); + fx.Store(stored); + var context = Single("photo.jpg"); + + (await NextCalledAsync(fx, context)).Should().BeTrue(); + + context.Classification!.Primary.Should().Be(new FileClassification(FileOutcome.ContentChanged, stored)); + context.AssetId.Should().Be(stored.Id); + context.AttemptId.Should().NotBe(stored.Id); + await fx.UnitOfWork.DidNotReceive().SaveChangesAsync(Arg.Any()); + } + + [Fact] + public async Task InvokeAsync_UnchangedHeicWithEditedMov_LeavesBothRowsAndSavesOnlyTheHeicRefreshAsync() + { + var fx = new ClassificationFixture(); + var heic = Stored("photo.heic"); + var mov = StoredMov("photo.mov"); + mov.ContentHash = "0ldm0v"; + mov.ProxyFiles = [new ProxyFile { AssetId = mov.Id, ProxyType = ProxyType.WebVideo }]; + heic.PairedAssetId = mov.Id; + fx.Store(heic, mov); + + (await NextCalledAsync(fx, Pair())).Should().BeFalse(); + + heic.LastSeenOnNas.Should().Be(Now); + mov.ContentHash.Should().Be("0ldm0v"); + mov.LastSeenOnNas.Should().Be(FirstSeen); + await fx.UnitOfWork.Received(1).SaveChangesAsync(Arg.Any()); + } + // ========================================================================== // Known content // ========================================================================== diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ContentHashingMiddlewareTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ContentHashingMiddlewareTests.cs index bdd603c..c3dcbc5 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ContentHashingMiddlewareTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ContentHashingMiddlewareTests.cs @@ -12,7 +12,7 @@ public sealed class ContentHashingMiddlewareTests { Item = new SingleFileItem(path, "photo.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.Empty, + AttemptId = Guid.Empty, }; // ========================================================================== @@ -66,13 +66,13 @@ public async Task InvokeAsync_SameContent_ProducesSameHashAsync() { Item = new SingleFileItem("/abs/a.jpg", "a.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.Empty, + AttemptId = Guid.Empty, }; var contextB = new IngestionContext { Item = new SingleFileItem("/abs/b.jpg", "b.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.Empty, + AttemptId = Guid.Empty, }; await new ContentHashingMiddleware(fs).InvokeAsync(contextA, (_, _) => Task.CompletedTask, CancellationToken.None); @@ -93,13 +93,13 @@ public async Task InvokeAsync_DifferentContent_ProducesDifferentHashAsync() { Item = new SingleFileItem("/abs/a.jpg", "a.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.Empty, + AttemptId = Guid.Empty, }; var contextB = new IngestionContext { Item = new SingleFileItem("/abs/b.jpg", "b.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.Empty, + AttemptId = Guid.Empty, }; await new ContentHashingMiddleware(fs).InvokeAsync(contextA, (_, _) => Task.CompletedTask, CancellationToken.None); @@ -120,7 +120,7 @@ public async Task InvokeAsync_LivePhotoPairItem_SetsSecondaryHashAsync() { Item = new LivePhotoPairItem("/abs/photo.heic", "photo.heic", "/abs/photo.mov", "photo.mov"), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.Empty, + AttemptId = Guid.Empty, }; await new ContentHashingMiddleware(fs).InvokeAsync(context, (_, _) => Task.CompletedTask, CancellationToken.None); diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ExifExtractionMiddlewareTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ExifExtractionMiddlewareTests.cs index 8459852..c8dd0b5 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ExifExtractionMiddlewareTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ExifExtractionMiddlewareTests.cs @@ -37,7 +37,7 @@ private static byte[] JpegWithOrientation(byte orientation) { Item = new SingleFileItem(path, "photo.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.NewGuid(), + AttemptId = Guid.NewGuid(), ContentHash = contentHash, }; diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ImageProxyMiddlewareTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ImageProxyMiddlewareTests.cs index e1aa3dd..0c08fa0 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ImageProxyMiddlewareTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/ImageProxyMiddlewareTests.cs @@ -86,6 +86,7 @@ private static IngestionContext MakeContext(MediaType mediaType = MediaType.Imag { Item = new SingleFileItem("/nas/photo.jpg", "photo.jpg", mediaType), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/nas" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), ContentHash = "0123456789abcdef", SecondaryHash = "fedcba9876543210", @@ -483,6 +484,7 @@ public void CanInvoke_LivePhotoItem_ReturnsTrue() { Item = new LivePhotoPairItem("/nas/photo.heic", "photo.heic", "/nas/photo.mov", "photo.mov"), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/nas" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), ContentHash = "0123456789abcdef", SecondaryHash = "fedcba9876543210", @@ -505,6 +507,7 @@ public async Task InvokeAsync_LivePhotoPairItem_GeneratesProxyFilesAsync() { Item = new LivePhotoPairItem("/nas/photo.heic", "photo.heic", "/nas/photo.mov", "photo.mov"), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/nas" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), ContentHash = "0123456789abcdef", SecondaryHash = "fedcba9876543210", diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/LoggingMiddlewareTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/LoggingMiddlewareTests.cs index 28bc534..8c04379 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/LoggingMiddlewareTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/LoggingMiddlewareTests.cs @@ -13,7 +13,7 @@ public sealed class LoggingMiddlewareTests { Item = new SingleFileItem(absolutePath, "photo.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.NewGuid(), + AttemptId = Guid.NewGuid(), }; // ========================================================================== diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/PersistenceMiddlewareTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/PersistenceMiddlewareTests.cs index 2ef4dd9..5e7f2e1 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/PersistenceMiddlewareTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/PersistenceMiddlewareTests.cs @@ -38,6 +38,7 @@ private static IOutcomeWriter Writer(FileOutcome outcome) { Item = new SingleFileItem("/abs/photo.jpg", "photo.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), AssetId = AssetId, Classification = new(new FileClassification(outcome, null), null, PairingAction.None), }; diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/VideoProxyMiddlewareTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/VideoProxyMiddlewareTests.cs index c987124..247d1e3 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/VideoProxyMiddlewareTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Middlewares/VideoProxyMiddlewareTests.cs @@ -62,6 +62,7 @@ private static IngestionContext MakeContext(MediaType mediaType = MediaType.Vide { Item = new SingleFileItem("/nas/video.mp4", "video.mp4", mediaType), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/nas" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), ContentHash = "0123456789abcdef", SecondaryHash = "fedcba9876543210", @@ -250,7 +251,7 @@ public async Task InvokeAsync_VideoItem_TranscodesToTemporaryPathAsync() await fixture.Build().InvokeAsync(context, NoOpNextAsync, CancellationToken.None); await fixture.Generator.Received(1).TranscodeAsync( - "/nas/video.mp4", $"/data/proxies/ab/cd/video_720p.mp4.{context.AssetId:N}.tmp", Arg.Any()); + "/nas/video.mp4", $"/data/proxies/ab/cd/video_720p.mp4.{context.AttemptId:N}.tmp", Arg.Any()); } [Fact] @@ -329,6 +330,7 @@ public void CanInvoke_LivePhotoPairItem_ReturnsFalse() { Item = new LivePhotoPairItem("/nas/photo.heic", "photo.heic", "/nas/photo.mov", "photo.mov"), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/nas" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), }; new TestFixture().Build().CanInvoke(context).Should().BeFalse(); diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs index 006bb07..6005a7d 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/ImageIngestionBlurhashTests.cs @@ -73,6 +73,7 @@ public async Task NewImage_IsPersistedWithItsBlurhashAndNoBlurhashProxyAsync() { Item = new SingleFileItem("/nas/photo.jpg", "photo.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/nas" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), ContentHash = "0123456789abcdef", Classification = new(new FileClassification(FileOutcome.New, null), null, PairingAction.None), diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs index 904eed5..c25e2f5 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionContextTests.cs @@ -13,6 +13,7 @@ public void ProxyFiles_DefaultsToEmptyList() { Item = new SingleFileItem("/abs/file.jpg", "file.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), }; context.ProxyFiles.Should().NotBeNull().And.BeEmpty(); @@ -25,6 +26,7 @@ public void Asset_CanBeSetAndRead() { Item = new SingleFileItem("/abs/file.jpg", "file.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), }; var asset = new MediaAsset(); @@ -39,6 +41,7 @@ public void ReleaseProxyFiles_DropsOnlyThatOwnersRegistrations() { Item = new SingleFileItem("/abs/file.jpg", "file.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), }; var movId = Guid.NewGuid(); @@ -58,6 +61,7 @@ public void ProxyFilesOf_ReturnsOnlyThatOwnersRegistrations() { Item = new SingleFileItem("/abs/file.jpg", "file.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.NewGuid(), }; var movId = Guid.NewGuid(); diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineBuilderTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineBuilderTests.cs index 3ca03dd..f126094 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineBuilderTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineBuilderTests.cs @@ -11,7 +11,7 @@ public sealed class IngestionPipelineBuilderTests { Item = new SingleFileItem("/abs/file.jpg", "file.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.NewGuid(), + AttemptId = Guid.NewGuid(), }; // ========================================================================== diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineRunnerTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineRunnerTests.cs index 7199950..1bae370 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineRunnerTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Pipeline/IngestionPipelineRunnerTests.cs @@ -1,5 +1,6 @@ using Anichron.Core.Domain; using Anichron.Worker.Ingestion; +using Anichron.Worker.Ingestion.Classification; using Anichron.Worker.Ingestion.Pipeline; using Anichron.Worker.Ingestion.Proxy; using Anichron.Worker.Tests.Unit.TestDoubles; @@ -35,7 +36,7 @@ public IngestionPipelineRunner Build(params IIngestionMiddleware[] middlewares) { Item = new SingleFileItem("/abs/file.jpg", "file.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/abs" }, - AssetId = Guid.Parse("abababab-abab-abab-abab-abababababab"), + AttemptId = Guid.Parse("abababab-abab-abab-abab-abababababab"), ContentHash = "0123456789abcdef", }; @@ -126,6 +127,32 @@ public async Task RunAsync_StepThrows_DeletesProxiesRegisteredDuringTheAttemptAs fixture.FileSystem.FileExists("/data/proxies/ab/cd/preview.jpg").Should().BeFalse(); } + [Fact] + public async Task RunAsync_EditedFileFailsAfterItsNewProxies_KeepsTheRowsPreviousProxiesAsync() + { + var fixture = new TestFixture(); + fixture.DirectoryStrategy.GetDirectory(Arg.Any(), "0ldc0ntent").Returns("0l/old"); + fixture.FileSystem.AddFile("/data/proxies/0l/old/thumbnail.jpg", new MockFileData([0x02])); + var stored = new MediaAsset + { + Id = Guid.Parse("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee"), + ContentHash = "0ldc0ntent", + ProxyFiles = [new ProxyFile { ProxyType = ProxyType.Thumbnail, ProxyPath = "0l/old/thumbnail.jpg" }], + }; + var context = MakeContext(); + context.AssetId = stored.Id; + context.Classification = new(new FileClassification(FileOutcome.ContentChanged, stored), null, PairingAction.None); + var runner = fixture.Build( + new ProxyWritingMiddleware(10, fixture.FileSystem, "ab/cd/thumbnail.jpg"), + new ThrowingMiddleware(20, new InvalidOperationException("boom"))); + + var act = () => runner.RunAsync(context, CancellationToken.None); + + await act.Should().ThrowAsync(); + fixture.FileSystem.FileExists("/data/proxies/ab/cd/thumbnail.jpg").Should().BeFalse(); + fixture.FileSystem.FileExists("/data/proxies/0l/old/thumbnail.jpg").Should().BeTrue(); + } + [Fact] public async Task RunAsync_StepThrows_PropagatesTheOriginalExceptionAsync() { diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/PairedMovProxyWriterTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/PairedMovProxyWriterTests.cs index 97462a9..2a10d28 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/PairedMovProxyWriterTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/PairedMovProxyWriterTests.cs @@ -52,6 +52,7 @@ public PairedMovProxyWriter Build() { Item = new LivePhotoPairItem("/nas/photo.heic", "photo.heic", "/nas/photo.mov", "photo.mov"), Config = new UserStorageConfig { Id = ConfigId, RootPath = "/nas" }, + AttemptId = Guid.NewGuid(), AssetId = Guid.Parse("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee"), ContentHash = "deadbeef", SecondaryHash = "cafebabe", diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/ProxyStagingWriterTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/ProxyStagingWriterTests.cs index 6b8c516..d00fe93 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/ProxyStagingWriterTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Proxy/ProxyStagingWriterTests.cs @@ -42,7 +42,7 @@ public ProxyStagingWriter Build() { Item = new SingleFileItem("/nas/photo.jpg", "photo.jpg", MediaType.Image), Config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/nas" }, - AssetId = AttemptId, + AttemptId = AttemptId, ContentHash = "0123456789abcdef", }; diff --git a/src/Anichron.Worker/Crawling/FileIngestionPipeline.cs b/src/Anichron.Worker/Crawling/FileIngestionPipeline.cs index 9b445e1..584fca5 100644 --- a/src/Anichron.Worker/Crawling/FileIngestionPipeline.cs +++ b/src/Anichron.Worker/Crawling/FileIngestionPipeline.cs @@ -128,7 +128,7 @@ private async Task ConsumeAsync( var runner = diScope.ServiceProvider.GetRequiredService(); try { - var context = new IngestionContext { Item = item, Config = config, AssetId = guidFactory.NewGuid() }; + var context = new IngestionContext { Item = item, Config = config, AttemptId = guidFactory.NewGuid() }; await runner.RunAsync(context, ct); } catch (Exception ex) when (IngestionShutdown.IsInProgress(ex, ct)) diff --git a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs index e76ee81..b277369 100644 --- a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs +++ b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs @@ -66,6 +66,7 @@ internal IServiceCollection AddIngestionServices() services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs b/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs new file mode 100644 index 0000000..927388a --- /dev/null +++ b/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs @@ -0,0 +1,58 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; +using Anichron.Worker.Ingestion.Pipeline; +using Anichron.Worker.Ingestion.Proxy; +using NodaTime; +using System.IO.Abstractions; + +namespace Anichron.Worker.Ingestion.Classification; + +internal sealed partial class ContentChangedWriter( + IMediaAssetRepository repository, + IngestionSaver saver, + IFileSystem fileSystem, + IClock clock, + PairedMovResolver pairedMovResolver, + PairedMovProxyWriter pairedMovProxyWriter, + ILogger logger) + : ProcessedContentWriter(saver, fileSystem, clock, pairedMovResolver, pairedMovProxyWriter, logger) +{ + public override FileOutcome Outcome => FileOutcome.ContentChanged; + + protected override async Task WriteAssetAsync(IngestionContext context, CancellationToken ct) + { + var asset = context.Classification!.Primary.Stored!; + Log.ContentChanged(logger, asset.Id, context.Item.RelativePath, asset.ContentHash, context.ContentHash!); + + await repository.LoadMetadataAsync(asset, ct); + DescribeContent(asset, asset.Metadata ??= new Metadata { AssetId = asset.Id }, context); + ReplaceProxyFiles(asset, context.ProxyFilesOf(asset.Id)); + return asset; + } + + private void ReplaceProxyFiles(MediaAsset asset, IReadOnlyList generated) + { + var stale = asset.ProxyFiles.Where(old => generated.All(proxy => proxy.ProxyType != old.ProxyType)); + foreach (var proxy in stale.ToList()) + repository.RemoveProxyFile(proxy); + + foreach (var proxy in generated) + { + if (asset.ProxyFiles.FirstOrDefault(old => old.ProxyType == proxy.ProxyType) is not { } existing) + { + repository.AddProxyFile(proxy); + continue; + } + + existing.ProxyPath = proxy.ProxyPath; + existing.SizeBytes = proxy.SizeBytes; + existing.CreatedAt = proxy.CreatedAt; + } + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Information, Message = "Asset {AssetId} at {RelativePath} changed content from {OldHash} to {NewHash}, updating it in place.")] + public static partial void ContentChanged(ILogger logger, Guid assetId, string relativePath, string oldHash, string newHash); + } +} diff --git a/src/Anichron.Worker/Ingestion/Classification/IngestionClassification.cs b/src/Anichron.Worker/Ingestion/Classification/IngestionClassification.cs index 12d557d..8eb9a5e 100644 --- a/src/Anichron.Worker/Ingestion/Classification/IngestionClassification.cs +++ b/src/Anichron.Worker/Ingestion/Classification/IngestionClassification.cs @@ -9,6 +9,7 @@ internal enum FileOutcome Moved = 2, DuplicateCopy = 3, PathTaken = 4, + ContentChanged = 5, } internal enum PairingAction @@ -25,5 +26,5 @@ internal sealed record IngestionClassification( FileClassification? Secondary, PairingAction Pairing) { - public bool NeedsProcessing => Primary.Outcome == FileOutcome.New; + public bool NeedsProcessing => Primary.Outcome is FileOutcome.New or FileOutcome.ContentChanged; } diff --git a/src/Anichron.Worker/Ingestion/Classification/IngestionClassifier.cs b/src/Anichron.Worker/Ingestion/Classification/IngestionClassifier.cs index fbdded0..ab7fa8c 100644 --- a/src/Anichron.Worker/Ingestion/Classification/IngestionClassifier.cs +++ b/src/Anichron.Worker/Ingestion/Classification/IngestionClassifier.cs @@ -19,14 +19,14 @@ public async Task ClassifyAsync(IngestionContext contex var rootPath = context.Config.RootPath; var primary = Classify(rows, contentHash, item.RelativePath, rootPath); var secondary = mov is null ? null : Classify(rows, secondaryHash, mov.RelativePath, rootPath); - return new(primary, secondary, PairingFor(primary.Stored, secondary)); + return new(primary, secondary, PairingFor(primary, secondary)); } private FileClassification Classify(IReadOnlyList rows, string contentHash, string relativePath, string rootPath) { var stored = rows.FirstOrDefault(row => !row.IsSoftDeleted && row.ContentHash == contentHash); if (stored is null) - return new(FileOutcome.New, null); + return ClassifyUnknownContent(rows, relativePath); if (stored.FilePath == relativePath) return new(FileOutcome.Unchanged, stored); if (fileSystem.File.Exists(fileSystem.Path.Combine(rootPath, stored.FilePath))) @@ -36,12 +36,18 @@ private FileClassification Classify(IReadOnlyList rows, string conte return new(FileOutcome.Moved, stored); } - private static PairingAction PairingFor(MediaAsset? storedPrimary, FileClassification? secondary) - => (storedPrimary, secondary) switch + private static FileClassification ClassifyUnknownContent(IReadOnlyList rows, string relativePath) + { + var atPath = rows.FirstOrDefault(row => !row.IsSoftDeleted && row.FilePath == relativePath); + return atPath is null ? new(FileOutcome.New, null) : new(FileOutcome.ContentChanged, atPath); + } + + private static PairingAction PairingFor(FileClassification primary, FileClassification? secondary) + => (primary, secondary) switch { - (null, _) or (_, null) => PairingAction.None, - ({ PairedAssetId: null }, _) => PairingAction.AttachMov, - ({ PairedAssetId: var pairedId }, { Stored.Id: var movId }) when pairedId == movId => PairingAction.BackfillMovProxy, + ({ Stored: null } or { Outcome: FileOutcome.ContentChanged }, _) or (_, null) => PairingAction.None, + ({ Stored.PairedAssetId: null }, _) => PairingAction.AttachMov, + ({ Stored.PairedAssetId: var pairedId }, { Stored.Id: var movId }) when pairedId == movId => PairingAction.BackfillMovProxy, _ => PairingAction.None, }; } diff --git a/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs b/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs index 2322972..a6d94ee 100644 --- a/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs +++ b/src/Anichron.Worker/Ingestion/Classification/NewContentWriter.cs @@ -7,75 +7,36 @@ namespace Anichron.Worker.Ingestion.Classification; -internal sealed partial class NewContentWriter( +internal sealed class NewContentWriter( IMediaAssetRepository repository, IngestionSaver saver, IFileSystem fileSystem, IClock clock, PairedMovResolver pairedMovResolver, PairedMovProxyWriter pairedMovProxyWriter, - ILogger logger) : IOutcomeWriter + ILogger logger) + : ProcessedContentWriter(saver, fileSystem, clock, pairedMovResolver, pairedMovProxyWriter, logger) { - public FileOutcome Outcome => FileOutcome.New; + public override FileOutcome Outcome => FileOutcome.New; - public async Task TryWriteAsync(IngestionContext context, CancellationToken ct) - { - var asset = BuildAsset(context); - MediaAsset? mov = null; - - if (context.Item.SecondaryFile is not null) - { - mov = pairedMovResolver.Resolve(context, context.Exif!.DateCaptured); - await pairedMovProxyWriter.WriteMissingAsync(context, mov, ct); - asset.PairedAssetId = mov.Id; - } - - repository.Add(asset); - foreach (var proxy in context.ProxyFilesOf(asset.Id)) - repository.AddProxyFile(proxy); - - if (await saver.TrySaveNewAsync(context, asset, mov?.Id, ct)) - return asset; - - Log.AlreadyPersisted(logger, context.Item.RelativePath); - return null; - } - - private MediaAsset BuildAsset(IngestionContext context) + protected override Task WriteAssetAsync(IngestionContext context, CancellationToken ct) { var relativePath = context.Item.RelativePath; - var exif = context.Exif!; + var metadata = new Metadata(); var asset = new MediaAsset { Id = context.AssetId, StorageConfigId = context.Config.Id, FilePath = relativePath, FileName = Path.GetFileName(relativePath), - ContentHash = context.ContentHash!, - MediaType = context.Item.PrimaryMediaType, IsSoftDeleted = false, - LastSeenOnNas = clock.GetCurrentInstant(), - Metadata = new Metadata - { - Width = exif.Width, - Height = exif.Height, - Orientation = exif.Orientation, - Latitude = exif.Latitude, - Longitude = exif.Longitude, - CameraMake = exif.CameraMake, - CameraModel = exif.CameraModel, - LensModel = exif.LensModel, - DurationInSeconds = exif.DurationInSeconds, - Blurhash = context.Blurhash, - }, + Metadata = metadata, }; - asset.SetCaptureDate(CaptureDate.Resolve(exif.DateCaptured, fileSystem, context.Item.AbsolutePath)); - return asset; - } + DescribeContent(asset, metadata, context); - private static partial class Log - { - [LoggerMessage(Level = LogLevel.Debug, Message = "Content of {RelativePath} was persisted concurrently by another consumer, skipping.")] - public static partial void AlreadyPersisted(ILogger logger, string relativePath); + repository.Add(asset); + foreach (var proxy in context.ProxyFilesOf(asset.Id)) + repository.AddProxyFile(proxy); + return Task.FromResult(asset); } } diff --git a/src/Anichron.Worker/Ingestion/Classification/ProcessedContentWriter.cs b/src/Anichron.Worker/Ingestion/Classification/ProcessedContentWriter.cs new file mode 100644 index 0000000..cd93378 --- /dev/null +++ b/src/Anichron.Worker/Ingestion/Classification/ProcessedContentWriter.cs @@ -0,0 +1,65 @@ +using Anichron.Core.Domain; +using Anichron.Worker.Ingestion.Pipeline; +using Anichron.Worker.Ingestion.Proxy; +using NodaTime; +using System.IO.Abstractions; + +namespace Anichron.Worker.Ingestion.Classification; + +internal abstract partial class ProcessedContentWriter( + IngestionSaver saver, + IFileSystem fileSystem, + IClock clock, + PairedMovResolver pairedMovResolver, + PairedMovProxyWriter pairedMovProxyWriter, + ILogger logger) : IOutcomeWriter +{ + public abstract FileOutcome Outcome { get; } + + public async Task TryWriteAsync(IngestionContext context, CancellationToken ct) + { + var asset = await WriteAssetAsync(context, ct); + MediaAsset? mov = null; + + if (context.Item.SecondaryFile is not null) + { + mov = pairedMovResolver.Resolve(context, context.Exif!.DateCaptured); + await pairedMovProxyWriter.WriteMissingAsync(context, mov, ct); + asset.PairedAssetId = mov.Id; + } + + if (await saver.TrySaveNewAsync(context, asset, mov?.Id, ct)) + return asset; + + Log.AlreadyPersisted(logger, context.Item.RelativePath); + return null; + } + + protected abstract Task WriteAssetAsync(IngestionContext context, CancellationToken ct); + + protected void DescribeContent(MediaAsset asset, Metadata metadata, IngestionContext context) + { + var exif = context.Exif!; + asset.ContentHash = context.ContentHash!; + asset.SetCaptureDate(CaptureDate.Resolve(exif.DateCaptured, fileSystem, context.Item.AbsolutePath)); + asset.MediaType = context.Item.PrimaryMediaType; + asset.LastSeenOnNas = clock.GetCurrentInstant(); + + metadata.Width = exif.Width; + metadata.Height = exif.Height; + metadata.Orientation = exif.Orientation; + metadata.Latitude = exif.Latitude; + metadata.Longitude = exif.Longitude; + metadata.CameraMake = exif.CameraMake; + metadata.CameraModel = exif.CameraModel; + metadata.LensModel = exif.LensModel; + metadata.DurationInSeconds = exif.DurationInSeconds; + metadata.Blurhash = context.Blurhash; + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Debug, Message = "Content of {RelativePath} was persisted concurrently by another consumer, skipping.")] + public static partial void AlreadyPersisted(ILogger logger, string relativePath); + } +} diff --git a/src/Anichron.Worker/Ingestion/Middlewares/ClassificationMiddleware.cs b/src/Anichron.Worker/Ingestion/Middlewares/ClassificationMiddleware.cs index d4408e7..78aa934 100644 --- a/src/Anichron.Worker/Ingestion/Middlewares/ClassificationMiddleware.cs +++ b/src/Anichron.Worker/Ingestion/Middlewares/ClassificationMiddleware.cs @@ -16,6 +16,7 @@ public async Task InvokeAsync(IngestionContext context, IngestionDelegate next, { var classification = await classifier.ClassifyAsync(context, ct); context.Classification = classification; + context.AssetId = classification.Primary.Stored?.Id ?? context.AttemptId; if (classification.NeedsProcessing) { await next(context, ct); diff --git a/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs b/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs index d9f057e..26c448c 100644 --- a/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs +++ b/src/Anichron.Worker/Ingestion/Pipeline/IngestionContext.cs @@ -11,7 +11,9 @@ internal sealed class IngestionContext public required IngestionItem Item { get; init; } public required UserStorageConfig Config { get; init; } - public required Guid AssetId { get; init; } + public required Guid AttemptId { get; init; } + + public Guid AssetId { get; set; } public string? ContentHash { get; set; } public string? SecondaryHash { get; set; } diff --git a/src/Anichron.Worker/Ingestion/Pipeline/IngestionPipelineRunner.cs b/src/Anichron.Worker/Ingestion/Pipeline/IngestionPipelineRunner.cs index f22ec5e..76de390 100644 --- a/src/Anichron.Worker/Ingestion/Pipeline/IngestionPipelineRunner.cs +++ b/src/Anichron.Worker/Ingestion/Pipeline/IngestionPipelineRunner.cs @@ -56,7 +56,7 @@ private void CleanUpDirectory(string proxyRoot, IngestionContext context, string { var directory = fileSystem.Path.Combine( proxyRoot, proxyDirectoryStrategy.GetDirectory(context.Config.Id, contentHash)); - DiscardTemporaryFiles(directory, context.AssetId); + DiscardTemporaryFiles(directory, context.AttemptId); TryRemoveIfEmpty(directory); } diff --git a/src/Anichron.Worker/Ingestion/Proxy/ProxyStagingWriter.cs b/src/Anichron.Worker/Ingestion/Proxy/ProxyStagingWriter.cs index 4a82059..cb41087 100644 --- a/src/Anichron.Worker/Ingestion/Proxy/ProxyStagingWriter.cs +++ b/src/Anichron.Worker/Ingestion/Proxy/ProxyStagingWriter.cs @@ -38,7 +38,7 @@ internal async Task WriteAsync( CancellationToken ct) { var proxyAbsolutePath = fileSystem.Path.Combine(StoragePath.ProxiesRoot, relativePath); - var temporaryAbsolutePath = TemporaryPathFor(proxyAbsolutePath, context.AssetId); + var temporaryAbsolutePath = TemporaryPathFor(proxyAbsolutePath, context.AttemptId); try { if (fileSystem.Path.GetDirectoryName(proxyAbsolutePath) is { } directory) From 6e2fd391eceef4dee8347853e79b938b7e47491f Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Mon, 19 Oct 2026 08:52:54 +1300 Subject: [PATCH 4/4] docs: Name ContentChanged where CLAUDE.md lists the outcomes and writers (#250) Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 44d016b..b12f470 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,7 +153,7 @@ The API reads files only through two narrow stores in `Anichron.Infrastructure/S - Generates proxy types: `Thumbnail`, `FullPreview`, `WebVideo` (the `ProxyType` enum, persisted as text), and a BlurHash string for images, stored in `Metadata.Blurhash` - Image proxies are auto-oriented once after decoding: upright pixels, orientation tag 1. `Metadata.Orientation` applies only when serving originals — never rotate a proxy by it - Image proxy sizes are edge-based, so portrait and landscape get the same treatment: preview long edge `PreviewLongEdge` (1920), thumbnail short edge `ThumbnailShortEdge` (300), blurhash sample long edge `BlurhashSampleLongEdge` (64). Nothing is ever upscaled -- Each `IImageProxyGenerator`, and the `BlurhashEncoder`, declares a `ProxyBound`; `ImageProxyMiddleware` decodes once to `Rgb24`, shrinks the shared image to each bound largest-first, and hands every step **its own copy** at that size. The encoder's string goes on the `IngestionContext` and `NewContentWriter` writes it to `Metadata`. Generators may crop/recolour/resize their copy freely; only the middleware resizes the shared image +- Each `IImageProxyGenerator`, and the `BlurhashEncoder`, declares a `ProxyBound`; `ImageProxyMiddleware` decodes once to `Rgb24`, shrinks the shared image to each bound largest-first, and hands every step **its own copy** at that size. The encoder's string goes on the `IngestionContext` and the content writer (`ProcessedContentWriter.DescribeContent`) writes it to `Metadata`. Generators may crop/recolour/resize their copy freely; only the middleware resizes the shared image - FFmpeg transcodes video with runtime GPU detection: QuickSync (`h264_qsv`) → NVENC (`h264_nvenc`) → AMF (`h264_amf`) → software (`libx264`) - Burst detection (**planned, not yet implemented**): will group rapid-fire sequences and assign a `PrimaryAssetId` cover. The `Burst` entity and schema exist; no detection code is wired into the Worker yet. - Reconciliation (**planned, Epic 4**): a periodic scan soft-deletes assets whose file is gone (interactions survive) and restores them when it reappears (#33, #34, #37). Moves are already handled at ingest (#35): when the hash matches an active row, `KnownFileReconciler` refreshes `LastSeenOnNas` if the path is unchanged (`Unchanged`), moves the row to the new path if its stored file is gone (`Moved`), and leaves it alone if the stored file still exists (`DuplicateCopy`, reported by #36). A Live Photo's MOV row gets the same treatment by its own hash. A move onto a path another row already holds, soft-deleted rows included (`PathTaken`), is logged as a warning and skipped; if a concurrent consumer takes the path after classification, the save fails and the item is retried on the next crawl. A file **edited in place** (hash unknown, but an active row holds the path: `ContentChanged`) is processed like new content and `ContentChangedWriter` updates that row: new hash, capture date, media type, `Metadata` and proxy rows (updated by type), while `Id`, interactions, `BurstId` and `PairedAssetId` are kept. A soft-deleted row at the path still blocks it (restore, #34). An edited Live Photo **MOV** keeps its stored row unchanged for now @@ -183,7 +183,7 @@ Each user has a private set of `UserStorageConfig` entries pointing to NAS roots Processing uses a bounded-concurrency `Channel` pipeline. The crawler produces file paths; N consumer tasks process them concurrently. Default concurrency: `Worker:MaxConcurrentFiles = 4`. Processing is always idempotent — files already in the DB (matched by `(StorageConfigId, ContentHash)`) skip extraction and proxy generation; only the row's `FilePath`/`LastSeenOnNas` are updated, and a file whose bytes changed at a stored path updates that row in place (see Reconciliation above). -**Classify once, then one handler per outcome (#254).** Right after hashing, `ClassificationMiddleware` asks `IngestionClassifier`, which loads every relevant row in one query (`FindByHashesOrPathsAsync`: active rows by either half's hash, any row at either half's path), and gives each half a `FileOutcome` (`New`, `Unchanged`, `Moved`, `DuplicateCopy`, `PathTaken`) plus a `PairingAction` for the item (`None`, `AttachMov`, `BackfillMovProxy`). No other step asks the database whether content is known, except `IngestionSaver` checking whether a concurrent consumer's row landed after a lost insert race. A `New` primary continues through EXIF, proxies and `PersistenceMiddleware`, which hands the write to the `IOutcomeWriter` for that outcome (`NewContentWriter` inserts). Every other outcome stops the item after `LivePhotoPairer` and `KnownFileReconciler` have updated the stored rows. All saves go through `IngestionSaver`, and unique violations are recognised by index name with `IsUniqueViolation` (`Anichron.Core/Data/UniqueViolations.cs`). A new outcome, such as content changed at a known path (#250), is a classifier branch, a `FileOutcome` value and its handler or writer; no other middleware changes. When a Live Photo's HEIC is already stored without its MOV (the halves arrived in different crawls), `LivePhotoPairer` attaches the MOV to the stored row and commits before the halves are reconciled; `PairedMovResolver` reuses the MOV row the classifier found rather than inserting a duplicate. The paired MOV is its own asset and owns its `WebVideo` proxy, written into the MOV's own content-hash directory by `PairedMovProxyWriter` only when that MOV has none yet — on first ingest, on late pairing, and on any later crawl of an already-paired HEIC whose MOV still lacks it (self-heal). +**Classify once, then one handler per outcome (#254).** Right after hashing, `ClassificationMiddleware` asks `IngestionClassifier`, which loads every relevant row in one query (`FindByHashesOrPathsAsync`: active rows by either half's hash, any row at either half's path), and gives each half a `FileOutcome` (`New`, `ContentChanged`, `Unchanged`, `Moved`, `DuplicateCopy`, `PathTaken`) plus a `PairingAction` for the item (`None`, `AttachMov`, `BackfillMovProxy`). No other step asks the database whether content is known, except `IngestionSaver` checking whether a concurrent consumer's row landed after a lost insert race. A `New` or `ContentChanged` primary continues through EXIF, proxies and `PersistenceMiddleware`, which hands the write to the `IOutcomeWriter` for that outcome (`NewContentWriter` inserts, `ContentChangedWriter` updates the stored row in place; both share `ProcessedContentWriter`). Every other outcome stops the item after `LivePhotoPairer` and `KnownFileReconciler` have updated the stored rows. All saves go through `IngestionSaver`, and unique violations are recognised by index name with `IsUniqueViolation` (`Anichron.Core/Data/UniqueViolations.cs`). A new outcome, as content changed at a known path (#250) was, is a classifier branch, a `FileOutcome` value and its handler or writer; no other middleware changes. When a Live Photo's HEIC is already stored without its MOV (the halves arrived in different crawls), `LivePhotoPairer` attaches the MOV to the stored row and commits before the halves are reconciled; `PairedMovResolver` reuses the MOV row the classifier found rather than inserting a duplicate. The paired MOV is its own asset and owns its `WebVideo` proxy, written into the MOV's own content-hash directory by `PairedMovProxyWriter` only when that MOV has none yet — on first ingest, on late pairing, and on any later crawl of an already-paired HEIC whose MOV still lacks it (self-heal). ## Infrastructure