diff --git a/CLAUDE.md b/CLAUDE.md index 79f1963..61b02ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,7 +148,7 @@ The API reads files only through two narrow stores in `Anichron.Infrastructure/S ### Worker Media Processing - Crawls NAS on a configurable interval (`Worker:CrawlIntervalHours`, default 4 h; immediate crawl on startup) -- The crawl walks folder by folder and never enters `CrawlExclusions`: any dot-folder or dot-file (macOS/Linux/QNAP/ZFS/sync-tool metadata), NAS and OS system folders (`@eaDir`, `#recycle`, `@Recycle`, `$RECYCLE.BIN`, `lost+found`, Nextcloud `files_trashbin`/`files_versions`, …) and app library packages (`*.photoslibrary`, `*.lrdata`, …, which hold thumbnails and renders). `Worker:ExcludedDirectories` adds comma-separated folder names (not paths; rejected at startup) on top. An unreadable subfolder is logged and skipped; an unreadable or missing root still fails the crawl +- The crawl walks folder by folder and never enters `CrawlExclusions`: any dot-folder or dot-file (macOS/Linux/QNAP/ZFS/sync-tool metadata), NAS and OS system folders (`@eaDir`, `#recycle`, `@Recycle`, `$RECYCLE.BIN`, `lost+found`, Nextcloud `files_trashbin`/`files_versions`, …) and app library packages (`*.photoslibrary`, `*.lrdata`, …, which hold thumbnails and renders). `Worker:ExcludedDirectories` adds comma-separated folder names (not paths; rejected at startup) on top. An unreadable subfolder is logged and skipped; an unreadable or missing root fails that storage config's crawl, which is logged as an error, and the Worker moves on to the next config (#282) - Computes XXHash64 `ContentHash` → extracts EXIF via MetadataExtractor → writes `MediaAsset` + `Metadata` - Generates proxy types: `Thumbnail`, `FullPreview`, `WebVideo` (the `ProxyType` enum, persisted as text), and a BlurHash string for images, stored in `Metadata.Blurhash` - Image proxies are auto-oriented once after decoding: upright pixels, orientation tag 1. `Metadata.Orientation` applies only when serving originals — never rotate a proxy by it diff --git a/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs b/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs index d678fdf..9f6559c 100644 --- a/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs @@ -6,6 +6,7 @@ using Anichron.Worker.Settings; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using CrawlingWorker = Anichron.Worker.Crawling.Worker; @@ -36,6 +37,8 @@ public TestFixture() .Returns([]); } + public FakeLogger Logger { get; } = new(); + public CrawlingWorker Build(TimeProvider? timeProvider = null) => new( scopeFactory, @@ -43,7 +46,7 @@ public CrawlingWorker Build(TimeProvider? timeProvider = null) Pipeline, timeProvider ?? new FakeTimeProvider(), Options.Create(new WorkerSettings { CrawlIntervalHours = IntervalHours }), - Substitute.For>()); + Logger); } // ========================================================================== @@ -173,4 +176,89 @@ public async Task CrawlAllAsync_ActiveConfig_RunsPipelineWithCorrectConfigAsync( await fixture.Pipeline.Received(1).RunAsync(config, Arg.Any()); } + + // ========================================================================== + // A failing config + // ========================================================================== + + [Fact] + public async Task CrawlAllAsync_ConfigCrawlFails_LogsItsRootAndCrawlsTheNextConfigAsync() + { + var fixture = new TestFixture(); + var userId = Guid.NewGuid(); + var missing = new UserStorageConfig { Id = Guid.NewGuid(), UserId = userId, RootPath = "/data/originals/mira" }; + var next = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/alice" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([missing, next]); + fixture.Pipeline.RunAsync(missing, Arg.Any()) + .Returns(Task.FromException(new DirectoryNotFoundException("gone"))); + var crawlingWorker = fixture.Build(); + + await crawlingWorker.CrawlAllAsync(Ct); + + await fixture.Pipeline.Received(1).RunAsync(next, Arg.Any()); + fixture.Logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Error) + .Which.Should().Match(r => + r.Message == $"Crawl of /data/originals/mira (userId={userId}) failed." + && r.Exception is DirectoryNotFoundException); + } + + [Fact] + public async Task CrawlAllAsync_ShutdownDuringACrawl_StopsWithoutLoggingAFailureAsync() + { + var fixture = new TestFixture(); + var first = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/mira" }; + var second = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/alice" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([first, second]); + using var shutdown = new CancellationTokenSource(); + await shutdown.CancelAsync(); + fixture.Pipeline.RunAsync(first, Arg.Any()).Returns(Task.FromCanceled(shutdown.Token)); + var crawlingWorker = fixture.Build(); + + var act = () => crawlingWorker.CrawlAllAsync(shutdown.Token); + + await act.Should().ThrowAsync(); + await fixture.Pipeline.DidNotReceive().RunAsync(second, Arg.Any()); + fixture.Logger.Collector.GetSnapshot().Should().NotContain(r => r.Level == LogLevel.Error); + } + + [Fact] + public async Task CrawlAllAsync_CancellationWithoutShutdown_IsLoggedAsACrawlFailureAsync() + { + var fixture = new TestFixture(); + var config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/mira" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([config]); + fixture.Pipeline.RunAsync(config, Arg.Any()) + .Returns(Task.FromException(new OperationCanceledException("timed out"))); + var crawlingWorker = fixture.Build(); + + await crawlingWorker.CrawlAllAsync(Ct); + + fixture.Logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Error) + .Which.Exception.Should().BeOfType(); + } + + [Fact] + public async Task ExecuteAsync_CrawlFails_KeepsRunningAndCrawlsAgainAfterTheIntervalAsync() + { + await using var timeline = new Timeline(); + var fixture = new TestFixture(); + var config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/mira" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([config]); + fixture.Pipeline.RunAsync(config, Arg.Any()) + .Returns(_ => + { + timeline.Record("crawl"); + return Task.FromException(new DirectoryNotFoundException("gone")); + }); + var worker = fixture.Build(timeline.Clock); + await worker.StartAsync(Ct); + timeline.Watch(worker.ExecuteTask!, () => worker.StopAsync(CancellationToken.None)); + await timeline.ExpectAsync(new Called("crawl"), new TimerArmed(TimeSpan.FromHours(3))); + + timeline.Clock.Advance(TimeSpan.FromHours(3)); + + await timeline.ExpectAsync( + new TimerFired(TimeSpan.FromHours(3)), new Called("crawl"), new TimerArmed(TimeSpan.FromHours(3))); + worker.ExecuteTask!.IsCompleted.Should().BeFalse(); + } } diff --git a/src/Anichron.Worker/Crawling/Worker.cs b/src/Anichron.Worker/Crawling/Worker.cs index 8f84d7f..6ba7fe5 100644 --- a/src/Anichron.Worker/Crawling/Worker.cs +++ b/src/Anichron.Worker/Crawling/Worker.cs @@ -1,5 +1,6 @@ using Anichron.Core.Data.Repository; using Anichron.Core.Domain; +using Anichron.Worker.Ingestion; using Anichron.Worker.Scheduling; using Anichron.Worker.Settings; using Microsoft.Extensions.Options; @@ -39,10 +40,17 @@ internal async Task CrawlAllAsync(CancellationToken ct) await CrawlAsync(config, ct); } - private Task CrawlAsync(UserStorageConfig config, CancellationToken ct) + private async Task CrawlAsync(UserStorageConfig config, CancellationToken ct) { Log.CrawlingPath(logger, config.RootPath, config.UserId); - return ingestionPipeline.RunAsync(config, ct); + try + { + await ingestionPipeline.RunAsync(config, ct); + } + catch (Exception ex) when (!IngestionShutdown.IsInProgress(ex, ct)) + { + Log.CrawlFailed(logger, config.RootPath, config.UserId, ex); + } } private static partial class Log @@ -52,5 +60,8 @@ private static partial class Log [LoggerMessage(Level = LogLevel.Information, Message = "Crawling {RootPath} (userId={UserId}).")] public static partial void CrawlingPath(ILogger logger, string rootPath, Guid userId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Crawl of {RootPath} (userId={UserId}) failed.")] + public static partial void CrawlFailed(ILogger logger, string rootPath, Guid userId, Exception ex); } } diff --git a/src/Anichron.Worker/Ingestion/IngestionShutdown.cs b/src/Anichron.Worker/Ingestion/IngestionShutdown.cs index f24271e..1ff0fb9 100644 --- a/src/Anichron.Worker/Ingestion/IngestionShutdown.cs +++ b/src/Anichron.Worker/Ingestion/IngestionShutdown.cs @@ -2,10 +2,11 @@ namespace Anichron.Worker.Ingestion; internal static class IngestionShutdown { - // The one definition of "this is a clean shutdown, not an ingestion failure". Both the - // compensation in the pipeline runner and the consumer loop's failure handling branch on it, - // and they must never drift apart: a shutdown keeps the proxies it already completed and stops - // taking new files, where a failure cleans up after itself and the crawl moves on. + // The one definition of "this is a clean shutdown, not an ingestion failure". The compensation + // in the pipeline runner, the consumer loop's failure handling and the crawl loop's per-config + // failure handling all branch on it, and they must never drift apart: a shutdown keeps the + // proxies it already completed and stops taking new files and configs, where a failure cleans + // up after itself and the crawl moves on. internal static bool IsInProgress(Exception exception, CancellationToken ct) => exception is OperationCanceledException && ct.IsCancellationRequested; }