From 8bc50a93ae31a2ce4487041a5e4c81b454570632 Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Sun, 18 Oct 2026 21:18:00 +1300 Subject: [PATCH 1/2] fix(worker): One storage config with a missing root no longer stops the Worker (#282) Worker.CrawlAllAsync now logs a failed crawl of one storage config, naming its root and user, and moves on to the next config. Before, the exception faulted the crawl BackgroundService, the default StopHost shut down the whole Worker, and restart: unless-stopped turned that into a crash loop that also blocked every other user's crawl in all-user mode. A shutdown mid-crawl still propagates (IngestionShutdown.IsInProgress) and is not logged as a failure. FileIngestionPipeline still throws for a missing root; only the per-config loop catches it. The work queue (#281) replaces this loop later. Closes #282. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- .../Crawling/WorkerTests.cs | 89 ++++++++++++++++++- src/Anichron.Worker/Crawling/Worker.cs | 15 +++- 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 79f1963..61b02ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,7 +148,7 @@ The API reads files only through two narrow stores in `Anichron.Infrastructure/S ### Worker Media Processing - Crawls NAS on a configurable interval (`Worker:CrawlIntervalHours`, default 4 h; immediate crawl on startup) -- The crawl walks folder by folder and never enters `CrawlExclusions`: any dot-folder or dot-file (macOS/Linux/QNAP/ZFS/sync-tool metadata), NAS and OS system folders (`@eaDir`, `#recycle`, `@Recycle`, `$RECYCLE.BIN`, `lost+found`, Nextcloud `files_trashbin`/`files_versions`, …) and app library packages (`*.photoslibrary`, `*.lrdata`, …, which hold thumbnails and renders). `Worker:ExcludedDirectories` adds comma-separated folder names (not paths; rejected at startup) on top. An unreadable subfolder is logged and skipped; an unreadable or missing root still fails the crawl +- The crawl walks folder by folder and never enters `CrawlExclusions`: any dot-folder or dot-file (macOS/Linux/QNAP/ZFS/sync-tool metadata), NAS and OS system folders (`@eaDir`, `#recycle`, `@Recycle`, `$RECYCLE.BIN`, `lost+found`, Nextcloud `files_trashbin`/`files_versions`, …) and app library packages (`*.photoslibrary`, `*.lrdata`, …, which hold thumbnails and renders). `Worker:ExcludedDirectories` adds comma-separated folder names (not paths; rejected at startup) on top. An unreadable subfolder is logged and skipped; an unreadable or missing root fails that storage config's crawl, which is logged as an error, and the Worker moves on to the next config (#282) - Computes XXHash64 `ContentHash` → extracts EXIF via MetadataExtractor → writes `MediaAsset` + `Metadata` - Generates proxy types: `Thumbnail`, `FullPreview`, `WebVideo` (the `ProxyType` enum, persisted as text), and a BlurHash string for images, stored in `Metadata.Blurhash` - Image proxies are auto-oriented once after decoding: upright pixels, orientation tag 1. `Metadata.Orientation` applies only when serving originals — never rotate a proxy by it diff --git a/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs b/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs index d678fdf..33b85c4 100644 --- a/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs @@ -6,6 +6,7 @@ using Anichron.Worker.Settings; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using CrawlingWorker = Anichron.Worker.Crawling.Worker; @@ -36,6 +37,8 @@ public TestFixture() .Returns([]); } + public FakeLogger Logger { get; } = new(); + public CrawlingWorker Build(TimeProvider? timeProvider = null) => new( scopeFactory, @@ -43,7 +46,7 @@ public CrawlingWorker Build(TimeProvider? timeProvider = null) Pipeline, timeProvider ?? new FakeTimeProvider(), Options.Create(new WorkerSettings { CrawlIntervalHours = IntervalHours }), - Substitute.For>()); + Logger); } // ========================================================================== @@ -173,4 +176,88 @@ public async Task CrawlAllAsync_ActiveConfig_RunsPipelineWithCorrectConfigAsync( await fixture.Pipeline.Received(1).RunAsync(config, Arg.Any()); } + + // ========================================================================== + // A failing config + // ========================================================================== + + [Fact] + public async Task CrawlAllAsync_ConfigCrawlFails_LogsItsRootAndCrawlsTheNextConfigAsync() + { + var fixture = new TestFixture(); + var userId = Guid.NewGuid(); + var missing = new UserStorageConfig { Id = Guid.NewGuid(), UserId = userId, RootPath = "/data/originals/mira" }; + var next = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/alice" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([missing, next]); + fixture.Pipeline.RunAsync(missing, Arg.Any()) + .Returns(Task.FromException(new DirectoryNotFoundException("gone"))); + var crawlingWorker = fixture.Build(); + + await crawlingWorker.CrawlAllAsync(Ct); + + await fixture.Pipeline.Received(1).RunAsync(next, Arg.Any()); + fixture.Logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Error) + .Which.Should().Match(r => + r.Message == $"Crawl of /data/originals/mira (userId={userId}) failed; crawling the next storage config." + && r.Exception is DirectoryNotFoundException); + } + + [Fact] + public async Task CrawlAllAsync_ShutdownDuringACrawl_StopsWithoutLoggingAFailureAsync() + { + var fixture = new TestFixture(); + var first = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/mira" }; + var second = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/alice" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([first, second]); + using var shutdown = new CancellationTokenSource(); + await shutdown.CancelAsync(); + fixture.Pipeline.RunAsync(first, Arg.Any()).Returns(Task.FromCanceled(shutdown.Token)); + var crawlingWorker = fixture.Build(); + + var act = () => crawlingWorker.CrawlAllAsync(shutdown.Token); + + await act.Should().ThrowAsync(); + await fixture.Pipeline.DidNotReceive().RunAsync(second, Arg.Any()); + fixture.Logger.Collector.GetSnapshot().Should().NotContain(r => r.Level == LogLevel.Error); + } + + [Fact] + public async Task CrawlAllAsync_CancellationWithoutShutdown_IsLoggedAsACrawlFailureAsync() + { + var fixture = new TestFixture(); + var config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/mira" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([config]); + fixture.Pipeline.RunAsync(config, Arg.Any()) + .Returns(Task.FromException(new OperationCanceledException("timed out"))); + var crawlingWorker = fixture.Build(); + + await crawlingWorker.CrawlAllAsync(Ct); + + fixture.Logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Error); + } + + [Fact] + public async Task ExecuteAsync_CrawlFails_KeepsRunningAndCrawlsAgainAfterTheIntervalAsync() + { + await using var timeline = new Timeline(); + var fixture = new TestFixture(); + var config = new UserStorageConfig { Id = Guid.NewGuid(), UserId = Guid.NewGuid(), RootPath = "/data/originals/mira" }; + fixture.ConfigRepository.GetAllActiveAsync(Arg.Any()).Returns([config]); + fixture.Pipeline.RunAsync(config, Arg.Any()) + .Returns(_ => + { + timeline.Record("crawl"); + return Task.FromException(new DirectoryNotFoundException("gone")); + }); + var worker = fixture.Build(timeline.Clock); + await worker.StartAsync(Ct); + timeline.Watch(worker.ExecuteTask!, () => worker.StopAsync(CancellationToken.None)); + await timeline.ExpectAsync(new Called("crawl"), new TimerArmed(TimeSpan.FromHours(3))); + + timeline.Clock.Advance(TimeSpan.FromHours(3)); + + await timeline.ExpectAsync( + new TimerFired(TimeSpan.FromHours(3)), new Called("crawl"), new TimerArmed(TimeSpan.FromHours(3))); + worker.ExecuteTask!.IsCompleted.Should().BeFalse(); + } } diff --git a/src/Anichron.Worker/Crawling/Worker.cs b/src/Anichron.Worker/Crawling/Worker.cs index 8f84d7f..34b4103 100644 --- a/src/Anichron.Worker/Crawling/Worker.cs +++ b/src/Anichron.Worker/Crawling/Worker.cs @@ -1,5 +1,6 @@ using Anichron.Core.Data.Repository; using Anichron.Core.Domain; +using Anichron.Worker.Ingestion; using Anichron.Worker.Scheduling; using Anichron.Worker.Settings; using Microsoft.Extensions.Options; @@ -39,10 +40,17 @@ internal async Task CrawlAllAsync(CancellationToken ct) await CrawlAsync(config, ct); } - private Task CrawlAsync(UserStorageConfig config, CancellationToken ct) + private async Task CrawlAsync(UserStorageConfig config, CancellationToken ct) { Log.CrawlingPath(logger, config.RootPath, config.UserId); - return ingestionPipeline.RunAsync(config, ct); + try + { + await ingestionPipeline.RunAsync(config, ct); + } + catch (Exception ex) when (!IngestionShutdown.IsInProgress(ex, ct)) + { + Log.CrawlFailed(logger, config.RootPath, config.UserId, ex); + } } private static partial class Log @@ -52,5 +60,8 @@ private static partial class Log [LoggerMessage(Level = LogLevel.Information, Message = "Crawling {RootPath} (userId={UserId}).")] public static partial void CrawlingPath(ILogger logger, string rootPath, Guid userId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Crawl of {RootPath} (userId={UserId}) failed; crawling the next storage config.")] + public static partial void CrawlFailed(ILogger logger, string rootPath, Guid userId, Exception ex); } } From 33dd0a893b80ef08e70ff5599e0ceef1cad2727b Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Sun, 18 Oct 2026 21:31:21 +1300 Subject: [PATCH 2/2] refactor(worker): Review follow-ups for #282 The crawl failure message no longer claims a next config exists, the IngestionShutdown comment names its third caller, and the cancellation test checks the logged exception's type. Co-Authored-By: Claude Opus 5.5 --- src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs | 5 +++-- src/Anichron.Worker/Crawling/Worker.cs | 2 +- src/Anichron.Worker/Ingestion/IngestionShutdown.cs | 9 +++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs b/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs index 33b85c4..9f6559c 100644 --- a/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Crawling/WorkerTests.cs @@ -198,7 +198,7 @@ public async Task CrawlAllAsync_ConfigCrawlFails_LogsItsRootAndCrawlsTheNextConf await fixture.Pipeline.Received(1).RunAsync(next, Arg.Any()); fixture.Logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Error) .Which.Should().Match(r => - r.Message == $"Crawl of /data/originals/mira (userId={userId}) failed; crawling the next storage config." + r.Message == $"Crawl of /data/originals/mira (userId={userId}) failed." && r.Exception is DirectoryNotFoundException); } @@ -233,7 +233,8 @@ public async Task CrawlAllAsync_CancellationWithoutShutdown_IsLoggedAsACrawlFail await crawlingWorker.CrawlAllAsync(Ct); - fixture.Logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Error); + fixture.Logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Error) + .Which.Exception.Should().BeOfType(); } [Fact] diff --git a/src/Anichron.Worker/Crawling/Worker.cs b/src/Anichron.Worker/Crawling/Worker.cs index 34b4103..6ba7fe5 100644 --- a/src/Anichron.Worker/Crawling/Worker.cs +++ b/src/Anichron.Worker/Crawling/Worker.cs @@ -61,7 +61,7 @@ private static partial class Log [LoggerMessage(Level = LogLevel.Information, Message = "Crawling {RootPath} (userId={UserId}).")] public static partial void CrawlingPath(ILogger logger, string rootPath, Guid userId); - [LoggerMessage(Level = LogLevel.Error, Message = "Crawl of {RootPath} (userId={UserId}) failed; crawling the next storage config.")] + [LoggerMessage(Level = LogLevel.Error, Message = "Crawl of {RootPath} (userId={UserId}) failed.")] public static partial void CrawlFailed(ILogger logger, string rootPath, Guid userId, Exception ex); } } diff --git a/src/Anichron.Worker/Ingestion/IngestionShutdown.cs b/src/Anichron.Worker/Ingestion/IngestionShutdown.cs index f24271e..1ff0fb9 100644 --- a/src/Anichron.Worker/Ingestion/IngestionShutdown.cs +++ b/src/Anichron.Worker/Ingestion/IngestionShutdown.cs @@ -2,10 +2,11 @@ namespace Anichron.Worker.Ingestion; internal static class IngestionShutdown { - // The one definition of "this is a clean shutdown, not an ingestion failure". Both the - // compensation in the pipeline runner and the consumer loop's failure handling branch on it, - // and they must never drift apart: a shutdown keeps the proxies it already completed and stops - // taking new files, where a failure cleans up after itself and the crawl moves on. + // The one definition of "this is a clean shutdown, not an ingestion failure". The compensation + // in the pipeline runner, the consumer loop's failure handling and the crawl loop's per-config + // failure handling all branch on it, and they must never drift apart: a shutdown keeps the + // proxies it already completed and stops taking new files and configs, where a failure cleans + // up after itself and the crawl moves on. internal static bool IsInProgress(Exception exception, CancellationToken ct) => exception is OperationCanceledException && ct.IsCancellationRequested; }