From c78b540794350347f880d5e42f1707cb21e29dcf Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Fri, 23 Oct 2026 20:52:15 +1300 Subject: [PATCH 1/4] feat(worker): Camera-labelled bursts through a BurstDetection work item (#297) After a crawl, the photos of a storage config that share Apple's BurstUUID form one burst, with the earliest member as the stored cover. - BurstDetection work kind, one item per storage config with dedup key burst-detection:{id}. The config's Crawl enqueues it once the crawl ran; StorageConfigSchedules, now shared with Crawl (decision 3B), schedules it at the crawl interval as a fallback. - BurstChecks table: one row per checked photo (candidate flag, BurstUUID). Detection reads each photo's original once (1A + 2A); the first run checks the whole library. A missing root fails the item before any photo is marked, so an unmounted share cannot mark photos as non-candidates. - Each run repairs the config's bursts first (fewer than two active members: deleted; otherwise the earliest active member becomes cover; memberless bursts deleted), then checks unchecked Image and LivePhoto rows oldest first, in batches, joining or merging bursts by label. - Bursts.PrimaryAssetId is nullable with ON DELETE SET NULL instead of RESTRICT, so deleting a storage config or a cover no longer fails; the Burst query filter keeps coverless bursts visible. - ContentChangedWriter clears the burst mark of a file edited in place. - MetadataExtractor 2.9.3 yields an AppleMakernoteDirectory from the maintainer's sample HEICs (no BurstUUID in them), confirming the label read before building on it. - ExifDate.Parse is shared by the EXIF middleware and the reader; it relies on NodaTime returning a failed result for null text. - GLOSSARY.md with Burst and Cover, from docs/26-burst-design. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 21 +- GLOSSARY.md | 13 + docs/schema.sql | 26 +- .../Data/ModelInvariantsTests.cs | 16 +- .../Repository/MediaAssetRepositoryTests.cs | 28 + .../Data/SoftDeleteQueryFilterTests.cs | 11 +- .../Work/PayloadsTests.cs | 23 + src/Anichron.Core/Data/AnichronDbContext.cs | 41 +- .../Data/Repository/MediaAssetRepository.cs | 11 + src/Anichron.Core/Domain/Burst.cs | 9 +- src/Anichron.Core/Domain/BurstCheck.cs | 18 + src/Anichron.Core/Domain/WorkKind.cs | 1 + .../20261023073023_BurstDetection.Designer.cs | 733 ++++++++++++++++++ .../20261023073023_BurstDetection.cs | 105 +++ .../AnichronDbContextModelSnapshot.cs | 46 +- src/Anichron.Core/Work/IWorkPayload.cs | 8 + src/Anichron.Core/Work/Payloads.cs | 15 +- .../Data/BurstDeleteBehaviourTests.cs | 68 ++ .../Bursts/BurstDetectionTests.cs | 579 ++++++++++++++ .../Bursts/OriginalPhotoReaderTests.cs | 90 +++ .../Crawling/CrawlTests.cs | 67 +- .../WorkerServiceCollectionExtensionsTests.cs | 5 +- .../ContentChangedWriterTests.cs | 12 +- .../Work/StorageConfigSchedulesTests.cs | 90 +++ .../Work/WorkExecutorTests.cs | 12 +- .../Work/WorkHandlerRegistryTests.cs | 17 +- src/Anichron.Worker/Bursts/BurstDetection.cs | 167 ++++ .../Bursts/OriginalPhotoReader.cs | 56 ++ src/Anichron.Worker/Crawling/Crawl.cs | 25 +- .../WorkerServiceCollectionExtensions.cs | 6 +- .../Classification/ContentChangedWriter.cs | 2 + src/Anichron.Worker/Ingestion/ExifDate.cs | 16 + .../Middlewares/ExifExtractionMiddleware.cs | 10 +- .../Work/StorageConfigSchedules.cs | 31 + 34 files changed, 2245 insertions(+), 133 deletions(-) create mode 100644 GLOSSARY.md create mode 100644 src/Anichron.Core/Domain/BurstCheck.cs create mode 100644 src/Anichron.Core/Migrations/20261023073023_BurstDetection.Designer.cs create mode 100644 src/Anichron.Core/Migrations/20261023073023_BurstDetection.cs create mode 100644 src/Anichron.Tests.Integration/Data/BurstDeleteBehaviourTests.cs create mode 100644 src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs create mode 100644 src/Anichron.Worker.Tests.Unit/Bursts/OriginalPhotoReaderTests.cs create mode 100644 src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs create mode 100644 src/Anichron.Worker/Bursts/BurstDetection.cs create mode 100644 src/Anichron.Worker/Bursts/OriginalPhotoReader.cs create mode 100644 src/Anichron.Worker/Ingestion/ExifDate.cs create mode 100644 src/Anichron.Worker/Work/StorageConfigSchedules.cs diff --git a/CLAUDE.md b/CLAUDE.md index cac88118..f67eabb4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,10 +97,10 @@ Four projects with strict separation of concerns: | `Anichron.Core` | Single source of truth: domain models, `AnichronDbContext`, all Fluent API config, shared utilities (XXHash64, path parsing). Zero dependencies on other projects (every other project references Core, so a reference back would be circular and fail the build). | | `Anichron.Infrastructure` | Shared host plumbing used by API and Worker: configuration (`DatabaseConfiguration` reads the connection from `POSTGRES_CONNECTION__*` env vars or Docker secrets; INI bootstrap; `ConfigurationList`), migration under a PostgreSQL advisory lock, and the API's read-only stores (`IProxyStore`, `IOriginalStore`). Each host wires its own DI. | | `Anichron.API` | ASP.NET Core Minimal APIs (the only web project: `libraries-are-not-web` keeps ASP.NET Core out of Core, Infrastructure and Worker, and the two hosts never use each other) — owns all identity/auth/admin writes (users, storage configs, refresh tokens, invites, lockout); read-only for media data. Media endpoints so far: proxy serving (#43), On This Day (`GET /api/v1/flashbacks/on-this-day`, #38) and the caller's star/like/hide state (`PATCH /api/v1/assets/{id}/interaction`, #46); originals (#44) are deferred past v1.0. All routes prefixed `/api/v1/`. | -| `Anichron.Worker` | `BackgroundService`s — NAS crawling, EXIF extraction, FFmpeg transcoding, proxy generation, and the work queue's executor and scheduler (maintenance such as expired-refresh-token cleanup runs as work items); reconciliation is planned (Epic 4) | +| `Anichron.Worker` | `BackgroundService`s — NAS crawling, EXIF extraction, FFmpeg transcoding, proxy generation, burst detection, and the work queue's executor and scheduler (maintenance such as expired-refresh-token cleanup runs as work items); reconciliation is planned (Epic 4) | **Write ownership is split by domain, not centralized.** -- **Worker is the only writer of *media* data** — `MediaAsset`, `Metadata`, `ProxyFile`, `Burst`, plus all reconciliation and soft-deletes happen here. The API never writes media: outside its DI wiring it never touches `AnichronDbContext` (architecture rule `api-reaches-data-through-repositories`), and the media write repository stays out of it (`only-worker-writes-media`). Its one identity-side write is the `TokenCleanup` work item's handler deleting expired refresh tokens. +- **Worker is the only writer of *media* data** — `MediaAsset`, `Metadata`, `ProxyFile`, `Burst`, `BurstCheck`, plus all reconciliation and soft-deletes happen here. The API never writes media: outside its DI wiring it never touches `AnichronDbContext` (architecture rule `api-reaches-data-through-repositories`), and the media write repository stays out of it (`only-worker-writes-media`). Its one identity-side write is the `TokenCleanup` work item's handler deleting expired refresh tokens. - **API owns all *identity/auth/admin* writes** — registration, login/refresh-token issuance and revocation, lockout state, invites, and admin user/storage-config CRUD all persist through API services (`AuthService`, `AdminUserService`, `AdminStorageConfigService`, `TokenService`, etc.) via `IUnitOfWork.SaveChangesAsync`. The API is read-only *only with respect to media*. The one exception is `AssetInteraction` (star/like/hide, #46): per-user state, written by the API through `IAssetInteractionWriter` (Core), which takes and returns plain records so no API type touches the entity. It checks only `VisibleTo` (never `NotHiddenFor`, so a hidden asset can be unhidden), and a first write that loses the `(UserId, AssetId)` unique race reloads and reapplies once. - **Work items** are inserted only through `IWorkQueue` (architecture rule `only-work-queue-inserts-work-items`); the API's only other write is an admin's cancel through `IWorkItemCanceller` (`work-items-change-only-through-queue-or-cancel`: outside the Worker, only `EfWorkQueue`, `EfWorkItemReader` and `EfWorkItemCanceller` depend on the `WorkItem` entity; raw SQL naming the table is beyond the rule and is for review); every other state change belongs to the Worker's executor, and handlers live only in the Worker (`work-handlers-live-in-worker`). See "Work queue" below. - Both hosts apply EF migrations at startup, serialised by a PostgreSQL advisory lock (`MigrateWithAdvisoryLockAsync`). @@ -120,7 +120,7 @@ into PostgreSQL and therefore into `docs/schema.sql` and any schema-doc tool poi database. If you find yourself about to explain *why* a column exists in a Markdown file, put it in a `HasComment` instead — see `docs/adr/0004-schema-documentation.md`. -**Ten entities**, all EF Core config via **Fluent API only** (architecture rule `domain-has-no-data-annotations`). Columns are +**Eleven entities**, all EF Core config via **Fluent API only** (architecture rule `domain-has-no-data-annotations`). Columns are **PascalCase** — the only explicitly-named column in the whole model is `Invites.xmin`. Table names do not always match entity names (`AssetInteraction` → `Interactions`, `UserStorageConfig` → `StorageConfigs`): @@ -128,7 +128,8 @@ do not always match entity names (`AssetInteraction` → `Interactions`, `UserSt - `MediaAsset` — central entity: `FilePath`, `ContentHash` (XXHash64), `Month`/`Day`/`Year` integers (set only by the `DateCaptured` setter), `MediaType`, `IsSoftDeleted`, and a `PairedAssetId` self-reference (HEIC parent → MOV child) - `Metadata` — 1:1 with `MediaAsset`; EXIF data (dimensions, GPS, raw EXIF orientation tag 1–8, camera make/model/lens, video duration) and the image's `Blurhash` string (a column, not a proxy file) - `ProxyFile` — one row per generated artefact, each a file at `ProxyPath` on the local SSD. `ProxyType` is `Thumbnail`, `WebVideo` or `FullPreview` -- `Burst` — groups rapid-fire photos with a `PrimaryAssetId` cover; prevents duplicate memories in flashbacks +- `Burst` — groups photos of one storage config taken moments apart, so a flashback shows one memory; `PrimaryAssetId` is the stored cover (the earliest member), nullable with `SET NULL` so deleting it never fails +- `BurstCheck` — one row per photo burst detection has checked (no row: unchecked), with whether it is a candidate and Apple's `BurstUUID` (`CameraBurstId`); cascades with its asset - `AssetInteraction` — per-user state (starred, liked, hidden, view count, last viewed); unique on `(UserId, AssetId)` - `UserStorageConfig` — NAS root paths per user; `RootPath` is **globally unique** — a path cannot be assigned to more than one storage config across all users. Paths are stored canonical (absolute, `.`/`..`/`//` collapsed, no trailing slash) via `StoragePath.Canonicalize`, so `/p` and `/p/` cannot both be assigned. A `RootPath` must be a folder strictly inside `StoragePath.OriginalsRoot` (`/data/originals`), never the root itself, and may not contain or be contained by another config's root (`StoragePath.Overlaps`) — nested roots would index the same files twice - `User` — owns storage configs and interactions @@ -143,7 +144,7 @@ do not always match entity names (`AssetInteraction` → `Interactions`, `UserSt - Unique index on `(StorageConfigId, FilePath)` — one asset row per path within a config, soft-deleted rows included - Unique index on `ProxyFiles (AssetId, ProxyType)` (`ProxyFileIndexNames.OnePerAssetAndType`) — at most one proxy of each kind per asset, so two attempts racing to add the same proxy (e.g. a paired MOV's backfilled `WebVideo`) cannot both land; the loser is a `LostIngestionRace` - Filtered index `IX_MediaAssets_Active` on `IsSoftDeleted`, filtered to `"IsSoftDeleted" = false` — excludes trashed assets from active gallery queries -- Global query filters hide soft-deleted assets: on `MediaAsset`, and on `Metadata`, `ProxyFile`, `AssetInteraction` and `Burst` through their asset. This is the **only** global filter (pinned by `ModelInvariantsTests`) — per-user rules (hidden, ownership) are query-level, because the Worker shares `AnichronDbContext` and has no user +- Global query filters hide soft-deleted assets: on `MediaAsset`, and on `Metadata`, `ProxyFile`, `AssetInteraction`, `BurstCheck` and `Burst` through their asset (a burst whose cover is null stays visible). This is the **only** global filter (pinned by `ModelInvariantsTests`) — per-user rules (hidden, ownership) are query-level, because the Worker shares `AnichronDbContext` and has no user - `User.Username` and `User.Email` carry an EF value converter (`IdentityNormalizer.Normalize`: trim + lowercase) that applies to every write **and** to every equality comparison, so repository lookups need no manual normalization. ⚠️ The InMemory provider does not convert query parameters, so read-side behaviour is tested via `ToQueryString()` on the Npgsql provider, not InMemory - **NodaTime for every date and time.** `System.IO` hands back `DateTime` (e.g. a file's last-write time); convert it at that boundary. Domain types never hold one (architecture rule `domain-uses-nodatime`) @@ -169,8 +170,8 @@ The API reads files only through two narrow stores in `Anichron.Infrastructure/S - Image proxy sizes are edge-based, so portrait and landscape get the same treatment: preview long edge `PreviewLongEdge` (1920), thumbnail short edge `ThumbnailShortEdge` (300), blurhash sample long edge `BlurhashSampleLongEdge` (64). Nothing is ever upscaled - Each `IImageProxyGenerator`, and the `BlurhashEncoder`, declares a `ProxyBound`; `ImageProxyMiddleware` decodes once to `Rgb24`, shrinks the shared image to each bound largest-first, and hands every step **its own copy** at that size. The encoder's string goes on the `IngestionContext` and the content writer (`ProcessedContentWriter.DescribeContent`) writes it to `Metadata`. Generators may crop/recolour/resize their copy freely; only the middleware resizes the shared image - FFmpeg transcodes video with runtime GPU detection: QuickSync (`h264_qsv`) → NVENC (`h264_nvenc`) → AMF (`h264_amf`) → software (`libx264`) -- Burst detection (**planned, not yet implemented**): will group rapid-fire sequences and assign a `PrimaryAssetId` cover. The `Burst` entity and schema exist; no detection code is wired into the Worker yet. -- Reconciliation (**planned, Epic 4**): a periodic scan soft-deletes assets whose file is gone (interactions survive) and restores them when it reappears (#33, #34, #37). Moves are already handled at ingest (#35): when the hash matches an active row, `KnownFileReconciler` refreshes `LastSeenOnNas` if the path is unchanged (`Unchanged`), moves the row to the new path if its stored file is gone (`Moved`), and leaves it alone if the stored file still exists (`DuplicateCopy`, reported by #36). A Live Photo's MOV row gets the same treatment by its own hash. A move onto a path another row already holds, soft-deleted rows included (`PathTaken`), is logged as a warning and skipped; if a concurrent consumer takes the path after classification, the save fails and the item is retried on the next crawl. A file **edited in place** (hash unknown, but an active row holds the path: `ContentChanged`) is processed like new content and `ContentChangedWriter` updates that row: new hash, capture date, media type, `Metadata` and proxy rows (updated by type), while `Id`, interactions, `BurstId` and `PairedAssetId` are kept. A soft-deleted row at the path still blocks it (restore, #34). An edited Live Photo **MOV** keeps its stored row unchanged for now +- Burst detection (#297): a `BurstDetection` work item per storage config, enqueued by the config's `Crawl` when it finishes and scheduled at the crawl interval as a fallback (one dedup key, so never twice). `BurstDetectionHandler` first repairs the config's bursts (fewer than 2 active members: deleted; otherwise the earliest active member becomes the cover; memberless bursts are deleted too), then checks each unchecked photo (`Image` or `LivePhoto`, never `Video`, so a paired MOV never joins) oldest first, in batches: it reads the original once through `IOriginalPhotoReader` (EXIF capture time, Apple `BurstUUID`) and writes its `BurstCheck` row. A candidate (EXIF capture time, camera make and model, a thumbnail) joins every checked candidate of the config with the same `BurstUUID`, regardless of time, creating or merging bursts; each photo saves with its burst changes. A missing root fails the item before anything is marked. Similarity grouping is #298 +- Reconciliation (**planned, Epic 4**): a periodic scan soft-deletes assets whose file is gone (interactions survive) and restores them when it reappears (#33, #34, #37). Moves are already handled at ingest (#35): when the hash matches an active row, `KnownFileReconciler` refreshes `LastSeenOnNas` if the path is unchanged (`Unchanged`), moves the row to the new path if its stored file is gone (`Moved`), and leaves it alone if the stored file still exists (`DuplicateCopy`, reported by #36). A Live Photo's MOV row gets the same treatment by its own hash. A move onto a path another row already holds, soft-deleted rows included (`PathTaken`), is logged as a warning and skipped; if a concurrent consumer takes the path after classification, the save fails and the item is retried on the next crawl. A file **edited in place** (hash unknown, but an active row holds the path: `ContentChanged`) is processed like new content and `ContentChangedWriter` updates that row: new hash, capture date, media type, `Metadata` and proxy rows (updated by type), while `Id`, interactions and `PairedAssetId` are kept. Its burst mark is cleared (`BurstId` null, `BurstCheck` row deleted), so the next burst detection checks it again; a moved file keeps both. A soft-deleted row at the path still blocks it (restore, #34). An edited Live Photo **MOV** keeps its stored row unchanged for now - Duplicate content is **reported, never merged** (#36). `SharedContentReporter` runs at the end of each config's `Crawl` item: it logs one `Information` summary for the crawled config (copies within it, counted by `DuplicateCopyTally` during that crawl in this Worker, and pairs of its active rows sharing a `ContentHash` with the same user's other active configs), with each pair at `Debug`, and the counts go into the crawl's result. A pair across two configs is reported by each of their crawls. A failed report is logged and does not fail the crawl. Duplicates across different users are expected and never reported. Two new identical copies racing in one crawl are first reported on the next crawl - Failure ownership: `ProxyStagingWriter` is the single place a proxy reaches its final name (architecture rule `proxies-reach-final-name-only-in-staging-writer`) — it writes under a per-attempt `..tmp` suffix (so concurrent attempts on the same content never share a temporary file), renames into place once complete, and registers the proxy on the `IngestionContext` at that moment; the proxy middlewares only supply the bytes. If a step throws, `IngestionPipelineRunner` deletes exactly those registered proxies plus its own temporary files, and removes the directory only if it is left empty — never recursively, since a concurrent attempt on the same content shares it until one of them persists. An attempt that loses a uniqueness race (`LostIngestionRace`: the active content-hash index or the one-proxy-per-asset-and-type index) first releases its paired MOV's registrations, because the winner owns those content-derived files. A clean shutdown is treated differently: the in-flight temporary file is dropped, but proxies already completed in that run are kept, and the consumer loop re-throws the cancellation instead of recording an item failure and draining the queue. Each attempt has its own `IngestionContext.AttemptId`, which names temporary files and scopes compensation; `IngestionContext.AssetId` is the row that owns the proxies, set by classification (the stored row for `ContentChanged`, the attempt id for new content). A hard kill, the storage-config cascade delete, and the old content directory of a file edited in place are out of reach in-process and belong to the orphan sweeper (#174); the old directory is deliberately not deleted after commit, because a copy of the old content elsewhere can be ingested into it in the same crawl. @@ -196,15 +197,15 @@ Each user has a private set of `UserStorageConfig` entries pointing to NAS roots ### Work queue -Durable work for the Worker lives in `WorkItems` (#276, ADR 0008). Its kinds are the crawl (`Crawl`, one per storage config) and periodic maintenance (`TokenCleanup`, `WorkItemRetention`); admin requests (#174) follow. +Durable work for the Worker lives in `WorkItems` (#276, ADR 0008). Its kinds are the crawl (`Crawl`) and burst detection (`BurstDetection`), each one per storage config, and periodic maintenance (`TokenCleanup`, `WorkItemRetention`); admin requests (#174) follow. - **Producing:** `IWorkQueue.EnqueueAsync` (`EfWorkQueue`, Core) is the only insert. A payload with a `DedupKey` coalesces: while an item with that key is `Queued`, enqueueing returns it (a partial unique index absorbs racing inserts). - **Scope:** a scoped item carries the `StorageConfigId` it belongs to (`IWorkPayload.StorageConfigId`). A per-user Worker (`WORKER__USER`) claims unscoped items and items of its user's configs; an all-user Worker claims everything. The served user is `WorkerState.ResolvedUserId`, read at each claim and tick, never cached at construction. -- **Scheduling:** every Worker runs `WorkScheduler`. Each poll (`Worker:WorkPollIntervalSeconds`, 5 s) it recovers expired leases, then asks each `IWorkScheduleSource` for its schedules (a fixed `WorkSchedule` is its own source; `CrawlSchedules` returns one per active config the Worker serves) and whether each is due: nothing queued or running with its key and nothing finished within its interval. No timer state is kept, so overdue work runs right after downtime. +- **Scheduling:** every Worker runs `WorkScheduler`. Each poll (`Worker:WorkPollIntervalSeconds`, 5 s) it recovers expired leases, then asks each `IWorkScheduleSource` for its schedules (a fixed `WorkSchedule` is its own source; `StorageConfigSchedules` returns one per active config the Worker serves, at the crawl interval, for any `IStorageConfigPayload`) and whether each is due: nothing queued or running with its key and nothing finished within its interval. No timer state is kept, so overdue work runs right after downtime. - **Executing:** `WorkExecutor` claims the oldest queued item (`FOR UPDATE SKIP LOCKED`), one at a time, under a lease (`WorkLeaseSeconds`, 60 s) that a heartbeat (`WorkHeartbeatSeconds`, 2 s) extends while writing the progress snapshot. `UpdatedAt` is the change cursor, so a heartbeat moves it only when the snapshot changed; `LeaseUntil` is what says the Worker is alive. Every write about a running item is fenced by `(Id, Attempt)`. A fenced heartbeat cancels the handler and writes nothing more; a shutdown releases the item without a stall; a handler exception fails it with no retry. An expired lease requeues the item and counts a stall; the `WorkMaxStalls`th (3) fails it as stalled. An item handed back (release or recovery) while another with its key is already queued ends `Cancelled` as superseded, since the queued one does the same work. A Worker claims only the kinds it knows, so a newer kind waits for a newer Worker. `WorkItemRetention` deletes items finished more than `WorkItemRetentionDays` (30) ago. - **Cancelling:** `POST /api/v1/admin/work-items/{id}/cancel` (#280) goes through `EfWorkItemCanceller`, one status-guarded `UPDATE`: a queued item ends `Cancelled` at once (`200`), a running one gets `CancelRequestedAt` (`202`), a finished one is `409` and an unknown id `404`. The heartbeat reads the request and cancels the handler but keeps beating until it returns; the item then ends `Cancelled` with its last progress snapshot (and a summary, if the handler returned one), shown as partial (`ResultIsPartial`). A handler that fails after the request still fails. A requested item is never queued again: release and lease recovery end it `Cancelled`, even at the stall limit. A cancelled periodic item counts as that run for due-ness, so the next one follows a full interval later; were it not counted, the scheduler would enqueue it again at the next poll and undo the cancel. - **Admin view (#279):** `GET /api/v1/admin/work-items` (filtered by status and kind, newest request first, paged), `.../{id}`, and `.../events`, a Server-Sent Events stream that re-queries `UpdatedAt` every `WorkItemEvents:IntervalSeconds` (2 s). An event's id is the cursor, and the list carries the cursor read before its page, so a client that lists and then subscribes with it as `Last-Event-ID` misses nothing; an item may arrive twice and is merged by id. The API reads items only through `IWorkItemReader` (Core), as `WorkItemView`. A result is shown once an item succeeded or was cancelled (then marked partial), an error once it failed or was cancelled. -- **Adding a kind:** a `WorkKind` value, a payload record in Core implementing `IWorkPayload` (static `Kind`, optional `DedupKey`), an `IWorkHandler` in the Worker registered with `AddWorkHandler`, and for a periodic kind a `WorkSchedule` subclass registered as an `IWorkScheduleSource` (or a source that builds several). `WorkHandlerRegistry` fails startup unless every kind has exactly one payload and one handler; nothing switches on the kind. +- **Adding a kind:** a `WorkKind` value, a payload record in Core implementing `IWorkPayload` (static `Kind`, optional `DedupKey`), an `IWorkHandler` in the Worker registered with `AddWorkHandler`, and for a periodic kind a `WorkSchedule` subclass registered as an `IWorkScheduleSource` (or a source that builds several; a per-config kind implements `IStorageConfigPayload` and registers `StorageConfigSchedules`). `WorkHandlerRegistry` fails startup unless every kind has exactly one payload and one handler; nothing switches on the kind. - **Handler contract:** delivery is at least once. Each run starts from scratch and converges, side effects are safe to repeat, and the returned summary becomes `Result`. ### Worker Processing Model diff --git a/GLOSSARY.md b/GLOSSARY.md new file mode 100644 index 00000000..7e31d12f --- /dev/null +++ b/GLOSSARY.md @@ -0,0 +1,13 @@ +# Anichron + +A self-hosted photo library that resurfaces a user's own photos and videos from their NAS as daily flashbacks. + +## Language + +**Burst**: +Photos of the same scene from one storage config, taken moments apart, shown as one memory. Either the camera marked them as one burst, or each one looks like a neighbouring frame taken shortly before or after it. +_Avoid_: Moment, similar group, near-duplicates + +**Cover**: +The member of a burst that represents it wherever the burst is shown as one memory: the earliest member the viewing user has not hidden. +_Avoid_: Primary photo, key photo diff --git a/docs/schema.sql b/docs/schema.sql index 1171623b..b99f7353 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -102,14 +102,26 @@ COMMENT ON COLUMN "WorkItems"."ProgressNote" IS 'The latest progress snapshot (d COMMENT ON COLUMN "WorkItems"."Result" IS 'The handler''s results summary, written when the item succeeds, or when it is cancelled after its handler returned one; a Cancelled item''s results are partial.'; +CREATE TABLE "BurstChecks" ( + "AssetId" uuid NOT NULL, + "CheckedAt" timestamp with time zone NOT NULL, + "IsCandidate" boolean NOT NULL, + "CameraBurstId" text, + CONSTRAINT "PK_BurstChecks" PRIMARY KEY ("AssetId") +); +COMMENT ON TABLE "BurstChecks" IS 'One row per photo burst detection has checked; no row means unchecked. Owned by the BurstDetection work item, which reads the original once when it checks a photo. A file edited in place loses its row, so it is checked again.'; +COMMENT ON COLUMN "BurstChecks"."IsCandidate" IS 'The photo has an EXIF capture time, a camera make and model, and a thumbnail; only candidates are grouped. False also when the original could not be read.'; +COMMENT ON COLUMN "BurstChecks"."CameraBurstId" IS 'Apple''s BurstUUID from the original''s maker note: photos of one storage config sharing it form one burst. Null when the camera wrote none.'; + + CREATE TABLE "Bursts" ( "Id" uuid NOT NULL, - "PrimaryAssetId" uuid NOT NULL, + "PrimaryAssetId" uuid, "CreatedAt" timestamp with time zone NOT NULL, CONSTRAINT "PK_Bursts" PRIMARY KEY ("Id") ); -COMMENT ON TABLE "Bursts" IS 'Groups a rapid-fire sequence so a flashback shows one memory rather than nine near-identical frames. The table and schema exist; burst DETECTION is not yet implemented in the Worker, so in a running system this table is empty.'; -COMMENT ON COLUMN "Bursts"."PrimaryAssetId" IS 'The frame chosen to represent the burst. Restrict on delete: losing the cover would leave a group that cannot be rendered.'; +COMMENT ON TABLE "Bursts" IS 'Groups photos of one storage config taken moments apart, so a flashback shows one memory rather than nine near-identical frames. Written only by the Worker''s BurstDetection work item, which also repairs bursts: one left with fewer than two active members is deleted, and so is one with no members at all.'; +COMMENT ON COLUMN "Bursts"."PrimaryAssetId" IS 'The stored cover, the earliest member. SetNull on delete, so deleting the cover''s row (a storage config removal cascading to its assets) cannot fail on the burst; the next BurstDetection run picks a new cover or deletes the burst.'; CREATE TABLE "MediaAssets" ( @@ -192,6 +204,9 @@ COMMENT ON COLUMN "ProxyFiles"."ProxyPath" IS 'Path relative to the proxy root, COMMENT ON COLUMN "ProxyFiles"."ProxyType" IS 'Thumbnail, WebVideo or FullPreview, persisted as text. The BlurHash placeholder is not a proxy: it is the Metadata.Blurhash column.'; +CREATE INDEX "IX_BurstChecks_CameraBurstId" ON "BurstChecks" ("CameraBurstId"); + + CREATE INDEX "IX_Bursts_PrimaryAssetId" ON "Bursts" ("PrimaryAssetId"); @@ -267,4 +282,7 @@ CREATE INDEX "IX_WorkItems_StorageConfigId" ON "WorkItems" ("StorageConfigId"); CREATE INDEX "IX_WorkItems_UpdatedAt" ON "WorkItems" ("UpdatedAt"); -ALTER TABLE "Bursts" ADD CONSTRAINT "FK_Bursts_MediaAssets_PrimaryAssetId" FOREIGN KEY ("PrimaryAssetId") REFERENCES "MediaAssets" ("Id") ON DELETE RESTRICT; +ALTER TABLE "BurstChecks" ADD CONSTRAINT "FK_BurstChecks_MediaAssets_AssetId" FOREIGN KEY ("AssetId") REFERENCES "MediaAssets" ("Id") ON DELETE CASCADE; + + +ALTER TABLE "Bursts" ADD CONSTRAINT "FK_Bursts_MediaAssets_PrimaryAssetId" FOREIGN KEY ("PrimaryAssetId") REFERENCES "MediaAssets" ("Id") ON DELETE SET NULL; diff --git a/src/Anichron.Core.Tests.Unit/Data/ModelInvariantsTests.cs b/src/Anichron.Core.Tests.Unit/Data/ModelInvariantsTests.cs index 460d4e63..9b88c37e 100644 --- a/src/Anichron.Core.Tests.Unit/Data/ModelInvariantsTests.cs +++ b/src/Anichron.Core.Tests.Unit/Data/ModelInvariantsTests.cs @@ -46,10 +46,24 @@ public void QueryFilters_AreSoftDeleteOnly() .ToList(); filters.Select(filter => filter.Entity).Should().BeEquivalentTo( - [nameof(MediaAsset), nameof(Metadata), nameof(ProxyFile), nameof(AssetInteraction), nameof(Burst)], + [nameof(MediaAsset), nameof(Metadata), nameof(ProxyFile), nameof(AssetInteraction), nameof(Burst), nameof(BurstCheck)], "soft-delete is the only global filter: on the asset and on what hangs off it, one filter each"); filters.Should().AllSatisfy(filter => filter.Expression.Should().Contain( nameof(MediaAsset.IsSoftDeleted), "per-user rules (hidden, ownership) stay query-level, because the Worker shares the context and has no user")); } + + [Fact] + public void BurstCover_IsSetToNullWhenItsAssetIsDeleted() + { + using var context = AnichronDbContexts.NpgsqlModel(); + var burst = context.Model.FindEntityType(typeof(Burst))!; + + var cover = burst.GetForeignKeys().Should().ContainSingle( + "a burst points at one asset, its cover; the members point at the burst").Which; + cover.Properties.Select(p => p.Name).Should().Equal(nameof(Burst.PrimaryAssetId)); + cover.IsRequired.Should().BeFalse(); + cover.DeleteBehavior.Should().Be(DeleteBehavior.SetNull, + "a storage config's removal cascades to its assets and must not fail on a burst's cover"); + } } diff --git a/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs b/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs index d36d10da..cb05a8d8 100644 --- a/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs +++ b/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs @@ -178,6 +178,34 @@ public async Task LoadMetadataAsync_LoadsTheAssetsMetadataRow() tracked.Metadata.Width.Should().Be(640); } + [Fact] + public async Task RemoveBurstCheckAsync_CheckedPhoto_TracksItsRowForDeleteAsync() + { + var ct = TestContext.Current.CancellationToken; + var asset = AssetAt(ConfigA, "cafe", "a.jpg"); + await using var seed = CreateDb(out var name); + seed.MediaAssets.Add(asset); + seed.BurstChecks.Add(new BurstCheck { AssetId = asset.Id, IsCandidate = true }); + await seed.SaveChangesAsync(ct); + await using var db = AnichronDbContexts.InMemory(name); + + await new EfMediaAssetRepository(db).RemoveBurstCheckAsync(asset.Id, ct); + await db.SaveChangesAsync(ct); + + (await db.BurstChecks.CountAsync(ct)).Should().Be(0); + } + + [Fact] + public async Task RemoveBurstCheckAsync_UncheckedPhoto_ChangesNothingAsync() + { + var ct = TestContext.Current.CancellationToken; + await using var db = CreateDb(out _); + + await new EfMediaAssetRepository(db).RemoveBurstCheckAsync(Guid.NewGuid(), ct); + + db.ChangeTracker.HasChanges().Should().BeFalse(); + } + // ========================================================================== // FindSharedContentAsync // ========================================================================== diff --git a/src/Anichron.Core.Tests.Unit/Data/SoftDeleteQueryFilterTests.cs b/src/Anichron.Core.Tests.Unit/Data/SoftDeleteQueryFilterTests.cs index 4850e7a9..fd27358e 100644 --- a/src/Anichron.Core.Tests.Unit/Data/SoftDeleteQueryFilterTests.cs +++ b/src/Anichron.Core.Tests.Unit/Data/SoftDeleteQueryFilterTests.cs @@ -11,6 +11,7 @@ public sealed class SoftDeleteQueryFilterTests : IAsyncLifetime private readonly MediaAsset active = Asset("active.jpg", isSoftDeleted: false); private readonly MediaAsset deleted = Asset("deleted.jpg", isSoftDeleted: true); + private readonly Guid coverless = Guid.NewGuid(); private readonly User user = new() { Id = Guid.NewGuid(), Username = "alice", Email = "a@example.com", PasswordHash = "h" }; private static MediaAsset Asset(string name, bool isSoftDeleted) @@ -36,7 +37,10 @@ private void Seed() db.ProxyFiles.Add(new ProxyFile { Id = Guid.NewGuid(), AssetId = asset.Id, ProxyPath = asset.FileName }); db.Interactions.Add(new AssetInteraction { Id = Guid.NewGuid(), UserId = user.Id, AssetId = asset.Id }); db.Bursts.Add(new Burst { Id = Guid.NewGuid(), PrimaryAssetId = asset.Id }); + db.BurstChecks.Add(new BurstCheck { AssetId = asset.Id }); } + + db.Bursts.Add(new Burst { Id = coverless, PrimaryAssetId = null }); } public async ValueTask DisposeAsync() => await db.DisposeAsync(); @@ -62,8 +66,13 @@ public async Task Interactions_OnSoftDeletedAssetAreHiddenAsync() .Should().Equal(active.Id); [Fact] - public async Task Bursts_WhosePrimaryAssetIsSoftDeletedAreHiddenAsync() + public async Task Bursts_WhosePrimaryAssetIsSoftDeletedAreHiddenButCoverlessOnesAreNotAsync() => (await db.Bursts.Select(b => b.PrimaryAssetId).ToListAsync(TestContext.Current.CancellationToken)) + .Should().BeEquivalentTo([active.Id, (Guid?)null]); + + [Fact] + public async Task BurstChecks_OfSoftDeletedAssetAreHiddenAsync() + => (await db.BurstChecks.Select(c => c.AssetId).ToListAsync(TestContext.Current.CancellationToken)) .Should().Equal(active.Id); [Fact] diff --git a/src/Anichron.Core.Tests.Unit/Work/PayloadsTests.cs b/src/Anichron.Core.Tests.Unit/Work/PayloadsTests.cs index 3baef700..5964c89f 100644 --- a/src/Anichron.Core.Tests.Unit/Work/PayloadsTests.cs +++ b/src/Anichron.Core.Tests.Unit/Work/PayloadsTests.cs @@ -22,6 +22,29 @@ public void Crawl_IsOnePeriodicItemPerConfigScopedToIt() ScopeOf(new CrawlPayload(configId)).Should().Be(configId); } + [Fact] + public void BurstDetection_IsOnePeriodicItemPerConfigScopedToIt() + { + var configId = Guid.Parse("77777777-7777-7777-7777-777777777777"); + + Describe(new BurstDetectionPayload(configId)).Should().Be( + (WorkKind.BurstDetection, "burst-detection:77777777-7777-7777-7777-777777777777")); + ScopeOf(new BurstDetectionPayload(configId)).Should().Be(configId); + } + + [Fact] + public void PerConfigPayloads_AreBuiltForTheirConfig() + { + var configId = Guid.Parse("77777777-7777-7777-7777-777777777777"); + + (For(configId), For(configId)).Should().Be( + (new CrawlPayload(configId), new BurstDetectionPayload(configId))); + } + + private static TPayload For(Guid configId) + where TPayload : IStorageConfigPayload + => TPayload.For(configId); + [Fact] public void MaintenancePayloads_AreUnscoped() => (ScopeOf(new TokenCleanupPayload()), ScopeOf(new WorkItemRetentionPayload())).Should().Be((null, null)); diff --git a/src/Anichron.Core/Data/AnichronDbContext.cs b/src/Anichron.Core/Data/AnichronDbContext.cs index 5d5a7f21..dfb7ec74 100644 --- a/src/Anichron.Core/Data/AnichronDbContext.cs +++ b/src/Anichron.Core/Data/AnichronDbContext.cs @@ -16,6 +16,7 @@ public class AnichronDbContext(DbContextOptions options) : Db public DbSet Metadata => Set(); public DbSet ProxyFiles => Set(); public DbSet Bursts => Set(); + public DbSet BurstChecks => Set(); public DbSet Interactions => Set(); public DbSet RefreshTokens => Set(); public DbSet Invites => Set(); @@ -57,20 +58,44 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity(entity => { entity.ToTable(t => t.HasComment( - "Groups a rapid-fire sequence so a flashback shows one memory rather than nine " - + "near-identical frames. The table and schema exist; burst DETECTION is not yet " - + "implemented in the Worker, so in a running system this table is empty." + "Groups photos of one storage config taken moments apart, so a flashback shows one " + + "memory rather than nine near-identical frames. Written only by the Worker's " + + "BurstDetection work item, which also repairs bursts: one left with fewer than two " + + "active members is deleted, and so is one with no members at all." )); entity.Property(b => b.PrimaryAssetId).HasComment( - "The frame chosen to represent the burst. Restrict on delete: losing the cover " - + "would leave a group that cannot be rendered."); + "The stored cover, the earliest member. SetNull on delete, so deleting the cover's row " + + "(a storage config removal cascading to its assets) cannot fail on the burst; the " + + "next BurstDetection run picks a new cover or deletes the burst."); - // Primary Asset reference (1:1 style lookup) entity.HasOne(b => b.PrimaryAsset) .WithMany() .HasForeignKey(b => b.PrimaryAssetId) - .OnDelete(DeleteBehavior.Restrict); - entity.HasQueryFilter(b => !b.PrimaryAsset.IsSoftDeleted); + .OnDelete(DeleteBehavior.SetNull); + entity.HasQueryFilter(b => b.PrimaryAssetId == null || !b.PrimaryAsset!.IsSoftDeleted); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable(t => t.HasComment( + "One row per photo burst detection has checked; no row means unchecked. Owned by the " + + "BurstDetection work item, which reads the original once when it checks a photo. " + + "A file edited in place loses its row, so it is checked again." + )); + entity.Property(c => c.IsCandidate).HasComment( + "The photo has an EXIF capture time, a camera make and model, and a thumbnail; only " + + "candidates are grouped. False also when the original could not be read."); + entity.Property(c => c.CameraBurstId).HasComment( + "Apple's BurstUUID from the original's maker note: photos of one storage config " + + "sharing it form one burst. Null when the camera wrote none."); + entity.HasKey(c => c.AssetId); + + entity.HasOne(c => c.Asset) + .WithOne() + .HasForeignKey(c => c.AssetId) + .OnDelete(DeleteBehavior.Cascade); + entity.HasIndex(c => c.CameraBurstId); + entity.HasQueryFilter(c => !c.Asset.IsSoftDeleted); }); modelBuilder.Entity(entity => diff --git a/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs b/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs index 9d0b92f5..d0ae5f48 100644 --- a/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs +++ b/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs @@ -27,6 +27,11 @@ Task> FindByHashesOrPathsAsync( Task LoadMetadataAsync(MediaAsset asset, CancellationToken ct); + /// Clears the asset's burst-checked mark, if it has one, at the next save. + /// The asset whose mark is cleared. + /// Cancels the lookup. + Task RemoveBurstCheckAsync(Guid assetId, CancellationToken ct); + void Add(MediaAsset asset); void AddProxyFiles(IEnumerable proxyFiles); void RemoveProxyFile(ProxyFile proxyFile); @@ -70,6 +75,12 @@ internal static IQueryable SharedContentQuery( public Task LoadMetadataAsync(MediaAsset asset, CancellationToken ct) => db.Entry(asset).Reference(a => a.Metadata).LoadAsync(ct); + public async Task RemoveBurstCheckAsync(Guid assetId, CancellationToken ct) + { + if (await db.BurstChecks.FindAsync([assetId], ct) is { } check) + db.BurstChecks.Remove(check); + } + public void Add(MediaAsset asset) => db.MediaAssets.Add(asset); diff --git a/src/Anichron.Core/Domain/Burst.cs b/src/Anichron.Core/Domain/Burst.cs index a3335082..d39a641b 100644 --- a/src/Anichron.Core/Domain/Burst.cs +++ b/src/Anichron.Core/Domain/Burst.cs @@ -1,17 +1,18 @@ -namespace Anichron.Core.Domain; +namespace Anichron.Core.Domain; public class Burst { public Guid Id { get; set; } /// - /// Points to the specific asset that should represent this burst in the UI. + /// The stored cover: the earliest member. Null once the cover row is deleted, until burst + /// detection repairs the burst. /// - public Guid PrimaryAssetId { get; set; } + public Guid? PrimaryAssetId { get; set; } public Instant CreatedAt { get; set; } // Navigation Properties - public virtual MediaAsset PrimaryAsset { get; set; } = null!; + public virtual MediaAsset? PrimaryAsset { get; set; } public virtual ICollection Assets { get; set; } = []; } diff --git a/src/Anichron.Core/Domain/BurstCheck.cs b/src/Anichron.Core/Domain/BurstCheck.cs new file mode 100644 index 00000000..10682315 --- /dev/null +++ b/src/Anichron.Core/Domain/BurstCheck.cs @@ -0,0 +1,18 @@ +namespace Anichron.Core.Domain; + +public class BurstCheck +{ + public Guid AssetId { get; set; } + public virtual MediaAsset Asset { get; set; } = null!; + + public Instant CheckedAt { get; set; } + + /// + /// The photo has an EXIF capture time, a camera make and model, and a thumbnail, so it can be + /// grouped. + /// + public bool IsCandidate { get; set; } + + /// Apple's BurstUUID, read from the original's maker note. + public string? CameraBurstId { get; set; } +} diff --git a/src/Anichron.Core/Domain/WorkKind.cs b/src/Anichron.Core/Domain/WorkKind.cs index 17bf0198..c4c0b130 100644 --- a/src/Anichron.Core/Domain/WorkKind.cs +++ b/src/Anichron.Core/Domain/WorkKind.cs @@ -5,4 +5,5 @@ public enum WorkKind TokenCleanup = 1, WorkItemRetention = 2, Crawl = 3, + BurstDetection = 4, } diff --git a/src/Anichron.Core/Migrations/20261023073023_BurstDetection.Designer.cs b/src/Anichron.Core/Migrations/20261023073023_BurstDetection.Designer.cs new file mode 100644 index 00000000..5701ebbe --- /dev/null +++ b/src/Anichron.Core/Migrations/20261023073023_BurstDetection.Designer.cs @@ -0,0 +1,733 @@ +// +using System; +using Anichron.Core.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Anichron.Core.Migrations +{ + [DbContext(typeof(AnichronDbContext))] + [Migration("20261023073023_BurstDetection")] + partial class BurstDetection + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Anichron.Core.Domain.AssetInteraction", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("IsHidden") + .HasColumnType("boolean"); + + b.Property("IsLiked") + .HasColumnType("boolean"); + + b.Property("IsStarred") + .HasColumnType("boolean"); + + b.Property("LastViewed") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("ViewCount") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("AssetId"); + + b.HasIndex("UserId", "AssetId") + .IsUnique() + .HasDatabaseName("IX_Interactions_UserId_AssetId"); + + b.ToTable("Interactions", t => + { + t.HasComment("Per-user state for one asset. Never shared between users: two people looking at the same file each get their own row. Drives flashback weighting — hidden excludes, starred always includes and sorts first, liked raises DisplayWeight."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PrimaryAssetId") + .HasColumnType("uuid") + .HasComment("The stored cover, the earliest member. SetNull on delete, so deleting the cover's row (a storage config removal cascading to its assets) cannot fail on the burst; the next BurstDetection run picks a new cover or deletes the burst."); + + b.HasKey("Id"); + + b.HasIndex("PrimaryAssetId"); + + b.ToTable("Bursts", t => + { + t.HasComment("Groups photos of one storage config taken moments apart, so a flashback shows one memory rather than nine near-identical frames. Written only by the Worker's BurstDetection work item, which also repairs bursts: one left with fewer than two active members is deleted, and so is one with no members at all."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.BurstCheck", b => + { + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("CameraBurstId") + .HasColumnType("text") + .HasComment("Apple's BurstUUID from the original's maker note: photos of one storage config sharing it form one burst. Null when the camera wrote none."); + + b.Property("CheckedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsCandidate") + .HasColumnType("boolean") + .HasComment("The photo has an EXIF capture time, a camera make and model, and a thumbnail; only candidates are grouped. False also when the original could not be read."); + + b.HasKey("AssetId"); + + b.HasIndex("CameraBurstId"); + + b.ToTable("BurstChecks", t => + { + t.HasComment("One row per photo burst detection has checked; no row means unchecked. Owned by the BurstDetection work item, which reads the original once when it checks a photo. A file edited in place loses its row, so it is checked again."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Invite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedByUserId") + .HasColumnType("uuid"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(44) + .HasColumnType("character varying(44)") + .HasComment("Hash, never the token itself. 44 chars — base64 of a 32-byte value."); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UsedByUserId") + .HasColumnType("uuid"); + + b.Property("xmin") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.HasKey("Id"); + + b.HasIndex("CreatedByUserId"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UsedByUserId"); + + b.ToTable("Invites", t => + { + t.HasComment("Admin-issued registration tokens. Registration requires one; the first user (bootstrap admin) is exempt. Redemption is a read-then-write race, which is why this is the only table carrying a concurrency token."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("BurstId") + .HasColumnType("uuid"); + + b.Property("ContentHash") + .IsRequired() + .HasColumnType("text") + .HasComment("XXHash64 of the file contents. Dedup is deliberately CONFIG-SCOPED: the same bytes under two storage configs are two assets, which is what makes multi-user work. A global unique index on this column alone would break that and must never be added."); + + b.Property("DateCaptured") + .HasColumnType("timestamp without time zone"); + + b.Property("Day") + .HasColumnType("integer") + .HasComment("See Month — the pair backs IX_MediaAsset_Flashback."); + + b.Property("FileName") + .IsRequired() + .HasColumnType("text"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text") + .HasComment("Path relative to the storage config root. Not to be confused with ProxyFiles.ProxyPath, which is relative to the proxy root."); + + b.Property("IsSoftDeleted") + .HasColumnType("boolean") + .HasComment("Set by reconciliation when a file is gone from the NAS. Soft rather than hard so the user's interactions survive a temporarily unmounted share."); + + b.Property("LastSeenOnNas") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaType") + .IsRequired() + .HasColumnType("text"); + + b.Property("Month") + .HasColumnType("integer") + .HasComment("Stored apart from DateCaptured so \"On This Day\" is an index lookup on (Month, Day) rather than an EXTRACT() scan over every row."); + + b.Property("PairedAssetId") + .HasColumnType("uuid") + .HasComment("Live Photo pairing: the HEIC parent points at its MOV child. SetNull on delete so losing one half leaves the other a usable standalone asset."); + + b.Property("StorageConfigId") + .HasColumnType("uuid"); + + b.Property("Year") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BurstId"); + + b.HasIndex("IsSoftDeleted") + .HasDatabaseName("IX_MediaAssets_Active") + .HasFilter("\"IsSoftDeleted\" = false"); + + b.HasIndex("PairedAssetId"); + + b.HasIndex("Month", "Day") + .HasDatabaseName("IX_MediaAsset_Flashback"); + + b.HasIndex("StorageConfigId", "ContentHash") + .IsUnique() + .HasDatabaseName("ix_mediaassets_storageconfigid_contenthash_active") + .HasFilter("\"IsSoftDeleted\" = false"); + + b.HasIndex("StorageConfigId", "FilePath") + .IsUnique() + .HasDatabaseName("IX_MediaAssets_StorageConfigId_FilePath"); + + b.ToTable("MediaAssets", t => + { + t.HasComment("One row per media file, scoped to a storage config. Written ONLY by the Worker — the API never writes media data."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Metadata", b => + { + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("Blurhash") + .HasColumnType("text") + .HasComment("BlurHash placeholder string (4x3 components), computed by the Worker from the upright image. Kept here rather than as a proxy file so a response needs no disk read for it. Null for videos, and for images ingested before the column existed, which are not backfilled."); + + b.Property("CameraMake") + .HasColumnType("text"); + + b.Property("CameraModel") + .HasColumnType("text"); + + b.Property("DurationInSeconds") + .HasColumnType("integer"); + + b.Property("Height") + .HasColumnType("integer"); + + b.Property("Latitude") + .HasColumnType("double precision"); + + b.Property("LensModel") + .HasColumnType("text"); + + b.Property("Longitude") + .HasColumnType("double precision"); + + b.Property("Orientation") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1) + .HasComment("Raw EXIF orientation tag, 1-8 (1 = upright, also used when absent or invalid). Stored raw because 5 and 7 combine a rotation with a mirror and cannot be expressed as degrees; a reader converts, and for 5-8 swaps Width/Height. Proxies keep this tag on unrotated pixels, so do not rotate them twice."); + + b.Property("Width") + .HasColumnType("integer"); + + b.HasKey("AssetId"); + + b.ToTable("Metadata", t => + { + t.HasComment("EXIF data, 1:1 with MediaAsset and keyed by it — there is no separate Id. Populated by ExifExtractionMiddleware; every field is best-effort, because a file may carry no EXIF at all."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.ProxyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ProxyPath") + .IsRequired() + .HasColumnType("text") + .HasComment("Path relative to the proxy root, sharded by CONTENT rather than by ingestion attempt so a retry overwrites its own debris instead of stranding it. IProxyDirectoryStrategy is the single place that rule lives."); + + b.Property("ProxyType") + .IsRequired() + .HasColumnType("text") + .HasComment("Thumbnail, WebVideo or FullPreview, persisted as text. The BlurHash placeholder is not a proxy: it is the Metadata.Blurhash column."); + + b.Property("SizeBytes") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("AssetId", "ProxyType") + .IsUnique() + .HasDatabaseName("ix_proxyfiles_assetid_proxytype"); + + b.ToTable("ProxyFiles", t => + { + t.HasComment("One row per generated artefact on the local SSD. An asset has several, but at most one per ProxyType, so of two attempts racing to add the same proxy only one row lands. Written only by ProxyStagingWriter, at the moment the file reaches its final name."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.RefreshToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text") + .HasComment("Hash, never the token itself — a database disclosure must not hand out usable sessions."); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UserId", "ExpiresAt"); + + b.ToTable("RefreshTokens", t => + { + t.HasComment("Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanup work item. One of the few tables both services touch."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text") + .HasComment("Always stored trimmed and lowercased: an EF value converter applies IdentityNormalizer.Normalize on every write and to every equality comparison against this column, so the case-sensitive unique index cannot hold two spellings of one name."); + + b.Property("FailedLoginAttempts") + .HasColumnType("integer"); + + b.Property("IsAdmin") + .HasColumnType("boolean"); + + b.Property("IsDisabled") + .HasColumnType("boolean"); + + b.Property("LockedUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("MustChangePassword") + .HasColumnType("boolean") + .HasComment("Set by an admin reset. MustChangePasswordMiddleware then 403s every route except the handful needed to change it."); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text") + .HasComment("Argon2id. The parameters live with the hasher, not here, so rotating them does not require a migration."); + + b.Property("Username") + .IsRequired() + .HasColumnType("text") + .HasComment("Always stored trimmed and lowercased: an EF value converter applies IdentityNormalizer.Normalize on every write and to every equality comparison against this column, so the case-sensitive unique index cannot hold two spellings of one name."); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique() + .HasDatabaseName("ix_users_email"); + + b.HasIndex("Username") + .IsUnique() + .HasDatabaseName("ix_users_username"); + + b.ToTable("Users", t => + { + t.HasComment("Owns storage configs and interactions. All identity writes happen in the API; the Worker only reads."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("RootPath") + .IsRequired() + .HasColumnType("text") + .HasComment("GLOBALLY unique, not per user: a path cannot be assigned to two configs across all users. Two users indexing the same directory would duplicate every asset and let one user's config removal cascade away the other's rows."); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("RootPath") + .IsUnique() + .HasDatabaseName("IX_StorageConfigs_RootPath"); + + b.HasIndex("UserId"); + + b.ToTable("StorageConfigs", t => + { + t.HasComment("A NAS root assigned to one user. This is the boundary that scopes assets to a user and lets several Workers each monitor a separate config."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasComment("Incremented by every claim. Every write a Worker makes while running an item is fenced by (Id, Attempt), so an attempt that lost its lease changes nothing."); + + b.Property("CancelRequestedAt") + .HasColumnType("timestamp with time zone") + .HasComment("When an admin asked to cancel the item. A Queued item is cancelled at once; a Running one keeps running until the Worker's next heartbeat reads this and cancels its handler. A requested item is never queued again: release and lease recovery end it Cancelled. A cancelled periodic item counts as that run, so the next one follows a full interval later."); + + b.Property("ClaimedBy") + .HasColumnType("text") + .HasComment("The claiming Worker for display: its WORKER__USER or 'all', then its host name."); + + b.Property("DedupKey") + .HasColumnType("text") + .HasComment("Opt-in coalescing. While an item with this key is Queued, enqueueing the same key returns that item instead of adding a second; the partial unique index enforces it."); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Kind") + .IsRequired() + .HasColumnType("text") + .HasComment("WorkKind, persisted as text. Picks the payload type and the handler through the registry, never through a switch."); + + b.Property("LeaseUntil") + .HasColumnType("timestamp with time zone") + .HasComment("Extended by every heartbeat. A Running item past it belongs to a dead or paused Worker and goes back to the queue."); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasComment("The kind's IWorkPayload, serialised with System.Text.Json and NodaTime settings."); + + b.Property("ProgressDone") + .HasColumnType("bigint"); + + b.Property("ProgressNote") + .HasColumnType("text") + .HasComment("The latest progress snapshot (done, optional total, note), overwritten by each heartbeat; no history is kept."); + + b.Property("ProgressTotal") + .HasColumnType("bigint"); + + b.Property("RequestedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RequestedBy") + .HasColumnType("uuid") + .HasComment("The admin who asked for the work; null when a Worker's scheduler enqueued it."); + + b.Property("Result") + .HasColumnType("jsonb") + .HasComment("The handler's results summary, written when the item succeeds, or when it is cancelled after its handler returned one; a Cancelled item's results are partial."); + + b.Property("Stalls") + .HasColumnType("integer") + .HasComment("Lease expiries, counted by lease recovery. A graceful shutdown hands the item back without counting one; the item fails as stalled once the limit is reached."); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasComment("Queued, Running, Succeeded, Failed or Cancelled, persisted as text."); + + b.Property("StorageConfigId") + .HasColumnType("uuid") + .HasComment("The storage config the work belongs to, or null for unscoped work. A per-user Worker claims only its user's configs and unscoped work; deleting the config deletes its items."); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasComment("Moves on every insert, claim and transition, and on a heartbeat only when the progress snapshot changed, so a reader can ask what changed since a point in time."); + + b.HasKey("Id"); + + b.HasIndex("DedupKey") + .IsUnique() + .HasDatabaseName("ix_workitems_dedupkey_queued") + .HasFilter("\"Status\" = 'Queued' AND \"DedupKey\" IS NOT NULL"); + + b.HasIndex("LeaseUntil") + .HasDatabaseName("ix_workitems_lease") + .HasFilter("\"Status\" = 'Running'"); + + b.HasIndex("RequestedAt") + .HasDatabaseName("ix_workitems_claim") + .HasFilter("\"Status\" = 'Queued'"); + + b.HasIndex("RequestedBy"); + + b.HasIndex("StorageConfigId"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("WorkItems", t => + { + t.HasComment("Durable work for the Worker: periodic maintenance today, admin requests later. Only IWorkQueue inserts rows; the API's only other write is an admin's cancel, and every other state change belongs to the Worker's executor. Delivery is at least once, so handlers start from scratch and converge."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.AssetInteraction", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithMany("Interactions") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("Interactions") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "PrimaryAsset") + .WithMany() + .HasForeignKey("PrimaryAssetId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("PrimaryAsset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.BurstCheck", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithOne() + .HasForeignKey("Anichron.Core.Domain.BurstCheck", "AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Invite", b => + { + b.HasOne("Anichron.Core.Domain.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedByUserId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.User", "UsedBy") + .WithMany() + .HasForeignKey("UsedByUserId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("CreatedBy"); + + b.Navigation("UsedBy"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.HasOne("Anichron.Core.Domain.Burst", "Burst") + .WithMany("Assets") + .HasForeignKey("BurstId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.MediaAsset", "PairedAsset") + .WithMany() + .HasForeignKey("PairedAssetId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.UserStorageConfig", "StorageConfig") + .WithMany("Assets") + .HasForeignKey("StorageConfigId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Burst"); + + b.Navigation("PairedAsset"); + + b.Navigation("StorageConfig"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Metadata", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithOne("Metadata") + .HasForeignKey("Anichron.Core.Domain.Metadata", "AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.ProxyFile", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithMany("ProxyFiles") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.RefreshToken", b => + { + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("RefreshTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("StorageConfigs") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.HasOne("Anichron.Core.Domain.User", null) + .WithMany() + .HasForeignKey("RequestedBy") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.UserStorageConfig", null) + .WithMany() + .HasForeignKey("StorageConfigId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.Navigation("Assets"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.Navigation("Interactions"); + + b.Navigation("Metadata"); + + b.Navigation("ProxyFiles"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.User", b => + { + b.Navigation("Interactions"); + + b.Navigation("RefreshTokens"); + + b.Navigation("StorageConfigs"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.Navigation("Assets"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Anichron.Core/Migrations/20261023073023_BurstDetection.cs b/src/Anichron.Core/Migrations/20261023073023_BurstDetection.cs new file mode 100644 index 00000000..68a2778c --- /dev/null +++ b/src/Anichron.Core/Migrations/20261023073023_BurstDetection.cs @@ -0,0 +1,105 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace Anichron.Core.Migrations +{ + /// + public partial class BurstDetection : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_Bursts_MediaAssets_PrimaryAssetId", + table: "Bursts"); + + migrationBuilder.AlterTable( + name: "Bursts", + comment: "Groups photos of one storage config taken moments apart, so a flashback shows one memory rather than nine near-identical frames. Written only by the Worker's BurstDetection work item, which also repairs bursts: one left with fewer than two active members is deleted, and so is one with no members at all.", + oldComment: "Groups a rapid-fire sequence so a flashback shows one memory rather than nine near-identical frames. The table and schema exist; burst DETECTION is not yet implemented in the Worker, so in a running system this table is empty."); + + migrationBuilder.AlterColumn( + name: "PrimaryAssetId", + table: "Bursts", + type: "uuid", + nullable: true, + comment: "The stored cover, the earliest member. SetNull on delete, so deleting the cover's row (a storage config removal cascading to its assets) cannot fail on the burst; the next BurstDetection run picks a new cover or deletes the burst.", + oldClrType: typeof(Guid), + oldType: "uuid", + oldComment: "The frame chosen to represent the burst. Restrict on delete: losing the cover would leave a group that cannot be rendered."); + + migrationBuilder.CreateTable( + name: "BurstChecks", + columns: table => new + { + AssetId = table.Column(type: "uuid", nullable: false), + CheckedAt = table.Column(type: "timestamp with time zone", nullable: false), + IsCandidate = table.Column(type: "boolean", nullable: false, comment: "The photo has an EXIF capture time, a camera make and model, and a thumbnail; only candidates are grouped. False also when the original could not be read."), + CameraBurstId = table.Column(type: "text", nullable: true, comment: "Apple's BurstUUID from the original's maker note: photos of one storage config sharing it form one burst. Null when the camera wrote none.") + }, + constraints: table => + { + table.PrimaryKey("PK_BurstChecks", x => x.AssetId); + table.ForeignKey( + name: "FK_BurstChecks_MediaAssets_AssetId", + column: x => x.AssetId, + principalTable: "MediaAssets", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }, + comment: "One row per photo burst detection has checked; no row means unchecked. Owned by the BurstDetection work item, which reads the original once when it checks a photo. A file edited in place loses its row, so it is checked again."); + + migrationBuilder.CreateIndex( + name: "IX_BurstChecks_CameraBurstId", + table: "BurstChecks", + column: "CameraBurstId"); + + migrationBuilder.AddForeignKey( + name: "FK_Bursts_MediaAssets_PrimaryAssetId", + table: "Bursts", + column: "PrimaryAssetId", + principalTable: "MediaAssets", + principalColumn: "Id", + onDelete: ReferentialAction.SetNull); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_Bursts_MediaAssets_PrimaryAssetId", + table: "Bursts"); + + migrationBuilder.DropTable( + name: "BurstChecks"); + + migrationBuilder.AlterTable( + name: "Bursts", + comment: "Groups a rapid-fire sequence so a flashback shows one memory rather than nine near-identical frames. The table and schema exist; burst DETECTION is not yet implemented in the Worker, so in a running system this table is empty.", + oldComment: "Groups photos of one storage config taken moments apart, so a flashback shows one memory rather than nine near-identical frames. Written only by the Worker's BurstDetection work item, which also repairs bursts: one left with fewer than two active members is deleted, and so is one with no members at all."); + + migrationBuilder.AlterColumn( + name: "PrimaryAssetId", + table: "Bursts", + type: "uuid", + nullable: false, + defaultValue: Guid.Empty, + comment: "The frame chosen to represent the burst. Restrict on delete: losing the cover would leave a group that cannot be rendered.", + oldClrType: typeof(Guid), + oldType: "uuid", + oldNullable: true, + oldComment: "The stored cover, the earliest member. SetNull on delete, so deleting the cover's row (a storage config removal cascading to its assets) cannot fail on the burst; the next BurstDetection run picks a new cover or deletes the burst."); + + migrationBuilder.AddForeignKey( + name: "FK_Bursts_MediaAssets_PrimaryAssetId", + table: "Bursts", + column: "PrimaryAssetId", + principalTable: "MediaAssets", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + } + } +} diff --git a/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs b/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs index 37a84927..85143643 100644 --- a/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs +++ b/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs @@ -73,9 +73,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("CreatedAt") .HasColumnType("timestamp with time zone"); - b.Property("PrimaryAssetId") + b.Property("PrimaryAssetId") .HasColumnType("uuid") - .HasComment("The frame chosen to represent the burst. Restrict on delete: losing the cover would leave a group that cannot be rendered."); + .HasComment("The stored cover, the earliest member. SetNull on delete, so deleting the cover's row (a storage config removal cascading to its assets) cannot fail on the burst; the next BurstDetection run picks a new cover or deletes the burst."); b.HasKey("Id"); @@ -83,7 +83,33 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("Bursts", t => { - t.HasComment("Groups a rapid-fire sequence so a flashback shows one memory rather than nine near-identical frames. The table and schema exist; burst DETECTION is not yet implemented in the Worker, so in a running system this table is empty."); + t.HasComment("Groups photos of one storage config taken moments apart, so a flashback shows one memory rather than nine near-identical frames. Written only by the Worker's BurstDetection work item, which also repairs bursts: one left with fewer than two active members is deleted, and so is one with no members at all."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.BurstCheck", b => + { + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("CameraBurstId") + .HasColumnType("text") + .HasComment("Apple's BurstUUID from the original's maker note: photos of one storage config sharing it form one burst. Null when the camera wrote none."); + + b.Property("CheckedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsCandidate") + .HasColumnType("boolean") + .HasComment("The photo has an EXIF capture time, a camera make and model, and a thumbnail; only candidates are grouped. False also when the original could not be read."); + + b.HasKey("AssetId"); + + b.HasIndex("CameraBurstId"); + + b.ToTable("BurstChecks", t => + { + t.HasComment("One row per photo burst detection has checked; no row means unchecked. Owned by the BurstDetection work item, which reads the original once when it checks a photo. A file edited in place loses its row, so it is checked again."); }); }); @@ -556,12 +582,22 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasOne("Anichron.Core.Domain.MediaAsset", "PrimaryAsset") .WithMany() .HasForeignKey("PrimaryAssetId") - .OnDelete(DeleteBehavior.Restrict) - .IsRequired(); + .OnDelete(DeleteBehavior.SetNull); b.Navigation("PrimaryAsset"); }); + modelBuilder.Entity("Anichron.Core.Domain.BurstCheck", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithOne() + .HasForeignKey("Anichron.Core.Domain.BurstCheck", "AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + modelBuilder.Entity("Anichron.Core.Domain.Invite", b => { b.HasOne("Anichron.Core.Domain.User", "CreatedBy") diff --git a/src/Anichron.Core/Work/IWorkPayload.cs b/src/Anichron.Core/Work/IWorkPayload.cs index 7497cbce..cc2a7a52 100644 --- a/src/Anichron.Core/Work/IWorkPayload.cs +++ b/src/Anichron.Core/Work/IWorkPayload.cs @@ -18,3 +18,11 @@ public interface IWorkPayload /// Guid? StorageConfigId => null; } + +/// Work that runs once per storage config, periodically for every config a Worker serves. +/// The payload type itself. +public interface IStorageConfigPayload : IWorkPayload + where TSelf : IStorageConfigPayload +{ + static abstract TSelf For(Guid storageConfigId); +} diff --git a/src/Anichron.Core/Work/Payloads.cs b/src/Anichron.Core/Work/Payloads.cs index b9977c05..c0848d15 100644 --- a/src/Anichron.Core/Work/Payloads.cs +++ b/src/Anichron.Core/Work/Payloads.cs @@ -16,11 +16,24 @@ public sealed record WorkItemRetentionPayload : IWorkPayload string IWorkPayload.DedupKey => "work-item-retention"; } -public sealed record CrawlPayload(Guid StorageConfigId) : IWorkPayload +public sealed record CrawlPayload(Guid StorageConfigId) : IStorageConfigPayload { public static WorkKind Kind => WorkKind.Crawl; + public static CrawlPayload For(Guid storageConfigId) => new(storageConfigId); + string IWorkPayload.DedupKey => $"crawl:{StorageConfigId}"; Guid? IWorkPayload.StorageConfigId => StorageConfigId; } + +public sealed record BurstDetectionPayload(Guid StorageConfigId) : IStorageConfigPayload +{ + public static WorkKind Kind => WorkKind.BurstDetection; + + public static BurstDetectionPayload For(Guid storageConfigId) => new(storageConfigId); + + string IWorkPayload.DedupKey => $"burst-detection:{StorageConfigId}"; + + Guid? IWorkPayload.StorageConfigId => StorageConfigId; +} diff --git a/src/Anichron.Tests.Integration/Data/BurstDeleteBehaviourTests.cs b/src/Anichron.Tests.Integration/Data/BurstDeleteBehaviourTests.cs new file mode 100644 index 00000000..6bcd8513 --- /dev/null +++ b/src/Anichron.Tests.Integration/Data/BurstDeleteBehaviourTests.cs @@ -0,0 +1,68 @@ +using Anichron.Core.Domain; +using Microsoft.EntityFrameworkCore; + +namespace Anichron.Tests.Integration.Data; + +public sealed class BurstDeleteBehaviourTests(PostgresServer server) +{ + private static readonly Instant Now = Instant.FromUtc(2026, 10, 23, 9, 0); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private sealed record Library(string ConnectionString, Guid ConfigId, Guid BurstId, Guid CoverId, Guid OtherId); + + private async Task LibraryAsync() + { + var connectionString = await server.CreateMigratedDatabaseAsync(Ct); + var (user, config, burst) = (Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid()); + MediaAsset Photo(string name) => new() + { + Id = Guid.NewGuid(), + StorageConfigId = config, + BurstId = burst, + FilePath = name, + FileName = name, + ContentHash = name, + MediaType = MediaType.Image, + }; + var (cover, other) = (Photo("a.heic"), Photo("b.heic")); + + await using var db = PostgresServer.Context(connectionString); + db.Users.Add(new User { Id = user, Username = "mira", Email = "mira@example.com", PasswordHash = "hash" }); + db.StorageConfigs.Add(new UserStorageConfig { Id = config, UserId = user, RootPath = "/data/originals/mira", IsActive = true }); + db.Bursts.Add(new Burst { Id = burst, CreatedAt = Now }); + db.MediaAssets.AddRange(cover, other); + db.BurstChecks.AddRange( + new BurstCheck { AssetId = cover.Id, CheckedAt = Now, IsCandidate = true, CameraBurstId = "label" }, + new BurstCheck { AssetId = other.Id, CheckedAt = Now, IsCandidate = true, CameraBurstId = "label" }); + await db.SaveChangesAsync(Ct); + await db.Bursts.Where(b => b.Id == burst).ExecuteUpdateAsync(s => s.SetProperty(b => b.PrimaryAssetId, cover.Id), Ct); + return new Library(connectionString, config, burst, cover.Id, other.Id); + } + + [Fact] + public async Task DeletingTheStorageConfig_RemovesItsAssetsAndChecksAndLeavesTheBurstCoverlessAsync() + { + var library = await LibraryAsync(); + await using var db = PostgresServer.Context(library.ConnectionString); + + await db.StorageConfigs.Where(c => c.Id == library.ConfigId).ExecuteDeleteAsync(Ct); + + (await db.MediaAssets.IgnoreQueryFilters().CountAsync(Ct)).Should().Be(0); + (await db.BurstChecks.IgnoreQueryFilters().CountAsync(Ct)).Should().Be(0); + (await db.Bursts.IgnoreQueryFilters().SingleAsync(Ct)).Should().BeEquivalentTo( + new { Id = library.BurstId, PrimaryAssetId = (Guid?)null }); + } + + [Fact] + public async Task DeletingTheCover_ClearsItButKeepsTheOtherMemberAsync() + { + var library = await LibraryAsync(); + await using var db = PostgresServer.Context(library.ConnectionString); + + await db.MediaAssets.Where(a => a.Id == library.CoverId).ExecuteDeleteAsync(Ct); + + (await db.Bursts.SingleAsync(Ct)).PrimaryAssetId.Should().BeNull(); + (await db.MediaAssets.SingleAsync(Ct)).Should().BeEquivalentTo(new { Id = library.OtherId, BurstId = (Guid?)library.BurstId }); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs b/src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs new file mode 100644 index 00000000..8de250af --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs @@ -0,0 +1,579 @@ +using Anichron.Core; +using Anichron.Core.Data; +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Testing.Data; +using Anichron.Worker.Bursts; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using NodaTime; +using System.IO.Abstractions.TestingHelpers; + +namespace Anichron.Worker.Tests.Unit.Bursts; + +public sealed class BurstDetectionTests : IAsyncDisposable +{ + private const string Root = "/data/originals/mira"; + private const string BurstA = "3F2504E0-4F89-11D3-9A0C-0305E82C3301"; + private const string BurstB = "9C1B7E44-0D2A-4C55-8F6B-2E1D3A4B5C6D"; + + private static readonly Guid MiraId = Guid.Parse("66666666-6666-6666-6666-666666666666"); + private static readonly Guid ConfigId = Guid.Parse("11111111-0000-0000-0000-000000000000"); + private static readonly Guid OtherConfigId = Guid.Parse("22222222-0000-0000-0000-000000000000"); + private static readonly Instant Now = Instant.FromUtc(2026, 10, 23, 9, 0); + private static readonly LocalDateTime Noon = new(2024, 7, 15, 12, 0, 0); + + private readonly string databaseName = Guid.NewGuid().ToString(); + private readonly AnichronDbContext db; + private readonly IOriginalPhotoReader reader = Substitute.For(); + private readonly MockFileSystem fileSystem = new(); + private readonly IClock clock = Substitute.For(); + private readonly IGuidFactory guids = Substitute.For(); + private readonly IWorkProgress progress = Substitute.For(); + private readonly FakeLogger logger = new(); + private int nextGuid; + + public BurstDetectionTests() + { + db = AnichronDbContexts.InMemory(databaseName); + fileSystem.AddDirectory(Root); + clock.GetCurrentInstant().Returns(Now); + guids.NewGuid().Returns(_ => Guid.Parse($"bbbbbbbb-0000-0000-0000-{++nextGuid:D12}")); + reader.Read(Arg.Any()).Returns(new OriginalPhotoFacts(Noon, null)); + Seed(new User { Id = MiraId, Username = "mira", Email = "mira@example.com", PasswordHash = "h" }); + Seed(new UserStorageConfig { Id = ConfigId, UserId = MiraId, RootPath = Root, IsActive = true }); + Seed(new UserStorageConfig { Id = OtherConfigId, UserId = MiraId, RootPath = "/data/originals/other", IsActive = true }); + } + + public async ValueTask DisposeAsync() => await db.DisposeAsync(); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private void Seed(params object[] entities) + { + using var seed = AnichronDbContexts.InMemory(databaseName); + seed.AddRange(entities); + seed.SaveChanges(); + } + + // A candidate unless told otherwise: a thumbnail, a camera make and model, captured minutes after noon. + private MediaAsset Photo( + string name, + int minutes = 0, + MediaType type = MediaType.Image, + Guid? config = null, + bool thumbnail = true, + string? make = "Apple", + string? model = "iPhone 15 Pro", + bool softDeleted = false, + Guid? burstId = null) + { + var id = Guid.NewGuid(); + var asset = new MediaAsset + { + Id = id, + StorageConfigId = config ?? ConfigId, + FilePath = $"2024/{name}", + FileName = name, + ContentHash = name, + MediaType = type, + DateCaptured = Noon.PlusMinutes(minutes), + IsSoftDeleted = softDeleted, + BurstId = burstId, + Metadata = new Metadata { AssetId = id, CameraMake = make, CameraModel = model }, + ProxyFiles = thumbnail + ? [new ProxyFile { Id = Guid.NewGuid(), AssetId = id, ProxyType = ProxyType.Thumbnail, ProxyPath = name }] + : [], + }; + Seed(asset); + return asset; + } + + private void Labelled(MediaAsset photo, string? label) + => reader.Read($"{Root}/{photo.FilePath}").Returns(new OriginalPhotoFacts(Noon, label)); + + private void Checked(MediaAsset photo, string? label, bool candidate = true) + => Seed(new BurstCheck { AssetId = photo.Id, CheckedAt = Now, IsCandidate = candidate, CameraBurstId = label }); + + private Guid StoredBurst(int createdMinutesAgo, params MediaAsset[] members) + { + var id = Guid.NewGuid(); + Seed(new Burst { Id = id, PrimaryAssetId = members.FirstOrDefault()?.Id, CreatedAt = Now.Minus(Duration.FromMinutes(createdMinutesAgo)) }); + using var seed = AnichronDbContexts.InMemory(databaseName); + foreach (var member in members) + seed.MediaAssets.IgnoreQueryFilters().Single(a => a.Id == member.Id).BurstId = id; + seed.SaveChanges(); + return id; + } + + private void ClearCover(Guid burstId) + { + using var seed = AnichronDbContexts.InMemory(databaseName); + seed.Bursts.IgnoreQueryFilters().Single(b => b.Id == burstId).PrimaryAssetId = null; + seed.SaveChanges(); + } + + private Task RunAsync() => RunForAsync(ConfigId, Ct); + + private Task RunForAsync(Guid configId, CancellationToken ct) + => new BurstDetectionHandler(db, reader, fileSystem, clock, guids, logger) + .RunAsync(new BurstDetectionPayload(configId), progress, ct); + + private Dictionary BurstsOf(params MediaAsset[] photos) + { + using var read = AnichronDbContexts.InMemory(databaseName); + var ids = photos.Select(p => p.Id).ToList(); + var stored = read.MediaAssets.IgnoreQueryFilters().Where(a => ids.Contains(a.Id)).ToDictionary(a => a.Id, a => a.BurstId); + return photos.ToDictionary(p => p.FileName, p => stored[p.Id]); + } + + private List StoredBursts() + { + using var read = AnichronDbContexts.InMemory(databaseName); + return [.. read.Bursts.IgnoreQueryFilters().OrderBy(b => b.CreatedAt)]; + } + + private List StoredChecks() + { + using var read = AnichronDbContexts.InMemory(databaseName); + return [.. read.BurstChecks.IgnoreQueryFilters()]; + } + + // ========================================================================== + // Skipping and failing + // ========================================================================== + + [Fact] + public async Task RunAsync_InactiveConfig_SkipsWithoutReadingAnythingAsync() + { + var inactive = Guid.NewGuid(); + Seed(new UserStorageConfig { Id = inactive, UserId = MiraId, RootPath = "/data/originals/off", IsActive = false }); + Photo("a.heic", config: inactive); + + var result = await RunForAsync(inactive, Ct); + + result.Should().Be(new BurstDetectionSkipped("The storage config is missing or inactive.")); + reader.DidNotReceiveWithAnyArgs().Read(default!); + StoredChecks().Should().BeEmpty(); + logger.Collector.GetSnapshot().Should().ContainSingle().Which.Message.Should().Be( + $"Skipping burst detection for storage config {inactive}: it is missing or inactive."); + } + + [Fact] + public async Task RunAsync_ConfigGone_SkipsAsync() + => (await RunForAsync(Guid.NewGuid(), Ct)).Should().BeOfType(); + + [Fact] + public async Task RunAsync_RootMissing_FailsWithoutMarkingAnyPhotoAsync() + { + Photo("a.heic"); + fileSystem.Directory.Delete(Root); + + var act = () => RunAsync(); + + await act.Should().ThrowAsync().WithMessage($"*{Root}*"); + StoredChecks().Should().BeEmpty(); + } + + // ========================================================================== + // Camera-labelled grouping + // ========================================================================== + + [Fact] + public async Task RunAsync_PhotosWithOneBurstUuid_FormOneBurstWithTheEarliestAsCoverAsync() + { + var (a, b, c) = (Photo("a.heic", minutes: 2), Photo("b.heic", minutes: 1), Photo("c.heic", minutes: 3)); + foreach (var photo in new[] { a, b, c }) + Labelled(photo, BurstA); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(Checked: 3, Created: 1, Merged: 0, Dissolved: 0)); + var burst = StoredBursts().Should().ContainSingle().Which; + burst.Should().BeEquivalentTo(new { PrimaryAssetId = b.Id, CreatedAt = Now }); + BurstsOf(a, b, c).Values.Should().AllBeEquivalentTo(burst.Id); + } + + [Fact] + public async Task RunAsync_SameBurstUuidDaysApart_StillOneBurstAsync() + { + var (a, b) = (Photo("a.heic"), Photo("b.heic", minutes: 3 * 24 * 60)); + Labelled(a, BurstA); + Labelled(b, BurstA); + + await RunAsync(); + + StoredBursts().Should().ContainSingle().Which.PrimaryAssetId.Should().Be(a.Id); + } + + [Fact] + public async Task RunAsync_TwoBurstUuids_TwoBurstsAndUnlabelledPhotosStayAloneAsync() + { + var (a1, a2, b1, b2, plain) = (Photo("a1.heic"), Photo("a2.heic", 1), Photo("b1.heic", 2), Photo("b2.heic", 3), Photo("p.heic", 4)); + Labelled(a1, BurstA); + Labelled(a2, BurstA); + Labelled(b1, BurstB); + Labelled(b2, BurstB); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(5, 2, 0, 0)); + var bursts = BurstsOf(a1, a2, b1, b2, plain); + var (burstA, burstB) = (bursts["a1.heic"], bursts["b1.heic"]); + new[] { burstA, burstB }.Should().OnlyHaveUniqueItems().And.NotContainNulls(); + bursts.Should().BeEquivalentTo(new Dictionary + { + ["a1.heic"] = burstA, + ["a2.heic"] = burstA, + ["b1.heic"] = burstB, + ["b2.heic"] = burstB, + ["p.heic"] = null, + }); + } + + [Fact] + public async Task RunAsync_LonePhotoWithABurstUuid_FormsNoBurstAsync() + { + var a = Photo("a.heic"); + Labelled(a, BurstA); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(1, 0, 0, 0)); + StoredBursts().Should().BeEmpty(); + BurstsOf(a)["a.heic"].Should().BeNull(); + } + + [Fact] + public async Task RunAsync_SameBurstUuidInAnotherStorageConfig_IsNotJoinedAsync() + { + var mine = Photo("a.heic"); + var theirs = Photo("b.heic", config: OtherConfigId); + Checked(theirs, BurstA); + Labelled(mine, BurstA); + + await RunAsync(); + + StoredBursts().Should().BeEmpty(); + } + + [Fact] + public async Task RunAsync_LivePhotoHalfJoins_VideosAndPairedMovsAreNeverCheckedAsync() + { + var still = Photo("a.jpg"); + var live = Photo("b.heic", minutes: 1, type: MediaType.LivePhoto); + var mov = Photo("b.mov", minutes: 1, type: MediaType.Video); + Labelled(still, BurstA); + Labelled(live, BurstA); + Labelled(mov, BurstA); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(2, 1, 0, 0)); + BurstsOf(still, live, mov).Should().BeEquivalentTo(new Dictionary + { + ["a.jpg"] = StoredBursts().Single().Id, + ["b.heic"] = StoredBursts().Single().Id, + ["b.mov"] = null, + }); + StoredChecks().Select(c => c.AssetId).Should().BeEquivalentTo([still.Id, live.Id]); + reader.DidNotReceive().Read($"{Root}/{mov.FilePath}"); + } + + [Theory] + [InlineData(false, "Apple", "iPhone 15 Pro", true)] + [InlineData(true, null, "iPhone 15 Pro", true)] + [InlineData(true, "Apple", null, true)] + [InlineData(true, "Apple", "iPhone 15 Pro", false)] + public async Task RunAsync_PhotoLackingWhatACandidateNeeds_IsCheckedButNotGroupedAsync( + bool exifTime, string? make, string? model, bool thumbnail) + { + var good = Photo("a.heic"); + var lacking = Photo("b.heic", minutes: 1, thumbnail: thumbnail, make: make, model: model); + Labelled(good, BurstA); + reader.Read($"{Root}/{lacking.FilePath}").Returns(new OriginalPhotoFacts(exifTime ? Noon : null, BurstA)); + var other = Photo("c.heic", minutes: 2); + Labelled(other, BurstA); + + await RunAsync(); + + BurstsOf(lacking)["b.heic"].Should().BeNull(); + StoredChecks().Single(c => c.AssetId == lacking.Id) + .Should().BeEquivalentTo(new { IsCandidate = false, CameraBurstId = BurstA, CheckedAt = Now }); + StoredBursts().Should().ContainSingle().Which.PrimaryAssetId.Should().Be(good.Id); + } + + [Fact] + public async Task RunAsync_LabelledPhotoAlreadyCheckedAsNoCandidate_IsNotJoinedAsync() + { + var old = Photo("a.heic"); + Checked(old, BurstA, candidate: false); + var fresh = Photo("b.heic", minutes: 1); + Labelled(fresh, BurstA); + + await RunAsync(); + + StoredBursts().Should().BeEmpty(); + } + + [Fact] + public async Task RunAsync_UnreadableOriginal_IsCheckedAsNoCandidateAsync() + { + var a = Photo("a.heic"); + reader.Read($"{Root}/{a.FilePath}").Returns(OriginalPhotoFacts.Unknown); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(1, 0, 0, 0)); + StoredChecks().Should().ContainSingle().Which.Should().BeEquivalentTo( + new BurstCheck { AssetId = a.Id, CheckedAt = Now, IsCandidate = false, CameraBurstId = null }, + o => o.Excluding(c => c.Asset)); + } + + // ========================================================================== + // The checked mark and incremental runs + // ========================================================================== + + [Fact] + public async Task RunAsync_MarksEachPhotoOnceAndASecondRunChecksNothingAsync() + { + var (a, b) = (Photo("a.heic"), Photo("b.heic", 1)); + Labelled(a, BurstA); + Labelled(b, BurstA); + await RunAsync(); + reader.ClearReceivedCalls(); + + var second = await RunAsync(); + + second.Should().Be(new BurstDetectionResult(0, 0, 0, 0)); + reader.DidNotReceiveWithAnyArgs().Read(default!); + StoredChecks().Should().BeEquivalentTo( + [ + new { AssetId = a.Id, IsCandidate = true, CameraBurstId = BurstA, CheckedAt = Now }, + new { AssetId = b.Id, IsCandidate = true, CameraBurstId = BurstA, CheckedAt = Now }, + ]); + StoredBursts().Should().ContainSingle(); + } + + [Fact] + public async Task RunAsync_NewPhotoWithTheLabelOfAStoredBurst_JoinsItAndBecomesItsCoverIfEarlierAsync() + { + var (a, b) = (Photo("a.heic", 5), Photo("b.heic", 6)); + Checked(a, BurstA); + Checked(b, BurstA); + var stored = StoredBurst(60, a, b); + var earlier = Photo("c.heic", 4); + Labelled(earlier, BurstA); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(1, 0, 0, 0)); + BurstsOf(a, b, earlier).Values.Should().AllBeEquivalentTo(stored); + StoredBursts().Should().ContainSingle().Which.PrimaryAssetId.Should().Be(earlier.Id); + reader.DidNotReceive().Read($"{Root}/{a.FilePath}"); + } + + [Fact] + public async Task RunAsync_NewPhotoWithTheLabelOfACheckedLonePhoto_FormsABurstWithItAsync() + { + var lone = Photo("a.heic", 1); + Checked(lone, BurstA); + var fresh = Photo("b.heic", 0); + Labelled(fresh, BurstA); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(1, 1, 0, 0)); + var burst = StoredBursts().Should().ContainSingle().Which; + burst.PrimaryAssetId.Should().Be(fresh.Id); + BurstsOf(lone, fresh).Values.Should().AllBeEquivalentTo(burst.Id); + } + + [Fact] + public async Task RunAsync_NewPhotoJoiningTwoStoredBursts_MergesThemIntoTheOlderAsync() + { + var (a, b, c, d) = (Photo("a.heic", 4), Photo("b.heic", 5), Photo("c.heic", 2), Photo("d.heic", 3)); + Checked(a, BurstA); + Checked(b, BurstA); + Checked(c, BurstA); + Checked(d, BurstA); + var older = StoredBurst(60, a, b); + StoredBurst(30, c, d); + var fresh = Photo("e.heic", 9); + Labelled(fresh, BurstA); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(1, 0, 1, 0)); + StoredBursts().Should().ContainSingle().Which.Should().BeEquivalentTo(new { Id = older, PrimaryAssetId = c.Id }); + BurstsOf(a, b, c, d, fresh).Values.Should().AllBeEquivalentTo(older); + } + + [Fact] + public async Task RunAsync_ProcessesPhotosOldestFirstAndReportsEachAsync() + { + var (late, early) = (Photo("a.heic", 9), Photo("b.heic", 1)); + + await RunAsync(); + + Received.InOrder(() => + { + reader.Read($"{Root}/{early.FilePath}"); + progress.Report(1, 2, "Checking photos for bursts"); + reader.Read($"{Root}/{late.FilePath}"); + progress.Report(2, 2, "Checking photos for bursts"); + }); + } + + [Fact] + public async Task RunAsync_SameCaptureTime_ProcessesTheLowerIdFirstAsync() + { + var (a, b) = (Photo("a.heic"), Photo("b.heic")); + var (first, second) = a.Id.CompareTo(b.Id) < 0 ? (a, b) : (b, a); + + await RunAsync(); + + Received.InOrder(() => + { + reader.Read($"{Root}/{first.FilePath}"); + reader.Read($"{Root}/{second.FilePath}"); + }); + } + + [Fact] + public async Task RunAsync_JoiningTwoBurstsCreatedAtOnce_MergesIntoTheLowerIdAsync() + { + var (a, b, c, d) = (Photo("a.heic", 1), Photo("b.heic", 2), Photo("c.heic", 3), Photo("d.heic", 4)); + foreach (var photo in new[] { a, b, c, d }) + Checked(photo, BurstA); + var (one, two) = (StoredBurst(60, a, b), StoredBurst(60, c, d)); + var fresh = Photo("e.heic", 9); + Labelled(fresh, BurstA); + + await RunAsync(); + + StoredBursts().Should().ContainSingle().Which.Id.Should().Be(one.CompareTo(two) < 0 ? one : two); + } + + [Fact] + public async Task RunAsync_SameCaptureTime_CoverIsTheLowerIdAsync() + { + var (a, b) = (Photo("a.heic"), Photo("b.heic")); + Labelled(a, BurstA); + Labelled(b, BurstA); + var lower = a.Id.CompareTo(b.Id) < 0 ? a.Id : b.Id; + + await RunAsync(); + + StoredBursts().Single().PrimaryAssetId.Should().Be(lower); + } + + [Fact] + public async Task RunAsync_MoreThanOneBatch_ChecksEveryPhotoAndKeepsNoneTrackedAsync() + { + var photos = Enumerable.Range(0, BurstDetectionHandler.BatchSize + 1).Select(i => Photo($"{i}.heic", i)).ToList(); + Labelled(photos[0], BurstA); + Labelled(photos[^1], BurstA); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(BurstDetectionHandler.BatchSize + 1, 1, 0, 0)); + StoredChecks().Should().HaveCount(BurstDetectionHandler.BatchSize + 1); + db.ChangeTracker.Entries().Should().BeEmpty(); + } + + [Fact] + public async Task RunAsync_Cancelled_StopsBeforeSavingThePhotoBeingCheckedAsync() + { + var (a, b) = (Photo("a.heic"), Photo("b.heic", 1)); + using var cancel = new CancellationTokenSource(); + reader.Read($"{Root}/{a.FilePath}").Returns(_ => + { + cancel.Cancel(); + return new OriginalPhotoFacts(Noon, null); + }); + + var act = () => RunForAsync(ConfigId, cancel.Token); + + await act.Should().ThrowAsync(); + StoredChecks().Should().BeEmpty(); + reader.DidNotReceive().Read($"{Root}/{b.FilePath}"); + } + + [Fact] + public async Task RunAsync_LogsTheSummaryAsync() + { + var (a, b) = (Photo("a.heic"), Photo("b.heic", 1)); + Labelled(a, BurstA); + Labelled(b, BurstA); + + await RunAsync(); + + logger.Collector.GetSnapshot().Should().ContainSingle(r => r.Level == LogLevel.Information).Which.Message.Should().Be( + $"Burst detection for storage config {ConfigId} checked 2 photo(s): 1 burst(s) created, 0 merged, 0 dissolved."); + } + + // ========================================================================== + // Repair + // ========================================================================== + + [Fact] + public async Task RunAsync_BurstLeftWithOneActiveMember_IsDissolvedAsync() + { + var (a, gone) = (Photo("a.heic"), Photo("b.heic", 1, softDeleted: true)); + StoredBurst(60, a, gone); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(1, 0, 0, 1)); + StoredBursts().Should().BeEmpty(); + BurstsOf(a, gone).Values.Should().AllSatisfy(id => id.Should().BeNull()); + } + + [Fact] + public async Task RunAsync_BurstWithoutMembers_IsDeletedAsync() + { + StoredBurst(60); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(0, 0, 0, 1)); + StoredBursts().Should().BeEmpty(); + } + + [Fact] + public async Task RunAsync_CoverSoftDeleted_EarliestActiveMemberBecomesCoverAsync() + { + var (cover, b, c) = (Photo("a.heic", 0, softDeleted: true), Photo("b.heic", 7), Photo("c.heic", 5)); + var id = StoredBurst(60, cover, b, c); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(2, 0, 0, 0)); + StoredBursts().Should().ContainSingle().Which.Should().BeEquivalentTo(new { Id = id, PrimaryAssetId = c.Id }); + } + + [Fact] + public async Task RunAsync_CoverlessBurst_GetsItsEarliestMemberAsCoverAsync() + { + var (a, b) = (Photo("a.heic", 3), Photo("b.heic", 2)); + ClearCover(StoredBurst(60, a, b)); + + await RunAsync(); + + StoredBursts().Single().PrimaryAssetId.Should().Be(b.Id); + } + + [Fact] + public async Task RunAsync_AnotherConfigsBurst_IsLeftAloneAsync() + { + var (a, gone) = (Photo("a.heic", config: OtherConfigId), Photo("b.heic", 1, config: OtherConfigId, softDeleted: true)); + StoredBurst(60, a, gone); + + var result = await RunAsync(); + + result.Should().Be(new BurstDetectionResult(0, 0, 0, 0)); + StoredBursts().Should().ContainSingle(); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Bursts/OriginalPhotoReaderTests.cs b/src/Anichron.Worker.Tests.Unit/Bursts/OriginalPhotoReaderTests.cs new file mode 100644 index 00000000..1cbc0782 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Bursts/OriginalPhotoReaderTests.cs @@ -0,0 +1,90 @@ +using Anichron.Worker.Bursts; +using MetadataExtractor.Formats.Exif; +using MetadataExtractor.Formats.Exif.Makernotes; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using NodaTime; +using System.IO.Abstractions.TestingHelpers; +using System.Text; + +namespace Anichron.Worker.Tests.Unit.Bursts; + +public sealed class OriginalPhotoReaderTests +{ + private readonly FakeLogger logger = new(); + + // A JPEG whose EXIF holds only DateTimeOriginal: IFD0 points at a sub-IFD with that one tag. + private static byte[] JpegCapturedAt(string exifDate) + { + var tiff = new List(); + tiff.AddRange([0x49, 0x49, 0x2A, 0x00, 0x08, 0x00, 0x00, 0x00]); // little endian, IFD0 at 8 + tiff.AddRange([0x01, 0x00, 0x69, 0x87, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 26, 0x00, 0x00, 0x00, 0, 0, 0, 0]); // ExifOffset = 26 + tiff.AddRange([0x01, 0x00, 0x03, 0x90, 0x02, 0x00, 20, 0x00, 0x00, 0x00, 44, 0x00, 0x00, 0x00, 0, 0, 0, 0]); // DateTimeOriginal at 44 + tiff.AddRange(Encoding.ASCII.GetBytes(exifDate + "\0")); + var length = tiff.Count + 8; + return [0xFF, 0xD8, 0xFF, 0xE1, (byte)(length >> 8), (byte)length, .. "Exif\0\0"u8, .. tiff, 0xFF, 0xD9]; + } + + private OriginalPhotoFacts Read(MockFileSystem fileSystem) + => new OriginalPhotoReader(fileSystem, logger).Read("/data/originals/mira/a.jpg"); + + [Fact] + public void Read_PhotoWithACaptureTime_ReadsIt() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/data/originals/mira/a.jpg"] = new(JpegCapturedAt("2024:07:15 13:22:01")), + }); + + Read(fileSystem).Should().Be(new OriginalPhotoFacts(new LocalDateTime(2024, 7, 15, 13, 22, 1), null)); + logger.Collector.Count.Should().Be(0); + } + + [Fact] + public void Read_MissingOriginal_LogsAWarningAndKnowsNothing() + { + var facts = Read(new MockFileSystem()); + + facts.Should().Be(new OriginalPhotoFacts(null, null)); + logger.Collector.GetSnapshot().Should().ContainSingle().Which.Should().Match(r => + r.Level == LogLevel.Warning + && r.Message == "Could not read /data/originals/mira/a.jpg for burst detection; it will not be grouped." + && r.Exception is FileNotFoundException); + } + + [Fact] + public void Describe_AppleBurst_ReadsTheBurstUuidAndCaptureTime() + { + var subIfd = new ExifSubIfdDirectory(); + subIfd.Set(ExifDirectoryBase.TagDateTimeOriginal, "2023:12:24 18:00:05"); + var apple = new AppleMakernoteDirectory(); + apple.Set(AppleMakernoteDirectory.TagBurstUuid, "3F2504E0-4F89-11D3-9A0C-0305E82C3301"); + + OriginalPhotoReader.Describe([subIfd, apple]).Should().Be(new OriginalPhotoFacts( + new LocalDateTime(2023, 12, 24, 18, 0, 5), "3F2504E0-4F89-11D3-9A0C-0305E82C3301")); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Describe_BlankBurstUuid_IsNoLabel(string blank) + { + var apple = new AppleMakernoteDirectory(); + apple.Set(AppleMakernoteDirectory.TagBurstUuid, blank); + + OriginalPhotoReader.Describe([apple]).CameraBurstId.Should().BeNull(); + } + + [Fact] + public void Describe_NoExifOrMakerNote_KnowsNothing() + => OriginalPhotoReader.Describe([new ExifIfd0Directory()]).Should().Be(new OriginalPhotoFacts(null, null)); + + [Fact] + public void Describe_UnparsableCaptureTime_HasNone() + { + var subIfd = new ExifSubIfdDirectory(); + subIfd.Set(ExifDirectoryBase.TagDateTimeOriginal, " : : : : "); + + OriginalPhotoReader.Describe([subIfd]).ExifCapturedAt.Should().BeNull(); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs b/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs index 9a0c7c36..7f0e2605 100644 --- a/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs @@ -2,13 +2,8 @@ using Anichron.Core.Domain; using Anichron.Core.Work; using Anichron.Worker.Crawling; -using Anichron.Worker.Settings; -using Anichron.Worker.Work; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Testing; -using Microsoft.Extensions.Options; -using NodaTime; namespace Anichron.Worker.Tests.Unit.Crawling; @@ -20,6 +15,7 @@ public sealed class CrawlTests private readonly IFileIngestionPipeline pipeline = Substitute.For(); private readonly IWorkProgress progress = Substitute.For(); private readonly ISharedContentReporter reporter = Substitute.For(); + private readonly IWorkQueue queue = Substitute.For(); private readonly FakeLogger logger = new(); private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -33,7 +29,7 @@ public sealed class CrawlTests }; private Task RunAsync(Guid configId) - => new CrawlHandler(configs, pipeline, reporter, logger).RunAsync(new CrawlPayload(configId), progress, Ct); + => new CrawlHandler(configs, pipeline, reporter, queue, logger).RunAsync(new CrawlPayload(configId), progress, Ct); // ========================================================================== // CrawlHandler @@ -53,6 +49,7 @@ public async Task RunAsync_ActiveConfig_CrawlsItAndReturnsTheCountsWithItsDuplic var result = await RunAsync(config.Id); result.Should().Be(new CrawlResult(5, 4, 1) { Duplicates = new DuplicateCounts(2, 1) }); + await queue.Received(1).EnqueueAsync(new BurstDetectionPayload(config.Id), null, Ct); logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Level == LogLevel.Information && e.Message.Contains("/data/originals/mira")); } @@ -69,6 +66,7 @@ public async Task RunAsync_DuplicateReportFails_LogsItAndStillSucceedsWithoutThe var result = await RunAsync(config.Id); result.Should().Be(new CrawlResult(1, 1, 0) { Duplicates = null }); + await queue.Received(1).EnqueueAsync(new BurstDetectionPayload(config.Id), null, Ct); logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Level == LogLevel.Error) .Which.Should().Match(e => e.Message == $"Reporting duplicate content for storage config {config.Id} failed." @@ -89,11 +87,12 @@ public async Task RunAsync_ShutdownDuringTheDuplicateReport_PropagatesAsync() throw new OperationCanceledException(shutdown.Token); }); - var act = () => new CrawlHandler(configs, pipeline, reporter, logger) + var act = () => new CrawlHandler(configs, pipeline, reporter, queue, logger) .RunAsync(new CrawlPayload(config.Id), progress, shutdown.Token); await act.Should().ThrowAsync(); logger.Collector.GetSnapshot().Should().NotContain(e => e.Level == LogLevel.Error); + await queue.DidNotReceiveWithAnyArgs().EnqueueAsync(default!, default, Ct); } [Fact] @@ -106,6 +105,7 @@ public async Task RunAsync_InactiveConfig_SkipsWithoutCrawlingAsync() result.Should().Be(new CrawlSkipped("The storage config is missing or inactive.")); await pipeline.DidNotReceiveWithAnyArgs().RunAsync(default!, default!, Ct); + await queue.DidNotReceiveWithAnyArgs().EnqueueAsync(default!, default, Ct); logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Message.Contains(config.Id.ToString())); } @@ -130,6 +130,7 @@ public async Task RunAsync_RootMissing_FailsOnlyThisItemByThrowingAsync() await act.Should().ThrowAsync(); await reporter.DidNotReceiveWithAnyArgs().ReportAsync(default!, default!, Ct); + await queue.DidNotReceiveWithAnyArgs().EnqueueAsync(default!, default, Ct); } [Fact] @@ -142,56 +143,6 @@ public async Task RunAsync_ShutdownMidCrawl_PropagatesTheCancellationAsync() var act = () => RunAsync(config.Id); await act.Should().ThrowAsync(); - } - - // ========================================================================== - // CrawlSchedules - // ========================================================================== - - private static async Task> SchedulesAsync( - CrawlSchedules source, IUserStorageConfigRepository repository) - { - await using var services = new ServiceCollection().AddSingleton(repository).BuildServiceProvider(); - return await source.SchedulesAsync(services, Ct); - } - - [Fact] - public async Task SchedulesAsync_AllUserWorker_SchedulesEveryActiveConfigAtTheCrawlIntervalAsync() - { - var (a, b) = (Config(), Config()); - configs.GetAllActiveAsync(Ct).Returns([a, b]); - var source = new CrawlSchedules(new WorkerState(), Options.Create(new WorkerSettings { CrawlIntervalHours = 6 })); - - var schedules = await SchedulesAsync(source, configs); - - schedules.Select(s => (s.DedupKey, s.Interval)).Should().Equal( - ($"crawl:{a.Id}", Duration.FromHours(6)), ($"crawl:{b.Id}", Duration.FromHours(6))); - } - - [Fact] - public async Task SchedulesAsync_PerUserWorker_SchedulesOnlyItsUsersActiveConfigsAsync() - { - var mine = Config(); - configs.GetActiveByUserIdAsync(MiraId, Ct).Returns([mine]); - configs.GetAllActiveAsync(Ct).Returns([mine, Config()]); - var state = new WorkerState(); - var source = new CrawlSchedules(state, Options.Create(new WorkerSettings())); - state.ResolvedUserId = MiraId; - - var schedules = await SchedulesAsync(source, configs); - - schedules.Should().ContainSingle().Which.Should().BeEquivalentTo( - new { DedupKey = $"crawl:{mine.Id}", Interval = Duration.FromHours(4) }); - } - - [Fact] - public async Task CrawlSchedule_EnqueuesACrawlOfItsConfigAsync() - { - var configId = Guid.NewGuid(); - var queue = Substitute.For(); - - await new CrawlSchedule(configId, Duration.FromHours(4)).EnqueueAsync(queue, Ct); - - await queue.Received(1).EnqueueAsync(new CrawlPayload(configId), null, Ct); + await queue.DidNotReceiveWithAnyArgs().EnqueueAsync(default!, default, Ct); } } diff --git a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs index af90e4e9..77565b6a 100644 --- a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs @@ -3,6 +3,7 @@ using Anichron.Core.Data.Repository; using Anichron.Core.Domain; using Anichron.Core.Work; +using Anichron.Worker.Bursts; using Anichron.Worker.Crawling; using Anichron.Worker.Infrastructure; using Anichron.Worker.Ingestion; @@ -225,6 +226,7 @@ public void AddWorkQueueServices_RegistersEachKindsHandlerAsScoped() (typeof(IWorkHandler), typeof(TokenCleanupHandler)), (typeof(IWorkHandler), typeof(WorkItemRetentionHandler)), (typeof(IWorkHandler), typeof(CrawlHandler)), + (typeof(IWorkHandler), typeof(BurstDetectionHandler)), ]); } @@ -235,7 +237,8 @@ public void AddWorkQueueServices_RegistersEachPeriodicKindsSchedule() .Select(d => (d.ImplementationType, d.Lifetime)).Should().BeEquivalentTo([ (typeof(TokenCleanupSchedule), ServiceLifetime.Singleton), (typeof(WorkItemRetentionSchedule), ServiceLifetime.Singleton), - (typeof(CrawlSchedules), ServiceLifetime.Singleton), + (typeof(StorageConfigSchedules), ServiceLifetime.Singleton), + (typeof(StorageConfigSchedules), ServiceLifetime.Singleton), ]); [Fact] diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs index 67f8c73d..1089b10c 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs @@ -157,12 +157,22 @@ public async Task TryWriteAsync_KeepsTheStoredRowAndItsIdentityAsync() asset.Should().BeSameAs(fx.Stored); asset.Id.Should().Be(StoredId); - asset.BurstId.Should().Be(BurstId); asset.FilePath.Should().Be("2022/photo.jpg"); asset.Interactions.Should().Equal(interactions); fx.Repository.DidNotReceive().Add(Arg.Any()); } + [Fact] + public async Task TryWriteAsync_ClearsTheBurstMarkSoTheEditedPhotoIsCheckedAgainAsync() + { + var fx = new TestFixture(); + + var asset = await fx.WriteAsync(fx.Context()); + + asset!.BurstId.Should().BeNull(); + await fx.Repository.Received(1).RemoveBurstCheckAsync(StoredId, Arg.Any()); + } + [Fact] public async Task TryWriteAsync_DescribesTheNewContentAsync() { diff --git a/src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs b/src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs new file mode 100644 index 00000000..7e2371d3 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs @@ -0,0 +1,90 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Worker.Crawling; +using Anichron.Worker.Settings; +using Anichron.Worker.Work; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Work; + +public sealed class StorageConfigSchedulesTests +{ + private static readonly Guid MiraId = Guid.Parse("66666666-6666-6666-6666-666666666666"); + + private readonly IUserStorageConfigRepository configs = Substitute.For(); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static UserStorageConfig Config() => new() + { + Id = Guid.NewGuid(), + UserId = MiraId, + RootPath = "/data/originals/mira", + IsActive = true, + }; + + private async Task> SchedulesAsync(IWorkScheduleSource source) + { + await using var services = new ServiceCollection().AddSingleton(configs).BuildServiceProvider(); + return await source.SchedulesAsync(services, Ct); + } + + [Fact] + public async Task SchedulesAsync_AllUserWorker_SchedulesEveryActiveConfigAtTheCrawlIntervalAsync() + { + var (a, b) = (Config(), Config()); + configs.GetAllActiveAsync(Ct).Returns([a, b]); + var source = new StorageConfigSchedules( + new WorkerState(), Options.Create(new WorkerSettings { CrawlIntervalHours = 6 })); + + var schedules = await SchedulesAsync(source); + + schedules.Select(s => (s.DedupKey, s.Interval)).Should().Equal( + ($"crawl:{a.Id}", Duration.FromHours(6)), ($"crawl:{b.Id}", Duration.FromHours(6))); + } + + [Fact] + public async Task SchedulesAsync_PerUserWorker_SchedulesOnlyItsUsersActiveConfigsAsync() + { + var mine = Config(); + configs.GetActiveByUserIdAsync(MiraId, Ct).Returns([mine]); + configs.GetAllActiveAsync(Ct).Returns([mine, Config()]); + var state = new WorkerState(); + var source = new StorageConfigSchedules(state, Options.Create(new WorkerSettings())); + state.ResolvedUserId = MiraId; + + var schedules = await SchedulesAsync(source); + + schedules.Should().ContainSingle().Which.Should().BeEquivalentTo( + new { DedupKey = $"crawl:{mine.Id}", Interval = Duration.FromHours(4) }); + } + + [Fact] + public async Task SchedulesAsync_BurstDetection_SchedulesItPerConfigAtTheCrawlIntervalAsync() + { + var config = Config(); + configs.GetAllActiveAsync(Ct).Returns([config]); + var source = new StorageConfigSchedules( + new WorkerState(), Options.Create(new WorkerSettings { CrawlIntervalHours = 6 })); + + var schedules = await SchedulesAsync(source); + + schedules.Should().ContainSingle().Which.Should().BeEquivalentTo( + new { DedupKey = $"burst-detection:{config.Id}", Interval = Duration.FromHours(6) }); + } + + [Fact] + public async Task StorageConfigSchedule_EnqueuesItsPayloadAsync() + { + var configId = Guid.NewGuid(); + var queue = Substitute.For(); + + await new StorageConfigSchedule(new BurstDetectionPayload(configId), Duration.FromHours(4)) + .EnqueueAsync(queue, Ct); + + await queue.Received(1).EnqueueAsync(new BurstDetectionPayload(configId), null, Ct); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Work/WorkExecutorTests.cs b/src/Anichron.Worker.Tests.Unit/Work/WorkExecutorTests.cs index 341269d2..4094a4ff 100644 --- a/src/Anichron.Worker.Tests.Unit/Work/WorkExecutorTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Work/WorkExecutorTests.cs @@ -30,6 +30,11 @@ public sealed record Crawl : IWorkPayload public static WorkKind Kind => WorkKind.Crawl; } + public sealed record Bursts : IWorkPayload + { + public static WorkKind Kind => WorkKind.BurstDetection; + } + private static readonly TimeSpan Heartbeat = TimeSpan.FromSeconds(2); private static readonly TimeSpan Poll = TimeSpan.FromSeconds(5); private static readonly Duration Lease = Duration.FromSeconds(60); @@ -98,8 +103,11 @@ private async Task StartAsync(params ClaimedWork[] items) foreach (var item in items) queued.Enqueue(item); var registry = new WorkHandlerRegistry( - [WorkHandlerRegistration.For(), WorkHandlerRegistration.For(), WorkHandlerRegistration.For()], - [typeof(Cleanup), typeof(Retention), typeof(Crawl)]); + [ + WorkHandlerRegistration.For(), WorkHandlerRegistration.For(), + WorkHandlerRegistration.For(), WorkHandlerRegistration.For(), + ], + [typeof(Cleanup), typeof(Retention), typeof(Crawl), typeof(Bursts)]); executor = new WorkExecutor( store, registry, workerState, ScopeFactories.Providing(services => services.AddSingleton(handler)), timeline.Clock, Options.Create(new WorkerSettings()), logger); diff --git a/src/Anichron.Worker.Tests.Unit/Work/WorkHandlerRegistryTests.cs b/src/Anichron.Worker.Tests.Unit/Work/WorkHandlerRegistryTests.cs index d1edc1f7..978a3ced 100644 --- a/src/Anichron.Worker.Tests.Unit/Work/WorkHandlerRegistryTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Work/WorkHandlerRegistryTests.cs @@ -18,13 +18,14 @@ public sealed record NumberedRetentionPayload(int Number) : IWorkPayload } private static readonly Type[] RealPayloads = - [typeof(TokenCleanupPayload), typeof(WorkItemRetentionPayload), typeof(CrawlPayload)]; + [typeof(TokenCleanupPayload), typeof(WorkItemRetentionPayload), typeof(CrawlPayload), typeof(BurstDetectionPayload)]; private static WorkHandlerRegistration[] RealHandlers => [ WorkHandlerRegistration.For(), WorkHandlerRegistration.For(), WorkHandlerRegistration.For(), + WorkHandlerRegistration.For(), ]; private static Action Building(IEnumerable registrations, params Type[] payloads) @@ -47,12 +48,13 @@ public void Constructor_FromDependencyInjection_ChecksTheCorePayloads() act.Should().Throw() .Which.Message.Should().Be(string.Join(Environment.NewLine, "WorkKind.WorkItemRetention needs exactly one handler, found none.", - "WorkKind.Crawl needs exactly one handler, found none.")); + "WorkKind.Crawl needs exactly one handler, found none.", + "WorkKind.BurstDetection needs exactly one handler, found none.")); } [Fact] public void Constructor_KindWithoutPayload_FailsNamingIt() - => Building(RealHandlers, typeof(TokenCleanupPayload), typeof(CrawlPayload)).Should().Throw() + => Building(RealHandlers, typeof(TokenCleanupPayload), typeof(CrawlPayload), typeof(BurstDetectionPayload)).Should().Throw() .WithMessage("WorkKind.WorkItemRetention needs exactly one payload type, found none."); [Fact] @@ -68,7 +70,7 @@ public void Constructor_KindWithTwoHandlers_FailsNamingThem() [Fact] public void Constructor_HandlerForAnotherPayloadOfTheKind_FailsNamingBoth() - => Building([WorkHandlerRegistration.For(), RealHandlers[1], RealHandlers[2]], RealPayloads) + => Building([WorkHandlerRegistration.For(), .. RealHandlers[1..]], RealPayloads) .Should().Throw() .WithMessage("WorkKind.TokenCleanup's handler takes OtherCleanupPayload, not its payload TokenCleanupPayload."); @@ -78,7 +80,8 @@ public void Constructor_SeveralProblems_ReportsEachOnItsOwnLine() .Which.Message.Should().Be(string.Join(Environment.NewLine, "WorkKind.TokenCleanup needs exactly one handler, found none.", "WorkKind.WorkItemRetention needs exactly one handler, found none.", - "WorkKind.Crawl needs exactly one handler, found none.")); + "WorkKind.Crawl needs exactly one handler, found none.", + "WorkKind.BurstDetection needs exactly one handler, found none.")); [Fact] public async Task RunAsync_DispatchesTheDeserialisedPayloadToTheKindsHandlerAsync() @@ -89,8 +92,8 @@ public async Task RunAsync_DispatchesTheDeserialisedPayloadToTheKindsHandlerAsyn handler.RunAsync(new NumberedRetentionPayload(7), progress, cts.Token).Returns("summary"); await using var services = new ServiceCollection().AddSingleton(handler).BuildServiceProvider(); var registry = new WorkHandlerRegistry( - [RealHandlers[0], WorkHandlerRegistration.For(), RealHandlers[2]], - [typeof(TokenCleanupPayload), typeof(NumberedRetentionPayload), typeof(CrawlPayload)]); + [RealHandlers[0], WorkHandlerRegistration.For(), .. RealHandlers[2..]], + [typeof(TokenCleanupPayload), typeof(NumberedRetentionPayload), typeof(CrawlPayload), typeof(BurstDetectionPayload)]); var work = new ClaimedWork(new WorkAttempt(Guid.NewGuid(), 1), WorkKind.WorkItemRetention, /*lang=json,strict*/ """{"number":7}"""); var result = await registry.RunAsync(work, services, progress, cts.Token); diff --git a/src/Anichron.Worker/Bursts/BurstDetection.cs b/src/Anichron.Worker/Bursts/BurstDetection.cs new file mode 100644 index 00000000..5c18a180 --- /dev/null +++ b/src/Anichron.Worker/Bursts/BurstDetection.cs @@ -0,0 +1,167 @@ +using Anichron.Core; +using Anichron.Core.Data; +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Microsoft.EntityFrameworkCore; +using NodaTime; +using System.IO.Abstractions; + +namespace Anichron.Worker.Bursts; + +public sealed record BurstDetectionResult(int Checked, int Created, int Merged, int Dissolved); + +public sealed record BurstDetectionSkipped(string Reason); + +internal sealed partial class BurstDetectionHandler( + AnichronDbContext db, + IOriginalPhotoReader reader, + IFileSystem fileSystem, + IClock clock, + IGuidFactory guids, + ILogger logger) : IWorkHandler +{ + internal const int BatchSize = 100; + + public async Task RunAsync(BurstDetectionPayload payload, IWorkProgress progress, CancellationToken ct) + { + var config = await db.StorageConfigs.FirstOrDefaultAsync(c => c.Id == payload.StorageConfigId, ct); + if (config is not { IsActive: true }) + { + Log.Skipped(logger, payload.StorageConfigId); + return new BurstDetectionSkipped("The storage config is missing or inactive."); + } + + if (!fileSystem.Directory.Exists(config.RootPath)) + throw new DirectoryNotFoundException($"The storage config's root {config.RootPath} is missing."); + + var tally = new Tally(); + await RepairAsync(config.Id, tally, ct); + await CheckAsync(config, tally, progress, ct); + + var result = new BurstDetectionResult(tally.Checked, tally.Created, tally.Merged, tally.Dissolved); + Log.Finished(logger, config.Id, result.Checked, result.Created, result.Merged, result.Dissolved); + return result; + } + + private async Task RepairAsync(Guid configId, Tally tally, CancellationToken ct) + { + var bursts = await db.Bursts.IgnoreQueryFilters() + .Include(b => b.Assets) + .Where(b => b.Assets.All(a => a.StorageConfigId == configId)) + .ToListAsync(ct); + foreach (var burst in bursts) + { + if (burst.Assets.Count(a => !a.IsSoftDeleted) < 2) + { + db.Bursts.Remove(burst); + tally.Dissolved++; + } + else + { + burst.PrimaryAssetId = Earliest(burst.Assets).Id; + } + } + + await db.SaveChangesAsync(ct); + } + + private async Task CheckAsync(UserStorageConfig config, Tally tally, IWorkProgress progress, CancellationToken ct) + { + var pending = db.MediaAssets.Where(a => a.StorageConfigId == config.Id + && (a.MediaType == MediaType.Image || a.MediaType == MediaType.LivePhoto) + && !db.BurstChecks.Any(c => c.AssetId == a.Id)); + var total = await pending.CountAsync(ct); + + while (await pending + .Include(a => a.Metadata) + .Include(a => a.ProxyFiles) + .OrderBy(a => a.DateCaptured).ThenBy(a => a.Id) + .Take(BatchSize) + .ToListAsync(ct) is { Count: > 0 } batch) + { + foreach (var photo in batch) + { + await CheckAsync(config, photo, tally, ct); + progress.Report(++tally.Checked, total, "Checking photos for bursts"); + } + + db.ChangeTracker.Clear(); + } + } + + private async Task CheckAsync(UserStorageConfig config, MediaAsset photo, Tally tally, CancellationToken ct) + { + var facts = reader.Read(fileSystem.Path.Combine(config.RootPath, photo.FilePath)); + var check = new BurstCheck + { + AssetId = photo.Id, + CheckedAt = clock.GetCurrentInstant(), + IsCandidate = facts.ExifCapturedAt is not null + && photo.Metadata is { CameraMake: not null, CameraModel: not null } + && photo.ProxyFiles.Any(p => p.ProxyType == ProxyType.Thumbnail), + CameraBurstId = facts.CameraBurstId, + }; + db.BurstChecks.Add(check); + if (check is { IsCandidate: true, CameraBurstId: { } label }) + await JoinCameraBurstAsync(config.Id, photo, label, tally, ct); + + await db.SaveChangesAsync(ct); + } + + private async Task JoinCameraBurstAsync(Guid configId, MediaAsset photo, string label, Tally tally, CancellationToken ct) + { + var labelled = await db.BurstChecks + .Where(c => c.IsCandidate && c.CameraBurstId == label && c.Asset.StorageConfigId == configId) + .Select(c => c.Asset) + .ToListAsync(ct); + if (labelled.Count == 0) + return; + + var burstIds = labelled.Select(a => a.BurstId).OfType().ToList(); + var bursts = await db.Bursts.IgnoreQueryFilters() + .Include(b => b.Assets) + .Where(b => burstIds.Contains(b.Id)) + .OrderBy(b => b.CreatedAt).ThenBy(b => b.Id) + .ToListAsync(ct); + List members = [.. bursts.SelectMany(b => b.Assets), .. labelled, photo]; + + var target = bursts.FirstOrDefault(); + if (target is null) + { + target = new Burst { Id = guids.NewGuid(), CreatedAt = clock.GetCurrentInstant() }; + db.Bursts.Add(target); + tally.Created++; + } + + // Removing a burst sets its tracked members' BurstId to null, so they are moved after. + foreach (var merged in bursts.Skip(1)) + { + db.Bursts.Remove(merged); + tally.Merged++; + } + + foreach (var member in members) + member.BurstId = target.Id; + target.PrimaryAssetId = Earliest(members).Id; + } + + private static MediaAsset Earliest(IEnumerable members) + => members.Where(a => !a.IsSoftDeleted).MinBy(a => (a.DateCaptured, a.Id))!; + + private sealed class Tally + { + public int Checked { get; set; } + public int Created { get; set; } + public int Merged { get; set; } + public int Dissolved { get; set; } + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Information, Message = "Skipping burst detection for storage config {Id}: it is missing or inactive.")] + public static partial void Skipped(ILogger logger, Guid id); + + [LoggerMessage(Level = LogLevel.Information, Message = "Burst detection for storage config {Id} checked {Checked} photo(s): {Created} burst(s) created, {Merged} merged, {Dissolved} dissolved.")] + public static partial void Finished(ILogger logger, Guid id, int @checked, int created, int merged, int dissolved); + } +} diff --git a/src/Anichron.Worker/Bursts/OriginalPhotoReader.cs b/src/Anichron.Worker/Bursts/OriginalPhotoReader.cs new file mode 100644 index 00000000..ddb58664 --- /dev/null +++ b/src/Anichron.Worker/Bursts/OriginalPhotoReader.cs @@ -0,0 +1,56 @@ +using Anichron.Worker.Ingestion; +using MetadataExtractor; +using MetadataExtractor.Formats.Exif; +using MetadataExtractor.Formats.Exif.Makernotes; +using NodaTime; +using System.IO.Abstractions; + +namespace Anichron.Worker.Bursts; + +/// What burst detection reads from a photo's original. +/// The EXIF capture time, or null when the original has none. +/// Apple's BurstUUID, or null when the camera wrote none. +internal sealed record OriginalPhotoFacts(LocalDateTime? ExifCapturedAt, string? CameraBurstId) +{ + internal static OriginalPhotoFacts Unknown { get; } = new(null, null); +} + +internal interface IOriginalPhotoReader +{ + /// Reads the original once; an unreadable one is logged and yields no facts. + /// The original's path in the container. + OriginalPhotoFacts Read(string absolutePath); +} + +internal sealed partial class OriginalPhotoReader(IFileSystem fileSystem, ILogger logger) + : IOriginalPhotoReader +{ + public OriginalPhotoFacts Read(string absolutePath) + { + try + { + using var stream = fileSystem.File.OpenRead(absolutePath); + return Describe(ImageMetadataReader.ReadMetadata(stream)); + } + catch (Exception ex) + { + Log.Unreadable(logger, absolutePath, ex); + return OriginalPhotoFacts.Unknown; + } + } + + internal static OriginalPhotoFacts Describe(IReadOnlyList directories) + { + var captured = directories.OfType().FirstOrDefault()?.GetString(ExifDirectoryBase.TagDateTimeOriginal); + var burstId = directories.OfType().FirstOrDefault()?.GetString(AppleMakernoteDirectory.TagBurstUuid); + return new( + ExifDate.Parse(captured), + string.IsNullOrWhiteSpace(burstId) ? null : burstId); + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Warning, Message = "Could not read {Path} for burst detection; it will not be grouped.")] + public static partial void Unreadable(ILogger logger, string path, Exception exception); + } +} diff --git a/src/Anichron.Worker/Crawling/Crawl.cs b/src/Anichron.Worker/Crawling/Crawl.cs index e4a5bbbd..9ca266ea 100644 --- a/src/Anichron.Worker/Crawling/Crawl.cs +++ b/src/Anichron.Worker/Crawling/Crawl.cs @@ -2,10 +2,6 @@ using Anichron.Core.Domain; using Anichron.Core.Work; using Anichron.Worker.Ingestion; -using Anichron.Worker.Settings; -using Anichron.Worker.Work; -using Microsoft.Extensions.Options; -using NodaTime; namespace Anichron.Worker.Crawling; @@ -15,6 +11,7 @@ internal sealed partial class CrawlHandler( IUserStorageConfigRepository configs, IFileIngestionPipeline pipeline, ISharedContentReporter reporter, + IWorkQueue queue, ILogger logger) : IWorkHandler { public async Task RunAsync(CrawlPayload payload, IWorkProgress progress, CancellationToken ct) @@ -28,7 +25,9 @@ internal sealed partial class CrawlHandler( Log.Crawling(logger, config.RootPath, config.UserId); var counts = await pipeline.RunAsync(config, progress, ct); - return counts with { Duplicates = await ReportDuplicatesAsync(config, ct) }; + var result = counts with { Duplicates = await ReportDuplicatesAsync(config, ct) }; + await queue.EnqueueAsync(new BurstDetectionPayload(config.Id), null, ct); + return result; } private async Task ReportDuplicatesAsync(UserStorageConfig config, CancellationToken ct) @@ -57,19 +56,3 @@ private static partial class Log public static partial void Skipped(ILogger logger, Guid id); } } - -public sealed class CrawlSchedule(Guid storageConfigId, Duration interval) - : WorkSchedule(new CrawlPayload(storageConfigId), interval); - -public sealed class CrawlSchedules(WorkerState workerState, IOptions options) : IWorkScheduleSource -{ - public async Task> SchedulesAsync(IServiceProvider services, CancellationToken ct) - { - var repository = services.GetRequiredService(); - var served = workerState.ResolvedUserId is { } userId - ? await repository.GetActiveByUserIdAsync(userId, ct) - : await repository.GetAllActiveAsync(ct); - var interval = Duration.FromHours(options.Value.CrawlIntervalHours); - return [.. served.Select(config => new CrawlSchedule(config.Id, interval))]; - } -} diff --git a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs index 62cfbcb7..ff961e94 100644 --- a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs +++ b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs @@ -3,6 +3,7 @@ using Anichron.Core.Data.Repository; using Anichron.Core.Work; using Anichron.Infrastructure.Configuration; +using Anichron.Worker.Bursts; using Anichron.Worker.Crawling; using Anichron.Worker.Ingestion; using Anichron.Worker.Ingestion.Classification; @@ -58,9 +59,11 @@ internal IServiceCollection AddWorkQueueServices() services.AddWorkHandler(); services.AddWorkHandler(); services.AddWorkHandler(); + services.AddWorkHandler(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton>(); + services.AddSingleton>(); return services; } @@ -79,6 +82,7 @@ internal IServiceCollection AddIngestionServices() services.AddSingleton(); services.AddSingleton(); services.AddScoped(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddImageProxyServices(); diff --git a/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs b/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs index e8401b8e..02e64803 100644 --- a/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs +++ b/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs @@ -27,6 +27,8 @@ protected override async Task WriteAssetAsync(IngestionContext conte await repository.LoadMetadataAsync(asset, ct); DescribeContent(asset, asset.Metadata ??= new Metadata { AssetId = asset.Id }, context); ReplaceProxyFiles(asset, context.ProxyFilesOf(asset.Id)); + asset.BurstId = null; + await repository.RemoveBurstCheckAsync(asset.Id, ct); return asset; } diff --git a/src/Anichron.Worker/Ingestion/ExifDate.cs b/src/Anichron.Worker/Ingestion/ExifDate.cs new file mode 100644 index 00000000..39def3ed --- /dev/null +++ b/src/Anichron.Worker/Ingestion/ExifDate.cs @@ -0,0 +1,16 @@ +using NodaTime; +using NodaTime.Text; + +namespace Anichron.Worker.Ingestion; + +internal static class ExifDate +{ + // EXIF dates use colons as date separators ("2024:07:15 13:22:01"), unlike ISO 8601. + private static LocalDateTimePattern Pattern + => LocalDateTimePattern.CreateWithInvariantCulture("yyyy:MM:dd HH:mm:ss"); + + /// The date, or null when the raw value is missing or not an EXIF date. + /// An EXIF date-time tag's value. + internal static LocalDateTime? Parse(string? raw) + => Pattern.Parse(raw!) is { Success: true } result ? result.Value : null; +} diff --git a/src/Anichron.Worker/Ingestion/Middlewares/ExifExtractionMiddleware.cs b/src/Anichron.Worker/Ingestion/Middlewares/ExifExtractionMiddleware.cs index 2039fccb..0ee96d06 100644 --- a/src/Anichron.Worker/Ingestion/Middlewares/ExifExtractionMiddleware.cs +++ b/src/Anichron.Worker/Ingestion/Middlewares/ExifExtractionMiddleware.cs @@ -3,7 +3,6 @@ using MetadataExtractor.Formats.Exif; using MetadataExtractor.Formats.QuickTime; using NodaTime; -using NodaTime.Text; using System.IO.Abstractions; namespace Anichron.Worker.Ingestion.Middlewares; @@ -12,10 +11,6 @@ internal sealed partial class ExifExtractionMiddleware( IFileSystem fileSystem, ILogger logger) : IIngestionMiddleware { - // EXIF dates use colons as date separators ("2024:07:15 13:22:01"), unlike ISO 8601. - private static LocalDateTimePattern ExifDatePattern - => LocalDateTimePattern.CreateWithInvariantCulture("yyyy:MM:dd HH:mm:ss"); - public int Order => IngestionOrder.ExifExtraction; public bool CanInvoke(IngestionContext context) => true; @@ -113,9 +108,8 @@ private static int GetOrientation(ExifIfd0Directory? cameraInfo) var raw = captureDetails?.GetString(ExifDirectoryBase.TagDateTimeOriginal); if (raw is null) return null; - var result = ExifDatePattern.Parse(raw); - if (result.Success) - return result.Value; + if (ExifDate.Parse(raw) is { } date) + return date; Log.DateParseFailed(logger, raw); return null; diff --git a/src/Anichron.Worker/Work/StorageConfigSchedules.cs b/src/Anichron.Worker/Work/StorageConfigSchedules.cs new file mode 100644 index 00000000..bdf90132 --- /dev/null +++ b/src/Anichron.Worker/Work/StorageConfigSchedules.cs @@ -0,0 +1,31 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Work; +using Anichron.Worker.Crawling; +using Anichron.Worker.Settings; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Work; + +public sealed class StorageConfigSchedule(TPayload payload, Duration interval) + : WorkSchedule(payload, interval) + where TPayload : IWorkPayload; + +/// One schedule per active storage config the Worker serves, at the crawl interval. +/// The per-config kind scheduled. +/// Names the user a per-user Worker serves. +/// Supplies the crawl interval. +public sealed class StorageConfigSchedules(WorkerState workerState, IOptions options) + : IWorkScheduleSource + where TPayload : IStorageConfigPayload +{ + public async Task> SchedulesAsync(IServiceProvider services, CancellationToken ct) + { + var repository = services.GetRequiredService(); + var served = workerState.ResolvedUserId is { } userId + ? await repository.GetActiveByUserIdAsync(userId, ct) + : await repository.GetAllActiveAsync(ct); + var interval = Duration.FromHours(options.Value.CrawlIntervalHours); + return [.. served.Select(config => new StorageConfigSchedule(TPayload.For(config.Id), interval))]; + } +} From ad77211bfefda2be2811aa47400ba24c34847d84 Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Fri, 23 Oct 2026 20:55:29 +1300 Subject: [PATCH 2/4] refactor(worker): Drop a rationale comment from BurstDetectionHandler (#297) The members are moved after the merged bursts are removed because removing a burst sets its tracked members' BurstId to null immediately; the merge tests pin that order. Co-Authored-By: Claude Opus 5.5 --- src/Anichron.Worker/Bursts/BurstDetection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Anichron.Worker/Bursts/BurstDetection.cs b/src/Anichron.Worker/Bursts/BurstDetection.cs index 5c18a180..dca54146 100644 --- a/src/Anichron.Worker/Bursts/BurstDetection.cs +++ b/src/Anichron.Worker/Bursts/BurstDetection.cs @@ -133,7 +133,6 @@ private async Task JoinCameraBurstAsync(Guid configId, MediaAsset photo, string tally.Created++; } - // Removing a burst sets its tracked members' BurstId to null, so they are moved after. foreach (var merged in bursts.Skip(1)) { db.Bursts.Remove(merged); From 6549149a45a76aba482e13d7c2b4d31e40c2de86 Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Fri, 23 Oct 2026 21:04:42 +1300 Subject: [PATCH 3/4] test(worker): Pin that a moved file keeps its burst (#297) Co-Authored-By: Claude Opus 5.5 --- .../KnownFileReconcilerTests.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/Anichron.Worker.Tests.Unit/Ingestion/Classification/KnownFileReconcilerTests.cs diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/KnownFileReconcilerTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/KnownFileReconcilerTests.cs new file mode 100644 index 00000000..cd4991e7 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/KnownFileReconcilerTests.cs @@ -0,0 +1,26 @@ +using Anichron.Core.Domain; +using Anichron.Worker.Ingestion.Classification; +using Microsoft.Extensions.Logging.Testing; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Ingestion.Classification; + +public sealed class KnownFileReconcilerTests +{ + private static readonly Instant Now = Instant.FromUtc(2026, 10, 23, 9, 0); + + [Fact] + public void Reconcile_MovedFile_KeepsItsBurst() + { + var burstId = Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"); + var stored = new MediaAsset { Id = Guid.NewGuid(), FilePath = "2024/a.heic", FileName = "a.heic", BurstId = burstId }; + var clock = Substitute.For(); + clock.GetCurrentInstant().Returns(Now); + + var reconciled = new KnownFileReconciler(clock, new DuplicateCopyTally(), new FakeLogger()) + .Reconcile(new FileClassification(FileOutcome.Moved, stored), "2025/a.heic"); + + reconciled.Should().BeTrue(); + stored.Should().BeEquivalentTo(new { FilePath = "2025/a.heic", BurstId = (Guid?)burstId, LastSeenOnNas = Now }); + } +} From f1101c6ab5848610b7c0f4fe51f4d07b46b89547 Mon Sep 17 00:00:00 2001 From: bitbiter-dev Date: Fri, 23 Oct 2026 21:29:21 +1300 Subject: [PATCH 4/4] refactor(worker): Burst detection through IBurstRepository, shared config guard (#297) - IBurstRepository (Core) holds every burst query and the burst-checked mark, so the handler no longer touches AnichronDbContext and BurstChecks is written through one route (ContentChangedWriter clears the mark through it too). The includes for a burst's members live in one place. - ActiveStorageConfigs is the one "missing or inactive config" guard for per-config handlers; Crawl and BurstDetection both return StorageConfigSkipped. CrawlSkipped is gone: same JSON shape, one type. - BurstDetectionResult is the run's counter itself; the private Tally that mirrored it is gone. - WorkSchedule is no longer abstract, so StorageConfigSchedules builds it directly and the empty StorageConfigSchedule subclass is gone. - ExifDate.Parse handles a missing value explicitly instead of relying on NodaTime returning a failed parse for null; tests pin null, empty and ISO input. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- .../Data/Repository/BurstRepositoryTests.cs | 40 ++++++ .../Repository/MediaAssetRepositoryTests.cs | 28 ----- .../Data/Repository/BurstRepository.cs | 108 ++++++++++++++++ .../Data/Repository/MediaAssetRepository.cs | 11 -- .../Bursts/BurstDetectionTests.cs | 43 ++++--- .../Crawling/CrawlTests.cs | 13 +- .../WorkerServiceCollectionExtensionsTests.cs | 2 + .../ContentChangedWriterTests.cs | 5 +- .../Ingestion/ExifDateTests.cs | 19 +++ .../Work/StorageConfigSchedulesTests.cs | 9 +- src/Anichron.Worker/Bursts/BurstDetection.cs | 116 +++++++----------- src/Anichron.Worker/Crawling/Crawl.cs | 15 +-- .../WorkerServiceCollectionExtensions.cs | 2 + .../Classification/ContentChangedWriter.cs | 3 +- src/Anichron.Worker/Ingestion/ExifDate.cs | 2 +- .../Work/ActiveStorageConfigs.cs | 35 ++++++ .../Work/StorageConfigSchedules.cs | 6 +- src/Anichron.Worker/Work/WorkSchedule.cs | 2 +- 19 files changed, 300 insertions(+), 161 deletions(-) create mode 100644 src/Anichron.Core.Tests.Unit/Data/Repository/BurstRepositoryTests.cs create mode 100644 src/Anichron.Core/Data/Repository/BurstRepository.cs create mode 100644 src/Anichron.Worker.Tests.Unit/Ingestion/ExifDateTests.cs create mode 100644 src/Anichron.Worker/Work/ActiveStorageConfigs.cs diff --git a/CLAUDE.md b/CLAUDE.md index f67eabb4..787e76b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -170,7 +170,7 @@ The API reads files only through two narrow stores in `Anichron.Infrastructure/S - Image proxy sizes are edge-based, so portrait and landscape get the same treatment: preview long edge `PreviewLongEdge` (1920), thumbnail short edge `ThumbnailShortEdge` (300), blurhash sample long edge `BlurhashSampleLongEdge` (64). Nothing is ever upscaled - Each `IImageProxyGenerator`, and the `BlurhashEncoder`, declares a `ProxyBound`; `ImageProxyMiddleware` decodes once to `Rgb24`, shrinks the shared image to each bound largest-first, and hands every step **its own copy** at that size. The encoder's string goes on the `IngestionContext` and the content writer (`ProcessedContentWriter.DescribeContent`) writes it to `Metadata`. Generators may crop/recolour/resize their copy freely; only the middleware resizes the shared image - FFmpeg transcodes video with runtime GPU detection: QuickSync (`h264_qsv`) → NVENC (`h264_nvenc`) → AMF (`h264_amf`) → software (`libx264`) -- Burst detection (#297): a `BurstDetection` work item per storage config, enqueued by the config's `Crawl` when it finishes and scheduled at the crawl interval as a fallback (one dedup key, so never twice). `BurstDetectionHandler` first repairs the config's bursts (fewer than 2 active members: deleted; otherwise the earliest active member becomes the cover; memberless bursts are deleted too), then checks each unchecked photo (`Image` or `LivePhoto`, never `Video`, so a paired MOV never joins) oldest first, in batches: it reads the original once through `IOriginalPhotoReader` (EXIF capture time, Apple `BurstUUID`) and writes its `BurstCheck` row. A candidate (EXIF capture time, camera make and model, a thumbnail) joins every checked candidate of the config with the same `BurstUUID`, regardless of time, creating or merging bursts; each photo saves with its burst changes. A missing root fails the item before anything is marked. Similarity grouping is #298 +- Burst detection (#297): a `BurstDetection` work item per storage config, enqueued by the config's `Crawl` when it finishes and scheduled at the crawl interval as a fallback (one dedup key, so never twice). `BurstDetectionHandler` reads and writes through `IBurstRepository` (Core). It first repairs the config's bursts (fewer than 2 active members: deleted; otherwise the earliest active member becomes the cover; memberless bursts are deleted too), then checks each unchecked photo (`Image` or `LivePhoto`, never `Video`, so a paired MOV never joins) oldest first, in batches: it reads the original once through `IOriginalPhotoReader` (EXIF capture time, Apple `BurstUUID`) and writes its `BurstCheck` row. A candidate (EXIF capture time, camera make and model, a thumbnail) joins every checked candidate of the config with the same `BurstUUID`, regardless of time, creating or merging bursts; each photo saves with its burst changes. A missing root fails the item before anything is marked. Per-config handlers skip a missing or inactive config through `ActiveStorageConfigs`, returning `StorageConfigSkipped`. Similarity grouping is #298 - Reconciliation (**planned, Epic 4**): a periodic scan soft-deletes assets whose file is gone (interactions survive) and restores them when it reappears (#33, #34, #37). Moves are already handled at ingest (#35): when the hash matches an active row, `KnownFileReconciler` refreshes `LastSeenOnNas` if the path is unchanged (`Unchanged`), moves the row to the new path if its stored file is gone (`Moved`), and leaves it alone if the stored file still exists (`DuplicateCopy`, reported by #36). A Live Photo's MOV row gets the same treatment by its own hash. A move onto a path another row already holds, soft-deleted rows included (`PathTaken`), is logged as a warning and skipped; if a concurrent consumer takes the path after classification, the save fails and the item is retried on the next crawl. A file **edited in place** (hash unknown, but an active row holds the path: `ContentChanged`) is processed like new content and `ContentChangedWriter` updates that row: new hash, capture date, media type, `Metadata` and proxy rows (updated by type), while `Id`, interactions and `PairedAssetId` are kept. Its burst mark is cleared (`BurstId` null, `BurstCheck` row deleted), so the next burst detection checks it again; a moved file keeps both. A soft-deleted row at the path still blocks it (restore, #34). An edited Live Photo **MOV** keeps its stored row unchanged for now - Duplicate content is **reported, never merged** (#36). `SharedContentReporter` runs at the end of each config's `Crawl` item: it logs one `Information` summary for the crawled config (copies within it, counted by `DuplicateCopyTally` during that crawl in this Worker, and pairs of its active rows sharing a `ContentHash` with the same user's other active configs), with each pair at `Debug`, and the counts go into the crawl's result. A pair across two configs is reported by each of their crawls. A failed report is logged and does not fail the crawl. Duplicates across different users are expected and never reported. Two new identical copies racing in one crawl are first reported on the next crawl - Failure ownership: `ProxyStagingWriter` is the single place a proxy reaches its final name (architecture rule `proxies-reach-final-name-only-in-staging-writer`) — it writes under a per-attempt `..tmp` suffix (so concurrent attempts on the same content never share a temporary file), renames into place once complete, and registers the proxy on the `IngestionContext` at that moment; the proxy middlewares only supply the bytes. If a step throws, `IngestionPipelineRunner` deletes exactly those registered proxies plus its own temporary files, and removes the directory only if it is left empty — never recursively, since a concurrent attempt on the same content shares it until one of them persists. An attempt that loses a uniqueness race (`LostIngestionRace`: the active content-hash index or the one-proxy-per-asset-and-type index) first releases its paired MOV's registrations, because the winner owns those content-derived files. A clean shutdown is treated differently: the in-flight temporary file is dropped, but proxies already completed in that run are kept, and the consumer loop re-throws the cancellation instead of recording an item failure and draining the queue. Each attempt has its own `IngestionContext.AttemptId`, which names temporary files and scopes compensation; `IngestionContext.AssetId` is the row that owns the proxies, set by classification (the stored row for `ContentChanged`, the attempt id for new content). A hard kill, the storage-config cascade delete, and the old content directory of a file edited in place are out of reach in-process and belong to the orphan sweeper (#174); the old directory is deliberately not deleted after commit, because a copy of the old content elsewhere can be ingested into it in the same crawl. diff --git a/src/Anichron.Core.Tests.Unit/Data/Repository/BurstRepositoryTests.cs b/src/Anichron.Core.Tests.Unit/Data/Repository/BurstRepositoryTests.cs new file mode 100644 index 00000000..33486862 --- /dev/null +++ b/src/Anichron.Core.Tests.Unit/Data/Repository/BurstRepositoryTests.cs @@ -0,0 +1,40 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; +using Anichron.Testing.Data; +using Microsoft.EntityFrameworkCore; + +namespace Anichron.Core.Tests.Unit.Data.Repository; + +public sealed class BurstRepositoryTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task RemoveCheckAsync_CheckedPhoto_DeletesItsRowAtTheNextSaveAsync() + { + var name = Guid.NewGuid().ToString(); + var asset = new MediaAsset { Id = Guid.NewGuid(), StorageConfigId = Guid.NewGuid(), FilePath = "a.jpg", FileName = "a.jpg", ContentHash = "cafe" }; + await using (var seed = AnichronDbContexts.InMemory(name)) + { + seed.MediaAssets.Add(asset); + seed.BurstChecks.Add(new BurstCheck { AssetId = asset.Id, IsCandidate = true }); + await seed.SaveChangesAsync(Ct); + } + + await using var db = AnichronDbContexts.InMemory(name); + await new EfBurstRepository(db).RemoveCheckAsync(asset.Id, Ct); + await db.SaveChangesAsync(Ct); + + (await db.BurstChecks.CountAsync(Ct)).Should().Be(0); + } + + [Fact] + public async Task RemoveCheckAsync_UncheckedPhoto_ChangesNothingAsync() + { + await using var db = AnichronDbContexts.InMemory(); + + await new EfBurstRepository(db).RemoveCheckAsync(Guid.NewGuid(), Ct); + + db.ChangeTracker.HasChanges().Should().BeFalse(); + } +} diff --git a/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs b/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs index cb05a8d8..d36d10da 100644 --- a/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs +++ b/src/Anichron.Core.Tests.Unit/Data/Repository/MediaAssetRepositoryTests.cs @@ -178,34 +178,6 @@ public async Task LoadMetadataAsync_LoadsTheAssetsMetadataRow() tracked.Metadata.Width.Should().Be(640); } - [Fact] - public async Task RemoveBurstCheckAsync_CheckedPhoto_TracksItsRowForDeleteAsync() - { - var ct = TestContext.Current.CancellationToken; - var asset = AssetAt(ConfigA, "cafe", "a.jpg"); - await using var seed = CreateDb(out var name); - seed.MediaAssets.Add(asset); - seed.BurstChecks.Add(new BurstCheck { AssetId = asset.Id, IsCandidate = true }); - await seed.SaveChangesAsync(ct); - await using var db = AnichronDbContexts.InMemory(name); - - await new EfMediaAssetRepository(db).RemoveBurstCheckAsync(asset.Id, ct); - await db.SaveChangesAsync(ct); - - (await db.BurstChecks.CountAsync(ct)).Should().Be(0); - } - - [Fact] - public async Task RemoveBurstCheckAsync_UncheckedPhoto_ChangesNothingAsync() - { - var ct = TestContext.Current.CancellationToken; - await using var db = CreateDb(out _); - - await new EfMediaAssetRepository(db).RemoveBurstCheckAsync(Guid.NewGuid(), ct); - - db.ChangeTracker.HasChanges().Should().BeFalse(); - } - // ========================================================================== // FindSharedContentAsync // ========================================================================== diff --git a/src/Anichron.Core/Data/Repository/BurstRepository.cs b/src/Anichron.Core/Data/Repository/BurstRepository.cs new file mode 100644 index 00000000..9e92bf18 --- /dev/null +++ b/src/Anichron.Core/Data/Repository/BurstRepository.cs @@ -0,0 +1,108 @@ +using Anichron.Core.Domain; +using Microsoft.EntityFrameworkCore; + +namespace Anichron.Core.Data.Repository; + +/// Bursts and the burst-checked mark, written only by the Worker's burst detection. +public interface IBurstRepository +{ + /// + /// The config's bursts, and bursts with no members left, each with every member, soft-deleted + /// ones included. + /// + /// The config whose bursts are loaded. + /// Cancels the query. + Task> GetInConfigWithMembersAsync(Guid storageConfigId, CancellationToken ct); + + /// The bursts with every member, oldest first (then by id). + /// The bursts to load. + /// Cancels the query. + Task> GetWithMembersAsync(IReadOnlyCollection burstIds, CancellationToken ct); + + Task CountUncheckedPhotosAsync(Guid storageConfigId, CancellationToken ct); + + /// + /// The config's oldest active photos without a burst-checked mark, with their metadata and + /// proxy rows. + /// + /// The config whose photos are loaded. + /// At most this many. + /// Cancels the query. + Task> GetUncheckedPhotosAsync(Guid storageConfigId, int count, CancellationToken ct); + + /// The config's checked candidates the camera labelled with this burst id. + /// The config searched. + /// Apple's BurstUUID. + /// Cancels the query. + Task> GetCandidatesLabelledAsync(Guid storageConfigId, string cameraBurstId, CancellationToken ct); + + void Add(Burst burst); + void Remove(Burst burst); + void AddCheck(BurstCheck check); + + /// Clears the asset's burst-checked mark, if it has one, at the next save. + /// The asset whose mark is cleared. + /// Cancels the lookup. + Task RemoveCheckAsync(Guid assetId, CancellationToken ct); + + /// Stops tracking everything loaded so far, so a long run does not hold it all. + void ForgetLoaded(); +} + +public sealed class EfBurstRepository(AnichronDbContext db) : IBurstRepository +{ + public async Task> GetInConfigWithMembersAsync(Guid storageConfigId, CancellationToken ct) + => await WithMembers() + .Where(b => b.Assets.All(a => a.StorageConfigId == storageConfigId)) + .ToListAsync(ct); + + public async Task> GetWithMembersAsync(IReadOnlyCollection burstIds, CancellationToken ct) + => await WithMembers() + .Where(b => burstIds.Contains(b.Id)) + .OrderBy(b => b.CreatedAt).ThenBy(b => b.Id) + .ToListAsync(ct); + + public Task CountUncheckedPhotosAsync(Guid storageConfigId, CancellationToken ct) + => UncheckedPhotos(storageConfigId).CountAsync(ct); + + public async Task> GetUncheckedPhotosAsync(Guid storageConfigId, int count, CancellationToken ct) + => await UncheckedPhotos(storageConfigId) + .Include(a => a.Metadata) + .Include(a => a.ProxyFiles) + .OrderBy(a => a.DateCaptured).ThenBy(a => a.Id) + .Take(count) + .ToListAsync(ct); + + public async Task> GetCandidatesLabelledAsync( + Guid storageConfigId, string cameraBurstId, CancellationToken ct) + => await db.BurstChecks + .Where(c => c.IsCandidate && c.CameraBurstId == cameraBurstId && c.Asset.StorageConfigId == storageConfigId) + .Select(c => c.Asset) + .ToListAsync(ct); + + public void Add(Burst burst) + => db.Bursts.Add(burst); + + public void Remove(Burst burst) + => db.Bursts.Remove(burst); + + public void AddCheck(BurstCheck check) + => db.BurstChecks.Add(check); + + public async Task RemoveCheckAsync(Guid assetId, CancellationToken ct) + { + if (await db.BurstChecks.FindAsync([assetId], ct) is { } check) + db.BurstChecks.Remove(check); + } + + public void ForgetLoaded() + => db.ChangeTracker.Clear(); + + private IQueryable WithMembers() + => db.Bursts.IgnoreQueryFilters().Include(b => b.Assets); + + private IQueryable UncheckedPhotos(Guid storageConfigId) + => db.MediaAssets.Where(a => a.StorageConfigId == storageConfigId + && (a.MediaType == MediaType.Image || a.MediaType == MediaType.LivePhoto) + && !db.BurstChecks.Any(c => c.AssetId == a.Id)); +} diff --git a/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs b/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs index d0ae5f48..9d0b92f5 100644 --- a/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs +++ b/src/Anichron.Core/Data/Repository/MediaAssetRepository.cs @@ -27,11 +27,6 @@ Task> FindByHashesOrPathsAsync( Task LoadMetadataAsync(MediaAsset asset, CancellationToken ct); - /// Clears the asset's burst-checked mark, if it has one, at the next save. - /// The asset whose mark is cleared. - /// Cancels the lookup. - Task RemoveBurstCheckAsync(Guid assetId, CancellationToken ct); - void Add(MediaAsset asset); void AddProxyFiles(IEnumerable proxyFiles); void RemoveProxyFile(ProxyFile proxyFile); @@ -75,12 +70,6 @@ internal static IQueryable SharedContentQuery( public Task LoadMetadataAsync(MediaAsset asset, CancellationToken ct) => db.Entry(asset).Reference(a => a.Metadata).LoadAsync(ct); - public async Task RemoveBurstCheckAsync(Guid assetId, CancellationToken ct) - { - if (await db.BurstChecks.FindAsync([assetId], ct) is { } check) - db.BurstChecks.Remove(check); - } - public void Add(MediaAsset asset) => db.MediaAssets.Add(asset); diff --git a/src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs b/src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs index 8de250af..82ef208b 100644 --- a/src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Bursts/BurstDetectionTests.cs @@ -1,9 +1,11 @@ using Anichron.Core; using Anichron.Core.Data; +using Anichron.Core.Data.Repository; using Anichron.Core.Domain; using Anichron.Core.Work; using Anichron.Testing.Data; using Anichron.Worker.Bursts; +using Anichron.Worker.Work; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Testing; @@ -32,6 +34,7 @@ public sealed class BurstDetectionTests : IAsyncDisposable private readonly IGuidFactory guids = Substitute.For(); private readonly IWorkProgress progress = Substitute.For(); private readonly FakeLogger logger = new(); + private readonly FakeLogger activeLogger = new(); private int nextGuid; public BurstDetectionTests() @@ -117,7 +120,9 @@ private void ClearCover(Guid burstId) private Task RunAsync() => RunForAsync(ConfigId, Ct); private Task RunForAsync(Guid configId, CancellationToken ct) - => new BurstDetectionHandler(db, reader, fileSystem, clock, guids, logger) + => new BurstDetectionHandler( + new ActiveStorageConfigs(new EfUserStorageConfigRepository(db), activeLogger), + new EfBurstRepository(db), db, reader, fileSystem, clock, guids, logger) .RunAsync(new BurstDetectionPayload(configId), progress, ct); private Dictionary BurstsOf(params MediaAsset[] photos) @@ -153,16 +158,16 @@ public async Task RunAsync_InactiveConfig_SkipsWithoutReadingAnythingAsync() var result = await RunForAsync(inactive, Ct); - result.Should().Be(new BurstDetectionSkipped("The storage config is missing or inactive.")); + result.Should().Be(StorageConfigSkipped.MissingOrInactive); reader.DidNotReceiveWithAnyArgs().Read(default!); StoredChecks().Should().BeEmpty(); - logger.Collector.GetSnapshot().Should().ContainSingle().Which.Message.Should().Be( - $"Skipping burst detection for storage config {inactive}: it is missing or inactive."); + activeLogger.Collector.GetSnapshot().Should().ContainSingle().Which.Message.Should().Be( + $"Skipping BurstDetection of storage config {inactive}: it is missing or inactive."); } [Fact] public async Task RunAsync_ConfigGone_SkipsAsync() - => (await RunForAsync(Guid.NewGuid(), Ct)).Should().BeOfType(); + => (await RunForAsync(Guid.NewGuid(), Ct)).Should().BeOfType(); [Fact] public async Task RunAsync_RootMissing_FailsWithoutMarkingAnyPhotoAsync() @@ -189,7 +194,7 @@ public async Task RunAsync_PhotosWithOneBurstUuid_FormOneBurstWithTheEarliestAsC var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(Checked: 3, Created: 1, Merged: 0, Dissolved: 0)); + result.Should().Be(new BurstDetectionResult { Checked = 3, Created = 1 }); var burst = StoredBursts().Should().ContainSingle().Which; burst.Should().BeEquivalentTo(new { PrimaryAssetId = b.Id, CreatedAt = Now }); BurstsOf(a, b, c).Values.Should().AllBeEquivalentTo(burst.Id); @@ -218,7 +223,7 @@ public async Task RunAsync_TwoBurstUuids_TwoBurstsAndUnlabelledPhotosStayAloneAs var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(5, 2, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 5, Created = 2 }); var bursts = BurstsOf(a1, a2, b1, b2, plain); var (burstA, burstB) = (bursts["a1.heic"], bursts["b1.heic"]); new[] { burstA, burstB }.Should().OnlyHaveUniqueItems().And.NotContainNulls(); @@ -240,7 +245,7 @@ public async Task RunAsync_LonePhotoWithABurstUuid_FormsNoBurstAsync() var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(1, 0, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 1 }); StoredBursts().Should().BeEmpty(); BurstsOf(a)["a.heic"].Should().BeNull(); } @@ -270,7 +275,7 @@ public async Task RunAsync_LivePhotoHalfJoins_VideosAndPairedMovsAreNeverChecked var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(2, 1, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 2, Created = 1 }); BurstsOf(still, live, mov).Should().BeEquivalentTo(new Dictionary { ["a.jpg"] = StoredBursts().Single().Id, @@ -325,7 +330,7 @@ public async Task RunAsync_UnreadableOriginal_IsCheckedAsNoCandidateAsync() var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(1, 0, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 1 }); StoredChecks().Should().ContainSingle().Which.Should().BeEquivalentTo( new BurstCheck { AssetId = a.Id, CheckedAt = Now, IsCandidate = false, CameraBurstId = null }, o => o.Excluding(c => c.Asset)); @@ -346,7 +351,7 @@ public async Task RunAsync_MarksEachPhotoOnceAndASecondRunChecksNothingAsync() var second = await RunAsync(); - second.Should().Be(new BurstDetectionResult(0, 0, 0, 0)); + second.Should().Be(new BurstDetectionResult()); reader.DidNotReceiveWithAnyArgs().Read(default!); StoredChecks().Should().BeEquivalentTo( [ @@ -368,7 +373,7 @@ public async Task RunAsync_NewPhotoWithTheLabelOfAStoredBurst_JoinsItAndBecomesI var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(1, 0, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 1 }); BurstsOf(a, b, earlier).Values.Should().AllBeEquivalentTo(stored); StoredBursts().Should().ContainSingle().Which.PrimaryAssetId.Should().Be(earlier.Id); reader.DidNotReceive().Read($"{Root}/{a.FilePath}"); @@ -384,7 +389,7 @@ public async Task RunAsync_NewPhotoWithTheLabelOfACheckedLonePhoto_FormsABurstWi var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(1, 1, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 1, Created = 1 }); var burst = StoredBursts().Should().ContainSingle().Which; burst.PrimaryAssetId.Should().Be(fresh.Id); BurstsOf(lone, fresh).Values.Should().AllBeEquivalentTo(burst.Id); @@ -405,7 +410,7 @@ public async Task RunAsync_NewPhotoJoiningTwoStoredBursts_MergesThemIntoTheOlder var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(1, 0, 1, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 1, Merged = 1 }); StoredBursts().Should().ContainSingle().Which.Should().BeEquivalentTo(new { Id = older, PrimaryAssetId = c.Id }); BurstsOf(a, b, c, d, fresh).Values.Should().AllBeEquivalentTo(older); } @@ -478,7 +483,7 @@ public async Task RunAsync_MoreThanOneBatch_ChecksEveryPhotoAndKeepsNoneTrackedA var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(BurstDetectionHandler.BatchSize + 1, 1, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = BurstDetectionHandler.BatchSize + 1, Created = 1 }); StoredChecks().Should().HaveCount(BurstDetectionHandler.BatchSize + 1); db.ChangeTracker.Entries().Should().BeEmpty(); } @@ -526,7 +531,7 @@ public async Task RunAsync_BurstLeftWithOneActiveMember_IsDissolvedAsync() var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(1, 0, 0, 1)); + result.Should().Be(new BurstDetectionResult { Checked = 1, Dissolved = 1 }); StoredBursts().Should().BeEmpty(); BurstsOf(a, gone).Values.Should().AllSatisfy(id => id.Should().BeNull()); } @@ -538,7 +543,7 @@ public async Task RunAsync_BurstWithoutMembers_IsDeletedAsync() var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(0, 0, 0, 1)); + result.Should().Be(new BurstDetectionResult { Dissolved = 1 }); StoredBursts().Should().BeEmpty(); } @@ -550,7 +555,7 @@ public async Task RunAsync_CoverSoftDeleted_EarliestActiveMemberBecomesCoverAsyn var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(2, 0, 0, 0)); + result.Should().Be(new BurstDetectionResult { Checked = 2 }); StoredBursts().Should().ContainSingle().Which.Should().BeEquivalentTo(new { Id = id, PrimaryAssetId = c.Id }); } @@ -573,7 +578,7 @@ public async Task RunAsync_AnotherConfigsBurst_IsLeftAloneAsync() var result = await RunAsync(); - result.Should().Be(new BurstDetectionResult(0, 0, 0, 0)); + result.Should().Be(new BurstDetectionResult()); StoredBursts().Should().ContainSingle(); } } diff --git a/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs b/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs index 7f0e2605..5ac4f3d8 100644 --- a/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Crawling/CrawlTests.cs @@ -2,6 +2,7 @@ using Anichron.Core.Domain; using Anichron.Core.Work; using Anichron.Worker.Crawling; +using Anichron.Worker.Work; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Testing; @@ -17,6 +18,7 @@ public sealed class CrawlTests private readonly ISharedContentReporter reporter = Substitute.For(); private readonly IWorkQueue queue = Substitute.For(); private readonly FakeLogger logger = new(); + private readonly FakeLogger activeLogger = new(); private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -29,7 +31,7 @@ public sealed class CrawlTests }; private Task RunAsync(Guid configId) - => new CrawlHandler(configs, pipeline, reporter, queue, logger).RunAsync(new CrawlPayload(configId), progress, Ct); + => new CrawlHandler(new ActiveStorageConfigs(configs, activeLogger), configs, pipeline, reporter, queue, logger).RunAsync(new CrawlPayload(configId), progress, Ct); // ========================================================================== // CrawlHandler @@ -87,7 +89,7 @@ public async Task RunAsync_ShutdownDuringTheDuplicateReport_PropagatesAsync() throw new OperationCanceledException(shutdown.Token); }); - var act = () => new CrawlHandler(configs, pipeline, reporter, queue, logger) + var act = () => new CrawlHandler(new ActiveStorageConfigs(configs, activeLogger), configs, pipeline, reporter, queue, logger) .RunAsync(new CrawlPayload(config.Id), progress, shutdown.Token); await act.Should().ThrowAsync(); @@ -103,10 +105,11 @@ public async Task RunAsync_InactiveConfig_SkipsWithoutCrawlingAsync() var result = await RunAsync(config.Id); - result.Should().Be(new CrawlSkipped("The storage config is missing or inactive.")); + result.Should().Be(new StorageConfigSkipped("The storage config is missing or inactive.")); await pipeline.DidNotReceiveWithAnyArgs().RunAsync(default!, default!, Ct); await queue.DidNotReceiveWithAnyArgs().EnqueueAsync(default!, default, Ct); - logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Message.Contains(config.Id.ToString())); + activeLogger.Collector.GetSnapshot().Should().ContainSingle().Which.Message.Should().Be( + $"Skipping Crawl of storage config {config.Id}: it is missing or inactive."); } [Fact] @@ -114,7 +117,7 @@ public async Task RunAsync_ConfigGone_SkipsWithoutCrawlingAsync() { var result = await RunAsync(Guid.NewGuid()); - result.Should().BeOfType(); + result.Should().BeOfType(); await pipeline.DidNotReceiveWithAnyArgs().RunAsync(default!, default!, Ct); } diff --git a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs index 77565b6a..d8d330e9 100644 --- a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs @@ -343,6 +343,8 @@ public void AddWorkerCoreServices_BindsWorkerSettingsFromTheWorkerSection() { typeof(IUserStorageConfigRepository), typeof(EfUserStorageConfigRepository) }, { typeof(IRefreshTokenRepository), typeof(EfRefreshTokenRepository) }, { typeof(IMediaAssetRepository), typeof(EfMediaAssetRepository) }, + { typeof(IBurstRepository), typeof(EfBurstRepository) }, + { typeof(ActiveStorageConfigs), typeof(ActiveStorageConfigs) }, { typeof(IDatabaseMigrator), typeof(EfDatabaseMigrator) }, { typeof(IWorkQueue), typeof(EfWorkQueue) }, }; diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs index 1089b10c..a39c9029 100644 --- a/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/Classification/ContentChangedWriterTests.cs @@ -38,6 +38,7 @@ public sealed class ContentChangedWriterTests private sealed class TestFixture { public IMediaAssetRepository Repository { get; } = Substitute.For(); + public IBurstRepository Bursts { get; } = Substitute.For(); public IUnitOfWork UnitOfWork { get; } = Substitute.For(); public IGuidFactory GuidFactory { get; } = Substitute.For(); public IVideoProxyGenerator MovGenerator { get; } = Substitute.For(); @@ -86,7 +87,7 @@ public ContentChangedWriter Build() MovGenerator.TranscodeAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(call => FileSystem.File.WriteAllBytesAsync(call.ArgAt(1), [0x01])); var stagingWriter = new ProxyStagingWriter(FileSystem, clock, GuidFactory, new FakeLogger()); - return new(Repository, new IngestionSaver(UnitOfWork, Repository), FileSystem, clock, + return new(Repository, Bursts, new IngestionSaver(UnitOfWork, Repository), FileSystem, clock, new PairedMovResolver(Repository, FileSystem, clock, GuidFactory), new PairedMovProxyWriter( [MovGenerator], new TwoLevelHexShardStrategy(), stagingWriter, Repository, @@ -170,7 +171,7 @@ public async Task TryWriteAsync_ClearsTheBurstMarkSoTheEditedPhotoIsCheckedAgain var asset = await fx.WriteAsync(fx.Context()); asset!.BurstId.Should().BeNull(); - await fx.Repository.Received(1).RemoveBurstCheckAsync(StoredId, Arg.Any()); + await fx.Bursts.Received(1).RemoveCheckAsync(StoredId, Arg.Any()); } [Fact] diff --git a/src/Anichron.Worker.Tests.Unit/Ingestion/ExifDateTests.cs b/src/Anichron.Worker.Tests.Unit/Ingestion/ExifDateTests.cs new file mode 100644 index 00000000..69670374 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Ingestion/ExifDateTests.cs @@ -0,0 +1,19 @@ +using Anichron.Worker.Ingestion; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Ingestion; + +public sealed class ExifDateTests +{ + [Fact] + public void Parse_ExifDate_ReadsItWithColonsAsDateSeparators() + => ExifDate.Parse("2024:07:15 13:22:01").Should().Be(new LocalDateTime(2024, 7, 15, 13, 22, 1)); + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("2024-07-15 13:22:01")] + [InlineData(" : : : : ")] + public void Parse_MissingOrNotAnExifDate_IsNull(string? raw) + => ExifDate.Parse(raw).Should().BeNull(); +} diff --git a/src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs b/src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs index 7e2371d3..6d29cb7d 100644 --- a/src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Work/StorageConfigSchedulesTests.cs @@ -77,13 +77,16 @@ public async Task SchedulesAsync_BurstDetection_SchedulesItPerConfigAtTheCrawlIn } [Fact] - public async Task StorageConfigSchedule_EnqueuesItsPayloadAsync() + public async Task Schedule_EnqueuesItsConfigsPayloadAsync() { var configId = Guid.NewGuid(); var queue = Substitute.For(); - await new StorageConfigSchedule(new BurstDetectionPayload(configId), Duration.FromHours(4)) - .EnqueueAsync(queue, Ct); + configs.GetAllActiveAsync(Ct).Returns([new UserStorageConfig { Id = configId, UserId = MiraId, RootPath = "/r", IsActive = true }]); + var schedules = await SchedulesAsync( + new StorageConfigSchedules(new WorkerState(), Options.Create(new WorkerSettings()))); + + await schedules.Single().EnqueueAsync(queue, Ct); await queue.Received(1).EnqueueAsync(new BurstDetectionPayload(configId), null, Ct); } diff --git a/src/Anichron.Worker/Bursts/BurstDetection.cs b/src/Anichron.Worker/Bursts/BurstDetection.cs index dca54146..f9a81189 100644 --- a/src/Anichron.Worker/Bursts/BurstDetection.cs +++ b/src/Anichron.Worker/Bursts/BurstDetection.cs @@ -1,19 +1,26 @@ using Anichron.Core; using Anichron.Core.Data; +using Anichron.Core.Data.Repository; using Anichron.Core.Domain; using Anichron.Core.Work; -using Microsoft.EntityFrameworkCore; +using Anichron.Worker.Work; using NodaTime; using System.IO.Abstractions; namespace Anichron.Worker.Bursts; -public sealed record BurstDetectionResult(int Checked, int Created, int Merged, int Dissolved); - -public sealed record BurstDetectionSkipped(string Reason); +public sealed record BurstDetectionResult +{ + public int Checked { get; set; } + public int Created { get; set; } + public int Merged { get; set; } + public int Dissolved { get; set; } +} internal sealed partial class BurstDetectionHandler( - AnichronDbContext db, + ActiveStorageConfigs activeConfigs, + IBurstRepository bursts, + IUnitOfWork unitOfWork, IOriginalPhotoReader reader, IFileSystem fileSystem, IClock clock, @@ -24,37 +31,28 @@ internal sealed partial class BurstDetectionHandler( public async Task RunAsync(BurstDetectionPayload payload, IWorkProgress progress, CancellationToken ct) { - var config = await db.StorageConfigs.FirstOrDefaultAsync(c => c.Id == payload.StorageConfigId, ct); - if (config is not { IsActive: true }) - { - Log.Skipped(logger, payload.StorageConfigId); - return new BurstDetectionSkipped("The storage config is missing or inactive."); - } + if (await activeConfigs.FindAsync(BurstDetectionPayload.Kind, payload.StorageConfigId, ct) is not { } config) + return StorageConfigSkipped.MissingOrInactive; if (!fileSystem.Directory.Exists(config.RootPath)) throw new DirectoryNotFoundException($"The storage config's root {config.RootPath} is missing."); - var tally = new Tally(); - await RepairAsync(config.Id, tally, ct); - await CheckAsync(config, tally, progress, ct); + var result = new BurstDetectionResult(); + await RepairAsync(config.Id, result, ct); + await CheckAsync(config, result, progress, ct); - var result = new BurstDetectionResult(tally.Checked, tally.Created, tally.Merged, tally.Dissolved); Log.Finished(logger, config.Id, result.Checked, result.Created, result.Merged, result.Dissolved); return result; } - private async Task RepairAsync(Guid configId, Tally tally, CancellationToken ct) + private async Task RepairAsync(Guid configId, BurstDetectionResult result, CancellationToken ct) { - var bursts = await db.Bursts.IgnoreQueryFilters() - .Include(b => b.Assets) - .Where(b => b.Assets.All(a => a.StorageConfigId == configId)) - .ToListAsync(ct); - foreach (var burst in bursts) + foreach (var burst in await bursts.GetInConfigWithMembersAsync(configId, ct)) { if (burst.Assets.Count(a => !a.IsSoftDeleted) < 2) { - db.Bursts.Remove(burst); - tally.Dissolved++; + bursts.Remove(burst); + result.Dissolved++; } else { @@ -62,34 +60,25 @@ private async Task RepairAsync(Guid configId, Tally tally, CancellationToken ct) } } - await db.SaveChangesAsync(ct); + await unitOfWork.SaveChangesAsync(ct); } - private async Task CheckAsync(UserStorageConfig config, Tally tally, IWorkProgress progress, CancellationToken ct) + private async Task CheckAsync(UserStorageConfig config, BurstDetectionResult result, IWorkProgress progress, CancellationToken ct) { - var pending = db.MediaAssets.Where(a => a.StorageConfigId == config.Id - && (a.MediaType == MediaType.Image || a.MediaType == MediaType.LivePhoto) - && !db.BurstChecks.Any(c => c.AssetId == a.Id)); - var total = await pending.CountAsync(ct); - - while (await pending - .Include(a => a.Metadata) - .Include(a => a.ProxyFiles) - .OrderBy(a => a.DateCaptured).ThenBy(a => a.Id) - .Take(BatchSize) - .ToListAsync(ct) is { Count: > 0 } batch) + var total = await bursts.CountUncheckedPhotosAsync(config.Id, ct); + while (await bursts.GetUncheckedPhotosAsync(config.Id, BatchSize, ct) is { Count: > 0 } batch) { foreach (var photo in batch) { - await CheckAsync(config, photo, tally, ct); - progress.Report(++tally.Checked, total, "Checking photos for bursts"); + await CheckAsync(config, photo, result, ct); + progress.Report(++result.Checked, total, "Checking photos for bursts"); } - db.ChangeTracker.Clear(); + bursts.ForgetLoaded(); } } - private async Task CheckAsync(UserStorageConfig config, MediaAsset photo, Tally tally, CancellationToken ct) + private async Task CheckAsync(UserStorageConfig config, MediaAsset photo, BurstDetectionResult result, CancellationToken ct) { var facts = reader.Read(fileSystem.Path.Combine(config.RootPath, photo.FilePath)); var check = new BurstCheck @@ -101,42 +90,34 @@ private async Task CheckAsync(UserStorageConfig config, MediaAsset photo, Tally && photo.ProxyFiles.Any(p => p.ProxyType == ProxyType.Thumbnail), CameraBurstId = facts.CameraBurstId, }; - db.BurstChecks.Add(check); + bursts.AddCheck(check); if (check is { IsCandidate: true, CameraBurstId: { } label }) - await JoinCameraBurstAsync(config.Id, photo, label, tally, ct); + await JoinCameraBurstAsync(config.Id, photo, label, result, ct); - await db.SaveChangesAsync(ct); + await unitOfWork.SaveChangesAsync(ct); } - private async Task JoinCameraBurstAsync(Guid configId, MediaAsset photo, string label, Tally tally, CancellationToken ct) + private async Task JoinCameraBurstAsync( + Guid configId, MediaAsset photo, string label, BurstDetectionResult result, CancellationToken ct) { - var labelled = await db.BurstChecks - .Where(c => c.IsCandidate && c.CameraBurstId == label && c.Asset.StorageConfigId == configId) - .Select(c => c.Asset) - .ToListAsync(ct); + var labelled = await bursts.GetCandidatesLabelledAsync(configId, label, ct); if (labelled.Count == 0) return; - var burstIds = labelled.Select(a => a.BurstId).OfType().ToList(); - var bursts = await db.Bursts.IgnoreQueryFilters() - .Include(b => b.Assets) - .Where(b => burstIds.Contains(b.Id)) - .OrderBy(b => b.CreatedAt).ThenBy(b => b.Id) - .ToListAsync(ct); - List members = [.. bursts.SelectMany(b => b.Assets), .. labelled, photo]; + var joined = await bursts.GetWithMembersAsync([.. labelled.Select(a => a.BurstId).OfType()], ct); + List members = [.. joined.SelectMany(b => b.Assets), .. labelled, photo]; - var target = bursts.FirstOrDefault(); - if (target is null) + if (joined is not [var target, ..]) { target = new Burst { Id = guids.NewGuid(), CreatedAt = clock.GetCurrentInstant() }; - db.Bursts.Add(target); - tally.Created++; + bursts.Add(target); + result.Created++; } - foreach (var merged in bursts.Skip(1)) + foreach (var merged in joined.Skip(1)) { - db.Bursts.Remove(merged); - tally.Merged++; + bursts.Remove(merged); + result.Merged++; } foreach (var member in members) @@ -147,19 +128,8 @@ private async Task JoinCameraBurstAsync(Guid configId, MediaAsset photo, string private static MediaAsset Earliest(IEnumerable members) => members.Where(a => !a.IsSoftDeleted).MinBy(a => (a.DateCaptured, a.Id))!; - private sealed class Tally - { - public int Checked { get; set; } - public int Created { get; set; } - public int Merged { get; set; } - public int Dissolved { get; set; } - } - private static partial class Log { - [LoggerMessage(Level = LogLevel.Information, Message = "Skipping burst detection for storage config {Id}: it is missing or inactive.")] - public static partial void Skipped(ILogger logger, Guid id); - [LoggerMessage(Level = LogLevel.Information, Message = "Burst detection for storage config {Id} checked {Checked} photo(s): {Created} burst(s) created, {Merged} merged, {Dissolved} dissolved.")] public static partial void Finished(ILogger logger, Guid id, int @checked, int created, int merged, int dissolved); } diff --git a/src/Anichron.Worker/Crawling/Crawl.cs b/src/Anichron.Worker/Crawling/Crawl.cs index 9ca266ea..67863036 100644 --- a/src/Anichron.Worker/Crawling/Crawl.cs +++ b/src/Anichron.Worker/Crawling/Crawl.cs @@ -2,12 +2,12 @@ using Anichron.Core.Domain; using Anichron.Core.Work; using Anichron.Worker.Ingestion; +using Anichron.Worker.Work; namespace Anichron.Worker.Crawling; -public sealed record CrawlSkipped(string Reason); - internal sealed partial class CrawlHandler( + ActiveStorageConfigs activeConfigs, IUserStorageConfigRepository configs, IFileIngestionPipeline pipeline, ISharedContentReporter reporter, @@ -16,12 +16,8 @@ internal sealed partial class CrawlHandler( { public async Task RunAsync(CrawlPayload payload, IWorkProgress progress, CancellationToken ct) { - var config = await configs.FindByIdAsync(payload.StorageConfigId, ct); - if (config is not { IsActive: true }) - { - Log.Skipped(logger, payload.StorageConfigId); - return new CrawlSkipped("The storage config is missing or inactive."); - } + if (await activeConfigs.FindAsync(CrawlPayload.Kind, payload.StorageConfigId, ct) is not { } config) + return StorageConfigSkipped.MissingOrInactive; Log.Crawling(logger, config.RootPath, config.UserId); var counts = await pipeline.RunAsync(config, progress, ct); @@ -51,8 +47,5 @@ private static partial class Log [LoggerMessage(Level = LogLevel.Error, Message = "Reporting duplicate content for storage config {Id} failed.")] public static partial void DuplicateReportFailed(ILogger logger, Guid id, Exception ex); - - [LoggerMessage(Level = LogLevel.Information, Message = "Skipping the crawl of storage config {Id}: it is missing or inactive.")] - public static partial void Skipped(ILogger logger, Guid id); } } diff --git a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs index ff961e94..db17277b 100644 --- a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs +++ b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs @@ -46,6 +46,8 @@ internal IServiceCollection AddWorkerDataServices(IConfiguration configuration) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(sp => sp.GetRequiredService()); services.AddScoped(); diff --git a/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs b/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs index 02e64803..e1a669ac 100644 --- a/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs +++ b/src/Anichron.Worker/Ingestion/Classification/ContentChangedWriter.cs @@ -9,6 +9,7 @@ namespace Anichron.Worker.Ingestion.Classification; internal sealed partial class ContentChangedWriter( IMediaAssetRepository repository, + IBurstRepository bursts, IngestionSaver saver, IFileSystem fileSystem, IClock clock, @@ -28,7 +29,7 @@ protected override async Task WriteAssetAsync(IngestionContext conte DescribeContent(asset, asset.Metadata ??= new Metadata { AssetId = asset.Id }, context); ReplaceProxyFiles(asset, context.ProxyFilesOf(asset.Id)); asset.BurstId = null; - await repository.RemoveBurstCheckAsync(asset.Id, ct); + await bursts.RemoveCheckAsync(asset.Id, ct); return asset; } diff --git a/src/Anichron.Worker/Ingestion/ExifDate.cs b/src/Anichron.Worker/Ingestion/ExifDate.cs index 39def3ed..764092ec 100644 --- a/src/Anichron.Worker/Ingestion/ExifDate.cs +++ b/src/Anichron.Worker/Ingestion/ExifDate.cs @@ -12,5 +12,5 @@ private static LocalDateTimePattern Pattern /// The date, or null when the raw value is missing or not an EXIF date. /// An EXIF date-time tag's value. internal static LocalDateTime? Parse(string? raw) - => Pattern.Parse(raw!) is { Success: true } result ? result.Value : null; + => raw is { } text && Pattern.Parse(text) is { Success: true } result ? result.Value : null; } diff --git a/src/Anichron.Worker/Work/ActiveStorageConfigs.cs b/src/Anichron.Worker/Work/ActiveStorageConfigs.cs new file mode 100644 index 00000000..d38bd81f --- /dev/null +++ b/src/Anichron.Worker/Work/ActiveStorageConfigs.cs @@ -0,0 +1,35 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; + +namespace Anichron.Worker.Work; + +/// The result of a per-config item whose storage config is missing or inactive. +/// Why nothing was done. +public sealed record StorageConfigSkipped(string Reason) +{ + public static StorageConfigSkipped MissingOrInactive { get; } = new("The storage config is missing or inactive."); +} + +internal sealed partial class ActiveStorageConfigs( + IUserStorageConfigRepository configs, + ILogger logger) +{ + /// The config, or null (logged) when it is missing or inactive and the item is skipped. + /// The kind of the item asking. + /// The item's storage config. + /// Cancels the lookup. + public async Task FindAsync(WorkKind kind, Guid storageConfigId, CancellationToken ct) + { + if (await configs.FindByIdAsync(storageConfigId, ct) is { IsActive: true } config) + return config; + + Log.Skipped(logger, kind, storageConfigId); + return null; + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Information, Message = "Skipping {Kind} of storage config {Id}: it is missing or inactive.")] + public static partial void Skipped(ILogger logger, WorkKind kind, Guid id); + } +} diff --git a/src/Anichron.Worker/Work/StorageConfigSchedules.cs b/src/Anichron.Worker/Work/StorageConfigSchedules.cs index bdf90132..3c743cfc 100644 --- a/src/Anichron.Worker/Work/StorageConfigSchedules.cs +++ b/src/Anichron.Worker/Work/StorageConfigSchedules.cs @@ -7,10 +7,6 @@ namespace Anichron.Worker.Work; -public sealed class StorageConfigSchedule(TPayload payload, Duration interval) - : WorkSchedule(payload, interval) - where TPayload : IWorkPayload; - /// One schedule per active storage config the Worker serves, at the crawl interval. /// The per-config kind scheduled. /// Names the user a per-user Worker serves. @@ -26,6 +22,6 @@ public async Task> SchedulesAsync(IServiceProvider ? await repository.GetActiveByUserIdAsync(userId, ct) : await repository.GetAllActiveAsync(ct); var interval = Duration.FromHours(options.Value.CrawlIntervalHours); - return [.. served.Select(config => new StorageConfigSchedule(TPayload.For(config.Id), interval))]; + return [.. served.Select(config => new WorkSchedule(TPayload.For(config.Id), interval))]; } } diff --git a/src/Anichron.Worker/Work/WorkSchedule.cs b/src/Anichron.Worker/Work/WorkSchedule.cs index 9bb0945f..566199ff 100644 --- a/src/Anichron.Worker/Work/WorkSchedule.cs +++ b/src/Anichron.Worker/Work/WorkSchedule.cs @@ -18,7 +18,7 @@ public interface IWorkScheduleSource Task> SchedulesAsync(IServiceProvider services, CancellationToken ct); } -public abstract class WorkSchedule(TPayload payload, Duration interval) : IWorkSchedule, IWorkScheduleSource +public class WorkSchedule(TPayload payload, Duration interval) : IWorkSchedule, IWorkScheduleSource where TPayload : IWorkPayload { public string DedupKey { get; } = payload.DedupKey