diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a00a4160..706e2280 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: run: dotnet build src/ - name: Test - run: dotnet test src/ --no-build --collect:"XPlat Code Coverage" --results-directory coverage --settings src/coverlet.runsettings + run: dotnet test src/ --no-build -p:TestSuite=Unit --collect:"XPlat Code Coverage" --results-directory coverage --settings src/coverlet.runsettings - name: Generate coverage report uses: danielpalme/ReportGenerator-GitHub-Action@v5 @@ -91,6 +91,21 @@ jobs: - name: Schema documentation is current run: scripts/derive-db-docs.sh --check + integration: + name: Integration tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.0.x' + + - name: Integration tests + run: dotnet test src/ -p:TestSuite=Integration + mutation: name: Mutation runs-on: ubuntu-latest @@ -204,7 +219,7 @@ jobs: gate: name: CI gate runs-on: ubuntu-latest - needs: [checks, mutation, docker] + needs: [checks, integration, mutation, docker] if: always() permissions: contents: read diff --git a/CLAUDE.md b/CLAUDE.md index f37c17b4..d181b9d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,9 +29,15 @@ dotnet run --project src/Anichron.Worker ## Testing ```bash -# Run all tests +# Run the unit tests (the default suite) dotnet test +# Run the integration tests (needs a running Docker daemon) +dotnet test -p:TestSuite=Integration + +# Run both +dotnet test -p:TestSuite=All + # Run tests for a specific project dotnet test src/SomeProject.Tests.Unit/SomeProject.Tests.Unit.csproj @@ -39,7 +45,11 @@ dotnet test src/SomeProject.Tests.Unit/SomeProject.Tests.Unit.csproj dotnet test --filter "FullyQualifiedName=Namespace.ClassName.MethodName" ``` -Test projects must be named `*.Tests.Unit` — `Directory.Build.props` auto-applies the test SDK to any project matching that pattern: xUnit v3, **NSubstitute** (not Moq), FluentAssertions, `System.IO.Abstractions.TestingHelpers`, and coverlet. Internal members are exposed to test projects via `InternalsVisibleTo`. +An integration test project, or a filter matching integration tests, runs nothing without `-p:TestSuite=Integration` (or `All`). + +Test projects are recognised by name: `*.Tests.Unit` for unit tests and `*.Tests.Integration` for integration tests. `Directory.Build.props` applies the test SDK to both: xUnit v3, **NSubstitute** (not Moq), FluentAssertions, `System.IO.Abstractions.TestingHelpers`, and coverlet. `Directory.Build.targets` makes only the projects in the selected `TestSuite` (`Unit` by default, `Integration` or `All`) test projects, so `dotnet test` and Stryker never run the integration suite. Internal members are exposed to test projects via `InternalsVisibleTo`. + +**Unit or integration?** A test belongs in `Anichron.Tests.Integration` only when its subject is behaviour that only real PostgreSQL has: `ExecuteUpdate`/`ExecuteDelete`, transactions, advisory locks, row locking, constraint and partial-index enforcement. Everything else is a unit test, including a query's SQL shape (assert on `ToQueryString()` with `AnichronDbContexts.NpgsqlModel()`). The integration project starts one PostgreSQL container per run (`PostgresServer`, the same image as `src/docker-compose.yml`). Each test gets its own database, from `CreateMigratedDatabaseAsync` (real migrations) or `CreateDatabaseAsync` (empty). Its CI job is `Integration tests`, and it runs outside mutation testing, so the members it covers keep their `// Stryker disable all` and ADR 0005 rows. Shared test infrastructure lives in `Anichron.Testing`, referenced by the test projects only and tested by `Anichron.Testing.Tests.Unit`; it is excluded from mutation (ADR 0005, rule 4). Use it instead of a private copy: @@ -85,11 +95,12 @@ Four projects with strict separation of concerns: | `Anichron.Core` | Single source of truth: domain models, `AnichronDbContext`, all Fluent API config, shared utilities (XXHash64, path parsing). Zero dependencies on other projects (every other project references Core, so a reference back would be circular and fail the build). | | `Anichron.Infrastructure` | Shared host plumbing used by API and Worker: configuration (`DatabaseConfiguration` reads the connection from `POSTGRES_CONNECTION__*` env vars or Docker secrets; INI bootstrap; `ConfigurationList`), migration under a PostgreSQL advisory lock, and the API's read-only stores (`IProxyStore`, `IOriginalStore`). Each host wires its own DI. | | `Anichron.API` | ASP.NET Core Minimal APIs (the only web project: `libraries-are-not-web` keeps ASP.NET Core out of Core, Infrastructure and Worker, and the two hosts never use each other) — owns all identity/auth/admin writes (users, storage configs, refresh tokens, invites, lockout); read-only for media data. Media endpoints so far: proxy serving (#43) and On This Day (`GET /api/v1/flashbacks/on-this-day`, #38); originals (#44) are deferred past v1.0. All routes prefixed `/api/v1/`. | -| `Anichron.Worker` | `BackgroundService`s — NAS crawling, EXIF extraction, FFmpeg transcoding, proxy generation, expired-refresh-token cleanup; reconciliation is planned (Epic 4) | +| `Anichron.Worker` | `BackgroundService`s — NAS crawling, EXIF extraction, FFmpeg transcoding, proxy generation, and the work queue's executor and scheduler (maintenance such as expired-refresh-token cleanup runs as work items); reconciliation is planned (Epic 4) | **Write ownership is split by domain, not centralized.** -- **Worker is the only writer of *media* data** — `MediaAsset`, `Metadata`, `ProxyFile`, `Burst`, plus all reconciliation and soft-deletes happen here. The API never writes media: outside its DI wiring it never touches `AnichronDbContext` (architecture rule `api-reaches-data-through-repositories`), and the media write repository stays out of it (`only-worker-writes-media`). Its one identity-side write is `TokenCleanupService` deleting expired refresh tokens. +- **Worker is the only writer of *media* data** — `MediaAsset`, `Metadata`, `ProxyFile`, `Burst`, plus all reconciliation and soft-deletes happen here. The API never writes media: outside its DI wiring it never touches `AnichronDbContext` (architecture rule `api-reaches-data-through-repositories`), and the media write repository stays out of it (`only-worker-writes-media`). Its one identity-side write is the `TokenCleanup` work item's handler deleting expired refresh tokens. - **API owns all *identity/auth/admin* writes** — registration, login/refresh-token issuance and revocation, lockout state, invites, and admin user/storage-config CRUD all persist through API services (`AuthService`, `AdminUserService`, `AdminStorageConfigService`, `TokenService`, etc.) via `IUnitOfWork.SaveChangesAsync`. The API is read-only *only with respect to media*. The one exception is `AssetInteraction` (star/like/hide, #46): per-user state, to be written by the API. +- **Work items** are inserted only through `IWorkQueue` (architecture rule `only-work-queue-inserts-work-items`); every later state change belongs to the Worker's executor, and handlers live only in the Worker (`work-handlers-live-in-worker`). See "Work queue" below. - Both hosts apply EF migrations at startup, serialised by a PostgreSQL advisory lock (`MigrateWithAdvisoryLockAsync`). A planned `Anichron.UI` (NextJS/React) for the Instagram Story-style flashback experience is documented but not yet in the solution. @@ -107,7 +118,7 @@ into PostgreSQL and therefore into `docs/schema.sql` and any schema-doc tool poi database. If you find yourself about to explain *why* a column exists in a Markdown file, put it in a `HasComment` instead — see `docs/adr/0004-schema-documentation.md`. -**Nine entities**, all EF Core config via **Fluent API only** (architecture rule `domain-has-no-data-annotations`). Columns are +**Ten entities**, all EF Core config via **Fluent API only** (architecture rule `domain-has-no-data-annotations`). Columns are **PascalCase** — the only explicitly-named column in the whole model is `Invites.xmin`. Table names do not always match entity names (`AssetInteraction` → `Interactions`, `UserStorageConfig` → `StorageConfigs`): @@ -119,8 +130,9 @@ do not always match entity names (`AssetInteraction` → `Interactions`, `UserSt - `AssetInteraction` — per-user state (starred, liked, hidden, view count, last viewed); unique on `(UserId, AssetId)` - `UserStorageConfig` — NAS root paths per user; `RootPath` is **globally unique** — a path cannot be assigned to more than one storage config across all users. Paths are stored canonical (absolute, `.`/`..`/`//` collapsed, no trailing slash) via `StoragePath.Canonicalize`, so `/p` and `/p/` cannot both be assigned. A `RootPath` must be a folder strictly inside `StoragePath.OriginalsRoot` (`/data/originals`), never the root itself, and may not contain or be contained by another config's root (`StoragePath.Overlaps`) — nested roots would index the same files twice - `User` — owns storage configs and interactions -- `RefreshToken` — issued and revoked by the API, expired rows deleted by the Worker's `TokenCleanupService`; `TokenHash` unique +- `RefreshToken` — issued and revoked by the API, expired rows deleted by the Worker's `TokenCleanup` work item; `TokenHash` unique - `Invite` — admin-issued registration tokens; `TokenHash` unique, and the only entity carrying an optimistic-concurrency token (`xmin`) +- `WorkItem` — one durable unit of Worker work (`Kind`, jsonb `Payload`, optional `DedupKey`, status, lease, attempt, progress snapshot, result or error); see "Work queue" ### Key EF Core / Database Decisions @@ -179,6 +191,16 @@ Each user has a private set of `UserStorageConfig` entries pointing to NAS roots **Asset lists are read through `IAssetViewReader`** (`Anichron.Core/Data/Repository/AssetViewReader.cs`, #253). It always applies `VisibleTo` and `NotHiddenFor` to the caller, then the caller's `IAssetSelection` (e.g. `OnThisDay`, which filters only on `MediaAsset` columns), then left-joins the caller's own interaction row, all projected into a flat `AssetView` in one SQL statement. A new list view adds a selection class; it never changes the reader, and no API type touches a media entity (architecture rule `api-reads-media-through-views`). Grouping and ordering (`FlashbackYears`: years newest first, starred first, then capture time) happen in memory on the result. +### Work queue + +Durable work for the Worker lives in `WorkItems` (#276, ADR 0008). Today its kinds are periodic maintenance (`TokenCleanup`, `WorkItemRetention`); the crawl (#281) and admin requests (#174) follow. + +- **Producing:** `IWorkQueue.EnqueueAsync` (`EfWorkQueue`, Core) is the only insert. A payload with a `DedupKey` coalesces: while an item with that key is `Queued`, enqueueing returns it (a partial unique index absorbs racing inserts). +- **Scheduling:** every Worker runs `WorkScheduler`. Each poll (`Worker:WorkPollIntervalSeconds`, 5 s) it recovers expired leases, then asks each `IWorkSchedule` whether its kind is due: nothing queued or running with its key and nothing finished within its interval. No timer state is kept, so overdue work runs right after downtime. +- **Executing:** `WorkExecutor` claims the oldest queued item (`FOR UPDATE SKIP LOCKED`), one at a time, under a lease (`WorkLeaseSeconds`, 60 s) that a heartbeat (`WorkHeartbeatSeconds`, 2 s) extends while writing the progress snapshot. Every write about a running item is fenced by `(Id, Attempt)`. A fenced heartbeat cancels the handler and writes nothing more; a shutdown releases the item without a stall; a handler exception fails it with no retry. An expired lease requeues the item and counts a stall; the `WorkMaxStalls`th (3) fails it as stalled. An item handed back (release or recovery) while another with its key is already queued ends `Cancelled` as superseded, since the queued one does the same work. A Worker claims only the kinds it knows, so a newer kind waits for a newer Worker. `WorkItemRetention` deletes items finished more than `WorkItemRetentionDays` (30) ago. +- **Adding a kind:** a `WorkKind` value, a payload record in Core implementing `IWorkPayload` (static `Kind`, optional `DedupKey`), an `IWorkHandler` in the Worker registered with `AddWorkHandler`, and for a periodic kind a `WorkSchedule` subclass. `WorkHandlerRegistry` fails startup unless every kind has exactly one payload and one handler; nothing switches on the kind. +- **Handler contract:** delivery is at least once. Each run starts from scratch and converges, side effects are safe to repeat, and the returned summary becomes `Result`. + ### Worker Processing Model Processing uses a bounded-concurrency `Channel` pipeline. The crawler produces file paths; N consumer tasks process them concurrently. Default concurrency: `Worker:MaxConcurrentFiles = 4`. Processing is always idempotent — files already in the DB (matched by `(StorageConfigId, ContentHash)`) skip extraction and proxy generation; only the row's `FilePath`/`LastSeenOnNas` are updated, and a file whose bytes changed at a stored path updates that row in place (see Reconciliation above). diff --git a/docs/adr/0005-mutation-scope-excludes-unkillable-and-wiring-code.md b/docs/adr/0005-mutation-scope-excludes-unkillable-and-wiring-code.md index 796d7ef3..5d372835 100644 --- a/docs/adr/0005-mutation-scope-excludes-unkillable-and-wiring-code.md +++ b/docs/adr/0005-mutation-scope-excludes-unkillable-and-wiring-code.md @@ -86,11 +86,15 @@ The directive is left bare — the reason lives here, not in the code. | --- | --- | | `AnichronDbContext.ExecuteInTransactionAsync` (both overloads) | Commit versus no commit is only observable on a provider with real transactions. | | `EfRefreshTokenRepository.RevokeAllActiveByUserIdAsync` and `DeleteExpiredAsync` | `ExecuteUpdate`/`ExecuteDelete` are not supported by the EF InMemory provider. | +| `EfWorkQueue.TryInsertAsync` | Absorbing a lost race needs the partial unique index on `DedupKey`, which the InMemory provider does not enforce. The rest of `EnqueueAsync` is in scope. | +| `EfWorkItemStore` members except `IsDueAsync`, and its `Running`/`Expired` filters | The claim is raw SQL with `FOR UPDATE SKIP LOCKED`; every transition is an `ExecuteUpdate`/`ExecuteDelete` fenced by `(Id, Attempt)`. `IsDueAsync` is plain LINQ and in scope. | These are **not** tested against a second SQL engine such as SQLite. A substitute engine diverges from PostgreSQL in exactly the places this model relies on (partial-index filters, `xmin`, NodaTime types, collation), so a green test there would claim coverage it does not -have. They stay excluded until tests run against real PostgreSQL. +have. Where they are tested, it is against real PostgreSQL in `Anichron.Tests.Integration` +(`DeleteExpiredAsync` and `MigrateWithAdvisoryLockAsync` so far). That project is outside the +Stryker run, so the members stay excluded from mutation. `WorkerSettings.FfmpegPath` is **not** in that list. It is a real default and is tested. `RootPath` lost its default in #221: it names one user's folder inside `/data/originals`, so no value is right @@ -107,7 +111,7 @@ for every deployment. The proxy root is no longer a setting at all since #43: it `ServiceCollection` and resolve the configured options. - **`Log.*` hides four mutants that existing tests do kill**: the two log calls in - `LoggingMiddleware`, `TokenCleanupService`'s deleted-count log, and `PwnedPasswordClient`'s + `LoggingMiddleware`, `TokenCleanupService`'s deleted-count log (now `TokenCleanupHandler`'s), and `PwnedPasswordClient`'s "check unavailable" warning. Accepted — the tests stay and still guard those lines; they just no longer count toward the score. `LoggingMiddleware` is left with almost no mutants of its own as a result, which is expected for a class whose only behaviour is logging. diff --git a/docs/adr/0008-hand-rolled-postgresql-work-queue.md b/docs/adr/0008-hand-rolled-postgresql-work-queue.md new file mode 100644 index 00000000..26b7034f --- /dev/null +++ b/docs/adr/0008-hand-rolled-postgresql-work-queue.md @@ -0,0 +1,43 @@ +# A hand-rolled PostgreSQL work queue, not a job library + +Status: accepted (2026-10-18, #276). + +Work the Worker does on request or on a schedule is a row in `WorkItems`, in our EF model. A Worker +claims a row with `FOR UPDATE SKIP LOCKED`, holds it under a lease that a heartbeat extends, and +records the outcome in the same row. No job library is used. + +## What it has to do + +The API and the Worker share only the database. The queue has to let the API ask for work that only +a Worker can do, route a request to a Worker that has the right folder mounted (a per-user worker +sees one user's originals), recover an item whose Worker died, and give an admin a list of what is +queued, running and finished, with progress, from our own API. + +## Libraries considered + +The full evaluation is on #273. In short: + +| Library | Why not | +| --- | --- | +| Hangfire + Hangfire.PostgreSql | Feasible, but its schema lives outside the EF model: not in `docs/schema.sql`, no `HasComment` (ADR 0004), created at startup beside our advisory-locked migrations. Jobs are serialised method calls on types both hosts share, so a rename breaks queued items. Newtonsoft.Json and `DateTime` at its boundary. No progress model, and per-job queue routing on PostgreSQL was unverified. | +| TickerQ | Enqueue-in-one-host, run-in-another and scope routing are undocumented; EF-only crash recovery unclear. | +| Quartz.NET (clustered) | A scheduler, not a work queue: any node runs any job, so routing would need one scheduler per user. No progress. | +| Wolverine | A messaging framework and programming model for both hosts; no job-status model to list. | +| Rebus | A message bus without a status view. | +| MassTransit | Commercial since 2026. | + +## Consequences + +- One table, one model: `WorkItems` is in `docs/schema.sql`, its columns carry comments, and every + time is NodaTime. +- We own the concurrency code. The claim, the fenced transitions and lease recovery are PostgreSQL-only, + so they carry `// Stryker disable all` (ADR 0005) and are covered instead by contract tests on real + PostgreSQL in `Anichron.Tests.Integration`: concurrent claimers, skip-locked, zombie attempts, the + stall limit, release without a stall and dedup coalescing. +- Delivery is at least once. A handler starts from scratch and converges, and its side effects are + safe to repeat. Every write a Worker makes about a running item is fenced by `(Id, Attempt)`, so an + attempt that lost its lease changes nothing. +- Due-ness comes from the table (nothing queued or running with the kind's key, nothing finished + within its interval), so no host keeps timer state and overdue work runs right after downtime. +- `LISTEN/NOTIFY`, priorities and automatic retry are left out; each can be added without changing + the handler contract. diff --git a/docs/schema.sql b/docs/schema.sql index 19092f6e..83357625 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -43,7 +43,7 @@ CREATE TABLE "RefreshTokens" ( CONSTRAINT "PK_RefreshTokens" PRIMARY KEY ("Id"), CONSTRAINT "FK_RefreshTokens_Users_UserId" FOREIGN KEY ("UserId") REFERENCES "Users" ("Id") ON DELETE CASCADE ); -COMMENT ON TABLE "RefreshTokens" IS 'Issued and revoked by the API; expired rows are swept by the Worker''s TokenCleanupService. One of the few tables both services touch.'; +COMMENT ON TABLE "RefreshTokens" IS 'Issued and revoked by the API; expired rows are swept by the Worker''s TokenCleanup work item. One of the few tables both services touch.'; COMMENT ON COLUMN "RefreshTokens"."TokenHash" IS 'Hash, never the token itself — a database disclosure must not hand out usable sessions.'; @@ -59,6 +59,44 @@ COMMENT ON TABLE "StorageConfigs" IS 'A NAS root assigned to one user. This is t COMMENT ON COLUMN "StorageConfigs"."RootPath" IS 'GLOBALLY unique, not per user: a path cannot be assigned to two configs across all users. Two users indexing the same directory would duplicate every asset and let one user''s config removal cascade away the other''s rows.'; +CREATE TABLE "WorkItems" ( + "Id" uuid NOT NULL, + "Kind" text NOT NULL, + "Payload" jsonb NOT NULL, + "DedupKey" text, + "RequestedBy" uuid, + "RequestedAt" timestamp with time zone NOT NULL, + "Status" text NOT NULL, + "Attempt" integer NOT NULL, + "Stalls" integer NOT NULL, + "ClaimedBy" text, + "StartedAt" timestamp with time zone, + "LeaseUntil" timestamp with time zone, + "FinishedAt" timestamp with time zone, + "UpdatedAt" timestamp with time zone NOT NULL, + "ProgressDone" bigint NOT NULL, + "ProgressTotal" bigint, + "ProgressNote" text, + "Result" jsonb, + "Error" text, + CONSTRAINT "PK_WorkItems" PRIMARY KEY ("Id"), + CONSTRAINT "FK_WorkItems_Users_RequestedBy" FOREIGN KEY ("RequestedBy") REFERENCES "Users" ("Id") ON DELETE SET NULL +); +COMMENT ON TABLE "WorkItems" IS 'Durable work for the Worker: periodic maintenance today, admin requests later. Only IWorkQueue inserts rows; every later state change belongs to the Worker''s executor. Delivery is at least once, so handlers start from scratch and converge.'; +COMMENT ON COLUMN "WorkItems"."Kind" IS 'WorkKind, persisted as text. Picks the payload type and the handler through the registry, never through a switch.'; +COMMENT ON COLUMN "WorkItems"."Payload" IS 'The kind''s IWorkPayload, serialised with System.Text.Json and NodaTime settings.'; +COMMENT ON COLUMN "WorkItems"."DedupKey" IS 'Opt-in coalescing. While an item with this key is Queued, enqueueing the same key returns that item instead of adding a second; the partial unique index enforces it.'; +COMMENT ON COLUMN "WorkItems"."RequestedBy" IS 'The admin who asked for the work; null when a Worker''s scheduler enqueued it.'; +COMMENT ON COLUMN "WorkItems"."Status" IS 'Queued, Running, Succeeded, Failed or Cancelled, persisted as text.'; +COMMENT ON COLUMN "WorkItems"."Attempt" IS 'Incremented by every claim. Every write a Worker makes while running an item is fenced by (Id, Attempt), so an attempt that lost its lease changes nothing.'; +COMMENT ON COLUMN "WorkItems"."Stalls" IS 'Lease expiries, counted by lease recovery. A graceful shutdown hands the item back without counting one; the item fails as stalled once the limit is reached.'; +COMMENT ON COLUMN "WorkItems"."ClaimedBy" IS 'The claiming Worker for display: its WORKER__USER or ''all'', then its host name.'; +COMMENT ON COLUMN "WorkItems"."LeaseUntil" IS 'Extended by every heartbeat. A Running item past it belongs to a dead or paused Worker and goes back to the queue.'; +COMMENT ON COLUMN "WorkItems"."UpdatedAt" IS 'Moves on every insert, claim, heartbeat and transition, so a reader can ask what changed since a point in time.'; +COMMENT ON COLUMN "WorkItems"."ProgressNote" IS 'The latest progress snapshot (done, optional total, note), overwritten by each heartbeat; no history is kept.'; +COMMENT ON COLUMN "WorkItems"."Result" IS 'The handler''s results summary, written when the item succeeds.'; + + CREATE TABLE "Bursts" ( "Id" uuid NOT NULL, "PrimaryAssetId" uuid NOT NULL, @@ -206,4 +244,19 @@ CREATE UNIQUE INDEX ix_users_email ON "Users" ("Email"); CREATE UNIQUE INDEX ix_users_username ON "Users" ("Username"); +CREATE INDEX ix_workitems_claim ON "WorkItems" ("RequestedAt") WHERE "Status" = 'Queued'; + + +CREATE UNIQUE INDEX ix_workitems_dedupkey_queued ON "WorkItems" ("DedupKey") WHERE "Status" = 'Queued' AND "DedupKey" IS NOT NULL; + + +CREATE INDEX ix_workitems_lease ON "WorkItems" ("LeaseUntil") WHERE "Status" = 'Running'; + + +CREATE INDEX "IX_WorkItems_RequestedBy" ON "WorkItems" ("RequestedBy"); + + +CREATE INDEX "IX_WorkItems_UpdatedAt" ON "WorkItems" ("UpdatedAt"); + + ALTER TABLE "Bursts" ADD CONSTRAINT "FK_Bursts_MediaAssets_PrimaryAssetId" FOREIGN KEY ("PrimaryAssetId") REFERENCES "MediaAssets" ("Id") ON DELETE RESTRICT; diff --git a/src/Anichron.API.Tests.Unit/packages.lock.json b/src/Anichron.API.Tests.Unit/packages.lock.json index ff81714b..dd7490dc 100644 --- a/src/Anichron.API.Tests.Unit/packages.lock.json +++ b/src/Anichron.API.Tests.Unit/packages.lock.json @@ -776,6 +776,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -972,6 +973,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.API/packages.lock.json b/src/Anichron.API/packages.lock.json index e43fe4ff..f742b5de 100644 --- a/src/Anichron.API/packages.lock.json +++ b/src/Anichron.API/packages.lock.json @@ -470,6 +470,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -485,6 +486,15 @@ "System.IO.Abstractions": "[22.3.0, )" } }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Architecture.Tests.Unit/ArchitectureRules.cs b/src/Anichron.Architecture.Tests.Unit/ArchitectureRules.cs index 52071bf1..d42018db 100644 --- a/src/Anichron.Architecture.Tests.Unit/ArchitectureRules.cs +++ b/src/Anichron.Architecture.Tests.Unit/ArchitectureRules.cs @@ -1,6 +1,7 @@ using Anichron.Core.Data; using Anichron.Core.Data.Repository; using Anichron.Core.Domain; +using Anichron.Core.Work; using ArchUnitNET.Fluent; using Microsoft.Extensions.Logging; using System.IO.Abstractions; @@ -25,10 +26,13 @@ internal static class ArchitectureRules public const string NoNotImplementedException = "no-not-implemented-exception"; public const string DomainHasNoDataAnnotations = "domain-has-no-data-annotations"; public const string DomainUsesNodaTime = "domain-uses-nodatime"; + public const string OnlyWorkQueueInsertsWorkItems = "only-work-queue-inserts-work-items"; + public const string WorkHandlersLiveInWorker = "work-handlers-live-in-worker"; private const string Domain = "Anichron.Core.Domain"; private const string ApiWiring = "Anichron.API.Infrastructure"; private const string StagingWriter = "Anichron.Worker.Ingestion.Proxy.ProxyStagingWriter"; + private const string WorkQueue = "Anichron.Core.Work.EfWorkQueue"; public static IReadOnlyList All { get; } = [ @@ -70,6 +74,13 @@ internal static class ArchitectureRules new(DomainUsesNodaTime, Types().That().ResideInNamespace(Domain) .Should().NotDependOnAny(typeof(DateTime), typeof(DateTimeOffset))), + new(OnlyWorkQueueInsertsWorkItems, + Types().That().ResideInAssembly(ApiAssembly, CoreAssembly, InfrastructureAssembly, WorkerAssembly) + .And().DoNotHaveFullName(WorkQueue) + .Should().NotCallAny(MethodMembers().That().AreConstructors().And().AreDeclaredIn(typeof(WorkItem)))), + new(WorkHandlersLiveInWorker, + Types().That().ImplementInterface(typeof(IWorkHandler<>)) + .Should().ResideInAssembly(WorkerAssembly)), ]; public static ArchitectureRule Named(string name) => All.Single(rule => rule.Name == name); diff --git a/src/Anichron.Architecture.Tests.Unit/packages.lock.json b/src/Anichron.Architecture.Tests.Unit/packages.lock.json index 903d6ef2..37af82cb 100644 --- a/src/Anichron.Architecture.Tests.Unit/packages.lock.json +++ b/src/Anichron.Architecture.Tests.Unit/packages.lock.json @@ -807,6 +807,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -1010,6 +1011,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Core.Tests.Unit/Work/PayloadsTests.cs b/src/Anichron.Core.Tests.Unit/Work/PayloadsTests.cs new file mode 100644 index 00000000..0d1d81f6 --- /dev/null +++ b/src/Anichron.Core.Tests.Unit/Work/PayloadsTests.cs @@ -0,0 +1,19 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; + +namespace Anichron.Core.Tests.Unit.Work; + +public sealed class PayloadsTests +{ + private static (WorkKind Kind, string? DedupKey) Describe(TPayload payload) + where TPayload : IWorkPayload + => (TPayload.Kind, payload.DedupKey); + + [Fact] + public void TokenCleanup_IsOnePeriodicItemAtATime() + => Describe(new TokenCleanupPayload()).Should().Be((WorkKind.TokenCleanup, "token-cleanup")); + + [Fact] + public void WorkItemRetention_IsOnePeriodicItemAtATime() + => Describe(new WorkItemRetentionPayload()).Should().Be((WorkKind.WorkItemRetention, "work-item-retention")); +} diff --git a/src/Anichron.Core.Tests.Unit/Work/WorkJsonTests.cs b/src/Anichron.Core.Tests.Unit/Work/WorkJsonTests.cs new file mode 100644 index 00000000..6bd5ec21 --- /dev/null +++ b/src/Anichron.Core.Tests.Unit/Work/WorkJsonTests.cs @@ -0,0 +1,45 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; +using NodaTime; +using System.Text.Json; + +namespace Anichron.Core.Tests.Unit.Work; + +public sealed class WorkJsonTests +{ + private sealed record Timed(Instant At, string Name) : IWorkPayload + { + public static WorkKind Kind => WorkKind.TokenCleanup; + } + + private sealed record Summary(int Deleted, Instant Before); + + private static readonly Instant Moment = Instant.FromUtc(2026, 10, 18, 9, 30); + + [Fact] + public void Serialize_WritesCamelCaseAndIsoInstants() + => WorkJson.Serialize(new Timed(Moment, "sweep")) + .Should().Be(/*lang=json,strict*/ """{"at":"2026-10-18T09:30:00Z","name":"sweep"}"""); + + [Fact] + public void Deserialize_ReadsWhatSerializeWrote() + => WorkJson.Deserialize(WorkJson.Serialize(new Timed(Moment, "sweep"))) + .Should().Be(new Timed(Moment, "sweep")); + + [Fact] + public void Deserialize_NullPayload_ThrowsNamingTheKind() + { + var act = () => WorkJson.Deserialize("null"); + + act.Should().Throw().WithMessage("The payload of a TokenCleanup item is null."); + } + + [Fact] + public void SerializeResult_WritesTheRuntimeTypesProperties() + => WorkJson.SerializeResult(new Summary(3, Moment)) + .Should().Be(/*lang=json,strict*/ """{"deleted":3,"before":"2026-10-18T09:30:00Z"}"""); + + [Fact] + public void SerializeResult_NoResult_IsNull() + => WorkJson.SerializeResult(null).Should().BeNull(); +} diff --git a/src/Anichron.Core.Tests.Unit/Work/WorkQueueTests.cs b/src/Anichron.Core.Tests.Unit/Work/WorkQueueTests.cs new file mode 100644 index 00000000..75e7a1fd --- /dev/null +++ b/src/Anichron.Core.Tests.Unit/Work/WorkQueueTests.cs @@ -0,0 +1,142 @@ +using Anichron.Core.Data; +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Testing.Data; +using Microsoft.EntityFrameworkCore; +using NodaTime; +using NSubstitute; + +namespace Anichron.Core.Tests.Unit.Work; + +public sealed class WorkQueueTests +{ + private sealed record Unkeyed(int Number) : IWorkPayload + { + public static WorkKind Kind => WorkKind.WorkItemRetention; + } + + private sealed record Keyed(string Key) : IWorkPayload + { + public static WorkKind Kind => WorkKind.TokenCleanup; + + string IWorkPayload.DedupKey => Key; + } + + private static readonly Instant Now = Instant.FromUtc(2026, 10, 18, 9, 0); + private static readonly Guid AdminId = Guid.Parse("11111111-1111-1111-1111-111111111111"); + + private readonly string database = Guid.NewGuid().ToString(); + private readonly IGuidFactory guids = Substitute.For(); + private readonly IClock clock = Substitute.For(); + + public WorkQueueTests() + { + clock.GetCurrentInstant().Returns(Now); + guids.NewGuid().Returns(_ => Guid.NewGuid()); + } + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private AnichronDbContext Context() => AnichronDbContexts.InMemory(database); + + private async Task EnqueueAsync(TPayload payload, Guid? requestedBy = null) + where TPayload : IWorkPayload + { + await using var db = Context(); + return await new EfWorkQueue(db, clock, guids).EnqueueAsync(payload, requestedBy, Ct); + } + + private async Task> ItemsAsync() + { + await using var db = Context(); + return await db.WorkItems.ToListAsync(Ct); + } + + private async Task SetStatusAsync(Guid id, WorkItemStatus status) + { + await using var db = Context(); + (await db.WorkItems.SingleAsync(w => w.Id == id, Ct)).Status = status; + await db.SaveChangesAsync(Ct); + } + + [Fact] + public async Task EnqueueAsync_AddsAQueuedItemDescribingTheRequestAsync() + { + var id = Guid.Parse("22222222-2222-2222-2222-222222222222"); + guids.NewGuid().Returns(id); + + var returned = await EnqueueAsync(new Keyed("sweep"), AdminId); + + returned.Should().Be(id); + var item = (await ItemsAsync()).Should().ContainSingle().Which; + item.Should().BeEquivalentTo(new + { + Id = id, + Kind = WorkKind.TokenCleanup, + Payload = /*lang=json,strict*/ """{"key":"sweep"}""", + DedupKey = "sweep", + RequestedBy = (Guid?)AdminId, + RequestedAt = Now, + Status = WorkItemStatus.Queued, + Attempt = 0, + Stalls = 0, + UpdatedAt = Now, + }); + } + + [Fact] + public async Task EnqueueAsync_Unkeyed_TakesTheKindFromThePayloadTypeAndKeepsNoKeyAsync() + { + await EnqueueAsync(new Unkeyed(1)); + + var item = (await ItemsAsync()).Should().ContainSingle().Which; + (item.Kind, item.DedupKey, item.RequestedBy).Should().Be((WorkKind.WorkItemRetention, null, null)); + } + + [Fact] + public async Task EnqueueAsync_UnkeyedTwice_AddsTwoItemsAsync() + { + await EnqueueAsync(new Unkeyed(1)); + await EnqueueAsync(new Unkeyed(1)); + + (await ItemsAsync()).Should().HaveCount(2); + } + + [Fact] + public async Task EnqueueAsync_KeyAlreadyQueued_ReturnsThatItemAndAddsNothingAsync() + { + var waiting = await EnqueueAsync(new Keyed("sweep")); + + var returned = await EnqueueAsync(new Keyed("sweep")); + + returned.Should().Be(waiting); + (await ItemsAsync()).Should().ContainSingle(); + } + + [Theory] + [InlineData(WorkItemStatus.Running)] + [InlineData(WorkItemStatus.Succeeded)] + [InlineData(WorkItemStatus.Failed)] + [InlineData(WorkItemStatus.Cancelled)] + public async Task EnqueueAsync_KeyHeldOnlyByAnItemNoLongerQueued_AddsANewItemAsync(WorkItemStatus status) + { + var earlier = await EnqueueAsync(new Keyed("sweep")); + await SetStatusAsync(earlier, status); + + var returned = await EnqueueAsync(new Keyed("sweep")); + + returned.Should().NotBe(earlier); + (await ItemsAsync()).Should().HaveCount(2); + } + + [Fact] + public async Task EnqueueAsync_AnotherKeyQueued_AddsANewItemAsync() + { + var other = await EnqueueAsync(new Keyed("delete:a")); + + var returned = await EnqueueAsync(new Keyed("delete:b")); + + returned.Should().NotBe(other); + (await ItemsAsync()).Should().HaveCount(2); + } +} diff --git a/src/Anichron.Core.Tests.Unit/packages.lock.json b/src/Anichron.Core.Tests.Unit/packages.lock.json index 04008d25..b53b8755 100644 --- a/src/Anichron.Core.Tests.Unit/packages.lock.json +++ b/src/Anichron.Core.Tests.Unit/packages.lock.json @@ -393,6 +393,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -474,6 +475,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Core/Anichron.Core.csproj b/src/Anichron.Core/Anichron.Core.csproj index ae276579..af1a5025 100644 --- a/src/Anichron.Core/Anichron.Core.csproj +++ b/src/Anichron.Core/Anichron.Core.csproj @@ -6,6 +6,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/Anichron.Core/Data/AnichronDbContext.cs b/src/Anichron.Core/Data/AnichronDbContext.cs index f2ad85f0..775e2f7e 100644 --- a/src/Anichron.Core/Data/AnichronDbContext.cs +++ b/src/Anichron.Core/Data/AnichronDbContext.cs @@ -19,6 +19,7 @@ public class AnichronDbContext(DbContextOptions options) : Db public DbSet Interactions => Set(); public DbSet RefreshTokens => Set(); public DbSet Invites => Set(); + public DbSet WorkItems => Set(); // Stryker disable all public async Task ExecuteInTransactionAsync(Func> action, CancellationToken ct = default) @@ -206,7 +207,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) { entity.ToTable(t => t.HasComment( "Issued and revoked by the API; expired rows are swept by the Worker's " - + "TokenCleanupService. One of the few tables both services touch." + + "TokenCleanup work item. One of the few tables both services touch." )); entity.Property(r => r.TokenHash).HasComment( "Hash, never the token itself — a database disclosure must not hand out usable " @@ -310,5 +311,62 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .WithOne(a => a.StorageConfig) .HasForeignKey(a => a.StorageConfigId); }); + + modelBuilder.Entity(entity => + { + entity.ToTable(t => t.HasComment( + "Durable work for the Worker: periodic maintenance today, admin requests later. " + + "Only IWorkQueue inserts rows; every later state change belongs to the Worker's " + + "executor. Delivery is at least once, so handlers start from scratch and converge." + )); + entity.Property(w => w.Kind).HasConversion().HasComment( + "WorkKind, persisted as text. Picks the payload type and the handler through the " + + "registry, never through a switch."); + entity.Property(w => w.Payload).HasColumnType("jsonb").HasComment( + "The kind's IWorkPayload, serialised with System.Text.Json and NodaTime settings."); + entity.Property(w => w.DedupKey).HasComment( + "Opt-in coalescing. While an item with this key is Queued, enqueueing the same key " + + "returns that item instead of adding a second; the partial unique index enforces it."); + entity.Property(w => w.RequestedBy).HasComment( + "The admin who asked for the work; null when a Worker's scheduler enqueued it."); + entity.Property(w => w.Status).HasConversion().HasComment( + "Queued, Running, Succeeded, Failed or Cancelled, persisted as text."); + entity.Property(w => w.Attempt).HasComment( + "Incremented by every claim. Every write a Worker makes while running an item is " + + "fenced by (Id, Attempt), so an attempt that lost its lease changes nothing."); + entity.Property(w => w.Stalls).HasComment( + "Lease expiries, counted by lease recovery. A graceful shutdown hands the item back " + + "without counting one; the item fails as stalled once the limit is reached."); + entity.Property(w => w.ClaimedBy).HasComment( + "The claiming Worker for display: its WORKER__USER or 'all', then its host name."); + entity.Property(w => w.LeaseUntil).HasComment( + "Extended by every heartbeat. A Running item past it belongs to a dead or paused " + + "Worker and goes back to the queue."); + entity.Property(w => w.UpdatedAt).HasComment( + "Moves on every insert, claim, heartbeat and transition, so a reader can ask what " + + "changed since a point in time."); + entity.Property(w => w.ProgressNote).HasComment( + "The latest progress snapshot (done, optional total, note), overwritten by each " + + "heartbeat; no history is kept."); + entity.Property(w => w.Result).HasColumnType("jsonb").HasComment( + "The handler's results summary, written when the item succeeds."); + + entity.HasIndex(w => w.RequestedAt) + .HasFilter(@"""Status"" = 'Queued'") + .HasDatabaseName("ix_workitems_claim"); + entity.HasIndex(w => w.DedupKey) + .IsUnique() + .HasFilter(@"""Status"" = 'Queued' AND ""DedupKey"" IS NOT NULL") + .HasDatabaseName(WorkItemIndexNames.QueuedDedupKeyUnique); + entity.HasIndex(w => w.LeaseUntil) + .HasFilter(@"""Status"" = 'Running'") + .HasDatabaseName("ix_workitems_lease"); + entity.HasIndex(w => w.UpdatedAt); + + entity.HasOne() + .WithMany() + .HasForeignKey(w => w.RequestedBy) + .OnDelete(DeleteBehavior.SetNull); + }); } } diff --git a/src/Anichron.Core/Data/WorkItemIndexNames.cs b/src/Anichron.Core/Data/WorkItemIndexNames.cs new file mode 100644 index 00000000..1ab413a3 --- /dev/null +++ b/src/Anichron.Core/Data/WorkItemIndexNames.cs @@ -0,0 +1,6 @@ +namespace Anichron.Core.Data; + +public static class WorkItemIndexNames +{ + public const string QueuedDedupKeyUnique = "ix_workitems_dedupkey_queued"; +} diff --git a/src/Anichron.Core/Domain/WorkItem.cs b/src/Anichron.Core/Domain/WorkItem.cs new file mode 100644 index 00000000..66c4c14a --- /dev/null +++ b/src/Anichron.Core/Domain/WorkItem.cs @@ -0,0 +1,26 @@ +namespace Anichron.Core.Domain; + +public class WorkItem +{ + public Guid Id { get; set; } + public WorkKind Kind { get; set; } + public required string Payload { get; set; } + public string? DedupKey { get; set; } + public Guid? RequestedBy { get; set; } + public Instant RequestedAt { get; set; } + + public WorkItemStatus Status { get; set; } + public int Attempt { get; set; } + public int Stalls { get; set; } + public string? ClaimedBy { get; set; } + public Instant? StartedAt { get; set; } + public Instant? LeaseUntil { get; set; } + public Instant? FinishedAt { get; set; } + public Instant UpdatedAt { get; set; } + + public long ProgressDone { get; set; } + public long? ProgressTotal { get; set; } + public string? ProgressNote { get; set; } + public string? Result { get; set; } + public string? Error { get; set; } +} diff --git a/src/Anichron.Core/Domain/WorkItemStatus.cs b/src/Anichron.Core/Domain/WorkItemStatus.cs new file mode 100644 index 00000000..40de71ee --- /dev/null +++ b/src/Anichron.Core/Domain/WorkItemStatus.cs @@ -0,0 +1,10 @@ +namespace Anichron.Core.Domain; + +public enum WorkItemStatus +{ + Queued = 1, + Running = 2, + Succeeded = 3, + Failed = 4, + Cancelled = 5, +} diff --git a/src/Anichron.Core/Domain/WorkKind.cs b/src/Anichron.Core/Domain/WorkKind.cs new file mode 100644 index 00000000..3374d5df --- /dev/null +++ b/src/Anichron.Core/Domain/WorkKind.cs @@ -0,0 +1,7 @@ +namespace Anichron.Core.Domain; + +public enum WorkKind +{ + TokenCleanup = 1, + WorkItemRetention = 2, +} diff --git a/src/Anichron.Core/Migrations/20261018080728_AddWorkItems.Designer.cs b/src/Anichron.Core/Migrations/20261018080728_AddWorkItems.Designer.cs new file mode 100644 index 00000000..b21c753d --- /dev/null +++ b/src/Anichron.Core/Migrations/20261018080728_AddWorkItems.Designer.cs @@ -0,0 +1,681 @@ +// +using System; +using Anichron.Core.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Anichron.Core.Migrations +{ + [DbContext(typeof(AnichronDbContext))] + [Migration("20261018080728_AddWorkItems")] + partial class AddWorkItems + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Anichron.Core.Domain.AssetInteraction", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("IsHidden") + .HasColumnType("boolean"); + + b.Property("IsLiked") + .HasColumnType("boolean"); + + b.Property("IsStarred") + .HasColumnType("boolean"); + + b.Property("LastViewed") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("ViewCount") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("AssetId"); + + b.HasIndex("UserId", "AssetId") + .IsUnique(); + + b.ToTable("Interactions", t => + { + t.HasComment("Per-user state for one asset. Never shared between users: two people looking at the same file each get their own row. Drives flashback weighting — hidden excludes, starred always includes and sorts first, liked raises DisplayWeight."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PrimaryAssetId") + .HasColumnType("uuid") + .HasComment("The frame chosen to represent the burst. Restrict on delete: losing the cover would leave a group that cannot be rendered."); + + b.HasKey("Id"); + + b.HasIndex("PrimaryAssetId"); + + b.ToTable("Bursts", t => + { + t.HasComment("Groups a rapid-fire sequence so a flashback shows one memory rather than nine near-identical frames. The table and schema exist; burst DETECTION is not yet implemented in the Worker, so in a running system this table is empty."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Invite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedByUserId") + .HasColumnType("uuid"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(44) + .HasColumnType("character varying(44)") + .HasComment("Hash, never the token itself. 44 chars — base64 of a 32-byte value."); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UsedByUserId") + .HasColumnType("uuid"); + + b.Property("xmin") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.HasKey("Id"); + + b.HasIndex("CreatedByUserId"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UsedByUserId"); + + b.ToTable("Invites", t => + { + t.HasComment("Admin-issued registration tokens. Registration requires one; the first user (bootstrap admin) is exempt. Redemption is a read-then-write race, which is why this is the only table carrying a concurrency token."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("BurstId") + .HasColumnType("uuid"); + + b.Property("ContentHash") + .IsRequired() + .HasColumnType("text") + .HasComment("XXHash64 of the file contents. Dedup is deliberately CONFIG-SCOPED: the same bytes under two storage configs are two assets, which is what makes multi-user work. A global unique index on this column alone would break that and must never be added."); + + b.Property("DateCaptured") + .HasColumnType("timestamp without time zone"); + + b.Property("Day") + .HasColumnType("integer") + .HasComment("See Month — the pair backs IX_MediaAsset_Flashback."); + + b.Property("FileName") + .IsRequired() + .HasColumnType("text"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text") + .HasComment("Path relative to the storage config root. Not to be confused with ProxyFiles.ProxyPath, which is relative to the proxy root."); + + b.Property("IsSoftDeleted") + .HasColumnType("boolean") + .HasComment("Set by reconciliation when a file is gone from the NAS. Soft rather than hard so the user's interactions survive a temporarily unmounted share."); + + b.Property("LastSeenOnNas") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaType") + .IsRequired() + .HasColumnType("text"); + + b.Property("Month") + .HasColumnType("integer") + .HasComment("Stored apart from DateCaptured so \"On This Day\" is an index lookup on (Month, Day) rather than an EXTRACT() scan over every row."); + + b.Property("PairedAssetId") + .HasColumnType("uuid") + .HasComment("Live Photo pairing: the HEIC parent points at its MOV child. SetNull on delete so losing one half leaves the other a usable standalone asset."); + + b.Property("StorageConfigId") + .HasColumnType("uuid"); + + b.Property("Year") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BurstId"); + + b.HasIndex("IsSoftDeleted") + .HasDatabaseName("IX_MediaAssets_Active") + .HasFilter("\"IsSoftDeleted\" = false"); + + b.HasIndex("PairedAssetId"); + + b.HasIndex("Month", "Day") + .HasDatabaseName("IX_MediaAsset_Flashback"); + + b.HasIndex("StorageConfigId", "ContentHash") + .IsUnique() + .HasDatabaseName("ix_mediaassets_storageconfigid_contenthash_active") + .HasFilter("\"IsSoftDeleted\" = false"); + + b.HasIndex("StorageConfigId", "FilePath") + .IsUnique() + .HasDatabaseName("IX_MediaAssets_StorageConfigId_FilePath"); + + b.ToTable("MediaAssets", t => + { + t.HasComment("One row per media file, scoped to a storage config. Written ONLY by the Worker — the API never writes media data."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Metadata", b => + { + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("Blurhash") + .HasColumnType("text") + .HasComment("BlurHash placeholder string (4x3 components), computed by the Worker from the upright image. Kept here rather than as a proxy file so a response needs no disk read for it. Null for videos, and for images ingested before the column existed, which are not backfilled."); + + b.Property("CameraMake") + .HasColumnType("text"); + + b.Property("CameraModel") + .HasColumnType("text"); + + b.Property("DurationInSeconds") + .HasColumnType("integer"); + + b.Property("Height") + .HasColumnType("integer"); + + b.Property("Latitude") + .HasColumnType("double precision"); + + b.Property("LensModel") + .HasColumnType("text"); + + b.Property("Longitude") + .HasColumnType("double precision"); + + b.Property("Orientation") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1) + .HasComment("Raw EXIF orientation tag, 1-8 (1 = upright, also used when absent or invalid). Stored raw because 5 and 7 combine a rotation with a mirror and cannot be expressed as degrees; a reader converts, and for 5-8 swaps Width/Height. Proxies keep this tag on unrotated pixels, so do not rotate them twice."); + + b.Property("Width") + .HasColumnType("integer"); + + b.HasKey("AssetId"); + + b.ToTable("Metadata", t => + { + t.HasComment("EXIF data, 1:1 with MediaAsset and keyed by it — there is no separate Id. Populated by ExifExtractionMiddleware; every field is best-effort, because a file may carry no EXIF at all."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.ProxyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ProxyPath") + .IsRequired() + .HasColumnType("text") + .HasComment("Path relative to the proxy root, sharded by CONTENT rather than by ingestion attempt so a retry overwrites its own debris instead of stranding it. IProxyDirectoryStrategy is the single place that rule lives."); + + b.Property("ProxyType") + .IsRequired() + .HasColumnType("text") + .HasComment("Thumbnail, WebVideo or FullPreview, persisted as text. The BlurHash placeholder is not a proxy: it is the Metadata.Blurhash column."); + + b.Property("SizeBytes") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("AssetId", "ProxyType") + .IsUnique() + .HasDatabaseName("ix_proxyfiles_assetid_proxytype"); + + b.ToTable("ProxyFiles", t => + { + t.HasComment("One row per generated artefact on the local SSD. An asset has several, but at most one per ProxyType, so of two attempts racing to add the same proxy only one row lands. Written only by ProxyStagingWriter, at the moment the file reaches its final name."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.RefreshToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text") + .HasComment("Hash, never the token itself — a database disclosure must not hand out usable sessions."); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UserId", "ExpiresAt"); + + b.ToTable("RefreshTokens", t => + { + t.HasComment("Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanupService. One of the few tables both services touch."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text") + .HasComment("Always stored trimmed and lowercased: an EF value converter applies IdentityNormalizer.Normalize on every write and to every equality comparison against this column, so the case-sensitive unique index cannot hold two spellings of one name."); + + b.Property("FailedLoginAttempts") + .HasColumnType("integer"); + + b.Property("IsAdmin") + .HasColumnType("boolean"); + + b.Property("IsDisabled") + .HasColumnType("boolean"); + + b.Property("LockedUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("MustChangePassword") + .HasColumnType("boolean") + .HasComment("Set by an admin reset. MustChangePasswordMiddleware then 403s every route except the handful needed to change it."); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text") + .HasComment("Argon2id. The parameters live with the hasher, not here, so rotating them does not require a migration."); + + b.Property("Username") + .IsRequired() + .HasColumnType("text") + .HasComment("Always stored trimmed and lowercased: an EF value converter applies IdentityNormalizer.Normalize on every write and to every equality comparison against this column, so the case-sensitive unique index cannot hold two spellings of one name."); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique() + .HasDatabaseName("ix_users_email"); + + b.HasIndex("Username") + .IsUnique() + .HasDatabaseName("ix_users_username"); + + b.ToTable("Users", t => + { + t.HasComment("Owns storage configs and interactions. All identity writes happen in the API; the Worker only reads."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("RootPath") + .IsRequired() + .HasColumnType("text") + .HasComment("GLOBALLY unique, not per user: a path cannot be assigned to two configs across all users. Two users indexing the same directory would duplicate every asset and let one user's config removal cascade away the other's rows."); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("RootPath") + .IsUnique() + .HasDatabaseName("IX_StorageConfigs_RootPath"); + + b.HasIndex("UserId"); + + b.ToTable("StorageConfigs", t => + { + t.HasComment("A NAS root assigned to one user. This is the boundary that scopes assets to a user and lets several Workers each monitor a separate config."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasComment("Incremented by every claim. Every write a Worker makes while running an item is fenced by (Id, Attempt), so an attempt that lost its lease changes nothing."); + + b.Property("ClaimedBy") + .HasColumnType("text") + .HasComment("The claiming Worker for display: its WORKER__USER or 'all', then its host name."); + + b.Property("DedupKey") + .HasColumnType("text") + .HasComment("Opt-in coalescing. While an item with this key is Queued, enqueueing the same key returns that item instead of adding a second; the partial unique index enforces it."); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Kind") + .IsRequired() + .HasColumnType("text") + .HasComment("WorkKind, persisted as text. Picks the payload type and the handler through the registry, never through a switch."); + + b.Property("LeaseUntil") + .HasColumnType("timestamp with time zone") + .HasComment("Extended by every heartbeat. A Running item past it belongs to a dead or paused Worker and goes back to the queue."); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasComment("The kind's IWorkPayload, serialised with System.Text.Json and NodaTime settings."); + + b.Property("ProgressDone") + .HasColumnType("bigint"); + + b.Property("ProgressNote") + .HasColumnType("text") + .HasComment("The latest progress snapshot (done, optional total, note), overwritten by each heartbeat; no history is kept."); + + b.Property("ProgressTotal") + .HasColumnType("bigint"); + + b.Property("RequestedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RequestedBy") + .HasColumnType("uuid") + .HasComment("The admin who asked for the work; null when a Worker's scheduler enqueued it."); + + b.Property("Result") + .HasColumnType("jsonb") + .HasComment("The handler's results summary, written when the item succeeds."); + + b.Property("Stalls") + .HasColumnType("integer") + .HasComment("Lease expiries, counted by lease recovery. A graceful shutdown hands the item back without counting one; the item fails as stalled once the limit is reached."); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasComment("Queued, Running, Succeeded, Failed or Cancelled, persisted as text."); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasComment("Moves on every insert, claim, heartbeat and transition, so a reader can ask what changed since a point in time."); + + b.HasKey("Id"); + + b.HasIndex("DedupKey") + .IsUnique() + .HasDatabaseName("ix_workitems_dedupkey_queued") + .HasFilter("\"Status\" = 'Queued' AND \"DedupKey\" IS NOT NULL"); + + b.HasIndex("LeaseUntil") + .HasDatabaseName("ix_workitems_lease") + .HasFilter("\"Status\" = 'Running'"); + + b.HasIndex("RequestedAt") + .HasDatabaseName("ix_workitems_claim") + .HasFilter("\"Status\" = 'Queued'"); + + b.HasIndex("RequestedBy"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("WorkItems", t => + { + t.HasComment("Durable work for the Worker: periodic maintenance today, admin requests later. Only IWorkQueue inserts rows; every later state change belongs to the Worker's executor. Delivery is at least once, so handlers start from scratch and converge."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.AssetInteraction", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithMany("Interactions") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("Interactions") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "PrimaryAsset") + .WithMany() + .HasForeignKey("PrimaryAssetId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("PrimaryAsset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Invite", b => + { + b.HasOne("Anichron.Core.Domain.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedByUserId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.User", "UsedBy") + .WithMany() + .HasForeignKey("UsedByUserId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("CreatedBy"); + + b.Navigation("UsedBy"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.HasOne("Anichron.Core.Domain.Burst", "Burst") + .WithMany("Assets") + .HasForeignKey("BurstId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.MediaAsset", "PairedAsset") + .WithMany() + .HasForeignKey("PairedAssetId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.UserStorageConfig", "StorageConfig") + .WithMany("Assets") + .HasForeignKey("StorageConfigId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Burst"); + + b.Navigation("PairedAsset"); + + b.Navigation("StorageConfig"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Metadata", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithOne("Metadata") + .HasForeignKey("Anichron.Core.Domain.Metadata", "AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.ProxyFile", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithMany("ProxyFiles") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.RefreshToken", b => + { + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("RefreshTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("StorageConfigs") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.HasOne("Anichron.Core.Domain.User", null) + .WithMany() + .HasForeignKey("RequestedBy") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.Navigation("Assets"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.Navigation("Interactions"); + + b.Navigation("Metadata"); + + b.Navigation("ProxyFiles"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.User", b => + { + b.Navigation("Interactions"); + + b.Navigation("RefreshTokens"); + + b.Navigation("StorageConfigs"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.Navigation("Assets"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Anichron.Core/Migrations/20261018080728_AddWorkItems.cs b/src/Anichron.Core/Migrations/20261018080728_AddWorkItems.cs new file mode 100644 index 00000000..e412c9f3 --- /dev/null +++ b/src/Anichron.Core/Migrations/20261018080728_AddWorkItems.cs @@ -0,0 +1,88 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using NodaTime; + +#nullable disable + +namespace Anichron.Core.Migrations +{ + /// + public partial class AddWorkItems : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "WorkItems", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + Kind = table.Column(type: "text", nullable: false, comment: "WorkKind, persisted as text. Picks the payload type and the handler through the registry, never through a switch."), + Payload = table.Column(type: "jsonb", nullable: false, comment: "The kind's IWorkPayload, serialised with System.Text.Json and NodaTime settings."), + DedupKey = table.Column(type: "text", nullable: true, comment: "Opt-in coalescing. While an item with this key is Queued, enqueueing the same key returns that item instead of adding a second; the partial unique index enforces it."), + RequestedBy = table.Column(type: "uuid", nullable: true, comment: "The admin who asked for the work; null when a Worker's scheduler enqueued it."), + RequestedAt = table.Column(type: "timestamp with time zone", nullable: false), + Status = table.Column(type: "text", nullable: false, comment: "Queued, Running, Succeeded, Failed or Cancelled, persisted as text."), + Attempt = table.Column(type: "integer", nullable: false, comment: "Incremented by every claim. Every write a Worker makes while running an item is fenced by (Id, Attempt), so an attempt that lost its lease changes nothing."), + Stalls = table.Column(type: "integer", nullable: false, comment: "Lease expiries, counted by lease recovery. A graceful shutdown hands the item back without counting one; the item fails as stalled once the limit is reached."), + ClaimedBy = table.Column(type: "text", nullable: true, comment: "The claiming Worker for display: its WORKER__USER or 'all', then its host name."), + StartedAt = table.Column(type: "timestamp with time zone", nullable: true), + LeaseUntil = table.Column(type: "timestamp with time zone", nullable: true, comment: "Extended by every heartbeat. A Running item past it belongs to a dead or paused Worker and goes back to the queue."), + FinishedAt = table.Column(type: "timestamp with time zone", nullable: true), + UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false, comment: "Moves on every insert, claim, heartbeat and transition, so a reader can ask what changed since a point in time."), + ProgressDone = table.Column(type: "bigint", nullable: false), + ProgressTotal = table.Column(type: "bigint", nullable: true), + ProgressNote = table.Column(type: "text", nullable: true, comment: "The latest progress snapshot (done, optional total, note), overwritten by each heartbeat; no history is kept."), + Result = table.Column(type: "jsonb", nullable: true, comment: "The handler's results summary, written when the item succeeds."), + Error = table.Column(type: "text", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_WorkItems", x => x.Id); + table.ForeignKey( + name: "FK_WorkItems_Users_RequestedBy", + column: x => x.RequestedBy, + principalTable: "Users", + principalColumn: "Id", + onDelete: ReferentialAction.SetNull); + }, + comment: "Durable work for the Worker: periodic maintenance today, admin requests later. Only IWorkQueue inserts rows; every later state change belongs to the Worker's executor. Delivery is at least once, so handlers start from scratch and converge."); + + migrationBuilder.CreateIndex( + name: "ix_workitems_claim", + table: "WorkItems", + column: "RequestedAt", + filter: "\"Status\" = 'Queued'"); + + migrationBuilder.CreateIndex( + name: "ix_workitems_dedupkey_queued", + table: "WorkItems", + column: "DedupKey", + unique: true, + filter: "\"Status\" = 'Queued' AND \"DedupKey\" IS NOT NULL"); + + migrationBuilder.CreateIndex( + name: "ix_workitems_lease", + table: "WorkItems", + column: "LeaseUntil", + filter: "\"Status\" = 'Running'"); + + migrationBuilder.CreateIndex( + name: "IX_WorkItems_RequestedBy", + table: "WorkItems", + column: "RequestedBy"); + + migrationBuilder.CreateIndex( + name: "IX_WorkItems_UpdatedAt", + table: "WorkItems", + column: "UpdatedAt"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "WorkItems"); + } + } +} diff --git a/src/Anichron.Core/Migrations/20261018090838_SweepRefreshTokensAsWorkItem.Designer.cs b/src/Anichron.Core/Migrations/20261018090838_SweepRefreshTokensAsWorkItem.Designer.cs new file mode 100644 index 00000000..67302f22 --- /dev/null +++ b/src/Anichron.Core/Migrations/20261018090838_SweepRefreshTokensAsWorkItem.Designer.cs @@ -0,0 +1,681 @@ +// +using System; +using Anichron.Core.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NodaTime; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Anichron.Core.Migrations +{ + [DbContext(typeof(AnichronDbContext))] + [Migration("20261018090838_SweepRefreshTokensAsWorkItem")] + partial class SweepRefreshTokensAsWorkItem + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Anichron.Core.Domain.AssetInteraction", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("IsHidden") + .HasColumnType("boolean"); + + b.Property("IsLiked") + .HasColumnType("boolean"); + + b.Property("IsStarred") + .HasColumnType("boolean"); + + b.Property("LastViewed") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("ViewCount") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("AssetId"); + + b.HasIndex("UserId", "AssetId") + .IsUnique(); + + b.ToTable("Interactions", t => + { + t.HasComment("Per-user state for one asset. Never shared between users: two people looking at the same file each get their own row. Drives flashback weighting — hidden excludes, starred always includes and sorts first, liked raises DisplayWeight."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("PrimaryAssetId") + .HasColumnType("uuid") + .HasComment("The frame chosen to represent the burst. Restrict on delete: losing the cover would leave a group that cannot be rendered."); + + b.HasKey("Id"); + + b.HasIndex("PrimaryAssetId"); + + b.ToTable("Bursts", t => + { + t.HasComment("Groups a rapid-fire sequence so a flashback shows one memory rather than nine near-identical frames. The table and schema exist; burst DETECTION is not yet implemented in the Worker, so in a running system this table is empty."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Invite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedByUserId") + .HasColumnType("uuid"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(44) + .HasColumnType("character varying(44)") + .HasComment("Hash, never the token itself. 44 chars — base64 of a 32-byte value."); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UsedByUserId") + .HasColumnType("uuid"); + + b.Property("xmin") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.HasKey("Id"); + + b.HasIndex("CreatedByUserId"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UsedByUserId"); + + b.ToTable("Invites", t => + { + t.HasComment("Admin-issued registration tokens. Registration requires one; the first user (bootstrap admin) is exempt. Redemption is a read-then-write race, which is why this is the only table carrying a concurrency token."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("BurstId") + .HasColumnType("uuid"); + + b.Property("ContentHash") + .IsRequired() + .HasColumnType("text") + .HasComment("XXHash64 of the file contents. Dedup is deliberately CONFIG-SCOPED: the same bytes under two storage configs are two assets, which is what makes multi-user work. A global unique index on this column alone would break that and must never be added."); + + b.Property("DateCaptured") + .HasColumnType("timestamp without time zone"); + + b.Property("Day") + .HasColumnType("integer") + .HasComment("See Month — the pair backs IX_MediaAsset_Flashback."); + + b.Property("FileName") + .IsRequired() + .HasColumnType("text"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text") + .HasComment("Path relative to the storage config root. Not to be confused with ProxyFiles.ProxyPath, which is relative to the proxy root."); + + b.Property("IsSoftDeleted") + .HasColumnType("boolean") + .HasComment("Set by reconciliation when a file is gone from the NAS. Soft rather than hard so the user's interactions survive a temporarily unmounted share."); + + b.Property("LastSeenOnNas") + .HasColumnType("timestamp with time zone"); + + b.Property("MediaType") + .IsRequired() + .HasColumnType("text"); + + b.Property("Month") + .HasColumnType("integer") + .HasComment("Stored apart from DateCaptured so \"On This Day\" is an index lookup on (Month, Day) rather than an EXTRACT() scan over every row."); + + b.Property("PairedAssetId") + .HasColumnType("uuid") + .HasComment("Live Photo pairing: the HEIC parent points at its MOV child. SetNull on delete so losing one half leaves the other a usable standalone asset."); + + b.Property("StorageConfigId") + .HasColumnType("uuid"); + + b.Property("Year") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BurstId"); + + b.HasIndex("IsSoftDeleted") + .HasDatabaseName("IX_MediaAssets_Active") + .HasFilter("\"IsSoftDeleted\" = false"); + + b.HasIndex("PairedAssetId"); + + b.HasIndex("Month", "Day") + .HasDatabaseName("IX_MediaAsset_Flashback"); + + b.HasIndex("StorageConfigId", "ContentHash") + .IsUnique() + .HasDatabaseName("ix_mediaassets_storageconfigid_contenthash_active") + .HasFilter("\"IsSoftDeleted\" = false"); + + b.HasIndex("StorageConfigId", "FilePath") + .IsUnique() + .HasDatabaseName("IX_MediaAssets_StorageConfigId_FilePath"); + + b.ToTable("MediaAssets", t => + { + t.HasComment("One row per media file, scoped to a storage config. Written ONLY by the Worker — the API never writes media data."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Metadata", b => + { + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("Blurhash") + .HasColumnType("text") + .HasComment("BlurHash placeholder string (4x3 components), computed by the Worker from the upright image. Kept here rather than as a proxy file so a response needs no disk read for it. Null for videos, and for images ingested before the column existed, which are not backfilled."); + + b.Property("CameraMake") + .HasColumnType("text"); + + b.Property("CameraModel") + .HasColumnType("text"); + + b.Property("DurationInSeconds") + .HasColumnType("integer"); + + b.Property("Height") + .HasColumnType("integer"); + + b.Property("Latitude") + .HasColumnType("double precision"); + + b.Property("LensModel") + .HasColumnType("text"); + + b.Property("Longitude") + .HasColumnType("double precision"); + + b.Property("Orientation") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1) + .HasComment("Raw EXIF orientation tag, 1-8 (1 = upright, also used when absent or invalid). Stored raw because 5 and 7 combine a rotation with a mirror and cannot be expressed as degrees; a reader converts, and for 5-8 swaps Width/Height. Proxies keep this tag on unrotated pixels, so do not rotate them twice."); + + b.Property("Width") + .HasColumnType("integer"); + + b.HasKey("AssetId"); + + b.ToTable("Metadata", t => + { + t.HasComment("EXIF data, 1:1 with MediaAsset and keyed by it — there is no separate Id. Populated by ExifExtractionMiddleware; every field is best-effort, because a file may carry no EXIF at all."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.ProxyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("AssetId") + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ProxyPath") + .IsRequired() + .HasColumnType("text") + .HasComment("Path relative to the proxy root, sharded by CONTENT rather than by ingestion attempt so a retry overwrites its own debris instead of stranding it. IProxyDirectoryStrategy is the single place that rule lives."); + + b.Property("ProxyType") + .IsRequired() + .HasColumnType("text") + .HasComment("Thumbnail, WebVideo or FullPreview, persisted as text. The BlurHash placeholder is not a proxy: it is the Metadata.Blurhash column."); + + b.Property("SizeBytes") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("AssetId", "ProxyType") + .IsUnique() + .HasDatabaseName("ix_proxyfiles_assetid_proxytype"); + + b.ToTable("ProxyFiles", t => + { + t.HasComment("One row per generated artefact on the local SSD. An asset has several, but at most one per ProxyType, so of two attempts racing to add the same proxy only one row lands. Written only by ProxyStagingWriter, at the moment the file reaches its final name."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.RefreshToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RevokedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("TokenHash") + .IsRequired() + .HasColumnType("text") + .HasComment("Hash, never the token itself — a database disclosure must not hand out usable sessions."); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UserId", "ExpiresAt"); + + b.ToTable("RefreshTokens", t => + { + t.HasComment("Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanup work item. One of the few tables both services touch."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text") + .HasComment("Always stored trimmed and lowercased: an EF value converter applies IdentityNormalizer.Normalize on every write and to every equality comparison against this column, so the case-sensitive unique index cannot hold two spellings of one name."); + + b.Property("FailedLoginAttempts") + .HasColumnType("integer"); + + b.Property("IsAdmin") + .HasColumnType("boolean"); + + b.Property("IsDisabled") + .HasColumnType("boolean"); + + b.Property("LockedUntil") + .HasColumnType("timestamp with time zone"); + + b.Property("MustChangePassword") + .HasColumnType("boolean") + .HasComment("Set by an admin reset. MustChangePasswordMiddleware then 403s every route except the handful needed to change it."); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text") + .HasComment("Argon2id. The parameters live with the hasher, not here, so rotating them does not require a migration."); + + b.Property("Username") + .IsRequired() + .HasColumnType("text") + .HasComment("Always stored trimmed and lowercased: an EF value converter applies IdentityNormalizer.Normalize on every write and to every equality comparison against this column, so the case-sensitive unique index cannot hold two spellings of one name."); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique() + .HasDatabaseName("ix_users_email"); + + b.HasIndex("Username") + .IsUnique() + .HasDatabaseName("ix_users_username"); + + b.ToTable("Users", t => + { + t.HasComment("Owns storage configs and interactions. All identity writes happen in the API; the Worker only reads."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("RootPath") + .IsRequired() + .HasColumnType("text") + .HasComment("GLOBALLY unique, not per user: a path cannot be assigned to two configs across all users. Two users indexing the same directory would duplicate every asset and let one user's config removal cascade away the other's rows."); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("RootPath") + .IsUnique() + .HasDatabaseName("IX_StorageConfigs_RootPath"); + + b.HasIndex("UserId"); + + b.ToTable("StorageConfigs", t => + { + t.HasComment("A NAS root assigned to one user. This is the boundary that scopes assets to a user and lets several Workers each monitor a separate config."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasComment("Incremented by every claim. Every write a Worker makes while running an item is fenced by (Id, Attempt), so an attempt that lost its lease changes nothing."); + + b.Property("ClaimedBy") + .HasColumnType("text") + .HasComment("The claiming Worker for display: its WORKER__USER or 'all', then its host name."); + + b.Property("DedupKey") + .HasColumnType("text") + .HasComment("Opt-in coalescing. While an item with this key is Queued, enqueueing the same key returns that item instead of adding a second; the partial unique index enforces it."); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Kind") + .IsRequired() + .HasColumnType("text") + .HasComment("WorkKind, persisted as text. Picks the payload type and the handler through the registry, never through a switch."); + + b.Property("LeaseUntil") + .HasColumnType("timestamp with time zone") + .HasComment("Extended by every heartbeat. A Running item past it belongs to a dead or paused Worker and goes back to the queue."); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasComment("The kind's IWorkPayload, serialised with System.Text.Json and NodaTime settings."); + + b.Property("ProgressDone") + .HasColumnType("bigint"); + + b.Property("ProgressNote") + .HasColumnType("text") + .HasComment("The latest progress snapshot (done, optional total, note), overwritten by each heartbeat; no history is kept."); + + b.Property("ProgressTotal") + .HasColumnType("bigint"); + + b.Property("RequestedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RequestedBy") + .HasColumnType("uuid") + .HasComment("The admin who asked for the work; null when a Worker's scheduler enqueued it."); + + b.Property("Result") + .HasColumnType("jsonb") + .HasComment("The handler's results summary, written when the item succeeds."); + + b.Property("Stalls") + .HasColumnType("integer") + .HasComment("Lease expiries, counted by lease recovery. A graceful shutdown hands the item back without counting one; the item fails as stalled once the limit is reached."); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasComment("Queued, Running, Succeeded, Failed or Cancelled, persisted as text."); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasComment("Moves on every insert, claim, heartbeat and transition, so a reader can ask what changed since a point in time."); + + b.HasKey("Id"); + + b.HasIndex("DedupKey") + .IsUnique() + .HasDatabaseName("ix_workitems_dedupkey_queued") + .HasFilter("\"Status\" = 'Queued' AND \"DedupKey\" IS NOT NULL"); + + b.HasIndex("LeaseUntil") + .HasDatabaseName("ix_workitems_lease") + .HasFilter("\"Status\" = 'Running'"); + + b.HasIndex("RequestedAt") + .HasDatabaseName("ix_workitems_claim") + .HasFilter("\"Status\" = 'Queued'"); + + b.HasIndex("RequestedBy"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("WorkItems", t => + { + t.HasComment("Durable work for the Worker: periodic maintenance today, admin requests later. Only IWorkQueue inserts rows; every later state change belongs to the Worker's executor. Delivery is at least once, so handlers start from scratch and converge."); + }); + }); + + modelBuilder.Entity("Anichron.Core.Domain.AssetInteraction", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithMany("Interactions") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("Interactions") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "PrimaryAsset") + .WithMany() + .HasForeignKey("PrimaryAssetId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("PrimaryAsset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Invite", b => + { + b.HasOne("Anichron.Core.Domain.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedByUserId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.User", "UsedBy") + .WithMany() + .HasForeignKey("UsedByUserId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("CreatedBy"); + + b.Navigation("UsedBy"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.HasOne("Anichron.Core.Domain.Burst", "Burst") + .WithMany("Assets") + .HasForeignKey("BurstId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.MediaAsset", "PairedAsset") + .WithMany() + .HasForeignKey("PairedAssetId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Anichron.Core.Domain.UserStorageConfig", "StorageConfig") + .WithMany("Assets") + .HasForeignKey("StorageConfigId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Burst"); + + b.Navigation("PairedAsset"); + + b.Navigation("StorageConfig"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Metadata", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithOne("Metadata") + .HasForeignKey("Anichron.Core.Domain.Metadata", "AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.ProxyFile", b => + { + b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") + .WithMany("ProxyFiles") + .HasForeignKey("AssetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Asset"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.RefreshToken", b => + { + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("RefreshTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.HasOne("Anichron.Core.Domain.User", "User") + .WithMany("StorageConfigs") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.HasOne("Anichron.Core.Domain.User", null) + .WithMany() + .HasForeignKey("RequestedBy") + .OnDelete(DeleteBehavior.SetNull); + }); + + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => + { + b.Navigation("Assets"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.MediaAsset", b => + { + b.Navigation("Interactions"); + + b.Navigation("Metadata"); + + b.Navigation("ProxyFiles"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.User", b => + { + b.Navigation("Interactions"); + + b.Navigation("RefreshTokens"); + + b.Navigation("StorageConfigs"); + }); + + modelBuilder.Entity("Anichron.Core.Domain.UserStorageConfig", b => + { + b.Navigation("Assets"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Anichron.Core/Migrations/20261018090838_SweepRefreshTokensAsWorkItem.cs b/src/Anichron.Core/Migrations/20261018090838_SweepRefreshTokensAsWorkItem.cs new file mode 100644 index 00000000..46879846 --- /dev/null +++ b/src/Anichron.Core/Migrations/20261018090838_SweepRefreshTokensAsWorkItem.cs @@ -0,0 +1,28 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Anichron.Core.Migrations +{ + /// + public partial class SweepRefreshTokensAsWorkItem : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterTable( + name: "RefreshTokens", + comment: "Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanup work item. One of the few tables both services touch.", + oldComment: "Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanupService. One of the few tables both services touch."); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterTable( + name: "RefreshTokens", + comment: "Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanupService. One of the few tables both services touch.", + oldComment: "Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanup work item. One of the few tables both services touch."); + } + } +} diff --git a/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs b/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs index 1744dd63..3d6df1e1 100644 --- a/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs +++ b/src/Anichron.Core/Migrations/AnichronDbContextModelSnapshot.cs @@ -333,7 +333,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("RefreshTokens", t => { - t.HasComment("Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanupService. One of the few tables both services touch."); + t.HasComment("Issued and revoked by the API; expired rows are swept by the Worker's TokenCleanup work item. One of the few tables both services touch."); }); }); @@ -421,6 +421,106 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasComment("Incremented by every claim. Every write a Worker makes while running an item is fenced by (Id, Attempt), so an attempt that lost its lease changes nothing."); + + b.Property("ClaimedBy") + .HasColumnType("text") + .HasComment("The claiming Worker for display: its WORKER__USER or 'all', then its host name."); + + b.Property("DedupKey") + .HasColumnType("text") + .HasComment("Opt-in coalescing. While an item with this key is Queued, enqueueing the same key returns that item instead of adding a second; the partial unique index enforces it."); + + b.Property("Error") + .HasColumnType("text"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Kind") + .IsRequired() + .HasColumnType("text") + .HasComment("WorkKind, persisted as text. Picks the payload type and the handler through the registry, never through a switch."); + + b.Property("LeaseUntil") + .HasColumnType("timestamp with time zone") + .HasComment("Extended by every heartbeat. A Running item past it belongs to a dead or paused Worker and goes back to the queue."); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasComment("The kind's IWorkPayload, serialised with System.Text.Json and NodaTime settings."); + + b.Property("ProgressDone") + .HasColumnType("bigint"); + + b.Property("ProgressNote") + .HasColumnType("text") + .HasComment("The latest progress snapshot (done, optional total, note), overwritten by each heartbeat; no history is kept."); + + b.Property("ProgressTotal") + .HasColumnType("bigint"); + + b.Property("RequestedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RequestedBy") + .HasColumnType("uuid") + .HasComment("The admin who asked for the work; null when a Worker's scheduler enqueued it."); + + b.Property("Result") + .HasColumnType("jsonb") + .HasComment("The handler's results summary, written when the item succeeds."); + + b.Property("Stalls") + .HasColumnType("integer") + .HasComment("Lease expiries, counted by lease recovery. A graceful shutdown hands the item back without counting one; the item fails as stalled once the limit is reached."); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text") + .HasComment("Queued, Running, Succeeded, Failed or Cancelled, persisted as text."); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasComment("Moves on every insert, claim, heartbeat and transition, so a reader can ask what changed since a point in time."); + + b.HasKey("Id"); + + b.HasIndex("DedupKey") + .IsUnique() + .HasDatabaseName("ix_workitems_dedupkey_queued") + .HasFilter("\"Status\" = 'Queued' AND \"DedupKey\" IS NOT NULL"); + + b.HasIndex("LeaseUntil") + .HasDatabaseName("ix_workitems_lease") + .HasFilter("\"Status\" = 'Running'"); + + b.HasIndex("RequestedAt") + .HasDatabaseName("ix_workitems_claim") + .HasFilter("\"Status\" = 'Queued'"); + + b.HasIndex("RequestedBy"); + + b.HasIndex("UpdatedAt"); + + b.ToTable("WorkItems", t => + { + t.HasComment("Durable work for the Worker: periodic maintenance today, admin requests later. Only IWorkQueue inserts rows; every later state change belongs to the Worker's executor. Delivery is at least once, so handlers start from scratch and converge."); + }); + }); + modelBuilder.Entity("Anichron.Core.Domain.AssetInteraction", b => { b.HasOne("Anichron.Core.Domain.MediaAsset", "Asset") @@ -537,6 +637,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("User"); }); + modelBuilder.Entity("Anichron.Core.Domain.WorkItem", b => + { + b.HasOne("Anichron.Core.Domain.User", null) + .WithMany() + .HasForeignKey("RequestedBy") + .OnDelete(DeleteBehavior.SetNull); + }); + modelBuilder.Entity("Anichron.Core.Domain.Burst", b => { b.Navigation("Assets"); diff --git a/src/Anichron.Core/Work/IWorkHandler.cs b/src/Anichron.Core/Work/IWorkHandler.cs new file mode 100644 index 00000000..9ceaed9b --- /dev/null +++ b/src/Anichron.Core/Work/IWorkHandler.cs @@ -0,0 +1,13 @@ +namespace Anichron.Core.Work; + +/// +/// Runs one kind of work item. Delivery is at least once: each run starts from scratch and +/// converges, and its side effects are safe to repeat. The returned summary, if any, becomes the +/// item's result. +/// +/// The payload of the kind this handler runs. +public interface IWorkHandler + where TPayload : IWorkPayload +{ + Task RunAsync(TPayload payload, IWorkProgress progress, CancellationToken ct); +} diff --git a/src/Anichron.Core/Work/IWorkPayload.cs b/src/Anichron.Core/Work/IWorkPayload.cs new file mode 100644 index 00000000..5f33d800 --- /dev/null +++ b/src/Anichron.Core/Work/IWorkPayload.cs @@ -0,0 +1,14 @@ +using Anichron.Core.Domain; + +namespace Anichron.Core.Work; + +public interface IWorkPayload +{ + static abstract WorkKind Kind { get; } + + /// + /// Coalesces requests: while an item with this key is queued, enqueueing another returns it. + /// Null, the default, keeps every request a separate item. + /// + string? DedupKey => null; +} diff --git a/src/Anichron.Core/Work/IWorkProgress.cs b/src/Anichron.Core/Work/IWorkProgress.cs new file mode 100644 index 00000000..1b22f3f0 --- /dev/null +++ b/src/Anichron.Core/Work/IWorkProgress.cs @@ -0,0 +1,15 @@ +namespace Anichron.Core.Work; + +public interface IWorkProgress +{ + /// Replaces the snapshot. + /// Units of work finished so far. + /// Units of work in all, or null while that is unknown. + /// A short description of the current step. + void Report(long done, long? total = null, string? note = null); +} + +public sealed record WorkProgress(long Done, long? Total, string? Note) +{ + public static WorkProgress None { get; } = new(0, null, null); +} diff --git a/src/Anichron.Core/Work/Payloads.cs b/src/Anichron.Core/Work/Payloads.cs new file mode 100644 index 00000000..a619a063 --- /dev/null +++ b/src/Anichron.Core/Work/Payloads.cs @@ -0,0 +1,17 @@ +using Anichron.Core.Domain; + +namespace Anichron.Core.Work; + +public sealed record TokenCleanupPayload : IWorkPayload +{ + public static WorkKind Kind => WorkKind.TokenCleanup; + + string IWorkPayload.DedupKey => "token-cleanup"; +} + +public sealed record WorkItemRetentionPayload : IWorkPayload +{ + public static WorkKind Kind => WorkKind.WorkItemRetention; + + string IWorkPayload.DedupKey => "work-item-retention"; +} diff --git a/src/Anichron.Core/Work/WorkJson.cs b/src/Anichron.Core/Work/WorkJson.cs new file mode 100644 index 00000000..af1e094b --- /dev/null +++ b/src/Anichron.Core/Work/WorkJson.cs @@ -0,0 +1,22 @@ +using NodaTime.Serialization.SystemTextJson; +using System.Text.Json; + +namespace Anichron.Core.Work; + +public static class WorkJson +{ + public static JsonSerializerOptions Options + => new JsonSerializerOptions(JsonSerializerDefaults.Web).ConfigureForNodaTime(DateTimeZoneProviders.Tzdb); + + public static string Serialize(TPayload payload) + where TPayload : IWorkPayload + => JsonSerializer.Serialize(payload, Options); + + public static TPayload Deserialize(string json) + where TPayload : IWorkPayload + => JsonSerializer.Deserialize(json, Options) + ?? throw new JsonException($"The payload of a {TPayload.Kind} item is null."); + + public static string? SerializeResult(object? result) + => result is null ? null : JsonSerializer.Serialize(result, result.GetType(), Options); +} diff --git a/src/Anichron.Core/Work/WorkQueue.cs b/src/Anichron.Core/Work/WorkQueue.cs new file mode 100644 index 00000000..2b8c3784 --- /dev/null +++ b/src/Anichron.Core/Work/WorkQueue.cs @@ -0,0 +1,72 @@ +using Anichron.Core.Data; +using Anichron.Core.Domain; +using Microsoft.EntityFrameworkCore; + +namespace Anichron.Core.Work; + +public interface IWorkQueue +{ + /// + /// Adds a queued item, or returns the id of the queued item that already holds the payload's + /// dedup key. Saves at once, so call it on a context with no other pending changes. + /// + /// The payload type, which names the kind. + /// What the handler receives. + /// The requesting admin, or null when scheduled. + /// Cancels the enqueue. + /// The id of the queued item. + Task EnqueueAsync(TPayload payload, Guid? requestedBy, CancellationToken ct) + where TPayload : IWorkPayload; +} + +public sealed class EfWorkQueue(AnichronDbContext db, IClock clock, IGuidFactory guids) : IWorkQueue +{ + public async Task EnqueueAsync(TPayload payload, Guid? requestedBy, CancellationToken ct) + where TPayload : IWorkPayload + { + var dedupKey = payload.DedupKey; + while (true) + { + if (dedupKey is not null) + { + var waiting = await db.WorkItems + .Where(w => w.DedupKey == dedupKey && w.Status == WorkItemStatus.Queued) + .Select(w => (Guid?)w.Id) + .FirstOrDefaultAsync(ct); + if (waiting is { } id) + return id; + } + + var now = clock.GetCurrentInstant(); + var item = new WorkItem + { + Id = guids.NewGuid(), + Kind = TPayload.Kind, + Payload = WorkJson.Serialize(payload), + DedupKey = dedupKey, + RequestedBy = requestedBy, + RequestedAt = now, + Status = WorkItemStatus.Queued, + UpdatedAt = now, + }; + if (await TryInsertAsync(item, ct)) + return item.Id; + } + } + + // Stryker disable all + private async Task TryInsertAsync(WorkItem item, CancellationToken ct) + { + db.WorkItems.Add(item); + try + { + await db.SaveChangesAsync(ct); + return true; + } + catch (DbUpdateException ex) when (ex.IsUniqueViolation(WorkItemIndexNames.QueuedDedupKeyUnique)) + { + db.Entry(item).State = EntityState.Detached; + return false; + } + } +} diff --git a/src/Anichron.Core/packages.lock.json b/src/Anichron.Core/packages.lock.json index 0f16457f..5bbd4b69 100644 --- a/src/Anichron.Core/packages.lock.json +++ b/src/Anichron.Core/packages.lock.json @@ -46,6 +46,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "Direct", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "Direct", "requested": "[10.0.3, )", diff --git a/src/Anichron.Infrastructure.Tests.Unit/packages.lock.json b/src/Anichron.Infrastructure.Tests.Unit/packages.lock.json index bc483160..7112cd50 100644 --- a/src/Anichron.Infrastructure.Tests.Unit/packages.lock.json +++ b/src/Anichron.Infrastructure.Tests.Unit/packages.lock.json @@ -424,6 +424,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -541,6 +542,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Infrastructure/packages.lock.json b/src/Anichron.Infrastructure/packages.lock.json index 43c4d31d..4d57925c 100644 --- a/src/Anichron.Infrastructure/packages.lock.json +++ b/src/Anichron.Infrastructure/packages.lock.json @@ -230,6 +230,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -250,6 +251,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql.EntityFrameworkCore.PostgreSQL": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Testing.Tests.Unit/packages.lock.json b/src/Anichron.Testing.Tests.Unit/packages.lock.json index 94545596..7d9f36c4 100644 --- a/src/Anichron.Testing.Tests.Unit/packages.lock.json +++ b/src/Anichron.Testing.Tests.Unit/packages.lock.json @@ -381,6 +381,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -474,6 +475,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Testing/packages.lock.json b/src/Anichron.Testing/packages.lock.json index fadcd762..2b15c10e 100644 --- a/src/Anichron.Testing/packages.lock.json +++ b/src/Anichron.Testing/packages.lock.json @@ -204,6 +204,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -237,6 +238,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Tests.Integration/Anichron.Tests.Integration.csproj b/src/Anichron.Tests.Integration/Anichron.Tests.Integration.csproj new file mode 100644 index 00000000..5e3a27fa --- /dev/null +++ b/src/Anichron.Tests.Integration/Anichron.Tests.Integration.csproj @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/src/Anichron.Tests.Integration/Data/MigrateWithAdvisoryLockTests.cs b/src/Anichron.Tests.Integration/Data/MigrateWithAdvisoryLockTests.cs new file mode 100644 index 00000000..0415079e --- /dev/null +++ b/src/Anichron.Tests.Integration/Data/MigrateWithAdvisoryLockTests.cs @@ -0,0 +1,70 @@ +using Anichron.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace Anichron.Tests.Integration.Data; + +public sealed class MigrateWithAdvisoryLockTests(PostgresServer server) +{ + [Fact] + public async Task MigrateWithAdvisoryLockAsync_TwoConcurrentMigrators_BothCompleteAndApplyEachMigrationOnceAsync() + { + var ct = TestContext.Current.CancellationToken; + var connectionString = await server.CreateDatabaseAsync(ct); + await using var first = PostgresServer.Context(connectionString); + await using var second = PostgresServer.Context(connectionString); + + await Task.WhenAll( + first.Database.MigrateWithAdvisoryLockAsync(ct), + second.Database.MigrateWithAdvisoryLockAsync(ct)); + + await using var check = PostgresServer.Context(connectionString); + var history = await check.Database + .SqlQueryRaw("SELECT \"MigrationId\" AS \"Value\" FROM \"__EFMigrationsHistory\" ORDER BY \"MigrationId\"") + .ToListAsync(ct); + history.Should().NotBeEmpty().And.Equal(check.Database.GetMigrations()); + } + + [Fact] + public async Task MigrateWithAdvisoryLockAsync_Finished_ReleasesTheLockAsync() + { + var ct = TestContext.Current.CancellationToken; + var connectionString = await server.CreateDatabaseAsync(ct); + await using (var db = PostgresServer.Context(connectionString)) + await db.Database.MigrateWithAdvisoryLockAsync(ct); + + await using var other = await OpenAsync(connectionString, ct); + + (await TryLockAsync(other, ct)).Should().BeTrue(); + } + + [Fact] + public async Task MigrateWithAdvisoryLockAsync_LockHeldElsewhere_GivesUpWithoutMigratingAsync() + { + var ct = TestContext.Current.CancellationToken; + var connectionString = await server.CreateDatabaseAsync(ct); + await using var holder = await OpenAsync(connectionString, ct); + (await TryLockAsync(holder, ct)).Should().BeTrue(); + + await using var db = PostgresServer.Context(connectionString); + var migrate = () => db.Database.MigrateWithAdvisoryLockAsync(ct, maxAttempts: 1); + + await migrate.Should().ThrowAsync(); + (await db.Database.GetAppliedMigrationsAsync(ct)).Should().BeEmpty(); + } + + private static async Task OpenAsync(string connectionString, CancellationToken ct) + { + var connection = new NpgsqlConnection( + new NpgsqlConnectionStringBuilder(connectionString) { Pooling = false }.ConnectionString); + await connection.OpenAsync(ct); + return connection; + } + + private static async Task TryLockAsync(NpgsqlConnection connection, CancellationToken ct) + { + await using var command = new NpgsqlCommand("SELECT pg_try_advisory_lock(@id)", connection); + command.Parameters.AddWithValue("id", PostgresConstants.MigrationAdvisoryLockId); + return (bool)(await command.ExecuteScalarAsync(ct))!; + } +} diff --git a/src/Anichron.Tests.Integration/Data/RefreshTokenRepositoryTests.cs b/src/Anichron.Tests.Integration/Data/RefreshTokenRepositoryTests.cs new file mode 100644 index 00000000..8c0f5c1c --- /dev/null +++ b/src/Anichron.Tests.Integration/Data/RefreshTokenRepositoryTests.cs @@ -0,0 +1,68 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; +using Microsoft.EntityFrameworkCore; + +namespace Anichron.Tests.Integration.Data; + +public sealed class RefreshTokenRepositoryTests(PostgresServer server) +{ + private static readonly Instant Now = Instant.FromUtc(2026, 10, 18, 12, 0); + + [Fact] + public async Task DeleteExpiredAsync_DeletesOnlyTokensExpiredBeforeTheCutoff_AndCountsThemAsync() + { + var ct = TestContext.Current.CancellationToken; + var connectionString = await server.CreateMigratedDatabaseAsync(ct); + await SeedAsync(connectionString, ct, + ("long-gone", Now - Duration.FromDays(30)), + ("just-expired", Now - Duration.FromMilliseconds(1)), + ("expires-now", Now), + ("still-valid", Now + Duration.FromDays(7))); + + await using (var db = PostgresServer.Context(connectionString)) + { + var deleted = await new EfRefreshTokenRepository(db).DeleteExpiredAsync(Now, ct); + + deleted.Should().Be(2); + } + + await using var check = PostgresServer.Context(connectionString); + (await check.RefreshTokens.Select(t => t.TokenHash).ToListAsync(ct)) + .Should().BeEquivalentTo("expires-now", "still-valid"); + } + + [Fact] + public async Task DeleteExpiredAsync_NothingExpired_DeletesNothingAsync() + { + var ct = TestContext.Current.CancellationToken; + var connectionString = await server.CreateMigratedDatabaseAsync(ct); + await SeedAsync(connectionString, ct, ("still-valid", Now + Duration.FromDays(7))); + + await using var db = PostgresServer.Context(connectionString); + var deleted = await new EfRefreshTokenRepository(db).DeleteExpiredAsync(Now, ct); + + deleted.Should().Be(0); + (await db.RefreshTokens.CountAsync(ct)).Should().Be(1); + } + + private static async Task SeedAsync( + string connectionString, CancellationToken ct, params (string Hash, Instant ExpiresAt)[] tokens) + { + await using var db = PostgresServer.Context(connectionString); + var user = new User { Id = Guid.NewGuid(), Username = "mira", Email = "mira@example.com", PasswordHash = "hash" }; + db.Users.Add(user); + foreach (var (hash, expiresAt) in tokens) + { + db.RefreshTokens.Add(new RefreshToken + { + Id = Guid.NewGuid(), + UserId = user.Id, + TokenHash = hash, + CreatedAt = expiresAt - Duration.FromDays(14), + ExpiresAt = expiresAt, + }); + } + + await db.SaveChangesAsync(ct); + } +} diff --git a/src/Anichron.Tests.Integration/GlobalUsings.cs b/src/Anichron.Tests.Integration/GlobalUsings.cs new file mode 100644 index 00000000..46b07a5c --- /dev/null +++ b/src/Anichron.Tests.Integration/GlobalUsings.cs @@ -0,0 +1,3 @@ +global using FluentAssertions; +global using NodaTime; +global using Xunit; diff --git a/src/Anichron.Tests.Integration/PostgresServer.cs b/src/Anichron.Tests.Integration/PostgresServer.cs new file mode 100644 index 00000000..839e1128 --- /dev/null +++ b/src/Anichron.Tests.Integration/PostgresServer.cs @@ -0,0 +1,46 @@ +using Anichron.Core.Data; +using Microsoft.EntityFrameworkCore; +using Npgsql; +using Testcontainers.PostgreSql; + +[assembly: AssemblyFixture(typeof(Anichron.Tests.Integration.PostgresServer))] + +namespace Anichron.Tests.Integration; + +public sealed class PostgresServer : IAsyncLifetime +{ + public const string Image = "postgres:17-alpine"; + + private readonly PostgreSqlContainer container = new PostgreSqlBuilder(Image).Build(); + + public async ValueTask InitializeAsync() => await container.StartAsync(); + + public ValueTask DisposeAsync() => container.DisposeAsync(); + + public async Task CreateDatabaseAsync(CancellationToken ct) + { + var name = $"test_{Guid.NewGuid():N}"; + await using (var connection = new NpgsqlConnection(container.GetConnectionString())) + { + await connection.OpenAsync(ct); + await using var command = connection.CreateCommand(); + command.CommandText = $"CREATE DATABASE \"{name}\""; + await command.ExecuteNonQueryAsync(ct); + } + + return new NpgsqlConnectionStringBuilder(container.GetConnectionString()) { Database = name }.ConnectionString; + } + + public async Task CreateMigratedDatabaseAsync(CancellationToken ct) + { + var connectionString = await CreateDatabaseAsync(ct); + await using var db = Context(connectionString); + await db.Database.MigrateAsync(ct); + return connectionString; + } + + public static AnichronDbContext Context(string connectionString) + => new(new DbContextOptionsBuilder() + .UseNpgsql(connectionString, o => o.UseNodaTime()) + .Options); +} diff --git a/src/Anichron.Tests.Integration/PostgresServerTests.cs b/src/Anichron.Tests.Integration/PostgresServerTests.cs new file mode 100644 index 00000000..791e6e2c --- /dev/null +++ b/src/Anichron.Tests.Integration/PostgresServerTests.cs @@ -0,0 +1,22 @@ +namespace Anichron.Tests.Integration; + +public sealed class PostgresServerTests +{ + [Fact] + public void Image_IsThePostgresImageTheComposeFileDeploys() + { + var compose = File.ReadAllLines(Path.Combine(SourceRoot(), "docker-compose.yml")); + + compose.Select(line => line.Trim()) + .Should().ContainSingle(line => line.StartsWith("image: postgres:", StringComparison.Ordinal)) + .Which.Should().Be($"image: {PostgresServer.Image}"); + } + + private static string SourceRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Anichron.slnx"))) + directory = directory.Parent; + return directory?.FullName ?? throw new InvalidOperationException("Anichron.slnx not found above the test output."); + } +} diff --git a/src/Anichron.Tests.Integration/Work/MaintenanceHandlerTests.cs b/src/Anichron.Tests.Integration/Work/MaintenanceHandlerTests.cs new file mode 100644 index 00000000..f046cb47 --- /dev/null +++ b/src/Anichron.Tests.Integration/Work/MaintenanceHandlerTests.cs @@ -0,0 +1,70 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Worker.Maintenance; +using Anichron.Worker.Settings; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; + +namespace Anichron.Tests.Integration.Work; + +public sealed class MaintenanceHandlerTests(PostgresServer server) +{ + private static readonly Duration Lease = Duration.FromSeconds(60); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task TokenCleanup_SecondRun_ConvergesAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await using (var db = work.CreateDbContext()) + { + var user = new User { Id = Guid.NewGuid(), Username = "mira", Email = "mira@example.com", PasswordHash = "hash" }; + db.Users.Add(user); + db.RefreshTokens.AddRange( + new RefreshToken { Id = Guid.NewGuid(), UserId = user.Id, TokenHash = "expired", ExpiresAt = WorkDatabase.Start - Duration.FromDays(1) }, + new RefreshToken { Id = Guid.NewGuid(), UserId = user.Id, TokenHash = "valid", ExpiresAt = WorkDatabase.Start + Duration.FromDays(1) }); + await db.SaveChangesAsync(Ct); + } + + var first = await RunTokenCleanupAsync(work); + var second = await RunTokenCleanupAsync(work); + + (first, second).Should().Be((new TokenCleanupResult(1), new TokenCleanupResult(0))); + await using var check = work.CreateDbContext(); + (await check.RefreshTokens.Select(t => t.TokenHash).ToListAsync(Ct)).Should().Equal("valid"); + } + + [Fact] + public async Task WorkItemRetention_SecondRun_ConvergesAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var old = WorkDatabase.Start - Duration.FromDays(31); + await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + var claimed = (await work.Store.ClaimAsync("worker", old, Lease, Ct))!; + await work.Store.SucceedAsync(claimed.Attempt, WorkProgress.None, null, old, Ct); + var pending = await work.EnqueueAsync(new UnkeyedPayload(2), Ct); + + var first = await RunRetentionAsync(work); + var second = await RunRetentionAsync(work); + + var cutoff = WorkDatabase.Start - Duration.FromDays(30); + (first, second).Should().Be((new WorkItemRetentionResult(1, cutoff), new WorkItemRetentionResult(0, cutoff))); + (await work.ItemsAsync(Ct)).Select(w => w.Id).Should().Equal(pending); + } + + private static async Task RunTokenCleanupAsync(WorkDatabase work) + { + await using var db = work.CreateDbContext(); + return await new TokenCleanupHandler(new EfRefreshTokenRepository(db), work.Clock, NullLogger.Instance) + .RunAsync(new TokenCleanupPayload(), Substitute.For(), Ct); + } + + private static Task RunRetentionAsync(WorkDatabase work) + => new WorkItemRetentionHandler( + work.Store, work.Clock, Options.Create(new WorkerSettings()), NullLogger.Instance) + .RunAsync(new WorkItemRetentionPayload(), Substitute.For(), Ct); +} diff --git a/src/Anichron.Tests.Integration/Work/WorkDatabase.cs b/src/Anichron.Tests.Integration/Work/WorkDatabase.cs new file mode 100644 index 00000000..bcf5317a --- /dev/null +++ b/src/Anichron.Tests.Integration/Work/WorkDatabase.cs @@ -0,0 +1,60 @@ +using Anichron.Core; +using Anichron.Core.Data; +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Worker.Work; +using Microsoft.EntityFrameworkCore; + +namespace Anichron.Tests.Integration.Work; + +public sealed record UnkeyedPayload(int Number) : IWorkPayload +{ + public static WorkKind Kind => WorkKind.TokenCleanup; +} + +public sealed record KeyedPayload(string Key) : IWorkPayload +{ + public static WorkKind Kind => WorkKind.TokenCleanup; + + string IWorkPayload.DedupKey => Key; +} + +internal sealed class StubClock(Instant now) : IClock +{ + public Instant Now { get; set; } = now; + + public Instant GetCurrentInstant() => Now; +} + +internal sealed class WorkDatabase(string connectionString) : IDbContextFactory +{ + public static readonly Instant Start = Instant.FromUtc(2026, 10, 18, 9, 0); + + public StubClock Clock { get; } = new(Start); + + public EfWorkItemStore Store => new(this); + + public static async Task CreateAsync(PostgresServer server, CancellationToken ct) + => new(await server.CreateMigratedDatabaseAsync(ct)); + + public AnichronDbContext CreateDbContext() => PostgresServer.Context(connectionString); + + public async Task EnqueueAsync(TPayload payload, CancellationToken ct) + where TPayload : IWorkPayload + { + await using var db = CreateDbContext(); + return await new EfWorkQueue(db, Clock, new TimeOrderedGuidFactory()).EnqueueAsync(payload, null, ct); + } + + public async Task ItemAsync(Guid id, CancellationToken ct) + { + await using var db = CreateDbContext(); + return await db.WorkItems.AsNoTracking().SingleAsync(w => w.Id == id, ct); + } + + public async Task> ItemsAsync(CancellationToken ct) + { + await using var db = CreateDbContext(); + return await db.WorkItems.AsNoTracking().OrderBy(w => w.RequestedAt).ToListAsync(ct); + } +} diff --git a/src/Anichron.Tests.Integration/Work/WorkItemStoreTests.cs b/src/Anichron.Tests.Integration/Work/WorkItemStoreTests.cs new file mode 100644 index 00000000..5f3dad41 --- /dev/null +++ b/src/Anichron.Tests.Integration/Work/WorkItemStoreTests.cs @@ -0,0 +1,331 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Worker.Work; +using Microsoft.EntityFrameworkCore; +using System.Collections.Concurrent; + +namespace Anichron.Tests.Integration.Work; + +public sealed class WorkItemStoreTests(PostgresServer server) +{ + private static readonly Duration Lease = Duration.FromSeconds(60); + private static readonly WorkProgress Halfway = new(5, 10, "halfway"); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static Instant At(int seconds) => WorkDatabase.Start + Duration.FromSeconds(seconds); + + [Fact] + public async Task ClaimAsync_ConcurrentClaimers_NeverGetTheSameItemAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + for (var i = 0; i < 20; i++) + await work.EnqueueAsync(new UnkeyedPayload(i), Ct); + var claimed = new ConcurrentBag(); + + await Task.WhenAll(Enumerable.Range(0, 8).Select(_ => Task.Run(async () => + { + while (await work.Store.ClaimAsync("worker", At(1), Lease, Ct) is { } item) + claimed.Add(item.Attempt.Id); + }, Ct))); + + claimed.Should().HaveCount(20).And.OnlyHaveUniqueItems(); + } + + [Fact] + public async Task ClaimAsync_OldestItemLockedByAnotherClaim_TakesTheNextWithoutWaitingAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var locked = await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + work.Clock.Now = At(1); + var next = await work.EnqueueAsync(new UnkeyedPayload(2), Ct); + await using var other = work.CreateDbContext(); + await using var transaction = await other.Database.BeginTransactionAsync(Ct); + await other.Database.ExecuteSqlAsync($"""SELECT 1 FROM "WorkItems" WHERE "Id" = {locked} FOR UPDATE""", Ct); + + var claimed = await work.Store.ClaimAsync("worker", At(10), Lease, Ct); + + claimed!.Attempt.Id.Should().Be(next); + } + + [Fact] + public async Task ClaimAsync_TakesTheOldestQueuedItemAndLeasesItAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var oldest = await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + work.Clock.Now = At(1); + await work.EnqueueAsync(new UnkeyedPayload(2), Ct); + + var claimed = await work.Store.ClaimAsync("all@host", At(10), Lease, Ct); + + claimed.Should().Be(new ClaimedWork(new WorkAttempt(oldest, 1), WorkKind.TokenCleanup, /*lang=json,strict*/ """{"number": 1}""")); + var item = await work.ItemAsync(oldest, Ct); + item.Status.Should().Be(WorkItemStatus.Running); + item.ClaimedBy.Should().Be("all@host"); + item.StartedAt.Should().Be(At(10)); + item.LeaseUntil.Should().Be(At(70)); + item.UpdatedAt.Should().Be(At(10)); + } + + [Fact] + public async Task ClaimAsync_NothingQueued_ReturnsNullAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + + (await work.Store.ClaimAsync("worker", At(0), Lease, Ct)).Should().BeNull(); + } + + [Fact] + public async Task HeartbeatAsync_HoldingAttempt_ExtendsTheLeaseAndStoresTheProgressAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + var claimed = (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!; + + var held = await work.Store.HeartbeatAsync(claimed.Attempt, Halfway, At(30), Lease, Ct); + + held.Should().BeTrue(); + var item = await work.ItemAsync(claimed.Attempt.Id, Ct); + item.LeaseUntil.Should().Be(At(90)); + (item.ProgressDone, item.ProgressTotal, item.ProgressNote).Should().Be((5L, 10L, "halfway")); + item.UpdatedAt.Should().Be(At(30)); + } + + [Fact] + public async Task AfterAReclaim_TheOldAttemptsHeartbeatAndCompletionChangeNothingAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + var zombie = (await work.Store.ClaimAsync("paused", At(0), Lease, Ct))!; + await work.Store.RecoverExpiredLeasesAsync(At(61), 3, Ct); + var current = (await work.Store.ClaimAsync("alive", At(62), Lease, Ct))!; + + var held = await work.Store.HeartbeatAsync(zombie.Attempt, Halfway, At(63), Lease, Ct); + await work.Store.SucceedAsync(zombie.Attempt, Halfway, /*lang=json,strict*/ """{"done": true}""", At(64), Ct); + await work.Store.FailAsync(zombie.Attempt, Halfway, "boom", At(64), Ct); + await work.Store.ReleaseAsync(zombie.Attempt, At(64), Ct); + + held.Should().BeFalse(); + current.Attempt.Should().Be(new WorkAttempt(zombie.Attempt.Id, 2)); + var item = await work.ItemAsync(current.Attempt.Id, Ct); + item.Status.Should().Be(WorkItemStatus.Running); + item.ClaimedBy.Should().Be("alive"); + (item.Result, item.Error, item.ProgressNote).Should().Be((null, null, null)); + item.UpdatedAt.Should().Be(At(62)); + } + + [Fact] + public async Task RecoverExpiredLeasesAsync_RequeuesTwiceThenFailsTheThirdStallAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var id = await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + var stalls = new List<(WorkItemStatus, int)>(); + + for (var round = 0; round < 3; round++) + { + var claimedAt = At(round * 100); + await work.Store.ClaimAsync("worker", claimedAt, Lease, Ct); + await work.Store.RecoverExpiredLeasesAsync(claimedAt + Duration.FromSeconds(61), 3, Ct); + var item = await work.ItemAsync(id, Ct); + stalls.Add((item.Status, item.Stalls)); + } + + stalls.Should().Equal((WorkItemStatus.Queued, 1), (WorkItemStatus.Queued, 2), (WorkItemStatus.Failed, 3)); + var failed = await work.ItemAsync(id, Ct); + failed.Error.Should().Be(EfWorkItemStore.StalledError); + failed.FinishedAt.Should().Be(At(261)); + } + + [Fact] + public async Task RecoverExpiredLeasesAsync_LeaseStillValid_LeavesTheItemRunningAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var id = await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + await work.Store.ClaimAsync("worker", At(0), Lease, Ct); + + await work.Store.RecoverExpiredLeasesAsync(At(60), 3, Ct); + + var item = await work.ItemAsync(id, Ct); + (item.Status, item.Stalls).Should().Be((WorkItemStatus.Running, 0)); + } + + [Fact] + public async Task RecoverExpiredLeasesAsync_Requeued_ClearsTheClaimAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var id = await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + await work.Store.ClaimAsync("worker", At(0), Lease, Ct); + + await work.Store.RecoverExpiredLeasesAsync(At(61), 3, Ct); + + var item = await work.ItemAsync(id, Ct); + (item.ClaimedBy, item.StartedAt, item.LeaseUntil).Should().Be((null, null, null)); + item.UpdatedAt.Should().Be(At(61)); + } + + [Fact] + public async Task ReleaseAsync_QueuesTheItemWithoutAStallAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + var claimed = (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!; + + await work.Store.ReleaseAsync(claimed.Attempt, At(5), Ct); + + var item = await work.ItemAsync(claimed.Attempt.Id, Ct); + (item.Status, item.Stalls, item.Attempt).Should().Be((WorkItemStatus.Queued, 0, 1)); + (item.ClaimedBy, item.LeaseUntil).Should().Be((null, null)); + (await work.Store.ClaimAsync("worker", At(6), Lease, Ct))!.Attempt.Should().Be(claimed.Attempt with { Attempt = 2 }); + } + + [Fact] + public async Task ReleaseAsync_SameKeyAlreadyQueued_SupersedesTheReleasedItemAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + var running = (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!; + var waiting = await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + + await work.Store.ReleaseAsync(running.Attempt, At(5), Ct); + + var released = await work.ItemAsync(running.Attempt.Id, Ct); + (released.Status, released.Error, released.Stalls).Should().Be( + (WorkItemStatus.Cancelled, EfWorkItemStore.SupersededError, 0)); + released.FinishedAt.Should().Be(At(5)); + (await work.ItemAsync(waiting, Ct)).Status.Should().Be(WorkItemStatus.Queued); + } + + [Fact] + public async Task RecoverExpiredLeasesAsync_SameKeyAlreadyQueued_SupersedesItAndStillRecoversTheRestAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + var keyed = (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!; + work.Clock.Now = At(1); + var other = await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + await work.Store.ClaimAsync("worker", At(1), Lease, Ct); + var waiting = await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + + await work.Store.RecoverExpiredLeasesAsync(At(62), 3, Ct); + + var superseded = await work.ItemAsync(keyed.Attempt.Id, Ct); + (superseded.Status, superseded.Error, superseded.Stalls).Should().Be( + (WorkItemStatus.Cancelled, EfWorkItemStore.SupersededError, 1)); + (await work.ItemAsync(other, Ct)).Status.Should().Be(WorkItemStatus.Queued); + (await work.ItemAsync(waiting, Ct)).Status.Should().Be(WorkItemStatus.Queued); + } + + [Fact] + public async Task RecoverExpiredLeasesAsync_TwoExpiredItemsShareAKey_RequeuesOneAndSupersedesTheOtherAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + await work.Store.ClaimAsync("worker", At(0), Lease, Ct); + work.Clock.Now = At(1); + await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + await work.Store.ClaimAsync("worker", At(1), Lease, Ct); + + await work.Store.RecoverExpiredLeasesAsync(At(62), 3, Ct); + + (await work.ItemsAsync(Ct)).Select(w => w.Status) + .Should().BeEquivalentTo([WorkItemStatus.Queued, WorkItemStatus.Cancelled]); + } + + [Fact] + public async Task ClaimAsync_KindThisWorkerDoesNotKnow_LeavesItQueuedAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var unknown = Guid.NewGuid(); + var requestedAt = At(-10); + const string payload = "{}"; + await using (var db = work.CreateDbContext()) + { + await db.Database.ExecuteSqlAsync($""" + INSERT INTO "WorkItems" ("Id", "Kind", "Payload", "RequestedAt", "Status", "Attempt", "Stalls", "UpdatedAt", "ProgressDone") + VALUES ({unknown}, 'FromANewerRelease', CAST({payload} AS jsonb), {requestedAt}, 'Queued', 0, 0, {requestedAt}, 0) + """, Ct); + } + + var known = await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + + (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!.Attempt.Id.Should().Be(known); + (await work.Store.ClaimAsync("worker", At(0), Lease, Ct)).Should().BeNull(); + await using var check = work.CreateDbContext(); + (await check.Database + .SqlQuery($"""SELECT "Status" AS "Value" FROM "WorkItems" WHERE "Id" = {unknown}""") + .ToListAsync(Ct)).Should().Equal("Queued"); + } + + [Fact] + public async Task SucceedAsync_StoresTheResultAndTheFinalProgressAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + var claimed = (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!; + + await work.Store.SucceedAsync(claimed.Attempt, Halfway, /*lang=json,strict*/ """{"deleted": 3}""", At(9), Ct); + + var item = await work.ItemAsync(claimed.Attempt.Id, Ct); + item.Status.Should().Be(WorkItemStatus.Succeeded); + item.Result.Should().Be(/*lang=json,strict*/ """{"deleted": 3}"""); + (item.ProgressDone, item.ProgressTotal, item.ProgressNote).Should().Be((5L, 10L, "halfway")); + (item.FinishedAt, item.UpdatedAt).Should().Be((At(9), At(9))); + } + + [Fact] + public async Task FailAsync_StoresTheErrorAndTheLastProgressButNoResultAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await work.EnqueueAsync(new UnkeyedPayload(1), Ct); + var claimed = (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!; + + await work.Store.FailAsync(claimed.Attempt, Halfway, "InvalidOperationException: boom", At(9), Ct); + + var item = await work.ItemAsync(claimed.Attempt.Id, Ct); + item.Status.Should().Be(WorkItemStatus.Failed); + item.Error.Should().Be("InvalidOperationException: boom"); + item.Result.Should().BeNull(); + (item.ProgressDone, item.ProgressTotal, item.ProgressNote).Should().Be((5L, 10L, "halfway")); + (item.FinishedAt, item.UpdatedAt).Should().Be((At(9), At(9))); + } + + [Fact] + public async Task DeleteFinishedBeforeAsync_DeletesOnlyItemsThatFinishedBeforeTheCutoffAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + await FinishedAsync(work, At(10)); + var recent = await FinishedAsync(work, At(100)); + var queued = await work.EnqueueAsync(new UnkeyedPayload(3), Ct); + + var deleted = await work.Store.DeleteFinishedBeforeAsync(At(100), Ct); + + deleted.Should().Be(1); + (await work.ItemsAsync(Ct)).Select(w => w.Id).Should().BeEquivalentTo([recent, queued]); + } + + [Fact] + public async Task IsDueAsync_FollowsTheLastFinishedItemWithTheKeyAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + (await work.Store.IsDueAsync("sweep", At(0), Ct)).Should().BeTrue(); + + var id = await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + (await work.Store.IsDueAsync("sweep", At(0), Ct)).Should().BeFalse(); + + var claimed = (await work.Store.ClaimAsync("worker", At(0), Lease, Ct))!; + (await work.Store.IsDueAsync("sweep", At(0), Ct)).Should().BeFalse(); + + await work.Store.SucceedAsync(claimed.Attempt, WorkProgress.None, null, At(50), Ct); + (await work.Store.IsDueAsync("sweep", At(49), Ct)).Should().BeFalse(); + (await work.Store.IsDueAsync("sweep", At(50), Ct)).Should().BeTrue(); + id.Should().Be(claimed.Attempt.Id); + } + + private static async Task FinishedAsync(WorkDatabase work, Instant finishedAt) + { + await work.EnqueueAsync(new UnkeyedPayload(0), Ct); + var claimed = (await work.Store.ClaimAsync("worker", finishedAt, Lease, Ct))!; + await work.Store.SucceedAsync(claimed.Attempt, WorkProgress.None, null, finishedAt, Ct); + return claimed.Attempt.Id; + } +} diff --git a/src/Anichron.Tests.Integration/Work/WorkQueueTests.cs b/src/Anichron.Tests.Integration/Work/WorkQueueTests.cs new file mode 100644 index 00000000..ed74b5a2 --- /dev/null +++ b/src/Anichron.Tests.Integration/Work/WorkQueueTests.cs @@ -0,0 +1,72 @@ +using Anichron.Core.Domain; + +namespace Anichron.Tests.Integration.Work; + +public sealed class WorkQueueTests(PostgresServer server) +{ + private static readonly Duration Lease = Duration.FromSeconds(60); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task EnqueueAsync_KeyAlreadyQueued_ReturnsTheWaitingItemAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var first = await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + + var second = await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + + second.Should().Be(first); + (await work.ItemsAsync(Ct)).Should().ContainSingle(); + } + + [Fact] + public async Task EnqueueAsync_KeyOnlyRunning_AddsANewQueuedItemAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + var running = await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + await work.Store.ClaimAsync("worker", work.Clock.Now, Lease, Ct); + + var queued = await work.EnqueueAsync(new KeyedPayload("sweep"), Ct); + + queued.Should().NotBe(running); + (await work.ItemAsync(queued, Ct)).Status.Should().Be(WorkItemStatus.Queued); + } + + [Fact] + public async Task EnqueueAsync_ConcurrentRequestsForOneKey_ProduceOneQueuedItemAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + + var ids = await Task.WhenAll(Enumerable.Range(0, 10) + .Select(_ => Task.Run(() => work.EnqueueAsync(new KeyedPayload("token-cleanup"), Ct), Ct))); + + var item = (await work.ItemsAsync(Ct)).Should().ContainSingle().Which; + ids.Should().AllBeEquivalentTo(item.Id); + } + + [Fact] + public async Task EnqueueAsync_DifferentKeys_StaySeparateItemsAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + + await work.EnqueueAsync(new KeyedPayload("delete:a"), Ct); + await work.EnqueueAsync(new KeyedPayload("delete:b"), Ct); + + (await work.ItemsAsync(Ct)).Select(w => w.DedupKey).Should().BeEquivalentTo("delete:a", "delete:b"); + } + + [Fact] + public async Task EnqueueAsync_StoresTheKindAndPayloadAsJsonAsync() + { + var work = await WorkDatabase.CreateAsync(server, Ct); + + var id = await work.EnqueueAsync(new UnkeyedPayload(7), Ct); + + var item = await work.ItemAsync(id, Ct); + item.Kind.Should().Be(WorkKind.TokenCleanup); + item.Payload.Should().Be(/*lang=json,strict*/ """{"number": 7}"""); + item.DedupKey.Should().BeNull(); + item.RequestedAt.Should().Be(WorkDatabase.Start); + } +} diff --git a/src/Anichron.Tests.Integration/packages.lock.json b/src/Anichron.Tests.Integration/packages.lock.json new file mode 100644 index 00000000..a55ebfe0 --- /dev/null +++ b/src/Anichron.Tests.Integration/packages.lock.json @@ -0,0 +1,866 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "coverlet.collector": { + "type": "Direct", + "requested": "[10.1.0, )", + "resolved": "10.1.0", + "contentHash": "SlYCIHnBYxINSUpR75SiM5JEmnx11pqT+Ac/C/fUqTTA0dLPl9eYzmFXKKuxmQhL3ZdTQl4PBD3YdAxAauaKLQ==" + }, + "FluentAssertions": { + "type": "Direct", + "requested": "[8.11.0, )", + "resolved": "8.11.0", + "contentHash": "dZqOxys9uY/0vO5BAwFtDECkFNSr9vWGwHa1fwZyuM1xAMbI4wiPTUBR2nH5e6R2/bQkXegxFY1B/oU1yqNNOw==" + }, + "Microsoft.NET.Test.Sdk": { + "type": "Direct", + "requested": "[18.5.1, )", + "resolved": "18.5.1", + "contentHash": "SfqVaLiIqAbRWuPg5BP4QFwBIirQj/YIL8Dhxl6zntBKbXp0cQykoV480SmwG+yRMiWptxEI6NbHQuGSZ8b97w==", + "dependencies": { + "Microsoft.CodeCoverage": "18.5.1", + "Microsoft.TestPlatform.TestHost": "18.5.1" + } + }, + "Microsoft.VisualStudio.Threading.Analyzers": { + "type": "Direct", + "requested": "[18.12.1, )", + "resolved": "18.12.1", + "contentHash": "sQt7NpnselT1gP1a7bTfhmfHYV0jbavst/8mOoWTMSjXhXzWPssqgBoIwrxNsgJUa7XQzD2raSRMAFfmlepxNw==" + }, + "NSubstitute": { + "type": "Direct", + "requested": "[6.2.0, )", + "resolved": "6.2.0", + "contentHash": "05oaAmBAvYYUQb/NTb9/kWBDjq0iS772X4t428WJcHRcH+5ULJx6rTbEvqkZJBIHWRLI8MlU3et3F5/gkEHKIw==", + "dependencies": { + "Castle.Core": "5.1.1" + } + }, + "Roslynator.Analyzers": { + "type": "Direct", + "requested": "[5.0.1, )", + "resolved": "5.0.1", + "contentHash": "sCZ0Di4B937wKa9pwKbdsE6Njr8A/FdCLw28+yDKyMaDZ/f8JWxcFn6NOCukzKhCG/eA6M2DbXlKkrPi0g0lIA==" + }, + "SonarAnalyzer.CSharp": { + "type": "Direct", + "requested": "[10.35.0.4138, )", + "resolved": "10.35.0.4138", + "contentHash": "T5o/tuyPw9pHRkyWxW07TkSri4I4pLpqwH922JuhphDjHAM7BUHaGE67h/w+65evHsIar8mYB2Ey6OmOLj9QDw==" + }, + "System.IO.Abstractions.TestingHelpers": { + "type": "Direct", + "requested": "[22.3.0, )", + "resolved": "22.3.0", + "contentHash": "18NddvxRHmpwirzjhMOZ2u0oCdlxUvNQV2hQXPIygEnbAoo0sYOzWY4Jf5WQs/QSOIOBYSFpvq3A003d7HFmTQ==", + "dependencies": { + "TestableIO.System.IO.Abstractions.TestingHelpers": "22.3.0" + } + }, + "Testcontainers.PostgreSql": { + "type": "Direct", + "requested": "[4.16.0, )", + "resolved": "4.16.0", + "contentHash": "QqvFS8Oj+i6deH3QJOMd7jgFF9P3h7RZ37wOaoR97ZahMcSYMYCb+COR4nUVznT1d7EavEcZdUs1QV0ow69JiA==", + "dependencies": { + "Testcontainers": "4.16.0" + } + }, + "xunit.runner.visualstudio": { + "type": "Direct", + "requested": "[3.1.5, )", + "resolved": "3.1.5", + "contentHash": "tKi7dSTwP4m5m9eXPM2Ime4Kn7xNf4x4zT9sdLO/G4hZVnQCRiMTWoSZqI/pYTVeI27oPPqHBKYI/DjJ9GsYgA==" + }, + "xunit.v3": { + "type": "Direct", + "requested": "[3.2.2, )", + "resolved": "3.2.2", + "contentHash": "L+4/4y0Uqcg8/d6hfnxhnwh4j9FaeULvefTwrk30rr1o4n/vdPfyUQ8k0yzH8VJx7bmFEkDdcRfbtbjEHlaYcA==", + "dependencies": { + "xunit.v3.mtp-v1": "[3.2.2]" + } + }, + "Blurhash.Core": { + "type": "Transitive", + "resolved": "4.2.1", + "contentHash": "miLOWO/pS7M35zj31fiCT9oO+/JxjRPxfEA+R/rjZPC86jMjZfaa9h5zZLaclw6S3bvg6KgeVhhRQnmltE8zug==" + }, + "BouncyCastle.Cryptography": { + "type": "Transitive", + "resolved": "2.7.0", + "contentHash": "U+12df8UEWHgBi04YVf/Lgi2dy3SItlIYvHjjEVa/BngCQIzDCDRBk50DDByCfDvSbe5pRNFr3b7UrVK2kMcLw==" + }, + "Castle.Core": { + "type": "Transitive", + "resolved": "5.1.1", + "contentHash": "rpYtIczkzGpf+EkZgDr9CClTdemhsrwA/W5hMoPjLkRFnXzH44zDLoovXeKtmxb1ykXK9aJVODSpiJml8CTw2g==", + "dependencies": { + "System.Diagnostics.EventLog": "6.0.0" + } + }, + "Docker.DotNet.Enhanced": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "nGicLwvd42FhRk+khY5uS6cx49ErNdwYKnYBg0F4m4BDKLp/R77AVmmN9xAiqI3W/wN5ZCHkdUhgxf5ORkZuFQ==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3", + "Docker.DotNet.Enhanced.LegacyHttp": "4.3.3", + "Docker.DotNet.Enhanced.NPipe": "4.3.3", + "Docker.DotNet.Enhanced.NativeHttp": "4.3.3", + "Docker.DotNet.Enhanced.Unix": "4.3.3", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "Docker.DotNet.Enhanced.Handler.Abstractions": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "9Cp8hOgtynixcDoAs9lnEaQosluojSYmiW3fsLsLIVfZjlq/fznSIZNUhnmyT4Xo1Iyuok/y49WL/25O47u0Pw==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "Docker.DotNet.Enhanced.LegacyHttp": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "7j3M16emv9PAQN7VwFn23xLYNj8GJmwPOcogveHkaWnOCqiC+anRaNKQwqIBNApM1AuwZKivehTKTPmmrjUUnw==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.NativeHttp": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "iNzK+jRFeEMobSA7l/h4ARwCKOOefOWtVN5/RB0ft6/6H6IQXvVUuOgGyZAjYLBT7TsyClRYno2B904f3dtBuQ==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.NPipe": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "ZTLYufuEfY0e6qLOgeH9QgXx2KYuoABRVaY5A8rsggyLgYqbDj9rCRfVAhHPCUv83S7pVxDHy+Tvm/BnxjWVpg==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.Unix": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "ypo8qNbmvHw1t9VfpRTMogCw2vht6VjkXzlGYUUeP2H2bf83USURdla1maW1njn2oq2rfLUFOGMfmt+A37QU2w==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.X509": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "oBDibWezEv4hgj3RIQxI3DVcxkNV1MdrD0d/jhjUu+h3DL+qc0wlkQva15kkwMatXmC/hWp1VP0DMoFXe+BmEw==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Microsoft.ApplicationInsights": { + "type": "Transitive", + "resolved": "2.23.0", + "contentHash": "nWArUZTdU7iqZLycLKWe0TDms48KKGE6pONH2terYNa8REXiqixrMOkf1sk5DHGMaUTqONU2YkS4SAXBhLStgw==" + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg==" + }, + "Microsoft.CodeCoverage": { + "type": "Transitive", + "resolved": "18.5.1", + "contentHash": "vMFDR1ZjqzzgKmM0zrPie7Gv9Y+ZppjODB5Quzu9Eq0TlIusUfUCYFPEawO91zQuqwzvdFbJSU7WHNtjStffJQ==" + }, + "Microsoft.EntityFrameworkCore.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "ARw+tZ3eiTosnK5nJPUGEzeUQcjyqCqs3qKA3lPtIRH5yJo3qIgxm7AvkZVIT9iWkS2083UsWe3Rj1MysW3p5A==" + }, + "Microsoft.EntityFrameworkCore.Analyzers": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "zBx6VeEz/hkhlm0BJ2V+9xebUuDDIszcKvurJceQHuZPEXuXubrL6Lneo9VZiliRyJbQlB2oNKLkJXd3gHteQA==" + }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "UOJqcdNMoFcV/5RUcOlVKq9pPmbrcmbpLx5iYz9QtrmMcWRQzlL5Q/NZRfbPPLNyIEc1HWjjs5Z7nIwqewU1sg==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "q+At65RACuCtGwUDVI08TmmrC73YFouimMkJAtlhsGhFcU/dVpqYEKpNBOF9TUxkJ6aefEOShqe7StLK7q3NqA==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "DY4C2qzTt2lOIY3VoscBH/TCwTolI5DKOfgJLuI+k1QUvRNT5vTMS58AOerNiihdVRStO0wEpLeRWA8jytzHBQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "yJJl/b0+W+GEWbu+7viw6C3h2BoNkB69Vlz7s/4e+hm4mYbJqfhdcDSi9sCXJw47blsBaiQd3UDPgDtop92+ug==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.CommandLine": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "RwdNZ9JdGs8eZmeL5qRe7ugp0Irk3KVkWpC/w1c8kjJwu1R4HvCXYz9kwHZxn45WiaRBG5lphQKtqnBaJOQC+Q==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.EnvironmentVariables": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "qJK2dm9/AxyPVfqqBaZtjkUFl2hnBR/CQnMJtSO5LdEoyATMqvZjeQVY+8l0oa6PM6qlVe28iMEyXIRQ0wyZjA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.FileExtensions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "kwC1DY55CsqPmDtJAz+7alD3G+gJcM+XwHSZrcRO8mJGYB9hRq1qC6ZH8DL0SPW9WtJ/Bae5ikzPic8SxSv6Vw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Json": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "acCWM2+WR67kBAz6gAhD2zG+IZH9Q2VpKv5le7Msiu0vAMsnnLXpnV3BAzxd8/i+uOEo/dfb2XRYTynRVTDvEg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.UserSecrets": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "kQ/Tx972unP+ueQL8SSninthIj6w6XP2o0VnJPFC5HQIimPMwEN704MOspEgDkmm4FUL5SuxLEU0bNiW4HEZFA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Json": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "G2GrGWNAPZBsii2F6AU8n+JT2tC1/I0Ml/e9ZyNCgPjf3hZqronP7o0X91aAynCmVeae8KkSIUNXXeXCNRmdVQ==" + }, + "Microsoft.Extensions.Diagnostics": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "J9ODOrffj1mK3Et69elNsLGRguesPxvThTdVpFmQX7fLNPhIOgqHQfLdGnTBEwNAV3yf8yiyVlpDZNw2qN87Nw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.13", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.13" + } + }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "oZzqZze8ed1fLS20HtIrkPxQXGRG/koq09srIw2ptnDHm9RHfwB++d35hosB/cXc55TH7Pshv85O0txongjCIg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "BZDwtN3uoWRghDi5lRRZPDjMMCg2Ds7J6YEYjqka/TLjqnzquEpXyvchrIMeoubwT1ns23ow6PPeAoBHJO81Rw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.FileProviders.Physical": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "U7HkgGCNPICN5k7NVK6dmLsSPSPeO/EbNcuxOl/5qdC9Ahjp9UZWKOvlnUC+rYurmoxIlwyv9e5j0BaQlTOELA==", + "dependencies": { + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileSystemGlobbing": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.FileSystemGlobbing": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "vQVzmAbnzlhXSXA2caYnlxtG9GXxEwtj5bODiWLu06A6iqeJmfBtlfKjuOSJGRy0/CFaNUrm7u9nxEUlljc2Rg==" + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "zIrbe9vkIFXxW3mOApIOWXw+xMwX7Cgsptr3ZSlglJt4hW2qp4fB/JU1NHgAevteBNH4PicvU9DsFPHxlnnloA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "fqbiPy4nUp4H2XHKAVQXKN2y3T2h4cFIx1DAAgE6iY9OHkN/OA1cKFi+geJKRWGuX6pS5SFbsjIPWHJqrxE0eg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "aSZEhEig4fvNiw2zs8Z6NyDlniKtq6YjcF3DyHuqTEHcZ2VKA/6itMPZ+dW+yWABWExlDcQwwdbqX1Rcb7W8WQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Configuration": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "OyYJOaE9WbBx1Hp+Dd2zrqeFM0JwH5M/x0VL6tNUv3V18vIAVpix8UjhvY9eW/iXpguEHy68JO/Z9YkBOwftIw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Console": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "i9rn0rHkiCxfNVbf4DRG7LOFcKPy/BioAxS07w6daWvIpLNRjskqDJcGGaYVFWndsq0iTRaS8Msry0t5fRYQug==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Configuration": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Debug": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "yBig0DNMq+3UgliwxYlUC48dg0fPqvqI5B3ljzmLTrpR6maoKTOO5i9QnPgQUmQoaQWZMgu74fuF6oSDihb6pg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.EventLog": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "OAK40lXhgMuvyNwvzW3fxLvvucN8yr5XS7dQRcPmtnPf+Pt1lpbMgao5T4Wa6VdFagtIeDZNgPtV5Ig4xstziQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "System.Diagnostics.EventLog": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.EventSource": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "SxJu3oczk+TJtHvpgAmCVzEkzx69/obA6GZCbGrbqYqc981YOAyfEprySBdFX7Yc5qatHJKHNTlpGeeBAQkZcA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "tmMJgdrE4uhf1JcNs9CV9moAHW+hmDuvgMiPpnQxarKaUxqI2eN33sYK5S4l9Uw+rG9PJvlm4qPxMhvolTDTaQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "xVO5HhATc09WXuS/RaDmx8ZeWS0mTpT1L0tkEDMSBHtAnNpNsDr0ge0wIO2jinCWWqpq5dL1tRTzpYNVlvyqLw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "ZjTwxzhiw2TMEzIbUDkMF0dfDB2I4NQgIfKD0ELu9/5Mk+SRGdKZcD0/lvBrRCAPo6KN2r0BW86MRancoyJkoA==" + }, + "Microsoft.Testing.Extensions.Telemetry": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "No5AudZMmSb+uNXjlgL2y3/stHD2IT4uxqc5yHwkE+/nNux9jbKcaJMvcp9SwgP4DVD8L9/P3OUz8mmmcvEIdQ==", + "dependencies": { + "Microsoft.ApplicationInsights": "2.23.0", + "Microsoft.Testing.Platform": "1.9.1" + } + }, + "Microsoft.Testing.Extensions.TrxReport.Abstractions": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "AL46Xe1WBi85Ntd4mNPvat5ZSsZ2uejiVqoKCypr8J3wK0elA5xJ3AN4G/Q4GIwzUFnggZoH/DBjnr9J18IO/g==", + "dependencies": { + "Microsoft.Testing.Platform": "1.9.1" + } + }, + "Microsoft.Testing.Platform": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "QafNtNSmEI0zazdebnsIkDKmFtTSpmx/5PLOjURWwozcPb3tvRxzosQSL8xwYNM1iPhhKiBksXZyRSE2COisrA==" + }, + "Microsoft.Testing.Platform.MSBuild": { + "type": "Transitive", + "resolved": "1.9.1", + "contentHash": "oTUtyR4X/s9ytuiNA29FGsNCCH0rNmY5Wdm14NCKLjTM1cT9edVSlA+rGS/mVmusPqcP0l/x9qOnMXg16v87RQ==", + "dependencies": { + "Microsoft.Testing.Platform": "1.9.1" + } + }, + "Microsoft.TestPlatform.ObjectModel": { + "type": "Transitive", + "resolved": "18.5.1", + "contentHash": "KNZd+M0S0rz5eNAln0pbZX+A/RbokYZCbGKx4fN4CkhtWhkz6nSJDO+9LGYjRE4d0WPVriJ2JnVubkjt3+PpMg==" + }, + "Microsoft.TestPlatform.TestHost": { + "type": "Transitive", + "resolved": "18.5.1", + "contentHash": "RM+3JNHEoHOCFXzVntUcIiYxzPjzBN0N8wto6HYXi76YyBTZ/3CeRL8U+Pk5zx3AUrOmHxDvKJwGUCdElU9bJg==", + "dependencies": { + "Microsoft.TestPlatform.ObjectModel": "18.5.1", + "Newtonsoft.Json": "13.0.3" + } + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.3", + "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" + }, + "Npgsql.NodaTime": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "6ba1CM/zAoI7bTuBV7cFhTR3qqUsY5FqqqNoOh7dHoFHcyEEpIsMui3mC3Swyc+oyAJG0Zf8DCiXPnAf+sW09A==", + "dependencies": { + "NodaTime": "3.2.2", + "Npgsql": "10.0.3" + } + }, + "SharpZipLib": { + "type": "Transitive", + "resolved": "1.4.2", + "contentHash": "yjj+3zgz8zgXpiiC3ZdF/iyTBbz2fFvMxZFEBPUcwZjIvXOf37Ylm+K58hqMfIBt5JgU/Z2uoUS67JmTLe973A==" + }, + "SSH.NET": { + "type": "Transitive", + "resolved": "2026.0.0", + "contentHash": "Yu9dirPq8l3oaat0+OQ7K0nUf5MmYltpia5UGqsApTG4zTPvBC1cxbNnC3NERij26dUST0A3Ef1QdHSn5ArbWQ==", + "dependencies": { + "BouncyCastle.Cryptography": "2.7.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "System.Diagnostics.EventLog": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "DkNjtQyPR5Ud/zHnWHdsOuKbFRL9nIC92Q76z0NmHVDoQm40toLtsGFTIWq7toBoVa6fo/DInL3PLxY3Asixyw==" + }, + "TestableIO.System.IO.Abstractions": { + "type": "Transitive", + "resolved": "22.3.0", + "contentHash": "2dqQ5AEE61NkQ/0F31mhOXYhgzZOtC+RUuJajwYa7gPgZBqJVe9Mc/hnqXEbBSDOxGZ7hYsmULjzK3yqewB3FA==", + "dependencies": { + "Testably.Abstractions.FileSystem.Interface": "10.4.0" + } + }, + "TestableIO.System.IO.Abstractions.TestingHelpers": { + "type": "Transitive", + "resolved": "22.3.0", + "contentHash": "kJsKJjXcjSvh0H89afGlsngJHj8n7f1q7juNiLUCbtJr+m208TG6z2RfjCVlGosfgaHBnNhgs0zFy2Z0V1XnZA==", + "dependencies": { + "TestableIO.System.IO.Abstractions.Wrappers": "22.3.0", + "Testably.Abstractions.FileSystem.Interface": "10.4.0" + } + }, + "TestableIO.System.IO.Abstractions.Wrappers": { + "type": "Transitive", + "resolved": "22.3.0", + "contentHash": "9TsOEcwIGJXHjkZ/lIpg6KnoRjxpZZ/LpMvNddcXEvmcxWKZdbhSy2UWMpy3oN8klbQ3dcpxbp9AddQk0zE+Kg==", + "dependencies": { + "Testably.Abstractions.FileSystem.Interface": "10.4.0" + } + }, + "Testably.Abstractions.FileSystem.Interface": { + "type": "Transitive", + "resolved": "10.4.0", + "contentHash": "7G5NzIjs7544Dit0uuvYOOAzD8E4alQS2ENPGT3HV1Ks/1r55O7X4d9g6L273n70IDkgqNdBCmrkWxJLn0xCZA==" + }, + "Testcontainers": { + "type": "Transitive", + "resolved": "4.16.0", + "contentHash": "JmMUZk6ZXQovbGQjaMB+YxOR0PbJTdYjOYjRcrm5LrA2dhLVMuCbhVGGunzCGIQfFMUB5Lno2LPOHMMuaGmuIg==", + "dependencies": { + "Docker.DotNet.Enhanced": "4.3.3", + "Docker.DotNet.Enhanced.X509": "4.3.3", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3", + "SSH.NET": "2026.0.0", + "SharpZipLib": "1.4.2" + } + }, + "XmpCore": { + "type": "Transitive", + "resolved": "6.1.10.1", + "contentHash": "/Tp2/e39e7z8hLtS4TXt0FNNZHbf5yNFfHS0/59soXlgOzZlyRZivvzWpvujyaWkTWWgecp/1nHxkuc5vntn7A==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "1.27.0", + "contentHash": "y/pxIQaLvk/kxAoDkZW9GnHLCEqzwl5TW0vtX3pweyQpjizB9y3DXhb9pkw2dGeUqhLjsxvvJM1k89JowU6z3g==" + }, + "xunit.v3.assert": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "BPciBghgEEaJN/JG00QfCYDfEfnLgQhfnYEy+j1izoeHVNYd5+3Wm8GJ6JgYysOhpBPYGE+sbf75JtrRc7jrdA==" + }, + "xunit.v3.common": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "Hj775PEH6GTbbg0wfKRvG2hNspDCvTH9irXhH4qIWgdrOSV1sQlqPie+DOvFeigsFg2fxSM3ZAaaCDQs+KreFA==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0" + } + }, + "xunit.v3.core.mtp-v1": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "Ga5aA2Ca9ktz+5k3g5ukzwfexwoqwDUpV6z7atSEUvqtd6JuybU1XopHqg1oFd78QdTfZgZE9h5sHpO4qYIi5w==", + "dependencies": { + "Microsoft.Testing.Extensions.Telemetry": "1.9.1", + "Microsoft.Testing.Extensions.TrxReport.Abstractions": "1.9.1", + "Microsoft.Testing.Platform": "1.9.1", + "Microsoft.Testing.Platform.MSBuild": "1.9.1", + "xunit.v3.extensibility.core": "[3.2.2]", + "xunit.v3.runner.inproc.console": "[3.2.2]" + } + }, + "xunit.v3.extensibility.core": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "srY8z/oMPvh/t8axtO2DwrHajhFMH7tnqKildvYrVQIfICi8fOn3yIBWkVPAcrKmHMwvXRJ/XsQM3VMR6DOYfQ==", + "dependencies": { + "xunit.v3.common": "[3.2.2]" + } + }, + "xunit.v3.mtp-v1": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "O41aAzYKBT5PWqATa1oEWVNCyEUypFQ4va6K0kz37dduV3EKzXNMaV2UnEhufzU4Cce1I33gg0oldS8tGL5I0A==", + "dependencies": { + "xunit.analyzers": "1.27.0", + "xunit.v3.assert": "[3.2.2]", + "xunit.v3.core.mtp-v1": "[3.2.2]" + } + }, + "xunit.v3.runner.common": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "/hkHkQCzGrugelOAehprm7RIWdsUFVmIVaD6jDH/8DNGCymTlKKPTbGokD5czbAfqfex47mBP0sb0zbHYwrO/g==", + "dependencies": { + "Microsoft.Win32.Registry": "[5.0.0]", + "xunit.v3.common": "[3.2.2]" + } + }, + "xunit.v3.runner.inproc.console": { + "type": "Transitive", + "resolved": "3.2.2", + "contentHash": "ulWOdSvCk+bPXijJZ73bth9NyoOHsAs1ZOvamYbCkD4DNLX/Bd29Ve2ZNUwBbK0MqfIYWXHZViy/HKrdEC/izw==", + "dependencies": { + "xunit.v3.extensibility.core": "[3.2.2]", + "xunit.v3.runner.common": "[3.2.2]" + } + }, + "anichron.core": { + "type": "Project", + "dependencies": { + "Microsoft.EntityFrameworkCore": "[10.0.13, )", + "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", + "Npgsql": "[10.0.3, )", + "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", + "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" + } + }, + "anichron.infrastructure": { + "type": "Project", + "dependencies": { + "Anichron.Core": "[1.0.0, )", + "Microsoft.EntityFrameworkCore": "[10.0.13, )", + "Microsoft.EntityFrameworkCore.Relational": "[10.0.13, )", + "Microsoft.Extensions.Configuration": "[10.0.13, )", + "Microsoft.Extensions.Configuration.Ini": "[10.0.13, )", + "Npgsql": "[10.0.3, )", + "System.IO.Abstractions": "[22.3.0, )" + } + }, + "anichron.worker": { + "type": "Project", + "dependencies": { + "Anichron.Core": "[1.0.0, )", + "Anichron.Infrastructure": "[1.0.0, )", + "Blurhash.ImageSharp": "[4.1.1, )", + "MetadataExtractor": "[2.9.3, )", + "Microsoft.EntityFrameworkCore": "[10.0.13, )", + "Microsoft.Extensions.Configuration.Ini": "[10.0.13, )", + "Microsoft.Extensions.Hosting": "[10.0.13, )", + "Microsoft.Extensions.Options.DataAnnotations": "[10.0.13, )", + "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", + "SixLabors.ImageSharp": "[4.1.3, )", + "System.IO.Abstractions": "[22.3.0, )", + "System.IO.Hashing": "[10.0.13, )" + } + }, + "Blurhash.ImageSharp": { + "type": "CentralTransitive", + "requested": "[4.1.1, )", + "resolved": "4.1.1", + "contentHash": "wH77j2afRQ+yGmcLv730SsCKTvdYyyiSZsf7GMs95UgTRK7Mf06vP6dx5jWsWLRk0S9KxGOeXVJLKTo1VwviqQ==", + "dependencies": { + "Blurhash.Core": "4.2.1", + "SixLabors.ImageSharp": "4.1.1" + } + }, + "MetadataExtractor": { + "type": "CentralTransitive", + "requested": "[2.9.3, )", + "resolved": "2.9.3", + "contentHash": "7dBcqlPC2Zc/bsZtXnv8nCWe5h5WsLHszzHtxnpcsVy4MwlD3KQ3wVNoiYBm0a3rzt5hRwAosPvLPhlXrV5lyg==", + "dependencies": { + "XmpCore": "6.1.10.1" + } + }, + "Microsoft.EntityFrameworkCore": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "u/z4nTQUJ5nJEXAQekgQEQZkh0delp4gL+ESHn7Z6k037c8HC5mSNu1/YdBgxGm7DtLjhOxktNl+aQMzw6wN7A==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Abstractions": "10.0.13", + "Microsoft.EntityFrameworkCore.Analyzers": "10.0.13", + "Microsoft.Extensions.Caching.Memory": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13" + } + }, + "Microsoft.EntityFrameworkCore.Relational": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "QU/sbTtDR5wVAJRVecC1GwM7mwH8bvKOZNm+lulzMr5rhjdb8DDEFIqqdhF08JIo51DPv5uQdZ1nXoZsD5pTag==", + "dependencies": { + "Microsoft.EntityFrameworkCore": "10.0.13", + "Microsoft.Extensions.Caching.Memory": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "inhujYI/nikhmLxYI6ZEcaCgZ28ipJsKMPPxxr2XWf3vM0Q2P+E9R3hDHABZOLBxzD3gjpgcbSH/+hd2+gvI2A==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Ini": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "9qZZbQQZhbQlEbByHq3/a2MZMYNK0nkdsgaNJQvVWdbZpx5i94JR+Aa9NeLW7gp8sZfRfJENqtOYWYDKj/1e8g==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "4tnWU9vrU/A8NwYe9Z6wvieZ0EBb/uM82LAT+edNhdepOBWREZ7M7mzb941EIDPJaeOFdbVrDd5+7s9e89U6bQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Hosting": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "Q+hmPoDqeHipHyJcY7mDCQiLDUcn9jIgjoCThcgaOgjLTTT3ApTe8p6M2+twsDyfKuuCsKhSNlnNjaaggtXxRA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.Configuration.CommandLine": "10.0.13", + "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.13", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.13", + "Microsoft.Extensions.Configuration.Json": "10.0.13", + "Microsoft.Extensions.Configuration.UserSecrets": "10.0.13", + "Microsoft.Extensions.DependencyInjection": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Diagnostics": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Configuration": "10.0.13", + "Microsoft.Extensions.Logging.Console": "10.0.13", + "Microsoft.Extensions.Logging.Debug": "10.0.13", + "Microsoft.Extensions.Logging.EventLog": "10.0.13", + "Microsoft.Extensions.Logging.EventSource": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Options.DataAnnotations": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "aq+s0s0gm/EejOIPK/CteIrm7MoP34B7eudiFoiyC/1XH8rD/Cf2Xzu+fejEypF3CDGcaesA1YbrirQcJ5rQyg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "NodaTime": { + "type": "CentralTransitive", + "requested": "[3.3.5, )", + "resolved": "3.3.5", + "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" + }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, + "Npgsql": { + "type": "CentralTransitive", + "requested": "[10.0.3, )", + "resolved": "10.0.3", + "contentHash": "7nb5YzXuvWWJxB0J8DiyL3we+X4FOctZrt0fIBnucOIaIevFEEwGQVZKtiu9olXdlNAK1eNgqSral6r/jlhI4w==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0" + } + }, + "Npgsql.EntityFrameworkCore.PostgreSQL": { + "type": "CentralTransitive", + "requested": "[10.0.3, )", + "resolved": "10.0.3", + "contentHash": "IPGrrZnRkuW7OlHDhUESZz4G5DLkW7Nej/O3Cx+0iTsgyU5XJxBgpsvTHLloo3WWuAKKbDHXBvWPVkX1deRh1Q==", + "dependencies": { + "Microsoft.EntityFrameworkCore": "[10.0.4, 11.0.0)", + "Microsoft.EntityFrameworkCore.Relational": "[10.0.4, 11.0.0)", + "Npgsql": "10.0.3" + } + }, + "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": { + "type": "CentralTransitive", + "requested": "[10.0.3, )", + "resolved": "10.0.3", + "contentHash": "V84fTKMoEBsi5bAKVWrLDWimGqC6Dhn7ioBtoTpsfjpRmTcpv7FqhSqlqREO+6sHFH2sSXEZHEPWJTPsuKhfjA==", + "dependencies": { + "Npgsql.EntityFrameworkCore.PostgreSQL": "10.0.3", + "Npgsql.NodaTime": "10.0.3" + } + }, + "SixLabors.ImageSharp": { + "type": "CentralTransitive", + "requested": "[4.1.3, )", + "resolved": "4.1.3", + "contentHash": "VzzKPgdC0ZupEg937iKpg2cEGXudnFyQRwiHfIvtCKSlFUaxVbLjW3AztDhyk06z+4NzD7IAlMf7fqU57WXyBQ==", + "dependencies": { + "System.IO.Hashing": "8.0.0" + } + }, + "System.IO.Abstractions": { + "type": "CentralTransitive", + "requested": "[22.3.0, )", + "resolved": "22.3.0", + "contentHash": "dHMHwd6rl1CFJvdD9jnahy3ZjU67huvUCNhitH+7mzg8ZfLocc5ZQyKmUQMUJiQdElARWJBw0XQQzP6BCKsglA==", + "dependencies": { + "TestableIO.System.IO.Abstractions": "22.3.0", + "TestableIO.System.IO.Abstractions.Wrappers": "22.3.0" + } + }, + "System.IO.Hashing": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "ZlcywyQ7FJmgOV2nctdfOEPGS0kRK8T2CdOoPr6F4HPYpmjSo6GQfj3yTyOzNO6B+2U84/JKSPkJJMWqLmVsMw==" + } + } + } +} \ No newline at end of file diff --git a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerIniDefaultsTests.cs b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerIniDefaultsTests.cs index 3798e057..83135e1a 100644 --- a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerIniDefaultsTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerIniDefaultsTests.cs @@ -15,6 +15,12 @@ public void Entries_HaveTheDocumentedDefaults() ("Worker", "CrawlIntervalHours", "4"), ("Worker", "MaxConcurrentFiles", "4"), ("Worker", "TokenCleanupIntervalHours", "24"), + ("Worker", "WorkPollIntervalSeconds", "5"), + ("Worker", "WorkHeartbeatSeconds", "2"), + ("Worker", "WorkLeaseSeconds", "60"), + ("Worker", "WorkMaxStalls", "3"), + ("Worker", "WorkItemRetentionDays", "30"), + ("Worker", "WorkItemRetentionIntervalHours", "24"), ("Worker", "ThumbnailShortEdge", "300"), ("Worker", "ThumbnailJpegQuality", "75"), ("Worker", "PreviewLongEdge", "1920"), diff --git a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs index ed479f37..fd7b6cca 100644 --- a/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Infrastructure/WorkerServiceCollectionExtensionsTests.cs @@ -1,14 +1,18 @@ using Anichron.Core; using Anichron.Core.Data; using Anichron.Core.Data.Repository; +using Anichron.Core.Domain; +using Anichron.Core.Work; using Anichron.Worker.Crawling; using Anichron.Worker.Infrastructure; using Anichron.Worker.Ingestion; using Anichron.Worker.Ingestion.Classification; using Anichron.Worker.Ingestion.Pipeline; using Anichron.Worker.Ingestion.Proxy; +using Anichron.Worker.Maintenance; using Anichron.Worker.Settings; using Anichron.Worker.Startup; +using Anichron.Worker.Work; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -195,13 +199,67 @@ public void AddIngestionServices_RegistersProxyStagingWriter_AsSingleton() // ========================================================================== [Fact] - public void AddWorkerHostedServices_RegistersFourHostedServices() + public void AddWorkerHostedServices_MigratesFirstThenRunsTheWorkQueueAndTheCrawl() { var services = new ServiceCollection(); services.AddWorkerHostedServices(); - services.Count(d => d.ServiceType == typeof(IHostedService)).Should().Be(4); + services.Where(d => d.ServiceType == typeof(IHostedService)).Select(d => d.ImplementationType).Should().Equal( + typeof(DatabaseMigratorService), typeof(WorkerInitializer), typeof(WorkScheduler), typeof(WorkExecutor), + typeof(Worker.Crawling.Worker)); + } + + // ========================================================================== + // AddWorkQueueServices + // ========================================================================== + + [Fact] + public void AddWorkQueueServices_RegistersEachKindsHandlerAsScoped() + { + var services = new ServiceCollection(); + + services.AddWorkQueueServices(); + + services.Where(d => d.Lifetime == ServiceLifetime.Scoped) + .Select(d => (d.ServiceType, d.ImplementationType)).Should().BeEquivalentTo([ + (typeof(IWorkHandler), typeof(TokenCleanupHandler)), + (typeof(IWorkHandler), typeof(WorkItemRetentionHandler)), + ]); + } + + [Fact] + public void AddWorkQueueServices_RegistersEachPeriodicKindsSchedule() + => new ServiceCollection().AddWorkQueueServices() + .Where(d => d.ServiceType == typeof(IWorkSchedule)) + .Select(d => (d.ImplementationType, d.Lifetime)).Should().BeEquivalentTo([ + (typeof(TokenCleanupSchedule), ServiceLifetime.Singleton), + (typeof(WorkItemRetentionSchedule), ServiceLifetime.Singleton), + ]); + + [Fact] + public void AddWorkQueueServices_RegistersTheRegistryAsASingleton() + => new ServiceCollection().AddWorkQueueServices() + .Should().ContainSingle(d => d.ServiceType == typeof(WorkHandlerRegistry)) + .Which.Lifetime.Should().Be(ServiceLifetime.Singleton); + + [Fact] + public void TheWholeWorker_ResolvesEveryHostedServiceWithACompleteKindRegistry() + { + var services = DataServices(); + services.AddSingleton(new ConfigurationBuilder().Build()); + services.AddLogging(); + services.AddWorkerCoreServices().AddIngestionServices().AddWorkQueueServices().AddWorkerHostedServices(); + using var provider = services.BuildServiceProvider(new ServiceProviderOptions { ValidateScopes = true, ValidateOnBuild = true }); + + var hosted = provider.GetServices(); + + hosted.Should().HaveCount(5); + provider.GetRequiredService().Should().NotBeNull(); + using var scope = provider.CreateScope(); + provider.GetServices().Select(r => r.Kind).Should().BeEquivalentTo(Enum.GetValues()); + foreach (var registration in provider.GetServices()) + scope.ServiceProvider.GetRequiredService(typeof(IWorkHandler<>).MakeGenericType(registration.PayloadType)); } // ========================================================================== @@ -280,8 +338,25 @@ public void AddWorkerCoreServices_BindsWorkerSettingsFromTheWorkerSection() { typeof(IRefreshTokenRepository), typeof(EfRefreshTokenRepository) }, { typeof(IMediaAssetRepository), typeof(EfMediaAssetRepository) }, { typeof(IDatabaseMigrator), typeof(EfDatabaseMigrator) }, + { typeof(IWorkQueue), typeof(EfWorkQueue) }, }; + [Fact] + public void AddWorkerDataServices_RegistersTheWorkItemStoreAsASingleton() + => DataServices().Should().ContainSingle(d => d.ServiceType == typeof(IWorkItemStore)) + .Which.Should().Match(d => + d.ImplementationType == typeof(EfWorkItemStore) && d.Lifetime == ServiceLifetime.Singleton); + + [Fact] + public void AddWorkerDataServices_ContextFactoryCreatesNpgsqlContexts() + { + using var provider = DataServices().BuildServiceProvider(); + + using var context = provider.GetRequiredService>().CreateDbContext(); + + context.Database.GetConnectionString().Should().Contain("Host=db.internal"); + } + private static ServiceCollection DataServices() { var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary diff --git a/src/Anichron.Worker.Tests.Unit/Maintenance/TokenCleanupServiceTests.cs b/src/Anichron.Worker.Tests.Unit/Maintenance/TokenCleanupServiceTests.cs deleted file mode 100644 index 7b7b1d7f..00000000 --- a/src/Anichron.Worker.Tests.Unit/Maintenance/TokenCleanupServiceTests.cs +++ /dev/null @@ -1,167 +0,0 @@ -using Anichron.Core.Data.Repository; -using Anichron.Testing.DependencyInjection; -using Anichron.Testing.Time; -using Anichron.Worker.Maintenance; -using Anichron.Worker.Settings; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Testing; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Time.Testing; -using NodaTime; - -namespace Anichron.Worker.Tests.Unit.Maintenance; - -public sealed class TokenCleanupServiceTests -{ - private sealed class TestFixture - { - public IRefreshTokenRepository Repository { get; } = Substitute.For(); - public IClock Clock { get; } = Substitute.For(); - public Instant Now { get; } - - public IServiceScopeFactory ScopeFactory { get; } - - public TestFixture() - { - Now = Instant.FromUtc(2026, 5, 12, 10, 0, 0); - Clock.GetCurrentInstant().Returns(Now); - - ScopeFactory = ScopeFactories.Providing(services => services.AddSingleton(Repository)); - } - - public TokenCleanupService Build( - TimeProvider? timeProvider = null, double intervalHours = 24, ILogger? logger = null) - { - var settings = Options.Create(new WorkerSettings { TokenCleanupIntervalHours = intervalHours }); - return new TokenCleanupService( - ScopeFactory, Clock, timeProvider ?? new FakeTimeProvider(), settings, - logger ?? Substitute.For>()); - } - } - - // ========================================================================== - // RunCleanupAsync - // ========================================================================== - - [Fact] - public async Task RunCleanupAsync_CallsDeleteExpiredWithCurrentInstantAsync() - { - var fixture = new TestFixture(); - var service = fixture.Build(); - - await service.RunCleanupAsync(CancellationToken.None); - - await fixture.Repository.Received(1).DeleteExpiredAsync(fixture.Now, Arg.Any()); - } - - [Theory] - [InlineData(0)] - [InlineData(42)] - public async Task RunCleanupAsync_LogsDeletedCountAtInformationAsync(int deletedCount) - { - var fixture = new TestFixture(); - fixture.Repository.DeleteExpiredAsync(Arg.Any(), Arg.Any()).Returns(deletedCount); - var logger = new FakeLogger(); - var service = fixture.Build(logger: logger); - - await service.RunCleanupAsync(CancellationToken.None); - - logger.Collector.GetSnapshot().Should().ContainSingle() - .Which.Should().Match( - entry => entry.Level == LogLevel.Information && entry.Message.Contains(deletedCount.ToString(System.Globalization.CultureInfo.InvariantCulture))); - } - - [Fact] - public async Task RunCleanupAsync_PropagatesRepositoryExceptionAsync() - { - var fixture = new TestFixture(); - fixture.Repository.DeleteExpiredAsync(Arg.Any(), Arg.Any()) - .Returns(_ => throw new InvalidOperationException("DB error")); - var service = fixture.Build(); - - var act = async () => await service.RunCleanupAsync(CancellationToken.None); - - await act.Should().ThrowAsync().WithMessage("DB error"); - } - - // ========================================================================== - // ExecuteAsync — schedule - // ========================================================================== - - [Fact] - public async Task ExecuteAsync_Started_CleansUpAtOnceThenWaitsTheConfiguredIntervalAsync() - { - await using var timeline = new Timeline(); - var fixture = new TestFixture(); - RecordCleanups(fixture, timeline); - - await StartAsync(timeline, fixture.Build(timeline.Clock, intervalHours: 2)); - - await timeline.ExpectAsync(new Called("cleanup"), new TimerArmed(TimeSpan.FromHours(2))); - } - - [Fact] - public async Task ExecuteAsync_CleanupFails_LogsItAndCleansUpAgainAfterTheIntervalAsync() - { - await using var timeline = new Timeline(); - var fixture = new TestFixture(); - var calls = 0; - RecordCleanups(fixture, timeline, () => ++calls == 1 ? throw new InvalidOperationException("DB error") : 0); - var logger = new FakeLogger(); - await StartAsync(timeline, fixture.Build(timeline.Clock, intervalHours: 1, logger)); - await timeline.ExpectAsync(new Called("cleanup"), new TimerArmed(TimeSpan.FromHours(1))); - - timeline.Clock.Advance(TimeSpan.FromHours(1)); - - await timeline.ExpectAsync( - new TimerFired(TimeSpan.FromHours(1)), new Called("cleanup"), new TimerArmed(TimeSpan.FromHours(1))); - logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Level == LogLevel.Error); - } - - [Fact] - public async Task ExecuteAsync_CleanupCancelled_StopsInsteadOfLoggingAFailureAsync() - { - await using var timeline = new Timeline(); - var fixture = new TestFixture(); - RecordCleanups(fixture, timeline, () => throw new OperationCanceledException()); - var logger = new FakeLogger(); - - await StartAsync(timeline, fixture.Build(timeline.Clock, intervalHours: 1, logger)); - - await timeline.ExpectAsync(new Called("cleanup"), new Finished(TaskStatus.Canceled)); - logger.Collector.GetSnapshot().Should().BeEmpty(); - } - - [Fact] - public async Task ExecuteAsync_Stopped_EndsWithoutFaultingAsync() - { - await using var timeline = new Timeline(); - var fixture = new TestFixture(); - RecordCleanups(fixture, timeline); - var service = fixture.Build(timeline.Clock); - await StartAsync(timeline, service); - await timeline.ExpectAsync(new Called("cleanup"), new TimerArmed(TimeSpan.FromHours(24))); - - await service.StopAsync(Ct); - - await timeline.ExpectAsync(new Finished(TaskStatus.Canceled)); - } - - private static CancellationToken Ct => TestContext.Current.CancellationToken; - - private static void RecordCleanups(TestFixture fixture, Timeline timeline, Func? outcome = null) - => fixture.Repository.DeleteExpiredAsync(Arg.Any(), Arg.Any()) - .Returns(call => - { - call.Arg().ThrowIfCancellationRequested(); - timeline.Record("cleanup"); - return outcome?.Invoke() ?? 0; - }); - - private static async Task StartAsync(Timeline timeline, TokenCleanupService service) - { - await service.StartAsync(Ct); - timeline.Watch(service.ExecuteTask!, () => service.StopAsync(CancellationToken.None)); - } -} diff --git a/src/Anichron.Worker.Tests.Unit/Maintenance/TokenCleanupTests.cs b/src/Anichron.Worker.Tests.Unit/Maintenance/TokenCleanupTests.cs new file mode 100644 index 00000000..d05a2c35 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Maintenance/TokenCleanupTests.cs @@ -0,0 +1,68 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Work; +using Anichron.Worker.Maintenance; +using Anichron.Worker.Settings; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Maintenance; + +public sealed class TokenCleanupTests +{ + private static readonly Instant Now = Instant.FromUtc(2026, 5, 12, 10, 0, 0); + + private readonly IRefreshTokenRepository repository = Substitute.For(); + private readonly IClock clock = Substitute.For(); + private readonly FakeLogger logger = new(); + + public TokenCleanupTests() => clock.GetCurrentInstant().Returns(Now); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private Task RunAsync() + => new TokenCleanupHandler(repository, clock, logger) + .RunAsync(new TokenCleanupPayload(), Substitute.For(), Ct); + + [Fact] + public async Task RunAsync_DeletesTokensExpiredBeforeNowAsync() + { + await RunAsync(); + + await repository.Received(1).DeleteExpiredAsync(Now, Ct); + } + + [Theory] + [InlineData(0)] + [InlineData(42)] + public async Task RunAsync_ReportsTheDeletedCountAsTheResultAndInTheLogAsync(int deletedCount) + { + repository.DeleteExpiredAsync(Arg.Any(), Arg.Any()).Returns(deletedCount); + + var result = await RunAsync(); + + result.Should().Be(new TokenCleanupResult(deletedCount)); + logger.Collector.GetSnapshot().Should().ContainSingle() + .Which.Should().Match(entry => entry.Level == LogLevel.Information + && entry.Message.Contains(deletedCount.ToString(System.Globalization.CultureInfo.InvariantCulture))); + } + + [Fact] + public async Task RunAsync_RepositoryFails_PropagatesSoTheItemFailsAsync() + { + repository.DeleteExpiredAsync(Arg.Any(), Arg.Any()) + .Returns(_ => throw new InvalidOperationException("DB error")); + + var act = RunAsync; + + await act.Should().ThrowAsync().WithMessage("DB error"); + } + + [Theory] + [InlineData(24)] + [InlineData(2.5)] + public void Schedule_IsTheTokenCleanupKeyAtTheConfiguredInterval(double hours) + => new TokenCleanupSchedule(Options.Create(new WorkerSettings { TokenCleanupIntervalHours = hours })) + .Should().BeEquivalentTo(new { DedupKey = "token-cleanup", Interval = Duration.FromHours(hours) }); +} diff --git a/src/Anichron.Worker.Tests.Unit/Maintenance/WorkItemRetentionTests.cs b/src/Anichron.Worker.Tests.Unit/Maintenance/WorkItemRetentionTests.cs new file mode 100644 index 00000000..0daf1964 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Maintenance/WorkItemRetentionTests.cs @@ -0,0 +1,62 @@ +using Anichron.Core.Work; +using Anichron.Worker.Maintenance; +using Anichron.Worker.Settings; +using Anichron.Worker.Work; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Maintenance; + +public sealed class WorkItemRetentionTests +{ + private static readonly Instant Now = Instant.FromUtc(2026, 10, 18, 12, 0); + + private readonly IWorkItemStore store = Substitute.For(); + private readonly IClock clock = Substitute.For(); + private readonly FakeLogger logger = new(); + + public WorkItemRetentionTests() => clock.GetCurrentInstant().Returns(Now); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private Task RunAsync(WorkerSettings? settings = null) + => new WorkItemRetentionHandler(store, clock, Options.Create(settings ?? new WorkerSettings()), logger) + .RunAsync(new WorkItemRetentionPayload(), Substitute.For(), Ct); + + [Fact] + public async Task RunAsync_ByDefault_DeletesItemsFinishedMoreThanThirtyDaysAgoAsync() + { + await RunAsync(); + + await store.Received(1).DeleteFinishedBeforeAsync(Instant.FromUtc(2026, 9, 18, 12, 0), Ct); + } + + [Fact] + public async Task RunAsync_UsesTheConfiguredRetentionAsync() + { + await RunAsync(new WorkerSettings { WorkItemRetentionDays = 1.5 }); + + await store.Received(1).DeleteFinishedBeforeAsync(Instant.FromUtc(2026, 10, 17, 0, 0), Ct); + } + + [Fact] + public async Task RunAsync_ReportsTheDeletedCountAndCutoffAsTheResultAndInTheLogAsync() + { + store.DeleteFinishedBeforeAsync(Arg.Any(), Arg.Any()).Returns(7); + + var result = await RunAsync(); + + result.Should().Be(new WorkItemRetentionResult(7, Instant.FromUtc(2026, 9, 18, 12, 0))); + logger.Collector.GetSnapshot().Should().ContainSingle() + .Which.Should().Match(entry => entry.Level == LogLevel.Information && entry.Message.Contains('7')); + } + + [Theory] + [InlineData(24)] + [InlineData(6)] + public void Schedule_IsTheRetentionKeyAtTheConfiguredInterval(double hours) + => new WorkItemRetentionSchedule(Options.Create(new WorkerSettings { WorkItemRetentionIntervalHours = hours })) + .Should().BeEquivalentTo(new { DedupKey = "work-item-retention", Interval = Duration.FromHours(hours) }); +} diff --git a/src/Anichron.Worker.Tests.Unit/Settings/WorkerSettingsTests.cs b/src/Anichron.Worker.Tests.Unit/Settings/WorkerSettingsTests.cs index f1a64eeb..ad70e41d 100644 --- a/src/Anichron.Worker.Tests.Unit/Settings/WorkerSettingsTests.cs +++ b/src/Anichron.Worker.Tests.Unit/Settings/WorkerSettingsTests.cs @@ -167,6 +167,63 @@ public void Validate_TokenCleanupIntervalHoursZero_ReturnsFailed() IsValid(ValidSettings with { TokenCleanupIntervalHours = 0 }).Should().BeFalse(); } + // ========================================================================== + // Work queue + // ========================================================================== + + [Fact] + public void Defaults_WorkQueueTimingsAreTheDocumentedOnes() + => new WorkerSettings().Should().BeEquivalentTo(new + { + WorkPollIntervalSeconds = 5.0, + WorkHeartbeatSeconds = 2.0, + WorkLeaseSeconds = 60.0, + WorkMaxStalls = 3, + WorkItemRetentionDays = 30.0, + WorkItemRetentionIntervalHours = 24.0, + }); + + private static WorkerSettings WithZero(string setting) => setting switch + { + nameof(WorkerSettings.WorkPollIntervalSeconds) => ValidSettings with { WorkPollIntervalSeconds = 0 }, + nameof(WorkerSettings.WorkHeartbeatSeconds) => ValidSettings with { WorkHeartbeatSeconds = 0 }, + nameof(WorkerSettings.WorkLeaseSeconds) => ValidSettings with { WorkLeaseSeconds = 0 }, + nameof(WorkerSettings.WorkMaxStalls) => ValidSettings with { WorkMaxStalls = 0 }, + nameof(WorkerSettings.WorkItemRetentionDays) => ValidSettings with { WorkItemRetentionDays = 0 }, + _ => ValidSettings with { WorkItemRetentionIntervalHours = 0 }, + }; + + [Theory] + [InlineData(nameof(WorkerSettings.WorkPollIntervalSeconds))] + [InlineData(nameof(WorkerSettings.WorkHeartbeatSeconds))] + [InlineData(nameof(WorkerSettings.WorkLeaseSeconds))] + [InlineData(nameof(WorkerSettings.WorkMaxStalls))] + [InlineData(nameof(WorkerSettings.WorkItemRetentionDays))] + [InlineData(nameof(WorkerSettings.WorkItemRetentionIntervalHours))] + public void Validate_NonPositiveWorkQueueSetting_ReportsIt(string setting) + => GetFailures(WithZero(setting)).Should().Contain(failure => failure.MemberNames.Contains(setting)); + + [Theory] + [InlineData(2, 2)] + [InlineData(10, 2)] + public void Validate_LeaseNotLongerThanTheHeartbeat_ReportsTheLease(double heartbeat, double lease) + => GetFailures(ValidSettings with { WorkHeartbeatSeconds = heartbeat, WorkLeaseSeconds = lease }) + .Should().ContainSingle().Which.Should().BeEquivalentTo(new + { + MemberNames = new[] { nameof(WorkerSettings.WorkLeaseSeconds) }, + ErrorMessage = "Worker:WorkLeaseSeconds must be longer than Worker:WorkHeartbeatSeconds, or every lease expires between heartbeats.", + }); + + [Fact] + public void Validate_LeaseJustLongerThanTheHeartbeat_ReturnsSuccess() + => IsValid(ValidSettings with { WorkHeartbeatSeconds = 2, WorkLeaseSeconds = 2.5 }).Should().BeTrue(); + + [Fact] + public void Validate_LeaseAndAPathEntryBothWrong_ReportsThePathFirst() + => GetFailures(ValidSettings with { WorkLeaseSeconds = 1, ExcludedDirectories = "a/b" }) + .Select(failure => failure.MemberNames.Single()) + .Should().Equal(nameof(WorkerSettings.ExcludedDirectories), nameof(WorkerSettings.WorkLeaseSeconds)); + // ========================================================================== // FfmpegPath // ========================================================================== diff --git a/src/Anichron.Worker.Tests.Unit/Work/EfWorkItemStoreTests.cs b/src/Anichron.Worker.Tests.Unit/Work/EfWorkItemStoreTests.cs new file mode 100644 index 00000000..0052aaf8 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Work/EfWorkItemStoreTests.cs @@ -0,0 +1,92 @@ +using Anichron.Core.Data; +using Anichron.Core.Domain; +using Anichron.Testing.Data; +using Anichron.Worker.Work; +using Microsoft.EntityFrameworkCore; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Work; + +public sealed class EfWorkItemStoreTests +{ + private sealed class InMemoryContexts(string database) : IDbContextFactory + { + public AnichronDbContext CreateDbContext() => AnichronDbContexts.InMemory(database); + } + + private static readonly Instant Cutoff = Instant.FromUtc(2026, 10, 18, 9, 0); + + private readonly InMemoryContexts contexts = new(Guid.NewGuid().ToString()); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private async Task AddAsync(string key, WorkItemStatus status, Instant? finishedAt = null) + { + await using var db = contexts.CreateDbContext(); + db.WorkItems.Add(new WorkItem + { + Id = Guid.NewGuid(), + Kind = WorkKind.TokenCleanup, + Payload = "{}", + DedupKey = key, + Status = status, + FinishedAt = finishedAt, + }); + await db.SaveChangesAsync(Ct); + } + + private Task IsDueAsync() => new EfWorkItemStore(contexts).IsDueAsync("token-cleanup", Cutoff, Ct); + + [Fact] + public async Task IsDueAsync_NeverRan_IsDueAsync() + => (await IsDueAsync()).Should().BeTrue(); + + [Theory] + [InlineData(WorkItemStatus.Queued)] + [InlineData(WorkItemStatus.Running)] + public async Task IsDueAsync_AnItemIsPending_IsNotDueAsync(WorkItemStatus status) + { + await AddAsync("token-cleanup", status); + + (await IsDueAsync()).Should().BeFalse(); + } + + [Theory] + [InlineData(WorkItemStatus.Succeeded)] + [InlineData(WorkItemStatus.Failed)] + [InlineData(WorkItemStatus.Cancelled)] + public async Task IsDueAsync_FinishedAfterTheCutoff_IsNotDueAsync(WorkItemStatus status) + { + await AddAsync("token-cleanup", status, Cutoff + Duration.FromTicks(1)); + + (await IsDueAsync()).Should().BeFalse(); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + public async Task IsDueAsync_LastFinishedAtOrBeforeTheCutoff_IsDueAsync(long ticksEarlier) + { + await AddAsync("token-cleanup", WorkItemStatus.Succeeded, Cutoff - Duration.FromTicks(ticksEarlier)); + + (await IsDueAsync()).Should().BeTrue(); + } + + [Theory] + [InlineData(WorkItemStatus.Queued)] + [InlineData(WorkItemStatus.Running)] + public async Task IsDueAsync_OnlyAnotherKeyIsPending_IsDueAsync(WorkItemStatus status) + { + await AddAsync("work-item-retention", status); + + (await IsDueAsync()).Should().BeTrue(); + } + + [Fact] + public async Task IsDueAsync_AnotherKeyFinishedRecently_IsDueAsync() + { + await AddAsync("work-item-retention", WorkItemStatus.Succeeded, Cutoff + Duration.FromHours(1)); + + (await IsDueAsync()).Should().BeTrue(); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Work/WorkExecutorTests.cs b/src/Anichron.Worker.Tests.Unit/Work/WorkExecutorTests.cs new file mode 100644 index 00000000..0030e09e --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Work/WorkExecutorTests.cs @@ -0,0 +1,353 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Testing.DependencyInjection; +using Anichron.Testing.Time; +using Anichron.Worker.Settings; +using Anichron.Worker.Work; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Work; + +public sealed class WorkExecutorTests : IAsyncDisposable +{ + public sealed record Cleanup : IWorkPayload + { + public static WorkKind Kind => WorkKind.TokenCleanup; + } + + public sealed record Retention : IWorkPayload + { + public static WorkKind Kind => WorkKind.WorkItemRetention; + } + + private static readonly TimeSpan Heartbeat = TimeSpan.FromSeconds(2); + private static readonly TimeSpan Poll = TimeSpan.FromSeconds(5); + private static readonly Duration Lease = Duration.FromSeconds(60); + + private readonly Timeline timeline = new(); + private readonly IWorkItemStore store = Substitute.For(); + private readonly IWorkHandler handler = Substitute.For>(); + private readonly FakeLogger logger = new(); + private readonly Queue queued = new(); + private readonly List> runs = []; + private CancellationToken handlerToken; + private IWorkProgress? handlerProgress; + private WorkExecutor? executor; + + public WorkExecutorTests() + { + store.ClaimAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("claim"); + return queued.TryDequeue(out var work) ? work : null; + }); + store.HeartbeatAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("heartbeat"); + return true; + }); + store.When(s => s.SucceedAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any())) + .Do(_ => timeline.Record("succeed")); + store.When(s => s.FailAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any())) + .Do(_ => timeline.Record("fail")); + store.When(s => s.ReleaseAsync(Arg.Any(), Arg.Any(), Arg.Any())) + .Do(_ => timeline.Record("release")); + handler.RunAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + handlerProgress = call.Arg(); + handlerToken = call.Arg(); + var run = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handlerToken.Register(() => run.TrySetCanceled(handlerToken)); + runs.Add(run); + timeline.Record("run"); + return run.Task; + }); + } + + public ValueTask DisposeAsync() => timeline.DisposeAsync(); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private Instant Now => Instant.FromDateTimeOffset(timeline.Clock.GetUtcNow()); + + private static ClaimedWork Work(int attempt = 1) + => new(new WorkAttempt(Guid.Parse("33333333-3333-3333-3333-333333333333"), attempt), WorkKind.TokenCleanup, "{}"); + + private async Task StartAsync(params ClaimedWork[] items) + { + foreach (var item in items) + queued.Enqueue(item); + var registry = new WorkHandlerRegistry( + [WorkHandlerRegistration.For(), WorkHandlerRegistration.For()], + [typeof(Cleanup), typeof(Retention)]); + executor = new WorkExecutor( + store, registry, ScopeFactories.Providing(services => services.AddSingleton(handler)), + timeline.Clock, Options.Create(new WorkerSettings()), logger); + await executor.StartAsync(Ct); + timeline.Watch(executor.ExecuteTask!, () => executor.StopAsync(CancellationToken.None)); + } + + private async Task StartRunningAsync() + { + await StartAsync(Work()); + await timeline.ExpectAsync(new Called("claim"), new TimerArmed(Heartbeat), new Called("run")); + } + + // ========================================================================== + // Claiming + // ========================================================================== + + [Fact] + public async Task ExecuteAsync_NothingQueued_ClaimsAgainAfterThePollIntervalAsync() + { + await StartAsync(); + await timeline.ExpectAsync(new Called("claim"), new TimerArmed(Poll)); + + timeline.Clock.Advance(Poll); + + await timeline.ExpectAsync(new TimerFired(Poll), new Called("claim"), new TimerArmed(Poll)); + } + + [Fact] + public async Task ExecuteAsync_ClaimsAsThisWorkerWithTheConfiguredLeaseAsync() + { + await StartAsync(); + await timeline.ExpectAsync(new Called("claim"), new TimerArmed(Poll)); + + await store.Received(1).ClaimAsync($"all@{Environment.MachineName}", Now, Lease, Arg.Any()); + } + + [Theory] + [InlineData("", "all@nas")] + [InlineData("mira", "mira@nas")] + public void ClaimantName_IsTheWorkersUserOrAllThenItsHost(string user, string expected) + => WorkExecutor.ClaimantName(new WorkerSettings { User = user }, "nas").Should().Be(expected); + + [Fact] + public async Task ExecuteAsync_ClaimFails_LogsAndClaimsAgainAtTheNextPollAsync() + { + var calls = 0; + store.ClaimAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("claim"); + return ++calls == 1 ? throw new InvalidOperationException("db down") : null; + }); + await StartAsync(); + await timeline.ExpectAsync(new Called("claim"), new TimerArmed(Poll)); + + timeline.Clock.Advance(Poll); + + await timeline.ExpectAsync(new TimerFired(Poll), new Called("claim"), new TimerArmed(Poll)); + logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Level == LogLevel.Error); + } + + [Fact] + public async Task ExecuteAsync_TwoItemsQueued_RunsThemOneAtATimeAsync() + { + await StartAsync(Work(), Work(attempt: 2)); + await timeline.ExpectAsync(new Called("claim"), new TimerArmed(Heartbeat), new Called("run")); + + runs[0].SetResult(null); + + await timeline.ExpectAsync( + new Called("succeed"), new Called("claim"), new TimerArmed(Heartbeat), new Called("run")); + runs[1].SetResult(null); + await timeline.ExpectAsync(new Called("succeed"), new Called("claim"), new TimerArmed(Poll)); + } + + // ========================================================================== + // Heartbeat + // ========================================================================== + + [Fact] + public async Task ExecuteAsync_Running_HeartbeatsAtTheIntervalAsync() + { + await StartRunningAsync(); + timeline.Clock.Advance(Heartbeat - TimeSpan.FromTicks(1)); + timeline.Unread.Should().BeEmpty(); + + timeline.Clock.Advance(TimeSpan.FromTicks(1)); + await timeline.ExpectAsync(new TimerFired(Heartbeat), new Called("heartbeat"), new TimerArmed(Heartbeat)); + timeline.Clock.Advance(Heartbeat); + await timeline.ExpectAsync(new TimerFired(Heartbeat), new Called("heartbeat"), new TimerArmed(Heartbeat)); + } + + [Fact] + public async Task ExecuteAsync_Heartbeat_ExtendsTheLeaseAndSendsTheReportedProgressAsync() + { + await StartRunningAsync(); + handlerProgress!.Report(3, 8, "walking"); + + timeline.Clock.Advance(Heartbeat); + + await timeline.ExpectAsync(new TimerFired(Heartbeat), new Called("heartbeat"), new TimerArmed(Heartbeat)); + await store.Received(1).HeartbeatAsync( + Work().Attempt, new WorkProgress(3, 8, "walking"), Now, Lease, Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_HeartbeatFails_LogsAndKeepsHeartbeatingAsync() + { + var beats = 0; + store.HeartbeatAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("heartbeat"); + return ++beats == 1 ? throw new InvalidOperationException("db down") : true; + }); + await StartRunningAsync(); + + timeline.Clock.Advance(Heartbeat); + await timeline.ExpectAsync(new TimerFired(Heartbeat), new Called("heartbeat"), new TimerArmed(Heartbeat)); + timeline.Clock.Advance(Heartbeat); + + await timeline.ExpectAsync(new TimerFired(Heartbeat), new Called("heartbeat"), new TimerArmed(Heartbeat)); + handlerToken.IsCancellationRequested.Should().BeFalse(); + logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Level == LogLevel.Warning); + } + + [Fact] + public async Task ExecuteAsync_HeartbeatFenced_CancelsTheHandlerAndWritesNothingMoreAsync() + { + store.HeartbeatAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("heartbeat"); + return false; + }); + await StartRunningAsync(); + + timeline.Clock.Advance(Heartbeat); + + await timeline.ExpectAsync( + new TimerFired(Heartbeat), new Called("heartbeat"), new Called("claim"), new TimerArmed(Poll)); + handlerToken.IsCancellationRequested.Should().BeTrue(); + } + + [Fact] + public async Task ExecuteAsync_FencedHandlerFinishesAnyway_WritesNothingAsync() + { + var fenced = false; + store.HeartbeatAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("heartbeat"); + fenced = runs[0].TrySetResult("done anyway"); + return false; + }); + await StartRunningAsync(); + + timeline.Clock.Advance(Heartbeat); + + await timeline.ExpectAsync( + new TimerFired(Heartbeat), new Called("heartbeat"), new Called("claim"), new TimerArmed(Poll)); + fenced.Should().BeTrue(); + } + + // ========================================================================== + // Outcomes + // ========================================================================== + + [Fact] + public async Task ExecuteAsync_HandlerSucceeds_StoresItsResultAndFinalProgressAsync() + { + await StartRunningAsync(); + handlerProgress!.Report(3, 3, "done"); + + runs[0].SetResult(new { Deleted = 3 }); + + await timeline.ExpectAsync(new Called("succeed"), new Called("claim"), new TimerArmed(Poll)); + await store.Received(1).SucceedAsync( + Work().Attempt, new WorkProgress(3, 3, "done"), /*lang=json,strict*/ """{"deleted":3}""", Now, CancellationToken.None); + } + + [Fact] + public async Task ExecuteAsync_HandlerReturnsNoResult_StoresNoneAsync() + { + await StartRunningAsync(); + + runs[0].SetResult(null); + + await timeline.ExpectAsync(new Called("succeed"), new Called("claim"), new TimerArmed(Poll)); + await store.Received(1).SucceedAsync(Work().Attempt, WorkProgress.None, null, Now, CancellationToken.None); + } + + [Fact] + public async Task ExecuteAsync_HandlerThrows_FailsTheItemWithTheErrorAndLastProgressAsync() + { + await StartRunningAsync(); + handlerProgress!.Report(1, null, "walking"); + + runs[0].SetException(new InvalidOperationException("boom")); + + await timeline.ExpectAsync(new Called("fail"), new Called("claim"), new TimerArmed(Poll)); + await store.Received(1).FailAsync( + Work().Attempt, new WorkProgress(1, null, "walking"), "InvalidOperationException: boom", Now, CancellationToken.None); + logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Level == LogLevel.Error); + } + + [Fact] + public async Task ExecuteAsync_HandlerCancelsItselfWithoutAShutdown_FailsTheItemAsync() + { + await StartRunningAsync(); + + runs[0].SetCanceled(CancellationToken.None); + + await timeline.ExpectAsync(new Called("fail"), new Called("claim"), new TimerArmed(Poll)); + await store.Received(1).FailAsync( + Work().Attempt, WorkProgress.None, "TaskCanceledException: A task was canceled.", Now, CancellationToken.None); + } + + [Fact] + public async Task ExecuteAsync_HandlerThrowsSynchronously_FailsTheItemAsync() + { + handler.RunAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("run"); + throw new InvalidOperationException("bad payload"); + }); + await StartAsync(Work()); + + await timeline.ExpectAsync( + new Called("claim"), new TimerArmed(Heartbeat), new Called("run"), new Called("fail"), + new Called("claim"), new TimerArmed(Poll)); + } + + [Fact] + public async Task StopAsync_WhileRunning_CancelsTheHandlerAndReleasesTheItemAsync() + { + await StartRunningAsync(); + + await executor!.StopAsync(Ct); + + await timeline.ExpectAsync(new Called("release"), new Finished(TaskStatus.Canceled)); + handlerToken.IsCancellationRequested.Should().BeTrue(); + await store.Received(1).ReleaseAsync(Work().Attempt, Now, CancellationToken.None); + } + + [Fact] + public async Task StopAsync_WhileWaitingToPoll_EndsWithoutFaultingAsync() + { + await StartAsync(); + await timeline.ExpectAsync(new Called("claim"), new TimerArmed(Poll)); + + await executor!.StopAsync(Ct); + + await timeline.ExpectAsync(new Finished(TaskStatus.Canceled)); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Work/WorkHandlerRegistryTests.cs b/src/Anichron.Worker.Tests.Unit/Work/WorkHandlerRegistryTests.cs new file mode 100644 index 00000000..eb6b0f92 --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Work/WorkHandlerRegistryTests.cs @@ -0,0 +1,92 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Worker.Work; +using Microsoft.Extensions.DependencyInjection; + +namespace Anichron.Worker.Tests.Unit.Work; + +public sealed class WorkHandlerRegistryTests +{ + public sealed record OtherCleanupPayload : IWorkPayload + { + public static WorkKind Kind => WorkKind.TokenCleanup; + } + + public sealed record NumberedRetentionPayload(int Number) : IWorkPayload + { + public static WorkKind Kind => WorkKind.WorkItemRetention; + } + + private static readonly Type[] RealPayloads = [typeof(TokenCleanupPayload), typeof(WorkItemRetentionPayload)]; + + private static WorkHandlerRegistration[] RealHandlers + => [WorkHandlerRegistration.For(), WorkHandlerRegistration.For()]; + + private static Action Building(IEnumerable registrations, params Type[] payloads) + => () => _ = new WorkHandlerRegistry(registrations, payloads); + + [Fact] + public void PayloadTypesIn_Core_FindsEveryConcretePayload() + => WorkHandlerRegistry.PayloadTypesIn(typeof(IWorkPayload).Assembly) + .Should().BeEquivalentTo(RealPayloads); + + [Fact] + public void Constructor_OnePayloadAndOneHandlerPerKind_Builds() + => Building(RealHandlers, RealPayloads).Should().NotThrow(); + + [Fact] + public void Constructor_FromDependencyInjection_ChecksTheCorePayloads() + { + var act = () => new WorkHandlerRegistry([WorkHandlerRegistration.For()]); + + act.Should().Throw() + .WithMessage("WorkKind.WorkItemRetention needs exactly one handler, found none."); + } + + [Fact] + public void Constructor_KindWithoutPayload_FailsNamingIt() + => Building(RealHandlers, typeof(TokenCleanupPayload)).Should().Throw() + .WithMessage("WorkKind.WorkItemRetention needs exactly one payload type, found none."); + + [Fact] + public void Constructor_KindWithTwoPayloads_FailsNamingBoth() + => Building(RealHandlers, [.. RealPayloads, typeof(OtherCleanupPayload)]).Should().Throw() + .WithMessage("WorkKind.TokenCleanup needs exactly one payload type, found TokenCleanupPayload, OtherCleanupPayload."); + + [Fact] + public void Constructor_KindWithTwoHandlers_FailsNamingThem() + => Building([.. RealHandlers, WorkHandlerRegistration.For()], RealPayloads) + .Should().Throw() + .WithMessage("WorkKind.TokenCleanup needs exactly one handler, found TokenCleanupPayload, OtherCleanupPayload."); + + [Fact] + public void Constructor_HandlerForAnotherPayloadOfTheKind_FailsNamingBoth() + => Building([WorkHandlerRegistration.For(), RealHandlers[1]], RealPayloads) + .Should().Throw() + .WithMessage("WorkKind.TokenCleanup's handler takes OtherCleanupPayload, not its payload TokenCleanupPayload."); + + [Fact] + public void Constructor_SeveralProblems_ReportsEachOnItsOwnLine() + => Building([], RealPayloads).Should().Throw() + .Which.Message.Should().Be(string.Join(Environment.NewLine, + "WorkKind.TokenCleanup needs exactly one handler, found none.", + "WorkKind.WorkItemRetention needs exactly one handler, found none.")); + + [Fact] + public async Task RunAsync_DispatchesTheDeserialisedPayloadToTheKindsHandlerAsync() + { + var handler = Substitute.For>(); + var progress = Substitute.For(); + using var cts = new CancellationTokenSource(); + handler.RunAsync(new NumberedRetentionPayload(7), progress, cts.Token).Returns("summary"); + await using var services = new ServiceCollection().AddSingleton(handler).BuildServiceProvider(); + var registry = new WorkHandlerRegistry( + [RealHandlers[0], WorkHandlerRegistration.For()], + [typeof(TokenCleanupPayload), typeof(NumberedRetentionPayload)]); + var work = new ClaimedWork(new WorkAttempt(Guid.NewGuid(), 1), WorkKind.WorkItemRetention, /*lang=json,strict*/ """{"number":7}"""); + + var result = await registry.RunAsync(work, services, progress, cts.Token); + + result.Should().Be("summary"); + } +} diff --git a/src/Anichron.Worker.Tests.Unit/Work/WorkSchedulerTests.cs b/src/Anichron.Worker.Tests.Unit/Work/WorkSchedulerTests.cs new file mode 100644 index 00000000..a7617c6e --- /dev/null +++ b/src/Anichron.Worker.Tests.Unit/Work/WorkSchedulerTests.cs @@ -0,0 +1,155 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Anichron.Testing.DependencyInjection; +using Anichron.Testing.Time; +using Anichron.Worker.Settings; +using Anichron.Worker.Work; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Tests.Unit.Work; + +public sealed class WorkSchedulerTests : IAsyncDisposable +{ + public sealed record Keyed(string Key) : IWorkPayload + { + public static WorkKind Kind => WorkKind.TokenCleanup; + + string IWorkPayload.DedupKey => Key; + } + + private sealed class KeyedSchedule(string key, Duration interval) + : WorkSchedule(new Keyed(key), interval); + + private static readonly TimeSpan Poll = TimeSpan.FromSeconds(5); + private static readonly Duration Hourly = Duration.FromHours(1); + private static readonly Duration Daily = Duration.FromDays(1); + + private readonly Timeline timeline = new(); + private readonly IWorkItemStore store = Substitute.For(); + private readonly IWorkQueue queue = Substitute.For(); + private readonly FakeLogger logger = new(); + private readonly HashSet due = []; + private WorkScheduler? scheduler; + + public WorkSchedulerTests() + { + store.RecoverExpiredLeasesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("recover"); + return Task.CompletedTask; + }); + store.IsDueAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => due.Contains(call.Arg())); + queue.EnqueueAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + timeline.Record($"enqueue {call.Arg().Key}"); + return Guid.NewGuid(); + }); + } + + public ValueTask DisposeAsync() => timeline.DisposeAsync(); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private Instant Now => Instant.FromDateTimeOffset(timeline.Clock.GetUtcNow()); + + private async Task StartAsync(params IWorkSchedule[] schedules) + { + scheduler = new WorkScheduler( + store, schedules, ScopeFactories.Providing(services => services.AddSingleton(queue)), + timeline.Clock, Options.Create(new WorkerSettings()), logger); + await scheduler.StartAsync(Ct); + timeline.Watch(scheduler.ExecuteTask!, () => scheduler.StopAsync(CancellationToken.None)); + } + + [Fact] + public async Task ExecuteAsync_Started_RecoversLeasesThenEnqueuesWhatIsDueAsync() + { + due.Add("sweep"); + + await StartAsync(new KeyedSchedule("sweep", Hourly), new KeyedSchedule("purge", Daily)); + + await timeline.ExpectAsync(new Called("recover"), new Called("enqueue sweep"), new TimerArmed(Poll)); + await store.Received(1).RecoverExpiredLeasesAsync(Now, 3, Arg.Any()); + await queue.Received(1).EnqueueAsync(new Keyed("sweep"), null, Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_AsksEachScheduleWhetherItsLastRunIsOlderThanItsIntervalAsync() + { + await StartAsync(new KeyedSchedule("sweep", Hourly), new KeyedSchedule("purge", Daily)); + await timeline.ExpectAsync(new Called("recover"), new TimerArmed(Poll)); + + await store.Received(1).IsDueAsync("sweep", Now - Hourly, Arg.Any()); + await store.Received(1).IsDueAsync("purge", Now - Daily, Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_PollIntervalElapses_TicksAgainAtTheLaterTimeAsync() + { + await StartAsync(new KeyedSchedule("sweep", Hourly)); + await timeline.ExpectAsync(new Called("recover"), new TimerArmed(Poll)); + due.Add("sweep"); + + timeline.Clock.Advance(Poll); + + await timeline.ExpectAsync( + new TimerFired(Poll), new Called("recover"), new Called("enqueue sweep"), new TimerArmed(Poll)); + await store.Received(1).IsDueAsync("sweep", Now - Hourly, Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_TickFails_LogsAndTicksAgainAtTheNextPollAsync() + { + var calls = 0; + store.RecoverExpiredLeasesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => + { + timeline.Record("recover"); + return ++calls == 1 ? throw new InvalidOperationException("db down") : Task.CompletedTask; + }); + await StartAsync(); + await timeline.ExpectAsync(new Called("recover"), new TimerArmed(Poll)); + + timeline.Clock.Advance(Poll); + + await timeline.ExpectAsync(new TimerFired(Poll), new Called("recover"), new TimerArmed(Poll)); + logger.Collector.GetSnapshot().Should().ContainSingle(e => e.Level == LogLevel.Error); + } + + [Fact] + public async Task StopAsync_EndsWithoutFaultingAsync() + { + await StartAsync(); + await timeline.ExpectAsync(new Called("recover"), new TimerArmed(Poll)); + + await scheduler!.StopAsync(Ct); + + await timeline.ExpectAsync(new Finished(TaskStatus.Canceled)); + } + + [Fact] + public void WorkSchedule_TakesItsKeyFromThePayload() + => new KeyedSchedule("sweep", Hourly).Should().BeEquivalentTo(new { DedupKey = "sweep", Interval = Hourly }); + + [Fact] + public void WorkSchedule_PayloadWithoutAKey_IsRejected() + { + var act = () => new UnkeyedSchedule(); + + act.Should().Throw().WithMessage("A periodic WorkItemRetention item needs a dedup key."); + } + + public sealed record Unkeyed : IWorkPayload + { + public static WorkKind Kind => WorkKind.WorkItemRetention; + } + + private sealed class UnkeyedSchedule() : WorkSchedule(new Unkeyed(), Duration.FromHours(1)); +} diff --git a/src/Anichron.Worker.Tests.Unit/packages.lock.json b/src/Anichron.Worker.Tests.Unit/packages.lock.json index 0f7a33c2..1b7a7da3 100644 --- a/src/Anichron.Worker.Tests.Unit/packages.lock.json +++ b/src/Anichron.Worker.Tests.Unit/packages.lock.json @@ -566,6 +566,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -759,6 +760,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.Worker/Infrastructure/WorkerIniDefaults.cs b/src/Anichron.Worker/Infrastructure/WorkerIniDefaults.cs index 6430fca9..5a9f8de7 100644 --- a/src/Anichron.Worker/Infrastructure/WorkerIniDefaults.cs +++ b/src/Anichron.Worker/Infrastructure/WorkerIniDefaults.cs @@ -10,6 +10,12 @@ internal static class WorkerIniDefaults new IniEntry("Worker", "CrawlIntervalHours", () => "4"), new IniEntry("Worker", "MaxConcurrentFiles", () => "4"), new IniEntry("Worker", "TokenCleanupIntervalHours", () => "24"), + new IniEntry("Worker", "WorkPollIntervalSeconds", () => "5"), + new IniEntry("Worker", "WorkHeartbeatSeconds", () => "2"), + new IniEntry("Worker", "WorkLeaseSeconds", () => "60"), + new IniEntry("Worker", "WorkMaxStalls", () => "3"), + new IniEntry("Worker", "WorkItemRetentionDays", () => "30"), + new IniEntry("Worker", "WorkItemRetentionIntervalHours", () => "24"), new IniEntry("Worker", "ThumbnailShortEdge", () => "300"), new IniEntry("Worker", "ThumbnailJpegQuality", () => "75"), new IniEntry("Worker", "PreviewLongEdge", () => "1920"), diff --git a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs index b2773691..0bf94712 100644 --- a/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs +++ b/src/Anichron.Worker/Infrastructure/WorkerServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Anichron.Core; using Anichron.Core.Data; using Anichron.Core.Data.Repository; +using Anichron.Core.Work; using Anichron.Infrastructure.Configuration; using Anichron.Worker.Crawling; using Anichron.Worker.Ingestion; @@ -11,6 +12,7 @@ using Anichron.Worker.Maintenance; using Anichron.Worker.Settings; using Anichron.Worker.Startup; +using Anichron.Worker.Work; using Microsoft.EntityFrameworkCore; using NodaTime; using System.IO.Abstractions; @@ -38,7 +40,7 @@ internal IServiceCollection AddWorkerCoreServices() internal IServiceCollection AddWorkerDataServices(IConfiguration configuration) { var connectionString = DatabaseConfiguration.GetConnectionString(configuration, new FileSystem()); - services.AddDbContext(options => + services.AddDbContextFactory(options => options.UseNpgsql(connectionString, o => o.UseNodaTime())); services.AddScoped(); services.AddScoped(); @@ -46,9 +48,29 @@ internal IServiceCollection AddWorkerDataServices(IConfiguration configuration) services.AddScoped(); services.AddScoped(); services.AddScoped(sp => sp.GetRequiredService()); + services.AddScoped(); + services.AddSingleton(); return services; } + internal IServiceCollection AddWorkQueueServices() + { + services.AddSingleton(); + services.AddWorkHandler(); + services.AddWorkHandler(); + services.AddSingleton(); + services.AddSingleton(); + return services; + } + + private void AddWorkHandler() + where TPayload : IWorkPayload + where THandler : class, IWorkHandler + { + services.AddScoped, THandler>(); + services.AddSingleton(WorkHandlerRegistration.For()); + } + internal IServiceCollection AddIngestionServices() { services.AddSingleton(); @@ -97,7 +119,8 @@ internal IServiceCollection AddWorkerHostedServices() { services.AddHostedService(); services.AddHostedService(); - services.AddHostedService(); + services.AddHostedService(); + services.AddHostedService(); services.AddHostedService(); return services; } diff --git a/src/Anichron.Worker/Maintenance/TokenCleanup.cs b/src/Anichron.Worker/Maintenance/TokenCleanup.cs new file mode 100644 index 00000000..7fed8824 --- /dev/null +++ b/src/Anichron.Worker/Maintenance/TokenCleanup.cs @@ -0,0 +1,32 @@ +using Anichron.Core.Data.Repository; +using Anichron.Core.Work; +using Anichron.Worker.Settings; +using Anichron.Worker.Work; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Maintenance; + +public sealed record TokenCleanupResult(int Deleted); + +public sealed partial class TokenCleanupHandler( + IRefreshTokenRepository repository, + IClock clock, + ILogger logger) : IWorkHandler +{ + public async Task RunAsync(TokenCleanupPayload payload, IWorkProgress progress, CancellationToken ct) + { + var deleted = await repository.DeleteExpiredAsync(clock.GetCurrentInstant(), ct); + Log.TokensDeleted(logger, deleted); + return new TokenCleanupResult(deleted); + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Information, Message = "Deleted {Count} expired refresh token(s).")] + public static partial void TokensDeleted(ILogger logger, int count); + } +} + +public sealed class TokenCleanupSchedule(IOptions options) + : WorkSchedule(new TokenCleanupPayload(), Duration.FromHours(options.Value.TokenCleanupIntervalHours)); diff --git a/src/Anichron.Worker/Maintenance/TokenCleanupService.cs b/src/Anichron.Worker/Maintenance/TokenCleanupService.cs deleted file mode 100644 index 31f51443..00000000 --- a/src/Anichron.Worker/Maintenance/TokenCleanupService.cs +++ /dev/null @@ -1,51 +0,0 @@ -using Anichron.Core.Data.Repository; -using Anichron.Worker.Scheduling; -using Anichron.Worker.Settings; -using Microsoft.Extensions.Options; -using NodaTime; - -namespace Anichron.Worker.Maintenance; - -public sealed partial class TokenCleanupService( - IServiceScopeFactory scopeFactory, - IClock clock, - TimeProvider timeProvider, - IOptions options, - ILogger logger) : BackgroundService -{ - private readonly WorkerSettings settings = options.Value; - - protected override Task ExecuteAsync(CancellationToken stoppingToken) - => RecurringWork.RunAsync( - CleanUpOrLogAsync, TimeSpan.FromHours(settings.TokenCleanupIntervalHours), timeProvider, stoppingToken); - - private async Task CleanUpOrLogAsync(CancellationToken ct) - { - try - { - await RunCleanupAsync(ct); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - Log.CleanupFailed(logger, ex); - } - } - - internal async Task RunCleanupAsync(CancellationToken ct) - { - using var scope = scopeFactory.CreateScope(); - var repository = scope.ServiceProvider.GetRequiredService(); - var now = clock.GetCurrentInstant(); - var deleted = await repository.DeleteExpiredAsync(now, ct); - Log.TokensDeleted(logger, deleted); - } - - private static partial class Log - { - [LoggerMessage(Level = LogLevel.Information, Message = "Deleted {Count} expired refresh token(s).")] - public static partial void TokensDeleted(ILogger logger, int count); - - [LoggerMessage(Level = LogLevel.Error, Message = "Token cleanup failed; will retry on next tick.")] - public static partial void CleanupFailed(ILogger logger, Exception ex); - } -} diff --git a/src/Anichron.Worker/Maintenance/WorkItemRetention.cs b/src/Anichron.Worker/Maintenance/WorkItemRetention.cs new file mode 100644 index 00000000..dd4fef40 --- /dev/null +++ b/src/Anichron.Worker/Maintenance/WorkItemRetention.cs @@ -0,0 +1,34 @@ +using Anichron.Core.Work; +using Anichron.Worker.Settings; +using Anichron.Worker.Work; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Maintenance; + +public sealed record WorkItemRetentionResult(int Deleted, Instant FinishedBefore); + +public sealed partial class WorkItemRetentionHandler( + IWorkItemStore store, + IClock clock, + IOptions options, + ILogger logger) : IWorkHandler +{ + public async Task RunAsync(WorkItemRetentionPayload payload, IWorkProgress progress, CancellationToken ct) + { + var cutoff = clock.GetCurrentInstant() - Duration.FromDays(options.Value.WorkItemRetentionDays); + var deleted = await store.DeleteFinishedBeforeAsync(cutoff, ct); + Log.ItemsDeleted(logger, deleted); + return new WorkItemRetentionResult(deleted, cutoff); + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Information, Message = "Deleted {Count} finished work item(s) past retention.")] + public static partial void ItemsDeleted(ILogger logger, int count); + } +} + +public sealed class WorkItemRetentionSchedule(IOptions options) + : WorkSchedule( + new WorkItemRetentionPayload(), Duration.FromHours(options.Value.WorkItemRetentionIntervalHours)); diff --git a/src/Anichron.Worker/Program.cs b/src/Anichron.Worker/Program.cs index afb8346f..d546dd85 100644 --- a/src/Anichron.Worker/Program.cs +++ b/src/Anichron.Worker/Program.cs @@ -7,6 +7,7 @@ .AddWorkerCoreServices() .AddWorkerDataServices(builder.Configuration) .AddIngestionServices() + .AddWorkQueueServices() .AddWorkerHostedServices(); await builder.Build().RunAsync(); diff --git a/src/Anichron.Worker/Settings/WorkerSettings.cs b/src/Anichron.Worker/Settings/WorkerSettings.cs index 61743a34..b12e5bf0 100644 --- a/src/Anichron.Worker/Settings/WorkerSettings.cs +++ b/src/Anichron.Worker/Settings/WorkerSettings.cs @@ -22,6 +22,24 @@ public sealed record WorkerSettings : IValidatableObject [Range(0.0, double.MaxValue, MinimumIsExclusive = true)] public double TokenCleanupIntervalHours { get; init; } = 24; + [Range(0.0, double.MaxValue, MinimumIsExclusive = true)] + public double WorkPollIntervalSeconds { get; init; } = 5; + + [Range(0.0, double.MaxValue, MinimumIsExclusive = true)] + public double WorkHeartbeatSeconds { get; init; } = 2; + + [Range(0.0, double.MaxValue, MinimumIsExclusive = true)] + public double WorkLeaseSeconds { get; init; } = 60; + + [Range(1, int.MaxValue)] + public int WorkMaxStalls { get; init; } = 3; + + [Range(0.0, double.MaxValue, MinimumIsExclusive = true)] + public double WorkItemRetentionDays { get; init; } = 30; + + [Range(0.0, double.MaxValue, MinimumIsExclusive = true)] + public double WorkItemRetentionIntervalHours { get; init; } = 24; + [Range(1, int.MaxValue)] public int ThumbnailShortEdge { get; init; } = 300; @@ -50,9 +68,16 @@ public IReadOnlyList ExcludedDirectoryNames() => ConfigurationList.Parse(ExcludedDirectories, static name => name); public IEnumerable Validate(ValidationContext validationContext) - => ExcludedDirectoryNames() + { + var paths = ExcludedDirectoryNames() .Where(name => name.IndexOfAny(['/', '\\']) >= 0) .Select(name => new ValidationResult( $"Worker:ExcludedDirectories takes folder names, not paths: '{name}'.", [nameof(ExcludedDirectories)])); + return WorkLeaseSeconds > WorkHeartbeatSeconds + ? paths + : paths.Append(new ValidationResult( + "Worker:WorkLeaseSeconds must be longer than Worker:WorkHeartbeatSeconds, or every lease expires between heartbeats.", + [nameof(WorkLeaseSeconds)])); + } } diff --git a/src/Anichron.Worker/Work/EfWorkItemStore.cs b/src/Anichron.Worker/Work/EfWorkItemStore.cs new file mode 100644 index 00000000..99cc929f --- /dev/null +++ b/src/Anichron.Worker/Work/EfWorkItemStore.cs @@ -0,0 +1,191 @@ +using Anichron.Core.Data; +using Anichron.Core.Domain; +using Anichron.Core.Work; +using Microsoft.EntityFrameworkCore; +using NodaTime; +using Npgsql; + +namespace Anichron.Worker.Work; + +public sealed class EfWorkItemStore(IDbContextFactory contexts) : IWorkItemStore +{ + internal const string StalledError = "Stalled: the lease expired without a heartbeat too many times."; + internal const string SupersededError = "Superseded by the queued item with the same dedup key."; + + private sealed record ClaimRow(Guid Id, string Kind, string Payload, int Attempt); + + // Stryker disable all + public async Task ClaimAsync(string claimedBy, Instant now, Duration lease, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + var leaseUntil = now + lease; + var kinds = Enum.GetNames(); + var rows = await db.Database.SqlQuery($""" + UPDATE "WorkItems" + SET "Status" = 'Running', "Attempt" = "Attempt" + 1, "ClaimedBy" = {claimedBy}, + "StartedAt" = {now}, "LeaseUntil" = {leaseUntil}, "UpdatedAt" = {now}, + "ProgressDone" = 0, "ProgressTotal" = NULL, "ProgressNote" = NULL + WHERE "Id" = ( + SELECT "Id" FROM "WorkItems" + WHERE "Status" = 'Queued' AND "Kind" = ANY({kinds}) + ORDER BY "RequestedAt", "Id" + LIMIT 1 + FOR UPDATE SKIP LOCKED) + RETURNING "Id", "Kind", "Payload", "Attempt" + """).ToListAsync(ct); + return rows is [var row] + ? new ClaimedWork(new WorkAttempt(row.Id, row.Attempt), Enum.Parse(row.Kind), row.Payload) + : null; + } + + // Stryker disable all + public async Task HeartbeatAsync( + WorkAttempt attempt, WorkProgress progress, Instant now, Duration lease, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + var rows = await Running(db, attempt).ExecuteUpdateAsync(s => s + .SetProperty(w => w.LeaseUntil, now + lease) + .SetProperty(w => w.ProgressDone, progress.Done) + .SetProperty(w => w.ProgressTotal, progress.Total) + .SetProperty(w => w.ProgressNote, progress.Note) + .SetProperty(w => w.UpdatedAt, now), ct); + return rows == 1; + } + + // Stryker disable all + public async Task SucceedAsync( + WorkAttempt attempt, WorkProgress progress, string? result, Instant now, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + await Running(db, attempt).ExecuteUpdateAsync(s => s + .SetProperty(w => w.Status, WorkItemStatus.Succeeded) + .SetProperty(w => w.ProgressDone, progress.Done) + .SetProperty(w => w.ProgressTotal, progress.Total) + .SetProperty(w => w.ProgressNote, progress.Note) + .SetProperty(w => w.Result, result) + .SetProperty(w => w.FinishedAt, now) + .SetProperty(w => w.UpdatedAt, now), ct); + } + + // Stryker disable all + public async Task FailAsync( + WorkAttempt attempt, WorkProgress progress, string error, Instant now, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + await Running(db, attempt).ExecuteUpdateAsync(s => s + .SetProperty(w => w.Status, WorkItemStatus.Failed) + .SetProperty(w => w.ProgressDone, progress.Done) + .SetProperty(w => w.ProgressTotal, progress.Total) + .SetProperty(w => w.ProgressNote, progress.Note) + .SetProperty(w => w.Error, error) + .SetProperty(w => w.FinishedAt, now) + .SetProperty(w => w.UpdatedAt, now), ct); + } + + // Stryker disable all + public async Task ReleaseAsync(WorkAttempt attempt, Instant now, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + await ReturnToQueueAsync(db, attempt, expiredBefore: null, stall: 0, now, ct); + } + + // Stryker disable all + public async Task RecoverExpiredLeasesAsync(Instant now, int maxStalls, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + await Expired(db, now).Where(w => w.Stalls + 1 >= maxStalls).ExecuteUpdateAsync(s => s + .SetProperty(w => w.Status, WorkItemStatus.Failed) + .SetProperty(w => w.Stalls, w => w.Stalls + 1) + .SetProperty(w => w.Error, StalledError) + .SetProperty(w => w.FinishedAt, now) + .SetProperty(w => w.UpdatedAt, now), ct); + var expired = await Expired(db, now).Select(w => new WorkAttempt(w.Id, w.Attempt)).ToListAsync(ct); + foreach (var attempt in expired) + await ReturnToQueueAsync(db, attempt, expiredBefore: now, stall: 1, now, ct); + } + + // Stryker disable all + private static async Task ReturnToQueueAsync( + AnichronDbContext db, WorkAttempt attempt, Instant? expiredBefore, int stall, Instant now, CancellationToken ct) + { + var running = Running(db, attempt); + if (expiredBefore is { } before) + running = running.Where(w => w.LeaseUntil < before); + + for (var tries = 0; tries < 3; tries++) + { + if (await TryReturnToQueueOnceAsync(running, db, stall, now, ct)) + return; + } + + throw new InvalidOperationException( + $"Work item {attempt.Id} kept colliding with a queued item while returning to the queue."); + } + + // Stryker disable all + private static async Task TryReturnToQueueOnceAsync( + IQueryable running, AnichronDbContext db, int stall, Instant now, CancellationToken ct) + { + try + { + var requeued = await running + .Where(w => !db.WorkItems.Any(q => + q.Status == WorkItemStatus.Queued && q.DedupKey != null && q.DedupKey == w.DedupKey)) + .ExecuteUpdateAsync(s => s + .SetProperty(w => w.Status, WorkItemStatus.Queued) + .SetProperty(w => w.Stalls, w => w.Stalls + stall) + .SetProperty(w => w.ClaimedBy, (string?)null) + .SetProperty(w => w.StartedAt, (Instant?)null) + .SetProperty(w => w.LeaseUntil, (Instant?)null) + .SetProperty(w => w.UpdatedAt, now), ct); + if (requeued == 0) + { + await running.ExecuteUpdateAsync(s => s + .SetProperty(w => w.Status, WorkItemStatus.Cancelled) + .SetProperty(w => w.Stalls, w => w.Stalls + stall) + .SetProperty(w => w.Error, SupersededError) + .SetProperty(w => w.FinishedAt, now) + .SetProperty(w => w.UpdatedAt, now), ct); + } + + return true; + } + catch (Exception ex) when (IsQueuedDedupViolation(ex)) + { + return false; + } + } + + // Stryker disable all + private static bool IsQueuedDedupViolation(Exception ex) + => (ex as PostgresException ?? ex.InnerException as PostgresException) is + { + SqlState: PostgresErrorCodes.UniqueViolation, + ConstraintName: WorkItemIndexNames.QueuedDedupKeyUnique, + }; + + public async Task IsDueAsync(string dedupKey, Instant finishedBefore, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + return !await db.WorkItems.AnyAsync(w => w.DedupKey == dedupKey + && (w.Status == WorkItemStatus.Queued + || w.Status == WorkItemStatus.Running + || w.FinishedAt > finishedBefore), ct); + } + + // Stryker disable all + public async Task DeleteFinishedBeforeAsync(Instant cutoff, CancellationToken ct) + { + await using var db = await contexts.CreateDbContextAsync(ct); + return await db.WorkItems.Where(w => w.FinishedAt < cutoff).ExecuteDeleteAsync(ct); + } + + // Stryker disable all + private static IQueryable Running(AnichronDbContext db, WorkAttempt attempt) + => db.WorkItems.Where(w => + w.Id == attempt.Id && w.Attempt == attempt.Attempt && w.Status == WorkItemStatus.Running); + + // Stryker disable all + private static IQueryable Expired(AnichronDbContext db, Instant now) + => db.WorkItems.Where(w => w.Status == WorkItemStatus.Running && w.LeaseUntil < now); +} diff --git a/src/Anichron.Worker/Work/IWorkItemStore.cs b/src/Anichron.Worker/Work/IWorkItemStore.cs new file mode 100644 index 00000000..3f7fcf3c --- /dev/null +++ b/src/Anichron.Worker/Work/IWorkItemStore.cs @@ -0,0 +1,71 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; +using NodaTime; + +namespace Anichron.Worker.Work; + +public sealed record WorkAttempt(Guid Id, int Attempt); + +public sealed record ClaimedWork(WorkAttempt Attempt, WorkKind Kind, string Payload); + +/// +/// Every state change a Worker makes to a work item after it was enqueued. Writes about a running +/// item are fenced by its : once the item was recovered and claimed again, +/// the older attempt changes nothing. +/// +public interface IWorkItemStore +{ + /// The oldest queued item, now running under a lease, or null when none is queued. + /// The claiming Worker's display name. + /// The claim time. + /// How long the claim holds without a heartbeat. + /// Cancels the claim. + /// The claimed item, or null. + Task ClaimAsync(string claimedBy, Instant now, Duration lease, CancellationToken ct); + + /// Extends the lease and stores the progress snapshot. + /// The running attempt. + /// The latest snapshot. + /// The heartbeat time. + /// How long the lease holds from now. + /// Cancels the heartbeat. + /// False when the attempt no longer holds the item. + Task HeartbeatAsync(WorkAttempt attempt, WorkProgress progress, Instant now, Duration lease, CancellationToken ct); + + Task SucceedAsync(WorkAttempt attempt, WorkProgress progress, string? result, Instant now, CancellationToken ct); + + Task FailAsync(WorkAttempt attempt, WorkProgress progress, string error, Instant now, CancellationToken ct); + + /// Hands the item back to the queue without counting a stall. + /// The running attempt. + /// The release time. + /// Cancels the release. + /// A task that completes once the item is queued again. + Task ReleaseAsync(WorkAttempt attempt, Instant now, CancellationToken ct); + + /// + /// Puts running items whose lease expired back in the queue, counting a stall, and fails those + /// reaching as stalled. + /// + /// Leases ending before this have expired. + /// The stall that fails an item. + /// Cancels the recovery. + /// A task that completes once every expired lease is recovered. + Task RecoverExpiredLeasesAsync(Instant now, int maxStalls, CancellationToken ct); + + /// + /// True when no item with is queued or running and none finished + /// after . + /// + /// The periodic kind's key. + /// Now minus the kind's interval. + /// Cancels the query. + /// Whether the kind is due. + Task IsDueAsync(string dedupKey, Instant finishedBefore, CancellationToken ct); + + /// Deletes finished items that finished before . + /// Items finished before this go. + /// Cancels the delete. + /// How many items were deleted. + Task DeleteFinishedBeforeAsync(Instant cutoff, CancellationToken ct); +} diff --git a/src/Anichron.Worker/Work/WorkExecutor.cs b/src/Anichron.Worker/Work/WorkExecutor.cs new file mode 100644 index 00000000..3fb72ae0 --- /dev/null +++ b/src/Anichron.Worker/Work/WorkExecutor.cs @@ -0,0 +1,134 @@ +using Anichron.Core.Work; +using Anichron.Worker.Scheduling; +using Anichron.Worker.Settings; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Work; + +public sealed partial class WorkExecutor( + IWorkItemStore store, + WorkHandlerRegistry registry, + IServiceScopeFactory scopeFactory, + TimeProvider timeProvider, + IOptions options, + ILogger logger) : BackgroundService +{ + private readonly WorkerSettings settings = options.Value; + private readonly string claimant = ClaimantName(options.Value, Environment.MachineName); + + internal static string ClaimantName(WorkerSettings settings, string host) + => $"{(settings.User.Length == 0 ? "all" : settings.User)}@{host}"; + + protected override Task ExecuteAsync(CancellationToken stoppingToken) + => RecurringWork.RunAsync( + DrainOrLogAsync, TimeSpan.FromSeconds(settings.WorkPollIntervalSeconds), timeProvider, stoppingToken); + + private Duration Lease => Duration.FromSeconds(settings.WorkLeaseSeconds); + + private Instant Now => Instant.FromDateTimeOffset(timeProvider.GetUtcNow()); + + private async Task DrainOrLogAsync(CancellationToken stopping) + { + try + { + while (await store.ClaimAsync(claimant, Now, Lease, stopping) is { } work) + await RunAsync(work, stopping); + } + catch (Exception ex) when (!stopping.IsCancellationRequested) + { + Log.QueueFailed(logger, ex); + } + } + + private async Task RunAsync(ClaimedWork work, CancellationToken stopping) + { + using var running = CancellationTokenSource.CreateLinkedTokenSource(stopping); + using var done = new CancellationTokenSource(); + var progress = new WorkProgressSink(); + await using var scope = scopeFactory.CreateAsyncScope(); + var heartbeat = HeartbeatAsync(work.Attempt, progress, running, done.Token); + + object? result = null; + Exception? failure = null; + try + { + result = await Task.Run( + () => registry.RunAsync(work, scope.ServiceProvider, progress, running.Token), CancellationToken.None); + } + catch (Exception ex) + { + failure = ex; + } + + await done.CancelAsync(); + if (await heartbeat) + { + Log.Fenced(logger, work.Attempt.Id); + } + else if (failure is null) + { + await store.SucceedAsync( + work.Attempt, progress.Snapshot, WorkJson.SerializeResult(result), Now, CancellationToken.None); + } + else if (failure is OperationCanceledException && stopping.IsCancellationRequested) + { + await store.ReleaseAsync(work.Attempt, Now, CancellationToken.None); + Log.Released(logger, work.Attempt.Id); + stopping.ThrowIfCancellationRequested(); + } + else + { + await store.FailAsync( + work.Attempt, progress.Snapshot, $"{failure.GetType().Name}: {failure.Message}", Now, CancellationToken.None); + Log.Failed(logger, work.Attempt.Id, work.Kind, failure); + } + } + + private async Task HeartbeatAsync( + WorkAttempt attempt, WorkProgressSink progress, CancellationTokenSource running, CancellationToken done) + { + var interval = TimeSpan.FromSeconds(settings.WorkHeartbeatSeconds); + try + { + while (true) + { + await Task.Delay(interval, timeProvider, done); + try + { + if (!await store.HeartbeatAsync(attempt, progress.Snapshot, Now, Lease, CancellationToken.None)) + { + await running.CancelAsync(); + return true; + } + } + catch (Exception ex) + { + Log.HeartbeatFailed(logger, attempt.Id, ex); + } + } + } + catch (OperationCanceledException) when (done.IsCancellationRequested) + { + return false; + } + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Error, Message = "Claiming or finishing work failed; will retry at the next poll.")] + public static partial void QueueFailed(ILogger logger, Exception ex); + + [LoggerMessage(Level = LogLevel.Error, Message = "Work item {Id} ({Kind}) failed.")] + public static partial void Failed(ILogger logger, Guid id, Core.Domain.WorkKind kind, Exception ex); + + [LoggerMessage(Level = LogLevel.Information, Message = "Work item {Id} was handed back to the queue at shutdown.")] + public static partial void Released(ILogger logger, Guid id); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Work item {Id} was recovered by another worker; this attempt stopped and wrote nothing more.")] + public static partial void Fenced(ILogger logger, Guid id); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Heartbeat for work item {Id} failed; will retry.")] + public static partial void HeartbeatFailed(ILogger logger, Guid id, Exception ex); + } +} diff --git a/src/Anichron.Worker/Work/WorkHandlerRegistry.cs b/src/Anichron.Worker/Work/WorkHandlerRegistry.cs new file mode 100644 index 00000000..4942bec5 --- /dev/null +++ b/src/Anichron.Worker/Work/WorkHandlerRegistry.cs @@ -0,0 +1,76 @@ +using Anichron.Core.Domain; +using Anichron.Core.Work; +using System.Reflection; + +namespace Anichron.Worker.Work; + +public delegate Task WorkDispatch( + IServiceProvider services, string payload, IWorkProgress progress, CancellationToken ct); + +public sealed record WorkHandlerRegistration(WorkKind Kind, Type PayloadType, WorkDispatch Run) +{ + public static WorkHandlerRegistration For() + where TPayload : IWorkPayload + => new(TPayload.Kind, typeof(TPayload), (services, payload, progress, ct) + => services.GetRequiredService>() + .RunAsync(WorkJson.Deserialize(payload), progress, ct)); +} + +public sealed class WorkHandlerRegistry +{ + private readonly Dictionary handlers; + + public WorkHandlerRegistry(IEnumerable registrations) + : this(registrations, PayloadTypesIn(typeof(IWorkPayload).Assembly)) + { + } + + internal WorkHandlerRegistry(IEnumerable registrations, IEnumerable payloadTypes) + { + var registered = registrations.ToList(); + var payloads = payloadTypes.ToList(); + var problems = Enum.GetValues() + .SelectMany(kind => Problems(kind, payloads.Where(t => KindOf(t) == kind).ToList(), + registered.Where(r => r.Kind == kind).ToList())) + .ToList(); + if (problems.Count > 0) + throw new InvalidOperationException(string.Join(Environment.NewLine, problems)); + + handlers = registered.ToDictionary(r => r.Kind); + } + + public Task RunAsync( + ClaimedWork work, IServiceProvider services, IWorkProgress progress, CancellationToken ct) + => handlers[work.Kind].Run(services, work.Payload, progress, ct); + + internal static IEnumerable PayloadTypesIn(Assembly assembly) + => assembly.GetTypes().Where(t => t is { IsAbstract: false, IsInterface: false } + && t.IsAssignableTo(typeof(IWorkPayload))); + + private static IEnumerable Problems( + WorkKind kind, List payloads, List registrations) + { + if (payloads.Count != 1) + yield return $"WorkKind.{kind} needs exactly one payload type, found {Names(payloads)}."; + if (registrations.Count != 1) + yield return $"WorkKind.{kind} needs exactly one handler, found {Names(registrations.Select(r => r.PayloadType))}."; + else if (payloads is [var payload] && registrations[0].PayloadType != payload) + yield return $"WorkKind.{kind}'s handler takes {registrations[0].PayloadType.Name}, not its payload {payload.Name}."; + } + + private static string Names(IEnumerable types) + => types.Select(t => t.Name).ToList() is { Count: > 0 } names ? string.Join(", ", names) : "none"; + + private static WorkKind KindOf(Type payloadType) + => (WorkKind)typeof(PayloadKind) + .GetMethod(nameof(PayloadKind.Of))! + .MakeGenericMethod(payloadType) + .Invoke(null, null)!; + + private static class PayloadKind + { + public static WorkKind Of() + where TPayload : IWorkPayload + => TPayload.Kind; + } +} diff --git a/src/Anichron.Worker/Work/WorkProgressSink.cs b/src/Anichron.Worker/Work/WorkProgressSink.cs new file mode 100644 index 00000000..67575d3a --- /dev/null +++ b/src/Anichron.Worker/Work/WorkProgressSink.cs @@ -0,0 +1,13 @@ +using Anichron.Core.Work; + +namespace Anichron.Worker.Work; + +internal sealed class WorkProgressSink : IWorkProgress +{ + private WorkProgress snapshot = WorkProgress.None; + + public WorkProgress Snapshot => Volatile.Read(ref snapshot); + + public void Report(long done, long? total = null, string? note = null) + => Volatile.Write(ref snapshot, new WorkProgress(done, total, note)); +} diff --git a/src/Anichron.Worker/Work/WorkSchedule.cs b/src/Anichron.Worker/Work/WorkSchedule.cs new file mode 100644 index 00000000..6bebdd55 --- /dev/null +++ b/src/Anichron.Worker/Work/WorkSchedule.cs @@ -0,0 +1,24 @@ +using Anichron.Core.Work; +using NodaTime; + +namespace Anichron.Worker.Work; + +public interface IWorkSchedule +{ + string DedupKey { get; } + + Duration Interval { get; } + + Task EnqueueAsync(IWorkQueue queue, CancellationToken ct); +} + +public abstract class WorkSchedule(TPayload payload, Duration interval) : IWorkSchedule + where TPayload : IWorkPayload +{ + public string DedupKey { get; } = payload.DedupKey + ?? throw new InvalidOperationException($"A periodic {TPayload.Kind} item needs a dedup key."); + + public Duration Interval => interval; + + public Task EnqueueAsync(IWorkQueue queue, CancellationToken ct) => queue.EnqueueAsync(payload, null, ct); +} diff --git a/src/Anichron.Worker/Work/WorkScheduler.cs b/src/Anichron.Worker/Work/WorkScheduler.cs new file mode 100644 index 00000000..54053b82 --- /dev/null +++ b/src/Anichron.Worker/Work/WorkScheduler.cs @@ -0,0 +1,54 @@ +using Anichron.Core.Work; +using Anichron.Worker.Scheduling; +using Anichron.Worker.Settings; +using Microsoft.Extensions.Options; +using NodaTime; + +namespace Anichron.Worker.Work; + +public sealed partial class WorkScheduler( + IWorkItemStore store, + IEnumerable schedules, + IServiceScopeFactory scopeFactory, + TimeProvider timeProvider, + IOptions options, + ILogger logger) : BackgroundService +{ + private readonly WorkerSettings settings = options.Value; + + protected override Task ExecuteAsync(CancellationToken stoppingToken) + => RecurringWork.RunAsync( + TickOrLogAsync, TimeSpan.FromSeconds(settings.WorkPollIntervalSeconds), timeProvider, stoppingToken); + + private async Task TickOrLogAsync(CancellationToken ct) + { + try + { + await TickAsync(ct); + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + Log.TickFailed(logger, ex); + } + } + + private async Task TickAsync(CancellationToken ct) + { + var now = Instant.FromDateTimeOffset(timeProvider.GetUtcNow()); + await store.RecoverExpiredLeasesAsync(now, settings.WorkMaxStalls, ct); + foreach (var schedule in schedules) + { + if (!await store.IsDueAsync(schedule.DedupKey, now - schedule.Interval, ct)) + continue; + + await using var scope = scopeFactory.CreateAsyncScope(); + await schedule.EnqueueAsync(scope.ServiceProvider.GetRequiredService(), ct); + } + } + + private static partial class Log + { + [LoggerMessage(Level = LogLevel.Error, Message = "Scheduling work or recovering leases failed; will retry at the next poll.")] + public static partial void TickFailed(ILogger logger, Exception ex); + } +} diff --git a/src/Anichron.Worker/packages.lock.json b/src/Anichron.Worker/packages.lock.json index e395a89f..a1d51b0f 100644 --- a/src/Anichron.Worker/packages.lock.json +++ b/src/Anichron.Worker/packages.lock.json @@ -623,6 +623,7 @@ "dependencies": { "Microsoft.EntityFrameworkCore": "[10.0.13, )", "NodaTime": "[3.3.5, )", + "NodaTime.Serialization.SystemTextJson": "[1.4.0, )", "Npgsql": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL": "[10.0.3, )", "Npgsql.EntityFrameworkCore.PostgreSQL.NodaTime": "[10.0.3, )" @@ -677,6 +678,15 @@ "resolved": "3.3.5", "contentHash": "0qtGKHOYhQGCeUItP4RN8KJktsKD9j/B4IqmoecUL+ymReq69O71qEpZY6x0aQPZa5KDLXY41aEOOHZ9T6Uqgw==" }, + "NodaTime.Serialization.SystemTextJson": { + "type": "CentralTransitive", + "requested": "[1.4.0, )", + "resolved": "1.4.0", + "contentHash": "jGe0WLNAoRb3efOgsa5iecmMuzP+LNdtw2Rs4N9+2pS4YN+kg8e9Z48Jk47WWebcpqF81YdWT2KNInbXZXTS7w==", + "dependencies": { + "NodaTime": "[3.0.0, 4.0.0)" + } + }, "Npgsql": { "type": "CentralTransitive", "requested": "[10.0.3, )", diff --git a/src/Anichron.slnx b/src/Anichron.slnx index 12890329..bae422c4 100644 --- a/src/Anichron.slnx +++ b/src/Anichron.slnx @@ -10,6 +10,7 @@ + diff --git a/src/Directory.Build.props b/src/Directory.Build.props index b73b17ec..564bdd9e 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -34,14 +34,17 @@ $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)../sixlabors.lic')) - - True + + Unit + True + True + True 1701;1702;1591 - + IDE1006 @@ -51,7 +54,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -67,9 +70,9 @@ - + - + \ No newline at end of file diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets new file mode 100644 index 00000000..d2884d2e --- /dev/null +++ b/src/Directory.Build.targets @@ -0,0 +1,9 @@ + + + false + true + true + $(InTestSuite) + false + + diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 569115d7..1b6f5b45 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -23,6 +23,7 @@ + @@ -43,6 +44,7 @@ +