From 9ac8715166ad8ee194c26073d3971aa50c61608d Mon Sep 17 00:00:00 2001 From: BorgForge Codex Date: Sat, 5 Sep 2026 19:05:26 +0200 Subject: [PATCH 01/17] Define immutable job identity contract and fixtures (#471) --- docs/changelog.md | 5 + docs/maintainer/identity-dependencies.json | 409 ++ docs/maintainer/immutable-job-identity.md | 319 ++ docs/maintainer/release-workflow.md | 11 + tests/fixtures/immutable_job_id_v1/README.md | 90 + tests/fixtures/immutable_job_id_v1/base.json | 127 + tests/fixtures/immutable_job_id_v1/cases.json | 3701 +++++++++++++++++ tests/identity_contract_support.py | 235 ++ tests/test_immutable_job_identity_contract.py | 215 + 9 files changed, 5112 insertions(+) create mode 100644 docs/maintainer/identity-dependencies.json create mode 100644 docs/maintainer/immutable-job-identity.md create mode 100644 tests/fixtures/immutable_job_id_v1/README.md create mode 100644 tests/fixtures/immutable_job_id_v1/base.json create mode 100644 tests/fixtures/immutable_job_id_v1/cases.json create mode 100644 tests/identity_contract_support.py create mode 100644 tests/test_immutable_job_identity_contract.py diff --git a/docs/changelog.md b/docs/changelog.md index cd42415a..e348cd9c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,6 +6,11 @@ Das Plugin-Manifest `borg-backup-ui.plg` enthaelt nur noch eine kurze nutzerrele ## Unreleased +### Issue #471 (integration work for #447; not released) +- Defined the immutable job identity, migration safety and legacy-data contract. +- Added a source dependency inventory and synthetic migration fixtures with + reusable integrity assertions. No runtime migration is enabled in this phase. + ### Issue #458 - Clarified in the Job Wizard and both manuals that Borg retention values count time periods rather than archives per period. - Added the effective retention policy to the flow preview and made Repository Maintenance describe the maximum restore points per period. diff --git a/docs/maintainer/identity-dependencies.json b/docs/maintainer/identity-dependencies.json new file mode 100644 index 00000000..92090b3f --- /dev/null +++ b/docs/maintainer/identity-dependencies.json @@ -0,0 +1,409 @@ +{ + "schema_version": 1, + "issue": 471, + "baseline": "fcd2117", + "scan": { + "roots": [ + "borg_backup_ui.py", + "api", + "runtime/lib", + "runtime/scripts", + "ui/js", + "plugin" + ], + "extensions": [ + ".py", + ".js", + ".sh", + ".php", + ".page" + ], + "pattern": "job_key|backup_type|type_id|jobKey|backupType|existingJobKey|source_job_keys" + }, + "groups": [ + { + "id": "migration", + "owner_issue": 472, + "also": [ + 479 + ], + "target": "Direct read-only inventory, plan, snapshot, durable mapping/journal, resume and integrity verification; pending approval blocks writers.", + "files": [ + { + "path": "api/migrations/registry.py", + "anchor": "run_startup_migrations", + "role": "read/write migration state; currently applies immediately" + }, + { + "path": "api/migrations/audit.py", + "anchor": "migration", + "role": "audit writer; mask failures without secrets" + }, + { + "path": "api/migrations/canonical_backup_conf_v1.py", + "anchor": "detect", + "role": "known configuration normalization; must not escape snapshot/approval boundary" + }, + { + "path": "api/migration_api.py", + "anchor": "backup", + "role": "snapshot/status API and keep-five cleanup; protect identity recovery snapshots" + }, + { + "path": "api/startup_state.py", + "anchor": "maintenance", + "role": "startup mode owner; pending is not success" + } + ] + }, + { + "id": "jobs", + "owner_issue": 473, + "also": [ + 474, + 475, + 478 + ], + "target": "UUID filename/payload, exact aliases, full prefix and preserved effective settings; remove mutable identity from discovery and editing.", + "files": [ + { + "path": "api/jobs_api.py", + "anchor": "discover_jobs", + "role": "metadata discovery and JobManager identity/runtime access; lazy migrations are not read-only" + }, + { + "path": "api/wizard_api.py", + "anchor": "save_job", + "role": "create/edit metadata writer" + }, + { + "path": "api/job_source_paths.py", + "anchor": "JOB_SCHEMA_VERSION", + "role": "job schema and existing source-path normalization" + }, + { + "path": "api/archive_prefix.py", + "anchor": "archive_prefix_from_backup_type", + "role": "prefix history reader; silent invalid filtering is unsuitable for migration" + }, + { + "path": "ui/js/pages/wizard.js", + "anchor": "saveWizardJob", + "role": "wizard identity/edit state and separate schedule request" + }, + { + "path": "ui/js/pages/jobs.js", + "anchor": "job_key", + "role": "actions, edit selection, last run and live state" + }, + { + "path": "ui/js/core/app-core.js", + "anchor": "job_key", + "role": "shared job actions/state joins" + }, + { + "path": "api/config_api.py", + "anchor": "backup_type", + "role": "legacy type-specific config defaults; preserve effective behavior" + } + ] + }, + { + "id": "assignments", + "owner_issue": 474, + "also": [ + 473, + 475, + 477, + 478 + ], + "target": "Use UUID for job references, preserve repository/storage identity and service schedules; repository scope is not job identity.", + "files": [ + { + "path": "api/repositories_api.py", + "anchor": "save_job_repository_transaction", + "role": "assignment/reverse-link writer, reconciliation, maintenance filters and deletion guard" + }, + { + "path": "api/storage_profiles_api.py", + "anchor": "build_storage_repo_uri", + "role": "legacy Type-ID-derived repository URI construction" + }, + { + "path": "api/storage_objects_api.py", + "anchor": "write_storage_store", + "role": "canonical storage reader/writer; preserve storage identity and paths" + }, + { + "path": "api/repository_context.py", + "anchor": "job_key", + "role": "repository resolution for a selected job" + }, + { + "path": "api/check_api.py", + "anchor": "_job_retention", + "role": "repository check/prune/compact execution, job selection and archive-prefix filtering" + }, + { + "path": "api/inventory_store.py", + "anchor": "write", + "role": "shared atomic inventory writer; maintain gate/transaction boundary" + }, + { + "path": "api/schedule_api.py", + "anchor": "write_schedules", + "role": "UUID schedule map, managed cron, service entries; disable preflight orphan cleanup" + }, + { + "path": "ui/js/pages/storage.js", + "anchor": "job_key", + "role": "assignment/maintenance display and actions" + } + ] + }, + { + "id": "runtime", + "owner_issue": 475, + "also": [ + 474, + 477, + 479 + ], + "target": "Full job ID plus independent run ID for ownership; preserve readable snapshots/logs, safe recovery and prefix-union retention.", + "files": [ + { + "path": "api/wizard_runner.py", + "anchor": "job_key", + "role": "subprocess environment, paths, archive name, cache selection and maintenance" + }, + { + "path": "api/job_control.py", + "anchor": "run_id", + "role": "cancellation state and owner verification" + }, + { + "path": "runtime/lib/backup_job.py", + "anchor": "BORG_UI_JOB_KEY", + "role": "backup process, lock/log/status/recovery/notification writers" + }, + { + "path": "runtime/lib/status.py", + "anchor": "backup_type", + "role": "status/test dataclasses, filename discovery and payload joins" + }, + { + "path": "runtime/lib/runtime_recovery.py", + "anchor": "backup_type", + "role": "stopped target records and explicit recovery acknowledgement" + }, + { + "path": "runtime/lib/borg_runner.py", + "anchor": "prune", + "role": "Borg archive creation and combined-prefix retention; no job identity inference" + }, + { + "path": "runtime/lib/docker_manager.py", + "anchor": "logger", + "role": "descriptive Docker actions; ensure legacy type removal does not change restart behavior" + }, + { + "path": "runtime/lib/notifications.py", + "anchor": "backup_type", + "role": "notification event/message builders" + }, + { + "path": "runtime/lib/notification_events.py", + "anchor": "job_key", + "role": "queue, retry delivery history, dedup state and detached writer" + }, + { + "path": "api/smb_mount.py", + "anchor": "job_key", + "role": "job-configured mount/runtime ownership" + }, + { + "path": "ui/js/components/log-viewer.js", + "anchor": "log", + "role": "readable log UI; full-ID lookup supplied by callers" + } + ] + }, + { + "id": "reporting", + "owner_issue": 476, + "also": [ + 475, + 477, + 479 + ], + "target": "Configured IDs drive active counts; historical unassigned/deleted records remain distinct; preserve weekly conflicts and reminder dedup.", + "files": [ + { + "path": "api/status_api.py", + "anchor": "_auto_write_weekly_snapshot", + "role": "dashboard aggregation and both live weekly stores; GET path currently writes" + }, + { + "path": "api/history_api.py", + "anchor": "backup_type", + "role": "status filename parser and history selection" + }, + { + "path": "api/reports_api.py", + "anchor": "job_key", + "role": "report discovery and time series joins" + }, + { + "path": "api/report_mail_api.py", + "anchor": "job_key", + "role": "scheduled reports/labels and snapshot joins" + }, + { + "path": "api/notification_reminder_api.py", + "anchor": "job_key", + "role": "overdue schedule/status/test joins and reminder keys" + }, + { + "path": "api/homepage_widget_api.py", + "anchor": "job_key", + "role": "external widget aggregation" + }, + { + "path": "api/unraid_dashboard_widget.py", + "anchor": "backup_type", + "role": "Unraid cache writer and active job counters" + }, + { + "path": "plugin/widget-status.php", + "anchor": "cache", + "role": "read-only external widget cache endpoint; gate cache refresh elsewhere" + }, + { + "path": "plugin/borg-backup-ui-dashboard.page", + "anchor": "widget", + "role": "Unraid widget integration consumes canonical aggregate" + }, + { + "path": "ui/js/pages/dashboard.js", + "anchor": "backup_type", + "role": "active cards, actions and overlay state" + }, + { + "path": "ui/js/pages/history.js", + "anchor": "job_key", + "role": "history labels/filtering without filename-derived identity" + }, + { + "path": "ui/js/pages/reports.js", + "anchor": "jobKey", + "role": "report selectors and continuity" + } + ] + }, + { + "id": "restore", + "owner_issue": 477, + "also": [ + 474, + 475, + 476 + ], + "target": "Select full ID, current repository plus prefix history; preserve restore IDs, historical scope and UUID test results.", + "files": [ + { + "path": "api/restore_api.py", + "anchor": "_persist_restore_runs", + "role": "browse/context, async run map, history index/detail and locks" + }, + { + "path": "api/restore_tests_api.py", + "anchor": "job_key", + "role": "plan/policy/result filenames and proof scope" + }, + { + "path": "runtime/scripts/borg_restore_test.py", + "anchor": "backup_type", + "role": "standalone result/log writer, invocation and notification identity" + }, + { + "path": "api/archive_browser.py", + "anchor": "archive", + "role": "repository-owned read-only browse; do not restrict to configured jobs" + }, + { + "path": "ui/js/pages/restore.js", + "anchor": "job_key", + "role": "restore selection, execution and history" + }, + { + "path": "ui/js/pages/restore-tests.js", + "anchor": "job_key", + "role": "plan/manual test selection and result display" + } + ] + }, + { + "id": "transfer", + "owner_issue": 478, + "also": [ + 473, + 474, + 476, + 479 + ], + "target": "Versioned partial bundle remap, safe ID deletion, diagnostics without secrets; existing settings restore is not full configuration recovery.", + "files": [ + { + "path": "api/settings_transfer_api.py", + "anchor": "bbui-job-bundle-v2", + "role": "export/import collision rewrite and backup.conf restore" + }, + { + "path": "api/factory_reset_api.py", + "anchor": "job_key", + "role": "explicit destructive scope and store enumeration" + }, + { + "path": "api/system_health_api.py", + "anchor": "job_key", + "role": "integrity and ownership diagnostics" + }, + { + "path": "api/support_bundle_api.py", + "anchor": "status", + "role": "sanitized bundle inclusion and referential diagnostics" + }, + { + "path": "api/borg_key_store.py", + "anchor": "remove_repository_key", + "role": "repository-owned secrets; references persist even with old key-like names" + }, + { + "path": "ui/js/pages/settings.js", + "anchor": "job_key", + "role": "partial transfer, selected jobs, settings restore and diagnostics" + } + ] + }, + { + "id": "activation", + "owner_issue": 479, + "also": [ + 472, + 474, + 475, + 476, + 477, + 478 + ], + "target": "Last phase only: register cutover, gated HTTP/startup/worker lifecycle and end-to-end tests before candidate build.", + "files": [ + { + "path": "borg_backup_ui.py", + "anchor": "_evaluate_startup_migrations", + "role": "HTTP job parameters/routes, schedule cleanup, migration gate, all background writers" + } + ] + } + ] +} diff --git a/docs/maintainer/immutable-job-identity.md b/docs/maintainer/immutable-job-identity.md new file mode 100644 index 00000000..2599f840 --- /dev/null +++ b/docs/maintainer/immutable-job-identity.md @@ -0,0 +1,319 @@ +# Immutable job identity contract + +Issue #447, phase 1/9 (#471). Baseline: `origin/main` at `fcd2117` +(stable package 2026.09.05.1319). This is the target contract, **not a claim +that the current application or a migration already implements it**. + +## Delivery and precedence + +Work stays on `codex/issue-447-immutable-job-ids` in one draft PR. Phases +#471-#478 receive local automated tests; they are not independently merged, +installed, or published. #479 integrates and verifies the complete cutover +before the first test-channel candidate. Main remains available for hotfixes; +integrate relevant main changes before final verification. Never promote an +intermediate phase as an installable upgrade. + +This contract consolidates the #447 body, its migration-safety and +production-analysis comments, and the later prefix/retention clarification. +The following resolutions replace contradictory earlier suggestions: + +| Topic | Binding resolution | Owner | +| --- | --- | --- | +| Current prefix | Derive the legacy runner's actual `-backup` first, then retain valid existing prefixes. Do not blindly adopt an old list's first entry. | #472, #473 | +| Prefixes and prune | Apply one retention policy to the union of a job's prefixes in its current repository. Do not prune each prefix independently. | #474, #475 | +| Installation/startup | Detect and enter maintenance; require a verified snapshot and explicit administrator confirmation before conversion. Installation is not consent. | #472, #479 | +| Atomic migration | Crash-consistent journal and resumable per-file replacements across filesystems, not a fictional single cross-filesystem atomic rename. | #472, #479 | +| Aliases and reverse links | Keep the body's bounded `legacy_job_keys` and UUID repository reverse references. Suggestions to remove these stores were not approved scope changes. | #473, #474 | +| Status/log filenames | Readability is secondary; new records need collision-safe run identity, not just second-resolution time plus eight UUID characters. Existing logs are immutable. | #475 | +| Restore-test proof | Preserve the tested archive and its scope; retaining a result does not mean a new prefix or repository has been tested. | #477 | +| Full configuration restore | The current product restores `backup.conf` and imports partial job bundles; it has no complete installation restore feature. Do not invent one here. | #478 | +| Archived files/manifests | Inventory actual owned stores. Do not recursively migrate arbitrary `.status` files or implement hypothetical source manifests. | #472, #478 | + +Related issues #452 (repository-change confirmation), #459 (retention +options), #414 and #470 (future source features) remain separate. None may +reintroduce mutable job identity. An implementation that cannot satisfy this +contract must document the conflict before changing it. + +## C1. Canonical identity and metadata + +- `schema_version` is the integer `4` for canonical job metadata. Other + stores have independently versioned schemas; do not globally set them to 4. +- `job_id` is a canonical lowercase, hyphenated, RFC 4122-variant UUIDv4. + Generate it once using the platform UUID generator, not from a name, + prefix, timestamp, location, repository, or hash of personal information. +- The metadata filename is exactly `.json`. Full IDs must be unique + across the inventory; a valid UUID string with a mismatching filename is + not a valid canonical job. UUIDs are identifiers, not authorization tokens. +- New metadata has no active `job_key`, `backup_type`, `type_id`, or + `location`. Historical records may retain those values as descriptors. +- `name` is editable; `repository_key` selects one current repository; + `archive_prefixes` holds complete readable prefixes. Retain every other + operational setting, including fields the wizard does not currently expose. +- Before removing Type ID, materialize any effective Docker/VM/config + defaults still derived from it. Evaluate the existing metadata and runtime + precedence, not a guess based on names such as `appdata` or `vms`. +- `legacy_job_keys` is an ordered, duplicate-free list of exact legacy + identifiers evidenced by the migration. Each alias has at most one owner. + It is not a new foreign key, and edits never append new aliases. New or + duplicated/imported-as-new jobs start with an empty alias list. + +Example (operational fields omitted only in this illustration): + +```json +{ + "schema_version": 4, + "job_id": "11111111-1111-4111-8111-111111111111", + "name": "Synthetic documents", + "repository_key": "repo_docs", + "archive_prefixes": ["documents-backup", "config-backup"], + "legacy_job_keys": ["documents_local"] +} +``` + +All job-dependent APIs, scheduling, runtime, joins, caches and UI selections +use the full ID. A bounded legacy API adapter may resolve an exact known +alias centrally and report deprecation; no permanent dual writes. Never +interpret an arbitrary unknown key as a request to create a job. Reject +conflicting `job_id` and legacy-key arguments. Before release, document the +adapter's supported endpoints and removal boundary; an undocumented fallback +in individual readers is not an acceptable compatibility policy. + +## C2. Names, archive prefixes and repositories + +For an eligible legacy job, normalize prefixes as follows: + +1. Determine the current prefix from the actual legacy runner input: + `-backup`. +2. Validate all stored prefixes. A missing or empty list is a supported + legacy state. A wrong type, empty element, unsafe element or non-string + element blocks planning rather than being silently discarded. +3. Emit the derived current prefix followed by valid old prefixes in their + original order, removing exact duplicates. Do not infer aliases from + prefix history. + +For the new model, store the complete prefix entered by the user. Use the +existing safe ASCII character family `[A-Za-z0-9_.-]+`, with no path +separators, whitespace, wildcard or Borg `::` selector; `.` and `..` are not +usable prefixes. Do not require or append `-backup`. New archives use +`-` and the wizard previews that exact name. +Changing the prefix moves the new value to the front, retaining previous +values without duplicates. This is a naming change, not a new job. + +One job has one current repository. Multiple jobs may share a repository. +Do not enforce repository uniqueness, split repositories, move archives, or +change repository IDs. Preserve the repository deletion guard: any assigned +job prevents deletion. Convert `used_by`/`source_job_keys` to +`job_ids`/`source_job_ids` consistent with canonical job assignments, using +the inventory's existing assignment semantics. Never quietly repair an +unexplained conflicting active assignment. + +Archive discovery and retention use current plus historical prefixes, **only +in the current repository**. Retention operates on their combined archive +set as one job, preserving keep-N semantics. Reject ambiguous/overlapping +ownership of matching archive names between jobs in a shared repository +before destructive maintenance. Do not just compare current prefixes for +equality; historical and delimiter-prefix overlaps matter too. Ambiguity is +not permission to prune another job's archives. #459 owns policy expansion; +the identity migration must not alter existing retention values. + +Changing repositories keeps `job_id`; old archives remain in the old +repository and are not reachable through this job's Browse & Restore. +Repository-level read-only browsing (#464) is a separate, repository-owned +view and must still work for archives without a configured job. No repository +history is added to jobs. + +## C3. Evidence, legacy history and continuity + +Build a validated exact legacy-key-to-ID map before rewriting references. +Cross-check canonical filename, payload key, backup type and location. +Underscores inside a type are legitimate; do not split at the first +underscore. Explicit aliases require unique ownership and traceable evidence. +Names, similar prefixes and the mere existence of an archive are not identity +evidence. An operator-approved repair mapping, if supported later, must be +explicitly recorded and reviewed; do not generate one automatically. + +The reported `config` -> `pfsense` failure is a blocking fixture: +`pfsense_local.json` exists, but `schedules.json` still addresses +`config_local`. The current wizard first saves metadata/repository changes, +then updates schedules. `write_schedules` validates the entire old map, so +the orphan key can raise `Unknown job key: config_local`. Existing cron may +also still contain the old key. Prefix history alone cannot safely repair it. +Block the active reference before conversion; preserve old statuses/reports +as unassigned unless an authoritative mapping exists. Do not run automatic +orphan-schedule cleanup during preflight. + +| Record | Target treatment | +| --- | --- | +| Schedule, active restore, pending job notification, unresolved runtime recovery | Resolve exactly or block; disabled schedules are still configuration, not disposable history. | +| Unambiguously mapped active top-level `.status` | Enrich payload in place with schema/ID; preserve timestamps, outcome, archive, descriptors, filename and `log_file`. | +| Orphan/ambiguous historical status, delivery or finished restore | Retain original data and source provenance as explicitly unassigned; do not create a configured job or include it in active counters. | +| Unknown/malformed owned input | Block with a sanitized path/reason. Do not silently treat a failed read as an empty store. | +| Logs, unowned archive directories, recycle bin, cache contents | Preserve bytes and paths; not recursive migration inputs. | + +Old records lack some snapshots and usually lack `run_id`. Keep missing +values unknown; do not fabricate historical repository/name from the current +job, invent an old run ID, or rewrite log text. New runs carry full `job_id`, +independent `run_id`, and name/prefix/repository/location snapshots. New +status/log/control names must be collision-safe even for simultaneous jobs +with equal names/prefixes and equal timestamp seconds. Short UUID display +suffixes may be added, but never identify or join records. + +Dashboard and widgets start from configured IDs, overlaying runtime, +schedule, status and restore proof. Exactly one active row per job; unassigned +history cannot create ghost jobs. Reports/History join by ID and can display +run snapshots. Keep clearly separated access to deleted-job and unassigned +history. A name edit must not split reports or reset the last-run state. + +Weekly snapshots have two known live locations: configured `SNAPSHOT_FILE` +(default: parent of `STATUS_DIR` / `weekly-snapshots.json`) and legacy +`STATUS_DIR/weekly-snapshots.json`. Inventory both even if the current reader +ignores one. Deduplicate equal key/week/value observations with provenance; +retain both conflicting values with an explicit conflict marker instead of +choosing max, min or newest. Do not turn uncertainty into reported growth. + +Restore results use `.test` and retain the tested archive/prefix and +available original descriptors. A rename preserves proof; a prefix change +does not prove an archive under the new prefix. After a repository change, +old proof must not attest the current repository. New results replace the +per-job result atomically. Keep independent `restore_id` values for restore +execution/history. Restore index and detail must refer to the same job ID. + +Preserve notification queue/delivery IDs, attempts, retry times and reminder +deduplication state. Do not send again merely because a key changed. Explicit +system events/schedules such as the existing `restore_test` service entry +are not jobs and must not receive fabricated job IDs. Retain historical +orphan deliveries, but do not silently dispatch orphan pending job events. + +## C4. Migration eligibility and execution boundary + +Migration ID: `immutable_job_id_v1`. + +Planning classification and execution state are different: + +- `not_applicable`: genuinely empty installation, or a completely valid + already-converted inventory with no active mutable references. Preserve + existing IDs. Old unassigned historical data alone does not trigger replay. +- `applicable`: supported input with a fully consistent proposed mapping. + This is **not** permission to apply; execution remains `pending` until the + snapshot, quiescence and administrator gate have passed. +- `blocked`: unknown schema, corrupt owned data, ambiguous active reference, + duplicate ID/alias, unsafe ownership/path or unavailable required input. + +An interrupted attempt with a valid journal is a resume candidate, not a +fresh allocation. A partially converted installation without its journal is +blocked. Reuse every persisted allocated ID; never generate a second identity +after interruption. A future schema is not an empty/legacy installation. + +Execution uses `pending`, `applied`, `skipped`, `failed`, `blocked` or +`not_applicable`, with migration ID, timestamps, affected objects/actions, +source fingerprints and masked error details in the journal/audit. Follow +the central runner contract (`detect` mapping with boolean `required`, +`apply` mapping with validated status). The current runner cannot yet safely +represent a pending confirmation: phases #472/#479 must extend the gate and +tests. `pending` or `blocked` must never be treated as normal startup success. +After a failed migration, later migrations stay blocked without detection or +application. A known existing migration must not mutate input before the +appropriate snapshot/approval boundary either. + +Before any user-data rewrite: + +1. Enter maintenance and quiesce **all** writers: HTTP writes, scheduler, + backup/restore/test workers, detached backup and notification processes, + background cleanup, cache refresh and widgets. Do not kill an active job + just to make migration proceed. Show why it blocks. +2. Read owned stores directly without invoking lazy migration, discovery, + schedule pruning, snapshot auto-write or repair helpers. Capture the + exact planned file set and fingerprints. Validate configured roots, + mounts, symlinks, permissions and space without executing Borg. +3. Persist the ID map/plan durably. Create and verify an exact-file snapshot + of affected configuration/data and managed cron state. The snapshot is + not a recursive copy of a backup share or repository. Re-read and verify + bytes/checksums, not just existence or a successful copy return code. +4. Offer a protected download/export and explain that an independent copy is + required. Require explicit administrator acknowledgement bound to this + verified plan/snapshot. A click cannot verify that an external copy exists. + Do not expose plaintext secrets in a public/downloadable diagnostic bundle. +5. Revalidate all preconditions immediately before apply. Source changes + invalidate the plan/confirmation. After partial application, allow only + journaled old/new fingerprints; an unexplained external edit blocks resume. + +Stage replacements on each destination filesystem, flush, atomically rename +per file and verify again. Journal enough to resume after every boundary. +Keep the app in maintenance until referential-integrity verification passes. +Rebuild only managed cron entries at the final commit boundary; retain +unrelated cron content. Failure must not start writers against half-converted +data. Pending Docker/VM recovery with no live owner remains actionable and +must keep its exact stopped targets; migration must not mark them recovered. + +Snapshots stay protected until explicit administrator deletion; the existing +keep-five cleanup must not silently remove this recovery snapshot. Snapshot +restoration is not a plugin downgrade: Unraid supplies the installed/current +package. Document repair with that or a corrected version and auditable manual +data restoration. Do not promise an automatic package rollback. + +This validates states observed on each installation, not a claim that one +production copy represents every user. Unknown states block before rewrites. +Extend synthetic fixtures when new supported states are discovered. + +## C5. Owned storage and excluded data + +Resolve roots from configuration, not production paths in a migration script. +The machine-readable inventory in +[`identity-dependencies.json`](identity-dependencies.json) assigns source +readers/writers and boundary checks to phases #472-#479. + +| Store | Ownership and cutover | +| --- | --- | +| Job metadata under the canonical configured jobs directory | Exact immediate JSON files; validate before filename/payload conversion. Explicit known legacy locations need their own audited detection, not arbitrary recursion. | +| `config/repositories.json`, storage profiles | Convert job reverse references; preserve repository/storage identity, paths, encryption and secret references. | +| `config/schedules.json`, managed crontab section | Convert job references, preserve cron/enabled; service entries stay service entries. | +| Runtime control/locks/recovery | Quiesce live owners first; convert persistent ownership safely, never guess from a prefix. | +| Immediate `STATUS_DIR/*.status` | Classify from validated payload/evidence; no recursive glob. | +| Both known weekly snapshot locations | Read both; preserve conflicting/unassigned observations. | +| Actual configured restore-test directory, immediate `*.test` | Convert known owners to UUID filenames without guessing filename splits. | +| `config/restore-runs.json`, `config/restore-history/index.json`, `runs/.json` | Preserve restore execution IDs and links across all three stores. | +| `config/notification-queue.json`, `notification-deliveries.json`, `notification-state.json` | Convert job correlation/deduplication without dropping queue or retry metadata. | +| Widget caches | Derived data; rebuild only after cutover using canonical IDs, under the writer gate. | +| Existing referenced secrets and Borg caches | Validate ownership/reference existence without reading secret contents into diagnostics; do not rename a secret merely because its basename contains a job key. Preserve existing cache contents; any new namespace must avoid old ownership collisions. | +| Logs, `.Recycle.Bin`, runtime/vendor, plugin packages, nested status archives without a reader | Excluded from conversion. Preserve paths/bytes; do not traverse them as migration inputs. | + +The exact paths, source hashes, permissions and excluded boundaries belong in +the plan. No `rglob('*.json')` / `rglob('*.status')` over a data share. An +unknown record inside an owned store blocks; an unrelated file outside the +allowlist stays untouched. Neither rule justifies deleting unknown data. + +## C6. Lifecycle, transfer and deletion + +| Operation | Identity rule | +| --- | --- | +| Create or duplicate | New UUID; no imported aliases. Check prefix ownership in the selected repository separately. | +| Edit name, prefix, sources, schedule or repository | Retain UUID and all unrelated settings; no identity change. | +| Import as new | New destination UUID, even when the bundle contains one. Maintain a single import mapping for all selected dependent references. | +| Explicit update of selected target job | Retain target UUID; source UUID is provenance, not replacement identity. | +| Name/prefix/repository collision | Do not silently merge; explicit action/remap or reject. | +| Existing settings restore | Restore only its supported scope (`backup.conf`); do not pretend it restores jobs/history/secrets as a consistent installation. | +| Future complete-configuration restore | Preserve IDs only after full integrity validation; collisions require explicit remapping. Conditional contract, not a new #447 feature. | +| Delete job | Remove only the selected ID's active references and explicitly authorized artifacts. Retained history keeps former ID/descriptors; never delete by broad type/prefix patterns. | +| Factory reset | Preserve existing confirmation boundaries; enumerate any newly added owned stores. | + +The existing `bbui-job-bundle-v2` transfer is a partial bundle, not disaster +recovery. Phase #478 must define its versioned ID-based successor and reject +inconsistent references before writes. Preserve supported legacy import +through a bounded conversion, not a second canonical identity system. + +## Verification assets and remaining gates + +[`tests/fixtures/immutable_job_id_v1/README.md`](../../tests/fixtures/immutable_job_id_v1/README.md) +describes the synthetic fixtures, deterministic test UUID allocation and +normalized observation format. Phase #471 tests fixture schema, privacy, +expected graph integrity and negative mutations of reusable assertions. +These are **not migration execution tests**. #472 must run its real detector +and planner against these inputs, and #479 must test actual on-disk results, +interruption/resume, no-write blocking and UI/HTTP maintenance behavior. + +Before the first candidate: cover each journal write/rename boundary, stale +source fingerprints, unavailable mounts, disk/permission failures, live +workers, same-second runs, renamed jobs, partial imports, rejected corrupt +inputs and repeated startup. Verify snapshots independently, old archives +and log bytes unchanged, no wrong-job joins, and no background writes during +maintenance. A passing fixture schema test alone cannot authorize release. diff --git a/docs/maintainer/release-workflow.md b/docs/maintainer/release-workflow.md index b82f688a..7eadf1ff 100644 --- a/docs/maintainer/release-workflow.md +++ b/docs/maintainer/release-workflow.md @@ -22,6 +22,17 @@ package and, after explicit approval, to a stable release. Internal changes with `release-note::no` do not need a fragment. +### Approved integration exception: immutable job IDs (#447) + +Phases #471-#478 use the shared `codex/issue-447-immutable-job-ids` branch +and one long-lived draft PR. Run focused tests per phase; do not publish +incomplete intermediate migrations or create separate phase release PRs. +The final source preflight and first test-channel candidate are deferred to +#479, after the migration and all dependent workflows are testable together. +This does not relax the final preflight, explicit user test approval, or +separate stable promotion requirements below. Main can receive unrelated +hotfix PRs while this integration remains in draft. + Do not run `plugin/build.sh` directly. It is an internal builder that accepts only an exported and prepared source tree created by the deployment workflow. diff --git a/tests/fixtures/immutable_job_id_v1/README.md b/tests/fixtures/immutable_job_id_v1/README.md new file mode 100644 index 00000000..bcf5a8b0 --- /dev/null +++ b/tests/fixtures/immutable_job_id_v1/README.md @@ -0,0 +1,90 @@ +# Immutable identity fixtures (#471 / #447) + +These are synthetic contract examples, not anonymized copies of a complete +production installation. They capture structures and failure modes discussed +in #447. They contain no repository data, credentials or real user paths. +`backup.example.invalid`, `/fixture/` and the fixed UUIDs are test values only. + +## Format + +- `base.json` contains configuration, directories and file contents for one + legacy schema-3 job and its immediate dependencies. +- `cases.json` overlays whole files on that base: `{ "json": ... }` serializes + JSON; `{ "text": ... }` preserves exact text, including malformed JSON. + `null` removes a base file. There is no implicit recursive object merge. +- `allocation_order` injects deterministic UUIDv4 values into future planner + tests. Production must generate random UUIDs once and persist the mapping. +- `preconditions` describes controlled test hooks, not a production journal + format. In particular the journal case models a durable mapping and one + completed replacement, **not** a real verified snapshot or real journal + bytes. #472 must supply its real journal and filesystem failure adapters. +- `/fixture/` is relocated by the test materializer to an isolated directory + under repository-local `.release-tmp/`. No Borg commands, network calls, + crontab updates, actual process checks or production reads are performed. + +## Expected observations + +`expected` defines a normalized test observation, not the API response or +every dependent store's future wire format: + +- `classification`: `applicable`, `blocked` or `not_applicable`. +- `execution`: supported apply/resume **after confirmation**, pending without + user-data writes, or no user-data writes. Applicability is not approval. +- `reason_codes`: exact machine-readable fixture reason vocabulary; real + planner adapters must explicitly map any differently named diagnostics. +- `jobs`: complete expected canonical metadata keyed by the injected full ID. + Assert `.json` filenames separately with `assert_canonical_job_files`. +- `bindings`: original `file#JSON-pointer` evidence -> resulting job ID or + `null`. Values identify source records/references, not destination filenames. + Extract actual destination IDs in the adapter; never just echo this map. +- `unassigned`: every null binding has an explicit retention/classification + reason. No implicit deletion and no invented active job. +- `preserved`: original provenance -> exact value that must survive in the + destination (possibly enriched with additional fields). For an entire + historical object, compare all original members; new schema/ID fields may + coexist. Preserve both weekly observations even when values conflict. +- `unchanged_files`: byte-identical files at their original paths after apply. + Blocked/pending/not-applicable cases additionally require **all** user-data + paths and bytes unchanged. Migration's own dedicated audit/plan/snapshot + files must be checked separately, not silently excluded by a broad filter. + +Bindings also cover service-versus-job distinctions, disabled schedules, +restore result/index/detail continuity, notifications, runtime recovery and +both live weekly stores. Explicit complete prefixes no longer require a +`-backup` suffix. The old runner's suffix is derived only during conversion. + +## Test coverage and limitations + +`tests/identity_contract_support.py` supplies reusable assertions for UUIDs, +canonical metadata files, aliases, referential integrity, unassigned history, +preservation and no-write outcomes. Negative tests deliberately corrupt these +observations to ensure the assertions fail. + +In phase #471, tests validate the fixture data, source inventory and the +assertions themselves. The materialization self-check passes an expected +observation to the assertion only to validate the oracle; **it is not evidence +of an implemented or successful migration**. + +Phase #472 must execute the actual read-only detector/planner on each input, +inject the UUID allocator and model real source fingerprints/journal actions. +Phase #479 must execute the real migration and read back destination files, +test every interruption boundary, retry/resume, missing mounts/space, +permissions/symlinks, all live writers and the administrator gate. Extend +fixtures with concrete bytes and OS hooks as those implementations are added. +Do not replace a regression golden merely to make a wrong migration pass. + +The source dependency checklist is in +[`docs/maintainer/identity-dependencies.json`](../../../docs/maintainer/identity-dependencies.json). +Its test detects new files matching known mutable-key spellings and validates +reviewed source anchors. This is a regression guard, not a proof that a text +search can find every semantic dependency. Update anchors and ownership +deliberately as phases replace legacy code. Replace the phase-1 +"not registered" assertion only when #479 intentionally enables the migration. + +Run focused validation from the repository root: + +```bash +python -m pytest -q tests/test_immutable_job_identity_contract.py +``` + +No test-channel package or stable release is created in this phase. diff --git a/tests/fixtures/immutable_job_id_v1/base.json b/tests/fixtures/immutable_job_id_v1/base.json new file mode 100644 index 00000000..3e0902ff --- /dev/null +++ b/tests/fixtures/immutable_job_id_v1/base.json @@ -0,0 +1,127 @@ +{ + "schema_version": 1, + "provenance": "Entirely synthetic; structural cases transcribed from #447 discussion, not copied production files.", + "config": { + "BACKUP_SCRIPTS_DIR": "/fixture/data", + "STATUS_DIR": "/fixture/status", + "SNAPSHOT_FILE": "/fixture/weekly-snapshots.json", + "BORG_RESOURCE_LOCK_DIR": "/fixture/locks", + "RESTORE_TEST_STATUS_DIR": "/fixture/restore_tests" + }, + "directories": [ + "sources/config", + "sources/config/temporary", + "repositories/repo_docs", + "sources/photos" + ], + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "used_by": [ + "config_local" + ], + "source_job_keys": [ + "config_local" + ] + } + ] + } + }, + "data/config/storages.json": { + "json": { + "schema_version": 1, + "storages": [ + { + "storage_key": "storage_local", + "display_name": "Synthetic local", + "storage_type": "local", + "location": "local", + "identity": "local:/fixture/repositories", + "base_path": "/fixture/repositories" + } + ] + } + }, + "data/config/schedules.json": { + "json": { + "config_local": { + "cron": "0 8 * * *", + "enabled": true + } + } + }, + "status/2026-08-31_08-00-00_config_local.status": { + "json": { + "backup_type": "config", + "location": "local", + "timestamp": "2026-08-31 08:00:00", + "duration_seconds": 60, + "exit_code": 0, + "status": "success", + "archive_name": "config-backup-2026-08-31_08-00-00", + "log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "repository_size": 100, + "archive_original_size": 30 + } + }, + "logs/Borg-Backup_config--2026-08-31_08-00-00.log": { + "text": "Synthetic backup log; config_local; keep these bytes.\n" + } + } +} diff --git a/tests/fixtures/immutable_job_id_v1/cases.json b/tests/fixtures/immutable_job_id_v1/cases.json new file mode 100644 index 00000000..8458abca --- /dev/null +++ b/tests/fixtures/immutable_job_id_v1/cases.json @@ -0,0 +1,3701 @@ +{ + "schema_version": 1, + "allocation_order": [ + "11111111-1111-4111-8111-111111111111", + "22222222-2222-4222-8222-222222222222" + ], + "cases": [ + { + "id": "fresh", + "description": "No jobs, schedules or historical job data; no allocation or conversion.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/repositories.json": null, + "data/config/storages.json": null, + "data/config/schedules.json": null, + "status/2026-08-31_08-00-00_config_local.status": null, + "logs/Borg-Backup_config--2026-08-31_08-00-00.log": null + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "not_applicable", + "execution": "no_user_data_writes", + "reason_codes": [], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "legacy_without_prefixes", + "description": "Untouched legacy job derives the runner's current complete prefix.", + "files": {}, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "empty_prefix_list", + "description": "Empty list is supported; derive one current prefix.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": [], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "current_prefix_precedes_stored_history", + "description": "Stored first prefix is not necessarily the one the old runner creates.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": [ + "old-backup", + "config-backup", + "old-backup" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup", + "old-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "renamed_display_name", + "description": "Only label changed; legacy identity and run descriptors still resolve.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic renamed job", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic renamed job", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "reported_config_to_pfsense_orphan_schedule", + "description": "Reported support failure: stale config_local schedule is not proven to belong to pfsense.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/pfsense_local.json": { + "json": { + "schema_version": 3, + "job_key": "pfsense_local", + "backup_type": "pfsense", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": [ + "pfsense-backup", + "config-backup" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "used_by": [ + "pfsense_local" + ], + "source_job_keys": [ + "pfsense_local" + ] + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "orphan_active_schedule" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "renamed_prefix_unassigned_old_history", + "description": "No stale active schedule; prefix history alone cannot assign config_local status to pfsense.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/pfsense_local.json": { + "json": { + "schema_version": 3, + "job_key": "pfsense_local", + "backup_type": "pfsense", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": [ + "pfsense-backup", + "config-backup" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/schedules.json": { + "json": { + "pfsense_local": { + "cron": "0 8 * * *", + "enabled": true + } + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "used_by": [ + "pfsense_local" + ], + "source_job_keys": [ + "pfsense_local" + ] + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": [ + "pfsense-backup", + "config-backup" + ], + "job_id": "11111111-1111-4111-8111-111111111111", + "legacy_job_keys": [ + "pfsense_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "status/2026-08-31_08-00-00_config_local.status#": null, + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "data/config/schedules.json#/pfsense_local": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": { + "status/2026-08-31_08-00-00_config_local.status#": "no_authoritative_alias" + }, + "preserved": { + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "different_storage_types", + "description": "Two independent jobs across local and synthetic remote storage.", + "files": { + "data/config/jobs/photos_storagebox.json": { + "json": { + "schema_version": 3, + "job_key": "photos_storagebox", + "backup_type": "photos", + "location": "storagebox", + "name": "Synthetic photos", + "repository_key": "repo_remote", + "source_paths": [ + "/fixture/sources/photos" + ], + "exclude_paths": [], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "used_by": [ + "config_local" + ], + "source_job_keys": [ + "config_local" + ] + }, + { + "repository_key": "repo_remote", + "display_name": "Synthetic repository", + "storage_key": "storage_remote", + "relative_path": "repo_remote", + "encryption": "none", + "initialized": true, + "used_by": [ + "photos_storagebox" + ], + "source_job_keys": [ + "photos_storagebox" + ] + } + ] + } + }, + "data/config/storages.json": { + "json": { + "schema_version": 1, + "storages": [ + { + "storage_key": "storage_local", + "display_name": "Synthetic local", + "storage_type": "local", + "location": "local", + "identity": "local:/fixture/repositories", + "base_path": "/fixture/repositories" + }, + { + "storage_key": "storage_remote", + "display_name": "Synthetic remote", + "storage_type": "storagebox", + "location": "storagebox", + "host": "backup.example.invalid", + "user": "fixture-user", + "port": 23, + "base_path": "./borg" + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + }, + "22222222-2222-4222-8222-222222222222": { + "schema_version": 4, + "name": "Synthetic photos", + "repository_key": "repo_remote", + "source_paths": [ + "/fixture/sources/photos" + ], + "exclude_paths": [], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "22222222-2222-4222-8222-222222222222", + "archive_prefixes": [ + "photos-backup" + ], + "legacy_job_keys": [ + "photos_storagebox" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/1/used_by/0": "22222222-2222-4222-8222-222222222222", + "data/config/repositories.json#/repositories/1/source_job_keys/0": "22222222-2222-4222-8222-222222222222" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "shared_repository_distinct_prefixes", + "description": "Sharing a repository is valid when archive ownership is distinct.", + "files": { + "data/config/jobs/photos_local.json": { + "json": { + "schema_version": 3, + "job_key": "photos_local", + "backup_type": "photos", + "location": "local", + "name": "Synthetic photos", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/photos" + ], + "exclude_paths": [], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "used_by": [ + "config_local", + "photos_local" + ], + "source_job_keys": [ + "config_local", + "photos_local" + ] + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + }, + "22222222-2222-4222-8222-222222222222": { + "schema_version": 4, + "name": "Synthetic photos", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/photos" + ], + "exclude_paths": [], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "22222222-2222-4222-8222-222222222222", + "archive_prefixes": [ + "photos-backup" + ], + "legacy_job_keys": [ + "photos_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/1": "22222222-2222-4222-8222-222222222222", + "data/config/repositories.json#/repositories/0/source_job_keys/1": "22222222-2222-4222-8222-222222222222" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "overlapping_shared_repository_prefixes", + "description": "Another job claims config-backup through its historical prefix list.", + "files": { + "data/config/jobs/photos_local.json": { + "json": { + "schema_version": 3, + "job_key": "photos_local", + "backup_type": "photos", + "location": "local", + "name": "Synthetic photos", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/photos" + ], + "exclude_paths": [], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": [ + "photos-backup", + "config-backup" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "used_by": [ + "config_local", + "photos_local" + ], + "source_job_keys": [ + "config_local", + "photos_local" + ] + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "ambiguous_archive_ownership" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "underscores_in_type", + "description": "Exact payload/key evidence avoids the first-underscore filename parser bug.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/config_home_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_home_local", + "backup_type": "config_home", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/schedules.json": { + "json": { + "config_home_local": { + "cron": "0 8 * * *", + "enabled": true + } + } + }, + "status/2026-08-31_08-00-00_config_local.status": null, + "status/2026-08-31_08-00-00_config_home_local.status": { + "json": { + "backup_type": "config_home", + "location": "local", + "timestamp": "2026-08-31 08:00:00", + "duration_seconds": 60, + "exit_code": 0, + "status": "success", + "archive_name": "config_home-backup-2026-08-31_08-00-00", + "log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "repository_size": 100, + "archive_original_size": 30 + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "used_by": [ + "config_home_local" + ], + "source_job_keys": [ + "config_home_local" + ] + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config_home-backup" + ], + "legacy_job_keys": [ + "config_home_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_home_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_home_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "disabled_schedule_and_reminder", + "description": "Disabled cron and due-marker deduplication must survive unchanged.", + "files": { + "data/config/schedules.json": { + "json": { + "config_local": { + "cron": "0 8 * * *", + "enabled": false + }, + "restore_test": { + "cron": "0 9 * * 0", + "enabled": true + } + } + }, + "data/config/notification-state.json": { + "json": { + "schema_version": 1, + "last_sent": { + "backup_overdue:config_local:2026-08-31T08:00:00": 1788163200 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "data/config/notification-state.json#/last_sent/backup_overdue:config_local:2026-08-31T08:00:00": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": false + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00", + "data/config/schedules.json#/restore_test": { + "cron": "0 9 * * 0", + "enabled": true + }, + "data/config/notification-state.json#/last_sent/backup_overdue:config_local:2026-08-31T08:00:00": 1788163200 + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "restore_result_and_history", + "description": "Preserve test evidence and restore_id; canonical test filename uses full UUID.", + "files": { + "restore_tests/config_local.test": { + "json": { + "test_date": "2026-08-31 08:10:00", + "test_result": "PASS", + "test_level": 1, + "tested_archive": "config-backup-2026-08-31_08-00-00", + "tested_entries": 1 + } + }, + "data/config/restore-history/index.json": { + "json": { + "schema_version": 1, + "runs": [ + { + "restore_id": "20260831-081500-aaaaaaaa", + "state": "done", + "phase": "done", + "job_key": "config_local", + "archive": "config-backup-2026-08-31_08-00-00", + "started_at": "2026-08-31T08:15:00", + "finished_at": "2026-08-31T08:15:30", + "source_path": "source.txt", + "target_dir": "/fixture/restore-output", + "destination_path": "/fixture/restore-output/source.txt", + "error": "" + } + ] + } + }, + "data/config/restore-history/runs/20260831-081500-aaaaaaaa.json": { + "json": { + "schema_version": 1, + "source": "worker", + "restore_id": "20260831-081500-aaaaaaaa", + "state": "done", + "phase": "done", + "job_key": "config_local", + "archive": "config-backup-2026-08-31_08-00-00", + "started_at": "2026-08-31T08:15:00", + "finished_at": "2026-08-31T08:15:30", + "source_path": "source.txt", + "target_dir": "/fixture/restore-output", + "destination_path": "/fixture/restore-output/source.txt", + "error": "", + "lines": [ + "Synthetic restore completed." + ] + } + }, + "data/config/restore-runs.json": { + "json": { + "schema_version": 1, + "runs": {} + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "restore_tests/config_local.test#": "11111111-1111-4111-8111-111111111111", + "data/config/restore-history/index.json#/runs/0": "11111111-1111-4111-8111-111111111111", + "data/config/restore-history/runs/20260831-081500-aaaaaaaa.json#": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00", + "restore_tests/config_local.test#": { + "test_date": "2026-08-31 08:10:00", + "test_result": "PASS", + "test_level": 1, + "tested_archive": "config-backup-2026-08-31_08-00-00", + "tested_entries": 1 + }, + "data/config/restore-history/index.json#/runs/0/restore_id": "20260831-081500-aaaaaaaa" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "orphan_history_and_excluded_archives", + "description": "Deleted-job history is retained unassigned; nested/recycled copies are not live stores.", + "files": { + "status/2026-08-30_08-00-00_deleted_local.status": { + "json": { + "backup_type": "deleted", + "location": "local", + "timestamp": "2026-08-31 08:00:00", + "duration_seconds": 60, + "exit_code": 0, + "status": "success", + "archive_name": "deleted-backup-2026-08-30_08-00-00", + "log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "repository_size": 100, + "archive_original_size": 30 + } + }, + "status/archive/old.status": { + "text": "Historical opaque bytes.\n" + }, + "data/.Recycle.Bin/config/jobs/old.json": { + "text": "{invalid but not an input\n" + }, + "cache/config_local/files": { + "text": "Synthetic opaque cache bytes.\n" + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "status/2026-08-30_08-00-00_deleted_local.status#": null + }, + "unassigned": { + "status/2026-08-30_08-00-00_deleted_local.status#": "no_configured_job" + }, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/archive/old.status", + "data/.Recycle.Bin/config/jobs/old.json", + "cache/config_local/files" + ] + } + }, + { + "id": "corrupt_job_json", + "description": "Invalid owned job input must block before data conversion.", + "files": { + "data/config/jobs/config_local.json": { + "text": "{\"schema_version\":3," + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "invalid_json" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "non_object_job", + "description": "Invalid owned job input must block before data conversion.", + "files": { + "data/config/jobs/config_local.json": { + "json": [] + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "invalid_job_shape" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "future_schema", + "description": "Invalid owned job input must block before data conversion.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 99, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "unsupported_schema" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "invalid_prefix_element", + "description": "Invalid owned job input must block before data conversion.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": [ + "config-backup", + "../other-backup" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "invalid_archive_prefix" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "prefix_list_wrong_type", + "description": "Invalid owned job input must block before data conversion.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "archive_prefixes": "config-backup", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "invalid_archive_prefix" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "mismatched_legacy_key", + "description": "Invalid owned job input must block before data conversion.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "other_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "conflicting_legacy_identity" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "missing_repository", + "description": "Invalid owned job input must block before data conversion.", + "files": { + "data/config/jobs/config_local.json": { + "json": { + "schema_version": 3, + "job_key": "config_local", + "backup_type": "config", + "location": "local", + "name": "Synthetic configuration", + "repository_key": "repo_missing", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "dangling_repository" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "ambiguous_aliases", + "description": "Two canonical jobs claim the same exact legacy alias.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/11111111-1111-4111-8111-111111111111.json": { + "json": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/jobs/22222222-2222-4222-8222-222222222222.json": { + "json": { + "schema_version": 4, + "name": "Synthetic photos", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/photos" + ], + "exclude_paths": [], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "22222222-2222-4222-8222-222222222222", + "archive_prefixes": [ + "photos-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "duplicate_legacy_alias" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "duplicate_uuid", + "description": "Two files contain the same UUID; do not silently choose one.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/11111111-1111-4111-8111-111111111111.json": { + "json": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/jobs/22222222-2222-4222-8222-222222222222.json": { + "json": { + "schema_version": 4, + "name": "Synthetic duplicate", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "duplicate_job_id" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "already_migrated", + "description": "Validated canonical installation keeps IDs without reapplying conversion.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/11111111-1111-4111-8111-111111111111.json": { + "json": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "data/config/schedules.json": { + "json": { + "11111111-1111-4111-8111-111111111111": { + "cron": "0 8 * * *", + "enabled": true + } + } + }, + "data/config/repositories.json": { + "json": { + "schema_version": 1, + "repositories": [ + { + "repository_key": "repo_docs", + "display_name": "Synthetic repository", + "storage_key": "storage_local", + "relative_path": "repo_docs", + "encryption": "none", + "initialized": true, + "job_ids": [ + "11111111-1111-4111-8111-111111111111" + ], + "source_job_ids": [ + "11111111-1111-4111-8111-111111111111" + ] + } + ] + } + }, + "status/2026-08-31_08-00-00_config_local.status": { + "json": { + "backup_type": "config", + "location": "local", + "timestamp": "2026-08-31 08:00:00", + "duration_seconds": 60, + "exit_code": 0, + "status": "success", + "archive_name": "config-backup-2026-08-31_08-00-00", + "log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "repository_size": 100, + "archive_original_size": 30, + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111" + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "not_applicable", + "execution": "no_user_data_writes", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/11111111-1111-4111-8111-111111111111": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/job_ids/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_ids/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "queued_notifications", + "description": "Convert job correlation without resetting attempts, retries or event IDs.", + "files": { + "data/config/notification-queue.json": { + "json": { + "schema_version": 1, + "queue": [ + { + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "created_at": "2026-08-31T08:01:00Z", + "updated_at": "2026-08-31T08:02:00Z", + "next_attempt_at": 1788163380, + "attempts_made": 1, + "max_attempts": 3, + "backoff_seconds": 60, + "timeout_seconds": 10, + "profile_id": "fixture-profile", + "profile_name": "Synthetic notifications", + "provider": "fixture", + "event_type": "backup_success", + "source": "backup", + "job_key": "config_local", + "severity": "normal", + "title": "Synthetic success", + "body": "Synthetic backup completed." + } + ] + } + }, + "data/config/notification-deliveries.json": { + "json": { + "schema_version": 1, + "deliveries": [ + { + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "job_key": "config_local", + "status": "retrying", + "attempts_made": 1 + }, + { + "id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "job_key": "deleted_local", + "status": "sent", + "attempts_made": 1 + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "data/config/notification-queue.json#/queue/0": "11111111-1111-4111-8111-111111111111", + "data/config/notification-deliveries.json#/deliveries/0": "11111111-1111-4111-8111-111111111111", + "data/config/notification-deliveries.json#/deliveries/1": null + }, + "unassigned": { + "data/config/notification-deliveries.json#/deliveries/1": "no_configured_job" + }, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00", + "data/config/notification-queue.json#/queue/0/id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "data/config/notification-queue.json#/queue/0/attempts_made": 1, + "data/config/notification-queue.json#/queue/0/next_attempt_at": 1788163380, + "data/config/notification-queue.json#/queue/0/body": "Synthetic backup completed." + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "orphan_pending_notification", + "description": "Pending job events cannot be silently dropped, reassigned or sent.", + "files": { + "data/config/notification-queue.json": { + "json": { + "schema_version": 1, + "queue": [ + { + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "created_at": "2026-08-31T08:01:00Z", + "updated_at": "2026-08-31T08:02:00Z", + "next_attempt_at": 1788163380, + "attempts_made": 1, + "max_attempts": 3, + "backoff_seconds": 60, + "timeout_seconds": 10, + "profile_id": "fixture-profile", + "profile_name": "Synthetic notifications", + "provider": "fixture", + "event_type": "backup_success", + "source": "backup", + "job_key": "deleted_local", + "severity": "normal", + "title": "Synthetic success", + "body": "Synthetic backup completed." + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "orphan_active_notification" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "pending_runtime_recovery", + "description": "Dead owner with known job retains exact stopped targets and unresolved recovery.", + "files": { + "data/config/runtime-recovery.json": { + "json": { + "schema_version": 1, + "entries": [ + { + "id": "fixture-recovery", + "state": "pending_restart", + "kind": "docker", + "job_name": "Synthetic configuration", + "backup_type": "config", + "backup_location": "local", + "log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "pid": 99999999, + "stopped_at": "2026-08-31T08:00:00Z", + "restarted_at": "", + "message": "Synthetic interrupted run", + "targets": [ + { + "id": "fixture-container", + "name": "Synthetic container" + } + ] + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "data/config/runtime-recovery.json#/entries/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00", + "data/config/runtime-recovery.json#/entries/0/id": "fixture-recovery", + "data/config/runtime-recovery.json#/entries/0/state": "pending_restart", + "data/config/runtime-recovery.json#/entries/0/targets": [ + { + "id": "fixture-container", + "name": "Synthetic container" + } + ], + "data/config/runtime-recovery.json#/entries/0/stopped_at": "2026-08-31T08:00:00Z" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "live_writer", + "description": "Application shutdown alone is insufficient while a detached writer is active.", + "files": {}, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": false, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "writers_not_quiescent" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "equal_weekly_observations", + "description": "Both live snapshot locations are inputs; equal observations may deduplicate with provenance.", + "files": { + "weekly-snapshots.json": { + "json": { + "config_local": [ + { + "week": "2026-08-31", + "size": 100 + } + ] + } + }, + "status/weekly-snapshots.json": { + "json": { + "config_local": [ + { + "week": "2026-08-31", + "size": 100 + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "weekly-snapshots.json#/config_local/0": "11111111-1111-4111-8111-111111111111", + "status/weekly-snapshots.json#/config_local/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00", + "weekly-snapshots.json#/config_local/0": { + "week": "2026-08-31", + "size": 100 + }, + "status/weekly-snapshots.json#/config_local/0": { + "week": "2026-08-31", + "size": 100 + } + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "conflicting_weekly_observations", + "description": "Do not select a winning value for the same job/week; preserve both with a conflict classification.", + "files": { + "weekly-snapshots.json": { + "json": { + "config_local": [ + { + "week": "2026-08-31", + "size": 100 + } + ] + } + }, + "status/weekly-snapshots.json": { + "json": { + "config_local": [ + { + "week": "2026-08-31", + "size": 120 + } + ] + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "applied_after_confirmation", + "reason_codes": [ + "weekly_value_conflict_preserved" + ], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111", + "weekly-snapshots.json#/config_local/0": "11111111-1111-4111-8111-111111111111", + "status/weekly-snapshots.json#/config_local/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00", + "weekly-snapshots.json#/config_local/0": { + "week": "2026-08-31", + "size": 100 + }, + "status/weekly-snapshots.json#/config_local/0": { + "week": "2026-08-31", + "size": 120 + } + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "interrupted_with_journal", + "description": "Reuse journaled UUID after canonical metadata replacement; old dependent references remain.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/11111111-1111-4111-8111-111111111111.json": { + "json": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": { + "migration_id": "immutable_job_id_v1", + "state": "pending", + "id_map": { + "config_local": "11111111-1111-4111-8111-111111111111" + }, + "completed_actions": [ + "write_canonical_job" + ], + "old_or_new_fingerprints_verified": true + } + }, + "expected": { + "classification": "applicable", + "execution": "resumed_after_confirmation", + "reason_codes": [ + "resume_existing_mapping" + ], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "partial_without_journal", + "description": "Canonical job with old active references and no matching journal cannot be accepted as complete.", + "files": { + "data/config/jobs/config_local.json": null, + "data/config/jobs/11111111-1111-4111-8111-111111111111.json": { + "json": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + } + }, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "partial_migration_without_journal" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "source_changed_after_plan", + "description": "Invalidate snapshot/approval when an input changes after planning.", + "files": {}, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": false, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "source_fingerprint_changed" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + }, + { + "id": "awaiting_confirmation", + "description": "Supported input remains pending; user data and all active writers stay unchanged.", + "files": {}, + "preconditions": { + "administrator_confirmed": false, + "snapshot_verified": true, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "applicable", + "execution": "pending_no_user_data_writes", + "reason_codes": [], + "jobs": { + "11111111-1111-4111-8111-111111111111": { + "schema_version": 4, + "name": "Synthetic configuration", + "repository_key": "repo_docs", + "source_paths": [ + "/fixture/sources/config" + ], + "exclude_paths": [ + "/fixture/sources/config/temporary" + ], + "compression": "lz4", + "retention": { + "daily": "7", + "weekly": "4", + "monthly": "6", + "yearly": "3" + }, + "enabled": true, + "features": { + "docker": false, + "vm": false + }, + "description": "Synthetic fixture only", + "job_id": "11111111-1111-4111-8111-111111111111", + "archive_prefixes": [ + "config-backup" + ], + "legacy_job_keys": [ + "config_local" + ], + "file_activity": true, + "docker_control": { + "mode": "none", + "selected": [], + "ack_appdata_risk": false + }, + "vm_control": { + "mode": "none", + "selected": [], + "ack_domains_risk": false + }, + "restore_test_policy": { + "mode": "scheduled", + "interval_days": 30, + "validity_days": 30, + "level": 1, + "max_runtime_minutes": 0 + } + } + }, + "bindings": { + "data/config/schedules.json#/config_local": "11111111-1111-4111-8111-111111111111", + "status/2026-08-31_08-00-00_config_local.status#": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/used_by/0": "11111111-1111-4111-8111-111111111111", + "data/config/repositories.json#/repositories/0/source_job_keys/0": "11111111-1111-4111-8111-111111111111" + }, + "unassigned": {}, + "preserved": { + "data/config/schedules.json#/config_local": { + "cron": "0 8 * * *", + "enabled": true + }, + "status/2026-08-31_08-00-00_config_local.status#/archive_name": "config-backup-2026-08-31_08-00-00", + "status/2026-08-31_08-00-00_config_local.status#/log_file": "/fixture/logs/Borg-Backup_config--2026-08-31_08-00-00.log", + "status/2026-08-31_08-00-00_config_local.status#/timestamp": "2026-08-31 08:00:00" + }, + "unchanged_files": [ + "logs/Borg-Backup_config--2026-08-31_08-00-00.log" + ] + } + }, + { + "id": "snapshot_unverified", + "description": "Administrator acknowledgement cannot bypass snapshot verification.", + "files": {}, + "preconditions": { + "administrator_confirmed": true, + "snapshot_verified": false, + "writers_quiescent": true, + "source_fingerprints_match": true, + "journal": null + }, + "expected": { + "classification": "blocked", + "execution": "no_user_data_writes", + "reason_codes": [ + "snapshot_not_verified" + ], + "jobs": {}, + "bindings": {}, + "unassigned": {}, + "preserved": {}, + "unchanged_files": [] + } + } + ] +} diff --git a/tests/identity_contract_support.py b/tests/identity_contract_support.py new file mode 100644 index 00000000..5ce5012c --- /dev/null +++ b/tests/identity_contract_support.py @@ -0,0 +1,235 @@ +"""Test-only contract oracle for #447; never imported by application code. + +There is deliberately no migration/detection implementation here. Future +phases must project their real outputs into this observation format. +""" + +from copy import deepcopy +import json +from pathlib import Path, PurePosixPath +import re +from uuid import UUID + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" / "immutable_job_id_v1" +LEGACY_FIELDS = {"job_key", "backup_type", "type_id", "location"} +OBSERVATION_FIELDS = { + "classification", "execution", "reason_codes", "jobs", "bindings", + "unassigned", "preserved", "unchanged_files", +} + + +def read_json(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def assert_uuid4(value): + assert isinstance(value, str), "job_id must be a string" + try: + parsed = UUID(value) + except (ValueError, AttributeError) as exc: + raise AssertionError("invalid job_id") from exc + assert parsed.version == 4, "job_id must be UUIDv4" + assert str(parsed) == value, "job_id must be canonical lowercase UUID" + + +def assert_safe_relative_path(value): + assert isinstance(value, str) and value, "empty fixture path" + path = PurePosixPath(value) + assert not path.is_absolute() and ".." not in path.parts, "unsafe fixture path" + assert str(path) == value and "\\" not in value, "noncanonical fixture path" + + +def load_cases(): + base = read_json(FIXTURES / "base.json") + matrix = read_json(FIXTURES / "cases.json") + assert base["schema_version"] == matrix["schema_version"] == 1 + result = [] + for raw in matrix["cases"]: + case = deepcopy(raw) + files = deepcopy(base["files"]) + for name, value in case["files"].items(): + assert_safe_relative_path(name) + if value is None: + assert name in files, "fixture deletes a nonexistent base file" + del files[name] + else: + files[name] = value + case["files"] = files + case["config"] = deepcopy(base["config"]) + case["directories"] = list(base["directories"]) + case["allocation_order"] = list(matrix["allocation_order"]) + result.append(case) + return result + + +def source_value(files, reference): + """Read file#JSON-pointer evidence, including the whole payload with '#'.""" + name, separator, pointer = reference.partition("#") + assert separator, "source reference needs a JSON pointer" + assert_safe_relative_path(name) + value = files[name]["json"] + assert not pointer or pointer.startswith("/"), "invalid JSON pointer" + for part in pointer.split("/")[1:]: + part = part.replace("~1", "/").replace("~0", "~") + value = value[int(part)] if isinstance(value, list) else value[part] + return value + + +def assert_job_graph(jobs, bindings, unassigned): + aliases = {} + for job_id, job in jobs.items(): + assert_uuid4(job_id) + assert job["job_id"] == job_id, "duplicate or mismatched job ID" + assert type(job["schema_version"]) is int and job["schema_version"] == 4 + assert not LEGACY_FIELDS.intersection(job), "active legacy identity remains" + assert isinstance(job["name"], str) and job["name"].strip() + assert isinstance(job["repository_key"], str) and job["repository_key"] + prefixes = job["archive_prefixes"] + assert isinstance(prefixes, list) and prefixes, "current prefix missing" + assert all(isinstance(p, str) and re.fullmatch(r"[A-Za-z0-9_.-]+", p) + and p not in {".", ".."} for p in prefixes), "unsafe prefix" + assert len(prefixes) == len(set(prefixes)), "duplicate prefix" + legacy = job["legacy_job_keys"] + assert isinstance(legacy, list), "aliases must be a list" + assert all(isinstance(a, str) and re.fullmatch(r"[A-Za-z0-9_.-]+", a) + for a in legacy), "invalid alias" + assert len(legacy) == len(set(legacy)), "duplicate alias in job" + for alias in legacy: + assert alias not in aliases, "duplicate alias ownership" + aliases[alias] = job_id + for source, job_id in bindings.items(): + if job_id is None: + assert source in unassigned, "unassigned reference lacks classification" + else: + assert job_id in jobs, "dangling canonical reference" + assert source not in unassigned, "assigned history marked unassigned" + assert set(unassigned) == {s for s, v in bindings.items() if v is None} + assert all(isinstance(reason, str) and reason for reason in unassigned.values()) + + +def assert_canonical_job_files(files): + """Verify actual post-migration metadata, independently of fixture goldens.""" + jobs = {} + for path, payload in files.items(): + assert_uuid4(payload.get("job_id")) + job_id = payload["job_id"] + assert Path(path).name == job_id + ".json", "noncanonical metadata filename" + assert job_id not in jobs, "duplicate job ID across files" + jobs[job_id] = payload + assert_job_graph(jobs, {}, {}) + + +def assert_lifecycle_identity(operation, original, result): + """Check output from a real create/edit/import action in later phases.""" + assert operation in {"create", "edit", "duplicate", "import_new", "import_update"} + assert_job_graph({result["job_id"]: result}, {}, {}) + if operation in {"edit", "import_update"}: + assert result["job_id"] == original["job_id"], "existing identity changed" + assert result["legacy_job_keys"] == original["legacy_job_keys"], "aliases changed on edit" + else: + assert result["legacy_job_keys"] == [], "new job inherited legacy aliases" + if original is not None: + assert result["job_id"] != original["job_id"], "new job reused source identity" + + +def assert_fixture(case): + assert re.fullmatch(r"[a-z][a-z0-9_]+", case["id"]) + assert case["description"] + for name, entry in case["files"].items(): + assert_safe_relative_path(name) + assert isinstance(entry, dict) and set(entry) in ({"json"}, {"text"}) + if "text" in entry: + assert isinstance(entry["text"], str) + for path in case["directories"]: + assert_safe_relative_path(path) + for job_id in case["allocation_order"]: + assert_uuid4(job_id) + assert len(set(case["allocation_order"])) == len(case["allocation_order"]) + expected = case["expected"] + assert set(expected) == OBSERVATION_FIELDS + assert expected["classification"] in {"not_applicable", "applicable", "blocked"} + assert expected["execution"] in { + "applied_after_confirmation", "resumed_after_confirmation", + "no_user_data_writes", "pending_no_user_data_writes", + } + assert isinstance(expected["reason_codes"], list) + assert all(isinstance(r, str) and r for r in expected["reason_codes"]) + if expected["classification"] == "blocked": + assert expected["reason_codes"] and not expected["jobs"] + assert expected["execution"] == "no_user_data_writes" + if expected["classification"] == "not_applicable": + assert expected["execution"] == "no_user_data_writes" + conditions = case["preconditions"] + for key in ("administrator_confirmed", "snapshot_verified", "writers_quiescent", + "source_fingerprints_match"): + assert type(conditions[key]) is bool + if expected["execution"] in {"applied_after_confirmation", "resumed_after_confirmation"}: + assert all(conditions[k] for k in conditions if k != "journal") + if expected["execution"] == "resumed_after_confirmation": + assert conditions["journal"]["migration_id"] == "immutable_job_id_v1" + assert set(conditions["journal"]["id_map"].values()) <= set(expected["jobs"]) + assert_job_graph(expected["jobs"], expected["bindings"], expected["unassigned"]) + for reference in expected["bindings"]: + source_value(case["files"], reference) + for reference, value in expected["preserved"].items(): + assert source_value(case["files"], reference) == value, "invalid preservation golden" + for name in expected["unchanged_files"]: + assert name in case["files"] + + +def _relocate(value, root): + if isinstance(value, str): + return value.replace("/fixture/", root.as_posix().rstrip("/") + "/") + if isinstance(value, list): + return [_relocate(item, root) for item in value] + if isinstance(value, dict): + return {key: _relocate(item, root) for key, item in value.items()} + return value + + +def materialize(case, root): + """Write synthetic inputs ONLY inside a new empty repository-local test root. + + Returns a relocated copy, not the original case. OS failure/journal hooks + in preconditions must be supplied by the future real planner test adapter. + """ + root = Path(root).resolve() + assert root.is_relative_to(ROOT), "test data must stay in this repository" + root.mkdir(parents=True, exist_ok=True) + assert not any(root.iterdir()), "fixture root must be empty" + relocated = _relocate(deepcopy(case), root) + assert_fixture(relocated) + for directory in relocated["directories"]: + (root / directory).mkdir(parents=True, exist_ok=True) + for name, entry in relocated["files"].items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + data = (json.dumps(entry["json"], indent=2, ensure_ascii=True) + "\n" + if "json" in entry else entry["text"]) + path.write_text(data, encoding="utf-8") + return relocated + + +def tree_bytes(root): + """Exact test-tree snapshot; not a production migration inventory.""" + root = Path(root) + return {p.relative_to(root).as_posix(): p.read_bytes() + for p in root.rglob("*") if p.is_file()} + + +def assert_observation(case, actual, before, after): + """Check an independently projected real planner/applier result. + + `preserved` values must be read from their destination by that adapter, + keyed by original provenance. Merely echoing the input is not verification. + """ + assert set(actual) == OBSERVATION_FIELDS + assert_job_graph(actual["jobs"], actual["bindings"], actual["unassigned"]) + assert actual == case["expected"], "migration observation differs from contract" + if actual["execution"] in {"no_user_data_writes", "pending_no_user_data_writes"}: + assert before == after, "user data changed without apply permission" + for name in actual["unchanged_files"]: + assert name in before and name in after, "preserved file missing" + assert before[name] == after[name], "excluded/log file bytes changed" diff --git a/tests/test_immutable_job_identity_contract.py b/tests/test_immutable_job_identity_contract.py new file mode 100644 index 00000000..3fef968a --- /dev/null +++ b/tests/test_immutable_job_identity_contract.py @@ -0,0 +1,215 @@ +"""#471: fixture/contract validation, NOT an implemented migration test suite.""" + +from copy import deepcopy +import json +from pathlib import Path +import re +from tempfile import TemporaryDirectory + +import pytest + +from identity_contract_support import ( + ROOT, assert_canonical_job_files, assert_fixture, assert_job_graph, assert_lifecycle_identity, + assert_observation, assert_safe_relative_path, assert_uuid4, + load_cases, materialize, read_json, tree_bytes, +) + + +CASES = load_cases() +BY_ID = {case["id"]: case for case in CASES} + + +@pytest.fixture +def identity_root(): + # Also stays repository-local under the preflight's plain `pytest -q`. + parent = ROOT / ".release-tmp" + parent.mkdir(exist_ok=True) + with TemporaryDirectory(prefix="identity-471-", dir=parent) as directory: + yield Path(directory) + + +def test_fixture_ids_and_required_classifications(): + assert len(CASES) == len(BY_ID) + assert {c["expected"]["classification"] for c in CASES} == { + "applicable", "blocked", "not_applicable", + } + assert {"fresh", "legacy_without_prefixes", "different_storage_types", + "reported_config_to_pfsense_orphan_schedule", "disabled_schedule_and_reminder", + "restore_result_and_history", "orphan_history_and_excluded_archives", + "ambiguous_aliases", "corrupt_job_json", "interrupted_with_journal"} <= BY_ID.keys() + + +@pytest.mark.parametrize("case", CASES, ids=lambda c: c["id"]) +def test_fixture_schema_and_expected_graph(case): + assert_fixture(case) + + +@pytest.mark.parametrize("case", CASES, ids=lambda c: c["id"]) +def test_fixture_materialization_is_synthetic_and_preserves_source_bytes(case, identity_root): + relocated = materialize(case, identity_root / "installation") + before = tree_bytes(identity_root / "installation") + assert set(before) == set(case["files"]) + for name, entry in relocated["files"].items(): + if "json" in entry: + assert json.loads(before[name]) == entry["json"] + else: + assert before[name].decode("utf-8") == entry["text"] + # Oracle self-check only: this is NOT a real planner/applier result. + assert_observation(relocated, deepcopy(relocated["expected"]), before, before) + + +def test_fixtures_have_no_credentials_or_production_paths(): + def inspect(value): + if isinstance(value, dict): + for key, item in value.items(): + assert key.lower() not in { + "password", "passphrase", "token", "gh_token", "private_key", + "api_key", "secret", "apprise_url", + }, "do not commit credential-bearing fixtures" + if key == "host": + assert item.endswith(".invalid") + inspect(item) + elif isinstance(value, list): + for item in value: + inspect(item) + elif isinstance(value, str): + assert not re.search(r"\b(?:gh[pousr]_|github_pat_|sk-)[A-Za-z0-9_]{12,}", value) + assert "PRIVATE KEY-----" not in value + assert not re.search(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", value) + assert not any(p in value for p in ("/home/", "/boot/", "/mnt/", "@hetzner")) + if value.startswith("/"): + assert value.startswith("/fixture/") + for case in CASES: + inspect(case) + + +@pytest.mark.parametrize("value", [ + None, "", "config_local", "11111111", "11111111111141118111111111111111", + "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA", "11111111-1111-5111-8111-111111111111", + "11111111-1111-4111-0111-111111111111", +]) +def test_noncanonical_or_non_v4_ids_rejected(value): + with pytest.raises(AssertionError): + assert_uuid4(value) + + +@pytest.mark.parametrize("path", ["/etc/file", "../file", "a/../../file", "a\\file", "a//file", ""]) +def test_fixture_paths_cannot_escape_root(path): + with pytest.raises(AssertionError): + assert_safe_relative_path(path) + + +def test_duplicate_alias_and_uuid_assertions(): + jobs = deepcopy(BY_ID["shared_repository_distinct_prefixes"]["expected"]["jobs"]) + first, second = jobs + jobs[second]["legacy_job_keys"] = jobs[first]["legacy_job_keys"] + with pytest.raises(AssertionError, match="alias ownership"): + assert_job_graph(jobs, {}, {}) + with pytest.raises(AssertionError, match="noncanonical metadata filename"): + assert_canonical_job_files({"wrong.json": jobs[first]}) + with pytest.raises(AssertionError, match="duplicate job ID"): + assert_canonical_job_files({f"one/{first}.json": jobs[first], f"two/{first}.json": jobs[first]}) + + +def test_canonical_metadata_and_full_explicit_prefix(): + jobs = deepcopy(BY_ID["legacy_without_prefixes"]["expected"]["jobs"]) + job_id = next(iter(jobs)) + jobs[job_id]["archive_prefixes"] = ["complete-user-prefix"] + assert_canonical_job_files({f"jobs/{job_id}.json": jobs[job_id]}) + jobs[job_id]["job_key"] = "should_not_survive" + with pytest.raises(AssertionError, match="active legacy identity"): + assert_job_graph(jobs, {}, {}) + + +def test_dangling_reference_and_unassigned_classification_assertions(): + expected = deepcopy(BY_ID["legacy_without_prefixes"]["expected"]) + with pytest.raises(AssertionError, match="dangling"): + assert_job_graph(expected["jobs"], {"record": "22222222-2222-4222-8222-222222222222"}, {}) + with pytest.raises(AssertionError, match="lacks classification"): + assert_job_graph(expected["jobs"], {"record": None}, {}) + with pytest.raises(AssertionError, match="assigned history marked unassigned"): + assert_job_graph(expected["jobs"], {"record": next(iter(expected["jobs"]))}, {"record": "orphan"}) + + +@pytest.mark.parametrize("operation", ["create", "duplicate", "import_new", "edit", "import_update"]) +def test_lifecycle_identity_oracle(operation): + original = deepcopy(next(iter(BY_ID["legacy_without_prefixes"]["expected"]["jobs"].values()))) + result = deepcopy(original) + result["name"] = "Synthetic renamed job" + result["archive_prefixes"] = ["explicit-new-prefix", "config-backup"] + result["repository_key"] = "repo_other" + if operation in {"create", "duplicate", "import_new"}: + result["job_id"] = "22222222-2222-4222-8222-222222222222" + result["legacy_job_keys"] = [] + assert_lifecycle_identity(operation, original, result) + if operation in {"edit", "import_update"}: + result["job_id"] = "22222222-2222-4222-8222-222222222222" + else: + result["legacy_job_keys"] = ["config_local"] + with pytest.raises(AssertionError): + assert_lifecycle_identity(operation, original, result) + + +def test_reported_rename_is_blocking_and_not_repaired_by_prefix_history(): + case = BY_ID["reported_config_to_pfsense_orphan_schedule"] + assert "config_local" in case["files"]["data/config/schedules.json"]["json"] + assert "data/config/jobs/config_local.json" not in case["files"] + assert case["files"]["data/config/jobs/pfsense_local.json"]["json"]["archive_prefixes"] == [ + "pfsense-backup", "config-backup", + ] + assert case["expected"]["reason_codes"] == ["orphan_active_schedule"] + assert case["expected"]["classification"] == "blocked" + + +def test_current_prefix_is_derived_before_deduplicated_history(): + case = BY_ID["current_prefix_precedes_stored_history"] + assert case["files"]["data/config/jobs/config_local.json"]["json"]["archive_prefixes"][0] == "old-backup" + assert next(iter(case["expected"]["jobs"].values()))["archive_prefixes"] == ["config-backup", "old-backup"] + + +def test_oracle_rejects_lost_history_and_unauthorized_writes(): + case = BY_ID["orphan_history_and_excluded_archives"] + actual = deepcopy(case["expected"]) + actual["unassigned"] = {} + with pytest.raises(AssertionError): + assert_observation(case, actual, {}, {}) + blocked = BY_ID["reported_config_to_pfsense_orphan_schedule"] + with pytest.raises(AssertionError, match="without apply permission"): + assert_observation(blocked, deepcopy(blocked["expected"]), {"job": b"old"}, {"job": b"new"}) + with pytest.raises(AssertionError, match="bytes changed"): + before = {p: b"old" for p in case["expected"]["unchanged_files"]} + after = {p: b"new" for p in before} + assert_observation(case, deepcopy(case["expected"]), before, after) + + +def test_dependency_inventory_covers_current_source_scan(): + inventory = read_json(ROOT / "docs/maintainer/identity-dependencies.json") + pattern = re.compile(inventory["scan"]["pattern"]) + extensions = set(inventory["scan"]["extensions"]) + indexed = set() + for group in inventory["groups"]: + assert group["owner_issue"] in range(472, 480) + assert all(issue in range(472, 480) for issue in group["also"]) + assert group["target"] + for entry in group["files"]: + path = ROOT / entry["path"] + assert path.is_file(), entry["path"] + assert entry["anchor"] in path.read_text(encoding="utf-8"), entry["path"] + assert entry["role"] and entry["path"] not in indexed + indexed.add(entry["path"]) + scanned = set() + for name in inventory["scan"]["roots"]: + root = ROOT / name + paths = [root] if root.is_file() else root.rglob("*") + for path in paths: + if path.suffix in extensions and path.is_file(): + if pattern.search(path.read_text(encoding="utf-8")): + scanned.add(path.relative_to(ROOT).as_posix()) + assert scanned <= indexed, f"Mutable identity dependencies need owners: {sorted(scanned - indexed)}" + + +def test_fixture_helpers_are_not_production_migration_imports(): + for name in ("api/migrations/registry.py", "borg_backup_ui.py"): + text = (ROOT / name).read_text(encoding="utf-8") + assert "identity_contract_support" not in text + assert "immutable_job_id_v1" not in text From e8fb69e0edc3b3cf0b960fb6127f3e6689083433 Mon Sep 17 00:00:00 2001 From: BorgForge Codex Date: Sat, 5 Sep 2026 19:49:22 +0200 Subject: [PATCH 02/17] Build inactive identity migration foundation (#472) --- api/migrations/identity_records.py | 570 +++++++++++++ api/migrations/identity_storage.py | 755 ++++++++++++++++++ api/migrations/immutable_job_id_v1.py | 687 ++++++++++++++++ docs/changelog.md | 8 + docs/maintainer/identity-dependencies.json | 15 + .../identity-migration-foundation.md | 209 +++++ docs/maintainer/immutable-job-identity.md | 8 + tests/fixtures/immutable_job_id_v1/README.md | 14 +- tests/test_identity_planner.py | 683 ++++++++++++++++ tests/test_identity_records.py | 474 +++++++++++ tests/test_identity_storage.py | 522 ++++++++++++ 11 files changed, 3941 insertions(+), 4 deletions(-) create mode 100644 api/migrations/identity_records.py create mode 100644 api/migrations/identity_storage.py create mode 100644 api/migrations/immutable_job_id_v1.py create mode 100644 docs/maintainer/identity-migration-foundation.md create mode 100644 tests/test_identity_planner.py create mode 100644 tests/test_identity_records.py create mode 100644 tests/test_identity_storage.py diff --git a/api/migrations/identity_records.py b/api/migrations/identity_records.py new file mode 100644 index 00000000..072f0ce8 --- /dev/null +++ b/api/migrations/identity_records.py @@ -0,0 +1,570 @@ +"""Pure dependent-record projection for the inactive #447 identity migration. + +No file, process, Borg, scheduler or application helpers are called here. The +scanner supplies owned records as ``path -> {kind, data, ...}``; ``target_path`` +may be supplied for restore tests and a shared weekly destination. Results are +plans, never permission to write. ``sources`` identifies exact consumed files. + +Existing store schema versions are retained. ``identity_schema_version = 1`` +marks new enriched collections; canonical schema-1 records are also accepted. +Weekly observations have their own version-1 envelope with source provenance. +Historical descriptors are preserved, while active ``job_key`` references move +to ``legacy_job_key``. Later phases must implement readers for these shapes. +""" + +from __future__ import annotations + +from copy import deepcopy +import json +import re +from typing import Any +from uuid import UUID + + +_TERMINAL = {"done", "error", "aborted"} +_COLLECTIONS = { + "notification_queue": ("queue", True), + "notification_deliveries": ("deliveries", False), + "runtime_recovery": ("entries", True), + "restore_index": ("runs", False), +} +_DIRECT = {"status", "restore_test", "restore_detail", "control", "cancel_request", "resource_lock"} +_STATUS_FILENAME = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}_(.+)\.status$") +_RESTORE_ID = re.compile(r"^[A-Za-z0-9._-]+$") +_RESTORE_SHARED_FIELDS = ( + "state", "archive", "started_at", "finished_at", "source_path", "target_dir", + "destination_path", "conflict_mode", "preserve_owner", "repository_key", + "repository_snapshot", "job_name_snapshot", "archive_prefix_snapshot", +) + + +def _pointer(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _uuid(value: Any) -> bool: + if not isinstance(value, str): + return False + try: + parsed = UUID(value) + return str(parsed) == value and parsed.version == 4 and parsed.variant == "specified in RFC 4122" + except ValueError: + return False + + +def _json_key(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +class _Projection: + def __init__(self, records: dict, jobs: dict, aliases: dict, *, verify: bool = False): + self.input = records + self.jobs = jobs + self.aliases = aliases + self.verify = verify + self.records: dict = {} + self.bindings: list = [] + self.unassigned: list = [] + self.reasons: list = [] + self.restore_links: dict = {} + self.required = False + + def reason(self, code: str, source: str, locator: str = "", *, severity: str = "error") -> None: + item = {"code": code, "source": source, "locator": locator, "severity": severity} + if item not in self.reasons: + self.reasons.append(item) + + def schema(self, data: Any, source: str, locator: str = "") -> bool: + if not isinstance(data, dict): + self.reason("invalid_owned_record", source, locator) + return False + for field in ("schema_version", "identity_schema_version"): + if field in data and (type(data[field]) is not int or data[field] != 1): + self.reason("unsupported_record_schema", source, locator) + return False + return True + + def resolve(self, raw: Any, source: str, locator: str, *, active: bool, + code: str = "orphan_active_reference") -> str | None: + if not isinstance(raw, str) or not raw: + self.reason("invalid_identity_reference", source, locator) + return None + if raw in self.jobs and _uuid(raw): + return raw + if raw in self.aliases and self.aliases[raw] in self.jobs: + if active: + self.required = True + if self.verify and active: + self.reason("mutable_active_reference", source, locator) + return None + return self.aliases[raw] + if active: + self.reason(code, source, locator) + return None + + def bind(self, source: str, locator: str, job_id: str | None, legacy: str, + role: str, original: Any, reason: str = "no_configured_job") -> None: + self.bindings.append({"source": source, "locator": locator, "job_id": job_id, + "legacy_key": legacy, "role": role}) + if job_id is None and role != "system": + self.unassigned.append({"source": source, "locator": locator, + "reason": reason, "data": deepcopy(original)}) + + def restore_link(self, source: str, locator: str, job_id: str | None, + legacy: str, kind: str, row: dict) -> None: + restore_id = row.get("restore_id") + if (not isinstance(restore_id, str) or not _RESTORE_ID.fullmatch(restore_id) + or restore_id in {".", ".."}): + self.reason("invalid_restore_id", source, locator) + return + if kind == "restore_detail" and source.rsplit("/", 1)[-1] != restore_id + ".json": + self.reason("restore_detail_filename_mismatch", source, locator) + self.restore_links.setdefault(restore_id, []).append( + (source, locator, job_id, legacy, kind, row)) + + def row(self, data: Any, source: str, locator: str, kind: str, + *, active: bool = False, legacy: str = "") -> Any: + if not self.schema(data, source, locator): + return deepcopy(data) + original = deepcopy(data) + row = deepcopy(data) + if kind.startswith("restore_") and kind != "restore_test": + state = row.get("state") + if not isinstance(state, str) or state not in _TERMINAL | {"running"}: + self.reason("invalid_restore_state", source, locator) + active = state not in _TERMINAL + if kind in {"restore_index", "restore_detail"} and state not in _TERMINAL: + self.reason("nonterminal_restore_history", source, locator) + if kind == "restore_runs" and state in _TERMINAL: + self.reason("terminal_restore_in_active_store", source, locator) + if kind == "runtime_recovery": + if row.get("state") not in {"pending_restart", "restart_failed"}: + self.reason("unknown_recovery_state", source, locator) + if not isinstance(row.get("targets"), list) or not row["targets"]: + self.reason("invalid_recovery_targets", source, locator) + else: + target_ids = [] + for target in row["targets"]: + if (not isinstance(target, dict) or not isinstance(target.get("id"), str) + or not target["id"] or not isinstance(target.get("name"), str) or not target["name"]): + self.reason("invalid_recovery_targets", source, locator) + else: + target_ids.append(target["id"]) + if len(set(target_ids)) != len(target_ids): + self.reason("invalid_recovery_targets", source, locator) + if kind in {"control", "cancel_request", "resource_lock"}: + active = kind != "control" or row.get("finished") is not True + if ("run_id" in row and (not isinstance(row["run_id"], str) or not row["run_id"])): + self.reason("invalid_run_id", source, locator) + if "pid" in row and (type(row["pid"]) is not int or row["pid"] < 0): + self.reason("invalid_owner_pid", source, locator) + raw_key = row.get("job_key", row.get("legacy_job_key", "")) + if raw_key and not isinstance(raw_key, str): + self.reason("invalid_identity_reference", source, locator) + return row + payload_key = "" + type_field = "backup_type" + location_field = "backup_location" if kind == "runtime_recovery" else "location" + if type_field in row or location_field in row: + backup_type, location = row.get(type_field), row.get(location_field) + if not isinstance(backup_type, str) or not backup_type or not isinstance(location, str) or not location: + self.reason("invalid_identity_descriptor", source, locator) + return row + payload_key = backup_type + "_" + location + candidates = [value for value in (raw_key, payload_key, legacy) if value] + is_system = (kind in {"notification_queue", "notification_deliveries"} + and not row.get("job_id") + and ((raw_key == "restore_test" and row.get("source") == "restore_test") + or (not raw_key and row.get("source") == "system"))) + if kind == "resource_lock" and not row.get("job_id") and raw_key == "restore_test": + is_system = row.get("operation") == "restore_test" + if is_system: + self.bind(source, locator, None, raw_key, "system", original) + return row + if not active and row.get("identity_state") == "unassigned": + self.bind(source, locator, None, raw_key or legacy, "history", original, + row.get("identity_reason", "no_configured_job")) + if kind in {"restore_runs", "restore_index", "restore_detail"}: + self.restore_link(source, locator, None, raw_key or legacy, kind, row) + return row + if kind == "status" and not payload_key and not row.get("job_id"): + self.reason("invalid_status_identity", source, locator) + return row + conflict = len(set(candidates)) > 1 + resolved = {self.aliases.get(value, value if value in self.jobs else None) for value in candidates} + if conflict and len(resolved) == 1 and None not in resolved: + conflict = False + supplied_id = row.get("job_id") + if supplied_id is not None and not _uuid(supplied_id): + self.reason("invalid_job_id", source, locator) + return row + if supplied_id and any(value is not None and value != supplied_id for value in resolved): + conflict = True + key = candidates[0] if candidates else "" + code = {"notification_queue": "orphan_active_notification", + "runtime_recovery": "orphan_runtime_recovery", + "restore_runs": "orphan_active_restore"}.get(kind, "orphan_active_reference") + if conflict: + if active: + self.reason("conflicting_active_identity", source, locator) + elif self.verify or supplied_id: + self.reason("conflicting_canonical_identity", source, locator) + job_id = None + elif supplied_id: + job_id = self.resolve(supplied_id, source, locator, active=active, code=code) + if self.verify and active and "job_key" in row: + self.reason("mutable_active_reference", source, locator) + elif key: + job_id = self.resolve(key, source, locator, active=active, code=code) + if self.verify and job_id is not None: + self.reason("missing_canonical_job_id", source, locator) + else: + if active: + self.reason(code, source, locator) + job_id = None + historical_reason = "conflicting_identity_evidence" if conflict else "no_configured_job" + if not active and supplied_id and job_id is None and not conflict: + historical_reason = "deleted_job" + elif not active and job_id is None and not conflict and payload_key: + # A former prefix can explain the diagnostic, never establish an + # alias. Keep the historical record unassigned even with one hint. + former_prefix = str(row.get("backup_type")) + "-backup" + if any(former_prefix in job.get("archive_prefixes", []) for job in self.jobs.values()): + historical_reason = "no_authoritative_alias" + self.bind(source, locator, job_id, key, "active" if active else "history", original, historical_reason) + if job_id is not None: + row["job_id"] = job_id + if not supplied_id: + row.setdefault("schema_version", 1) + if active and "job_key" in row: + if "legacy_job_key" in row and row["legacy_job_key"] != row["job_key"]: + self.reason("conflicting_active_identity", source, locator) + row["legacy_job_key"] = row.pop("job_key") + else: + # Keep unknown historical IDs/descriptors, but never imply a live job. + row["identity_state"] = "unassigned" + row["identity_reason"] = historical_reason + row.setdefault("identity_schema_version", 1) + if kind in {"restore_runs", "restore_index", "restore_detail"}: + self.restore_link(source, locator, job_id, key, kind, row) + return row + + def emit(self, source: str, record: dict, data: Any, *, sources: list | None = None) -> None: + target = record.get("target_path", source) + if not isinstance(target, str) or not target.startswith("/"): + self.reason("invalid_target_path", source) + return + if target in self.records: + self.reason("target_collision", source) + return + self.records[target] = {**deepcopy(record), "data": data, + "sources": sources or [source], "target_path": target} + + def schedules(self, source: str, record: dict) -> None: + data = record["data"] + if not isinstance(data, dict): + self.reason("invalid_owned_record", source) + return + result = {} + for key, row in data.items(): + locator = "/" + _pointer(key) + if (not isinstance(row, dict) or not isinstance(row.get("cron"), str) + or ("enabled" in row and type(row["enabled"]) is not bool)): + self.reason("invalid_schedule", source, locator) + continue + cron = row["cron"] + if ((cron and (len(cron.split()) != 5 or any(not re.fullmatch(r"[\d*/,\-]+", part) for part in cron.split()))) + or (not cron and row.get("enabled", True))): + self.reason("invalid_schedule", source, locator) + continue + if key == "restore_test": + result[key] = deepcopy(row) + continue + job_id = self.resolve(key, source, locator, active=True, code="orphan_active_schedule") + if job_id is None: + continue + self.bind(source, locator, job_id, key, "active", row) + if job_id in result: + self.reason("duplicate_schedule_identity", source, locator) + else: + result[job_id] = deepcopy(row) + self.emit(source, record, result) + + def repositories(self, source: str, record: dict) -> None: + data = deepcopy(record["data"]) + if not self.schema(data, source) or not isinstance(data.get("repositories"), list): + self.reason("invalid_repository_store", source) + return + seen = set() + for index, row in enumerate(data["repositories"]): + locator = f"/repositories/{index}" + if not isinstance(row, dict) or not isinstance(row.get("repository_key"), str): + self.reason("invalid_repository", source, locator) + continue + repo = row["repository_key"] + if repo in seen: + self.reason("duplicate_repository_key", source, locator) + seen.add(repo) + expected = {job_id for job_id, job in self.jobs.items() if job.get("repository_key") == repo} + for old, new in (("used_by", "job_ids"), ("source_job_keys", "source_job_ids")): + if old in row and new in row: + self.reason("mixed_repository_identity", source, locator) + continue + field = old if old in row else new + if field == old: + self.required = True + values = row.get(field, []) + if not isinstance(values, list): + self.reason("invalid_repository_references", source, locator + "/" + field) + continue + converted = [] + for i, value in enumerate(values): + pointer = locator + f"/{field}/{i}" + if self.verify and field == old: + self.reason("mutable_active_reference", source, pointer) + job_id = self.resolve(value, source, pointer, active=True, code="orphan_repository_reference") + if job_id is not None: + self.bind(source, pointer, job_id, value, "active", value) + converted.append(job_id) + if len(set(converted)) != len(converted) or set(converted) != expected: + self.reason("repository_assignment_mismatch", source, locator + "/" + field) + row.pop(old, None) + row[new] = converted + self.emit(source, record, data) + + def reminders(self, source: str, record: dict) -> None: + data = deepcopy(record["data"]) + if not self.schema(data, source) or not isinstance(data.get("last_sent"), dict): + self.reason("invalid_reminder_store", source) + return + converted = {} + unassigned = deepcopy(data.get("unassigned", [])) + if not isinstance(unassigned, list): + self.reason("invalid_reminder_store", source) + return + for key, value in data["last_sent"].items(): + locator = "/last_sent/" + _pointer(key) + parts = key.split(":", 2) + if (len(parts) != 3 or not all(parts) or type(value) not in (int, float) + or value < 0): + self.reason("invalid_reminder_record", source, locator) + continue + event, job_key, due = parts + job_id = self.resolve(job_key, source, locator, active=False) + self.bind(source, locator, job_id, job_key, "history", value) + if job_id is None: + unassigned.append({"key": key, "value": value, "source": source, "locator": locator}) + continue + if self.verify and job_key != job_id: + self.reason("mutable_active_reference", source, locator) + if job_key != job_id: + self.required = True + new_key = f"{event}:{job_id}:{due}" + if new_key in converted and converted[new_key] != value: + self.reason("reminder_identity_collision", source, locator) + converted[new_key] = value + data["last_sent"] = converted + if unassigned: + data["unassigned"] = unassigned + self.emit(source, record, data) + + def weekly(self, sources: list[tuple[str, dict]]) -> None: + groups: dict = {} + targets = {record.get("target_path", source) for source, record in sources} + if len(targets) != 1: + self.reason("weekly_destination_mismatch", sources[0][0]) + return + canonical = all(isinstance(record["data"], dict) and "observations" in record["data"] + for _, record in sources) + if self.verify and (not canonical or len(sources) != 1): + self.reason("legacy_weekly_store", sources[0][0]) + for source, record in sources: + data = record["data"] + if not isinstance(data, dict): + self.reason("invalid_weekly_store", source) + continue + if "observations" in data: + if not self.schema(data, source) or not isinstance(data["observations"], list): + self.reason("invalid_weekly_store", source) + continue + entries = [(row.get("job_id") or row.get("legacy_job_key", ""), row, + f"/observations/{i}") for i, row in enumerate(data["observations"]) if isinstance(row, dict)] + if len(entries) != len(data["observations"]): + self.reason("invalid_weekly_record", source) + else: + entries = [] + for key, rows in data.items(): + if not isinstance(rows, list): + self.reason("invalid_weekly_store", source, "/" + _pointer(key)) + continue + entries.extend((key, row, f"/{_pointer(key)}/{i}") for i, row in enumerate(rows)) + for key, original, locator in entries: + if (not isinstance(original, dict) or not isinstance(original.get("week"), str) + or type(original.get("size")) is not int or original["size"] < 0): + self.reason("invalid_weekly_record", source, locator) + continue + supplied_id = original.get("job_id") + if supplied_id is not None and not _uuid(supplied_id): + self.reason("invalid_job_id", source, locator) + continue + legacy_id = self.aliases.get(original.get("legacy_job_key", "")) + if (supplied_id and legacy_id is not None and supplied_id != legacy_id + and original.get("identity_state") != "unassigned"): + self.reason("conflicting_canonical_identity", source, locator) + job_id = (None if original.get("identity_state") == "unassigned" + else self.resolve(key, source, locator, active=False)) + history_reason = "deleted_job" if supplied_id and job_id is None else "no_configured_job" + if original.get("identity_state") == "unassigned": + history_reason = original.get("identity_reason") or history_reason + self.bind(source, locator, job_id, key, "history", original, history_reason) + if self.verify and job_id and key != job_id: + self.reason("mutable_active_reference", source, locator) + row = deepcopy(original) + # A deleted job is absent from the active graph, not stripped + # of its former immutable identity in retained history. + row["job_id"] = supplied_id or job_id + if not canonical: + row["legacy_job_key"] = key + if job_id is None and (not self.verify or not supplied_id or "identity_state" in original): + row["identity_state"] = "unassigned" + if supplied_id: + row["identity_reason"] = history_reason + provenance = row.pop("source_records", [{"source": source, "locator": locator}]) + row.pop("conflict", None) + if not isinstance(provenance, list) or not provenance: + self.reason("invalid_weekly_provenance", source, locator) + continue + identity = _json_key(row) + if identity not in groups: + groups[identity] = {**row, "source_records": []} + for evidence in provenance: + if (not isinstance(evidence, dict) or not isinstance(evidence.get("source"), str) + or not isinstance(evidence.get("locator"), str)): + self.reason("invalid_weekly_provenance", source, locator) + elif evidence not in groups[identity]["source_records"]: + groups[identity]["source_records"].append(evidence) + observations = list(groups.values()) + by_week: dict = {} + for row in observations: + owner = row.get("job_id") or row.get("legacy_job_key") + by_week.setdefault((owner, row["week"]), set()).add(row["size"]) + for row in observations: + owner = row.get("job_id") or row.get("legacy_job_key") + if len(by_week[(owner, row["week"])]) > 1: + row["conflict"] = True + self.reason("weekly_value_conflict_preserved", sources[0][0], severity="warning") + payload = (deepcopy(sources[0][1]["data"]) if canonical and len(sources) == 1 else {}) + payload.update({"schema_version": 1, "identity_schema_version": 1, "observations": observations}) + source, record = sources[0] + if self.verify and canonical and len(sources) == 1 and payload != record["data"]: + self.reason("weekly_projection_mismatch", source) + self.emit(source, record, payload, sources=[path for path, _ in sources]) + + def run(self) -> dict: + weekly = [] + for source, record in sorted(self.input.items()): + if (not isinstance(source, str) or not source.startswith("/") or not isinstance(record, dict) + or "data" not in record or not isinstance(record.get("kind"), str)): + self.reason("invalid_owned_record", str(source)) + continue + kind, data = record["kind"], record["data"] + if kind == "weekly": + weekly.append((source, record)) + elif kind == "schedules": + self.schedules(source, record) + elif kind == "repositories": + self.repositories(source, record) + elif kind == "notification_state": + self.reminders(source, record) + elif kind in _DIRECT: + legacy = record.get("legacy_key", "") + if kind == "status": + match = _STATUS_FILENAME.fullmatch(source.rsplit("/", 1)[-1]) + legacy = match.group(1) if match else legacy + output = self.row(data, source, "", kind, legacy=legacy) + proof = data if self.verify else output + if (kind == "restore_test" and isinstance(proof, dict) + and proof.get("identity_state") != "unassigned" and _uuid(proof.get("job_id"))): + target = source if self.verify else record.get("target_path", source) + if isinstance(target, str) and target.rsplit("/", 1)[-1] != proof["job_id"] + ".test": + self.reason("restore_test_filename_mismatch", source) + self.emit(source, record, output) + elif kind in _COLLECTIONS: + field, active = _COLLECTIONS[kind] + if not self.schema(data, source) or not isinstance(data.get(field), list): + self.reason("invalid_owned_collection", source) + continue + output = deepcopy(data) + output[field] = [self.row(row, source, f"/{field}/{index}", kind, active=active) + for index, row in enumerate(data[field])] + self.emit(source, record, output) + elif kind == "restore_runs": + if not self.schema(data, source) or not isinstance(data.get("runs"), dict): + self.reason("invalid_owned_collection", source) + continue + output = deepcopy(data) + for restore_id, row in data["runs"].items(): + locator = "/runs/" + _pointer(restore_id) + if not isinstance(row, dict) or row.get("restore_id") != restore_id: + self.reason("restore_id_mismatch", source, locator) + output["runs"][restore_id] = self.row(row, source, locator, kind, active=True) + self.emit(source, record, output) + elif kind == "storages": + if self.schema(data, source) and isinstance(data.get("storages"), list): + self.emit(source, record, deepcopy(data)) + else: + self.reason("invalid_storage_store", source) + elif kind == "widget_cache": + # Derived caches cannot survive cutover as active legacy joins. + # Rebuild is owned by #476/#479 under the startup writer gate. + if self.schema(data, source): + self.reason("widget_rebuild_required", source, severity="warning") + else: + self.reason("unsupported_record_kind", source) + if weekly: + self.weekly(weekly) + for links in self.restore_links.values(): + assigned = {(entry[2], entry[5].get("job_id") or entry[3]) if entry[2] is None + else (entry[2], "") for entry in links} + if len(assigned) > 1: + for source, locator, *_ in links: + self.reason("restore_identity_mismatch", source, locator) + by_kind = {kind: [entry for entry in links if entry[4] == kind] + for kind in ("restore_index", "restore_detail", "restore_runs")} + for entries in by_kind.values(): + if len(entries) > 1: + for source, locator, *_ in entries: + self.reason("duplicate_restore_id", source, locator) + index, detail, active = (by_kind[kind] for kind in ("restore_index", "restore_detail", "restore_runs")) + if (index or detail) and active: + for source, locator, *_ in links: + self.reason("restore_active_history_collision", source, locator) + if index and not detail: + for source, locator, *_ in index: + self.reason("missing_restore_detail", source, locator) + if detail and not index: + for source, locator, *_ in detail: + self.reason("missing_restore_index_entry", source, locator) + if len(index) == len(detail) == 1: + summary, body = index[0][5], detail[0][5] + if any((field in summary) != (field in body) or summary.get(field) != body.get(field) + for field in _RESTORE_SHARED_FIELDS): + for source, locator, *_ in index + detail: + self.reason("restore_snapshot_mismatch", source, locator) + return {"records": self.records, "bindings": self.bindings, + "unassigned": self.unassigned, "reasons": self.reasons, + "required": self.required} + + +def project_records(records: dict, jobs: dict, aliases: dict) -> dict: + """Project validated owned stores without touching input objects or disk.""" + return _Projection(records, jobs, aliases).run() + + +def verify_records(records: dict, jobs: dict, aliases: dict | None = None) -> list[dict]: + """Check actual target records; active aliases are errors, never repaired. + + This is referential verification only. The scanner/journal verifier must + independently enforce filenames, ownership, completeness and exact bytes. + """ + return _Projection(records, jobs, aliases or {}, verify=True).run()["reasons"] diff --git a/api/migrations/identity_storage.py b/api/migrations/identity_storage.py new file mode 100644 index 00000000..a4a424ab --- /dev/null +++ b/api/migrations/identity_storage.py @@ -0,0 +1,755 @@ +"""Private, inactive planning/snapshot primitives for #472. + +Nothing in this module rewrites installation data, invokes Borg, registers a +migration, or starts workers. The caller must supply an exact allowlisted plan +and a dedicated state directory on a persistent filesystem supporting private +0700/0600 permissions and hard links (normally not the Unraid FAT /boot USB). +A verified snapshot is not an apply engine, +a downloadable support bundle, or proof of an independent external backup. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from datetime import datetime, timezone +import errno +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import stat +from typing import Callable +import uuid + + +MIGRATION_ID = "immutable_job_id_v1" +STATUSES = frozenset({"pending", "applied", "skipped", "failed", "blocked", "not_applicable"}) +PHASES = frozenset({"detect", "plan", "snapshot", "verify", "confirm", "apply", "resume", "commit"}) +REASON_CODES = frozenset({ + "approval_required", "input_changed", "inventory_changed", "invalid_plan", + "invalid_snapshot", "snapshot_incomplete", "snapshot_changed", "unsafe_path", + "state_conflict", "invalid_journal", "storage_unavailable", "insufficient_space", + "writers_active", "verification_failed", "interrupted", "not_applicable", + "state_filesystem_unsupported", +}) +_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0) +_DIRECTORY = getattr(os, "O_DIRECTORY", 0) + + +class IdentityStorageError(RuntimeError): + """Stable error codes only: never expose filesystem exception/secret text.""" + + def __init__(self, code: str): + self.code = code if isinstance(code, str) and code in REASON_CODES else "verification_failed" + super().__init__(self.code) + + +def _fail(code: str): + raise IdentityStorageError(code) from None + + +def _canonical(value) -> bytes: + try: + return json.dumps(value, sort_keys=True, ensure_ascii=True, allow_nan=False, + separators=(",", ":")).encode("utf-8") + except (TypeError, ValueError, OverflowError, RecursionError): + _fail("invalid_plan") + + +def _digest(value) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _absolute(path) -> Path: + try: + raw = os.fspath(path) + except TypeError: + _fail("unsafe_path") + if not isinstance(raw, str) or not raw.startswith("/") or "\x00" in raw: + _fail("unsafe_path") + if raw != os.path.normpath(raw) or raw.startswith("//"): + _fail("unsafe_path") + try: + raw.encode("utf-8") + except UnicodeError: + _fail("unsafe_path") + return Path(raw) + + +def _os_error(exc: OSError): + _fail("unsafe_path" if exc.errno in {errno.ELOOP, errno.ENOTDIR} else "storage_unavailable") + + +@contextmanager +def _directory(path, *, missing_ok=False): + """Walk using directory FDs; never follow even an intermediate symlink.""" + path = _absolute(path) + fd = None + try: + fd = os.open("/", os.O_RDONLY | _DIRECTORY | _NOFOLLOW) + for component in path.parts[1:]: + try: + child = os.open(component, os.O_RDONLY | _DIRECTORY | _NOFOLLOW, dir_fd=fd) + except FileNotFoundError: + if missing_ok: + os.close(fd) + fd = None + yield None + return + raise + os.close(fd) + fd = child + yield fd + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) + + +def _read_file(path, *, private=False): + path = _absolute(path) + with _directory(path.parent, missing_ok=True) as directory: + if directory is None: + return {"exists": False}, None + fd = None + try: + try: + fd = os.open(path.name, os.O_RDONLY | _NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) + except FileNotFoundError: + return {"exists": False}, None + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode): + _fail("unsafe_path") + if private and (stat.S_IMODE(before.st_mode) != 0o600 + or before.st_uid != os.getuid() or before.st_nlink != 1): + _fail("unsafe_path") + chunks = [] + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(fd) + identity = lambda info: (info.st_dev, info.st_ino, info.st_size, + info.st_mtime_ns, info.st_ctime_ns, info.st_mode) + if identity(before) != identity(after): + _fail("input_changed") + content = b"".join(chunks) + if len(content) != before.st_size: + _fail("input_changed") + return {"exists": True, "size": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "mode": stat.S_IMODE(before.st_mode)}, content + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) + + +def fingerprint_file(path) -> dict: + """Read exact regular-file bytes; absent destinations are explicit.""" + return _read_file(path)[0] + + +def read_fingerprinted_file(path): + """Return one stable descriptor read: ``(fingerprint, bytes_or_None)``.""" + return _read_file(path) + + +def read_file(path) -> bytes: + """Read a required regular file without following any symlink component.""" + fingerprint, content = _read_file(path) + if not fingerprint["exists"]: + _fail("storage_unavailable") + return content + + +def inventory_group(path, suffixes) -> dict: + """Capture an immediate allowlist, including an absent directory itself. + + No recursive scan is performed. Matching non-files/symlinks are rejected, + not silently hidden. Directory identity detects replaced/missing mounts. + """ + path = _absolute(path) + if path == Path("/") or not isinstance(suffixes, (list, tuple)) or not suffixes: + _fail("unsafe_path") + if any(not isinstance(item, str) or not item.startswith(".") + or "/" in item or "\\" in item or len(item) > 32 for item in suffixes): + _fail("unsafe_path") + suffixes = sorted(set(suffixes)) + with _directory(path, missing_ok=True) as directory: + if directory is None: + return {"path": str(path), "suffixes": suffixes, "exists": False, "entries": []} + before = os.fstat(directory) + names = sorted(name for name in os.listdir(directory) if name.endswith(tuple(suffixes))) + for name in names: + info = os.stat(name, dir_fd=directory, follow_symlinks=False) + if not stat.S_ISREG(info.st_mode): + _fail("unsafe_path") + after = os.fstat(directory) + if (before.st_mtime_ns, before.st_ctime_ns) != (after.st_mtime_ns, after.st_ctime_ns): + _fail("inventory_changed") + return {"path": str(path), "suffixes": suffixes, "exists": True, + "device": before.st_dev, "inode": before.st_ino, "entries": names} + + +def inventory_directories(path) -> dict: + """Capture a bounded directory-only control root, not a recursive tree. + + Every child must be a real directory. Callers inventory the allowed files + inside each child separately, so an added run directory invalidates a plan. + """ + path = _absolute(path) + if path == Path("/"): + _fail("unsafe_path") + with _directory(path, missing_ok=True) as directory: + if directory is None: + return {"path": str(path), "kind": "directories", "exists": False, "entries": []} + before = os.fstat(directory) + names = sorted(os.listdir(directory)) + for name in names: + info = os.stat(name, dir_fd=directory, follow_symlinks=False) + if not stat.S_ISDIR(info.st_mode): + _fail("unsafe_path") + after = os.fstat(directory) + if (before.st_mtime_ns, before.st_ctime_ns) != (after.st_mtime_ns, after.st_ctime_ns): + _fail("inventory_changed") + return {"path": str(path), "kind": "directories", "exists": True, + "device": before.st_dev, "inode": before.st_ino, "entries": names} + + +def _valid_fingerprint(value): + if not isinstance(value, dict) or type(value.get("exists")) is not bool: + _fail("invalid_plan") + if not value["exists"]: + if value != {"exists": False}: + _fail("invalid_plan") + return + if set(value) != {"exists", "size", "sha256", "mode"}: + _fail("invalid_plan") + if type(value["size"]) is not int or value["size"] < 0: + _fail("invalid_plan") + if type(value["mode"]) is not int or not 0 <= value["mode"] <= 0o7777: + _fail("invalid_plan") + checksum = value["sha256"] + if not isinstance(checksum, str) or len(checksum) != 64 or any(c not in "0123456789abcdef" for c in checksum): + _fail("invalid_plan") + + +def seal_plan(plan: dict) -> dict: + """Validate structural storage boundaries and hash the complete plan. + + Domain/referential validation belongs to the planner. Multiple aliases may + deliberately map to the same UUID. No UUID is allocated by this module. + """ + if not isinstance(plan, dict): + _fail("invalid_plan") + result = json.loads(_canonical(plan)) + claimed = result.pop("plan_id", None) + if type(result.get("schema_version")) is not int or result["schema_version"] != 1: + _fail("invalid_plan") + if result.get("migration_id") != MIGRATION_ID or not isinstance(result.get("id_map"), dict): + _fail("invalid_plan") + for alias, identity in result["id_map"].items(): + if not isinstance(alias, str) or not alias or not isinstance(identity, str): + _fail("invalid_plan") + try: + value = uuid.UUID(identity) + except (ValueError, AttributeError): + _fail("invalid_plan") + if str(value) != identity or value.version != 4 or value.variant != uuid.RFC_4122: + _fail("invalid_plan") + inputs = result.get("inputs") + if not isinstance(inputs, dict): + _fail("invalid_plan") + for path, value in inputs.items(): + _absolute(path) + _valid_fingerprint(value) + external = result.get("external_inputs", {}) + if not isinstance(external, dict): + _fail("invalid_plan") + for name, item in external.items(): + if (not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", name) + or not isinstance(item, dict) or set(item) != {"text", "kind"} + or not isinstance(item["text"], str) or item["kind"] != "crontab"): + _fail("invalid_plan") + try: + item["text"].encode("utf-8") + except UnicodeError: + _fail("invalid_plan") + groups = result.get("inventory_groups", []) + if not isinstance(groups, list): + _fail("invalid_plan") + seen_roots = set() + for group in groups: + if not isinstance(group, dict) or not isinstance(group.get("entries"), list): + _fail("invalid_plan") + path = _absolute(group.get("path")) + suffixes = group.get("suffixes") + directory_group = group.get("kind") == "directories" + if (not directory_group and (not isinstance(suffixes, list) or not suffixes) + or type(group.get("exists")) is not bool): + _fail("invalid_plan") + if str(path) in seen_roots: + _fail("invalid_plan") + seen_roots.add(str(path)) + for name in group["entries"]: + if not isinstance(name, str) or not name or name in {".", ".."} or "/" in name or "\\" in name: + _fail("invalid_plan") + if not directory_group and str(path / name) not in inputs: + _fail("invalid_plan") + actions = result.get("actions", []) + if not isinstance(actions, list): + _fail("invalid_plan") + action_ids = set() + for action in actions: + if not isinstance(action, dict) or not isinstance(action.get("id"), str) or not action["id"]: + _fail("invalid_plan") + if action["id"] in action_ids: + _fail("invalid_plan") + action_ids.add(action["id"]) + for field in ("source", "target"): + if action.get(field) is not None and str(_absolute(action[field])) not in inputs: + _fail("invalid_plan") + digest = _digest(result) + if claimed is not None and claimed != digest: + _fail("invalid_plan") + result["plan_id"] = digest + return result + + +def _check_overlap(plan, state_dir): + state_dir = _absolute(state_dir) + for name in plan["inputs"]: + path = _absolute(name) + if path == state_dir or state_dir in path.parents or path in state_dir.parents: + _fail("unsafe_path") + for group in plan.get("inventory_groups", []): + path = _absolute(group["path"]) + # A dedicated migration directory can be below config, but never below + # one of the exact scanned jobs/status/restore-test directories. + if state_dir == path or path in state_dir.parents or state_dir in path.parents: + _fail("unsafe_path") + + +def _private_directory(path, *, create=False): + path = _absolute(path) + if create: + with _directory(path.parent) as parent: + try: + os.mkdir(path.name, mode=0o700, dir_fd=parent) + os.fsync(parent) + except FileExistsError: + pass + except OSError as exc: + _os_error(exc) + with _directory(path) as directory: + info = os.fstat(directory) + if stat.S_IMODE(info.st_mode) != 0o700 or info.st_uid != os.getuid(): + _fail("unsafe_path") + return path + + +def _publish_once(path: Path, content: bytes): + """Durable publication without overwriting an existing pathname.""" + with _directory(path.parent) as directory: + temporary = ".stage-" + uuid.uuid4().hex + fd = None + try: + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _NOFOLLOW, + 0o600, dir_fd=directory) + view = memoryview(content) + while view: + written = os.write(fd, view) + if written <= 0: + _fail("storage_unavailable") + view = view[written:] + os.fsync(fd) + os.close(fd) + fd = None + try: + os.link(temporary, path.name, src_dir_fd=directory, dst_dir_fd=directory, + follow_symlinks=False) + except FileExistsError: + existing, raw = _read_file(path, private=True) + if not existing["exists"] or raw != content: + _fail("state_conflict") + except OSError as exc: + if exc.errno in {errno.EOPNOTSUPP, errno.ENOSYS, errno.EXDEV, errno.EPERM}: + _fail("state_filesystem_unsupported") + raise + os.unlink(temporary, dir_fd=directory) + temporary = None + os.fsync(directory) + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) + if temporary is not None: + try: + os.unlink(temporary, dir_fd=directory) + except OSError: + pass + + +def _read_json(path): + exists, raw = _read_file(path, private=True) + if not exists["exists"]: + _fail("snapshot_incomplete") + try: + return json.loads(raw, object_pairs_hook=_unique_json_pairs) + except (ValueError, UnicodeError): + _fail("state_conflict") + + +def _unique_json_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def load_plan(state_dir) -> dict: + """Read/validate the persisted allocation before considering a retry.""" + state_dir = _private_directory(state_dir) + result = seal_plan(_read_json(state_dir / "plan.json")) + _check_overlap(result, state_dir) + return result + + +def persist_plan(plan: dict, state_dir) -> dict: + plan = seal_plan(plan) + _check_overlap(plan, state_dir) + state_dir = _private_directory(state_dir, create=True) + _publish_once(state_dir / "plan.json", _canonical(plan)) + persisted = load_plan(state_dir) + if persisted != plan: + _fail("state_conflict") + return persisted + + +def verify_inputs(plan: dict): + plan = seal_plan(plan) + for path, expected in plan["inputs"].items(): + if fingerprint_file(path) != expected: + _fail("input_changed") + for expected in plan.get("inventory_groups", []): + current = (inventory_directories(expected["path"]) if expected.get("kind") == "directories" + else inventory_group(expected["path"], expected["suffixes"])) + if current != expected: + _fail("inventory_changed") + return True + + +def _snapshot_metadata(plan, snapshot, *, create=False): + """Persist creation identity once, including across interrupted copies. + + The timestamp is separate from the pre-existing sealed plan: snapshot + creation never silently changes that plan's digest or UUID allocation. + """ + path = snapshot / "metadata.json" + fingerprint, _ = _read_file(path, private=True) + if not fingerprint["exists"]: + if not create: + _fail("snapshot_incomplete") + # Once copying/commit has begun, a missing header is lost evidence, + # not permission to assign a fresh creation time to existing bytes. + if fingerprint_file(snapshot / "manifest.json")["exists"]: + _fail("snapshot_incomplete") + with _directory(snapshot / "files") as directory: + if os.listdir(directory): + _fail("snapshot_incomplete") + metadata = {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], + "created_at": datetime.now(timezone.utc).isoformat()} + _publish_once(path, _canonical(metadata)) + metadata = _read_json(path) + if not isinstance(metadata, dict) or not isinstance(metadata.get("created_at"), str): + _fail("invalid_snapshot") + try: + timestamp = datetime.fromisoformat(metadata["created_at"]) + except (ValueError, TypeError): + _fail("invalid_snapshot") + if timestamp.tzinfo != timezone.utc or timestamp.isoformat() != metadata["created_at"]: + _fail("invalid_snapshot") + expected = {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], "created_at": metadata["created_at"]} + if _canonical(metadata) != _canonical(expected): + _fail("invalid_snapshot") + return metadata + + +def _snapshot_manifest(plan, metadata): + entries = {} + for path, expected in plan["inputs"].items(): + entries[path] = {"artifact_kind": "file", "original": expected, + "blob": hashlib.sha256(path.encode("utf-8")).hexdigest() + ".bin" + if expected["exists"] else None} + external = {} + for name, item in plan.get("external_inputs", {}).items(): + raw = item["text"].encode("utf-8") + external[name] = {"artifact_kind": "external", "kind": item["kind"], "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + "blob": "external-" + name + ".bin"} + return {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], "entries": entries, + "created_at": metadata["created_at"], "id_map": plan["id_map"], + # Deliberately exclude projected `data` and any other action + # payload: secrets/configuration bytes belong only in private + # original blobs and the protected plan, never duplicated here. + "actions": [{key: action[key] for key in ("id", "kind", "source", "target") + if key in action} for action in plan.get("actions", [])], + "external_inputs": external, + "inventory_groups": plan.get("inventory_groups", [])} + + +def _snapshot_requirements(plan): + prerequisites = plan.get("prerequisites", {}) + if not isinstance(prerequisites, dict): + _fail("invalid_plan") + if "managed_cron_captured" in prerequisites and prerequisites["managed_cron_captured"] is not True: + _fail("snapshot_incomplete") + if prerequisites.get("managed_cron_captured") is True and "managed_cron" not in plan.get("external_inputs", {}): + _fail("snapshot_incomplete") + + +def create_snapshot(plan: dict, state_dir) -> dict: + """Copy exact original inputs, including existing destination originals. + + A partially copied snapshot can be completed only with the same persisted + plan, unchanged inputs and already-valid blobs. Corrupt blobs are never + overwritten. There is deliberately no export or restoration endpoint. + """ + plan = seal_plan(plan) + _snapshot_requirements(plan) + plan = persist_plan(plan, state_dir) + verify_inputs(plan) + state_dir = _private_directory(state_dir) + snapshot = _private_directory(state_dir / "snapshot", create=True) + blobs = _private_directory(snapshot / "files", create=True) + with _directory(blobs) as directory: + free = os.fstatvfs(directory) + required = sum(item["size"] for item in plan["inputs"].values() if item["exists"]) + required += sum(len(item["text"].encode("utf-8")) for item in plan.get("external_inputs", {}).values()) + if free.f_bavail * free.f_frsize < required + 65536: + _fail("insufficient_space") + metadata = _snapshot_metadata(plan, snapshot, create=True) + manifest = _snapshot_manifest(plan, metadata) + for path, entry in manifest["entries"].items(): + actual, content = _read_file(path) + if actual != entry["original"]: + _fail("input_changed") + if entry["blob"] is not None: + _publish_once(blobs / entry["blob"], content) + for name, entry in manifest["external_inputs"].items(): + _publish_once(blobs / entry["blob"], plan["external_inputs"][name]["text"].encode("utf-8")) + verify_inputs(plan) + _publish_once(snapshot / "manifest.json", _canonical(manifest)) + handle = {"path": str(snapshot), "plan_id": plan["plan_id"], "digest": _digest(manifest)} + verify_snapshot(plan, handle) + return handle + + +def verify_snapshot(plan: dict, snapshot: dict) -> dict: + """Verify completeness, binding and each private stored byte independently.""" + plan = seal_plan(plan) + _snapshot_requirements(plan) + if not isinstance(snapshot, dict) or set(snapshot) != {"path", "plan_id", "digest"}: + _fail("invalid_snapshot") + path = _absolute(snapshot["path"]) + if path.name != "snapshot" or snapshot["plan_id"] != plan["plan_id"]: + _fail("invalid_snapshot") + _private_directory(path) + if load_plan(path.parent) != plan: + _fail("invalid_snapshot") + metadata = _snapshot_metadata(plan, path) + manifest = _read_json(path / "manifest.json") + if _canonical(manifest) != _canonical(_snapshot_manifest(plan, metadata)) or _digest(manifest) != snapshot["digest"]: + _fail("invalid_snapshot") + blobs = _private_directory(path / "files") + expected_names = {entry["blob"] for entry in manifest["entries"].values() if entry["blob"]} + expected_names.update(entry["blob"] for entry in manifest["external_inputs"].values()) + with _directory(blobs) as directory: + # Interrupted private staging files are not recovery blobs. They carry + # no authority; committed snapshots must contain exactly their blobs. + if set(os.listdir(directory)) != expected_names: + _fail("snapshot_incomplete") + for entry in manifest["entries"].values(): + if entry["blob"]: + actual, _ = _read_file(blobs / entry["blob"], private=True) + original = entry["original"] + if not actual["exists"] or (actual["size"], actual["sha256"]) != (original["size"], original["sha256"]): + _fail("snapshot_changed") + for entry in manifest["external_inputs"].values(): + actual, _ = _read_file(blobs / entry["blob"], private=True) + if not actual["exists"] or (actual["size"], actual["sha256"]) != (entry["size"], entry["sha256"]): + _fail("snapshot_changed") + return manifest + + +def verify_preconditions(plan: dict, snapshot: dict, confirmation=None, *, + quiescence_check: Callable[[], bool] | None = None, + external_input_check: Callable[[], dict] | None = None) -> bool: + """Default-deny *library* gate; this neither applies data nor trusts a UI. + + Phase #479 must supply the real writer/maintenance check and authenticated + confirmation. Checking immediately here cannot replace locks held across + the eventual apply transaction. An acknowledgement is not external-copy + verification, and this function must never be described as such. + """ + plan = seal_plan(plan) + if (plan.get("classification") != "applicable" or plan.get("required") is not True + or plan.get("status") != "pending"): + _fail("invalid_plan") + prerequisites = plan.get("prerequisites") + if not isinstance(prerequisites, dict) or prerequisites.get("managed_cron_captured") is not True: + _fail("snapshot_incomplete") + _snapshot_requirements(plan) + if not isinstance(confirmation, dict) or confirmation.get("approved") is not True: + _fail("approval_required") + if (confirmation.get("independent_backup_acknowledged") is not True + or confirmation.get("plan_id") != plan["plan_id"] + or not isinstance(snapshot, dict) + or confirmation.get("snapshot_digest") != snapshot.get("digest")): + _fail("approval_required") + if quiescence_check is None: + _fail("writers_active") + try: + quiescent = quiescence_check() + except Exception: + _fail("writers_active") + if quiescent is not True: + _fail("writers_active") + verify_snapshot(plan, snapshot) + verify_inputs(plan) + if plan.get("external_inputs"): + if external_input_check is None: + _fail("input_changed") + try: + current = external_input_check() + except Exception: + _fail("input_changed") + if current != plan["external_inputs"]: + _fail("input_changed") + return True + + +def _journal_records(raw, plan): + records = [] + previous = None + if raw and not raw.endswith(b"\n"): + _fail("invalid_journal") + for line in raw.splitlines(): + try: + record = json.loads(line, object_pairs_hook=_unique_json_pairs) + except (ValueError, UnicodeError): + _fail("invalid_journal") + if not isinstance(record, dict): + _fail("invalid_journal") + expected = {"schema_version", "migration_id", "plan_id", "sequence", "timestamp", + "status", "phase", "reason_code", "action_ids", "previous", "digest"} + if (set(record) != expected or type(record["schema_version"]) is not int + or record["schema_version"] != 1 or record["migration_id"] != MIGRATION_ID): + _fail("invalid_journal") + if (record["plan_id"] != plan["plan_id"] or type(record["sequence"]) is not int + or record["sequence"] != len(records) + 1): + _fail("invalid_journal") + try: + timestamp = datetime.fromisoformat(record["timestamp"]) + if timestamp.utcoffset() is None: + _fail("invalid_journal") + except (ValueError, TypeError): + _fail("invalid_journal") + if (not isinstance(record["status"], str) or record["status"] not in STATUSES + or not isinstance(record["phase"], str) or record["phase"] not in PHASES): + _fail("invalid_journal") + if record["reason_code"] is not None and (not isinstance(record["reason_code"], str) or record["reason_code"] not in REASON_CODES): + _fail("invalid_journal") + known_actions = {action["id"] for action in plan.get("actions", [])} + if not isinstance(record["action_ids"], list) or any(not isinstance(item, str) or item not in known_actions for item in record["action_ids"]): + _fail("invalid_journal") + if record["previous"] != previous: + _fail("invalid_journal") + unsigned = dict(record) + claimed = unsigned.pop("digest") + if claimed != _digest(unsigned): + _fail("invalid_journal") + previous = claimed + records.append(record) + return records + + +def read_journal(state_dir) -> list: + state_dir = _private_directory(state_dir) + plan = load_plan(state_dir) + _, raw = _read_file(state_dir / "journal.jsonl", private=True) + return _journal_records(raw or b"", plan) + + +def append_journal(state_dir, plan: dict, status: str, phase: str, *, + reason_code=None, action_ids=None) -> dict: + """Append a private, fsynced, hash-linked event without free-form errors.""" + plan = seal_plan(plan) + if (not isinstance(status, str) or status not in STATUSES + or not isinstance(phase, str) or phase not in PHASES + or reason_code is not None and (not isinstance(reason_code, str) or reason_code not in REASON_CODES)): + _fail("invalid_journal") + action_ids = [] if action_ids is None else action_ids + known_actions = {action["id"] for action in plan.get("actions", [])} + if not isinstance(action_ids, list) or any(not isinstance(item, str) or item not in known_actions for item in action_ids): + _fail("invalid_journal") + state_dir = _private_directory(state_dir) + if load_plan(state_dir) != plan: + _fail("state_conflict") + with _directory(state_dir) as directory: + fd = None + try: + fd = os.open("journal.jsonl", os.O_RDWR | os.O_APPEND | os.O_CREAT | _NOFOLLOW, + 0o600, dir_fd=directory) + fcntl.flock(fd, fcntl.LOCK_EX) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_uid != os.getuid() or info.st_nlink != 1: + _fail("unsafe_path") + raw = b"" + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + raw += chunk + records = _journal_records(raw, plan) + record = {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], "sequence": len(records) + 1, + "timestamp": datetime.now(timezone.utc).isoformat(), + "status": status, "phase": phase, "reason_code": reason_code, + "action_ids": action_ids, + "previous": records[-1]["digest"] if records else None} + record["digest"] = _digest(record) + data = memoryview(_canonical(record) + b"\n") + while data: + written = os.write(fd, data) + if written <= 0: + _fail("storage_unavailable") + data = data[written:] + os.fsync(fd) + os.fsync(directory) + return record + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) diff --git a/api/migrations/immutable_job_id_v1.py b/api/migrations/immutable_job_id_v1.py new file mode 100644 index 00000000..d93b6d06 --- /dev/null +++ b/api/migrations/immutable_job_id_v1.py @@ -0,0 +1,687 @@ +"""Inactive, read-only identity migration planner (#472). + +Deliberately not registered, imported by startup, exposed by HTTP, or given an +apply() entry point. Proposed JSON replacements are private planning data; +only the separate snapshot/journal utilities may write a dedicated state dir. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +import os +from pathlib import Path +import re +import stat +from uuid import UUID, uuid4 + +from . import identity_storage as storage +from .identity_records import project_records, verify_records + + +MIGRATION_ID = "immutable_job_id_v1" +INTRODUCED_IN = "pending-issue-447" +MAX_JSON_BYTES = 64 * 1024 * 1024 +_KEY = re.compile(r"^[A-Za-z0-9_.-]+$") +_TYPE = re.compile(r"^[a-z0-9_]+$") +_LOCATIONS = {"local", "usb", "smb", "storagebox", "custom"} +_LEGACY = {"job_key", "backup_type", "type_id", "location"} + + +class PlanningError(ValueError): + def __init__(self, code, source=""): + self.code, self.source = code, str(source) + super().__init__(code) # Never interpolate raw JSON, config or errors. + + +def _fail(code, source=""): + raise PlanningError(code, source) + + +def _uuid(value): + try: + parsed = UUID(value) if isinstance(value, str) else None + except ValueError: + parsed = None + return parsed is not None and parsed.version == 4 and str(parsed) == value + + +def _path(value): + if not isinstance(value, (str, Path)): + _fail("unsafe_path") + raw = str(value) + if not raw.startswith("/") or raw != os.path.normpath(raw) or raw.startswith("//"): + _fail("unsafe_path") + if any(c in raw for c in ("\x00", "\n", "\r", "$")) or raw == "/": + _fail("unsafe_path") + path = Path(raw) + parts = path.parts + mount = None + if len(parts) > 2 and parts[1] == "mnt": + mount = Path(*parts[:4]) if parts[2] in {"disks", "remotes"} and len(parts) > 3 else Path(*parts[:3]) + elif len(parts) > 1 and parts[1] == "boot": + mount = Path("/boot") + if mount is not None and not mount.is_mount(): + _fail("required_mount_unavailable", path) + return path + + +def _strict_json(raw, source): + def pairs(items): + result = {} + for key, value in items: + if key in result: + _fail("duplicate_json_member", source) + result[key] = value + return result + if len(raw) > MAX_JSON_BYTES: + _fail("owned_input_too_large", source) + try: + return json.loads(raw, object_pairs_hook=pairs, + parse_constant=lambda _: _fail("invalid_json", source)) + except (UnicodeError, json.JSONDecodeError, RecursionError): + _fail("invalid_json", source) + + +def _read_conf(raw): + """Same literal decoding/forward-reference semantics as status.load_config. + + No shell, environment expansion or import of lazy runtime helpers. Only + relevant non-secret values are copied into proposed canonical metadata. + """ + values = {} + try: + lines = raw.decode("utf-8").splitlines() + except UnicodeError: + _fail("invalid_configuration_encoding") + for line in lines: + line = line.strip().removeprefix("readonly ") + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key, value = key.strip(), value.strip() + if value.startswith('"'): + try: + decoded, end = json.JSONDecoder().raw_decode(value) + tail = value[end:].strip() + if isinstance(decoded, str) and (not tail or tail.startswith("#")): + value = decoded + except json.JSONDecodeError: + pass + elif value.startswith("'") and value.rfind("'") > 0: + end = value.rfind("'") + tail = value[end + 1:].strip() + if not tail or tail.startswith("#"): + value = value[1:end] + else: + value = value.split(" #", 1)[0].rstrip() + if value.startswith("("): + continue + values[key] = re.sub(r"\$\{([^}]+)\}", lambda m: values.get(m[1], m[0]), value) + return values + + +class Inventory: + def __init__(self): + self.inputs, self.groups, self.records, self.raw = {}, [], {}, {} + + def file(self, path, kind=None, **metadata): + path = _path(path) + name = str(path) + fingerprint, raw = storage.read_fingerprinted_file(path) + previous = self.inputs.get(name) + if previous is not None and previous != fingerprint: + _fail("source_fingerprint_changed", path) + self.inputs[name] = fingerprint + if raw is None: + return None + self.raw[name] = raw + if kind is not None: + value = _strict_json(raw, path) + if not isinstance(value, dict): + _fail("invalid_job_shape" if kind == "job" else "invalid_store_shape", path) + old = self.records.get(name) + if old and old["kind"] != kind: + _fail("overlapping_owned_stores", path) + self.records[name] = {"kind": kind, "data": value, **metadata} + return value + return raw + + def group(self, directory, suffix, kind, **metadata): + directory = _path(directory) + group = storage.inventory_group(directory, [suffix]) + if group not in self.groups: + self.groups.append(group) + for name in group["entries"]: + self.file(directory / name, kind, **metadata) + return group + + +def _inventory(config, control_root): + scan = Inventory() + data = _path(config.get("BACKUP_SCRIPTS_DIR", "/boot/config/borg-backup")) + if data.name == "scripts": + data = data.parent + conf = scan.file(data / "config/backup.conf") + expanded = _read_conf(conf) if conf is not None else {} + effective = dict(config) + for key in ("STATUS_DIR", "RESTORE_TEST_STATUS_DIR", "BORG_RESOURCE_LOCK_DIR", + "RUNTIME_RECOVERY_FILE", "UNRAID_DASHBOARD_WIDGET_FILE"): + if expanded.get(key): + effective[key] = expanded[key] + plugin = _path(effective.get("PLUGIN_DIR") or "/boot/config/plugins/borg-backup-ui") + jobs_dir = data / "config/jobs" + scan.group(jobs_dir, ".json", "job") + # Known lazy-migration locations only; never traverse runtime/vendor/recycle bins. + scripts = _path(effective.get("BORG_SCRIPTS_DIR") or str(data / "scripts")) + for legacy in sorted({scripts / "config/jobs", plugin / "runtime/config/jobs"} - {jobs_dir}): + scan.group(legacy, ".json", "job", legacy_directory=True) + singleton = { + "repositories.json": "repositories", "storages.json": "storages", + "schedules.json": "schedules", "restore-runs.json": "restore_runs", + "restore-history/index.json": "restore_index", + "notification-queue.json": "notification_queue", + "notification-deliveries.json": "notification_deliveries", + "notification-state.json": "notification_state", + } + for filename, kind in singleton.items(): + scan.file(data / "config" / filename, kind) + scan.file(_path(effective.get("RUNTIME_RECOVERY_FILE") or str(data / "config/runtime-recovery.json")), "runtime_recovery") + scan.group(data / "config/restore-history/runs", ".json", "restore_detail") + status_dir = _path(effective.get("STATUS_DIR") or "/mnt/user/backup-status") + scan.group(status_dir, ".status", "status") + weekly = _path(effective.get("SNAPSHOT_FILE") or str(status_dir.parent / "weekly-snapshots.json")) + for candidate in sorted({weekly, status_dir / "weekly-snapshots.json"}): + scan.file(candidate, "weekly", target_path=str(weekly)) + candidates = [] + if effective.get("RESTORE_TEST_STATUS_DIR"): + candidates.append(_path(effective["RESTORE_TEST_STATUS_DIR"])) + candidates += [status_dir.parent / "restore-status", status_dir / "restore-tests"] + # Scan each known location: hidden stale results are not silently abandoned. + for directory in dict.fromkeys(candidates): + scan.group(directory, ".test", "restore_test") + lock_dir = _path(effective.get("BORG_RESOURCE_LOCK_DIR") or str(data / "locks")) + scan.group(lock_dir, ".json", "resource_lock") + # This independent worker lock does not prove quiescence; preserve/capture it. + scan.file(data / "locks/notification-delivery.lock") + controls = _path(control_root or "/run/borg-backup-ui/jobs") + children = storage.inventory_directories(controls) + scan.groups.append(children) + for run in children["entries"]: + group = scan.group(controls / run, ".json", "control") + for name in group["entries"]: + path = str(controls / run / name) + if name == "cancel.request.json": + scan.records[path]["kind"] = "cancel_request" + elif name != "state.json": + _fail("unknown_control_file", path) + widget = _path(effective.get("UNRAID_DASHBOARD_WIDGET_FILE") or str(plugin / "widget-status.json")) + scan.file(widget, "widget_cache") + return scan, data, jobs_dir, expanded + + +def _prefixes(meta, legacy, source): + values = meta.get("archive_prefixes", []) + if not isinstance(values, list) or any(not isinstance(v, str) or not _KEY.fullmatch(v) + or v in {".", ".."} for v in values): + _fail("invalid_archive_prefix", source) + if legacy and any(not re.fullmatch(r"[A-Za-z0-9_.-]+-backup", value) for value in values): + # The old reader silently ignored these. Adopting them would expand + # archive/prune ownership; dropping them would discard user data. + _fail("invalid_archive_prefix", source) + if not legacy and (not values or len(set(values)) != len(values)): + _fail("invalid_archive_prefix", source) + return list(dict.fromkeys(([meta["backup_type"] + "-backup"] if legacy else []) + values)) + + +def _validate_job(meta, source): + schema = meta.get("schema_version") + if type(schema) is not int or schema not in {1, 2, 3, 4}: + _fail("unsupported_schema", source) + if schema == 4: + if not _uuid(meta.get("job_id")): + _fail("invalid_job_id", source) + if _LEGACY.intersection(meta): + _fail("mutable_canonical_identity", source) + if "legacy_job_keys" not in meta: + _fail("invalid_legacy_alias", source) + else: + typ, location, key = meta.get("backup_type"), meta.get("location"), meta.get("job_key") + if not isinstance(typ, str) or not _TYPE.fullmatch(typ) or location not in _LOCATIONS: + _fail("conflicting_legacy_identity", source) + if key != f"{typ}_{location}" or Path(source).stem != key or "job_id" in meta: + _fail("conflicting_legacy_identity", source) + aliases = meta.get("legacy_job_keys", []) + if not isinstance(aliases, list) or any(not isinstance(a, str) or not _KEY.fullmatch(a) for a in aliases): + _fail("invalid_legacy_alias", source) + if len(set(aliases)) != len(aliases): + _fail("duplicate_legacy_alias", source) + _prefixes(meta, schema != 4, source) + if not isinstance(meta.get("name"), str) or not meta["name"].strip(): + _fail("invalid_job_name", source) + if not isinstance(meta.get("repository_key"), str) or not _KEY.fullmatch(meta["repository_key"]): + _fail("dangling_repository", source) + + +def _operational_defaults(meta, conf, source): + result = deepcopy(meta) + # Existing pure path converter does no writes. Ambiguous old strings may + # inspect source directories, but never silently split nonexistent paths. + try: + from ..job_source_paths import normalize_source_paths, upgrade_job_source_paths + except ImportError: + from job_source_paths import normalize_source_paths, upgrade_job_source_paths + try: + if meta["schema_version"] in {1, 2}: + result = upgrade_job_source_paths(result, job_key=meta["job_key"]) + elif normalize_source_paths(meta.get("source_paths")) != meta.get("source_paths"): + _fail("noncanonical_source_paths", source) + except ValueError: + _fail("invalid_source_paths", source) + if meta["schema_version"] == 4: + return result + tu = meta["backup_type"].upper() + # Env overrides unrelated to job metadata cannot be inferred safely. The + # startup coordinator must supply the actual expanded backup.conf, captured + # here, rather than invoking the runner which mutates process environment. + if not result.get("compression"): + result["compression"] = conf.get(f"COMPRESSION_{tu}", "lz4") + retention = result.get("retention", {}) + if not isinstance(retention, dict): + _fail("invalid_retention", source) + retention = deepcopy(retention) + if any(not isinstance(value, str) for value in retention.values()): + # Numeric zero follows a different truthiness path in the legacy + # runner. Neither silently filling defaults nor guessing intent is safe. + _fail("ambiguous_retention_shape", source) + for key, default in {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}.items(): + if not str(retention.get(key) or "").strip(): + retention[key] = conf.get(f"RETENTION_{tu}_{key.upper()}", default) + result["retention"] = retention + for kind in ("docker", "vm"): + key = kind + "_control" + if key not in result: + features = result.get("features", {}) + if not isinstance(features, dict) or type(features.get(kind, False)) is not bool: + _fail("invalid_runtime_control", source) + result[key] = {"mode": "all" if features.get(kind) else "none", "selected": [], + "ack_appdata_risk" if kind == "docker" else "ack_domains_risk": False} + elif not isinstance(result[key], dict): + _fail("invalid_runtime_control", source) + return result + + +def _plan_jobs(scan, jobs_dir, conf, allocator, journal): + jobs, aliases, sources, seen_legacy, canonical_ids = {}, {}, {}, set(), set() + rows = [(path, row["data"]) for path, row in sorted(scan.records.items()) if row["kind"] == "job"] + for source, meta in rows: + _validate_job(meta, source) + if meta["schema_version"] == 4: + job_id = meta["job_id"] + if job_id in canonical_ids: + _fail("duplicate_job_id", source) + canonical_ids.add(job_id) + for source, meta in rows: + legacy = meta["schema_version"] != 4 + if legacy: + key = meta["job_key"] + if any(alias != key for alias in meta.get("legacy_job_keys", [])) and journal is None: + _fail("unproven_legacy_alias", source) + if key in seen_legacy: + _fail("duplicate_legacy_identity", source) + seen_legacy.add(key) + proposed = journal.get("id_map", {}).get(key) if journal else None + if proposed is None: + try: + proposed = str(allocator()) + except Exception: + _fail("uuid_allocation_failed", source) + if not _uuid(proposed) or proposed in jobs or proposed in canonical_ids: + _fail("duplicate_job_id" if _uuid(proposed) else "invalid_job_id", source) + job_id = proposed + else: + job_id = meta["job_id"] + if Path(source) != jobs_dir / (job_id + ".json"): + _fail("noncanonical_metadata_filename", source) + target = _operational_defaults(meta, conf, source) + target["archive_prefixes"] = _prefixes(meta, legacy, source) + target["legacy_job_keys"] = list(dict.fromkeys(([meta["job_key"]] if legacy else []) + meta.get("legacy_job_keys", []))) + for alias in target["legacy_job_keys"]: + if alias in aliases: + _fail("duplicate_legacy_alias", source) + aliases[alias] = job_id + for key in _LEGACY: + target.pop(key, None) + target.update(schema_version=4, job_id=job_id) + jobs[job_id], sources[job_id] = target, source + if seen_legacy and canonical_ids: + # A mixed on-disk cutover is not a new installation to re-plan. The + # original persisted mapping/snapshot is required, even if aliases + # could currently resolve all remaining references. + _fail("partial_migration_without_journal") + return jobs, aliases, sources, bool(seen_legacy) + + +def _check_repositories(scan, jobs, aliases): + def collection(kind, name, key): + selected = [r["data"] for r in scan.records.values() if r["kind"] == kind] + if not selected: + return {} + raw = selected[0] + if type(raw.get("schema_version")) is not int or raw["schema_version"] != 1 or not isinstance(raw.get(name), list): + _fail("unsupported_store_schema") + result = {} + for row in raw[name]: + if not isinstance(row, dict) or not isinstance(row.get(key), str) or not _KEY.fullmatch(row[key]): + _fail("invalid_inventory_entry") + if row[key] in result: + _fail("duplicate_inventory_key") + result[row[key]] = row + return result + repos = collection("repositories", "repositories", "repository_key") + storages = collection("storages", "storages", "storage_key") + for job in jobs.values(): + if job["repository_key"] not in repos: + _fail("dangling_repository") + for key, repo in repos.items(): + if repo.get("storage_key") not in storages: + _fail("dangling_storage") + expected = {job_id for job_id, job in jobs.items() if job["repository_key"] == key} + for field in ("used_by", "source_job_keys", "job_ids", "source_job_ids"): + if field not in repo: + continue + values = repo[field] + if not isinstance(values, list) or any(not isinstance(v, str) for v in values): + _fail("invalid_repository_assignments") + mapped = [value if value in jobs else aliases.get(value) for value in values] + if None in mapped or set(mapped) != expected or len(set(mapped)) != len(mapped): + _fail("conflicting_repository_assignments") + for field in ("passphrase_ref", "keyfile_ref"): + if repo.get(field): + path = _path(repo[field]) + # Existence/type only; never read secret contents into the plan. + for parent in [*reversed(path.parents), path]: + if parent.is_symlink(): + _fail("unsafe_secret_reference") + try: + info = path.stat() + except OSError: + _fail("missing_secret_reference") + if not stat.S_ISREG(info.st_mode): + _fail("unsafe_secret_reference") + # Delimiter overlap also matters (p-* includes p-long-*). + ownership = [] + for job_id, job in jobs.items(): + for prefix in job["archive_prefixes"]: + for other_repo, other_prefix, other_id in ownership: + if other_repo == job["repository_key"] and other_id != job_id and ( + prefix == other_prefix or prefix.startswith(other_prefix + "-") or other_prefix.startswith(prefix + "-") + ): + _fail("ambiguous_archive_ownership") + ownership.append((job["repository_key"], prefix, job_id)) + + +def _check_live_owners(scan): + for source, record in scan.records.items(): + data = record["data"] + rows = data.get("entries", []) if record["kind"] == "runtime_recovery" else [data] + if record["kind"] not in {"runtime_recovery", "resource_lock", "control"}: + continue + if not isinstance(rows, list): + _fail("invalid_runtime_state", source) + for row in rows: + if not isinstance(row, dict): + _fail("invalid_runtime_state", source) + if row.get("finished") is True: + continue + pid = row.get("pid") + if pid is None: + continue # Shape/identity validation belongs to the projector. + if type(pid) is not int or pid <= 0: + _fail("invalid_runtime_pid", source) + try: + os.kill(pid, 0) + except ProcessLookupError: + continue + except (PermissionError, OSError): + _fail("writers_not_quiescent", source) + else: + _fail("writers_not_quiescent", source) + + +def _changed_active(scan, projected): + for record in scan.records.values(): + kind, data = record["kind"], record["data"] + if kind == "schedules" and any(key != "restore_test" and not _uuid(key) for key in data): + return True + if kind == "repositories" and any("used_by" in r or "source_job_keys" in r for r in data.get("repositories", [])): + return True + if projected.get("required"): + return True + for binding in projected["bindings"]: + if binding["job_id"] is None: + continue + record = scan.records[binding["source"]] + if record["kind"] in {"schedules", "repositories", "notification_state"}: + continue + row = record["data"] + for part in binding["locator"].split("/")[1:]: + part = part.replace("~1", "/").replace("~0", "~") + row = row[int(part)] if isinstance(row, list) else row[part] + if not isinstance(row, dict) or row.get("job_id") != binding["job_id"]: + return True + if binding["role"] == "active" and "job_key" in row: + return True + if record["kind"] == "restore_test" and Path(binding["source"]).stem != binding["job_id"]: + return True + return False + + +def encode_target_json(value): + """Frozen planned JSON encoding; the future applier must use these bytes.""" + return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=True, allow_nan=False) + "\n").encode("utf-8") + + +def _resume_check(scan, journal): + if journal is None: + return None + try: + saved = storage.seal_plan(journal) + if saved.get("plan_id") != journal.get("plan_id") or saved.get("classification") != "applicable": + _fail("invalid_migration_journal") + # Check the COMPLETE saved footprint, not just still-discoverable files. + # A vanished source is only a valid retirement when its exact canonical + # replacement is present. JSON-equivalent edits/chmod are not accepted. + replacements = {a["target"]: a.get("after") for a in saved.get("actions", []) + if a.get("kind") == "write_json"} + retired = {a["source"]: a["target"] for a in saved.get("actions", []) if a.get("kind") == "retire_source"} + actual_inputs = {path: storage.fingerprint_file(path) for path in saved["inputs"]} + if set(scan.inputs) - set(saved["inputs"]): + _fail("source_fingerprint_changed") + for path, actual in actual_inputs.items(): + if actual == saved["inputs"][path]: + continue + if (not actual["exists"] and path in retired + and actual_inputs.get(retired[path]) == replacements.get(retired[path]) + and replacements.get(retired[path]) is not None): + continue + if path in replacements and actual == replacements[path]: + continue + _fail("source_fingerprint_changed", path) + current_groups = {g["path"]: g for g in scan.groups} + if set(current_groups) != {g["path"] for g in saved["inventory_groups"]}: + _fail("source_fingerprint_changed") + for group in saved["inventory_groups"]: + current = current_groups[group["path"]] + if group.get("kind") == "directories": + if current != group: + _fail("source_fingerprint_changed", group["path"]) + continue + if {k: v for k, v in current.items() if k != "entries"} != {k: v for k, v in group.items() if k != "entries"}: + _fail("source_fingerprint_changed", group["path"]) + expected_names = {Path(path).name for path, fp in actual_inputs.items() + if str(Path(path).parent) == group["path"] and fp["exists"] + and Path(path).name.endswith(tuple(group["suffixes"]))} + if set(current["entries"]) != expected_names: + _fail("source_fingerprint_changed", group["path"]) + return saved + except storage.IdentityStorageError: + _fail("invalid_migration_journal") + + +def build_plan(config, *, uuid_factory=uuid4, journal_plan=None, control_root=None, cron_text=None): + """Return a proposed complete mapping without changing any installation file. + + Pass a plan loaded/validated from the private journal to reuse allocated + IDs. Fresh dry runs propose IDs; only persist_plan makes that mapping + durable. No caller-supplied boolean can authorize application here. + """ + scan = None + try: + scan, data, jobs_dir, conf = _inventory(config, control_root) + journal = _resume_check(scan, journal_plan) + if journal is not None: + # Secret contents are not migration inputs, but their referenced + # existence/type must still be checked on a resumed plan. + _check_repositories(scan, journal["jobs"], journal["aliases"]) + _check_live_owners(scan) + # Keep the original plan ID, complete original snapshot footprint, + # and UUID map. Do not re-plan from half-converted stores or produce + # a new snapshot of already converted data after an interruption. + return journal + jobs, aliases, sources, legacy = _plan_jobs(scan, jobs_dir, conf, uuid_factory, journal) + _check_repositories(scan, jobs, aliases) + _check_live_owners(scan) + records = {p: deepcopy(r) for p, r in scan.records.items() if r["kind"] not in {"job", "storages"}} + for path, row in records.items(): + if row["kind"] == "restore_test": + key = Path(path).stem + row["legacy_key"] = key + job_id = key if key in jobs else aliases.get(key) + if job_id: + row["target_path"] = str(Path(path).with_name(job_id + ".test")) + projected = project_records(records, jobs, aliases) + reasons = projected.get("reasons", []) + fatal = [r for r in reasons if r.get("severity") != "warning" and r["code"] != "weekly_value_conflict_preserved"] + if fatal: + return _blocked(fatal, scan) + mutable_active = _changed_active(scan, projected) + if jobs and not legacy and mutable_active: + _fail("partial_migration_without_journal") + required = legacy or mutable_active or bool(journal) + actions = [] + destinations = {} + def write(source, target, payload): + target = str(target) + existing = destinations.get(target) + if existing is not None and existing != payload: + _fail("conflicting_destination", target) + destinations[target] = payload + scan.file(target) + if source != target and scan.inputs[target]["exists"]: + if target not in scan.records or scan.records[target]["data"] != payload: + _fail("destination_already_exists", target) + if source == target and source in scan.records and scan.records[source]["data"] == payload: + return + encoded = encode_target_json(payload) + mode = scan.inputs[target].get("mode", scan.inputs[source].get("mode", 0o600)) + after = {"exists": True, "size": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(), "mode": mode} + actions.append({"kind": "write_json", "source": source, "target": target, + "data": payload, "after": after}) + if required: + for job_id, job in jobs.items(): + source = sources[job_id] + target = str(jobs_dir / (job_id + ".json")) + write(source, target, job) + if source != target: + actions.append({"kind": "retire_source", "source": source, "target": target}) + for target, row in projected["records"].items(): + origins = row.get("sources", [target]) + # Projector retains explicit source metadata on renamed records. + source = target if target in origins else row.get("source", origins[0]) + if source not in scan.inputs: + source = next((p for p, r in records.items() if r.get("target_path") == target), target) + if row["kind"] == "widget_cache": + actions.append({"kind": "rebuild_derived", "source": source, "target": target}) + continue + write(source, target, row["data"]) + for old in origins: + if old != target and old in scan.inputs: + actions.append({"kind": "retire_source", "source": old, "target": target}) + if source != target and source not in origins: + actions.append({"kind": "retire_source", "source": source, "target": target}) + # Derived caches deliberately have no projected payload: rebuilding + # must use the final verified graph, not copy old display keys. + for path, row in records.items(): + if row["kind"] == "widget_cache" and path not in projected["records"]: + actions.append({"kind": "rebuild_derived", "source": path, "target": path}) + for action in actions: + action["id"] = hashlib.sha256(json.dumps(action, sort_keys=True).encode()).hexdigest() + plan = { + "schema_version": 1, "migration_id": MIGRATION_ID, + "classification": "applicable" if required else "not_applicable", + "status": "pending" if required else "not_applicable", "required": bool(required), + "jobs": jobs, "aliases": aliases, "id_map": aliases, "job_sources": sources, + "inputs": scan.inputs, "inventory_groups": scan.groups, + "actions": actions, "records": projected["records"], + "bindings": projected.get("bindings", []), "unassigned": projected.get("unassigned", []), + "reasons": reasons + ([{"code": "resume_existing_mapping", "severity": "warning", "source": "", "locator": ""}] if journal else []), + "prerequisites": {"managed_cron_captured": isinstance(cron_text, str)}, + "external_inputs": {"managed_cron": {"kind": "crontab", "text": cron_text}} if isinstance(cron_text, str) else {}, + "activation_allowed": False, + } + # End-of-scan revalidation catches concurrent edits/additions, including + # destinations that were previously absent. Still no writer permission. + plan = storage.seal_plan(plan) + storage.verify_inputs(plan) + return plan + except PlanningError as exc: + return _blocked([{"code": exc.code, "source": exc.source, "locator": ""}], scan) + except storage.IdentityStorageError as exc: + return _blocked([{"code": exc.code, "source": "", "locator": ""}], scan) + except (OSError, ValueError, TypeError, KeyError, OverflowError, RecursionError): + return _blocked([{"code": "invalid_owned_state", "source": "", "locator": ""}], scan) + + +def _blocked(reasons, scan=None): + return {"migration_id": MIGRATION_ID, "required": True, "classification": "blocked", + "status": "blocked", "jobs": {}, "aliases": {}, "id_map": {}, "actions": [], + "records": {}, "bindings": [], "unassigned": [], "reasons": reasons, + "activation_allowed": False} + + +def detect(config, *, control_root=None): + """Runner-shaped read-only summary. Blocked never means required=False.""" + plan = build_plan(config, control_root=control_root) + return {key: plan[key] for key in ("migration_id", "required", "classification", "status", "reasons")} + + +def verify_target(config, *, control_root=None): + """Read actual target files, never authorize services from a proposed plan.""" + plan = build_plan(config, control_root=control_root) + reasons = list(plan.get("reasons", [])) + if plan["classification"] != "not_applicable": + reasons.append({"code": "identity_cutover_incomplete", "source": "", "locator": ""}) + else: + try: + # Verify source records, not replacements that could conceal a + # stale FK. Both scans must represent the same complete graph. + scan, _, _, _ = _inventory(config, control_root) + if scan.inputs != plan["inputs"] or scan.groups != plan["inventory_groups"]: + _fail("source_fingerprint_changed") + records = {p: r for p, r in scan.records.items() if r["kind"] not in {"job", "storages"}} + for path, row in records.items(): + if row["kind"] == "restore_test": + row["legacy_key"] = Path(path).stem + reasons.extend(verify_records(records, plan["jobs"], plan["aliases"])) + storage.verify_inputs(plan) + except PlanningError as exc: + reasons.append({"code": exc.code, "source": exc.source, "locator": ""}) + except storage.IdentityStorageError as exc: + reasons.append({"code": exc.code, "source": "", "locator": ""}) + except (OSError, ValueError, TypeError, KeyError, OverflowError, RecursionError): + reasons.append({"code": "invalid_owned_state", "source": "", "locator": ""}) + fatal = [r for r in reasons if r.get("severity") != "warning" or r["code"] == "widget_rebuild_required"] + return {"valid": not fatal, "reasons": reasons, "writable_services_allowed": False, + "activation_allowed": False, "migration_id": MIGRATION_ID} diff --git a/docs/changelog.md b/docs/changelog.md index e348cd9c..14958826 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,6 +6,14 @@ Das Plugin-Manifest `borg-backup-ui.plg` enthaelt nur noch eine kurze nutzerrele ## Unreleased +### Issue #472 (integration work for #447; not released) +- Added an inactive, read-only migration inventory and UUID planner with exact + legacy-reference mapping, archive ownership checks and preserved history. +- Added private persisted plans, checksummed snapshots, a chained audit journal + and fail-closed preconditions. Retries retain the original UUID allocation. +- Added real-file fixture, failure and integrity tests. Startup registration, + installation-data conversion and test-channel publication remain deferred. + ### Issue #471 (integration work for #447; not released) - Defined the immutable job identity, migration safety and legacy-data contract. - Added a source dependency inventory and synthetic migration fixtures with diff --git a/docs/maintainer/identity-dependencies.json b/docs/maintainer/identity-dependencies.json index 92090b3f..3a0ae816 100644 --- a/docs/maintainer/identity-dependencies.json +++ b/docs/maintainer/identity-dependencies.json @@ -29,6 +29,21 @@ ], "target": "Direct read-only inventory, plan, snapshot, durable mapping/journal, resume and integrity verification; pending approval blocks writers.", "files": [ + { + "path": "api/migrations/immutable_job_id_v1.py", + "anchor": "build_plan", + "role": "inactive direct inventory, UUID proposal, exact resume and target verification; never registered for startup" + }, + { + "path": "api/migrations/identity_records.py", + "anchor": "project_records", + "role": "pure persisted-record projection and independent reference validation; no filesystem writes" + }, + { + "path": "api/migrations/identity_storage.py", + "anchor": "persist_plan", + "role": "private immutable plan, verified byte snapshot and audit journal; no installation-data applier" + }, { "path": "api/migrations/registry.py", "anchor": "run_startup_migrations", diff --git a/docs/maintainer/identity-migration-foundation.md b/docs/maintainer/identity-migration-foundation.md new file mode 100644 index 00000000..d79a99b4 --- /dev/null +++ b/docs/maintainer/identity-migration-foundation.md @@ -0,0 +1,209 @@ +# Inactive identity-migration foundation + +Issue #472, phase 2/9 of #447. This documents the implemented planning and +private recovery-state primitives, not an installable migration. The binding +target model remains the [immutable job identity contract](immutable-job-identity.md). + +## Current boundary + +The implementation is deliberately absent from the migration registry, +application startup and HTTP routes. It has no `apply()` function and performs +no installation-data conversion, cron update, Borg operation or background +service activation. Calling a planner or verifier never grants permission to +start writers. Separate explicit storage calls may write only their dedicated +private migration-state directory. + +Phases #473-#478 still own the application consumers and writers. #479 must +provide the complete maintenance, confirmation, execution and final-verification +coordinator. There is no test-channel candidate for this intermediate phase; +the first candidate remains gated on the complete, testable cutover in #479. +The verifier is an identity/reference verifier, not a replacement for the full +job-options validation still owned by #473. Settings are preserved; the later +coordinator must combine both validators before allowing normal operation. + +## APIs and responsibilities + +| Module / entry point | Responsibility and limitation | +| --- | --- | +| `api/migrations/immutable_job_id_v1.py`: `build_plan(config, *, uuid_factory, journal_plan, control_root, cron_text)` | Read owned installation stores, validate identities and dependencies, and return a proposed plan. Optional arguments have defaults; tests inject UUIDs and isolated roots. `journal_plan` is the original validated persisted plan, not an arbitrary repair map. | +| `detect(config, *, control_root)` | Read-only, runner-shaped summary with boolean `required`, classification, execution status and reason codes. It is not registered with the runner. A blocked inventory never reports `required=False`. | +| `verify_target(config, *, control_root)` | Re-read actual on-disk stores and check canonical cutover integrity. It does not verify merely the proposed replacement objects. Even a valid result keeps `activation_allowed` and `writable_services_allowed` false. | +| `api/migrations/identity_records.py`: `project_records(records, jobs, aliases)` | Deterministic, no-I/O projection of dependent records. Preserve original historical evidence; block unresolved or conflicting active ownership. | +| `verify_records(records, jobs, aliases)` | Validate actual target references without repairing active mutable keys through aliases. Filesystem ownership and snapshot-byte checks remain separate responsibilities. | +| `api/migrations/identity_storage.py`: `seal_plan`, `persist_plan`, `load_plan` | Validate and content-hash a complete plan; durably publish and reload the same proposed allocation without overwriting a conflicting plan. | +| `create_snapshot`, `verify_snapshot`, `verify_inputs` | Copy and independently verify the exact captured originals; recheck file fingerprints and immediate directory inventories. No restoration or production replacement is performed. | +| `verify_preconditions` | Default-deny library gate requiring the bound confirmation, verified snapshot, unchanged inputs, explicit quiescence check and external-input recheck. It is not an authenticated UI or an execution transaction. | +| `append_journal`, `read_journal` | Maintain a private, sequenced, hash-linked JSONL record with fixed statuses, phases, reason codes and known action IDs. No free-form exception or payload text is accepted as a journal reason. | + +These are internal Python APIs for the staged implementation and automated +tests. They are not user-facing maintenance commands or instructions to run +against a production installation now. + +## Read-only inventory and proposed plan + +The scanner derives configured roots and reads only the owned stores listed +in contract C5, including both known weekly-snapshot locations and the exact +known legacy job locations. It records immediate directory membership as well +as individual files; a newly added file or disappeared input cannot be hidden +by checking only objects that remain discoverable. + +Reads reject symlinks, unsafe paths, non-regular matching inputs, malformed +JSON, duplicate JSON members, unsupported schemas and inconsistent ownership. +Known Unraid mount paths are checked for availability. The scanner does not +call application discovery, lazy migration, orphan-schedule cleanup, weekly +auto-write or repair helpers. Logs, unrelated nested files, recycle bins, +runtime/vendor contents and Borg repository/cache data are not recursively +inventoried or converted. + +A supported plan contains the full UUIDv4 job map, canonical proposed jobs, +exact aliases, record bindings, preserved unassigned history, input +fingerprints, directory inventories and proposed actions. `write_json` +actions include the exact expected destination bytes' fingerprint; +`retire_source` identifies only an individual superseded source and its +canonical replacement. These are descriptions, not executed writes. + +`build_plan` may propose fresh UUIDs in memory. The allocation becomes durable +only after explicit `persist_plan`. The full plan receives a `plan_id` +content digest; it is not a cryptographic signature or an administrator's +approval. An applicable plan remains `pending`. An empty or completely valid +canonical installation is `not_applicable`; uncertain or unsupported states +are `blocked` with no proposed production actions. + +The foundation preserves stopped runtime-recovery targets, notification retry +and reminder state, independent restore IDs and original history descriptors. +Historical UUIDs belonging to deleted jobs remain in the records while their +bindings to the active job graph remain unassigned. Weekly observations retain +source provenance: equal observations are deduplicated and conflicting values +remain explicit. Restore-history index/detail pairs must agree; the planner +does not invent missing peers or silently select one contradictory record. + +## Explicit private state directory + +The caller must supply `state_dir`. No production default is selected and no +automatic fallback to another directory or weaker permissions is implemented. +It must be outside the plan's owned input files and inventory roots, on an +available **persistent filesystem supporting private POSIX permissions and +hard links**. The process must own directories with mode `0700` and stored +files with mode `0600`. + +The Unraid FAT `/boot` USB filesystem is unsuitable for this private state +store. This does not prohibit reading job/configuration originals on `/boot`; +it means the protected plan, snapshot and journal must live on a suitable +separate filesystem. Lack of required permission or publication semantics +blocks storage; the code does not silently relax them. The later coordinator +must select and validate a persistent mounted location and keep it available +through the entire migration. Library calls alone do not prove persistence or +that every required mount will remain available. + +The private layout is: + +```text +state_dir/ + plan.json + journal.jsonl + snapshot/ + metadata.json + manifest.json + files/ + .bin + external-managed_cron.bin +``` + +The journal exists only after an explicit append. The external cron blob exists +only for a supplied capture. Publication uses private staging files, flushes +and exclusive final-name publication without overwriting a conflicting file. +`metadata.json` durably records the snapshot's UTC creation time before blob +copying; retries retain that time rather than inventing a new snapshot age. +The manifest binds it to the original plan, complete ID map and ordered action +summaries, and distinguishes file artifacts from externally captured cron. +Action summaries contain IDs, kinds and source/target paths, not replacement +payloads; the overall snapshot still remains private and potentially sensitive. +Snapshot verification checks the exact expected blob set, sizes, hashes and +binding to the persisted plan; existence alone is not sufficient. + +Treat **the entire state directory as secret-bearing**. Snapshot originals can +contain raw, secret-bearing configuration and notification content. Plan payloads +can also contain sensitive settings, paths, messages or fields retained from +existing records. Private file permissions are not encryption. Never attach +these files to public issues, logs, normal support packages or an unprotected +download. Only sanitized reason codes and deliberately selected metadata +belong in public diagnostics. A secure export/download design remains future +coordinator work; no such endpoint exists in this phase. + +## Confirmation, quiescence and cron + +`verify_preconditions` denies by default. It requires an applicable pending +plan, a verified snapshot, an explicit approval tied to `plan_id` and the +snapshot digest, and acknowledgement that an independent backup is required. +Acknowledgement is not proof that an external copy was actually made. + +The gate also requires a supplied quiescence callback returning exactly `True`. +Missing callbacks, failures or negative results block. Existing owner checks +in the scanner are useful evidence, but are not a complete exclusion barrier +against detached workers or new concurrent activity. #479 must authenticate +the administrator, enter maintenance, prevent all writers and retain the +necessary exclusion across snapshot, apply and verification. It must cover +backup/restore/test workers, notification delivery, scheduler, cleanup, +widgets and other write-capable background activity without killing a live +job to force progress. + +The planner never executes `crontab`. A future coordinator must capture actual +cron text and supply `cron_text`; omitting it does not mean the crontab is +empty. A plan without the capture may be inspected, but cannot pass snapshot +and execution preconditions. A genuinely empty captured crontab is supplied +explicitly as an empty string. The captured bytes are included in the private +snapshot, and an external-input callback must return the same capture before +the gate passes. Rebuilding only the managed cron section, preserving unrelated +entries, remains the final execution boundary in #479. + +## Interruption and resume boundaries + +A resume loads the original sealed plan and reuses its complete UUID map, +plan ID, original snapshot footprint and expected actions. It must not allocate +fresh identities or snapshot partially converted data as a new starting point. +The scanner checks every saved input, including files that have disappeared. + +Each observed file must match its exact original fingerprint or its plan's +exact expected post-replacement fingerprint. A superseded source may be +absent only when the matching canonical destination exists with the expected +replacement fingerprint. Equivalent parsed JSON is insufficient: unexplained +byte changes or permission changes block. Directory membership and unrelated +inputs are revalidated as well. Missing both a source and its replacement is +not a completed retirement. +Mixed legacy/UUID job files require the original plan even when their current +references could be resolved. Referenced repository secret files are checked +again for existence, regular-file type and symlinks, without reading their +contents into the plan. + +Snapshot creation can resume between completely published blobs when the +persisted plan, originals and completed blobs are unchanged. A crash during +publication may instead leave an incomplete staging/publication state, +unexpected snapshot file, inconsistent link count or truncated journal. +These conditions block verification and require explicit diagnosis; no +automatic cleanup of questionable recovery evidence is provided. A malformed +or truncated journal is not silently repaired or treated as an empty log. + +There is no production apply engine, automatic rollback or snapshot-restore +operation in #472. The eventual cross-filesystem migration must remain +crash-consistent and resumable, not claim a single atomic filesystem +transaction. Journal entries alone do not demonstrate that installation data +was converted or verified. Data recovery is distinct from downgrading the +installed Unraid plugin; the existing rollback limitations still apply. + +## Verification and remaining work + +The tests now exercise the actual read-only planner with the phase-1 synthetic +fixtures, pure record projection, default-deny gates and private snapshot and +journal primitives. Negative cases cover malformed/ambiguous inputs, changed +sources, partial states, denied permissions, snapshot corruption and preserved +historical identity. They do not exercise a production apply path because none +exists yet. + +The final coordinator must integrate the existing migration runner's pending +and maintenance behavior, authenticate confirmation, supply live cron and +writer checks, perform and journal the replacements, handle protected snapshot +retention/export, and verify complete startup and UI behavior before the first +test candidate. Passing this phase does not assert coverage of every existing +installation or guarantee migration success on unknown states. Unknown states +must continue to block before modification and gain explicit supported +fixtures when their semantics are understood. diff --git a/docs/maintainer/immutable-job-identity.md b/docs/maintainer/immutable-job-identity.md index 2599f840..e1cf9b11 100644 --- a/docs/maintainer/immutable-job-identity.md +++ b/docs/maintainer/immutable-job-identity.md @@ -311,6 +311,14 @@ These are **not migration execution tests**. #472 must run its real detector and planner against these inputs, and #479 must test actual on-disk results, interruption/resume, no-write blocking and UI/HTTP maintenance behavior. +Phase #472 now provides the inactive planner, dependent-record verification +and private plan/snapshot/journal primitives described in +[Identity-migration foundation](identity-migration-foundation.md). Its tests +exercise actual planning and private recovery-state operations against the +synthetic inputs. No production apply engine, startup registration or installable +test candidate is enabled; the execution and integration gates above remain +owned by #479. + Before the first candidate: cover each journal write/rename boundary, stale source fingerprints, unavailable mounts, disk/permission failures, live workers, same-second runs, renamed jobs, partial imports, rejected corrupt diff --git a/tests/fixtures/immutable_job_id_v1/README.md b/tests/fixtures/immutable_job_id_v1/README.md index bcf5a8b0..61b25949 100644 --- a/tests/fixtures/immutable_job_id_v1/README.md +++ b/tests/fixtures/immutable_job_id_v1/README.md @@ -12,12 +12,13 @@ in #447. They contain no repository data, credentials or real user paths. - `cases.json` overlays whole files on that base: `{ "json": ... }` serializes JSON; `{ "text": ... }` preserves exact text, including malformed JSON. `null` removes a base file. There is no implicit recursive object merge. -- `allocation_order` injects deterministic UUIDv4 values into future planner +- `allocation_order` injects deterministic UUIDv4 values into planner tests. Production must generate random UUIDs once and persist the mapping. - `preconditions` describes controlled test hooks, not a production journal format. In particular the journal case models a durable mapping and one completed replacement, **not** a real verified snapshot or real journal - bytes. #472 must supply its real journal and filesystem failure adapters. + bytes. The #472 integration tests supply real persisted-plan, snapshot, + journal and filesystem failure adapters rather than trusting these flags. - `/fixture/` is relocated by the test materializer to an isolated directory under repository-local `.release-tmp/`. No Borg commands, network calls, crontab updates, actual process checks or production reads are performed. @@ -65,8 +66,12 @@ assertions themselves. The materialization self-check passes an expected observation to the assertion only to validate the oracle; **it is not evidence of an implemented or successful migration**. -Phase #472 must execute the actual read-only detector/planner on each input, -inject the UUID allocator and model real source fingerprints/journal actions. +Phase #472 runs the actual read-only detector/planner on materialized inputs, +injects the UUID allocator and tests real source fingerprints/journal actions. +The four fixture scenarios requiring live-writer, changed-source, snapshot or +interruption evidence use separate real-state tests; fixture booleans cannot +authorize a migration. See `tests/test_identity_planner.py`, +`tests/test_identity_records.py` and `tests/test_identity_storage.py`. Phase #479 must execute the real migration and read back destination files, test every interruption boundary, retry/resume, missing mounts/space, permissions/symlinks, all live writers and the administrator gate. Extend @@ -85,6 +90,7 @@ Run focused validation from the repository root: ```bash python -m pytest -q tests/test_immutable_job_identity_contract.py +python -m pytest -q tests/test_identity_planner.py tests/test_identity_records.py tests/test_identity_storage.py ``` No test-channel package or stable release is created in this phase. diff --git a/tests/test_identity_planner.py b/tests/test_identity_planner.py new file mode 100644 index 00000000..a41d5b76 --- /dev/null +++ b/tests/test_identity_planner.py @@ -0,0 +1,683 @@ +"""#472: exercise the inactive planner against real synthetic files. + +The phase-1 execution goldens describe the eventual phase-9 cutover. Planning +never performs that cutover, even when a fixture models future confirmation. +""" + +from copy import deepcopy +import json +import os +from pathlib import Path +import sys +from tempfile import TemporaryDirectory +from uuid import UUID + +import pytest + +from identity_contract_support import ROOT, load_cases, materialize, source_value, tree_bytes + +API_ROOT = ROOT / "api" +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +from migrations import immutable_job_id_v1 as migration # noqa: E402 +from migrations import identity_storage as storage # noqa: E402 +from migrations import registry # noqa: E402 + + +CASES = load_cases() +BY_ID = {case["id"]: case for case in CASES} +# These hooks did not supply real OS/snapshot/journal bytes in phase 1. +# Their safety boundaries are tested separately, not pretended to be inputs +# from which an initial read-only scan could infer administrator consent. +GATE_ONLY_CASES = { + "live_writer", "source_changed_after_plan", "snapshot_unverified", + "interrupted_with_journal", +} + + +@pytest.fixture +def identity_root(): + parent = ROOT / ".release-tmp" + parent.mkdir(exist_ok=True) + with TemporaryDirectory(prefix="identity-472-planner-", dir=parent) as directory: + yield Path(directory) + + +def installation(case, root): + installed = root / "installation" + relocated = materialize(case, installed) + # Never inspect the host's live control/widget paths from a synthetic case. + relocated["config"]["PLUGIN_DIR"] = str(installed / "plugin") + return installed, relocated + + +def plan_for(case, installed): + values = iter(case["allocation_order"]) + return migration.build_plan( + case["config"], uuid_factory=lambda: UUID(next(values)), + control_root=installed / "run", + ) + + +def reason_codes(result): + return {item["code"] if isinstance(item, dict) else item + for item in result.get("reasons", [])} + + +def binding_projection(plan, installed): + result = {} + for item in plan.get("bindings", []): + source = Path(item["source"]).relative_to(installed).as_posix() + reference = source + "#" + item["locator"] + assert reference not in result, "multiple contradictory binding outcomes" + result[reference] = item["job_id"] + return result + + +def _pointer(value, pointer): + for part in pointer.split("/")[1:]: + part = part.replace("~1", "/").replace("~0", "~") + value = value[int(part)] if isinstance(value, list) else value[part] + return value + + +def _planned_value(plan, installed, reference): + """Read a destination object independently; never echo a source golden.""" + name, _, pointer = reference.partition("#") + source = str(installed / name) + targets = [record for target, record in plan["records"].items() + if source == str(target) or source in record.get("sources", [])] + assert len(targets) == 1, f"source lacks one destination: {name}" + data = targets[0]["data"] + if "observations" in data and "weekly" in name: + parts = pointer.split("/") + row_pointer = "/".join(parts[:3]) + observations = [row for row in data["observations"] + if {"source": source, "locator": row_pointer} + in row["source_records"]] + assert len(observations) == 1 + return _pointer(observations[0], "/" + "/".join(parts[3:])) if len(parts) > 3 else observations[0] + if name == "data/config/schedules.json": + prefix = pointer.split("/", 2)[1] + matching = [binding for binding in plan["bindings"] + if binding["source"] == source and binding["locator"] == "/" + prefix] + if matching: + pointer = "/" + matching[0]["job_id"] + pointer[len(prefix) + 1:] + if name == "data/config/notification-state.json" and pointer.startswith("/last_sent/"): + matching = [binding for binding in plan["bindings"] + if binding["source"] == source and binding["locator"] == pointer] + if matching and matching[0]["job_id"]: + event, old_key, due = pointer.removeprefix("/last_sent/").split(":", 2) + pointer = f"/last_sent/{event}:{matching[0]['job_id']}:{due}" + return _pointer(data, pointer) + + +def _assert_contains_original(actual, expected): + if isinstance(expected, dict): + assert isinstance(actual, dict) + for key, value in expected.items(): + assert key in actual, f"preserved member removed: {key}" + _assert_contains_original(actual[key], value) + elif isinstance(expected, list): + assert isinstance(actual, list) and len(actual) == len(expected) + for observed, value in zip(actual, expected): + _assert_contains_original(observed, value) + else: + assert actual == expected + + +@pytest.mark.parametrize("case", [c for c in CASES if c["id"] not in GATE_ONLY_CASES], + ids=lambda c: c["id"]) +def test_read_only_planner_matches_phase1_identity_and_binding_goldens(case, identity_root): + installed, relocated = installation(case, identity_root) + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + assert tree_bytes(installed) == before, "planning wrote installation data" + expected = relocated["expected"] + assert plan["classification"] == expected["classification"] + assert plan["jobs"] == expected["jobs"] + assert type(plan["required"]) is bool + assert plan["required"] is (plan["classification"] != "not_applicable") + assert plan["status"] == { + "applicable": "pending", "blocked": "blocked", + "not_applicable": "not_applicable", + }[plan["classification"]] + assert set(expected["reason_codes"]) <= reason_codes(plan) + if plan["classification"] != "blocked": + assert binding_projection(plan, installed) == expected["bindings"] + for reference, value in expected["preserved"].items(): + _assert_contains_original(_planned_value(plan, installed, reference), value) + for reference, value in expected["bindings"].items(): + if value is None: + _assert_contains_original(_planned_value(plan, installed, reference), + source_value(relocated["files"], reference)) + unassigned = { + Path(row["source"]).relative_to(installed).as_posix() + "#" + row["locator"]: + row["reason"] for row in plan["unassigned"] + } + assert unassigned == expected["unassigned"] + + +@pytest.mark.parametrize("case_id", ["source_changed_after_plan", "snapshot_unverified"]) +def test_fixture_gate_booleans_are_not_misrepresented_as_initial_detection(case_id, identity_root): + installed, relocated = installation(BY_ID[case_id], identity_root) + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + # Both fixtures have the same eligible input bytes as the base case. + # The separate pre-apply verifier must enforce the changed/unverified + # conditions using an actual plan/snapshot, not invisible fixture metadata. + assert plan["classification"] == "applicable" + assert plan["status"] == "pending" + assert tree_bytes(installed) == before + + +def test_planner_not_registered_and_has_no_user_data_apply_entry_point(): + assert migration not in registry.MIGRATIONS + assert all(getattr(item, "MIGRATION_ID", "") != "immutable_job_id_v1" + for item in registry.MIGRATIONS) + assert not hasattr(migration, "apply") + + +@pytest.mark.parametrize("replacement", [ + {"schema_version": True}, + {"schema_version": "3"}, + {"schema_version": 99}, + {"archive_prefixes": [""]}, + {"archive_prefixes": ["config-backup", 2]}, + {"archive_prefixes": ["config-backup", " "]}, + {"archive_prefixes": ["config-backup", "other*"]}, +]) +def test_unknown_or_unsafe_owned_metadata_blocks_without_writes(replacement, identity_root): + case = deepcopy(BY_ID["legacy_without_prefixes"]) + case["files"]["data/config/jobs/config_local.json"]["json"].update(replacement) + installed, relocated = installation(case, identity_root) + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert plan["required"] is True + assert not plan["jobs"] + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("path", [ + "data/config/repositories.json", "data/config/storages.json", + "data/config/schedules.json", "data/config/notification-queue.json", + "data/config/notification-deliveries.json", "data/config/notification-state.json", + "data/config/runtime-recovery.json", "data/config/restore-runs.json", + "status/2026-08-31_08-00-00_config_local.status", + "restore_tests/config_local.test", "weekly-snapshots.json", + "status/weekly-snapshots.json", +]) +def test_corrupt_owned_store_is_not_an_empty_or_ignored_store(path, identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + changed = installed / path + changed.parent.mkdir(parents=True, exist_ok=True) + changed.write_text("{invalid JSON", encoding="utf-8") + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert plan["required"] is True + assert tree_bytes(installed) == before + + +def test_duplicate_json_member_cannot_silently_choose_identity(identity_root): + case = deepcopy(BY_ID["legacy_without_prefixes"]) + job = case["files"]["data/config/jobs/config_local.json"]["json"] + encoded = json.dumps(job) + case["files"]["data/config/jobs/config_local.json"] = { + "text": encoded[:-1] + ', "job_key": "config_local"}', + } + installed, relocated = installation(case, identity_root) + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("path", [ + "data/config/jobs/config_local.json", + "status/2026-08-31_08-00-00_config_local.status", +]) +def test_owned_symlink_is_not_followed(path, identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + source = installed / path + outside = identity_root / "outside-owned-root.json" + source.rename(outside) + source.symlink_to(outside) + before = outside.read_bytes() + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert source.is_symlink() + assert outside.read_bytes() == before + + +def test_symlinked_owned_directory_is_not_followed(identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + source = installed / "data/config/jobs" + outside = identity_root / "outside-owned-jobs" + source.rename(outside) + source.symlink_to(outside, target_is_directory=True) + before = tree_bytes(outside) + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert source.is_symlink() + assert tree_bytes(outside) == before + + +def test_allocator_cannot_reuse_an_id_for_distinct_jobs(identity_root): + installed, relocated = installation(BY_ID["shared_repository_distinct_prefixes"], identity_root) + before = tree_bytes(installed) + duplicate = UUID(relocated["allocation_order"][0]) + plan = migration.build_plan(relocated["config"], uuid_factory=lambda: duplicate, + control_root=installed / "run") + assert plan["classification"] == "blocked" + assert not plan["jobs"] + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("kind", ["resource", "control", "recovery"]) +def test_real_live_owner_blocks_even_when_fixture_claims_quiescence(kind, identity_root): + case = deepcopy(BY_ID["legacy_without_prefixes"]) + if kind == "resource": + case["files"]["locks/fixture.lock.json"] = {"json": { + "schema_version": 1, "job_key": "config_local", "pid": os.getpid(), + "resource": "repo:fixture", "operation": "backup", + "run_id": "fixture-live-run", "started_at": "2026-08-31T08:00:00Z", + "updated_at": "2026-08-31T08:00:00Z", + }} + elif kind == "control": + case["files"]["run/fixture-live-run/state.json"] = {"json": { + "schema_version": 1, "job_key": "config_local", "pid": os.getpid(), + "run_id": "fixture-live-run", "phase": "backup", "finished": False, + }} + else: + case["files"]["data/config/runtime-recovery.json"] = deepcopy( + BY_ID["pending_runtime_recovery"]["files"]["data/config/runtime-recovery.json"] + ) + case["files"]["data/config/runtime-recovery.json"]["json"]["entries"][0]["pid"] = os.getpid() + installed, relocated = installation(case, identity_root) + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert tree_bytes(installed) == before + + +def test_invalid_resource_lock_is_not_assumed_inactive(identity_root): + case = deepcopy(BY_ID["legacy_without_prefixes"]) + case["files"]["locks/fixture.lock.json"] = {"text": "{not readable lock JSON"} + installed, relocated = installation(case, identity_root) + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert tree_bytes(installed) == before + + +def test_owned_fifo_is_rejected_without_reading_or_writing_it(identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + path = installed / "data/config/jobs/other_local.json" + os.mkfifo(path) + original = (installed / "data/config/jobs/config_local.json").read_bytes() + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert (installed / "data/config/jobs/config_local.json").read_bytes() == original + assert path.stat().st_size == 0 + + +@pytest.mark.parametrize("case_id,valid", [ + ("already_migrated", True), + ("partial_without_journal", False), + ("legacy_without_prefixes", False), +]) +def test_target_verifier_reads_actual_files_and_never_activates_writers(case_id, valid, identity_root): + installed, relocated = installation(BY_ID[case_id], identity_root) + before = tree_bytes(installed) + result = migration.verify_target(relocated["config"], control_root=installed / "run") + assert result["valid"] is valid + assert result["writable_services_allowed"] is False + assert tree_bytes(installed) == before + + +def test_ambiguous_numeric_zero_retention_is_blocked_not_silently_reinterpreted(identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + source = installed / "data/config/jobs/config_local.json" + data = json.loads(source.read_text(encoding="utf-8")) + data["retention"] = {"daily": 0, "weekly": 4, "monthly": 0, "yearly": 0} + source.write_text(json.dumps(data), encoding="utf-8") + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + # The legacy runner treats numeric 0 as missing but string "0" as a + # disabled tier. Neither guessing user intent nor silently inserting its + # fallback is a safe migration of this ambiguous, non-wizard shape. + assert plan["classification"] == "blocked" + assert not plan["jobs"] + assert tree_bytes(installed) == before + + +def test_resume_cannot_ignore_disappeared_restore_proof_without_canonical_replacement(identity_root): + installed, relocated = installation(BY_ID["restore_result_and_history"], identity_root) + plan = plan_for(relocated, installed) + assert plan["classification"] == "applicable" + persisted = storage.persist_plan(plan, identity_root / "migration-state") + (installed / "restore_tests/config_local.test").unlink() + before = tree_bytes(installed) + resumed = migration.build_plan(relocated["config"], journal_plan=persisted, + control_root=installed / "run") + assert resumed["classification"] == "blocked" + assert tree_bytes(installed) == before + + +def test_legacy_extra_alias_does_not_authorize_repair_of_reported_orphan_schedule(identity_root): + installed, relocated = installation(BY_ID["reported_config_to_pfsense_orphan_schedule"], identity_root) + source = installed / "data/config/jobs/pfsense_local.json" + data = json.loads(source.read_text(encoding="utf-8")) + # The old product never recorded this field as a migration identity proof. + # A stray field is not a reviewed repair map or a validated saved journal. + data["legacy_job_keys"] = ["config_local"] + source.write_text(json.dumps(data), encoding="utf-8") + before = tree_bytes(installed) + plan = plan_for(relocated, installed) + assert plan["classification"] == "blocked" + assert tree_bytes(installed) == before + + +def test_canonical_recovery_descriptors_are_not_active_mutable_identity(identity_root): + installed, relocated = installation(BY_ID["already_migrated"], identity_root) + payload = deepcopy(BY_ID["pending_runtime_recovery"]["files"]["data/config/runtime-recovery.json"]["json"]) + entry = payload["entries"][0] + entry["job_id"] = relocated["allocation_order"][0] + entry["log_file"] = str(installed / "logs/Borg-Backup_config--2026-08-31_08-00-00.log") + source = installed / "data/config/runtime-recovery.json" + source.write_text(json.dumps(payload), encoding="utf-8") + before = tree_bytes(installed) + result = migration.verify_target(relocated["config"], control_root=installed / "run") + assert result["valid"] is True + assert result["writable_services_allowed"] is False + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("kind", ["status", "weekly"]) +def test_canonical_inventory_cannot_hide_unconverted_known_job_history_as_orphan(kind, identity_root): + installed, relocated = installation(BY_ID["already_migrated"], identity_root) + if kind == "status": + path = installed / "status/2026-08-31_08-00-00_config_local.status" + payload = json.loads(path.read_text(encoding="utf-8")) + del payload["job_id"] + del payload["schema_version"] + else: + path = installed / "weekly-snapshots.json" + payload = {"config_local": [{"week": "2026-08-31", "size": 100}]} + path.write_text(json.dumps(payload), encoding="utf-8") + before = tree_bytes(installed) + result = migration.verify_target(relocated["config"], control_root=installed / "run") + assert result["valid"] is False + assert result["writable_services_allowed"] is False + assert tree_bytes(installed) == before + + +def _persisted_partial_job_plan(identity_root): + """Simulate a future applier boundary on fixture files, not plugin data.""" + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + original = plan_for(relocated, installed) + state_dir = identity_root / "migration-state" + persisted = storage.persist_plan(original, state_dir) + source = str(installed / "data/config/jobs/config_local.json") + write = next(action for action in persisted["actions"] + if action["kind"] == "write_json" and action["source"] == source) + retirement = next(action for action in persisted["actions"] + if action["kind"] == "retire_source" and action["source"] == source) + storage.append_journal(state_dir, persisted, "pending", "apply", action_ids=[write["id"]]) + target = Path(write["target"]) + target.write_bytes(migration.encode_target_json(write["data"])) + target.chmod(write["after"]["mode"]) + storage.append_journal(state_dir, persisted, "applied", "apply", action_ids=[write["id"]]) + storage.append_journal(state_dir, persisted, "pending", "apply", action_ids=[retirement["id"]]) + Path(source).unlink() + storage.append_journal(state_dir, persisted, "applied", "apply", action_ids=[retirement["id"]]) + assert len(storage.read_journal(state_dir)) == 4 + return installed, relocated, state_dir, persisted, target + + +def test_partial_plan_reuses_persisted_identity_and_original_snapshot_footprint(identity_root): + installed, relocated, state_dir, persisted, target = _persisted_partial_job_plan(identity_root) + before = tree_bytes(installed) + + def forbidden_allocator(): + raise AssertionError("resume allocated a second identity") + + resumed = migration.build_plan(relocated["config"], + uuid_factory=forbidden_allocator, + journal_plan=storage.load_plan(state_dir), + control_root=installed / "run") + assert resumed == persisted + assert target.name == relocated["allocation_order"][0] + ".json" + assert resumed["status"] == "pending" + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("change", ["different_json_encoding", "permissions"]) +def test_resume_rejects_unexplained_target_bytes_or_mode_changes(change, identity_root): + installed, relocated, state_dir, persisted, target = _persisted_partial_job_plan(identity_root) + if change == "different_json_encoding": + value = json.loads(target.read_text(encoding="utf-8")) + target.write_text(json.dumps(value), encoding="utf-8") + else: + target.chmod(0o777) + before = tree_bytes(installed) + resumed = migration.build_plan(relocated["config"], journal_plan=storage.load_plan(state_dir), + control_root=installed / "run") + assert resumed["classification"] == "blocked" + assert tree_bytes(installed) == before + + +def test_real_planner_snapshot_requires_cron_capture_and_bound_confirmation(identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + missing_cron = plan_for(relocated, installed) + with pytest.raises(storage.IdentityStorageError): + storage.create_snapshot(missing_cron, identity_root / "missing-cron-state") + before = tree_bytes(installed) + values = iter(relocated["allocation_order"]) + cron = "# Synthetic unrelated cron entry\n0 4 * * * /fixture/maintenance\n" + plan = migration.build_plan(relocated["config"], uuid_factory=lambda: UUID(next(values)), + control_root=installed / "run", cron_text=cron) + snapshot = storage.create_snapshot(plan, identity_root / "migration-state") + storage.verify_snapshot(plan, snapshot) + with pytest.raises(storage.IdentityStorageError): + storage.verify_preconditions(plan, snapshot, quiescence_check=lambda: True) + confirmation = { + "approved": True, "independent_backup_acknowledged": True, + "plan_id": plan["plan_id"], "snapshot_digest": snapshot["digest"], + } + with pytest.raises(storage.IdentityStorageError): + storage.verify_preconditions(plan, snapshot, confirmation) + assert storage.verify_preconditions( + plan, snapshot, confirmation, quiescence_check=lambda: True, + external_input_check=lambda: {"managed_cron": {"kind": "crontab", "text": cron}}, + ) is True + # Successful library precondition verification is still not an apply API. + assert plan["activation_allowed"] is False + assert tree_bytes(installed) == before + + +def test_actual_source_change_invalidates_real_planner_snapshot_and_confirmation(identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + plan = migration.build_plan(relocated["config"], control_root=installed / "run", cron_text="") + snapshot = storage.create_snapshot(plan, identity_root / "migration-state") + confirmation = { + "approved": True, "independent_backup_acknowledged": True, + "plan_id": plan["plan_id"], "snapshot_digest": snapshot["digest"], + } + source = installed / "data/config/jobs/config_local.json" + source.write_bytes(source.read_bytes() + b"\n") + before = tree_bytes(installed) + with pytest.raises(storage.IdentityStorageError): + storage.verify_preconditions( + plan, snapshot, confirmation, quiescence_check=lambda: True, + external_input_check=lambda: {"managed_cron": {"kind": "crontab", "text": ""}}, + ) + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("case_id", [ + "fresh", "already_migrated", "legacy_without_prefixes", "future_schema", +]) +def test_repeated_detect_is_idempotent_and_never_changes_data_or_config(case_id, identity_root): + installed, relocated = installation(BY_ID[case_id], identity_root) + config_before = deepcopy(relocated["config"]) + before = tree_bytes(installed) + first = migration.detect(relocated["config"], control_root=installed / "run") + second = migration.detect(relocated["config"], control_root=installed / "run") + assert first == second + assert first["classification"] == relocated["expected"]["classification"] + assert first["required"] is (first["classification"] != "not_applicable") + assert first["status"] != "applied" + assert tree_bytes(installed) == before + assert relocated["config"] == config_before + + +def test_missing_canonical_alias_list_is_not_blessed_by_proposal_defaults(identity_root): + installed, relocated = installation(BY_ID["already_migrated"], identity_root) + path = installed / "data/config/jobs" / (relocated["allocation_order"][0] + ".json") + payload = json.loads(path.read_text(encoding="utf-8")) + del payload["legacy_job_keys"] + path.write_text(json.dumps(payload), encoding="utf-8") + before = tree_bytes(installed) + result = migration.verify_target(relocated["config"], control_root=installed / "run") + assert result["valid"] is False + assert tree_bytes(installed) == before + + +def test_target_verifier_rejects_job_change_between_its_inventory_reads(identity_root, monkeypatch): + installed, relocated = installation(BY_ID["already_migrated"], identity_root) + path = installed / "data/config/jobs" / (relocated["allocation_order"][0] + ".json") + original_inventory = migration._inventory + calls = 0 + + def changing_inventory(config, control_root): + nonlocal calls + calls += 1 + if calls == 2: + payload = json.loads(path.read_text(encoding="utf-8")) + payload["job_id"] = relocated["allocation_order"][1] + path.write_text(json.dumps(payload), encoding="utf-8") + return original_inventory(config, control_root) + + monkeypatch.setattr(migration, "_inventory", changing_inventory) + result = migration.verify_target(relocated["config"], control_root=installed / "run") + assert calls == 2, "test must exercise the independent verification read" + assert result["valid"] is False + assert result["writable_services_allowed"] is False + + +def test_legacy_safe_text_prefix_without_old_suffix_does_not_gain_archive_ownership(identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + path = installed / "data/config/jobs/config_local.json" + payload = json.loads(path.read_text(encoding="utf-8")) + # The legacy reader ignored safe-looking entries without its -backup + # suffix. Treating this as a newly active prefix could claim other archives. + payload["archive_prefixes"] = ["config-backup", "other-safe-prefix"] + path.write_text(json.dumps(payload), encoding="utf-8") + before = tree_bytes(installed) + result = plan_for(relocated, installed) + assert result["classification"] == "blocked" + assert not result["jobs"] + assert tree_bytes(installed) == before + + +def test_existing_widget_cache_requires_explicit_deferred_rebuild_action(identity_root): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + path = installed / "plugin/widget-status.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({ + "schema_version": 1, "generated_at": "2026-08-31T08:01:00Z", + "jobs": {"total": 1, "running": 0, "successful": 1}, + }), encoding="utf-8") + before = tree_bytes(installed) + result = plan_for(relocated, installed) + assert result["classification"] == "applicable" + actions = [action for action in result["actions"] + if action["kind"] == "rebuild_derived" and action["source"] == str(path)] + assert len(actions) == 1, "warning-only omission does not plan cache invalidation" + assert actions[0]["target"] == str(path) + assert result["inputs"][str(path)]["exists"] is True + assert result["activation_allowed"] is False + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("error", [ + OSError("SYNTHETIC_SECRET_NOT_FOR_DIAGNOSTICS"), + storage.IdentityStorageError("unsafe_path"), +]) +def test_target_verifier_masks_second_scan_failures_and_returns_invalid(error, identity_root, monkeypatch): + installed, relocated = installation(BY_ID["already_migrated"], identity_root) + before = tree_bytes(installed) + original_inventory = migration._inventory + calls = 0 + + def failing_inventory(config, control_root): + nonlocal calls + calls += 1 + if calls == 2: + raise error + return original_inventory(config, control_root) + + monkeypatch.setattr(migration, "_inventory", failing_inventory) + result = migration.verify_target(relocated["config"], control_root=installed / "run") + assert calls == 2 + assert result["valid"] is False + assert result["writable_services_allowed"] is False + assert result["reasons"] + assert "SYNTHETIC_SECRET_NOT_FOR_DIAGNOSTICS" not in json.dumps(result) + assert tree_bytes(installed) == before + + +def test_mixed_legacy_and_canonical_jobs_require_the_original_migration_plan(identity_root): + installed, relocated = installation(BY_ID["shared_repository_distinct_prefixes"], identity_root) + job_id = relocated["allocation_order"][1] + canonical = relocated["expected"]["jobs"][job_id] + directory = installed / "data/config/jobs" + (directory / (job_id + ".json")).write_bytes(migration.encode_target_json(canonical)) + (directory / "photos_local.json").unlink() + # The remaining legacy job and both repository references are otherwise + # consistent. Their presence does not legitimize a partially converted job. + before = tree_bytes(installed) + result = plan_for(relocated, installed) + assert result["classification"] == "blocked" + assert "partial_migration_without_journal" in reason_codes(result) + assert tree_bytes(installed) == before + + +@pytest.mark.parametrize("change", ["missing", "symlink"]) +def test_resume_revalidates_secret_reference_without_copying_its_contents(change, identity_root, monkeypatch): + installed, relocated = installation(BY_ID["legacy_without_prefixes"], identity_root) + secret = installed / "data/secrets/repository.passphrase" + secret.parent.mkdir(parents=True) + secret.write_text("SYNTHETIC_PRIVATE_SECRET_CONTENT", encoding="utf-8") + repository_file = installed / "data/config/repositories.json" + repositories = json.loads(repository_file.read_text(encoding="utf-8")) + repositories["repositories"][0].update(encryption="repokey", passphrase_ref=str(secret)) + repository_file.write_text(json.dumps(repositories), encoding="utf-8") + other = identity_root / "another-secret-file" + actual_read = storage._read_file + + def reject_secret_content_read(path, **kwargs): + assert Path(path) not in {secret, other}, "planner read secret content" + return actual_read(path, **kwargs) + + monkeypatch.setattr(storage, "_read_file", reject_secret_content_read) + original = plan_for(relocated, installed) + assert original["classification"] == "applicable" + assert str(secret) not in original["inputs"] + assert "SYNTHETIC_PRIVATE_SECRET_CONTENT" not in json.dumps(original) + persisted = storage.persist_plan(original, identity_root / "migration-state") + secret.unlink() + if change == "symlink": + other.write_text("OTHER_SYNTHETIC_PRIVATE_CONTENT", encoding="utf-8") + secret.symlink_to(other) + before = tree_bytes(installed) + result = migration.build_plan(relocated["config"], journal_plan=persisted, + control_root=installed / "run") + assert result["classification"] == "blocked" + assert "SYNTHETIC_PRIVATE_SECRET_CONTENT" not in json.dumps(result) + assert "OTHER_SYNTHETIC_PRIVATE_CONTENT" not in json.dumps(result) + assert tree_bytes(installed) == before diff --git a/tests/test_identity_records.py b/tests/test_identity_records.py new file mode 100644 index 00000000..1d6a1f46 --- /dev/null +++ b/tests/test_identity_records.py @@ -0,0 +1,474 @@ +"""Pure store-projection regressions for inactive migration phase #472.""" + +from copy import deepcopy +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "api")) + +from migrations.identity_records import project_records, verify_records + + +JOB = "11111111-1111-4111-8111-111111111111" +OTHER = "22222222-2222-4222-8222-222222222222" +JOBS = {JOB: {"schema_version": 4, "job_id": JOB, "repository_key": "repo", + "legacy_job_keys": ["config_local"], "name": "Current name"}} +ALIASES = {"config_local": JOB} + + +def rec(kind, data, **metadata): + return {"kind": kind, "data": data, **metadata} + + +def project(kind, data, **metadata): + return project_records({"/fixture/input": rec(kind, data, **metadata)}, JOBS, ALIASES) + + +def codes(result): + return {reason["code"] for reason in result["reasons"] if reason["severity"] != "warning"} + + +def output(result): + return next(iter(result["records"].values()))["data"] + + +def test_schedules_preserve_disabled_and_system_entries_and_are_pure(): + data = {"config_local": {"cron": "0 8 * * *", "enabled": False, "future": {"x": 1}}, + "restore_test": {"cron": "0 9 * * 0", "enabled": True}} + before = deepcopy(data) + result = project("schedules", data) + assert not codes(result) + assert result["required"] + assert output(result) == {JOB: data["config_local"], "restore_test": data["restore_test"]} + assert data == before + assert verify_records(result["records"], JOBS, ALIASES) == [] + + +def test_unknown_disabled_schedule_is_not_discarded_or_inferred_from_name(): + result = project("schedules", {"old_config_local": {"cron": "0 8 * * *", "enabled": False}}) + assert codes(result) == {"orphan_active_schedule"} + + +def test_verifier_rejects_active_alias_even_when_projector_could_resolve_it(): + records = {"/fixture/schedules": rec("schedules", {"config_local": {"cron": "0 8 * * *"}})} + assert {r["code"] for r in verify_records(records, JOBS, ALIASES)} == {"mutable_active_reference"} + + +def test_duplicate_schedule_aliases_cannot_overwrite_one_another(): + result = project("schedules", {"config_local": {"cron": "0 8 * * *"}, JOB: {"cron": "0 9 * * *"}}) + assert "duplicate_schedule_identity" in codes(result) + + +def test_repository_links_convert_both_fields_but_preserve_repository_identity(): + row = {"repository_key": "repo", "used_by": ["config_local"], "source_job_keys": ["config_local"], + "storage_key": "storage", "passphrase_ref": "secrets/unchanged.ref", "metadata": {"x": 1}} + result = project("repositories", {"schema_version": 1, "repositories": [row]}) + target = output(result)["repositories"][0] + assert not codes(result) + assert target["job_ids"] == target["source_job_ids"] == [JOB] + assert "used_by" not in target and "source_job_keys" not in target + assert target["passphrase_ref"] == row["passphrase_ref"] + assert target["metadata"] == row["metadata"] + assert len(result["bindings"]) == 2 + assert verify_records(result["records"], JOBS, ALIASES) == [] + + +@pytest.mark.parametrize("changes", [ + {"used_by": []}, {"used_by": ["unknown_local"]}, {"used_by": ["config_local", "config_local"]}, + {"job_ids": [JOB]}, {"source_job_keys": "config_local"}, +]) +def test_repository_reverse_conflicts_block(changes): + row = {"repository_key": "repo", "used_by": ["config_local"], "source_job_keys": ["config_local"], **changes} + assert codes(project("repositories", {"schema_version": 1, "repositories": [row]})) + + +def test_status_enrichment_preserves_original_history_and_never_invents_snapshots(): + data = {"backup_type": "config", "location": "local", "timestamp": "2026-08-31 08:00:00", + "archive_name": "config-backup-example", "log_file": "/fixture/logs/old.log", + "borg_exit_code": 0} + result = project("status", data) + target = output(result) + assert not codes(result) + assert target == {**data, "schema_version": 1, "job_id": JOB} + for unknown in ("run_id", "job_name", "repository_key"): + assert unknown not in target + assert verify_records(result["records"], JOBS, ALIASES) == [] + + +def test_already_canonical_status_is_not_changed(): + data = {"schema_version": 1, "job_id": JOB, "backup_type": "config", "location": "local"} + result = project("status", data) + assert output(result) == data + assert not result["required"] + + +def test_status_filename_disagreement_is_unassigned_not_wrong_job(): + path = "/fixture/status/2026-08-31_08-00-00_photos_local.status" + result = project_records({path: rec("status", {"backup_type": "config", "location": "local"})}, JOBS, ALIASES) + assert not codes(result) + assert result["bindings"][0]["job_id"] is None + assert result["unassigned"][0]["reason"] == "conflicting_identity_evidence" + assert "job_id" not in output(result) + + +def test_underscore_legacy_identity_uses_full_exact_key(): + data = {"backup_type": "server_config_old", "location": "local"} + result = project_records({"/fixture/status/2026-08-31_08-00-00_server_config_old_local.status": rec("status", data)}, + JOBS, {"server_config_old_local": JOB}) + assert output(result)["job_id"] == JOB + + +def test_orphan_and_explicitly_unassigned_history_never_creates_a_job(): + data = {"backup_type": "deleted", "location": "local", "log_file": "/fixture/logs/deleted.log"} + result = project("status", data) + assert not codes(result) + assert not result["required"] + assert output(result)["identity_state"] == "unassigned" + assert result["unassigned"][0]["data"] == data + rerun = project_records(result["records"], JOBS, {**ALIASES, "deleted_local": JOB}) + assert output(rerun) == output(result) + assert rerun["bindings"][0]["job_id"] is None + + +def test_restore_test_filename_move_keeps_tested_scope(): + data = {"test_date": "2026-08-31 08:10:00", "test_result": "PASS", "tested_archive": "old-prefix-archive", + "repository": "/fixture/previous-repository", "tested_entries": ["old-name"]} + result = project("restore_test", data, legacy_key="config_local", target_path=f"/fixture/tests/{JOB}.test") + assert not codes(result) + assert list(result["records"]) == [f"/fixture/tests/{JOB}.test"] + assert all(output(result)[key] == value for key, value in data.items()) + + +def test_restore_history_and_active_runs_keep_independent_restore_ids(): + row = {"restore_id": "restore-example", "state": "done", "job_key": "config_local", "archive": "old-archive"} + records = {"/fixture/index": rec("restore_index", {"schema_version": 1, "runs": [row]}), + "/fixture/restore-example.json": rec("restore_detail", {"schema_version": 1, **row}), + "/fixture/runs": rec("restore_runs", {"schema_version": 1, "runs": {}})} + result = project_records(records, JOBS, ALIASES) + assert not codes(result) + assert result["records"]["/fixture/restore-example.json"]["data"]["job_key"] == "config_local" + assert result["records"]["/fixture/restore-example.json"]["data"]["restore_id"] == "restore-example" + assert {r["job_id"] for r in result["bindings"]} == {JOB} + assert verify_records(result["records"], JOBS, ALIASES) == [] + active = project("restore_runs", {"schema_version": 1, "runs": { + "restore-example": {**row, "state": "running"}}}) + assert output(active)["runs"]["restore-example"]["legacy_job_key"] == "config_local" + assert "job_key" not in output(active)["runs"]["restore-example"] + + +def test_restore_cross_store_mismatch_blocks(): + jobs = {**JOBS, OTHER: {"job_id": OTHER, "repository_key": "other"}} + records = {"/fixture/index": rec("restore_index", {"schema_version": 1, "runs": [ + {"restore_id": "restore-example", "state": "done", "job_id": JOB}]}), + "/fixture/restore-example.json": rec("restore_detail", {"restore_id": "restore-example", "state": "done", "job_id": OTHER})} + assert "restore_identity_mismatch" in codes(project_records(records, jobs, ALIASES)) + + +def test_notification_queue_preserves_retry_state_and_does_not_dispatch(): + row = {"id": "event-1", "job_key": "config_local", "attempts_made": 2, "next_attempt_at": 1788163380, + "body": "Original message", "source": "backup_job", "event_type": "backup_success"} + result = project("notification_queue", {"schema_version": 1, "queue": [row]}) + target = output(result)["queue"][0] + assert not codes(result) + assert target["job_id"] == JOB and target["legacy_job_key"] == "config_local" + assert "job_key" not in target + for key in ("id", "attempts_made", "next_attempt_at", "body"): + assert target[key] == row[key] + assert verify_records(result["records"], JOBS, ALIASES) == [] + + +def test_orphan_queue_blocks_but_delivery_is_retained_unassigned(): + row = {"id": "event-1", "job_key": "deleted_local", "source": "backup_job"} + assert codes(project("notification_queue", {"schema_version": 1, "queue": [row]})) == {"orphan_active_notification"} + result = project("notification_deliveries", {"schema_version": 1, "deliveries": [row]}) + assert not codes(result) + assert output(result)["deliveries"][0]["job_key"] == row["job_key"] + assert result["unassigned"][0]["reason"] == "no_configured_job" + + +def test_explicit_system_event_does_not_receive_fabricated_id(): + row = {"id": "event-1", "job_key": "restore_test", "source": "restore_test"} + result = project("notification_queue", {"schema_version": 1, "queue": [row]}) + assert not codes(result) + assert output(result)["queue"] == [row] + assert result["bindings"][0]["role"] == "system" + bad = {**row, "source": "backup_job"} + assert codes(project("notification_queue", {"schema_version": 1, "queue": [bad]})) + + +def test_reminder_due_marker_with_colons_and_timestamp_survive(): + key = "backup_overdue:config_local:2026-08-31T08:00:00" + result = project("notification_state", {"schema_version": 1, "last_sent": {key: 1788163200}}) + assert not codes(result) + assert output(result)["last_sent"] == {f"backup_overdue:{JOB}:2026-08-31T08:00:00": 1788163200} + assert result["bindings"][0]["locator"] == "/last_sent/" + key + assert verify_records(result["records"], JOBS, ALIASES) == [] + + +def test_unknown_reminders_retained_with_provenance_not_deleted(): + key = "backup_overdue:deleted_local:current" + result = project("notification_state", {"schema_version": 1, "last_sent": {key: 10}}) + assert output(result)["last_sent"] == {} + assert output(result)["unassigned"] == [{"key": key, "value": 10, "source": "/fixture/input", "locator": "/last_sent/" + key}] + + +def test_pending_recovery_keeps_targets_and_never_marks_restarted(): + row = {"id": "recovery-1", "state": "pending_restart", "backup_type": "config", "backup_location": "local", + "pid": 999999, "targets": [{"id": "container-id", "name": "original-container"}], + "stopped_at": "2026-08-31T08:00:00Z", "restarted_at": ""} + result = project("runtime_recovery", {"schema_version": 1, "entries": [row]}) + assert not codes(result) + target = output(result)["entries"][0] + assert target == {**row, "schema_version": 1, "job_id": JOB} + + +@pytest.mark.parametrize("kind", ["control", "cancel_request", "resource_lock"]) +def test_runtime_owner_reference_uses_id_without_changing_run(kind): + row = {"job_key": "config_local", "run_id": "original-run", "pid": 999999, "resource": "repository:repo"} + result = project(kind, row) + assert not codes(result) + assert output(result)["job_id"] == JOB + assert output(result)["run_id"] == "original-run" + assert output(result)["pid"] == 999999 + + +@pytest.mark.parametrize("second_size, count, conflict", [(100, 1, False), (101, 2, True)]) +def test_both_weekly_stores_deduplicate_equal_values_and_preserve_conflicts(second_size, count, conflict): + sources = {"/fixture/current-weekly": rec("weekly", {"config_local": [{"week": "2026-08-31", "size": 100}]}, + target_path="/fixture/current-weekly"), + "/fixture/legacy-weekly": rec("weekly", {"config_local": [{"week": "2026-08-31", "size": second_size}]}, + target_path="/fixture/current-weekly")} + result = project_records(sources, JOBS, ALIASES) + assert not codes(result) + rows = output(result)["observations"] + assert len(rows) == count + assert {row["size"] for row in rows} == {100, second_size} + assert sum(len(row["source_records"]) for row in rows) == 2 + assert all(bool(row.get("conflict")) == conflict for row in rows) + assert next(iter(result["records"].values()))["sources"] == sorted(sources) + again = project_records(result["records"], JOBS, ALIASES) + assert output(again) == output(result) + + +@pytest.mark.parametrize("kind,data", [ + ("status", []), ("status", {}), ("status", {"schema_version": 99}), + ("notification_queue", {"schema_version": 1, "queue": {}}), + ("notification_queue", {"schema_version": 1, "queue": [None]}), + ("notification_state", {"schema_version": 1, "last_sent": {"invalid": 1}}), + ("runtime_recovery", {"schema_version": 1, "entries": [{"state": "recovered"}]}), + ("unknown_kind", {}), ("widget_cache", {"schema_version": 99}), +]) +def test_unknown_owned_shapes_fail_closed(kind, data): + assert codes(project(kind, data)) + + +def test_widget_is_explicit_rebuild_gate_not_silently_projected(): + result = project("widget_cache", {"schema_version": 1, "jobs": [{"job_key": "config_local"}]}) + assert not codes(result) + assert not result["records"] + assert result["reasons"] == [{"code": "widget_rebuild_required", "source": "/fixture/input", "locator": "", "severity": "warning"}] + + +def test_verifier_detects_lost_weekly_conflict_marker(): + records = {"/fixture/weekly": rec("weekly", {"schema_version": 1, "identity_schema_version": 1, + "observations": [{"job_id": JOB, "legacy_job_key": "config_local", "week": "2026-08-31", "size": size, + "source_records": [{"source": f"/fixture/source-{size}", "locator": "/config_local/0"}]} + for size in (100, 101)]})} + assert "weekly_projection_mismatch" in {reason["code"] for reason in verify_records(records, JOBS, ALIASES)} + + +def test_verifier_detects_unassigned_restore_index_with_assigned_detail(): + common = {"restore_id": "restore-example", "state": "done", "job_key": "config_local"} + records = {"/fixture/index": rec("restore_index", {"schema_version": 1, "runs": [ + {**common, "identity_state": "unassigned"}]}), + "/fixture/restore-example.json": rec("restore_detail", {**common, "job_id": JOB})} + assert "restore_identity_mismatch" in {reason["code"] for reason in verify_records(records, JOBS, ALIASES)} + + +def test_conflicting_active_explicit_id_and_legacy_key_blocks(): + jobs = {**JOBS, OTHER: {"job_id": OTHER, "repository_key": "other"}} + result = project_records({"/fixture/control": rec("control", {"job_key": "config_local", "job_id": OTHER})}, + jobs, ALIASES) + assert "conflicting_active_identity" in codes(result) + + +def test_former_prefix_only_yields_diagnostic_not_an_alias(): + jobs = {JOB: {**JOBS[JOB], "archive_prefixes": ["pfsense-backup", "config-backup"], "legacy_job_keys": ["pfsense_local"]}} + result = project_records({"/fixture/status": rec("status", {"backup_type": "config", "location": "local"})}, + jobs, {"pfsense_local": JOB}) + assert result["bindings"][0]["job_id"] is None + assert result["unassigned"][0]["reason"] == "no_authoritative_alias" + + +@pytest.mark.parametrize("targets", [[None], [{"name": "only-name"}], [{"id": "a", "name": "A"}, {"id": "a", "name": "B"}]]) +def test_invalid_pending_recovery_targets_block(targets): + data = {"schema_version": 1, "entries": [{"state": "pending_restart", "backup_type": "config", + "backup_location": "local", "targets": targets}]} + assert "invalid_recovery_targets" in codes(project("runtime_recovery", data)) + + +def restore_pair(): + row = {"restore_id": "restore-example", "state": "done", "job_key": "config_local", "archive": "old-archive", + "source_path": "source.txt", "target_dir": "/fixture/output"} + return {"/fixture/index.json": rec("restore_index", {"schema_version": 1, "runs": [deepcopy(row)]}), + "/fixture/runs/restore-example.json": rec("restore_detail", {"schema_version": 1, **deepcopy(row)})} + + +@pytest.mark.parametrize("missing,code", [ + ("/fixture/index.json", "missing_restore_index_entry"), + ("/fixture/runs/restore-example.json", "missing_restore_detail"), +]) +def test_restore_history_requires_both_owned_records(missing, code): + records = restore_pair() + del records[missing] + assert code in codes(project_records(records, JOBS, ALIASES)) + + +@pytest.mark.parametrize("kind", ["index", "detail"]) +def test_duplicate_restore_history_ids_block(kind): + records = restore_pair() + if kind == "index": + rows = records["/fixture/index.json"]["data"]["runs"] + rows.append(deepcopy(rows[0])) + else: + records["/fixture/other/restore-example.json"] = deepcopy(records["/fixture/runs/restore-example.json"]) + assert "duplicate_restore_id" in codes(project_records(records, JOBS, ALIASES)) + + +@pytest.mark.parametrize("field,value", [ + ("state", "error"), ("archive", "different-archive"), ("source_path", "different.txt"), + ("target_dir", "/fixture/other-output"), ("repository_key", "other-repository"), +]) +def test_restore_summary_detail_snapshot_disagreement_blocks(field, value): + records = restore_pair() + records["/fixture/runs/restore-example.json"]["data"][field] = value + assert "restore_snapshot_mismatch" in codes(project_records(records, JOBS, ALIASES)) + + +def test_restore_detail_filename_is_exact_restore_id_not_display_name(): + records = restore_pair() + records["/fixture/runs/other-name.json"] = records.pop("/fixture/runs/restore-example.json") + assert "restore_detail_filename_mismatch" in codes(project_records(records, JOBS, ALIASES)) + + +def test_active_restore_must_not_collide_with_terminal_history(): + records = restore_pair() + row = {**records["/fixture/index.json"]["data"]["runs"][0], "state": "running"} + records["/fixture/restore-runs.json"] = rec("restore_runs", {"schema_version": 1, "runs": {"restore-example": row}}) + assert "restore_active_history_collision" in codes(project_records(records, JOBS, ALIASES)) + + +def test_deleted_job_restore_history_pair_is_retained_without_inventing_identity(): + records = restore_pair() + records["/fixture/index.json"]["data"]["runs"][0]["job_key"] = "deleted_local" + records["/fixture/runs/restore-example.json"]["data"]["job_key"] = "deleted_local" + result = project_records(records, JOBS, ALIASES) + assert not codes(result) + assert all(binding["job_id"] is None for binding in result["bindings"]) + assert verify_records(result["records"], JOBS, ALIASES) == [] + broken = deepcopy(result["records"]) + broken.pop("/fixture/index.json") + assert "missing_restore_index_entry" in {r["code"] for r in verify_records(broken, JOBS, ALIASES)} + + +@pytest.mark.parametrize("filename", ["config_local.test", f"{OTHER}.test", "display-name.test"]) +def test_verifier_rejects_mapped_restore_proof_under_wrong_filename(filename): + records = {"/fixture/tests/" + filename: rec("restore_test", {"schema_version": 1, "job_id": JOB, + "tested_archive": "old-archive", "test_result": "PASS"}, legacy_key=filename[:-5])} + assert "restore_test_filename_mismatch" in {r["code"] for r in verify_records(records, JOBS, ALIASES)} + + +def test_verifier_accepts_canonical_restore_test_and_unassigned_legacy_proof(): + records = {f"/fixture/tests/{JOB}.test": rec("restore_test", {"schema_version": 1, "job_id": JOB, + "tested_archive": "old-archive", "test_result": "PASS"}, legacy_key=JOB), + "/fixture/tests/deleted_local.test": rec("restore_test", {"identity_schema_version": 1, + "identity_state": "unassigned", "identity_reason": "no_configured_job", "tested_archive": "deleted-archive"}, + legacy_key="deleted_local")} + assert verify_records(records, JOBS, ALIASES) == [] + + +def test_conflicting_canonical_proof_cannot_pass_by_becoming_unassigned_in_verifier(): + records = {f"/fixture/tests/{OTHER}.test": rec("restore_test", {"schema_version": 1, "job_id": JOB}, + legacy_key=OTHER)} + jobs = {**JOBS, OTHER: {"job_id": OTHER, "repository_key": "other"}} + reasons = {reason["code"] for reason in verify_records(records, jobs, ALIASES)} + assert reasons == {"conflicting_canonical_identity", "restore_test_filename_mismatch"} + + +@pytest.mark.parametrize("kind,field", [("status", None), ("notification_deliveries", "deliveries")]) +def test_deleted_job_historical_record_keeps_its_former_immutable_id(kind, field): + row = {"schema_version": 1, "job_id": OTHER, "job_key": "deleted_local", "archive": "old-archive"} + data = {"schema_version": 1, field: [row]} if field else row + result = project(kind, data) + assert not codes(result) + target = output(result)[field][0] if field else output(result) + assert target["job_id"] == OTHER + assert target["job_key"] == "deleted_local" + assert target["identity_state"] == "unassigned" + assert target["identity_reason"] == "deleted_job" + assert result["bindings"][0]["job_id"] is None + assert result["unassigned"][0]["data"] == row + assert verify_records(result["records"], JOBS, ALIASES) == [] + # Reading older canonical history without the new diagnostic does not + # require a current configured owner or resurrect a job. + assert verify_records({"/fixture/input": rec(kind, data)}, JOBS, ALIASES) == [] + + +def test_deleted_job_restore_history_keeps_former_id_in_both_peers(): + records = restore_pair() + summary = records["/fixture/index.json"]["data"]["runs"][0] + detail = records["/fixture/runs/restore-example.json"]["data"] + for row in (summary, detail): + row["job_id"] = OTHER + row["job_key"] = "deleted_local" + result = project_records(records, JOBS, ALIASES) + assert not codes(result) + assert result["records"]["/fixture/index.json"]["data"]["runs"][0]["job_id"] == OTHER + assert result["records"]["/fixture/runs/restore-example.json"]["data"]["job_id"] == OTHER + assert all(binding["job_id"] is None for binding in result["bindings"]) + assert verify_records(result["records"], JOBS, ALIASES) == [] + + +def test_deleted_job_weekly_observation_never_loses_former_uuid(): + row = {"job_id": OTHER, "legacy_job_key": "deleted_local", "week": "2026-08-31", "size": 100, + "source_records": [{"source": "/fixture/old-weekly", "locator": "/deleted_local/0"}]} + data = {"schema_version": 1, "identity_schema_version": 1, "observations": [row]} + result = project("weekly", data) + assert not codes(result) + target = output(result)["observations"][0] + assert target["job_id"] == OTHER + assert target["identity_state"] == "unassigned" + assert target["identity_reason"] == "deleted_job" + assert target["source_records"] == row["source_records"] + assert result["bindings"][0]["job_id"] is None + assert result["unassigned"][0]["data"] == row + assert verify_records(result["records"], JOBS, ALIASES) == [] + assert verify_records({"/fixture/input": rec("weekly", data)}, JOBS, ALIASES) == [] + assert output(project_records(result["records"], JOBS, ALIASES)) == output(result) + + +@pytest.mark.parametrize("kind,field", [("control", None), ("notification_queue", "queue")]) +def test_former_uuid_never_authorizes_active_dangling_reference(kind, field): + row = {"schema_version": 1, "job_id": OTHER, "legacy_job_key": "deleted_local"} + data = {"schema_version": 1, field: [row]} if field else row + result = project(kind, data) + assert codes(result) + assert verify_records({"/fixture/input": rec(kind, data)}, JOBS, ALIASES) + + +def test_conflicting_canonical_history_blocks_plan_without_overwriting_original_id(): + row = {"schema_version": 1, "job_id": OTHER, "job_key": "config_local"} + result = project("status", row) + assert "conflicting_canonical_identity" in codes(result) + assert output(result)["job_id"] == OTHER + assert output(result)["job_key"] == "config_local" + + +def test_conflicting_canonical_weekly_identity_blocks_without_reassigning_uuid(): + row = {"job_id": OTHER, "legacy_job_key": "config_local", "week": "2026-08-31", "size": 100, + "source_records": [{"source": "/fixture/old-weekly", "locator": "/config_local/0"}]} + result = project("weekly", {"schema_version": 1, "identity_schema_version": 1, "observations": [row]}) + assert "conflicting_canonical_identity" in codes(result) + assert output(result)["observations"][0]["job_id"] == OTHER diff --git a/tests/test_identity_storage.py b/tests/test_identity_storage.py new file mode 100644 index 00000000..f9b69727 --- /dev/null +++ b/tests/test_identity_storage.py @@ -0,0 +1,522 @@ +"""#472 inactive, exact-file snapshot/plan/journal primitives.""" + +from copy import deepcopy +from datetime import datetime, timezone +import errno +import hashlib +import json +import os +from pathlib import Path +import sys +from tempfile import TemporaryDirectory +import traceback + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "api")) +from migrations import identity_storage as storage + +ID = "11111111-1111-4111-8111-111111111111" + + +@pytest.fixture +def installation(): + parent = ROOT / ".release-tmp" + parent.mkdir(exist_ok=True) + with TemporaryDirectory(prefix="identity-storage-472-", dir=parent) as temporary: + root = Path(temporary) + jobs = root / "data" / "config" / "jobs" + jobs.mkdir(parents=True) + source = jobs / "documents_local.json" + source.write_bytes(b'{"name":"Synthetic documents","schema_version":3}\n') + source.chmod(0o640) + destination = jobs / (ID + ".json") + conf = jobs.parent / "backup.conf" + conf.write_bytes(b"SYNTHETIC_SETTING=not-a-credential\n") + groups = [storage.inventory_group(jobs, [".json"])] + plan = storage.seal_plan({ + "schema_version": 1, "migration_id": storage.MIGRATION_ID, + "classification": "applicable", "status": "pending", "required": True, + "prerequisites": {"managed_cron_captured": True}, + "external_inputs": {"managed_cron": {"kind": "crontab", "text": ""}}, + "id_map": {"documents_local": ID}, + "inputs": {str(path): storage.fingerprint_file(path) for path in (source, destination, conf)}, + "inventory_groups": groups, + "actions": [{"id": "write_job_1", "kind": "write_json", "source": str(source), + "target": str(destination), "data": {"job_id": ID, "schema_version": 4}}], + }) + yield {"root": root, "source": source, "destination": destination, + "config": conf, "jobs": jobs, "plan": plan, "state": root / "migration-state"} + + +def reseal(plan): + plan.pop("plan_id", None) + return storage.seal_plan(plan) + + +def expect_error(code, callback): + with pytest.raises(storage.IdentityStorageError) as error: + callback() + assert error.value.code == code + assert str(error.value) == code + + +def confirmation(plan, snapshot): + return {"approved": True, "independent_backup_acknowledged": True, + "plan_id": plan["plan_id"], "snapshot_digest": snapshot["digest"]} + + +def test_secure_fingerprint_and_read_share_exact_bytes(installation): + source = installation["source"] + expected = source.read_bytes() + fingerprint, raw = storage.read_fingerprinted_file(source) + assert raw == storage.read_file(source) == expected + assert fingerprint == {"exists": True, "size": len(expected), + "mode": 0o640, "sha256": hashlib.sha256(expected).hexdigest()} + assert storage.fingerprint_file(installation["destination"]) == {"exists": False} + assert storage.read_fingerprinted_file(installation["root"] / "absent" / "nested") == ({"exists": False}, None) + expect_error("storage_unavailable", lambda: storage.read_file(installation["destination"])) + + +@pytest.mark.parametrize("kind", ["file", "ancestor", "fifo", "directory"]) +def test_no_symlinks_or_nonregular_source_files(installation, kind): + root = installation["root"] + source = root / "unsafe" + if kind == "file": + source.symlink_to(installation["source"]) + elif kind == "ancestor": + source.symlink_to(installation["jobs"], target_is_directory=True) + source = source / installation["source"].name + elif kind == "fifo": + os.mkfifo(source) + else: + source.mkdir() + expect_error("unsafe_path", lambda: storage.fingerprint_file(source)) + + +@pytest.mark.parametrize("path", ["relative", "/a/../b", "/a//b", "//a", "/a/./b", "/a/", "/a\x00b"]) +def test_noncanonical_paths_rejected(path): + expect_error("unsafe_path", lambda: storage.fingerprint_file(path)) + + +def test_seal_hash_binds_every_field_and_allows_multiple_evidenced_aliases(installation): + plan = deepcopy(installation["plan"]) + original_hash = plan["plan_id"] + plan["id_map"]["config_local"] = ID + expect_error("invalid_plan", lambda: storage.seal_plan(plan)) + plan = reseal(plan) + assert plan["plan_id"] != original_hash + assert len(set(plan["id_map"].values())) == 1 + plan["actions"][0]["data"]["unexpected"] = "must be included in digest" + expect_error("invalid_plan", lambda: storage.seal_plan(plan)) + + +def test_unknown_destination_cannot_be_excluded_from_snapshot(installation): + plan = deepcopy(installation["plan"]) + del plan["inputs"][str(installation["destination"])] + expect_error("invalid_plan", lambda: reseal(plan)) + + +@pytest.mark.parametrize("mutate", [ + lambda plan: plan["id_map"].update({"documents_local": "not-a-uuid"}), + lambda plan: plan.update(schema_version=True), + lambda plan: plan["inputs"].update({"relative": {"exists": False}}), + lambda plan: plan["actions"].append(deepcopy(plan["actions"][0])), + lambda plan: plan["inputs"].update({"/missing": {"exists": False, "sha256": "ignored"}}), +]) +def test_bad_plan_structures_fail_closed(installation, mutate): + plan = deepcopy(installation["plan"]) + mutate(plan) + with pytest.raises(storage.IdentityStorageError): + reseal(plan) + + +def test_persist_once_loads_identical_ids_and_never_replaces_a_different_plan(installation): + plan, state = installation["plan"], installation["state"] + assert storage.persist_plan(plan, state) == plan + assert storage.load_plan(state) == plan + assert storage.persist_plan(deepcopy(plan), state) == plan + changed = deepcopy(plan) + changed["id_map"]["documents_local"] = "22222222-2222-4222-8222-222222222222" + expect_error("state_conflict", lambda: storage.persist_plan(reseal(changed), state)) + assert storage.load_plan(state) == plan + assert state.stat().st_mode & 0o777 == 0o700 + assert (state / "plan.json").stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize("position", ["source_parent", "ancestor", "symlink", "world_readable"]) +def test_state_must_be_private_and_disjoint_from_sources(installation, position): + plan = installation["plan"] + state = installation["state"] + if position == "source_parent": + state = installation["jobs"] / "migration" + elif position == "ancestor": + state = installation["root"] / "data" + elif position == "symlink": + other = installation["root"] / "other-state" + other.mkdir(mode=0o700) + state.symlink_to(other) + else: + state.mkdir(mode=0o755) + expect_error("unsafe_path", lambda: storage.persist_plan(plan, state)) + assert not (installation["jobs"] / "plan.json").exists() + + +def test_snapshot_exact_originals_absence_permissions_and_idempotence(installation): + plan, state = installation["plan"], installation["state"] + before = {path: Path(path).read_bytes() for path, value in plan["inputs"].items() if value["exists"]} + snapshot = storage.create_snapshot(plan, state) + manifest = storage.verify_snapshot(plan, snapshot) + assert set(manifest["entries"]) == set(plan["inputs"]) + absent = manifest["entries"][str(installation["destination"])] + assert absent == {"artifact_kind": "file", "original": {"exists": False}, "blob": None} + for path, raw in before.items(): + blob = state / "snapshot" / "files" / manifest["entries"][path]["blob"] + assert blob.read_bytes() == raw + assert blob.stat().st_mode & 0o777 == 0o600 + assert Path(path).read_bytes() == raw + assert storage.create_snapshot(plan, state) == snapshot + assert not installation["destination"].exists() + + +@pytest.mark.parametrize("mutation", ["content", "permission", "removed", "destination_created", "new_member"]) +def test_source_changes_invalidate_snapshot_reuse(installation, mutation): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + source = installation["source"] + if mutation == "content": + source.write_bytes(b"changed") + elif mutation == "permission": + source.chmod(0o600) + elif mutation == "removed": + source.unlink() + elif mutation == "destination_created": + installation["destination"].write_text("{}") + else: + (installation["jobs"] / "new_job.json").write_text("{}") + code = "inventory_changed" if mutation == "new_member" else "input_changed" + expect_error(code, lambda: storage.create_snapshot(plan, state)) + # An intact old snapshot stays readable even when source data changes. + storage.verify_snapshot(plan, snapshot) + + +def test_missing_inventory_directory_becoming_present_invalidates_plan(installation): + path = installation["root"] / "new-status" + group = storage.inventory_group(path, [".status"]) + assert group == {"path": str(path), "suffixes": [".status"], "exists": False, "entries": []} + plan = deepcopy(installation["plan"]) + plan["inventory_groups"].append(group) + plan = reseal(plan) + path.mkdir() + expect_error("inventory_changed", lambda: storage.verify_inputs(plan)) + + +def test_control_directory_membership_is_bounded_and_revalidated(installation): + control = installation["root"] / "control" + assert storage.inventory_directories(control)["exists"] is False + control.mkdir() + (control / "run1").mkdir() + plan = deepcopy(installation["plan"]) + plan["inventory_groups"].append(storage.inventory_directories(control)) + plan = reseal(plan) + assert storage.verify_inputs(plan) + (control / "run2").mkdir() + expect_error("inventory_changed", lambda: storage.verify_inputs(plan)) + (control / "unknown.lock").write_text("anything") + expect_error("unsafe_path", lambda: storage.inventory_directories(control)) + + +def test_same_entries_in_replaced_inventory_root_do_not_match(installation): + jobs = installation["jobs"] + old = jobs.with_name("old-jobs") + jobs.rename(old) + jobs.mkdir() + source = installation["source"] + source.write_bytes((old / source.name).read_bytes()) + source.chmod(0o640) + expect_error("inventory_changed", lambda: storage.verify_inputs(installation["plan"])) + + +@pytest.mark.parametrize("mutation", ["blob", "missing_blob", "extra_blob", "manifest", "private_mode", "symlink_blob"]) +def test_tampered_or_incomplete_snapshot_is_not_retrusted(installation, mutation): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + manifest = storage.verify_snapshot(plan, snapshot) + blob = state / "snapshot" / "files" / manifest["entries"][str(installation["source"])]["blob"] + if mutation == "blob": + blob.write_bytes(b"forged") + elif mutation == "missing_blob": + blob.unlink() + elif mutation == "extra_blob": + blob.with_name("unplanned.bin").write_bytes(b"unplanned") + elif mutation == "private_mode": + blob.chmod(0o644) + elif mutation == "symlink_blob": + blob.unlink() + blob.symlink_to(installation["source"]) + else: + del manifest["entries"][str(installation["config"])] + (state / "snapshot" / "manifest.json").write_text(json.dumps(manifest)) + snapshot["digest"] = storage._digest(manifest) + with pytest.raises(storage.IdentityStorageError): + storage.verify_snapshot(plan, snapshot) + + +def test_existing_corrupt_blob_is_not_overwritten_by_retry(installation): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + manifest = storage.verify_snapshot(plan, snapshot) + blob = state / "snapshot" / "files" / manifest["entries"][str(installation["source"])]["blob"] + blob.write_bytes(b"corruption must remain visible") + expect_error("state_conflict", lambda: storage.create_snapshot(plan, state)) + assert blob.read_bytes() == b"corruption must remain visible" + + +def test_snapshot_manifest_does_not_accept_boolean_integer_substitution(installation): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + manifest = storage.verify_snapshot(plan, snapshot) + manifest["schema_version"] = True + (state / "snapshot" / "manifest.json").write_text(json.dumps(manifest)) + snapshot["digest"] = storage._digest(manifest) + expect_error("invalid_snapshot", lambda: storage.verify_snapshot(plan, snapshot)) + + +def test_private_state_cannot_alias_other_files_through_a_hard_link(installation): + plan, state = installation["plan"], installation["state"] + storage.persist_plan(plan, state) + os.link(state / "plan.json", installation["root"] / "other-link") + expect_error("unsafe_path", lambda: storage.load_plan(state)) + + +def test_duplicate_json_fields_in_persisted_plan_are_not_ignored(installation): + plan, state = installation["plan"], installation["state"] + storage.persist_plan(plan, state) + path = state / "plan.json" + path.write_bytes(b'{"schema_version":999,' + path.read_bytes()[1:]) + expect_error("state_conflict", lambda: storage.load_plan(state)) + + +def test_interrupted_snapshot_can_resume_original_uuid_allocation(installation, monkeypatch): + plan, state = installation["plan"], installation["state"] + publish = storage._publish_once + calls = [] + def interrupt(path, content): + if path.suffix == ".bin": + calls.append(path) + if len(calls) == 2: + raise storage.IdentityStorageError("interrupted") + return publish(path, content) + monkeypatch.setattr(storage, "_publish_once", interrupt) + expect_error("interrupted", lambda: storage.create_snapshot(plan, state)) + assert storage.load_plan(state)["id_map"] == plan["id_map"] + assert not (state / "snapshot" / "manifest.json").exists() + metadata_before = (state / "snapshot" / "metadata.json").read_bytes() + monkeypatch.setattr(storage, "_publish_once", publish) + snapshot = storage.create_snapshot(storage.load_plan(state), state) + storage.verify_snapshot(plan, snapshot) + assert (state / "snapshot" / "metadata.json").read_bytes() == metadata_before + assert not installation["destination"].exists() + + +def test_snapshot_manifest_records_stable_timestamp_allocation_actions_and_artifact_types(installation): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + manifest = storage.verify_snapshot(plan, snapshot) + metadata = json.loads((state / "snapshot" / "metadata.json").read_text()) + timestamp = datetime.fromisoformat(manifest["created_at"]) + assert timestamp.tzinfo == timezone.utc + assert manifest["created_at"] == metadata["created_at"] + assert manifest["id_map"] == plan["id_map"] + assert manifest["actions"] == [{key: value for key, value in action.items() if key != "data"} + for action in plan["actions"]] + assert all(entry["artifact_kind"] == "file" for entry in manifest["entries"].values()) + assert all(entry["artifact_kind"] == "external" for entry in manifest["external_inputs"].values()) + assert "data" not in manifest["actions"][0] + assert storage.create_snapshot(plan, state) == snapshot + assert storage.verify_snapshot(plan, snapshot)["created_at"] == manifest["created_at"] + assert storage.load_plan(state) == plan + + +@pytest.mark.parametrize("timestamp", [None, True, "", "invalid", "2026-09-05T12:00:00", "2026-09-05T12:00:00+02:00"]) +def test_corrupt_snapshot_creation_timestamp_blocks_verification_and_retry(installation, timestamp): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + path = state / "snapshot" / "metadata.json" + metadata = json.loads(path.read_text()) + metadata["created_at"] = timestamp + path.write_text(json.dumps(metadata)) + expect_error("invalid_snapshot", lambda: storage.verify_snapshot(plan, snapshot)) + expect_error("invalid_snapshot", lambda: storage.create_snapshot(plan, state)) + + +def test_missing_snapshot_creation_evidence_is_not_regenerated(installation): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + path = state / "snapshot" / "metadata.json" + path.unlink() + expect_error("snapshot_incomplete", lambda: storage.verify_snapshot(plan, snapshot)) + expect_error("snapshot_incomplete", lambda: storage.create_snapshot(plan, state)) + assert not path.exists() + + +@pytest.mark.parametrize("field", ["created_at", "actions", "id_map", "artifact_kind"]) +def test_snapshot_manifest_metadata_is_bound_to_private_metadata_and_plan(installation, field): + plan, state = installation["plan"], installation["state"] + snapshot = storage.create_snapshot(plan, state) + manifest = storage.verify_snapshot(plan, snapshot) + if field == "created_at": + manifest[field] = "2000-01-01T00:00:00+00:00" + elif field == "actions": + manifest[field] = [] + elif field == "id_map": + manifest[field] = {} + else: + manifest["entries"][str(installation["source"])][field] = "unknown" + (state / "snapshot" / "manifest.json").write_text(json.dumps(manifest)) + snapshot["digest"] = storage._digest(manifest) + expect_error("invalid_snapshot", lambda: storage.verify_snapshot(plan, snapshot)) + + +def test_unsupported_state_filesystem_does_not_fall_back_to_insecure_copy(installation, monkeypatch): + def unsupported(*args, **kwargs): + raise OSError(errno.EOPNOTSUPP, "do-not-log-this-sensitive-path") + monkeypatch.setattr(storage.os, "link", unsupported) + expect_error("state_filesystem_unsupported", lambda: storage.persist_plan(installation["plan"], installation["state"])) + assert not (installation["state"] / "plan.json").exists() + + +def test_masked_io_error_suppresses_sensitive_exception_chain(installation, monkeypatch): + def denied(*args, **kwargs): + raise OSError(errno.EACCES, "sensitive-context-must-not-escape") + monkeypatch.setattr(storage.os, "open", denied) + with pytest.raises(storage.IdentityStorageError) as error: + storage.fingerprint_file(installation["source"]) + rendered = "".join(traceback.format_exception(error.type, error.value, error.tb)) + assert "sensitive-context-must-not-escape" not in rendered + assert error.value.code == "storage_unavailable" + + +def test_disk_full_is_sanitized_and_never_changes_user_data(installation, monkeypatch): + class Full: + f_bavail = 0 + f_frsize = 4096 + monkeypatch.setattr(storage.os, "fstatvfs", lambda fd: Full()) + before = installation["source"].read_bytes() + expect_error("insufficient_space", lambda: storage.create_snapshot(installation["plan"], installation["state"])) + assert installation["source"].read_bytes() == before + assert not installation["destination"].exists() + + +@pytest.mark.parametrize("bad", [None, {}, {"approved": False}, {"approved": 1}]) +def test_preconditions_deny_without_explicit_bound_acknowledgement(installation, bad): + plan = installation["plan"] + snapshot = storage.create_snapshot(plan, installation["state"]) + expect_error("approval_required", lambda: storage.verify_preconditions(plan, snapshot, bad)) + + +@pytest.mark.parametrize("field", ["independent_backup_acknowledged", "plan_id", "snapshot_digest"]) +def test_confirmation_cannot_be_reused_for_other_plan_snapshot(installation, field): + plan = installation["plan"] + snapshot = storage.create_snapshot(plan, installation["state"]) + approved = confirmation(plan, snapshot) + approved[field] = False if field.endswith("acknowledged") else "wrong" + expect_error("approval_required", lambda: storage.verify_preconditions(plan, snapshot, approved, quiescence_check=lambda: True)) + + +@pytest.mark.parametrize("check", [None, lambda: False, lambda: 1]) +def test_confirmation_does_not_replace_independent_quiescence_check(installation, check): + plan = installation["plan"] + snapshot = storage.create_snapshot(plan, installation["state"]) + expect_error("writers_active", lambda: storage.verify_preconditions(plan, snapshot, confirmation(plan, snapshot), quiescence_check=check)) + + +def test_confirmed_snapshot_gate_is_read_only_and_rechecks_changed_inputs(installation): + plan = installation["plan"] + snapshot = storage.create_snapshot(plan, installation["state"]) + approved = confirmation(plan, snapshot) + assert storage.verify_preconditions(plan, snapshot, approved, quiescence_check=lambda: True, + external_input_check=lambda: deepcopy(plan["external_inputs"])) + assert not installation["destination"].exists() + installation["source"].write_text("changed after confirmation") + expect_error("input_changed", lambda: storage.verify_preconditions(plan, snapshot, approved, quiescence_check=lambda: True)) + + +def test_cron_is_explicitly_captured_private_and_revalidated(installation): + plan = deepcopy(installation["plan"]) + plan["prerequisites"] = {"managed_cron_captured": True} + plan["external_inputs"] = {"managed_cron": {"kind": "crontab", "text": "# unrelated synthetic cron\n* * * * * synthetic\n"}} + plan = reseal(plan) + snapshot = storage.create_snapshot(plan, installation["state"]) + manifest = storage.verify_snapshot(plan, snapshot) + entry = manifest["external_inputs"]["managed_cron"] + blob = Path(snapshot["path"]) / "files" / entry["blob"] + assert blob.read_text() == plan["external_inputs"]["managed_cron"]["text"] + approved = confirmation(plan, snapshot) + expect_error("input_changed", lambda: storage.verify_preconditions(plan, snapshot, approved, quiescence_check=lambda: True)) + assert storage.verify_preconditions(plan, snapshot, approved, quiescence_check=lambda: True, + external_input_check=lambda: deepcopy(plan["external_inputs"])) + expect_error("input_changed", lambda: storage.verify_preconditions(plan, snapshot, approved, quiescence_check=lambda: True, external_input_check=lambda: {})) + + +@pytest.mark.parametrize("captured", [False, True]) +def test_incomplete_cron_prerequisite_blocks_snapshot_before_state_writes(installation, captured): + plan = deepcopy(installation["plan"]) + plan["prerequisites"] = {"managed_cron_captured": captured} + plan.pop("external_inputs") + plan = reseal(plan) + expect_error("snapshot_incomplete", lambda: storage.create_snapshot(plan, installation["state"])) + assert not installation["state"].exists() + + +@pytest.mark.parametrize("field", ["classification", "required", "status", "prerequisites"]) +def test_gate_does_not_infer_missing_eligibility_fields(installation, field): + plan = deepcopy(installation["plan"]) + plan.pop(field) + plan = reseal(plan) + snapshot = storage.create_snapshot(plan, installation["state"]) + expected = "snapshot_incomplete" if field == "prerequisites" else "invalid_plan" + expect_error(expected, lambda: storage.verify_preconditions( + plan, snapshot, confirmation(plan, snapshot), quiescence_check=lambda: True, + external_input_check=lambda: deepcopy(plan["external_inputs"]))) + + +def test_journal_is_durable_hash_linked_private_and_only_accepts_safe_fields(installation): + plan, state = installation["plan"], installation["state"] + storage.persist_plan(plan, state) + first = storage.append_journal(state, plan, "pending", "plan") + second = storage.append_journal(state, plan, "blocked", "verify", reason_code="input_changed", action_ids=["write_job_1"]) + assert first["sequence"] == 1 + assert second["previous"] == first["digest"] + assert storage.read_journal(state) == [first, second] + path = state / "journal.jsonl" + assert path.stat().st_mode & 0o777 == 0o600 + text = path.read_text() + assert "Synthetic documents" not in text + assert str(installation["source"]) not in text + expect_error("invalid_journal", lambda: storage.append_journal(state, plan, "failed", "verify", reason_code="sensitive raw exception")) + assert storage.read_journal(state) == [first, second] + + +@pytest.mark.parametrize("mutation", ["partial", "changed", "wrong_plan", "unknown_field", "bad_type"]) +def test_torn_or_forged_journal_is_not_silently_repaired(installation, mutation): + plan, state = installation["plan"], installation["state"] + storage.persist_plan(plan, state) + event = storage.append_journal(state, plan, "pending", "plan") + path = state / "journal.jsonl" + if mutation == "partial": + path.write_bytes(path.read_bytes()[:-1]) + else: + if mutation == "changed": + event["status"] = "applied" + elif mutation == "wrong_plan": + event["plan_id"] = "different" + elif mutation == "bad_type": + event["status"] = [] + else: + event["secret"] = "not-allowed" + path.write_text(json.dumps(event) + "\n") + before = path.read_bytes() + expect_error("invalid_journal", lambda: storage.append_journal(state, plan, "pending", "resume")) + assert path.read_bytes() == before From da7cda0d35bc4a370664aa1dd3f965f831e0a09a Mon Sep 17 00:00:00 2001 From: BorgForge Codex Date: Sat, 5 Sep 2026 20:08:37 +0200 Subject: [PATCH 03/17] Document user-approved migration assistant flow (#479) --- .../identity-migration-foundation.md | 7 +++ docs/maintainer/immutable-job-identity.md | 58 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/docs/maintainer/identity-migration-foundation.md b/docs/maintainer/identity-migration-foundation.md index d79a99b4..ce0343fa 100644 --- a/docs/maintainer/identity-migration-foundation.md +++ b/docs/maintainer/identity-migration-foundation.md @@ -132,6 +132,13 @@ coordinator work; no such endpoint exists in this phase. ## Confirmation, quiescence and cron +The approved user-facing sequence is specified in +[contract C4.1](immutable-job-identity.md#c41-approved-user-initiated-migration-assistant-479): +automatic read-only detection, explicit preparation, verified snapshot, +mandatory user backup-check pause, then a separate explicit apply action. +This remains #479 implementation work; the foundation does not expose an +assistant or turn snapshot completion/acknowledgement into automatic execution. + `verify_preconditions` denies by default. It requires an applicable pending plan, a verified snapshot, an explicit approval tied to `plan_id` and the snapshot digest, and acknowledgement that an independent backup is required. diff --git a/docs/maintainer/immutable-job-identity.md b/docs/maintainer/immutable-job-identity.md index e1cf9b11..bbdfb254 100644 --- a/docs/maintainer/immutable-job-identity.md +++ b/docs/maintainer/immutable-job-identity.md @@ -255,6 +255,64 @@ This validates states observed on each installation, not a claim that one production copy represents every user. Unknown states block before rewrites. Extend synthetic fixtures when new supported states are discovered. +### C4.1 Approved user-initiated migration assistant (#479) + +Maintainer-approved on 2026-09-05. This is the required runtime flow for #479, +not an activation of the phase-2 libraries. The visible steps below are migration +execution steps, not the nine development issues. + +1. **Migration required.** Startup performs read-only detection. A required or + unclear migration keeps Borg Backup UI in maintenance, with normal plugin + functions and new scheduled/manual work blocked. Installation, startup and + opening the assistant are not consent to prepare or apply. Authentication, + migration status, safe diagnostics and protected recovery access remain + available. This does not block Unraid itself or its array/pool controls. +2. **Prepare migration.** An explicit administrator action starts prerequisite + checks and preparation. Wait for existing backup/restore/test and detached + notification work to finish safely; never kill it to force migration. + Validate the actual mounted snapshot location and only then capture a + quiescent inventory. If required storage is unavailable, wait/block without + creating recovery data on an unmounted path. Persist the complete plan and + original UUID allocation only in the dedicated private state directory. +3. **Create and verify backup.** Create the exact pre-change snapshot and + independently verify completeness, sizes and checksums. Show the storage + path, creation time, size and verification result. The application performs + this technical integrity check; users are not required to audit JSON files. +4. **Mandatory pause: check and save the backup.** Offer an authenticated, + protected download/export and require the administrator to save and check + an independent copy. Explain that it may contain credentials or other + confidential data. Require explicit acknowledgement, for example: "I have + saved and checked a separate copy of the backup. I understand that it may + contain confidential data." Acknowledgement cannot prove an external copy + exists and never replaces automatic verification. Snapshot completion, + download and acknowledgement must not automatically start conversion. +5. **Run migration now.** A second explicit administrator action authorizes + installation-data conversion, bound to this exact plan and snapshot digest. + Immediately recheck storage availability, snapshot integrity, unchanged + source/cron state and writer exclusion. Changes invalidate the previous + approval; do not silently generate a new plan under old consent. Display + apply and final-verification progress, with actionable masked failure + details. Only successful complete verification and the existing startup + gate permit normal operation and the scheduler to resume. + +Keep the current step, completed/remaining steps, waiting reason and any failure +visible. Persist progress using the existing migration statuses and audit +phases; UI progress must not imply that a pending step has already succeeded. +Closing the browser or restarting the plugin retains maintenance and the +original plan/UUIDs/snapshot/journal; it is not automatic apply consent or a +return to normal operation. Reconnect must show the durable stage and permit +only validated, explicit continuation of interrupted work. If the previously +authorized backend operation is still running, reconnect only observes its +progress; it must not start a duplicate operation. Unknown or inconsistent +partial states remain blocked. After a migration failure, the existing requirement for a +failure-free restart remains in force. A partial conversion has no general +"cancel and resume normal operation" or automatic plugin-downgrade action. + +#479 acceptance tests must exercise both user actions, the compulsory snapshot +pause, rejected/unbound approval, changed inputs after approval, unavailable +storage, live workers, browser closure, restart and safe continuation. Bypassing +the UI via an API call must not bypass the same authenticated state gates. + ## C5. Owned storage and excluded data Resolve roots from configuration, not production paths in a migration script. From 4fb6822f846c747de66edf575e658c28b7c95641 Mon Sep 17 00:00:00 2001 From: BorgForge Codex Date: Sat, 5 Sep 2026 20:52:35 +0200 Subject: [PATCH 04/17] Introduce canonical job metadata and ID-based wizard (#473) --- api/job_model.py | 184 +++++++++ api/job_store.py | 154 ++++++++ api/migrations/immutable_job_id_v1.py | 9 + api/wizard_api.py | 369 +++++------------- borg_backup_ui.py | 14 +- docs/changelog.md | 11 + docs/maintainer/canonical-job-wizard.md | 112 ++++++ docs/maintainer/identity-dependencies.json | 10 + release-notes/pending/473.md | 2 + tests/canonical_wizard_support.py | 49 +++ tests/canonical_wizard_ui.cjs | 89 +++++ tests/test_canonical_job_wizard.py | 353 +++++++++++++++++ tests/test_canonical_wizard_ui.py | 34 ++ ...st_issue_205_wizard_repository_feedback.py | 9 +- tests/test_issue_450_modal_dismissal.py | 2 +- tests/test_issue_458_retention_help.py | 2 +- tests/test_issue_463_file_activity_log.py | 39 +- tests/test_repository_objects.py | 15 +- tests/test_required_field_markers.py | 2 +- tests/test_restore_archive_filter.py | 63 ++- tests/test_wizard_remote_repo.py | 25 +- ui/i18n/de.json | 23 +- ui/i18n/en.json | 23 +- ui/index.html | 9 +- ui/js/components/app-bindings.js | 9 +- ui/js/pages/wizard.js | 124 +++--- ui/style.css | 6 + 27 files changed, 1334 insertions(+), 407 deletions(-) create mode 100644 api/job_model.py create mode 100644 api/job_store.py create mode 100644 docs/maintainer/canonical-job-wizard.md create mode 100644 release-notes/pending/473.md create mode 100644 tests/canonical_wizard_support.py create mode 100644 tests/canonical_wizard_ui.cjs create mode 100644 tests/test_canonical_job_wizard.py create mode 100644 tests/test_canonical_wizard_ui.py diff --git a/api/job_model.py b/api/job_model.py new file mode 100644 index 00000000..3e3fcc17 --- /dev/null +++ b/api/job_model.py @@ -0,0 +1,184 @@ +"""Canonical, immutable job metadata contract (#447, #473). + +Pure validation and edit operations. Legacy conversion belongs exclusively to +the explicit migration boundary, never to a read or an ordinary wizard save. +""" + +from copy import deepcopy +import re +from uuid import UUID + +from job_source_paths import normalize_source_paths + + +JOB_SCHEMA_VERSION = 4 +MUTABLE_IDENTITY_FIELDS = {"job_key", "backup_type", "type_id", "location"} +ARCHIVE_TIMESTAMP_PATTERN = "YYYY-MM-DD_HH-mm-ss" +_SAFE = re.compile(r"[A-Za-z0-9_.-]+") + + +class JobValidationError(ValueError): + def __init__(self, code, message): + self.api_code = code + super().__init__(message) + + +def validate_job_id(value): + try: + parsed = UUID(value) if isinstance(value, str) else None + except ValueError: + parsed = None + if parsed is None or parsed.version != 4 or str(parsed) != value: + raise JobValidationError("invalid_job_id", "A canonical UUIDv4 job_id is required") + return value + + +def validate_archive_prefix(value): + if not isinstance(value, str) or not _SAFE.fullmatch(value) or value in {".", ".."}: + raise JobValidationError("invalid_archive_prefix", "Archive prefix must use letters, digits, dots, underscores or hyphens; '.' and '..' are not allowed") + return value + + +def archive_name_preview(prefix): + return validate_archive_prefix(prefix) + "-" + ARCHIVE_TIMESTAMP_PATTERN + + +def updated_archive_prefixes(prefix, previous): + validate_archive_prefix(prefix) + if not isinstance(previous, list): + raise JobValidationError("invalid_archive_prefix", "Archive prefix history must be a list") + for value in previous: + validate_archive_prefix(value) + return list(dict.fromkeys([prefix, *previous])) + + +def validate_job(meta, *, filename=None): + """Validate without repairing, allocating an ID, or discarding fields.""" + if not isinstance(meta, dict) or type(meta.get("schema_version")) is not int or meta["schema_version"] != JOB_SCHEMA_VERSION: + raise JobValidationError("job_migration_required", "Job metadata requires the explicit identity migration") + job_id = validate_job_id(meta.get("job_id")) + if filename is not None and filename != job_id + ".json": + raise JobValidationError("invalid_job_filename", "Job filename does not match job_id") + if MUTABLE_IDENTITY_FIELDS.intersection(meta): + raise JobValidationError("mutable_job_identity", "Canonical metadata must not contain mutable identity fields") + if not isinstance(meta.get("name"), str) or not meta["name"].strip(): + raise JobValidationError("invalid_job_name", "Job name must not be empty") + repo = meta.get("repository_key") + if not isinstance(repo, str) or not _SAFE.fullmatch(repo): + raise JobValidationError("invalid_job_repository", "A repository_key is required") + prefixes = meta.get("archive_prefixes") + if not isinstance(prefixes, list) or not prefixes or updated_archive_prefixes(prefixes[0], prefixes) != prefixes: + raise JobValidationError("invalid_archive_prefix", "Archive prefixes must be nonempty, ordered and unique") + aliases = meta.get("legacy_job_keys") + if not isinstance(aliases, list) or any(not isinstance(a, str) or not _SAFE.fullmatch(a) for a in aliases) or len(set(aliases)) != len(aliases): + raise JobValidationError("invalid_job_aliases", "Legacy aliases must be an ordered list of unique exact identifiers") + if normalize_source_paths(meta.get("source_paths")) != meta.get("source_paths"): + raise JobValidationError("invalid_source_paths", "Source paths must be canonical") + for field in ("enabled", "file_activity", "mount_before_run", "unmount_after_run"): + if field in meta and type(meta[field]) is not bool: + raise JobValidationError("invalid_job_settings", "A boolean job setting has an unsupported value") + if "compression" in meta and (not isinstance(meta["compression"], str) or not meta["compression"].strip()): + raise JobValidationError("invalid_job_settings", "Compression must be a nonempty string") + features = meta.get("features", {}) + if not isinstance(features, dict) or any(type(features.get(kind, False)) is not bool for kind in ("docker", "vm")): + raise JobValidationError("invalid_runtime_control", "Unsupported job feature settings") + for kind in ("docker", "vm"): + control = meta.get(kind + "_control") + if control is None and kind + "_control" not in meta: + continue + modes = {"all", "selected", "none"} | ({"except_selected"} if kind == "docker" else set()) + if not isinstance(control, dict) or not isinstance(control.get("mode"), str) or control["mode"] not in modes: + raise JobValidationError("invalid_runtime_control", "Unsupported runtime control mode") + selected = control.get("selected", []) + if not isinstance(selected, list) or any(not isinstance(v, str) or not v.strip() for v in selected): + raise JobValidationError("invalid_runtime_control", "Invalid runtime selection") + ack = "ack_appdata_risk" if kind == "docker" else "ack_domains_risk" + if ack in control and type(control[ack]) is not bool: + raise JobValidationError("invalid_runtime_control", "Invalid runtime acknowledgement") + if "retention" in meta: + retention = meta["retention"] + if not isinstance(retention, dict) or any( + key in retention and (not isinstance(retention[key], str) or not re.fullmatch(r"[0-9]+", retention[key])) + for key in ("daily", "weekly", "monthly", "yearly") + ): + raise JobValidationError("invalid_retention", "Unsupported retention settings") + return meta + + +def validate_job_inventory(jobs): + """Reject alias collisions and shared-repository prefix ownership overlap.""" + aliases, ownership = {}, [] + for job_id, job in jobs.items(): + validate_job(job, filename=job_id + ".json") + for alias in job["legacy_job_keys"]: + if alias in aliases or (alias in jobs and alias != job_id): + raise JobValidationError("ambiguous_job_alias", "A legacy alias has conflicting owners") + aliases[alias] = job_id + for prefix in job["archive_prefixes"]: + for other_repo, other_prefix, other_id in ownership: + if other_repo == job["repository_key"] and other_id != job_id and ( + prefix == other_prefix or prefix.startswith(other_prefix + "-") or other_prefix.startswith(prefix + "-") + ): + raise JobValidationError("ambiguous_archive_ownership", "Archive prefixes overlap with another job in the selected repository") + ownership.append((job["repository_key"], prefix, job_id)) + + +def new_job_defaults(): + return { + "schema_version": JOB_SCHEMA_VERSION, "legacy_job_keys": [], + "description": "", "icon": "sonstiges", "icon_color": "", "enabled": True, + "standard": "wizard", "runner": "scriptless-wizard-runner", "script": "", + "mount_before_run": True, "unmount_after_run": True, + "exclude_paths": [], "compression": "lz4", "file_activity": False, + "features": {"docker": False, "vm": False}, + "docker_control": {"mode": "none", "selected": [], "ack_appdata_risk": False}, + "vm_control": {"mode": "none", "selected": [], "ack_domains_risk": False}, + "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, + } + + +def job_to_params(meta): + """Only expose wizard-owned fields, not unknown settings or secrets.""" + result = {key: deepcopy(meta[key]) for key in ( + "description", "icon", "icon_color", "source_paths", "exclude_paths", + "repository_key", "mount_before_run", "unmount_after_run", "compression", + "file_activity", "docker_control", "vm_control", + ) if key in meta} + result.update(job_name=meta.get("name", ""), archive_prefix=(meta.get("archive_prefixes") or [""])[0]) + for key, value in meta.get("retention", {}).items(): + if key in {"daily", "weekly", "monthly", "yearly"}: + result["keep_" + key] = value + return result + + +def apply_wizard_changes(params, *, existing=None, job_id, now, duplicate=False): + """Patch exposed fields; retain every other existing setting verbatim.""" + if existing is not None: + validate_job(existing) + if not duplicate and job_id != existing["job_id"]: + raise JobValidationError("immutable_job_id", "Editing cannot change job_id") + result = deepcopy(existing) if existing is not None else new_job_defaults() + fresh = existing is None or duplicate + result.update(schema_version=JOB_SCHEMA_VERSION, job_id=validate_job_id(job_id), updated_at=now) + if fresh: + result.update(created_at=now, legacy_job_keys=[]) + if "job_name" in params: + result["name"] = params["job_name"].strip() + prefix = params.get("archive_prefix", (result.get("archive_prefixes") or [""])[0]) + result["archive_prefixes"] = updated_archive_prefixes(prefix, [] if fresh else result.get("archive_prefixes", [])) + for key in ( + "description", "icon", "icon_color", "repository_key", "source_paths", "exclude_paths", + "compression", "file_activity", "mount_before_run", "unmount_after_run", + ): + if key in params: + result[key] = deepcopy(params[key]) + for kind in ("docker", "vm"): + key = kind + "_control" + if key in params: + result.setdefault(key, {}).update(deepcopy(params[key])) + result.setdefault("features", {})[kind] = result[key]["mode"] != "none" + for period in ("daily", "weekly", "monthly", "yearly"): + if "keep_" + period in params: + result.setdefault("retention", {})[period] = params["keep_" + period] + validate_job(result) + return result diff --git a/api/job_store.py b/api/job_store.py new file mode 100644 index 00000000..50bdfbb8 --- /dev/null +++ b/api/job_store.py @@ -0,0 +1,154 @@ +"""Strict schema-v4 metadata persistence for the #447 cutover (#473). + +No legacy discovery, conversion, reconciliation, or scheduler side effects. +Repository-wide readers/writers are converted in #474 before installation. +""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path + +from inventory_store import atomic_write_bytes, atomic_write_json, inventory_lock +from job_model import JobValidationError, validate_job, validate_job_id, validate_job_inventory +from migrations.identity_storage import inventory_group, read_fingerprinted_file + + +def read_json(path, *, missing=None): + """Read a regular file without following symlinks or accepting duplicate keys.""" + _, raw = read_fingerprinted_file(Path(path)) + if raw is None: + return deepcopy(missing) + + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate member") + result[key] = value + return result + + def invalid_constant(_): + raise ValueError("invalid constant") + + try: + if len(raw) > 64 * 1024 * 1024: + raise ValueError("too large") + value = json.loads(raw, object_pairs_hook=pairs, parse_constant=invalid_constant) + if not isinstance(value, dict): + raise ValueError("not an object") + return value + except (ValueError, UnicodeError, RecursionError): + raise JobValidationError("invalid_job_inventory", "An owned inventory file is malformed; no changes were made") from None + + +def job_revision(meta): + return hashlib.sha256(json.dumps(meta, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()).hexdigest() + + +def read_jobs(jobs_dir): + result = {} + for name in inventory_group(Path(jobs_dir), [".json"])["entries"]: + meta = read_json(Path(jobs_dir) / name) + validate_job(meta, filename=name) + result[meta["job_id"]] = meta + validate_job_inventory(result) + return result + + +def read_job(jobs_dir, job_id): + validate_job_id(job_id) + # Validate the inventory as a whole: another ambiguous owner is not safe. + jobs = read_jobs(jobs_dir) + if job_id not in jobs: + raise JobValidationError("unknown_job_id", "Unknown job_id; editing cannot create a job") + return jobs[job_id] + + +def read_repositories(path): + store = read_json(path, missing={"schema_version": 1, "repositories": []}) + if type(store.get("schema_version")) is not int or store["schema_version"] != 1 or not isinstance(store.get("repositories"), list): + raise JobValidationError("invalid_job_repository", "Unsupported repository inventory") + keys = set() + for row in store["repositories"]: + if not isinstance(row, dict) or not isinstance(row.get("repository_key"), str) or not row["repository_key"] or row["repository_key"] in keys: + raise JobValidationError("invalid_job_repository", "Invalid or duplicate repository entry") + keys.add(row["repository_key"]) + return store + + +def validate_assignments(jobs, store): + keys = {row["repository_key"] for row in store["repositories"]} + if any(job["repository_key"] not in keys for job in jobs.values()): + raise JobValidationError("invalid_job_repository", "A job references a missing repository") + for repo in store["repositories"]: + if {"used_by", "source_job_keys"}.intersection(repo): + raise JobValidationError("job_migration_required", "Repository assignments require the explicit identity migration") + expected = {job_id for job_id, job in jobs.items() if job["repository_key"] == repo["repository_key"]} + for field in ("job_ids", "source_job_ids"): + values = repo.get(field) + if not isinstance(values, list) or any(not isinstance(v, str) for v in values) or len(set(values)) != len(values) or set(values) != expected: + raise JobValidationError("conflicting_job_assignments", "Repository assignments do not match job metadata") + + +def save_job_transaction(jobs_dir, repository_path, build, *, source_id=None, expected_revision=None, duplicate=False): + """Serialize read/validate/patch/write; roll back ordinary I/O failures. + + Each replacement is durable, but the pair is not crash-atomic. A crash + between replacements leaves inconsistent assignments which strict readers + reject, never reconcile silently. The global cutover gate is owned by #479. + """ + jobs_dir, repository_path = Path(jobs_dir), Path(repository_path) + with inventory_lock(repository_path.parent): + jobs = read_jobs(jobs_dir) + store = read_repositories(repository_path) + validate_assignments(jobs, store) + existing = None + if source_id is not None: + validate_job_id(source_id) + existing = jobs.get(source_id) + if existing is None: + raise JobValidationError("unknown_job_id", "Unknown job_id; editing cannot create a job") + if expected_revision is not None and job_revision(existing) != expected_revision: + raise JobValidationError("job_edit_conflict", "The job changed since it was opened; reload before saving") + metadata = build(deepcopy(existing)) + validate_job(metadata) + job_id = metadata["job_id"] + fresh = source_id is None or duplicate + if fresh and job_id in jobs: + raise JobValidationError("duplicate_job_id", "Allocated job_id already exists") + if not fresh and source_id != job_id: + raise JobValidationError("immutable_job_id", "Editing cannot change job_id") + if metadata["legacy_job_keys"] != ([] if fresh else existing["legacy_job_keys"]): + raise JobValidationError("immutable_job_aliases", "Ordinary saves cannot add or change legacy aliases") + jobs[job_id] = metadata + validate_job_inventory(jobs) + next_store = deepcopy(store) + for repo in next_store["repositories"]: + # Only update the affected ID; retain ordering and unknown fields. + for field in ("job_ids", "source_job_ids"): + values = repo[field] + if repo["repository_key"] == metadata["repository_key"]: + if job_id not in values: + values.append(job_id) + elif job_id in values: + values.remove(job_id) + validate_assignments(jobs, next_store) + target = jobs_dir / (job_id + ".json") + _, before_job = read_fingerprinted_file(target) + _, before_repos = read_fingerprinted_file(repository_path) + try: + atomic_write_json(target, metadata) + if next_store != store: + atomic_write_json(repository_path, next_store) + except Exception: + # Never turn a failed transaction into success. If rollback itself + # fails, inconsistent inputs remain detectable by strict readers. + if before_job is None: + target.unlink(missing_ok=True) + else: + atomic_write_bytes(target, before_job) + if before_repos is not None: + atomic_write_bytes(repository_path, before_repos) + raise + return metadata, target diff --git a/api/migrations/immutable_job_id_v1.py b/api/migrations/immutable_job_id_v1.py index d93b6d06..4c121020 100644 --- a/api/migrations/immutable_job_id_v1.py +++ b/api/migrations/immutable_job_id_v1.py @@ -354,6 +354,15 @@ def _plan_jobs(scan, jobs_dir, conf, allocator, journal): for key in _LEGACY: target.pop(key, None) target.update(schema_version=4, job_id=job_id) + # Share the target model's validation, but never its creation/write path. + try: + try: + from ..job_model import validate_job + except ImportError: + from job_model import validate_job + validate_job(target, filename=job_id + ".json") + except ValueError as exc: + _fail(getattr(exc, "api_code", "invalid_job_settings"), source) jobs[job_id], sources[job_id] = target, source if seen_legacy and canonical_ids: # A mixed on-disk cutover is not a new installation to re-plan. The diff --git a/api/wizard_api.py b/api/wizard_api.py index 021a368f..fbfbb61d 100644 --- a/api/wizard_api.py +++ b/api/wizard_api.py @@ -5,18 +5,16 @@ scriptless wizard runner. """ -import json +from copy import deepcopy import os import re from datetime import datetime, timezone from pathlib import Path from typing import Optional -from job_source_paths import JOB_SCHEMA_VERSION, SourcePathValidationError, normalize_source_paths - - -def _type_upper(type_id: str) -> str: - return re.sub(r"[^A-Z0-9]", "_", type_id.upper()) +from job_source_paths import normalize_source_paths +from job_model import (JobValidationError, apply_wizard_changes, archive_name_preview, + job_to_params, new_job_defaults, validate_archive_prefix, validate_job_id) _RUNTIME_MODES = {"all", "selected", "none"} @@ -150,11 +148,15 @@ def _runtime_control_from_params(params: dict, kind: str, existing: Optional[dic existing = existing if isinstance(existing, dict) else {} legacy_key = "use_docker" if kind == "docker" else "use_vm" raw = params.get(f"{kind}_control") + if f"{kind}_control" in params and not isinstance(raw, dict): + raise JobValidationError("invalid_runtime_control", "Runtime control must be an object") source = raw if isinstance(raw, dict) else {} if not source and isinstance(existing.get(f"{kind}_control"), dict): source = existing.get(f"{kind}_control") or {} mode = str(source.get("mode") or "").strip().lower() + if source and mode not in _runtime_modes(kind): + raise JobValidationError("invalid_runtime_control", "Unsupported runtime control mode") if mode not in _runtime_modes(kind): mode = "all" if bool(params.get(legacy_key, False)) else "none" @@ -240,14 +242,22 @@ def validate_params( ui_config: Optional[dict] = None, require_runtime_ack: bool = True, ) -> None: - """Wirft ValueError bei ungültigen Parametern.""" - type_id = params.get("type_id", "").strip() - if not type_id: - raise ValueError("Type ID must not be empty") - if not re.fullmatch(r"[a-z0-9_]+", type_id): - raise ValueError("Type ID may contain only lowercase letters, digits, and underscores") - if not params.get("job_name", "").strip(): - raise ValueError("Job name must not be empty") + """Validate effective wizard fields; never convert legacy metadata.""" + from jobs_api import get_jobs_meta_dir + from job_store import read_jobs + mode, source_id = _request_identity(params) + if mode == "edit" and not allow_existing: + raise JobValidationError("invalid_wizard_mode", "Editing requires edit mode") + jobs = read_jobs(get_jobs_meta_dir(scripts_dir, data_root)) + existing = jobs.get(source_id) if source_id else None + if source_id and existing is None: + raise JobValidationError("unknown_job_id", "Unknown job_id; editing cannot create a job") + effective = job_to_params(existing if existing is not None else new_job_defaults()) + effective.update(deepcopy(params)) + params.update(effective) + validate_archive_prefix(params.get("archive_prefix")) + if not isinstance(params.get("job_name"), str) or not params["job_name"].strip(): + raise JobValidationError("invalid_job_name", "Job name must not be empty") retention = _retention_from_params(params) params["file_activity"] = _bool_value(params.get("file_activity"), default=False) for period, value in retention.items(): @@ -260,17 +270,16 @@ def validate_params( params["repo_path"] = _repository_path(selected_repo, ui_config) params["encryption"] = _repository_encryption(selected_repo, str(params.get("encryption", "repokey-blake2"))) - location = params.get("location", "local") - if location not in ("local", "usb", "smb", "storagebox"): - raise ValueError(f"Invalid location: {location!r}") from repository_context import storage_by_key selected_storage_key = str(selected_repo.get("storage_key") or "").strip() selected_storage = storage_by_key(ui_config or {}, selected_storage_key) repository_location = str(selected_storage.get("location") or selected_storage.get("storage_type") or "").strip().lower() if repository_location == "ssh": repository_location = "storagebox" - if repository_location != location: - raise ValueError("Selected repository does not match the selected storage location") + location = repository_location + if location not in {"local", "usb", "smb", "storagebox"}: + raise ValueError("Selected repository has an unsupported storage location") + params["location"] = location requested_storage_key = str(params.get("storage_key") or "").strip() if requested_storage_key and requested_storage_key != selected_storage_key: raise ValueError("Selected repository does not belong to the selected storage target") @@ -302,11 +311,8 @@ def validate_params( if not bool(vm_control.get("ack_domains_risk", False)): raise ValueError("VM domain backup risk must be acknowledged when not shutting down all VMs") - from jobs_api import get_jobs_meta_dir - job_key = f"{type_id}_{location}" - meta_target = get_jobs_meta_dir(scripts_dir, data_root) / f"{job_key}.json" - if meta_target.exists() and not allow_existing: - raise FileExistsError(f"Job already exists: {type_id}_{location}") + params["docker_control"] = docker_control + params["vm_control"] = vm_control def _repository_from_params(params: dict, ui_config: Optional[dict]) -> Optional[dict]: @@ -314,10 +320,11 @@ def _repository_from_params(params: dict, ui_config: Optional[dict]) -> Optional if not repository_key or not ui_config: return None try: - from repositories_api import read_repository_store - rows = read_repository_store(ui_config).get("repositories", []) - except Exception: - return None + from repositories_api import repositories_file + from job_store import read_repositories + rows = read_repositories(repositories_file(ui_config))["repositories"] + except OSError: + raise ValueError("Repository inventory is not readable") from None for row in rows if isinstance(rows, list) else []: if str(row.get("repository_key") or "").strip() == repository_key: return row @@ -343,147 +350,57 @@ def _repository_encryption(repo: Optional[dict], fallback: str = "repokey-blake2 return str(repo.get("encryption") or fallback).strip() or fallback -def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dict: - from archive_prefix import archive_prefix_from_backup_type, normalize_archive_prefixes - from jobs_api import discover_jobs, get_jobs_meta_dirs, resolve_data_root - from config_api import read_expanded_conf - - data_root = resolve_data_root(ui_config) - jobs = {j.key: j for j in discover_jobs(scripts_dir, data_root)} - if job_key not in jobs: - raise ValueError(f"Unknown job: {job_key}") - - info = jobs[job_key] - conf = read_expanded_conf(ui_config) - type_id = str(info.backup_type or "").lower() - location = str(info.location or "local").lower() - - # Prefer explicit wizard metadata values if available. - meta_source_paths: list[str] = [] - meta_exclude_paths: list[str] = [] - meta_compression = "" - meta_file_activity = False - meta_keep_daily = "" - meta_keep_weekly = "" - meta_keep_monthly = "" - meta_keep_yearly = "" - meta_repository_key = "" - meta_mount_before_run = True - meta_unmount_after_run = True - meta_archive_prefixes: list[str] = [] - meta: dict = {} - meta_docker_control = {"mode": "all" if bool(info.has_docker) else "none", "selected": [], "ack_appdata_risk": False} - meta_vm_control = {"mode": "all" if bool(info.has_vm) else "none", "selected": [], "ack_domains_risk": False} - for meta_dir in get_jobs_meta_dirs(scripts_dir, data_root): - meta_file = meta_dir / f"{job_key}.json" - if not meta_file.exists(): - continue - try: - candidate = json.loads(meta_file.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, UnicodeDecodeError, TypeError): - continue - if not isinstance(candidate, dict): - raise ValueError(f"Wizard metadata root is not an object: {job_key}") - try: - meta_source_paths = normalize_source_paths( - candidate.get("source_paths"), field=f"Job '{job_key}' source_paths" - ) - except SourcePathValidationError as exc: - raise ValueError( - f"Job '{job_key}' has not been migrated to structured source paths: {exc}. " - "Review Settings > System Health & Migration before editing or running this job." - ) from exc - try: - meta = candidate - meta_exclude_paths = _exclude_paths(meta.get("exclude_paths", [])) - meta_compression = str(meta.get("compression") or "").strip() - meta_file_activity = _bool_value(meta.get("file_activity"), default=False) - meta_ret = meta.get("retention") if isinstance(meta.get("retention"), dict) else {} - meta_keep_daily = str(meta_ret.get("daily") or "").strip() - meta_keep_weekly = str(meta_ret.get("weekly") or "").strip() - meta_keep_monthly = str(meta_ret.get("monthly") or "").strip() - meta_keep_yearly = str(meta_ret.get("yearly") or "").strip() - meta_repository_key = str(meta.get("repository_key") or "").strip() - meta_mount_before_run = bool(meta.get("mount_before_run", True)) - meta_unmount_after_run = bool(meta.get("unmount_after_run", True)) - meta_archive_prefixes = normalize_archive_prefixes([ - archive_prefix_from_backup_type(type_id), - *(meta.get("archive_prefixes") if isinstance(meta.get("archive_prefixes"), list) else []), - ]) - meta_docker_control = _runtime_control_from_meta(meta, "docker") - meta_vm_control = _runtime_control_from_meta(meta, "vm") - break - except (TypeError, ValueError): - continue +def _request_identity(params): + if not isinstance(params, dict): + raise ValueError("Wizard payload must be an object") + if {"type_id", "backup_type", "job_key", "existing_job_key", "legacy_job_keys", "archive_prefixes"}.intersection(params): + raise JobValidationError("legacy_wizard_request", "Wizard requests must use job_id and archive_prefix") + mode = params.get("_wizard_mode", "create") + if not isinstance(mode, str) or mode not in {"create", "edit", "duplicate"}: + raise JobValidationError("invalid_wizard_mode", "Unknown wizard operation") + source_id = params.get("job_id") + if mode == "create" and source_id is not None: + raise JobValidationError("immutable_job_id", "New job IDs are assigned by the server") + if mode != "create": + validate_job_id(source_id) + return mode, source_id + + +def load_job_for_wizard(job_id: str, scripts_dir: Path, ui_config: dict) -> dict: + from jobs_api import get_jobs_meta_dir, resolve_data_root + from job_store import read_job, read_json, job_revision + from repository_context import storage_by_key - from repository_context import RepositoryContextError, resolve_job_repository_context - if not meta: - raise ValueError(f"Wizard metadata is missing: {job_key}") - assignment_error = "" - try: - repository_context = resolve_job_repository_context( - ui_config, - job_key, - job=meta, - require_passphrase_file=False, - ) - repo_path = str(repository_context["repository_path"]) - except RepositoryContextError as exc: - repository_context = {} - repo_path = "" - assignment_error = str(exc) - compression = meta_compression or conf.get(f"COMPRESSION_{_type_upper(type_id)}", "lz4") - - # Prefer explicit job metadata name (JSON) over display label with location suffix. - # This keeps edited names stable (e.g. "Flash" stays "Flash", not "Flash - Lokal"). - from schedule_api import get_schedules - - schedule = get_schedules(ui_config).get(job_key) - if not isinstance(schedule, dict): - schedule = None - - params = { - "job_key": job_key, - "type_id": type_id, - "job_name": (info.name or "").strip() or info.display_name or job_key, - "description": info.description or "", - "icon": str(getattr(info, "icon", "") or "").strip().lower(), - "icon_color": str(getattr(info, "icon_color", "") or "").strip().lower(), - "location": location, - "use_docker": meta_docker_control["mode"] != "none", - "use_vm": meta_vm_control["mode"] != "none", - "docker_control": meta_docker_control, - "vm_control": meta_vm_control, - "source_paths": meta_source_paths, - "exclude_paths": meta_exclude_paths, - "repo_path": repo_path or "", - "repository_key": meta_repository_key, - "repository_assignment_error": assignment_error, - "mount_before_run": meta_mount_before_run, - "unmount_after_run": meta_unmount_after_run, - "compression": compression, - "file_activity": meta_file_activity, - "encryption": str(repository_context.get("encryption") or ""), - "passphrase": "", - "keep_daily": meta_keep_daily or conf.get(f"RETENTION_{_type_upper(type_id)}_DAILY", "7"), - "keep_weekly": meta_keep_weekly or conf.get(f"RETENTION_{_type_upper(type_id)}_WEEKLY", "4"), - "keep_monthly": meta_keep_monthly or conf.get(f"RETENTION_{_type_upper(type_id)}_MONTHLY", "6"), - "keep_yearly": meta_keep_yearly or conf.get(f"RETENTION_{_type_upper(type_id)}_YEARLY", "3"), - "standard": info.standard, - "archive_prefixes": meta_archive_prefixes or normalize_archive_prefixes([ - archive_prefix_from_backup_type(type_id), - ]), - "schedule": { - "cron": str(schedule.get("cron") or "").strip(), - "enabled": bool(schedule.get("enabled", True)), - } if schedule else None, - } + meta = read_job(get_jobs_meta_dir(scripts_dir, resolve_data_root(ui_config)), job_id) + params = job_to_params(meta) + params.setdefault("file_activity", False) + repo = _repository_from_params(params, ui_config) + if repo is None: + raise JobValidationError("invalid_job_repository", "The assigned repository is missing") + storage = storage_by_key(ui_config, repo.get("storage_key", "")) + location = str(storage.get("location") or storage.get("storage_type") or "") + if location == "ssh": + location = "storagebox" + # Read the UUID schedule map directly; get_schedules still owns a legacy + # discovery/cleanup path until #474. Reading an editor must never clean it. + schedules = read_json(resolve_data_root(ui_config) / "config" / "schedules.json", missing={}) + schedule = schedules.get(job_id) + if schedule is not None and not isinstance(schedule, dict): + raise JobValidationError("invalid_job_schedule", "The job schedule is malformed") + params.update( + job_id=job_id, revision=job_revision(meta), location=location, + storage_key=repo.get("storage_key", ""), repo_path=_repository_path(repo, ui_config), + encryption=_repository_encryption(repo), passphrase="", + archive_prefixes=list(meta["archive_prefixes"]), + archive_name_preview=archive_name_preview(meta["archive_prefixes"][0]), + schedule=deepcopy(schedule), + ) return params def generate_flow_preview(params: dict, ui_config: Optional[dict] = None, scripts_dir: Optional[Path] = None) -> dict: """Erzeugt eine textuelle Backup-Flow-Vorschau fuer den Wizard.""" - type_id = params["type_id"].strip() + prefix = validate_archive_prefix(params.get("archive_prefix")) location = params.get("location", "local") source_paths = normalize_source_paths(params.get("source_paths")) exclude_paths = _exclude_paths(params.get("exclude_paths", [])) @@ -543,7 +460,9 @@ def add_step(code: str, message: str, **params) -> None: } if location == "storagebox" else {"checked": False, "exists": False, "needs_init_confirm": False, "message": ""} return { "runner": "scriptless-wizard-runner", - "job_key": f"{type_id}_{location}", + "job_id": params.get("job_id"), + "job_name": params.get("job_name", ""), + "archive_name_preview": archive_name_preview(prefix), "summary": { "location": location, "repo": repo_path, @@ -569,109 +488,33 @@ def add_step(code: str, message: str, **params) -> None: def save_job(params: dict, scripts_dir: Path, data_root: Optional[Path] = None, ui_config: Optional[dict] = None) -> dict: - """Speichert Job-eigene Wizard-Metadaten mit kanonischer Repository-Referenz.""" - from archive_prefix import archive_prefix_from_backup_type, normalize_archive_prefixes + """Save one ID-based job and its assignments; never rename an identity.""" + from uuid import uuid4 from jobs_api import get_jobs_meta_dir - type_id = params["type_id"].strip() - location = params.get("location", "local") - description = params.get("description", "").strip() - icon = str(params.get("icon", "")).strip().lower() - icon_color = str(params.get("icon_color", "")).strip().lower() - retention = _retention_from_params(params) - file_activity = _bool_value(params.get("file_activity"), default=False) - selected_repo = _repository_from_params(params, ui_config) - if not selected_repo: - raise ValueError("Selected repository object was not found") - selected_repository_key = str((selected_repo or {}).get("repository_key") or params.get("repository_key") or "").strip() - - scripts_dir.mkdir(parents=True, exist_ok=True) - existing_job_key = str(params.get("existing_job_key", "")).strip() + from job_store import job_revision, save_job_transaction + from repositories_api import repositories_file - # ── Wizard-Metadaten schreiben (Phase 2) ───────────────────────────────── - job_key = f"{type_id}_{location}" - now_iso = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - jobs_meta_dir = get_jobs_meta_dir(scripts_dir, data_root) - jobs_meta_dir.mkdir(parents=True, exist_ok=True) - meta_path = jobs_meta_dir / f"{job_key}.json" - - existing = {} - if meta_path.exists(): - try: - existing = json.loads(meta_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, UnicodeDecodeError): - existing = {} - elif existing_job_key and existing_job_key != job_key: - old_meta_path = jobs_meta_dir / f"{existing_job_key}.json" - if old_meta_path.exists(): - try: - existing = json.loads(old_meta_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, UnicodeDecodeError): - existing = {} - - mount_before_run = bool(params.get("mount_before_run", existing.get("mount_before_run", True))) - unmount_after_run = bool(params.get("unmount_after_run", existing.get("unmount_after_run", True))) - docker_control = _runtime_control_from_params(params, "docker", existing) - vm_control = _runtime_control_from_params(params, "vm", existing) - archive_prefixes = normalize_archive_prefixes([ - archive_prefix_from_backup_type(type_id), - archive_prefix_from_backup_type(existing.get("backup_type")), - *(existing.get("archive_prefixes") if isinstance(existing.get("archive_prefixes"), list) else []), - ]) - - metadata = { - "schema_version": JOB_SCHEMA_VERSION, - "job_key": job_key, - "name": params.get("job_name", "").strip() or job_key, - "description": description, - "icon": icon, - "icon_color": icon_color, - "enabled": bool(existing.get("enabled", True)), - "standard": "wizard", - "backup_type": type_id, - "archive_prefixes": archive_prefixes, - "location": location, - "mount_before_run": mount_before_run if location == "smb" else True, - "unmount_after_run": unmount_after_run if location == "smb" else True, - "script": "", - "runner": "scriptless-wizard-runner", - "source_paths": normalize_source_paths(params.get("source_paths")), - "exclude_paths": _exclude_paths(params.get("exclude_paths", [])), - "features": { - "docker": docker_control["mode"] != "none", - "vm": vm_control["mode"] != "none", - }, - "docker_control": docker_control, - "vm_control": vm_control, - "compression": str(params.get("compression", "lz4")).strip() or "lz4", - "file_activity": file_activity, - "retention": retention, - "created_at": existing.get("created_at", now_iso), - "updated_at": now_iso, - } - metadata["repository_key"] = selected_repository_key - if isinstance(existing.get("restore_test_policy"), dict): - metadata["restore_test_policy"] = dict(existing["restore_test_policy"]) - - repo_config = ui_config or { + mode, source_id = _request_identity(params) + config = ui_config or { "BACKUP_SCRIPTS_DIR": str(data_root or (scripts_dir.parent if scripts_dir.name == "scripts" else scripts_dir)), } - from repositories_api import save_job_repository_transaction - previous_meta_path = jobs_meta_dir / f"{existing_job_key}.json" if existing_job_key else None - save_job_repository_transaction( - repo_config, - meta_path, - metadata, - selected_repository_key, - job_key, - previous_repository_key=str(existing.get("repository_key") or ""), - previous_job_key=existing_job_key or job_key, - previous_metadata_path=previous_meta_path, + original_params = deepcopy(params) + + def build(existing): + effective = job_to_params(existing if existing is not None else new_job_defaults()) + effective.update(deepcopy(original_params)) + validate_params(effective, scripts_dir, data_root, allow_existing=mode == "edit", ui_config=config) + # Allocate only on the write path, after validation, never on preview/read. + job_id = source_id if mode == "edit" else str(uuid4()) + now = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + return apply_wizard_changes(effective, existing=existing, job_id=job_id, now=now, duplicate=mode == "duplicate") + + metadata, target = save_job_transaction( + get_jobs_meta_dir(scripts_dir, data_root), repositories_file(config), build, + source_id=source_id, expected_revision=params.get("expected_revision"), duplicate=mode == "duplicate", ) - return { - "filename": "", - "path": "", - "script": "", - "metadata_path": str(meta_path), - "regenerated_script": False, + "job_id": metadata["job_id"], "revision": job_revision(metadata), + "job_name": metadata["name"], "archive_name_preview": archive_name_preview(metadata["archive_prefixes"][0]), + "filename": "", "path": "", "script": "", "metadata_path": str(target), "regenerated_script": False, } diff --git a/borg_backup_ui.py b/borg_backup_ui.py index 62bcdb07..795c1db4 100644 --- a/borg_backup_ui.py +++ b/borg_backup_ui.py @@ -2148,11 +2148,13 @@ def _get_wizard_job(self, qs: str) -> dict: from wizard_api import load_job_for_wizard from jobs_api import resolve_scripts_dir params = _pqs(qs) - job_key = (params.get("job_key") or [""])[0].strip() - if not job_key: - raise ValueError("job_key is required") + if "job_key" in params: + raise ValueError("The wizard requires job_id, not job_key") + job_id = (params.get("job_id") or [""])[0] + if len(params.get("job_id", [])) != 1: + raise ValueError("Exactly one job_id is required") scripts_dir = resolve_scripts_dir(self.config) - return {"job": load_job_for_wizard(job_key, scripts_dir, self.config)} + return {"job": load_job_for_wizard(job_id, scripts_dir, self.config)} def _get_wizard_source_dirs(self, qs: str) -> dict: from urllib.parse import parse_qs as _pqs @@ -2547,13 +2549,11 @@ def _post_wizard_preview(self) -> dict: return {"flow": generate_flow_preview(body, self.config, scripts_dir)} def _post_wizard_save(self) -> dict: - from wizard_api import validate_params, save_job + from wizard_api import save_job from jobs_api import resolve_scripts_dir, resolve_data_root body = self._read_json_body() scripts_dir = resolve_scripts_dir(self.config) data_root = resolve_data_root(self.config) - mode = str(body.get("_wizard_mode", "create")).strip().lower() - validate_params(body, scripts_dir, data_root, allow_existing=(mode == "edit"), ui_config=self.config) return save_job(body, scripts_dir, data_root, self.config) def _start_restore_test_from_body(self, body: dict) -> dict: diff --git a/docs/changelog.md b/docs/changelog.md index 14958826..7dca1fcc 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,6 +6,17 @@ Das Plugin-Manifest `borg-backup-ui.plg` enthaelt nur noch eine kurze nutzerrele ## Unreleased +### Issue #473 (integration work for #447; not released) +- Added strict schema-v4 job validation, UUID-named metadata and canonical + create/edit/duplicate operations, retaining unexposed operational settings. +- Replaced Type ID in the wizard with the complete archive prefix, an exact + timestamp-pattern preview and retained prefix history in German and English. +- Wizard requests use job_id; schedule save retries reuse the returned ID. + Added stale-editor detection and strict repository ownership checks. +- Legacy/corrupt inputs are rejected without implicit conversion. UUID runtime, + scheduling, general inventory APIs and the startup migration gate remain + subsequent phases; no independently installable package is produced. + ### Issue #472 (integration work for #447; not released) - Added an inactive, read-only migration inventory and UUID planner with exact legacy-reference mapping, archive ownership checks and preserved history. diff --git a/docs/maintainer/canonical-job-wizard.md b/docs/maintainer/canonical-job-wizard.md new file mode 100644 index 00000000..f5f082ac --- /dev/null +++ b/docs/maintainer/canonical-job-wizard.md @@ -0,0 +1,112 @@ +# Canonical job model and wizard boundary + +Issue #473, phase 3/9 of #447. Integration-only; **not installable or released**. +The binding contract remains [immutable-job-identity.md](immutable-job-identity.md). + +## Implemented boundary + +- `api/job_model.py`: strict UUIDv4/schema-v4/filename validation, complete + prefix validation and preview, ordered prefix history, alias/ownership + checks, and settings-preserving create/edit/duplicate metadata operations. +- `api/job_store.py`: direct strict JSON reads and a serialized metadata plus + repository-assignment write transaction. Reads reject duplicate JSON keys, + symlinks, malformed records, missing IDs and unsupported schemas. They do + not discover legacy jobs, migrate files or clean schedules. +- `api/wizard_api.py`: the wizard validates effective settings and existing + repository/storage selection. Location comes from that selection and is + returned for display; it is not persisted as job identity. +- `ui/js/pages/wizard.js`: name, complete archive prefix, repository selection; + UUID remains internal. A new job receives an editable name-based suggestion, + but manual prefix edits and existing jobs are never auto-renamed. The preview + is `-YYYY-MM-DD_HH-mm-ss`, with no hidden `-backup` suffix. + The existing history tooltip explains current-repository-only archive scope. + +Changing a prefix retains all earlier prefixes, including when switching back +to a previous value. Duplicating creates a new ID, clears aliases and old +prefix history, and requires independent archive ownership. It preserves +operational settings but does not copy schedule, status, logs or restore proof. +Multiple jobs may use the same repository with nonoverlapping archive scopes. +Name equality alone is not a conflict. + +## Wizard request contract + +This is the **target API**, not a published compatibility promise for the +current stable version. No mutable-key fallback or dual writer is installed. + +| Operation | Request | Result | +| --- | --- | --- | +| Load | `GET /api/wizard/job?job_id=` | Exposed job settings, `job_id`, `revision`, prefix/history/preview, derived location and saved UUID schedule | +| Preview | `POST /api/wizard/preview`, wizard fields and `_wizard_mode` | Effective flow and exact archive-name pattern; no allocated ID or metadata write | +| Create | `POST /api/wizard/save`, `_wizard_mode: "create"`, no ID | Server-generated UUID, `.json`, empty aliases | +| Edit | Same endpoint, `_wizard_mode: "edit"`, `job_id`, changed fields | Same ID/file, preserved unknown settings, updated prefix history | +| Duplicate | Same endpoint, `_wizard_mode: "duplicate"`, source `job_id`, new prefix/name | New ID/file, empty aliases/history except the chosen new prefix | + +The form sends `expected_revision` for edits/duplicates. It is the SHA-256 of +the complete opened metadata, checked again under the inventory lock. A stale +editor is rejected. Ordinary saves cannot submit `legacy_job_keys` or replace +`archive_prefixes`; the server owns both lists. Unknown IDs never imply create. + +Create/save returns `job_id`, `revision`, `job_name`, `archive_name_preview` +and the existing scriptless result fields. The UI uses the returned ID for +`PUT /api/schedules`; it never reconstructs a key from name/prefix/location. +If schedule saving fails after job creation, a retry edits the returned ID +instead of creating another job. General schedule API conversion is #474. + +## Persistence and failure limits + +The transaction reads the complete canonical job and repository inventories +under the existing inventory lock. Before changing anything it verifies +aliases, prefix ownership and both repository reverse-reference lists against +job assignments. It patches only the affected ID in `job_ids` and +`source_job_ids`, retaining order and unrelated repository fields. + +Normal I/O failures restore original bytes. Each file replacement is durable; +the metadata/repository pair is **not claimed to be crash-atomic**. A process +crash between replacements can leave a detectable assignment conflict, which +must block operation rather than trigger silent reconciliation. #479 must wire +the global startup/API gate for this state before any candidate is published. +The existing migration storage module supplies only pure/direct safe-read +primitives here; its planner, snapshotter and applier are never invoked by a +wizard read/save. + +The migration planner now validates its proposed schema-v4 jobs through the +same model. Legacy conversion remains explicit and inactive. These checks do +not claim full cross-store migration or whole-installation verification. + +## Remaining phase owners and release gate + +- #474: general job discovery and inventory APIs, repository normalizers, + deletion guards and maintenance, UUID schedule map/managed cron. Until then, + **do not run old repository writers on schema-v4 data**: they still normalize + legacy assignment fields. Do not feed new jobs into legacy discovery. +- #475: runner, archive creation with the exact prefix, status, logs, recovery + and notifications. Model + and preview tests are not evidence of an end-to-end backup yet. +- #476-#478: dashboards/reports, restore, imports and remaining + persistence boundaries. Duplicating through the wizard is available at its + API/entry-point boundary; general job actions are not cut over here. +- #479: approved manual migration assistant, snapshot/independent backup + acknowledgement, final verification and first installable test candidate. +- #452: repository-change warning and explicit confirmation is separate and + was not implemented at the start of #473. Nothing here replaces that issue + with a repository-history feature or claims the confirmation is complete. + +User manuals/screenshots describing the published product remain unchanged +until the integrated workflow is ready. The pending release-note fragment is +held for the final #447 candidate, not an intermediate release. + +## Focused verification + +- `test_canonical_job_wizard.py`: real metadata/assignment lifecycle, ID + retention, exact prefixes, unknown settings, stale revisions, corrupt inputs, + write-failure rollback, HTTP handler boundaries and synthetic planner output. +- `test_canonical_wizard_ui.py` plus `canonical_wizard_ui.cjs`: execute the + JavaScript collector/preview/save retry in both languages. Node.js is needed; + set `BBUI_TEST_NODE` if it is not on PATH. A skipped JS test is not a pass. +- Existing wizard, source/exclusion, retention, activity-log and schedule-load + regressions use explicit synthetic target fixtures. The test-only fixture + helper is not a migration engine and is not imported by production code. +- The legacy runner's file-activity tests remain independent until #475; + they do not pretend to prove the new wizard-to-runner end-to-end path. +- Full preflight, packages and Unraid migration/backup acceptance tests remain + deferred under the approved #447 integration exception. diff --git a/docs/maintainer/identity-dependencies.json b/docs/maintainer/identity-dependencies.json index 3a0ae816..01a40dab 100644 --- a/docs/maintainer/identity-dependencies.json +++ b/docs/maintainer/identity-dependencies.json @@ -81,6 +81,16 @@ ], "target": "UUID filename/payload, exact aliases, full prefix and preserved effective settings; remove mutable identity from discovery and editing.", "files": [ + { + "path": "api/job_model.py", + "anchor": "validate_job", + "role": "strict canonical schema-v4 identity, prefix lifecycle and settings-preserving wizard patches (#473)" + }, + { + "path": "api/job_store.py", + "anchor": "save_job_transaction", + "role": "strict canonical inventory reads and UUID repository assignment transaction; general API cutover remains #474" + }, { "path": "api/jobs_api.py", "anchor": "discover_jobs", diff --git a/release-notes/pending/473.md b/release-notes/pending/473.md new file mode 100644 index 00000000..d05f6413 --- /dev/null +++ b/release-notes/pending/473.md @@ -0,0 +1,2 @@ +- Separate the editable job name and complete archive prefix from the permanent internal job identity (#447, #473). +- Show the exact archive-name pattern in the Job Wizard and retain previous prefixes when editing, without resetting other job settings (#473). diff --git a/tests/canonical_wizard_support.py b/tests/canonical_wizard_support.py new file mode 100644 index 00000000..5ff0d5b5 --- /dev/null +++ b/tests/canonical_wizard_support.py @@ -0,0 +1,49 @@ +"""Explicit target-model fixture setup, never a production migration (#473).""" + +import json +from pathlib import Path +from uuid import uuid4 + + +def canonical_fixture(config): + """Convert only the synthetic jobs/reverse links/schedules a test supplied. + + This helper is not a migration test or evidence that real data is eligible. + The dedicated identity planner tests cover actual detection/projection. + """ + from job_model import new_job_defaults + + root = Path(config["BACKUP_SCRIPTS_DIR"]) + jobs_dir = root / "config" / "jobs" + ids, jobs = {}, [] + for path in sorted(jobs_dir.glob("*.json")): + old = json.loads(path.read_text()) + if old.get("schema_version") == 4: + jobs.append(old) + continue + key, typ = old["job_key"], old["backup_type"] + job_id = str(uuid4()) + meta = new_job_defaults() + meta.update(old) + meta.update(schema_version=4, job_id=job_id, legacy_job_keys=[key], + archive_prefixes=list(dict.fromkeys([typ + "-backup", *old.get("archive_prefixes", [])]))) + for field in ("job_key", "backup_type", "type_id", "location"): + meta.pop(field, None) + (jobs_dir / (job_id + ".json")).write_text(json.dumps(meta)) + path.unlink() + ids[key] = job_id + jobs.append(meta) + repos = root / "config" / "repositories.json" + if repos.exists(): + data = json.loads(repos.read_text()) + for repo in data["repositories"]: + repo.pop("used_by", None) + repo.pop("source_job_keys", None) + assigned = [j["job_id"] for j in jobs if j["repository_key"] == repo["repository_key"]] + repo.update(job_ids=assigned, source_job_ids=list(assigned)) + repos.write_text(json.dumps(data)) + schedules = root / "config" / "schedules.json" + if schedules.exists(): + data = json.loads(schedules.read_text()) + schedules.write_text(json.dumps({ids.get(key, key): value for key, value in data.items()})) + return ids diff --git a/tests/canonical_wizard_ui.cjs b/tests/canonical_wizard_ui.cjs new file mode 100644 index 00000000..85024d92 --- /dev/null +++ b/tests/canonical_wizard_ui.cjs @@ -0,0 +1,89 @@ +// Executable wizard contract tests (#473), without a backend or production data. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const root = path.resolve(__dirname, '..'); +const script = fs.readFileSync(path.join(root, 'ui/js/pages/wizard.js'), 'utf8'); +const jobId = '11111111-1111-4111-8111-111111111111'; + +async function run(lang) { + const dictionary = JSON.parse(fs.readFileSync(path.join(root, `ui/i18n/${lang}.json`), 'utf8')); + const elements = new Map(); + const element = (id) => { + if (!elements.has(id)) elements.set(id, { value: '', checked: false, hidden: false, textContent: '', innerHTML: '', + classList: { add() {}, remove() {}, contains() { return false; } } }); + return elements.get(id); + }; + const translate = (key, params = {}) => { + const value = key.split('.').reduce((v, k) => v?.[k], dictionary); + assert.equal(typeof value, 'string', `Missing ${lang} translation: ${key}`); + return value.replace(/\{(\w+)\}/g, (_, name) => String(params[name] ?? '')); + }; + const calls = []; + let failSchedule = true; + const context = vm.createContext({ + window: { BBUI: { components: { i18n: { t: translate } }, core: { getSchedulesData: () => ({}), setSchedulesData() {} } }, addEventListener() {} }, + document: { getElementById: element }, + escHtml: (s) => String(s).replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"'), + apiErrorMessage: () => 'Synthetic failure', jobsState: {}, refreshJobs: async () => {}, showMsg() {}, + fetch: async (url, options) => { + calls.push({ url, body: JSON.parse(options.body) }); + return { ok: url === '/api/wizard/save' || !failSchedule, status: 400, + json: async () => url === '/api/wizard/save' ? { job_id: jobId, revision: 'new-revision', job_name: 'Display name' } : {} }; + }, + }); + vm.runInContext(script, context); + const evalJs = (s) => vm.runInContext(s, context); + element('wiz-job-name').value = 'Meine schöne Daten'; + evalJs('wizardSuggestArchivePrefix()'); + assert.equal(element('wiz-archive-prefix').value, 'meine-schone-daten-backup'); + element('wiz-archive-prefix').value = 'Exact.Prefix_1'; + evalJs('wizardState.prefixEdited = true; wizardSuggestArchivePrefix()'); + assert.equal(element('wiz-archive-prefix').value, 'Exact.Prefix_1'); + evalJs("wizardState.archivePrefixes = ['old-prefix', 'Exact.Prefix_1']; wizardRenderArchivePrefixSummary()"); + const html = element('wiz-archive-prefix-summary').innerHTML; + assert.ok(html.includes('Exact.Prefix_1-YYYY-MM-DD_HH-mm-ss')); + assert.ok(!html.includes('Exact.Prefix_1-backup')); + assert.ok(html.includes('old-prefix-*')); + assert.ok(html.includes(dictionary.wizard.archivePrefixHistoryHint)); + assert.ok(html.includes('role="tooltip"')); + for (const invalid of ['', '.', '..', 'a/b', 'a b', 'a*', ' a', 'a\n', ''],stage:'waiting',status:'blocked'}; + await assistant.refresh(); + assert.ok(html.includes('<script>unsafe</script>') && !html.includes('