diff --git a/api/activity_log.py b/api/activity_log.py index 7cdb9d87..fb4ff500 100644 --- a/api/activity_log.py +++ b/api/activity_log.py @@ -15,14 +15,9 @@ WINDOW_BYTES = 65536 SEARCH_BYTES = 1024 * 1024 -_KEY = re.compile(r"^[A-Za-z0-9_.-]{1,128}$") -_RUN = re.compile(r"^[A-Za-z0-9_.-]{8,96}$") - - -def activity_log_path(directory: Path, job_key: str, run_id: str) -> Path: - if not _KEY.fullmatch(job_key) or not _RUN.fullmatch(run_id): - raise ValueError("Invalid activity log identity") - return directory / f"Borg-Backup_{job_key}--activity-{run_id}.log" +def activity_log_path(directory: Path, job_id: str, run_id: str, name: str = "job") -> Path: + from job_runs import log_filename + return directory / log_filename(job_id, run_id, name) def open_activity_file(path: Path): @@ -33,49 +28,48 @@ def open_activity_file(path: Path): return os.fdopen(fd, "rb") -def resolve_activity_run(config: dict, job_key: str, run_id: str = "") -> tuple[Path, dict]: - from jobs_api import JobManager, durable_running_states, _runtime_log_dir +def resolve_activity_run(config: dict, job_id: str, run_id: str = "") -> tuple[Path, dict]: + from jobs_api import JobManager, durable_running_states from job_control import read_control_state - - if not _KEY.fullmatch(job_key): - raise ValueError("Invalid job key") - memory = JobManager.get().get_state(job_key) - current = memory if memory.get("running") else durable_running_states(config).get(job_key, memory) - run_id = run_id or str(current.get("run_id") or "") - path = activity_log_path(_runtime_log_dir(config), job_key, run_id) + from job_model import validate_job_id + from job_runs import find_run_status, validate_run_id + validate_job_id(job_id) + validate_run_id(run_id) + memory = JobManager.get().get_state(job_id) + current = memory if memory.get("running") else durable_running_states(config).get(job_id, memory) if current.get("run_id") == run_id: - if not current.get("file_activity"): - raise ValueError("File activity is not enabled for this run") - # The configured directory may have changed since this run started. - path = Path(current["log_file"]) state = dict(current) else: - # Exact run filenames allow reconnecting after completion or a UI - # restart, without accepting arbitrary filesystem paths from clients. - state = {"running": False, "exit_code": None, "run_id": run_id} + state = find_run_status(config, job_id, run_id) control = read_control_state(run_id) - if control.get("job_key") == job_key: + if control.get("job_id") == job_id: + if not state: + state = {**control, "running": not control.get("finished")} state["phase"] = control.get("phase", "") if control.get("finished") and not state.get("running"): state["exit_code"] = control.get("exit_code") from activity_log_capture import capture_record, capture_path - capture = capture_record(job_key, run_id) + capture = capture_record(job_id, run_id) if capture: - path = capture_path(capture) + state.setdefault("run_id", run_id) + state["file_activity"] = True + state["log_file"] = str(capture_path(capture)) state["capture"] = capture if capture.get("status") in {"saved", "failed"}: state["running"] = False state["exit_code"] = capture.get("exit_code") state["log_persistence_failed"] = capture.get("status") == "failed" or (capture.get("status") == "running" and not state.get("running")) - return path, state + if not state or not state.get("file_activity") or not state.get("log_file"): + raise FileNotFoundError("File activity is not available for this run") + return Path(state["log_file"]), state @contextmanager -def open_activity_run(config: dict, job_key: str, run_id: str = ""): +def open_activity_run(config: dict, job_id: str, run_id: str = ""): # RAM can be released between resolving its path and opening it. The saved # location is published first, allowing a retry without resetting cursors. for attempt in range(2): - path, state = resolve_activity_run(config, job_key, run_id) + path, state = resolve_activity_run(config, job_id, run_id) try: handle = open_activity_file(path) break @@ -119,8 +113,8 @@ def read_window(handle, start: int, end: int, *, running: bool = False, align_st def get_activity_window(config: dict, qs: dict) -> dict: - job = (qs.get("job") or [""])[0] - run = (qs.get("run") or [""])[0] + job = (qs.get("job_id") or [""])[0] + run = (qs.get("run_id") or [""])[0] with open_activity_run(config, job, run) as (_path, state, handle): info = os.fstat(handle.fileno()) size = info.st_size diff --git a/api/activity_log_capture.py b/api/activity_log_capture.py index 7918c180..67628939 100644 --- a/api/activity_log_capture.py +++ b/api/activity_log_capture.py @@ -20,9 +20,13 @@ def file_identity(info) -> str: def read_record(path: Path) -> dict: try: data = json.loads(path.read_text(encoding="utf-8")) - required = ("job_key", "run_id", "active_file", "retained_file", "active_file_id", "started_at") + required = ("job_id", "run_id", "active_file", "retained_file", "active_file_id", "started_at") if not isinstance(data, dict) or any(not isinstance(data.get(key), str) or not data[key] for key in required): return {} + from job_model import validate_job_id + from job_runs import validate_run_id + validate_job_id(data["job_id"]) + validate_run_id(data["run_id"]) return data if data.get("status") in {"running", "saved", "failed"} else {} except (OSError, ValueError): return {} @@ -38,12 +42,12 @@ def write_record(path: Path, data: dict) -> None: temporary.unlink(missing_ok=True) -def capture_record(job_key: str, run_id: str) -> dict: +def capture_record(job_id: str, run_id: str) -> dict: from activity_log import activity_log_path - activity_log_path(CAPTURE_ROOT, job_key, run_id) # Validate both components. + activity_log_path(CAPTURE_ROOT, job_id, run_id) # Validate both components. record = read_record(CAPTURE_ROOT / run_id / "capture.json") - if record.get("job_key") != job_key or record.get("run_id") != run_id: + if record.get("job_id") != job_id or record.get("run_id") != run_id: return {} return record @@ -63,17 +67,17 @@ def open_capture_file(record_path: Path): raise -def prepare_capture(job_key: str, run_id: str, destination: Path) -> tuple[Path, Path]: +def prepare_capture(job_id: str, run_id: str, destination: Path, *, name: str = "job") -> tuple[Path, Path]: from activity_log import activity_log_path - retained = activity_log_path(destination, job_key, run_id) - active = activity_log_path(CAPTURE_ROOT / run_id, job_key, run_id) + retained = activity_log_path(destination, job_id, run_id, name) + active = activity_log_path(CAPTURE_ROOT / run_id, job_id, run_id, name) active.parent.mkdir(parents=True, mode=0o700) with os.fdopen(os.open(active, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as handle: identity = file_identity(os.fstat(handle.fileno())) record_path = active.parent / "capture.json" write_record(record_path, { - "job_key": job_key, "run_id": run_id, "status": "running", + "job_id": job_id, "run_id": run_id, "job_name_snapshot": name, "status": "running", "active_file": str(active), "retained_file": str(retained), "active_file_id": identity, "started_at": datetime.now(timezone.utc).isoformat(), }) @@ -141,7 +145,7 @@ def running_captures() -> list[dict]: except (TypeError, ValueError): continue rows.append({ - "job_key": record["job_key"], "run_id": record["run_id"], + "job_id": record["job_id"], "run_id": record["run_id"], "running": True, "exit_code": None, "file_activity": True, "log_file": record["active_file"], "pid": record["pid"], "start_time": record["started_at"], "source": "activity_capture", @@ -150,6 +154,17 @@ def running_captures() -> list[dict]: def supervise(record_path: Path, command: list[str]) -> int: + from migration_barrier import MigrationBlocked, writer_lease + config = {"BACKUP_SCRIPTS_DIR": os.environ.get("BORG_SCRIPT_DIR") or "/boot/config/borg-backup"} + try: + with writer_lease(config): + return _supervise_admitted(record_path, command) + except MigrationBlocked as exc: + print(f"Backup capture start blocked: {exc.reason}", flush=True) + return 2 + + +def _supervise_admitted(record_path: Path, command: list[str]) -> int: # This small process owns persistence independently of the WebUI process. # The runner and Borg inherit stdout/stderr pointing directly at the RAM # file, so browser speed cannot block them and no Python log list grows. diff --git a/api/check_api.py b/api/check_api.py index ee458c7a..9bd54df3 100644 --- a/api/check_api.py +++ b/api/check_api.py @@ -6,6 +6,7 @@ """ import subprocess +import sys import threading import time import json @@ -15,8 +16,6 @@ from pathlib import Path from typing import Generator, List, Optional -from archive_prefix import archive_prefix_from_job_key as _archive_prefix_from_job_key - class _CheckState: def __init__( @@ -83,12 +82,25 @@ def start_repository( action: str = "check", mode: str = "quick", *, - job_key: str = "", + job_id: str = "", ) -> tuple: """Start a maintenance action for one managed repository object.""" - with self._lock: - if self._state is not None and not self._state.finished: - return False, "A repository maintenance action is already running" + from migration_barrier import acquire_writer_lease + lease = acquire_writer_lease(config) + try: + with self._lock, lease.activate(): + result = self._start_repository_locked(config, repository_key, action, mode, + job_id=job_id, lease=lease) + if not result[0]: + lease.close() + return result + except BaseException: + lease.close() + raise + + def _start_repository_locked(self, config, repository_key, action, mode, *, job_id, lease): + if self._state is not None and not self._state.finished: + return False, "A repository maintenance action is already running" action = str(action or "check").strip().lower() mode = str(mode or "quick").strip().lower() @@ -116,6 +128,9 @@ def start_repository( repo_path = effective_repository_path(storage, str(repository.get("relative_path") or "")) if not repo_path: return False, "Repository path is missing" + # Validate the selected UUID and complete archive scope before a + # mount or any other operation with external side effects. + cmd = self._repository_command(config, repository, repo_path, action, mode, job_id=job_id) passphrase_ref = str(repository.get("passphrase_ref") or "").strip() passphrase_file = Path(passphrase_ref) if passphrase_ref else None if passphrase_file is not None and not passphrase_file.is_file(): @@ -134,7 +149,6 @@ def start_repository( config, encryption=str(repository.get("encryption") or ""), ) - cmd = self._repository_command(config, repository, repo_path, action, mode, job_key=job_key) except Exception as exc: return False, f"Repository information is not readable: {exc}" @@ -162,9 +176,9 @@ def start_repository( repository=repository, ) state.cleanup = cleanup + state.migration_lease = lease state.append_line(f"[Info] Starting repository {action}: {' '.join(cmd[:-1])} {repo_path}") - with self._lock: - self._state = state + self._state = state threading.Thread( target=self._reader, args=(state,), @@ -181,7 +195,7 @@ def _repository_command( action: str, mode: str, *, - job_key: str = "", + job_id: str = "", ) -> list[str]: if action == "check": return [ @@ -194,57 +208,41 @@ def _repository_command( "--progress", repo_path, ] - from repository_context import jobs_using_repository - used_by = jobs_using_repository(config, str(repository.get("repository_key") or "")) - job_keys = [str(item or "").strip() for item in used_by if str(item or "").strip()] - selected_job_key = str(job_key or "").strip() - if selected_job_key: - if selected_job_key not in job_keys: + from job_model import validate_job_id + from repository_context import jobs_using_repository, load_job_metadata + ids = jobs_using_repository(config, repository["repository_key"]) + if job_id: + validate_job_id(job_id) + if job_id not in ids: raise ValueError("The selected retention source job does not use this repository") - elif len(job_keys) > 1: - raise ValueError("Multiple backup jobs use this repository; select a retention source job") + elif len(ids) == 1: + job_id = ids[0] else: - selected_job_key = next(iter(job_keys), "") - if not selected_job_key: - raise ValueError("Prune requires a backup job with a retention policy") - retention = self._job_retention(config, selected_job_key) - archive_prefix = _archive_prefix_from_job_key(selected_job_key) - cmd = [ - "borg", "prune", "--lock-wait", self._LOCK_WAIT_SECONDS, - "--list", "--progress", - ] - if archive_prefix: - cmd.extend(["--glob-archives", f"{archive_prefix}-*"]) - retention_counts = [] - for key, option in (("daily", "--keep-daily"), ("weekly", "--keep-weekly"), ("monthly", "--keep-monthly"), ("yearly", "--keep-yearly")): - value = str(retention.get(key) or "").strip() - if value: - if not re.fullmatch(r"\d+", value): - raise ValueError("Retention values must be non-negative whole numbers") - normalized = str(int(value)) - retention_counts.append(int(normalized)) - cmd.extend([option, normalized]) - if not retention_counts: - raise ValueError("The selected job has no retention policy") - if not any(value > 0 for value in retention_counts): + raise ValueError("Select one backup job as the retention source") + metadata = load_job_metadata(config, job_id) + retention = metadata.get("retention", {}) + counts = [int(retention.get(period, "0")) for period in ("daily", "weekly", "monthly", "yearly")] + if not any(counts): raise ValueError("At least one retention value must be greater than zero") - cmd.append(repo_path) - return cmd - - @staticmethod - def _job_retention(config: dict, job_key: str) -> dict: - from jobs_api import get_jobs_meta_dirs, resolve_data_root, resolve_scripts_dir - scripts_dir = resolve_scripts_dir(config) - data_root = resolve_data_root(config) - for directory in get_jobs_meta_dirs(scripts_dir, data_root): - path = directory / f"{job_key}.json" - if not path.is_file(): - continue - payload = json.loads(path.read_text(encoding="utf-8")) - return payload.get("retention") if isinstance(payload.get("retention"), dict) else {} - raise ValueError(f"Job metadata not found: {job_key}") + from job_runs import create_run_context + from jobs_api import resolve_data_root + snapshot = create_run_context(config, job_id, require_enabled=False) + if snapshot["repository_snapshot"] != repo_path or snapshot["repository_key_snapshot"] != repository["repository_key"]: + raise ValueError("Repository assignment changed during maintenance preparation") + return [sys.executable, str(Path(__file__).with_name("retention_runner.py")), + job_id, snapshot["run_id"], str(resolve_data_root(config))] def _reader(self, state: _CheckState) -> None: + lease = getattr(state, "migration_lease", None) + if lease is None: + return self._reader_admitted(state) + try: + with lease.activate(): + self._reader_admitted(state) + finally: + lease.close() + + def _reader_admitted(self, state: _CheckState) -> None: last_emitted: Optional[str] = None def _emit(buf: List[str]) -> None: @@ -312,13 +310,37 @@ def _maintenance_result(state: _CheckState) -> dict: action_key = "verify_data" if state.action == "check" and state.mode == "verify_data" else state.action deleted_archives = [] freed_space = "" + archive_metadata = ( + r"\s+(?:\S+,\s+)?\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}" + r"(?:\s*[+-]\d{2}:?\d{2})?\s+\[[0-9a-f]{64}\]" + ) + log_timestamp = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}" + logger_prefix = re.compile( + r"^(?:(?:\[" + log_timestamp + r"\]|" + log_timestamp + r")\s+(?:INFO\s+)?|INFO\s+)" + ) for raw in lines: line = str(raw or "").strip() - match = re.search(r"\bPruning archive(?:\s*\([^)]*\))?\s*:\s*(.+)$", line, re.IGNORECASE) - if match: - archive = match.group(1).strip() - if archive and archive not in deleted_archives: - deleted_archives.append(archive) + # The manual runner adds INFO; backup runners also add a timestamp. + # Remove only these known prefixes so plan/dry-run text cannot match. + line = logger_prefix.sub("", line, count=1) + # Borg logs deletion before it finishes/commits. A failed process + # cannot confirm which of its announced deletions were persisted. + archive = "" + if exit_code == 0: + legacy = re.match(r"Pruning archive(?:\s*\([^)]*\))?\s*:\s*(.+)$", line, re.IGNORECASE) + if legacy: + payload = legacy.group(1).strip() + formatted = re.fullmatch(r"(.+?)" + archive_metadata, payload, re.IGNORECASE) + archive = formatted.group(1).strip() if formatted else payload + else: + deleted = re.fullmatch( + r"Deleting archive:\s*(.+?)" + archive_metadata + r"\s+\(\d+/\d+\)", + line, re.IGNORECASE, + ) + if deleted: + archive = deleted.group(1).strip() + if archive and archive not in deleted_archives: + deleted_archives.append(archive) freed = re.search( r"\bfreed(?:\s+about)?\s+([0-9]+(?:[.,][0-9]+)?\s*(?:[KMGTPE]i?B|bytes?))", line, @@ -417,39 +439,6 @@ def stream_output(self) -> Generator[str, None, None]: def get_check_jobs(config: dict) -> List[dict]: - """Gibt alle bekannten Jobs zurück (key + display_name) für den Selektor.""" - from jobs_api import discover_jobs, get_jobs_meta_dirs, resolve_data_root, resolve_scripts_dir - loc_label = {"local": "local", "usb": "usb", "smb": "smb", "storagebox": "storagebox", "custom": "custom"} - - def _label(name: str, location: str) -> str: - return f"{name} ({loc_label.get(location, location)})" - - scripts_dir = resolve_scripts_dir(config) - data_root = resolve_data_root(config) - jobs = discover_jobs(scripts_dir, data_root) - result = [ - {"key": j.key, "name": _label((j.name or j.display_name), j.location)} - for j in jobs - if not j.is_utility - ] - if result: - return result - - # Fallback: lies Wizard-Metadaten direkt, falls discover_jobs nichts liefert. - seen = set() - for meta_dir in get_jobs_meta_dirs(scripts_dir, data_root): - if not meta_dir.is_dir(): - continue - for meta_file in sorted(meta_dir.glob("*.json")): - try: - raw = json.loads(meta_file.read_text(encoding="utf-8")) - except Exception: - continue - key = str(raw.get("job_key") or "").strip() - if not key or key in seen: - continue - seen.add(key) - name = str(raw.get("name") or key).strip() - location = str(raw.get("location") or "").strip().lower() or "local" - result.append({"key": key, "name": _label(name, location)}) - return result + from jobs_api import discover_jobs, resolve_data_root, resolve_scripts_dir + return [{"job_id": job.job_id, "name": f"{job.name} ({job.location})"} + for job in discover_jobs(resolve_scripts_dir(config), resolve_data_root(config))] diff --git a/api/config_api.py b/api/config_api.py index 6acaeeb6..967cbe92 100644 --- a/api/config_api.py +++ b/api/config_api.py @@ -827,10 +827,10 @@ def get_settings_data(ui_config: dict, include_storagebox_setup: bool = True) -> data["storagebox_setup"] = get_storagebox_setup_status(ui_config, probe_auth=False) from storage_objects_api import settings_profiles_from_storages canonical_profiles = settings_profiles_from_storages(ui_config) - from repositories_api import read_repository_store + from repositories_api import read_repository_store_for_api from repositories_api import get_repository_info_refresh_status data["repository_info_refresh"] = get_repository_info_refresh_status(ui_config) - repository_rows = read_repository_store(ui_config).get("repositories", []) + repository_rows = read_repository_store_for_api(ui_config).get("repositories", []) refs_by_storage: Dict[str, List[str]] = {} repositories_by_storage: Dict[str, List[str]] = {} for repository in repository_rows: @@ -838,7 +838,7 @@ def get_settings_data(ui_config: dict, include_storagebox_setup: bool = True) -> if storage_key: label = str(repository.get("display_name") or repository.get("repository_key") or storage_key).strip() repositories_by_storage.setdefault(storage_key, []).append(label) - refs = [str(value) for value in repository.get("used_by", []) if str(value)] + refs = [f"{job['name']} ({job['job_id']})" for job in repository["jobs"]] refs_by_storage.setdefault(storage_key, []).extend(refs) data["local_profiles"] = [ { diff --git a/api/factory_reset_api.py b/api/factory_reset_api.py index e449c54d..f865085b 100644 --- a/api/factory_reset_api.py +++ b/api/factory_reset_api.py @@ -35,7 +35,8 @@ def _now() -> str: def _data_root(config: dict) -> Path: - return Path(str(config.get("BACKUP_SCRIPTS_DIR", "/boot/config/borg-backup")).strip() or "/boot/config/borg-backup") + from jobs_api import resolve_data_root + return resolve_data_root(config) def _configured_operational_root(config: dict) -> Path | None: @@ -119,11 +120,13 @@ def _active_operation_blockers(config: dict) -> list[dict[str, str]]: locked_jobs: set[str] = set() for row in active_resource_locks(config): - job_key = str(row.get("job_key") or "Backup job") - locked_jobs.add(job_key) + job_id = str(row.get("job_id") or "Backup job") + locked_jobs.add(job_id) blockers.append({ "type": "backup", - "name": job_key, + "name": job_id, + "job_id": str(row.get("job_id") or ""), + "run_id": str(row.get("run_id") or ""), }) for key, state in JobManager.get().get_all_states().items(): if key == "restore_test" or key in locked_jobs: @@ -154,7 +157,7 @@ def _active_operation_blockers(config: dict) -> list[dict[str, str]]: for row in list_restore_runs(config, 100).get("active", []): blockers.append({ "type": "restore", - "name": str(row.get("restore_id") or row.get("job_key") or "restore"), + "name": str(row.get("restore_id") or row.get("job_id") or "restore"), }) except Exception as exc: raise FactoryResetBlocked(f"Restore state cannot be verified: {exc}") from exc @@ -178,6 +181,12 @@ def factory_reset_status(config: dict) -> dict[str, Any]: roots.append(operational) repository_blockers = _repository_blockers(config, roots) operation_blockers = _active_operation_blockers(config) + from identity_lifecycle import owned_paths + outside = [name for name, (path, _) in owned_paths(config).items() + if path.exists() and not any(_inside(path, item) for item in roots)] + if outside: + raise FactoryResetBlocked("Identity stores are outside the confirmed reset roots; relocate them before reset: " + ", ".join(outside[:8])) + return { "ok": True, "server_name": socket.gethostname(), diff --git a/api/factory_reset_worker.py b/api/factory_reset_worker.py index ae844202..46eb0d1e 100644 --- a/api/factory_reset_worker.py +++ b/api/factory_reset_worker.py @@ -110,10 +110,20 @@ def perform_reset(marker: dict, *, production: bool = True) -> dict: if not example.is_file(): raise FileNotFoundError("backup.conf.example is missing") + controls = Path(str(marker.get("controls_root") or "/run/borg-backup-ui/jobs")) + if production and controls != Path('/run/borg-backup-ui/jobs'): + raise ValueError('Unexpected runtime control root') + if controls in {Path('/'), Path('/run'), Path('/run/borg-backup-ui')}: + raise ValueError('Unsafe runtime control root') _safe_remove(root, expected_config_root=True, production=production) if old_data is not None and old_data.resolve(strict=False) != root.resolve(strict=False): _safe_remove(old_data, production=production) + if controls.is_symlink(): + controls.unlink() + elif controls.exists(): + shutil.rmtree(controls) + for directory in (root / "config" / "jobs", root / "secrets", root / "locks", root / "scripts"): directory.mkdir(parents=True, exist_ok=True) shutil.copy2(example, root / "config" / "backup.conf") @@ -136,13 +146,18 @@ def main() -> int: marker = json.loads(marker_path.read_text(encoding="utf-8")) audit = Path(str(marker.get("audit_file") or "")) rc = str(marker.get("rc_script") or "/etc/rc.d/rc.borg_backup_ui") - time.sleep(2) - _audit(audit, "factory_reset_started", "started", marker) + from migration_barrier import MigrationBlocked, writer_lease try: - subprocess.run([rc, "stop"], timeout=30, check=False) - remove_plugin_cron_blocks() - result = perform_reset(marker) - _audit(audit, "factory_reset_completed", "success", marker, json.dumps(result, ensure_ascii=False)) + with writer_lease({"BACKUP_SCRIPTS_DIR": str(marker.get("configuration_root") or "")}): + time.sleep(2) + _audit(audit, "factory_reset_started", "started", marker) + subprocess.run([rc, "stop"], timeout=30, check=False) + remove_plugin_cron_blocks() + result = perform_reset(marker) + _audit(audit, "factory_reset_completed", "success", marker, json.dumps(result, ensure_ascii=False)) + except MigrationBlocked as exc: + _audit(audit, "factory_reset_blocked", "blocked", marker, exc.reason) + return 1 except Exception as exc: _audit(audit, "factory_reset_failed", "failed", marker, str(exc)) subprocess.run([rc, "start"], timeout=30, check=False) diff --git a/api/history_api.py b/api/history_api.py index 0e576051..dd8ab421 100644 --- a/api/history_api.py +++ b/api/history_api.py @@ -1,36 +1,9 @@ """history_api.py – Liest alle .status-Dateien und gibt sie als Liste zurück.""" -import json from datetime import datetime -from pathlib import Path - - - -def _fmt_bytes(b): - if b is None: - return None - for unit in ("B", "KB", "MB", "GB", "TB"): - if b < 1024: - return f"{b:.1f} {unit}" - b /= 1024 - return f"{b:.1f} PB" - - -def _fmt_duration(secs): - if secs is None: - return None - secs = int(secs) - h, rem = divmod(secs, 3600) - m, s = divmod(rem, 60) - if h: - return f"{h}h {m:02d}m" - if m: - return f"{m}m {s:02d}s" - return f"{s}s" def get_history_data(config: dict, filters: dict | None = None) -> dict: - status_dir = Path(config["STATUS_DIR"]) filters = filters or {} try: page = max(1, int(filters.get("page") or 1)) @@ -41,76 +14,29 @@ def get_history_data(config: dict, filters: dict | None = None) -> dict: except (TypeError, ValueError): per_page = 20 + from status_read_model import configured_jobs, history_rows, navigation_jobs, valid_job_id + jobs = configured_jobs(config) + all_rows = history_rows(config, jobs) + job_id = filters.get('job_id') or '' + if job_id and not valid_job_id(job_id): + raise ValueError('A canonical job_id is required') + scope = filters.get('scope') or 'all' + if scope not in {'all', 'configured', 'deleted', 'unassigned'}: + raise ValueError('Invalid history scope') entries = [] - location_counts = {location: 0 for location in ("storagebox", "usb", "smb", "local")} - known_types = {"flash", "appdata", "photos", "vms", "sonstiges"} - for f in sorted(status_dir.glob("*.status"), reverse=True): - # Filename: YYYY-MM-DD_HH-MM-SS_type_location.status - stem = f.stem - parts = stem.split("_") - if len(parts) < 4: + location_counts = {location: 0 for location in ('storagebox', 'usb', 'smb', 'local', 'unknown')} + for row in all_rows: + if job_id and (row['job_id'] != job_id or row['identity_scope'] == 'unassigned'): continue - date_part = parts[0] # 2026-03-01 - time_part = parts[1] # 02-15-43 - backup_type = parts[2] # flash / appdata / … - location = "_".join(parts[3:]) # local / usb / storagebox - - try: - raw = json.loads(f.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + if scope != 'all' and row['identity_scope'] != scope: continue - - status = raw.get("status", "unknown") - exit_code = raw.get("borg_exit_code", raw.get("exit_code")) - - # Apply filters - filt_type = str(filters.get("type") or "").strip().lower() - if filt_type: - bt_low = str(backup_type or "").strip().lower() - if filt_type == "custom": - if bt_low in known_types: - continue - elif filt_type != bt_low: - continue - if filters.get("status") and filters["status"] != status: + if filters.get('status') and row['status'] != filters['status']: continue - - normalized_location = str(raw.get("location", location) or location).strip().lower() - if normalized_location in location_counts: - location_counts[normalized_location] += 1 - if filters.get("location") and filters["location"] != normalized_location: + location = row['location'] if row['location'] in location_counts else 'unknown' + location_counts[location] += 1 + if filters.get('location') and filters['location'] != location: continue - - entries.append({ - "entry_kind": "backup_run", - "filename": f.name, - "date": date_part, - "time": time_part.replace("-", ":"), - "timestamp": raw.get("timestamp", f"{date_part} {time_part.replace('-', ':')}"), - "backup_type": raw.get("backup_type", backup_type), - "location": raw.get("location", location), - "status": status, - "exit_code": exit_code, - "duration_seconds": raw.get("duration_seconds"), - "duration_fmt": _fmt_duration(raw.get("duration_seconds")), - "original_size": raw.get("original_size"), - "original_size_fmt": _fmt_bytes(raw.get("original_size")), - "compressed_size": raw.get("compressed_size"), - "compressed_size_fmt": _fmt_bytes(raw.get("compressed_size")), - "deduplicated_size": raw.get("deduplicated_size"), - "deduplicated_size_fmt": _fmt_bytes(raw.get("deduplicated_size")), - "repository_size": raw.get("repository_size"), - "repository_size_fmt": _fmt_bytes(raw.get("repository_size")), - "files_count": raw.get("files_count"), - "archive_name": raw.get("archive_name"), - "log_file": raw.get("log_file"), - "error_message": raw.get("error_message"), - "skip_reason_code": raw.get("skip_reason_code", ""), - "skip_reason_text": raw.get("skip_reason_text", ""), - "repository_check_date": raw.get("repository_check_date"), - "repository_check_status": raw.get("repository_check_status"), - "repository_next_check": raw.get("repository_next_check"), - }) + entries.append(row) def _ts_key(entry: dict): ts = str(entry.get("timestamp") or "") @@ -130,6 +56,8 @@ def _ts_key(entry: dict): return { "entries": entries[start:end], + "jobs": navigation_jobs(jobs, all_rows), + "scope_counts": {scope: sum(row["identity_scope"] == scope for row in all_rows) for scope in ("configured", "deleted", "unassigned")}, "total": total, "page": page, "per_page": per_page, diff --git a/api/homepage_widget_api.py b/api/homepage_widget_api.py index bbb1106c..0b079bcb 100644 --- a/api/homepage_widget_api.py +++ b/api/homepage_widget_api.py @@ -16,89 +16,6 @@ def _data_root(config: dict) -> Path: return base.parent if base.name == "scripts" else base -def _read_jobs(config: dict) -> list[dict]: - jobs_dir = _data_root(config) / "config" / "jobs" - rows: list[dict] = [] - if not jobs_dir.is_dir(): - return rows - for path in sorted(jobs_dir.glob("*.json")): - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError, TypeError, ValueError): - continue - if not isinstance(raw, dict): - continue - key = str(raw.get("job_key") or path.stem).strip() - if not key: - continue - policy = raw.get("restore_test_policy") if isinstance(raw.get("restore_test_policy"), dict) else {} - location = str(raw.get("location") or "").strip().lower() - name = str(raw.get("name") or raw.get("backup_type") or key).strip() - location_label = { - "local": "Local", - "usb": "USB", - "smb": "SMB", - "storagebox": "Storagebox", - }.get(location, location.title()) - rows.append({ - "key": key, - "name": name, - "display_name": f"{name} - {location_label}" if location_label else name, - "location": location, - "enabled": bool(raw.get("enabled", True)), - "is_utility": bool(raw.get("is_utility", False)), - "restore_test_policy": policy, - }) - return rows - - -def _read_latest_backup_rows(config: dict) -> list[dict]: - from status import StatusStore - - status_dir = Path(str(config.get("STATUS_DIR") or "/mnt/user/backup-status")) - store = StatusStore(status_dir) - latest = store.get_latest_per_key(store.load()) - rows: list[dict] = [] - for key, status in latest.items(): - rows.append({ - "key": str(key), - "backup_type": str(getattr(status, "backup_type", "") or ""), - "location": str(getattr(status, "location", "") or ""), - "status": str(getattr(status, "status", "") or "").strip().lower(), - "timestamp": str(getattr(status, "timestamp", "") or ""), - }) - return rows - - -def _backup_overdue_count(config: dict, jobs: list[dict], latest_rows: list[dict], now: datetime) -> int: - from config_api import read_expanded_conf - from notification_reminder_api import ( - _backup_overdue_item, - _backup_overdue_tolerance_hours, - _latest_backup_status_by_key, - ) - from schedule_api import get_schedules - - effective = {**config, **read_expanded_conf(config)} - schedules = get_schedules(effective) - latest = _latest_backup_status_by_key(latest_rows) - by_key = {str(row.get("key") or ""): row for row in jobs} - tolerance = _backup_overdue_tolerance_hours(effective) - count = 0 - for job_key, schedule in schedules.items(): - if job_key == "restore_test" or not isinstance(schedule, dict) or not bool(schedule.get("enabled", True)): - continue - job = by_key.get(str(job_key)) - if not job or not job.get("enabled", True): - continue - item = _backup_overdue_item( - str(job_key), schedule, job, latest.get(str(job_key)) or {}, {}, now, 24, tolerance - ) - if str(item.get("state") or "").startswith("overdue_"): - count += 1 - return count - - def _restore_summary(config: dict, jobs: list[dict]) -> dict[str, int]: from config_api import read_expanded_conf from restore_tests_api import build_restore_verification_map @@ -115,6 +32,9 @@ def _restore_summary(config: dict, jobs: list[dict]) -> dict[str, int]: continue configured += 1 state = str(item.get("status") or "") + if state == 'stale' and item.get('reason') in {'target_unknown', 'target_changed', 'test_date_unknown'}: + never += 1 # Existing API counter for proof that remains open. + continue if state == "verified": verified += 1 elif state == "failed": @@ -134,51 +54,33 @@ def _restore_summary(config: dict, jobs: list[dict]) -> dict[str, int]: def build_homepage_widget_summary(config: dict, *, now: datetime | None = None) -> dict[str, Any]: """Build a stable, redacted and side-effect-free widget response.""" - from jobs_api import get_all_runtime_states - + from config_api import read_expanded_conf + from status_api import get_status_data + effective = {**config, **read_expanded_conf(config)} generated = now or datetime.now(timezone.utc) local_now = generated.astimezone().replace(tzinfo=None) if generated.tzinfo else generated - jobs = [row for row in _read_jobs(config) if not row.get("is_utility")] - enabled_jobs = [row for row in jobs if row.get("enabled", True)] - latest_rows = _read_latest_backup_rows(config) - latest_by_key = {str(row.get("key") or ""): row for row in latest_rows} - - counts = {"successful": 0, "warning": 0, "failed": 0, "skipped": 0, "never": 0} - for job in enabled_jobs: - status = str((latest_by_key.get(str(job.get("key") or "")) or {}).get("status") or "").lower() - if status == "success": - counts["successful"] += 1 - elif status == "warning": - counts["warning"] += 1 - elif status in {"error", "failed", "failure"}: - counts["failed"] += 1 - elif status == "skipped": - counts["skipped"] += 1 - else: - counts["never"] += 1 - - counts["overdue"] = _backup_overdue_count(config, enabled_jobs, latest_rows, local_now) - restore = _restore_summary(config, enabled_jobs) - - by_key = {str(row.get("key") or ""): row for row in enabled_jobs} - running_states = get_all_runtime_states(config) - active_jobs = [ - str(by_key[key].get("display_name") or by_key[key].get("name") or key) - for key, state in sorted(running_states.items()) - if key in by_key and isinstance(state, dict) and bool(state.get("running", False)) - ] + data = get_status_data(effective, write_snapshots=False, now=local_now) + jobs = data['backups'] + enabled_jobs = [row for row in jobs if row.get('enabled') is not False] + summary = data['summary'] + counts = {'successful': summary['success'], 'warning': summary['warning'], + 'failed': summary['error'], 'skipped': summary['skipped'], + 'never': summary['never'], 'unknown': summary['unknown']} + counts['overdue'] = sum(bool(row.get('backup_overdue')) for row in enabled_jobs if not row.get('running')) + restore = _restore_summary(effective, enabled_jobs) + active_jobs = [str(row.get('name') or 'Backup')[:160] for row in enabled_jobs if row.get('running')] critical = counts["failed"] + restore["failed"] attention = ( - counts["warning"] + counts["skipped"] + counts["never"] + counts["overdue"] + counts["warning"] + counts["skipped"] + counts["never"] + counts["unknown"] + restore["overdue"] + restore["never"] ) if critical: state, label, severity = "critical", "Critical", 3 - elif attention: - state, label, severity = "attention", "Attention required", 2 elif active_jobs: state, label, severity = "active", "Backup running", 1 + elif attention: + state, label, severity = "attention", "Attention required", 2 else: state, label, severity = "healthy", "Healthy", 0 @@ -204,5 +106,6 @@ def build_homepage_widget_summary(config: dict, *, now: datetime | None = None) **counts, }, "restore_tests": restore, - "active": {"count": len(active_jobs), "jobs": active_jobs}, + "active": {"count": len(active_jobs), "jobs": active_jobs, + "job_ids": [row["job_id"] for row in enabled_jobs if row.get("running")]}, } diff --git a/api/identity_lifecycle.py b/api/identity_lifecycle.py new file mode 100644 index 00000000..be766194 --- /dev/null +++ b/api/identity_lifecycle.py @@ -0,0 +1,252 @@ +"""Owned identity stores, bounded diagnostics and precise deletion (#447, #478). + +Logical names are a closed ownership registry, never paths supplied by a client. +Historical evidence is retained until its exact digest is explicitly confirmed. +""" +from copy import deepcopy +import hashlib +import json +from pathlib import Path + +from job_model import validate_job, validate_job_inventory, validate_job_id +from job_store import read_json +from repository_context import jobs_dir +from migrations.identity_records import verify_records +from migrations.identity_storage import inventory_group, inventory_directories + +SINGLETONS = { + 'repositories.json': 'repositories', 'storages.json': 'storages', 'schedules.json': 'schedules', + 'restore-runs.json': 'restore_runs', 'restore-history/index.json': 'restore_index', + 'notification-queue.json': 'notification_queue', 'notification-deliveries.json': 'notification_deliveries', + 'notification-state.json': 'notification_state', +} +HISTORY = {'status', 'restore_test', 'restore_detail', 'restore_index', 'notification_deliveries', 'notification_state', 'weekly'} + + +def store_layout(config): + root = jobs_dir(config).parent + status = Path(config.get('STATUS_DIR') or '/mnt/user/backup-status') + return {'config': root, 'status': status, + 'proof': Path(config.get('RESTORE_TEST_STATUS_DIR') or status.parent / 'restore-status'), + 'weekly': Path(config.get('SNAPSHOT_FILE') or status.parent / 'weekly-snapshots.json'), + 'recovery': Path(config.get('RUNTIME_RECOVERY_FILE') or root / 'runtime-recovery.json')} + + +def owned_paths(config, *, runtime=False): + layout = store_layout(config); root = layout['config'] + result = {'config/' + name: (root / name, kind) for name, kind in SINGLETONS.items()} + result.update({'weekly': (layout['weekly'], 'weekly'), 'recovery': (layout['recovery'], 'runtime_recovery')}) + groups = [('config/jobs', root / 'jobs', '.json', 'job'), + ('config/restore-history/runs', root / 'restore-history/runs', '.json', 'restore_detail'), + ('status', layout['status'], '.status', 'status'), ('proof', layout['proof'], '.test', 'restore_test')] + if runtime: + from jobs_api import resolve_resource_lock_dir + from job_runs import control_root + groups.append(('locks', resolve_resource_lock_dir(config), '.json', 'resource_lock')) + controls = control_root() + for run_id in inventory_directories(controls)['entries']: + for name in inventory_group(controls / run_id, ['.json'])['entries']: + kind = {'state.json': 'control', 'context.json': 'run_context', 'cancel.request.json': 'cancel_request'}.get(name, 'unknown') + result['controls/' + run_id + '/' + name] = (controls / run_id / name, kind) + for namespace, directory, suffix, kind in groups: + for name in inventory_group(directory, [suffix])['entries']: + result[namespace + '/' + name] = (directory / name, kind) + values = [str(path.absolute()) for path, _ in result.values()] + if len(values) != len(set(values)): + raise ValueError('Identity store locations overlap') + return result + + +def read_owned(config, *, runtime=False): + return {name: {'kind': kind, 'data': data} for name, (path, kind) in owned_paths(config, runtime=runtime).items() + if (data := read_json(path)) is not None} + + +def record_digest(data): + return hashlib.sha256(json.dumps(data, sort_keys=True, ensure_ascii=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest() + + +def record_references(records): + """Typed reference map, including map keys; snapshots retain descriptors.""" + references = [] + def walk(value, name, pointer): + if isinstance(value, dict): + for key, item in value.items(): + at = pointer + '/' + key.replace('~', '~0').replace('/', '~1') + if key == 'job_id' and item is not None: + references.append({'file': name, 'pointer': at, 'job_id': item, 'encoding': 'value'}) + elif key in {'job_ids', 'source_job_ids'} and isinstance(item, list): + references.extend({'file': name, 'pointer': at + '/' + str(i), 'job_id': job_id, 'encoding': 'value'} for i, job_id in enumerate(item)) + else: + walk(item, name, at) + elif isinstance(value, list): + for i, item in enumerate(value): walk(item, name, pointer + '/' + str(i)) + for name, record in sorted(records.items()): + data, kind = record['data'], record['kind'] + if kind == 'schedules': + for key in data: + if key != 'restore_test': references.append({'file': name, 'pointer': '/' + key, 'job_id': key, 'encoding': 'key'}) + elif kind == 'notification_state': + for key in data.get('last_sent', {}): + parts = key.split(':', 2) + if len(parts) != 3: raise ValueError('Invalid reminder identity key') + references.append({'file': name, 'pointer': '/last_sent/' + key.replace('~', '~0').replace('/', '~1'), 'job_id': parts[1], 'encoding': 'reminder'}) + walk(data, name, '') + for ref in references: validate_job_id(ref['job_id']) + return references + + +def remap_records(records, mapping): + output = deepcopy(records) + for ref in record_references(records): + job_id = mapping.get(ref['job_id'], ref['job_id']) + validate_job_id(job_id) + parts = [part.replace('~1', '/').replace('~0', '~') for part in ref['pointer'].split('/')[1:]] + parent = output[ref['file']]['data'] + for part in parts[:-1]: parent = parent[int(part)] if isinstance(parent, list) else parent[part] + key = parts[-1] + if ref['encoding'] == 'value': parent[int(key) if isinstance(parent, list) else key] = job_id + else: + new_key = job_id if ref['encoding'] == 'key' else ':'.join([key.split(':', 2)[0], job_id, key.split(':', 2)[2]]) + if new_key != key: + if new_key in parent: raise ValueError('Remapped identity keys collide') + parent[new_key] = parent.pop(key) + renamed = {} + for name, record in output.items(): + if record['kind'] in {'job', 'restore_test'} and record['data'].get('job_id') in mapping.values(): + name = name.rsplit('/', 1)[0] + '/' + record['data']['job_id'] + ('.json' if record['kind'] == 'job' else '.test') + if name in renamed: raise ValueError('Remapped owned files collide') + renamed[name] = record + return renamed + + +def validate_records(records): + jobs = {} + for name, record in records.items(): + if record['kind'] == 'job': + validate_job(record['data'], filename=name.rsplit('/', 1)[-1]) + job_id = record['data']['job_id'] + if job_id in jobs: raise ValueError('Duplicate job ID') + jobs[job_id] = record['data'] + validate_job_inventory(jobs) + from job_store import validate_assignments + from schedule_api import validate_schedules + repos = records.get('config/repositories.json', {}).get('data', {'repositories': []}) + validate_assignments(jobs, repos) + from job_transfer import indexed + from repository_context import resolve_job_repository_context + storages = indexed(records.get('config/storages.json', {}).get('data', {}).get('storages', []), 'storage_key') + repositories = indexed(repos['repositories'], 'repository_key') + if any(row.get('storage_key') not in storages for row in repositories.values()): + raise ValueError('A repository references a missing storage') + inventory = {'repositories': repositories, 'storages': storages} + for job_id, job in jobs.items(): + resolve_job_repository_context({}, job_id, job=job, inventory=inventory, require_passphrase_file=False) + validate_schedules(records.get('config/schedules.json', {}).get('data', {}), jobs) + reasons = verify_records({'/' + name: record for name, record in records.items() if record['kind'] != 'job'}, jobs) + errors = [reason['code'] for reason in reasons if reason.get('severity') == 'error'] + if errors: raise ValueError('Identity integrity check failed: ' + ', '.join(sorted(set(errors))[:8])) + record_references(records) + return jobs + + +def identity_health(config, *, limit=200): + from status_read_model import valid_job_id + findings, records, jobs = [], {}, {} + def add(code, name='', job_id='', category='error'): + findings.append({'code': code, 'file': str(name)[:240], 'job_id': valid_job_id(job_id), 'category': category}) + try: + paths = owned_paths(config, runtime=True) + except Exception: + return {'ok': False, 'findings': [{'code': 'owned_store_scan_failed', 'category': 'error'}], 'total': 1, 'truncated': False} + for name, (path, kind) in paths.items(): + try: + data = read_json(path) + if data is None: continue + records['/' + name] = {'kind': kind, 'data': data} + if kind == 'job': + job_id = data.get('job_id') + if job_id in jobs: add('duplicate_job_id', name, job_id) + if not job_id: add('missing_job_id', name) + validate_job(data, filename=path.name) + jobs[job_id] = data + except Exception as exc: + add(getattr(exc, 'api_code', 'owned_record_unreadable'), name) + try: validate_job_inventory(jobs) + except Exception as exc: add(getattr(exc, 'api_code', 'invalid_job_inventory')) + reasons = verify_records({name: row for name, row in records.items() if row['kind'] != 'job'}, jobs) + for reason in reasons: + add(reason['code'], reason['source'].lstrip('/'), category='warning' if reason.get('severity') == 'warning' else 'error') + for name, record in records.items(): + data, kind = record['data'], record['kind'] + # A deleted historical owner is evidence, never an active join or an + # automatic cleanup request. Legacy records remain visibly unassigned. + rows = [data] + for field in ('runs', 'deliveries', 'observations'): + value = data.get(field) + if isinstance(value, list): rows.extend(row for row in value if isinstance(row, dict)) + for row in rows: + if kind in HISTORY and row.get('job_id') and row['job_id'] not in jobs: + add('deleted_job_history_preserved', name.lstrip('/'), row['job_id'], 'history') + elif kind in HISTORY and (row.get('identity_state') == 'unassigned' or row.get('legacy_job_key') or row.get('job_key') or row.get('backup_type')) and not row.get('job_id'): + add('unresolved_legacy_history', name.lstrip('/'), category='history') + if kind == 'control' and row.get('finished') is True and row.get('job_id') not in jobs: + add('orphan_terminal_control', name.lstrip('/'), row.get('job_id'), 'cleanup_candidate') + return {'ok': not any(row['category'] == 'error' for row in findings), 'findings': findings[:limit], + 'total': len(findings), 'truncated': len(findings) > limit, + 'jobs': [{'job_id': j, 'name': row['name'][:160], 'repository_key': row['repository_key']} for j, row in list(jobs.items())[:limit]]} + + +def deletion_plan(config, job_id, confirmed_artifacts=None): + """Plan exact history removals, requiring matching content digests.""" + validate_job_id(job_id) + records = read_owned(config) + validate_records(records) + artifacts, changes, selected_rows = [], {}, {} + selected = confirmed_artifacts or [] + if not isinstance(selected, list) or any(not isinstance(item, str) for item in selected): + raise ValueError('Artifact confirmations must be a list of preview IDs') + paths = owned_paths(config) + for name, record in records.items(): + kind, data = record['kind'], record['data'] + if kind not in HISTORY: continue + if kind in {'status', 'restore_test', 'restore_detail'}: + rows = [('', data)] + elif kind == 'notification_state': + rows = [(key, {'job_id': key.split(':', 2)[1], 'value': val}) for key, val in data.get('last_sent', {}).items() if len(key.split(':', 2)) == 3] + else: + field = {'weekly': 'observations', 'restore_index': 'runs', 'notification_deliveries': 'deliveries'}[kind] + rows = [(str(i), row) for i, row in enumerate(data.get(field, []))] + for locator, row in rows: + if row.get('job_id') != job_id: continue + token = record_digest({'file': name, 'locator': locator, 'record': row}) + artifacts.append({'id': token, 'file': name, 'kind': kind, 'job_id': job_id, + 'run_id': str(row.get('run_id') or row.get('restore_id') or '')[:64]}) + if token not in selected: continue + target = paths[name][0] + if not locator and kind in {'status', 'restore_test', 'restore_detail'}: + changes[target] = None + elif kind == 'notification_state': + changes.setdefault(target, deepcopy(data))['last_sent'].pop(locator) + else: + after = changes.setdefault(target, deepcopy(data)) + selected_rows.setdefault(target, set()).add(int(locator)) + after[field] = [item for i, item in enumerate(data[field]) if i not in selected_rows[target]] + if set(selected) - {row['id'] for row in artifacts}: + raise ValueError('Artifact preview changed; reload before deleting') + # Restore index/detail are one record pair: require both explicitly. + next_records = deepcopy(records) + for name, (path, _) in paths.items(): + if path in changes: + if changes[path] is None: next_records.pop(name, None) + else: next_records[name]['data'] = changes[path] + validate_records(next_records) + for record in records.values(): + data, kind = record['data'], record['kind'] + if kind in {'notification_queue', 'runtime_recovery', 'restore_runs'}: + field = {'notification_queue': 'queue', 'runtime_recovery': 'entries', 'restore_runs': 'runs'}[kind] + rows = data.get(field, []) + rows = rows.values() if isinstance(rows, dict) else rows + if any(row.get('job_id') == job_id for row in rows): + raise ValueError('The job still has pending notifications, runtime recovery or restores; resolve them before deletion') + return {'artifacts': artifacts, 'changes': changes, 'deleted_count': len(selected)} diff --git a/api/identity_migration_api.py b/api/identity_migration_api.py new file mode 100644 index 00000000..927f7bc2 --- /dev/null +++ b/api/identity_migration_api.py @@ -0,0 +1,595 @@ +"""Explicit, durable administrator workflow for immutable job identity (#479). + +Only this coordinator starts preparation and application. GET/startup never +allocate persistent IDs, snapshot data, acknowledge, or resume conversion. +""" +from __future__ import annotations + +from contextlib import contextmanager +from datetime import datetime, timezone +import hashlib +import io +import json +import os +import re +import stat +from pathlib import Path +import subprocess +import tarfile +import threading +import uuid + +from migrations import identity_storage as storage +from migrations import immutable_job_id_v1 as identity +from migrations.identity_reasons import PLANNING_REASON_CODES + +MIGRATION_ID = identity.MIGRATION_ID +_ASSISTANTS = {} +_ASSISTANTS_LOCK = threading.RLock() +REASONS = storage.REASON_CODES | PLANNING_REASON_CODES | { + "cron_unavailable", "cron_write_failed", "persistent_private_storage_required", + "migration_operation_failed", "invalid_migration_location", "invalid_assistant_state", + "incomplete_preparation", "invalid_completion_evidence", "identity_cutover_incomplete", + "explicit_continuation_required", "restart_required", "original_migration_location_required", + "existing_preparation_must_be_preserved", "migration_not_applicable", "prepare_required", + "operation_in_progress", "verified_backup_pause_required", "independent_backup_ack_required", + "final_verification_failed", "startup_gate_failed", "writers_running", "legacy_workers_running", + "migration_maintenance", "gate_storage_unavailable", "unsafe_gate_path", "unsafe_gate_state", + "invalid_gate_state", "startup_validation_required", "migration_in_progress", "unsupported_managed_cron", +} +META_KEYS = {"migration_id", "status", "stage", "reason_codes", "acknowledged", "updated_at", "plan_id", "snapshot_digest"} +STAGES = {"required", "waiting", "preparing", "backup_ready", "acknowledged", "applying", "verifying", "interrupted", "complete"} + + +class MigrationRequestError(ValueError): + api_status = 409 + api_code = "identity_migration_blocked" + + +def _fail(code): + raise MigrationRequestError(code) + + +def _root(config): + path = storage._absolute(config.get("BACKUP_SCRIPTS_DIR") or "/boot/config/borg-backup") + return path.parent if path.name == "scripts" else path + + +def _control_root(config): + return config.get("BORG_UI_CONTROL_ROOT") or os.environ.get("BORG_UI_CONTROL_ROOT") or "/run/borg-backup-ui/jobs" + + +def _read_cron(): + result = subprocess.run(["crontab", "-l"], capture_output=True, text=True, timeout=10) + if result.returncode == 0: + return result.stdout + if result.returncode == 1 and not result.stdout and (not result.stderr.strip() or "no crontab for" in result.stderr.lower()): + return "" + _fail("cron_unavailable") + + +def _write_cron(text): + result = subprocess.run(["crontab", "-"], input=text, capture_output=True, text=True, timeout=10) + if result.returncode: + _fail("cron_write_failed") + + +def _render_cron(config, original, plan): + from migrations.identity_apply import replace_managed_cron + from schedule_api import schedule_lines + schedules = next((row["data"] for row in plan["records"].values() if row["kind"] == "schedules"), {}) + return replace_managed_cron(original, schedule_lines(config, schedules, plan["jobs"])) + + +def _validate_cron_admission(text): + from migrations.identity_apply import _cron_parts, _BEGIN, _END + _cron_parts(text) # Also reject duplicate, malformed or reversed markers. + if _BEGIN not in text: + return + block = text.split(_BEGIN + "\n", 1)[1].split(_END, 1)[0] + for line in block.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + # Previous shipped schedules call the guarded localhost HTTP boundary. + # Direct legacy scripts cannot be made safe by observing current PIDs. + if ("curl " not in line or not re.search(r"http://127\.0\.0\.1:[0-9]+/api/(jobs/run|restore-tests/run)", line) + or any(name in line for name in ("wizard_runner.py", "borg_backup_", "borg_restore_test.sh"))): + _fail("unsupported_managed_cron") + + +def _canonical_plan(config, plan): + """Include earlier automatic config rewrites in the approved byte snapshot.""" + from config_api import canonical_backup_conf_plan, get_backup_conf_schema_file + # The schema is a version-owned input; changing installed code invalidates consent. + plan = json.loads(json.dumps(plan)) + plan.pop("plan_id", None) + source = str(_root(config) / "config/backup.conf") + schema = str(get_backup_conf_schema_file(config)) + obsolete = str(_root(config) / "config/backup.conf.example") + source_fingerprint, source_raw = storage.read_fingerprinted_file(source) + if source_fingerprint != plan["inputs"][source]: + _fail("input_changed") + for path in (schema, obsolete): + fingerprint = storage.fingerprint_file(path) + if path in plan["inputs"] and plan["inputs"][path] != fingerprint: + _fail("input_changed") + plan["inputs"][path] = fingerprint + canonical = canonical_backup_conf_plan(config, source_content=(source_raw or b"").decode("utf-8")) + for path in (source, schema, obsolete): + if storage.fingerprint_file(path) != plan["inputs"][path]: + _fail("input_changed") + if canonical["changed"] or plan["inputs"][obsolete]["exists"]: + raw = canonical["content"].encode("utf-8") + action = {"kind": "write_bytes", "source": source, "target": source, + "text": canonical["content"], "after": {"exists": True, "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), "mode": plan["inputs"][source].get("mode", 0o600)}} + action["id"] = hashlib.sha256(json.dumps(action, sort_keys=True).encode()).hexdigest() + plan["actions"].append(action) + if plan["inputs"][obsolete]["exists"]: + action = {"kind": "retire_auxiliary", "source": obsolete, "target": source} + action["id"] = hashlib.sha256(json.dumps(action, sort_keys=True).encode()).hexdigest() + plan["actions"].append(action) + return storage.seal_plan(plan) + + +def _safe_code(exc): + # Never include exception messages, paths, config values or subprocess stderr. + code = getattr(exc, "code", "") or getattr(exc, "reason", "") + if isinstance(exc, MigrationRequestError): + code = str(exc) + return code if isinstance(code, str) and code in REASONS else "migration_operation_failed" + + +def _write_meta(path, value): + """Private atomic replacement, anchored to the already validated directory.""" + storage._private_directory(path.parent) + storage._read_file(path, private=True) + raw = storage._canonical(value) + with storage._directory(path.parent) as parent: + name = ".assistant-" + uuid.uuid4().hex + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(raw) + handle.flush() + os.fsync(handle.fileno()) + os.replace(name, path.name, src_dir_fd=parent, dst_dir_fd=parent) + os.fsync(parent) + finally: + try: + os.unlink(name, dir_fd=parent) + except FileNotFoundError: + pass + + +class IdentityMigrationAssistant: + def __init__(self, config, *, read_cron=None, write_cron=None, activate=None): + self.config = config + self.read_cron = read_cron or _read_cron + self.write_cron = write_cron or _write_cron + self.activate = activate + self._operation = threading.Lock() + self._view_lock = threading.RLock() + self._busy = False + self._failed_here = False + self._view = {"status": "pending", "stage": "required", "reason_codes": []} + self._last_preparation = None + self._snapshot_view = None + self._snapshot_signature = None + + @property + def selector(self): + return _root(self.config) / ".identity-migration-location.json" + + def _state_dir(self): + fingerprint, raw = storage.read_fingerprinted_file(self.selector) + if not fingerprint["exists"]: + return None + try: + value = json.loads(raw, object_pairs_hook=storage._unique_json_pairs) + if set(value) != {"migration_id", "state_dir"} or value["migration_id"] != MIGRATION_ID: + _fail("invalid_migration_location") + return self._validate_location(value["state_dir"]) + except (TypeError, ValueError): + _fail("invalid_migration_location") + + def _validate_location(self, value): + path = storage._absolute(value) + # Recovery is persistent, never /run, /tmp, /boot FAT, or a symlink. + if len(path.parts) < 2 or path.parts[1] in {"tmp", "run", "dev", "proc", "sys", "boot"}: + _fail("persistent_private_storage_required") + identity._path(str(path)) # mounted /mnt roots and all existing ancestors + with storage._directory(path.parent): + pass + return path + + def _validate_state_layout(self, state): + allowed = {"assistant.json", "plan.json", "journal.jsonl", "snapshot", ".capability-probe"} + names = set(p.name for p in state.iterdir()) + extra = names - allowed + if extra: + if not storage.fingerprint_file(state / "plan.json")["exists"]: + _fail("invalid_assistant_state") + journal = storage.read_journal(state) + started = any(row["phase"] in {"apply", "commit"} for row in journal) + meta = self._meta(state) + authorized_preflight = bool(meta and meta.get("acknowledged") and meta["stage"] in {"applying", "verifying", "interrupted"} + and any(row["phase"] == "confirm" and row["status"] == "applied" for row in journal)) + if not started and (not authorized_preflight or extra - {"apply-roots.json", "apply-cron.json"}): + _fail("invalid_assistant_state") + if any(name not in {"apply-roots.json", "apply-cron.json"} and not re.fullmatch(r"directory-[0-9a-f]{64}\.json", name) for name in extra): + _fail("invalid_assistant_state") + for name in names - {"snapshot"}: + storage._read_file(state / name, private=True) + + def _meta(self, state): + fp, raw = storage._read_file(state / "assistant.json", private=True) + if not fp["exists"]: + return None + try: + meta = json.loads(raw, object_pairs_hook=storage._unique_json_pairs) + if (not isinstance(meta, dict) or meta.get("migration_id") != MIGRATION_ID + or meta.get("stage") not in STAGES + or meta.get("status") not in {"pending", "blocked", "failed", "applied"} + or type(meta.get("acknowledged")) is not bool + or not isinstance(meta.get("reason_codes"), list)): + _fail("invalid_assistant_state") + if (set(meta) - META_KEYS or any(not isinstance(code, str) or code not in REASONS for code in meta["reason_codes"]) + or not isinstance(meta.get("updated_at"), str)): + _fail("invalid_assistant_state") + datetime.fromisoformat(meta["updated_at"]) + for key in ("plan_id", "snapshot_digest"): + if key in meta and (not isinstance(meta[key], str) or not re.fullmatch(r"[0-9a-f]{64}", meta[key])): + _fail("invalid_assistant_state") + if meta["acknowledged"] and not all(key in meta for key in ("plan_id", "snapshot_digest")): + _fail("invalid_assistant_state") + return meta + except (ValueError, TypeError): + _fail("invalid_assistant_state") + + def _save(self, state, **changes): + meta = self._meta(state) or {"migration_id": MIGRATION_ID, "status": "pending", "stage": "required", + "reason_codes": [], "acknowledged": False} + meta.update(changes) + meta["updated_at"] = datetime.now(timezone.utc).isoformat() + _write_meta(state / "assistant.json", meta) + with self._view_lock: + self._view = dict(meta) + return meta + + def _snapshot(self, state, plan): + manifest = storage._read_json(state / "snapshot/manifest.json") + handle = {"path": str(state / "snapshot"), "plan_id": plan["plan_id"], "digest": storage._digest(manifest)} + return handle, storage.verify_snapshot(plan, handle) + + def _snapshot_stat_signature(self, state, manifest): + # A cheap change detector avoids rehashing every blob on each UI poll. + # Full verification is still mandatory for every exported/approved copy. + blobs = state / "snapshot/files" + names = {e["blob"] for e in manifest["entries"].values() if e["blob"]} + names.update(e["blob"] for e in manifest["external_inputs"].values()) + with storage._directory(blobs) as directory: + if set(os.listdir(directory)) != names: + _fail("snapshot_changed") + paths = [state / "plan.json", state / "snapshot/metadata.json", state / "snapshot/manifest.json"] + paths.extend(blobs / name for name in sorted(names)) + result = [] + for path in paths: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + _fail("unsafe_path") + result.append((info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns, info.st_mode)) + return result + + def startup_detection(self): + """Read only, including on failed/interrupted attempts; never resumes.""" + try: + state = self._state_dir() + if state: + self._validate_state_layout(state) + if not storage.fingerprint_file(state / "plan.json")["exists"]: + if set(p.name for p in state.iterdir()) - {"assistant.json", ".capability-probe"}: + _fail("invalid_assistant_state") + self._view = {"status": "pending", "stage": "interrupted", "reason_codes": ["incomplete_preparation"]} + return {"required": True, "status": "pending", "reasons": ["incomplete_preparation"]} + plan = storage.load_plan(state) + journal = storage.read_journal(state) + meta = self._meta(state) + if meta is None: + self._view = {"status": "pending", "stage": "interrupted", "reason_codes": ["incomplete_preparation"]} + return {"required": True, "status": "pending", "reasons": ["incomplete_preparation"]} + if meta.get("stage") == "complete": + if not journal or journal[-1]["phase"] != "commit" or journal[-1]["status"] != "applied": + _fail("invalid_completion_evidence") + result = identity.verify_active_target(self.config, control_root=_control_root(self.config)) + if result.get("valid") is not True: + _fail(next((row["code"] for row in result.get("reasons", []) if row.get("code") in REASONS), "identity_cutover_incomplete")) + self._view = {**meta, "status": "applied", "stage": "complete"} + return {"required": False, "status": "applied", "reasons": []} + self._view = dict(meta) + if meta["stage"] in {"preparing", "applying", "verifying"} or meta["status"] == "failed": + self._view.update(stage="interrupted", status="pending", reason_codes=["explicit_continuation_required"]) + return {"required": True, "status": self._view["status"], "reasons": self._view.get("reason_codes", [])} + detected = identity.detect(self.config, control_root=_control_root(self.config)) + if not detected["required"]: + verified = identity.verify_active_target(self.config, control_root=_control_root(self.config)) + if verified.get("valid") is not True: + _fail("identity_cutover_incomplete") + self._view = {"status": detected["status"], "stage": "required" if detected["required"] else "complete", + "reason_codes": [r["code"] for r in detected["reasons"]]} + return detected + except Exception as exc: + code = _safe_code(exc) + self._view = {"status": "blocked", "stage": "waiting" if code in {"writers_not_quiescent", "writers_active", "storage_unavailable"} else "required", "reason_codes": [code]} + return {"required": True, "status": "blocked", "reasons": [code]} + + def status(self): + with self._view_lock: + result = dict(self._view) + if self._last_preparation is not None: + result["last_preparation"] = {**self._last_preparation, + "reason_codes": list(self._last_preparation["reason_codes"])} + result.update(migration_id=MIGRATION_ID, busy=self._busy, restart_required=self._failed_here, + can_resume=False, can_prepare=not self._busy and not self._failed_here, suggested_state_dir=str(_root(self.config) / ".identity-migration-v1")) + try: + state = self._state_dir() + if state: + result["state_dir"] = str(state) + if not storage.fingerprint_file(state / "plan.json")["exists"]: + return result + plan = storage.load_plan(state) + result["plan_id"] = plan["plan_id"] + journal = storage.read_journal(state) + completed = {a for row in journal if row["phase"] == "apply" and row["status"] == "applied" for a in row["action_ids"]} + result["progress"] = {"completed": len(completed), "total": len(plan["actions"])} + if (state / "snapshot/manifest.json").exists(): + if (self._snapshot_view is None or self._snapshot_stat_signature(state, self._snapshot_view[1]) != self._snapshot_signature): + self._snapshot_view = self._snapshot(state, plan) + self._snapshot_signature = self._snapshot_stat_signature(state, self._snapshot_view[1]) + snapshot, manifest = self._snapshot_view + result["snapshot_digest"] = snapshot["digest"] + result["snapshot"] = {"path": snapshot["path"], "created_at": manifest["created_at"], + "size_bytes": sum(e["original"].get("size", 0) for e in manifest["entries"].values()) + + sum(e["size"] for e in manifest["external_inputs"].values()), "verified": True} + meta = self._meta(state) + result["can_prepare"] = bool(not self._busy and not self._failed_here and meta + and not meta.get("acknowledged") and meta["stage"] in {"preparing", "interrupted", "required", "waiting"}) + result["can_resume"] = bool(not self._busy and not self._failed_here and meta + and meta.get("acknowledged") is True and result["stage"] == "interrupted") + except Exception as exc: + result.update(status="blocked", reason_codes=[_safe_code(exc)], can_resume=False, can_prepare=False) + return result + + def _launch(self, fn, *, background=True, stage=None): + if not self._operation.acquire(blocking=False): + return self.status() + self._busy = True + if stage is not None: + with self._view_lock: + self._view.update(status="pending", stage=stage, reason_codes=[]) + if stage == "preparing": + self._last_preparation = None + def run(): + try: + fn() + finally: + self._busy = False + self._operation.release() + if background: + threading.Thread(target=run, name="identity-migration", daemon=True).start() + else: + run() + return self.status() + + def prepare(self, body, *, background=True): + if self._failed_here: + _fail("restart_required") + requested = body.get("state_dir") or str(_root(self.config) / ".identity-migration-v1") + state = self._state_dir() + if state and str(state) != requested: + _fail("original_migration_location_required") + if state: + meta = self._meta(state) + if meta and (meta.get("acknowledged") or meta["stage"] in {"backup_ready", "complete"}): + _fail("existing_preparation_must_be_preserved") + return self._launch(lambda: self._prepare(requested), background=background, stage="preparing") + + def _prepare(self, requested): + from migration_barrier import block_writers, exclusive_migration + state = None + try: + block_writers(self.config) + with exclusive_migration(self.config): + # Validate the requested location without creating it, so an + # unrelated planning blocker cannot hide invalid path input. + validated_state = self._validate_location(requested) + existing = self._state_dir() + initial_plan = None + if existing is None: + cron_text = self.read_cron() + _validate_cron_admission(cron_text) + initial_plan = identity.build_plan(self.config, control_root=_control_root(self.config), cron_text=cron_text) + if initial_plan["classification"] != "applicable": + _fail("migration_not_applicable" if not initial_plan["required"] else next((row["code"] for row in initial_plan.get("reasons", []) if row.get("code") in REASONS), "invalid_plan")) + state = validated_state + storage._private_directory(state, create=True) + self._validate_state_layout(state) + # Verify private durable publication semantics before allocating UUIDs. + probe = state / ".capability-probe" + storage._publish_once(probe, b"identity-migration-storage-v1\n") + with storage._directory(state) as directory: + os.unlink(probe.name, dir_fd=directory) + os.fsync(directory) + existing = self._state_dir() + if existing is None: + # The nonsensitive fixed selector is persisted BEFORE the ID map. + with storage._directory(self.selector.parent) as parent: + fd = os.open(self.selector.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent) + with os.fdopen(fd, "wb") as handle: + handle.write(storage._canonical({"migration_id": MIGRATION_ID, "state_dir": str(state)})) + handle.flush() + os.fsync(handle.fileno()) + os.fsync(parent) + elif existing != state: + _fail("original_migration_location_required") + meta = self._meta(state) + if meta and meta["stage"] in {"backup_ready", "acknowledged", "applying", "verifying", "complete"}: + _fail("existing_preparation_must_be_preserved") + self._save(state, stage="preparing", status="pending", reason_codes=[], acknowledged=False) + if storage.fingerprint_file(state / "plan.json")["exists"]: + plan = storage.load_plan(state) + if any(row["phase"] in {"apply", "commit"} for row in storage.read_journal(state)): + _fail("explicit_continuation_required") + storage.verify_inputs(plan) + if self.read_cron() != plan["external_inputs"]["managed_cron"]["text"]: + _fail("input_changed") + else: + if initial_plan is None: + cron_text = self.read_cron() + _validate_cron_admission(cron_text) + initial_plan = identity.build_plan(self.config, control_root=_control_root(self.config), cron_text=cron_text) + plan = initial_plan + if plan["classification"] != "applicable": + _fail(plan.get("reasons", [{}])[0].get("code", "migration_not_applicable") if plan.get("reasons") else "migration_not_applicable") + plan = _canonical_plan(self.config, plan) + storage.persist_plan(plan, state) + storage.append_journal(state, plan, "pending", "plan") + snapshot = storage.create_snapshot(plan, state) + self._snapshot_view = self._snapshot(state, plan) + self._snapshot_signature = self._snapshot_stat_signature(state, self._snapshot_view[1]) + storage.append_journal(state, plan, "applied", "snapshot") + self._save(state, stage="backup_ready", plan_id=plan["plan_id"], snapshot_digest=snapshot["digest"], reason_codes=[]) + except Exception as exc: + code = _safe_code(exc) + with self._view_lock: + self._view = {"status": "blocked", "stage": "waiting" if code in {"writers_active", "writers_running", "legacy_workers_running"} else "required", "reason_codes": [code]} + # Readiness keeps rescanning the installation. Retain the + # explicit action's safe result separately for browser polls, + # including failures before a persistent location exists. + self._last_preparation = {"status": "blocked", "reason_codes": [code], + "updated_at": datetime.now(timezone.utc).isoformat()} + if state and (state / "assistant.json").is_file(): + self._save(state, **self._view) + + def _binding(self, body): + state = self._state_dir() + if state is None: + _fail("prepare_required") + plan = storage.load_plan(state) + try: + snapshot, manifest = self._snapshot(state, plan) + except Exception as exc: + self._snapshot_view = None + self._view.update(status="blocked", reason_codes=[_safe_code(exc)]) + raise + if body.get("plan_id") != plan["plan_id"] or body.get("snapshot_digest") != snapshot["digest"]: + _fail("approval_required") + return state, plan, snapshot, manifest + + def acknowledge(self, body): + from migration_barrier import exclusive_migration + if not self._operation.acquire(blocking=False): + _fail("operation_in_progress") + try: + with exclusive_migration(self.config): + return self._acknowledge(body) + finally: + self._operation.release() + + def _acknowledge(self, body): + if self._busy: + _fail("operation_in_progress") + state, plan, snapshot, _ = self._binding(body) + meta = self._meta(state) + if not meta or meta["stage"] not in {"backup_ready", "acknowledged"}: + _fail("verified_backup_pause_required") + if body.get("independent_backup_ack") is not True: + _fail("independent_backup_ack_required") + storage.verify_inputs(plan) + if self.read_cron() != plan["external_inputs"]["managed_cron"]["text"]: + _fail("input_changed") + storage.append_journal(state, plan, "applied", "confirm") + self._save(state, stage="acknowledged", acknowledged=True) + return self.status() + + def apply(self, body, *, background=True): + if self._failed_here: + _fail("restart_required") + if self._busy: + return self.status() + state, plan, snapshot, _ = self._binding(body) + meta = self._meta(state) + if not meta or meta.get("acknowledged") is not True or meta["stage"] not in {"acknowledged", "applying", "verifying", "interrupted"}: + _fail("independent_backup_ack_required") + approval = {"approved": True, "independent_backup_acknowledged": True, + "plan_id": plan["plan_id"], "snapshot_digest": snapshot["digest"]} + return self._launch(lambda: self._apply(state, approval), background=background) + + def _apply(self, state, approval): + from migration_barrier import exclusive_migration, clear_block, quiescence_held, block_writers + from migrations.identity_apply import apply_plan + try: + with exclusive_migration(self.config): + meta = self._meta(state) + if not meta or not meta.get("acknowledged") or meta.get("plan_id") != approval["plan_id"] or meta.get("snapshot_digest") != approval["snapshot_digest"]: + _fail("approval_required") + self._save(state, stage="applying", status="pending", reason_codes=[]) + result = apply_plan(self.config, state, approval=approval, quiescence_callback=lambda: quiescence_held(self.config), + read_cron=self.read_cron, write_cron=self.write_cron, + render_cron=lambda original, plan: _render_cron(self.config, original, plan), + control_root=_control_root(self.config)) + if result.get("status") != "applied": + _fail("final_verification_failed") + self._save(state, stage="verifying") + # The registry still validates all remaining migrations before release. + from migrations.registry import run_startup_migrations + self._save(state, stage="complete", status="applied") + summary = run_startup_migrations(self.config) + if summary.get("status") != "ok": + _fail("startup_gate_failed") + from startup_state import set_startup_state, normal_startup_state + set_startup_state(self.config, normal_startup_state(summary)) + clear_block(self.config) + if self.activate: + self.activate(self.config) + except Exception as exc: + self._failed_here = True + block_writers(self.config) + meta = self._meta(state) or {} + self._save(state, status="failed", stage="complete" if meta.get("stage") == "complete" else "interrupted", reason_codes=[_safe_code(exc)]) + from startup_state import set_startup_state, migration_maintenance_state + set_startup_state(self.config, migration_maintenance_state({"failed": [MIGRATION_ID]})) + + @contextmanager + def snapshot_files(self, body): + """Exact protected export set; caller streams it without public staging.""" + state, plan, snapshot, manifest = self._binding(body) + files = [("plan.json", state / "plan.json"), ("snapshot/metadata.json", state / "snapshot/metadata.json"), + ("snapshot/manifest.json", state / "snapshot/manifest.json")] + names = {e["blob"] for e in manifest["entries"].values() if e["blob"]} + names.update(e["blob"] for e in manifest["external_inputs"].values()) + files.extend(("snapshot/files/" + name, state / "snapshot/files" / name) for name in sorted(names)) + # Fixed short ASCII member names permit a deterministic USTAR stream. + def expected_bytes(raw): + return {"exists": True, "size": len(raw), "mode": 0o600, "sha256": hashlib.sha256(raw).hexdigest()} + metadata = {key: manifest[key] for key in ("schema_version", "migration_id", "plan_id", "created_at")} + expected = {"plan.json": expected_bytes(storage._canonical(plan)), + "snapshot/metadata.json": expected_bytes(storage._canonical(metadata)), + "snapshot/manifest.json": expected_bytes(storage._canonical(manifest))} + for entry in manifest["entries"].values(): + if entry["blob"]: + expected["snapshot/files/" + entry["blob"]] = {**entry["original"], "mode": 0o600} + for entry in manifest["external_inputs"].values(): + expected["snapshot/files/" + entry["blob"]] = {"exists": True, "mode": 0o600, "size": entry["size"], "sha256": entry["sha256"]} + members = [(name, path, expected[name]) for name, path in files] + size = sum(512 + ((fp["size"] + 511) // 512) * 512 for _, _, fp in members) + 1024 + size = ((size + tarfile.RECORDSIZE - 1) // tarfile.RECORDSIZE) * tarfile.RECORDSIZE + yield {"members": members, "size_bytes": size} + + +def get_assistant(config): + key = str(_root(config)) + with _ASSISTANTS_LOCK: + if key not in _ASSISTANTS: + _ASSISTANTS[key] = IdentityMigrationAssistant(config) + return _ASSISTANTS[key] diff --git a/api/identity_startup_watch.py b/api/identity_startup_watch.py new file mode 100644 index 00000000..5d066163 --- /dev/null +++ b/api/identity_startup_watch.py @@ -0,0 +1,88 @@ +"""Retry read-only startup readiness after storage/old workers recover (#479). + +Waiting is not migration consent. This watcher can resume ordinary startup only +when the identity assistant proves that no conversion or explicit continuation +is required. It never prepares a plan/snapshot, acknowledges, or applies one. +""" + +import threading + +from migration_barrier import data_root + + +_WATCHERS = {} +_WATCHERS_LOCK = threading.Lock() + + +def retry_startup_once(config, *, storage_ready, activate, assistant=None): + """Return ready/waiting/restart_required; callbacks retain startup policy. + + ``storage_ready`` must check every configured runtime mount without waiting + or creating directories. ``activate`` repeats the normal startup gate under + exclusive writer ownership before publishing readiness or starting services; + its True result means startup succeeded, even during empty first setup. + """ + if assistant is None: + from identity_migration_api import get_assistant + assistant = get_assistant(config) + # Serialize against both Prepare/Run actions, including the gap between + # read-only eligibility detection and the repeated startup gate. + if not assistant._operation.acquire(blocking=False): + return "waiting" + try: + if assistant._failed_here: + return "restart_required" + # An explicit assistant action may already have completed activation + # while this observer was asleep. Only an explicit normal state counts; + # absent startup state must still pass the detection/startup gates. + startup = config.get("_STARTUP_STATE") + if isinstance(startup, dict) and startup.get("mode") == "normal": + return "ready" + if assistant._busy: + return "waiting" + if storage_ready(config) is not True: + return "waiting" + result = assistant.startup_detection() + if not isinstance(result, dict) or result.get("required") is not False: + return "waiting" + if assistant._failed_here: + return "restart_required" + return "ready" if activate(config) is True else "waiting" + except Exception: + # A transient check cannot open the gate. Details belong to the normal + # startup/assistant diagnostics, never arbitrary exception strings here. + return "waiting" + finally: + assistant._operation.release() + + +def start_startup_readiness_watch(config, *, storage_ready, activate, + interval_seconds=10, stop_event=None): + """Start at most one readiness observer for this installation in the UI.""" + key = str(data_root(config)) + with _WATCHERS_LOCK: + existing = _WATCHERS.get(key) + if existing is not None and existing.is_alive(): + return existing + stop = stop_event or threading.Event() + interval = max(0.01, float(interval_seconds)) + + def watch(): + try: + while not stop.wait(interval): + result = retry_startup_once(config, storage_ready=storage_ready, activate=activate) + if result in {"ready", "restart_required"}: + return + finally: + with _WATCHERS_LOCK: + if _WATCHERS.get(key) is threading.current_thread(): + _WATCHERS.pop(key, None) + + thread = threading.Thread(target=watch, name="identity-startup-readiness", daemon=True) + _WATCHERS[key] = thread + try: + thread.start() + except BaseException: + _WATCHERS.pop(key, None) + raise + return thread diff --git a/api/job_actions.py b/api/job_actions.py new file mode 100644 index 00000000..e3dcc80f --- /dev/null +++ b/api/job_actions.py @@ -0,0 +1,113 @@ +"""UUID control-plane boundaries and transactions (#447, #474).""" + +from copy import deepcopy + +from inventory_store import inventory_lock +from job_model import JobValidationError, validate_job_id +from job_store import read_jobs, read_repositories, validate_assignments, write_transaction + + +def resolve_request_schedule_id(config, body, *, endpoint): + if "service" in body: + if body["service"] != "restore_test" or {"job_id", "job_key"}.intersection(body): + raise JobValidationError("invalid_schedule_service", "The restore-test service must be addressed separately from job IDs") + return "restore_test" + return resolve_request_job_id(config, body, endpoint=endpoint) + + +def resolve_request_job_id(config, body, *, endpoint): + from repository_context import jobs_dir + with inventory_lock(jobs_dir(config).parent): + jobs = read_jobs(jobs_dir(config)) + job_id = body.get("job_id") + if "job_id" in body: + validate_job_id(job_id) + if job_id not in jobs: + raise JobValidationError("unknown_job_id", "Unknown job_id") + if "job_key" in body: + raise JobValidationError("deprecated_job_key", "This endpoint requires job_id") + if job_id is None: + raise JobValidationError("invalid_job_id", "job_id is required") + return job_id + + +def _inventory(config): + from repository_context import jobs_dir + from repositories_api import repositories_file + jobs = read_jobs(jobs_dir(config)) + store = read_repositories(repositories_file(config)) + validate_assignments(jobs, store) + return jobs, store + + +def prepare_job_action(config, job_id, *, require_enabled=False): + from repository_context import jobs_dir, resolve_job_repository_context + from schedule_api import get_schedules + validate_job_id(job_id) + with inventory_lock(jobs_dir(config).parent): + jobs, _ = _inventory(config) + get_schedules(config) + if job_id not in jobs: + raise JobValidationError("unknown_job_id", "Unknown job_id") + job = jobs[job_id] + if require_enabled and not job.get("enabled", True): + raise JobValidationError("job_disabled", "The job is disabled") + return resolve_job_repository_context(config, job_id, job=job, require_passphrase_file=False) + + +def set_job_enabled(config, job_id, enabled): + from repository_context import jobs_dir + from schedule_api import get_schedules, schedule_lines, _update_crontab + if type(enabled) is not bool: + raise JobValidationError("invalid_job_settings", "enabled must be boolean") + validate_job_id(job_id) + with inventory_lock(jobs_dir(config).parent): + jobs, _ = _inventory(config) + if job_id not in jobs: + raise JobValidationError("unknown_job_id", "Unknown job_id") + schedules = get_schedules(config) + old_lines = schedule_lines(config, schedules, jobs) + job = deepcopy(jobs[job_id]) + job["enabled"] = enabled + jobs[job_id] = job + new_lines = schedule_lines(config, schedules, jobs) + write_transaction({jobs_dir(config) / (job_id + ".json"): job}, + after_write=lambda: _update_crontab(new_lines), + rollback_after=lambda: _update_crontab(old_lines)) + return {"saved": True, "job_id": job_id, "enabled": enabled} + + +def delete_job_configuration(config, job_id, *, confirmed_artifacts=None, preview=False): + """Remove exactly one job, its reverse links and schedule, preserving artifacts.""" + from repository_context import jobs_dir + from repositories_api import repositories_file + from schedule_api import get_schedules, schedule_lines, _update_crontab, _schedules_path + validate_job_id(job_id) + with inventory_lock(jobs_dir(config).parent): + jobs, store = _inventory(config) + if job_id not in jobs: + raise JobValidationError("unknown_job_id", "Unknown job_id") + from identity_lifecycle import deletion_plan + from jobs_api import get_job_runtime_state + if get_job_runtime_state(config, job_id).get("running"): + raise JobValidationError("job_running", "Wait for the running job before deleting it") + plan = deletion_plan(config, job_id, confirmed_artifacts) + if preview: + return {"job_id": job_id, "name": jobs[job_id]["name"], "artifacts": plan["artifacts"], "repository_preserved": True} + schedules = get_schedules(config) + old_lines = schedule_lines(config, schedules, jobs) + del jobs[job_id] + schedules.pop(job_id, None) + for repository in store["repositories"]: + for field in ("job_ids", "source_job_ids"): + repository[field] = [value for value in repository[field] if value != job_id] + validate_assignments(jobs, store) + new_lines = schedule_lines(config, schedules, jobs) + write_transaction({ + **plan["changes"], + jobs_dir(config) / (job_id + ".json"): None, + repositories_file(config): store, + _schedules_path(config): schedules, + }, after_write=lambda: _update_crontab(new_lines), + rollback_after=lambda: _update_crontab(old_lines)) + return {"deleted": True, "job_id": job_id, "deleted_metadata": 1, "deleted_artifacts": plan["deleted_count"]} diff --git a/api/job_control.py b/api/job_control.py index 9fae1bc0..a0ba6dde 100644 --- a/api/job_control.py +++ b/api/job_control.py @@ -4,7 +4,6 @@ import json import os -import re import signal import threading import time @@ -13,9 +12,10 @@ from typing import Any, Dict, Optional -CONTROL_ROOT = Path("/run/borg-backup-ui/jobs") -_RUN_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,96}$") -_JOB_KEY_RE = re.compile(r"^[A-Za-z0-9._-]{1,128}$") +from job_model import validate_job_id +from job_runs import control_root, validate_run_id + +CONTROL_ROOT = control_root() def _utc_now() -> str: @@ -41,20 +41,14 @@ def _atomic_json(path: Path, data: Dict[str, Any]) -> None: pass -def _safe_component(value: str, pattern: re.Pattern[str], label: str) -> str: - clean = str(value or "").strip() - if not pattern.fullmatch(clean): - raise ValueError(f"Invalid {label}") - return clean - - class JobControl: """Runner-owned state plus an API-owned cancellation marker.""" - def __init__(self, job_key: str, run_id: str, root: Path = CONTROL_ROOT) -> None: - self.job_key = _safe_component(job_key, _JOB_KEY_RE, "job key") - self.run_id = _safe_component(run_id, _RUN_ID_RE, "run id") - self.run_dir = Path(root) / self.run_id + def __init__(self, job_id: str, run_id: str, root: Path | None = None, *, snapshot: dict | None = None) -> None: + self.job_id = validate_job_id(job_id) + self.run_id = validate_run_id(run_id) + self.run_dir = Path(root or control_root()) / self.run_id + self.snapshot = dict(snapshot or {}) self.state_file = self.run_dir / "state.json" self.cancel_file = self.run_dir / "cancel.request.json" self._process_lock = threading.Lock() @@ -79,8 +73,9 @@ def update_phase( ) -> Dict[str, Any]: previous = read_control_state(self.run_id, self.run_dir.parent) data: Dict[str, Any] = { + **self.snapshot, "schema_version": 1, - "job_key": self.job_key, + "job_id": self.job_id, "run_id": self.run_id, "pid": os.getpid(), "phase": str(phase), @@ -122,7 +117,7 @@ def _monitor() -> None: self._monitor_thread = threading.Thread( target=_monitor, daemon=True, - name=f"cancel-monitor-{self.job_key}", + name=f"cancel-monitor-{self.job_id}", ) self._monitor_thread.start() @@ -135,14 +130,18 @@ def detach_process(self) -> None: self._active_process = None -def read_control_state(run_id: str, root: Path = CONTROL_ROOT) -> Dict[str, Any]: - safe_run_id = _safe_component(run_id, _RUN_ID_RE, "run id") - path = Path(root) / safe_run_id / "state.json" +def read_control_state(run_id: str, root: Path | None = None) -> Dict[str, Any]: + safe_run_id = validate_run_id(run_id) + path = Path(root or control_root()) / safe_run_id / "state.json" try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError, TypeError, ValueError): return {} - if not isinstance(data, dict): + if not isinstance(data, dict) or data.get("run_id") != safe_run_id: + return {} + try: + validate_job_id(data.get("job_id")) + except ValueError: return {} if (path.parent / "cancel.request.json").is_file(): data["cancel_requested"] = True @@ -150,27 +149,27 @@ def read_control_state(run_id: str, root: Path = CONTROL_ROOT) -> Dict[str, Any] def request_cancel( - job_key: str, + job_id: str, run_id: str, *, requested_by: str = "", - root: Path = CONTROL_ROOT, + root: Path | None = None, ) -> Dict[str, Any]: - safe_job_key = _safe_component(job_key, _JOB_KEY_RE, "job key") - safe_run_id = _safe_component(run_id, _RUN_ID_RE, "run id") + safe_job_id = validate_job_id(job_id) + safe_run_id = validate_run_id(run_id) state = read_control_state(safe_run_id, root) if not state or state.get("finished"): raise FileNotFoundError("The backup run is no longer active") - if str(state.get("job_key") or "") != safe_job_key: + if str(state.get("job_id") or "") != safe_job_id: raise ValueError("The run does not belong to this job") if not bool(state.get("cancel_allowed")): raise RuntimeError("Cancellation is no longer possible during runtime recovery") - run_dir = Path(root) / safe_run_id + run_dir = Path(root or control_root()) / safe_run_id marker = run_dir / "cancel.request.json" payload = { "schema_version": 1, - "job_key": safe_job_key, + "job_id": safe_job_id, "run_id": safe_run_id, "requested_at": _utc_now(), "requested_by": str(requested_by or ""), diff --git a/api/job_model.py b/api/job_model.py new file mode 100644 index 00000000..e3191b77 --- /dev/null +++ b/api/job_model.py @@ -0,0 +1,195 @@ +"""Canonical, immutable job metadata contract (#447, #473). + +Pure validation and edit operations. Legacy conversion belongs exclusively to +the explicit migration boundary, never to a read or an ordinary wizard save. +""" + +from copy import deepcopy +import re +from uuid import UUID + +from job_source_paths import normalize_source_paths + + +JOB_SCHEMA_VERSION = 4 +MUTABLE_IDENTITY_FIELDS = {"job_key", "backup_type", "type_id", "location"} +ARCHIVE_TIMESTAMP_PATTERN = "YYYY-MM-DD_HH-mm-ss" +_SAFE = re.compile(r"[A-Za-z0-9_.-]+") + + +class JobValidationError(ValueError): + def __init__(self, code, message): + self.api_code = code + super().__init__(message) + + +def validate_job_id(value): + try: + parsed = UUID(value) if isinstance(value, str) else None + except ValueError: + parsed = None + if parsed is None or parsed.version != 4 or str(parsed) != value: + raise JobValidationError("invalid_job_id", "A canonical UUIDv4 job_id is required") + return value + + +def validate_archive_prefix(value): + if not isinstance(value, str) or not _SAFE.fullmatch(value) or value in {".", ".."}: + raise JobValidationError("invalid_archive_prefix", "Archive prefix must use letters, digits, dots, underscores or hyphens; '.' and '..' are not allowed") + return value + + +def archive_name_preview(prefix): + return validate_archive_prefix(prefix) + "-" + ARCHIVE_TIMESTAMP_PATTERN + + +def updated_archive_prefixes(prefix, previous): + validate_archive_prefix(prefix) + if not isinstance(previous, list): + raise JobValidationError("invalid_archive_prefix", "Archive prefix history must be a list") + for value in previous: + validate_archive_prefix(value) + return list(dict.fromkeys([prefix, *previous])) + + +def validate_job(meta, *, filename=None): + """Validate without repairing, allocating an ID, or discarding fields.""" + if not isinstance(meta, dict) or type(meta.get("schema_version")) is not int or meta["schema_version"] != JOB_SCHEMA_VERSION: + raise JobValidationError("job_migration_required", "Job metadata requires the explicit identity migration") + job_id = validate_job_id(meta.get("job_id")) + if filename is not None and filename != job_id + ".json": + raise JobValidationError("invalid_job_filename", "Job filename does not match job_id") + if MUTABLE_IDENTITY_FIELDS.intersection(meta): + raise JobValidationError("mutable_job_identity", "Canonical metadata must not contain mutable identity fields") + if not isinstance(meta.get("name"), str) or not meta["name"].strip(): + raise JobValidationError("invalid_job_name", "Job name must not be empty") + repo = meta.get("repository_key") + if not isinstance(repo, str) or not _SAFE.fullmatch(repo): + raise JobValidationError("invalid_job_repository", "A repository_key is required") + prefixes = meta.get("archive_prefixes") + if not isinstance(prefixes, list) or not prefixes or updated_archive_prefixes(prefixes[0], prefixes) != prefixes: + raise JobValidationError("invalid_archive_prefix", "Archive prefixes must be nonempty, ordered and unique") + aliases = meta.get("legacy_job_keys") + if not isinstance(aliases, list) or any(not isinstance(a, str) or not _SAFE.fullmatch(a) for a in aliases) or len(set(aliases)) != len(aliases): + raise JobValidationError("invalid_job_aliases", "Legacy aliases must be an ordered list of unique exact identifiers") + if normalize_source_paths(meta.get("source_paths")) != meta.get("source_paths"): + raise JobValidationError("invalid_source_paths", "Source paths must be canonical") + for field in ("enabled", "file_activity", "mount_before_run", "unmount_after_run"): + if field in meta and type(meta[field]) is not bool: + raise JobValidationError("invalid_job_settings", "A boolean job setting has an unsupported value") + if "compression" in meta and (not isinstance(meta["compression"], str) or not meta["compression"].strip()): + raise JobValidationError("invalid_job_settings", "Compression must be a nonempty string") + features = meta.get("features", {}) + if not isinstance(features, dict) or any(type(features.get(kind, False)) is not bool for kind in ("docker", "vm")): + raise JobValidationError("invalid_runtime_control", "Unsupported job feature settings") + for kind in ("docker", "vm"): + control = meta.get(kind + "_control") + if control is None and kind + "_control" not in meta: + continue + modes = {"all", "selected", "none"} | ({"except_selected"} if kind == "docker" else set()) + if not isinstance(control, dict) or not isinstance(control.get("mode"), str) or control["mode"] not in modes: + raise JobValidationError("invalid_runtime_control", "Unsupported runtime control mode") + selected = control.get("selected", []) + if not isinstance(selected, list) or any(not isinstance(v, str) or not v.strip() for v in selected): + raise JobValidationError("invalid_runtime_control", "Invalid runtime selection") + ack = "ack_appdata_risk" if kind == "docker" else "ack_domains_risk" + if ack in control and type(control[ack]) is not bool: + raise JobValidationError("invalid_runtime_control", "Invalid runtime acknowledgement") + if "retention" in meta: + retention = meta["retention"] + if not isinstance(retention, dict) or any( + key in retention and (not isinstance(retention[key], str) or not re.fullmatch(r"[0-9]+", retention[key])) + for key in ("daily", "weekly", "monthly", "yearly") + ): + raise JobValidationError("invalid_retention", "Unsupported retention settings") + if "cache_reference" in meta: + cache = meta["cache_reference"] + if not isinstance(cache, dict) or any( + not isinstance(cache.get(key), str) or not cache[key].startswith("/") + or any(ch in cache[key] for ch in ("\x00", "\n", "\r")) + for key in ("directory", "check_flag_file") + ): + raise JobValidationError("invalid_cache_reference", "Cache references must be explicit absolute paths") + if not isinstance(cache.get("repository_key"), str) or not _SAFE.fullmatch(cache["repository_key"]): + raise JobValidationError("invalid_cache_reference", "Cache check markers require their original repository reference") + return meta + + +def validate_job_inventory(jobs): + """Reject alias collisions and shared-repository prefix ownership overlap.""" + aliases, ownership = {}, [] + for job_id, job in jobs.items(): + validate_job(job, filename=job_id + ".json") + for alias in job["legacy_job_keys"]: + if alias in aliases or (alias in jobs and alias != job_id): + raise JobValidationError("ambiguous_job_alias", "A legacy alias has conflicting owners") + aliases[alias] = job_id + for prefix in job["archive_prefixes"]: + for other_repo, other_prefix, other_id in ownership: + if other_repo == job["repository_key"] and other_id != job_id and ( + prefix == other_prefix or prefix.startswith(other_prefix + "-") or other_prefix.startswith(prefix + "-") + ): + raise JobValidationError("ambiguous_archive_ownership", "Archive prefixes overlap with another job in the selected repository") + ownership.append((job["repository_key"], prefix, job_id)) + + +def new_job_defaults(): + return { + "schema_version": JOB_SCHEMA_VERSION, "legacy_job_keys": [], + "description": "", "icon": "sonstiges", "icon_color": "", "enabled": True, + "standard": "wizard", "runner": "scriptless-wizard-runner", "script": "", + "mount_before_run": True, "unmount_after_run": True, + "exclude_paths": [], "compression": "lz4", "file_activity": False, + "features": {"docker": False, "vm": False}, + "docker_control": {"mode": "none", "selected": [], "ack_appdata_risk": False}, + "vm_control": {"mode": "none", "selected": [], "ack_domains_risk": False}, + "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, + } + + +def job_to_params(meta): + """Only expose wizard-owned fields, not unknown settings or secrets.""" + result = {key: deepcopy(meta[key]) for key in ( + "description", "icon", "icon_color", "source_paths", "exclude_paths", + "repository_key", "mount_before_run", "unmount_after_run", "compression", + "file_activity", "docker_control", "vm_control", + ) if key in meta} + result.update(job_name=meta.get("name", ""), archive_prefix=(meta.get("archive_prefixes") or [""])[0]) + for key, value in meta.get("retention", {}).items(): + if key in {"daily", "weekly", "monthly", "yearly"}: + result["keep_" + key] = value + return result + + +def apply_wizard_changes(params, *, existing=None, job_id, now, duplicate=False): + """Patch exposed fields; retain every other existing setting verbatim.""" + if existing is not None: + validate_job(existing) + if not duplicate and job_id != existing["job_id"]: + raise JobValidationError("immutable_job_id", "Editing cannot change job_id") + result = deepcopy(existing) if existing is not None else new_job_defaults() + fresh = existing is None or duplicate + result.update(schema_version=JOB_SCHEMA_VERSION, job_id=validate_job_id(job_id), updated_at=now) + if fresh: + result.update(created_at=now, legacy_job_keys=[]) + result.pop("cache_reference", None) + if "job_name" in params: + result["name"] = params["job_name"].strip() + prefix = params.get("archive_prefix", (result.get("archive_prefixes") or [""])[0]) + result["archive_prefixes"] = updated_archive_prefixes(prefix, [] if fresh else result.get("archive_prefixes", [])) + for key in ( + "description", "icon", "icon_color", "repository_key", "source_paths", "exclude_paths", + "compression", "file_activity", "mount_before_run", "unmount_after_run", + ): + if key in params: + result[key] = deepcopy(params[key]) + for kind in ("docker", "vm"): + key = kind + "_control" + if key in params: + result.setdefault(key, {}).update(deepcopy(params[key])) + result.setdefault("features", {})[kind] = result[key]["mode"] != "none" + for period in ("daily", "weekly", "monthly", "yearly"): + if "keep_" + period in params: + result.setdefault("retention", {})[period] = params["keep_" + period] + validate_job(result) + return result diff --git a/api/job_presentation.py b/api/job_presentation.py new file mode 100644 index 00000000..cf60a576 --- /dev/null +++ b/api/job_presentation.py @@ -0,0 +1,37 @@ +"""Preserve legacy automatic presentation without retaining mutable identity.""" + +ICON_KEYS = frozenset({ + 'flash', 'appdata', 'photos', 'vms', 'sonstiges', 'docker', 'folder', + 'cloud', 'archive', 'database', 'server', 'home', 'music', 'video', + 'documents', 'code', 'camera', 'usb', 'shield', +}) +LEGACY_TYPE_COLORS = {'flash': 'blue', 'appdata': 'orange', 'photos': 'violet', 'vms': 'green'} + + +def legacy_automatic_icon(meta): + """Only materialize a recognized old automatic icon; explicit values win.""" + if str(meta.get('icon') or '').strip(): + return None + previous = str(meta.get('backup_type') or '').strip().lower() + return previous if previous in ICON_KEYS else None + + +def legacy_presentation_defaults(meta): + """Keep the previous list appearance using existing explicit palette keys. + + Legacy cards colored an explicitly selected icon by backup_type, while the + editor preview already used the icon itself. Preserve the recorded jobs' + list colors when switching to the consistent icon-based canonical UI. + """ + updates = {} + automatic = legacy_automatic_icon(meta) + if automatic is not None: + updates['icon'] = automatic + icon = str(meta.get('icon') or '').strip().lower() + previous = str(meta.get('backup_type') or '').strip().lower() + if icon in ICON_KEYS and icon != previous and not str(meta.get('icon_color') or '').strip(): + if previous in LEGACY_TYPE_COLORS: + updates['icon_color'] = LEGACY_TYPE_COLORS[previous] + elif icon in LEGACY_TYPE_COLORS: + updates['icon_color'] = 'gray' + return updates diff --git a/api/job_runs.py b/api/job_runs.py new file mode 100644 index 00000000..bbddea93 --- /dev/null +++ b/api/job_runs.py @@ -0,0 +1,154 @@ +"""Immutable, private start-time context for a backup run (#475). + +The HTTP boundary creates this once. The runner never re-resolves an editable +job to obtain its sources, target, retention or runtime-control settings. +Only ``descriptors`` may be exposed in status/log responses. +""" + +from copy import deepcopy +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import re +from uuid import uuid4 + +from job_model import JobValidationError, validate_job, validate_job_id + + +def validate_run_id(value): + try: + return validate_job_id(value) + except ValueError as exc: + raise JobValidationError("invalid_run_id", "A canonical UUIDv4 run_id is required") from exc + + +def control_root(): + return Path(os.environ.get("BORG_UI_CONTROL_ROOT") or "/run/borg-backup-ui/jobs") + + +def log_filename(job_id, run_id, name): + validate_job_id(job_id) + validate_run_id(run_id) + slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(name)).strip(".-_")[:48] or "job" + return f"Borg-Backup_{slug}_{job_id[:8]}--{run_id}.log" + + +def descriptors(context): + return {key: deepcopy(context[key]) for key in ( + "job_id", "run_id", "job_name_snapshot", "archive_prefix_snapshot", + "archive_prefixes_snapshot", "repository_key_snapshot", + "repository_snapshot", "location_snapshot", "started_at", "log_file", "file_activity", + )} + + +def create_run_context(config, job_id, *, require_enabled=True): + from config_api import read_expanded_conf + from inventory_store import inventory_lock + from job_actions import prepare_job_action + from repository_context import jobs_dir, resolve_job_repository_context + with inventory_lock(jobs_dir(config).parent): + context = prepare_job_action(config, job_id, require_enabled=require_enabled) + # Validate the credential reference before creating a run or subprocess. + context = resolve_job_repository_context(config, job_id, job=context["job"]) + settings = read_expanded_conf(config) + run_id = str(uuid4()) + payload = { + "schema_version": 1, "job_id": job_id, "run_id": run_id, + "started_at": datetime.now(timezone.utc).isoformat(), + "job_name_snapshot": context["name"], + "archive_prefix_snapshot": context["archive_prefix"], + "archive_prefixes_snapshot": context["archive_prefixes"], + "repository_key_snapshot": context["repository_key"], + "repository_snapshot": context["repository_path"], + "location_snapshot": context["location"], + "context": context, "settings": settings, + "log_file": str(Path(settings.get("GLOBAL_LOG_DIR") or "/mnt/user/Logs") / log_filename(job_id, run_id, context["name"])), + "file_activity": bool(context["job"].get("file_activity")), + } + validate_run_context(payload, job_id, run_id) + directory = control_root() / run_id + directory.mkdir(parents=True, mode=0o700) # UUID collision never overwrites. + path = directory / "context.json" + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as handle: + json.dump(payload, handle, ensure_ascii=False) + handle.flush() + os.fsync(handle.fileno()) + return deepcopy(payload) + + +def read_run_context(job_id, run_id): + from job_store import read_json + validate_job_id(job_id) + validate_run_id(run_id) + payload = read_json(control_root() / run_id / "context.json") + return validate_run_context(payload, job_id, run_id) + + +def validate_run_context(payload, job_id, run_id): + """Pure validation, also used by the migration's owned-record verifier.""" + validate_job_id(job_id) + validate_run_id(run_id) + if not isinstance(payload, dict): + raise FileNotFoundError("Run context is not available") + context = payload.get("context", {}) + if not isinstance(context, dict): + raise ValueError("Run repository context is missing") + if payload.get("schema_version") != 1 or payload.get("job_id") != job_id or payload.get("run_id") != run_id: + raise JobValidationError("conflicting_run_identity", "Run context does not match the requested identity") + job = context.get("job") + validate_job(job) + expected = { + "job_id": job["job_id"], "job_name_snapshot": job["name"], + "archive_prefix_snapshot": job["archive_prefixes"][0], + "archive_prefixes_snapshot": job["archive_prefixes"], + "repository_key_snapshot": job["repository_key"], + "repository_snapshot": context.get("repository_path"), + "location_snapshot": context.get("location"), + "file_activity": bool(job.get("file_activity")), + } + if any(payload.get(key) != value for key, value in expected.items()): + raise JobValidationError("conflicting_run_identity", "Run snapshot descriptors are inconsistent") + if not isinstance(payload.get("settings"), dict): + raise ValueError("Run settings are missing") + if type(payload.get("file_activity")) is not bool: + raise ValueError("Run file-activity setting is invalid") + path = payload.get("log_file") + if (not isinstance(path, str) or not Path(path).is_absolute() + or Path(path).name != log_filename(job_id, run_id, job["name"])): + raise ValueError("Run log reference is invalid") + return payload + + +def maintenance_context_unchanged(config, snapshot): + """Do not prune an old target after it may have acquired another owner.""" + from job_actions import prepare_job_action + try: + current = prepare_job_action(config, snapshot["job_id"]) + except (ValueError, OSError): + return False + return (current["repository_key"] == snapshot["repository_key_snapshot"] + and current["repository_path"] == snapshot["repository_snapshot"] + and set(snapshot["archive_prefixes_snapshot"]).issubset(current["archive_prefixes"])) + + +def find_run_status(config, job_id, run_id=""): + """Locate a finished run by payload identity, without filename inference.""" + validate_job_id(job_id) + if run_id: + validate_run_id(run_id) + from wizard_runner import _ensure_runtime_import_paths + _ensure_runtime_import_paths(Path(config["BACKUP_SCRIPTS_DIR"])) + from lib.status import StatusStore + directory = Path(config.get("STATUS_DIR") or "/mnt/user/backup-status") + rows = [row for row in StatusStore(directory).load() + if row.job_id == job_id and row.identity_state != "unassigned" + and (not run_id or row.run_id == run_id)] + if not rows: + return {} + status = max(rows, key=lambda row: (row.timestamp, row.run_id)) + return {"job_id": status.job_id, "run_id": status.run_id, + "job_name_snapshot": status.job_name_snapshot, + "running": False, "exit_code": status.exit_code, + "phase": "skipped" if status.status == "skipped" else ("completed" if status.exit_code < 2 else status.status), + "file_activity": status.file_activity, "log_file": status.log_file} diff --git a/api/job_store.py b/api/job_store.py new file mode 100644 index 00000000..f0ed9dd0 --- /dev/null +++ b/api/job_store.py @@ -0,0 +1,186 @@ +"""Strict schema-v4 inventory and configuration transactions (#447, #473, #474). + +No implicit discovery, conversion or reconciliation of legacy user data. +The explicit job action/schedule callers own managed cron updates. +""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path + +from inventory_store import atomic_write_bytes, atomic_write_json, inventory_lock +from job_model import JobValidationError, validate_job, validate_job_id, validate_job_inventory +from migrations.identity_storage import inventory_group, read_fingerprinted_file + + +def read_json(path, *, missing=None): + """Read a regular file without following symlinks or accepting duplicate keys.""" + _, raw = read_fingerprinted_file(Path(path)) + if raw is None: + return deepcopy(missing) + + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate member") + result[key] = value + return result + + def invalid_constant(_): + raise ValueError("invalid constant") + + try: + if len(raw) > 64 * 1024 * 1024: + raise ValueError("too large") + value = json.loads(raw, object_pairs_hook=pairs, parse_constant=invalid_constant) + if not isinstance(value, dict): + raise ValueError("not an object") + return value + except (ValueError, UnicodeError, RecursionError): + raise JobValidationError("invalid_job_inventory", "An owned inventory file is malformed; no changes were made") from None + + +def job_revision(meta): + return hashlib.sha256(json.dumps(meta, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()).hexdigest() + + +def read_jobs(jobs_dir): + result = {} + for name in inventory_group(Path(jobs_dir), [".json"])["entries"]: + meta = read_json(Path(jobs_dir) / name) + validate_job(meta, filename=name) + result[meta["job_id"]] = meta + validate_job_inventory(result) + return result + + +def read_job(jobs_dir, job_id): + validate_job_id(job_id) + # Validate the inventory as a whole: another ambiguous owner is not safe. + jobs = read_jobs(jobs_dir) + if job_id not in jobs: + raise JobValidationError("unknown_job_id", "Unknown job_id; editing cannot create a job") + return jobs[job_id] + + +def read_repositories(path): + store = read_json(path, missing={"schema_version": 1, "repositories": []}) + if type(store.get("schema_version")) is not int or store["schema_version"] != 1 or not isinstance(store.get("repositories"), list): + raise JobValidationError("invalid_job_repository", "Unsupported repository inventory") + keys = set() + for row in store["repositories"]: + if not isinstance(row, dict) or not isinstance(row.get("repository_key"), str) or not row["repository_key"] or row["repository_key"] in keys: + raise JobValidationError("invalid_job_repository", "Invalid or duplicate repository entry") + keys.add(row["repository_key"]) + return store + + +def validate_assignments(jobs, store): + keys = {row["repository_key"] for row in store["repositories"]} + if any(job["repository_key"] not in keys for job in jobs.values()): + raise JobValidationError("invalid_job_repository", "A job references a missing repository") + for repo in store["repositories"]: + if {"used_by", "source_job_keys"}.intersection(repo): + raise JobValidationError("job_migration_required", "Repository assignments require the explicit identity migration") + expected = {job_id for job_id, job in jobs.items() if job["repository_key"] == repo["repository_key"]} + for field in ("job_ids", "source_job_ids"): + values = repo.get(field) + if not isinstance(values, list) or any(not isinstance(v, str) for v in values) or len(set(values)) != len(values) or set(values) != expected: + raise JobValidationError("conflicting_job_assignments", "Repository assignments do not match job metadata") + + +def write_transaction(changes, *, after_write=None, rollback_after=None): + """Durable replacements with byte rollback on ordinary failures. + + Caller holds inventory_lock. A crash between files remains detectable; + the guarded activation/recovery workflow in #479 owns crash recovery. + None removes one explicitly selected metadata file, never artifacts. + """ + snapshots = {Path(path): read_fingerprinted_file(Path(path))[1] for path in changes} + callback_started = False + try: + for path, payload in changes.items(): + if payload is None: + Path(path).unlink(missing_ok=True) + elif isinstance(payload, bytes): + atomic_write_bytes(Path(path), payload) + else: + atomic_write_json(Path(path), payload) + if after_write: + callback_started = True + return after_write() + except Exception as original: + failures = [] + for path, raw in snapshots.items(): + try: + if raw is None: + path.unlink(missing_ok=True) + else: + atomic_write_bytes(path, raw) + except Exception as exc: + failures.append(exc) + if callback_started and rollback_after: + try: + rollback_after() + except Exception as exc: + failures.append(exc) + if failures: + raise JobValidationError("job_transaction_recovery_required", "Inventory or cron rollback failed; recovery is required before further writes") from original + raise + + +def save_job_transaction(jobs_dir, repository_path, build, *, source_id=None, expected_revision=None, duplicate=False): + """Serialize read/validate/patch/write; roll back ordinary I/O failures. + + Each replacement is durable, but the pair is not crash-atomic. A crash + between replacements leaves inconsistent assignments which strict readers + reject, never reconcile silently. The global cutover gate is owned by #479. + """ + jobs_dir, repository_path = Path(jobs_dir), Path(repository_path) + with inventory_lock(repository_path.parent): + jobs = read_jobs(jobs_dir) + store = read_repositories(repository_path) + validate_assignments(jobs, store) + from schedule_api import validate_schedules + schedules = read_json(repository_path.parent / "schedules.json", missing={}) + validate_schedules(schedules, jobs) + existing = None + if source_id is not None: + validate_job_id(source_id) + existing = jobs.get(source_id) + if existing is None: + raise JobValidationError("unknown_job_id", "Unknown job_id; editing cannot create a job") + if expected_revision is not None and job_revision(existing) != expected_revision: + raise JobValidationError("job_edit_conflict", "The job changed since it was opened; reload before saving") + metadata = build(deepcopy(existing)) + validate_job(metadata) + job_id = metadata["job_id"] + fresh = source_id is None or duplicate + if fresh and job_id in jobs: + raise JobValidationError("duplicate_job_id", "Allocated job_id already exists") + if not fresh and source_id != job_id: + raise JobValidationError("immutable_job_id", "Editing cannot change job_id") + if metadata["legacy_job_keys"] != ([] if fresh else existing["legacy_job_keys"]): + raise JobValidationError("immutable_job_aliases", "Ordinary saves cannot add or change legacy aliases") + jobs[job_id] = metadata + validate_job_inventory(jobs) + next_store = deepcopy(store) + for repo in next_store["repositories"]: + # Only update the affected ID; retain ordering and unknown fields. + for field in ("job_ids", "source_job_ids"): + values = repo[field] + if repo["repository_key"] == metadata["repository_key"]: + if job_id not in values: + values.append(job_id) + elif job_id in values: + values.remove(job_id) + validate_assignments(jobs, next_store) + validate_schedules(schedules, jobs) + target = jobs_dir / (job_id + ".json") + changes = {target: metadata} + if next_store != store: + changes[repository_path] = next_store + write_transaction(changes) + return metadata, target diff --git a/api/job_transfer.py b/api/job_transfer.py new file mode 100644 index 00000000..49b34dce --- /dev/null +++ b/api/job_transfer.py @@ -0,0 +1,294 @@ +"""Explicit schema-v4 job transfer plans and atomic application (#447, #478). + +Source IDs select bundle objects; only an explicit target selects a live job. +Historical records belong to full configuration recovery, never a new job copy. +""" +from copy import deepcopy +import base64 +import hashlib +import json +from pathlib import Path +import re +from uuid import uuid4 + +from inventory_store import inventory_lock +from job_model import JobValidationError, validate_job, validate_job_inventory, validate_job_id +from job_store import read_jobs, read_json, read_repositories, validate_assignments, write_transaction +from repository_context import jobs_dir, resolve_job_repository_context +from schedule_api import validate_schedules, schedule_lines + +FORMAT = "bbui-job-bundle-v3" + + +def fail(code, message): + raise JobValidationError(code, message) + + +def decode_json(text): + def pairs(items): + out = {} + for key, value in items: + if key in out: + fail("invalid_transfer_bundle", "Duplicate JSON members are not supported") + out[key] = value + return out + try: + if len(text) > 64 * 1024 * 1024: + raise ValueError() + return json.loads(text, object_pairs_hook=pairs, + parse_constant=lambda _: fail("invalid_transfer_bundle", "Invalid JSON constant")) + except (ValueError, UnicodeError, RecursionError): + fail("invalid_transfer_bundle", "The transfer file is invalid or too large") + + +def indexed(rows, key): + if not isinstance(rows, list): + fail("invalid_transfer_bundle", "An inventory list is missing") + result = {} + for row in rows: + value = row.get(key) if isinstance(row, dict) else None + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+", value) or value in {".", ".."} or value in result: + fail("invalid_transfer_bundle", "Invalid or duplicate inventory identifier") + result[value] = deepcopy(row) + return result + + +def reference_map(bundle): + return {"schema_version": 1, "jobs": { + job["job_id"]: {"repository_key": job["repository_key"], + "scheduled": job["job_id"] in bundle["schedules"]} + for job in bundle["jobs"] + }, "repositories": {row["repository_key"]: {"storage_key": row["storage_key"], + "job_ids": sorted(row["job_ids"])} for row in bundle["repositories"]}} + + +def validate_bundle(bundle): + if not isinstance(bundle, dict) or bundle.get("format") != FORMAT: + fail("job_transfer_upgrade_required", "This operation requires a v3 job export from a migrated installation. Keep older backups for explicit configuration migration.") + allowed = {"format", "exported_at", "jobs", "repositories", "storages", "schedules", "references", "passphrase_meta", "keyfile_meta"} + if set(bundle) - allowed: + fail("unsupported_transfer_dependency", "The bundle includes unsupported dependent objects; nothing was written") + jobs = indexed(bundle.get("jobs"), "job_id") + validate_job_inventory(jobs) + repositories = indexed(bundle.get("repositories"), "repository_key") + storages = indexed(bundle.get("storages"), "storage_key") + validate_assignments(jobs, {"repositories": list(repositories.values())}) + validate_schedules(bundle.get("schedules"), jobs) + if "restore_test" in bundle["schedules"]: + fail("unsupported_transfer_dependency", "Service schedules require full configuration recovery") + required_repos = {row["repository_key"] for row in jobs.values()} + if set(repositories) != required_repos or {row.get("storage_key") for row in repositories.values()} != set(storages): + fail("invalid_transfer_references", "Bundle dependencies are missing or outside the selected jobs") + if any(row.get("schema_version", 1) != 1 for row in [*repositories.values(), *storages.values()]): + fail("invalid_transfer_bundle", "Unsupported inventory schema") + if bundle.get("references") != reference_map(bundle): + fail("invalid_transfer_references", "The job reference map is incomplete or inconsistent") + return jobs, repositories, storages + + +def export_bundle(config, selected_ids=None): + root = jobs_dir(config).parent + with inventory_lock(root): + jobs = read_jobs(root / "jobs") + repos = read_repositories(root / "repositories.json") + validate_assignments(jobs, repos) + schedules = validate_schedules(read_json(root / "schedules.json", missing={}), jobs) + selected = set(jobs if selected_ids is None else selected_ids) + if selected - jobs.keys(): + fail("unknown_job_id", "The selection includes an unknown job_id") + repositories = [deepcopy(row) for row in repos["repositories"] if any(jobs[j]["repository_key"] == row["repository_key"] for j in selected)] + for row in repositories: + for field in ("job_ids", "source_job_ids"): + row[field] = [job_id for job_id in row[field] if job_id in selected] + storage_keys = {row["storage_key"] for row in repositories} + storages = read_json(root / "storages.json", missing={"schema_version": 1, "storages": []}) + bundle = {"format": FORMAT, "jobs": [jobs[j] for j in sorted(selected)], "repositories": repositories, + "storages": [row for row in storages["storages"] if row["storage_key"] in storage_keys], + "schedules": {j: row for j, row in schedules.items() if j in selected}} + bundle["references"] = reference_map(bundle) + validate_bundle(bundle) + return bundle + + +def preview_bundle(config, bundle): + jobs, repos, storages = validate_bundle(bundle) + current = read_jobs(jobs_dir(config)) + rows = [] + for job_id, job in jobs.items(): + repo = repos[job["repository_key"]] + rows.append({"job_id": job_id, "name": job["name"][:200], "archive_prefix": job["archive_prefixes"][0], + "repository_key": job["repository_key"], "features": job.get("features", {}), + "schedule": bundle["schedules"].get(job_id, {}), "conflict": "exists" if job_id in current else "new", + "suggested_mode": "new", "repository": {"display_name": repo.get("display_name", repo["repository_key"]), + "repository_key": repo["repository_key"], "path": repo.get("relative_path", "")}, + "passphrase": {"status": "present" if bundle.get("passphrase_meta", {}).get(repo["repository_key"], {}).get("exists") else "missing"}}) + return {"format": FORMAT, "job_count": len(rows), "jobs": rows, + "current_jobs": [{"job_id": j, "name": job["name"][:200], "repository_key": job["repository_key"]} for j, job in current.items()]} + + +def _compatible(current, incoming, fields): + def value(row, key): + item = row.get(key) + return '' if item is None else item + return all(value(current, key) == value(incoming, key) for key in fields) + + +def plan_import(config, bundle, *, mode="new", selected_jobs=None, per_job_mode=None, + target_jobs=None, archive_prefixes=None, import_jobs=True, secret_payload=None): + source, source_repos, source_storages = validate_bundle(bundle) + root = jobs_dir(config).parent + jobs = read_jobs(root / "jobs") + repos_store = read_repositories(root / "repositories.json") + validate_assignments(jobs, repos_store) + storage_store = read_json(root / "storages.json", missing={"schema_version": 1, "storages": []}) + repos = indexed(repos_store["repositories"], "repository_key") + storages = indexed(storage_store["storages"], "storage_key") + schedules = validate_schedules(read_json(root / "schedules.json", missing={}), jobs) + old_lines = schedule_lines(config, schedules, jobs) + selected = set(source if selected_jobs is None else selected_jobs) + if selected - source.keys(): + fail("unknown_job_id", "Selection refers to jobs outside the bundle") + modes, targets, prefixes = per_job_mode or {}, target_jobs or {}, archive_prefixes or {} + for mapping in (modes, targets, prefixes): + if not isinstance(mapping, dict) or set(mapping) - selected: + fail("invalid_transfer_selection", "Import choices must reference selected source IDs") + if mode not in {"new", "merge", "skip"}: + fail("invalid_transfer_mode", "Choose new, merge with an explicit target, or skip") + result_jobs, changes, remap, report, selected_repos = deepcopy(jobs), {}, {}, [], set() + for source_id, raw in source.items(): + action = modes.get(source_id, mode) + if source_id not in selected or action == "skip": + report.append({"job_id": source_id, "status": "skipped_unselected" if source_id not in selected else "skipped"}) + continue + if action not in {"new", "merge"}: + fail("invalid_transfer_mode", "Choose new or an explicitly selected merge target") + if action == "new": + if source_id in targets or not import_jobs: + fail("explicit_import_target_required", "Secret-only imports require an explicitly selected existing target job") + target_id = str(uuid4()) + if target_id in jobs: + fail("duplicate_job_id", "Allocated job_id already exists") + patched = deepcopy(raw) + patched["legacy_job_keys"] = [] + patched.pop("cache_reference", None) + else: + target_id = validate_job_id(targets.get(source_id)) + if target_id not in jobs: + fail("unknown_job_id", "The selected merge target no longer exists") + patched = {**deepcopy(jobs[target_id]), **deepcopy(raw)} + patched["legacy_job_keys"] = deepcopy(jobs[target_id]["legacy_job_keys"]) + patched.pop("cache_reference", None) + if "cache_reference" in jobs[target_id]: + patched["cache_reference"] = deepcopy(jobs[target_id]["cache_reference"]) + if not import_jobs and raw["repository_key"] != jobs[target_id]["repository_key"]: + fail("invalid_transfer_target", "Secret-only import must target a job using the same repository") + if target_id in remap.values(): + fail("duplicate_transfer_target", "Each source job needs a distinct target") + remap[source_id] = target_id + patched["job_id"] = target_id + if source_id in prefixes: + patched["archive_prefixes"] = [prefixes[source_id]] + if action == "merge": + patched["archive_prefixes"] = list(dict.fromkeys([*patched["archive_prefixes"], *jobs[target_id]["archive_prefixes"]])) + validate_job(patched) + if import_jobs: + result_jobs[target_id] = patched + changes[root / "jobs" / (target_id + ".json")] = patched + if source_id in bundle["schedules"]: + schedules[target_id] = deepcopy(bundle["schedules"][source_id]) + selected_repos.add(raw["repository_key"]) + report.append({"job_id": source_id, "target_job_id": target_id, "status": action if import_jobs else "secrets", "name": patched["name"]}) + for key in selected_repos: + incoming = source_repos[key] + storage_key = incoming["storage_key"] + storage = source_storages[storage_key] + if storage_key in storages and not _compatible(storages[storage_key], storage, ( + "storage_type", "location", "identity", "base_path", "mount_path", "host", "user", "port", "server", "share")): + fail("transfer_storage_collision", "A storage identifier refers to a different target") + storages.setdefault(storage_key, deepcopy(storage)) + if key in repos: + if not _compatible(repos[key], incoming, ("storage_key", "relative_path", "encryption", "borg_repository_id")): + fail("transfer_repository_collision", "A repository identifier refers to a different repository") + else: + repo = deepcopy(incoming) + repo.pop("keyfile_ref", None) + # Imported host paths never select write destinations. + if repo.get("passphrase_ref"): + repo["passphrase_ref"] = str(root.parent / "secrets" / (".borg-passphrase-" + key)) + repos[key] = repo + validate_job_inventory(result_jobs) + for repo in repos.values(): + expected = [j for j, raw in result_jobs.items() if raw["repository_key"] == repo["repository_key"]] + repo["job_ids"] = expected + repo["source_job_ids"] = expected[:] + next_repos = {**repos_store, "repositories": list(repos.values())} + validate_assignments(result_jobs, next_repos) + inventory = {"repositories": repos, "storages": storages} + for target_id in remap.values(): + resolve_job_repository_context(config, target_id, job=result_jobs[target_id], inventory=inventory, require_passphrase_file=False) + secret_counts = plan_secrets(config, secret_payload, selected_repos, repos, changes) if secret_payload is not None else {} + if remap: + changes[root / "repositories.json"] = next_repos + changes[root / "storages.json"] = {**storage_store, "storages": list(storages.values())} + changes[root / "schedules.json"] = schedules + return {"changes": changes, "old_lines": old_lines, "new_lines": schedule_lines(config, schedules, result_jobs), + "result": {"report": report, "id_map": remap, "imported_count": len(remap) if import_jobs else 0, + "scheduled_count": sum(j in bundle["schedules"] for j in remap) if import_jobs else 0, + "repository_inventory": {"repositories": len(selected_repos), "storages": len({repos[r]['storage_key'] for r in selected_repos})}, + **secret_counts}} + + +def secret_bytes(row): + if not isinstance(row, dict): + fail("invalid_transfer_secret", "Malformed protected file") + try: + content = base64.b64decode(row["content_b64"], validate=True) + if not content or len(content) > 1024 * 1024 or hashlib.sha256(content).hexdigest() != row["sha256"]: + raise ValueError() + return content + except (ValueError, KeyError, TypeError): + fail("invalid_transfer_secret", "Protected file content or digest is invalid") + + +def plan_secrets(config, payload, selected_repos, repositories, changes): + from borg_key_store import borg_keys_dir, find_key_file + from migrations.identity_storage import read_fingerprinted_file + counts = {"restored_passphrases": 0, "restored_keyfiles": 0} + for collection, count in (("passphrase_files", "restored_passphrases"), ("key_files", "restored_keyfiles")): + files = payload.get(collection, {}) + if not isinstance(files, dict): + fail("invalid_transfer_secret", "Invalid protected file collection") + for key, row in files.items(): + if collection == "key_files" and isinstance(row, dict) and row.get("exists") is False: + continue + content = secret_bytes(row) # Validate the complete included payload before any writes. + if key not in selected_repos: + continue + repo = repositories[key] + if collection == "passphrase_files": + target = Path(repo.get("passphrase_ref") or jobs_dir(config).parent.parent / "secrets" / (".borg-passphrase-" + key)) + repo["passphrase_ref"] = str(target) + else: + repository_id = repo.get("borg_repository_id", "") + if not re.fullmatch(r"[0-9a-f]{64}", repository_id) or row.get("repository_id") != repository_id or content.splitlines()[0] != ("BORG_KEY " + repository_id).encode(): + fail("invalid_transfer_secret", "Borg key repository identity does not match") + target = find_key_file(borg_keys_dir(config), repository_id) or borg_keys_dir(config) / ("bbui-" + repository_id) + repo["keyfile_ref"] = str(target) + old = read_fingerprinted_file(target)[1] + if old is not None and old != content: + fail("transfer_secret_collision", "A protected file differs from the existing file; restore it through the explicit secrets workflow") + if target in changes and changes[target] != content: + fail("transfer_secret_collision", "Two repositories refer to conflicting protected files") + changes[target] = content + counts[count] += 1 + return counts + + +def apply_import(config, bundle, *, dry_run=True, **choices): + from schedule_api import _update_crontab + with inventory_lock(jobs_dir(config).parent): + plan = plan_import(config, bundle, **choices) + if not dry_run and plan["changes"]: + write_transaction(plan["changes"], after_write=lambda: _update_crontab(plan["new_lines"]), + rollback_after=lambda: _update_crontab(plan["old_lines"])) + return {**plan["result"], "dry_run": bool(dry_run)} diff --git a/api/jobs_api.py b/api/jobs_api.py index 4681e320..39eb864f 100644 --- a/api/jobs_api.py +++ b/api/jobs_api.py @@ -7,10 +7,7 @@ import json import io -import copy import os -import re -import shutil import subprocess import sys import threading @@ -23,20 +20,16 @@ DEFAULT_DATA_ROOT = Path("/boot/config/borg-backup") SSE_HEARTBEAT_INTERVAL_SECONDS = 15.0 -_JOB_KEY_RX = re.compile(r"^[a-zA-Z0-9_.-]+$") _RUNTIME_MODES = {"all", "selected", "none"} _DOCKER_RUNTIME_MODES = _RUNTIME_MODES | {"except_selected"} -_JOB_DISCOVERY_CACHE_TTL_SECONDS = 5.0 -_job_discovery_cache: dict[str, dict] = {} -_job_discovery_cache_lock = threading.Lock() -_job_metadata_migrations: set[str] = set() -def _validate_job_key(job_key: str) -> str: - key = str(job_key or "").strip() - if not _JOB_KEY_RX.fullmatch(key): - raise ValueError("Invalid job key") - return key +def _validate_runtime_id(job_id: str) -> str: + # Restore verification remains a separate service during its #477 cutover. + if job_id == "restore_test": + return job_id + from job_model import validate_job_id + return validate_job_id(job_id) def _control_state_for_run(run_id: str) -> dict: @@ -61,9 +54,9 @@ def _control_state_for_run(run_id: str) -> dict: } -def cancel_job(config: dict, job_key: str, run_id: str, requested_by: str = "") -> dict: +def cancel_job(config: dict, job_id: str, run_id: str, requested_by: str = "") -> dict: """Request cooperative cancellation for the currently active run.""" - key = _validate_job_key(job_key) + key = _validate_runtime_id(job_id) runtime = get_job_runtime_state(config, key) active_run_id = str(runtime.get("run_id") or "").strip() if not runtime.get("running") or not active_run_id: @@ -167,16 +160,23 @@ def active_resource_locks(config: dict) -> List[dict]: continue if not isinstance(raw, dict): continue - job_key = str(raw.get("job_key") or "").strip() + job_id = str(raw.get("job_id") or "").strip() try: - job_key = _validate_job_key(job_key) + if not job_id and raw.get("service") in {"restore", "restore_test"} and raw.get("operation") == raw["service"]: + pass + else: + job_id = _validate_runtime_id(job_id) except ValueError: continue pid = _safe_int(raw.get("pid"), 0) if not _pid_alive(pid): continue + from activity_log_capture import process_token + if raw.get("process_start") and process_token(pid) != raw["process_start"]: + continue rows.append({ - "job_key": job_key, + **{key: raw[key] for key in ("job_name_snapshot", "archive_prefix_snapshot", "repository_key_snapshot", "repository_snapshot", "location_snapshot") if key in raw}, + "job_id": job_id, "pid": pid, "resource": str(raw.get("resource") or "").strip(), "operation": str(raw.get("operation") or "backup").strip().lower() or "backup", @@ -205,40 +205,16 @@ def _runtime_log_dir(config: dict) -> Path: return Path(configured or "/mnt/user/Logs") -def _fallback_runtime_log(config: dict, job_key: str, started_at: str) -> str: - log_dir = _runtime_log_dir(config) - if not log_dir.is_dir(): - return "" - try: - candidates = sorted( - log_dir.glob(f"Borg-Backup_{job_key}--*.log"), - key=lambda path: path.stat().st_mtime, - reverse=True, - ) - except OSError: - return "" - if not candidates: - return "" - if started_at: - try: - started = datetime.fromisoformat(started_at.replace("Z", "+00:00")) - if started.tzinfo is None: - started = started.replace(tzinfo=timezone.utc) - if candidates[0].stat().st_mtime < started.timestamp() - 120: - return "" - except (OSError, ValueError): - pass - return str(candidates[0]) - - def durable_running_states(config: dict) -> Dict[str, dict]: """Aggregate live runner locks into one durable state per job.""" grouped: Dict[str, dict] = {} for lock in active_resource_locks(config): if str(lock.get("operation") or "backup").strip().lower() != "backup": continue - job_key = str(lock.get("job_key") or "") - current = grouped.setdefault(job_key, { + job_id = str(lock.get("job_id") or "") + current = grouped.setdefault(job_id, { + **{key: value for key, value in lock.items() if key.endswith("_snapshot")}, + "job_id": job_id, "running": True, "exit_code": None, "start_time": str(lock.get("started_at") or ""), @@ -257,17 +233,15 @@ def durable_running_states(config: dict) -> Dict[str, dict]: current["run_id"] = str(lock.get("run_id")) from activity_log_capture import running_captures for capture in running_captures(): - grouped.setdefault(capture['job_key'], capture) - for job_key, state in grouped.items(): - if not state.get("log_file"): - state["log_file"] = _fallback_runtime_log(config, job_key, str(state.get("start_time") or "")) + grouped.setdefault(capture['job_id'], capture) + for job_id, state in grouped.items(): state["log_available"] = bool(state.get("log_file") and Path(str(state["log_file"])).is_file()) state.update(_control_state_for_run(str(state.get("run_id") or ""))) return grouped -def get_job_runtime_state(config: dict, job_key: str) -> dict: - key = _validate_job_key(job_key) +def get_job_runtime_state(config: dict, job_id: str) -> dict: + key = _validate_runtime_id(job_id) memory = JobManager.get().get_state(key) if memory.get("running"): return memory @@ -276,9 +250,9 @@ def get_job_runtime_state(config: dict, job_key: str) -> dict: def get_all_runtime_states(config: dict) -> Dict[str, dict]: states = JobManager.get().get_all_states() - for job_key, durable in durable_running_states(config).items(): - if not states.get(job_key, {}).get("running"): - states[job_key] = durable + for job_id, durable in durable_running_states(config).items(): + if not states.get(job_id, {}).get("running"): + states[job_id] = durable return states @@ -311,37 +285,13 @@ def get_jobs_meta_dirs(scripts_dir: Path, data_root: Path | None = None) -> List return [get_jobs_meta_dir(scripts_dir, data_root)] -def migrate_jobs_metadata_dir(scripts_dir: Path, data_root: Path | None = None) -> None: - """One-time migration: move legacy jobs/*.json into canonical config/jobs/.""" - preferred = get_jobs_meta_dir(scripts_dir, data_root) - preferred.mkdir(parents=True, exist_ok=True) - sources = [ - scripts_dir / "config" / "jobs", - Path("/boot/config/plugins/borg-backup-ui/runtime/config/jobs"), - ] - for legacy in sources: - if legacy == preferred or not legacy.is_dir(): - continue - for src in legacy.glob("*.json"): - dst = preferred / src.name - if dst.exists(): - continue - try: - src.rename(dst) - except OSError: - try: - shutil.copy2(src, dst) - src.unlink() - except OSError: - continue - - @dataclass class JobInfo: - key: str - backup_type: str + job_id: str + repository_key: str + archive_prefixes: list[str] location: str - script_path: Optional[Path] + script_path: Optional[Path] = None name: str = "" has_docker: bool = False has_vm: bool = False @@ -367,10 +317,7 @@ class JobInfo: @property def display_name(self) -> str: - loc_label = {"local": "Lokal", "usb": "USB", "smb": "SMB", "storagebox": "Storagebox"}.get( - self.location, self.location - ) - return f"{self.backup_type.capitalize()} – {loc_label}" + return self.name class _JobState: @@ -380,6 +327,7 @@ def __init__(self, proc: subprocess.Popen, start_time: datetime, run_id: str, lo self.run_id = run_id self.log_file = log_file self.capture_record_file = capture_record_file + self.run_snapshot = {} self.line_count = 0 self.lines: List[str] = [] self.finished = False @@ -421,28 +369,42 @@ def get(cls) -> "JobManager": def start( self, - job_key: str, + job_id: str, command: List[str], cwd: Path, extra_env: Optional[Dict[str, str]] = None, + *, run_context: dict | None = None, ) -> tuple: + with self._lock: + return self._start_locked(job_id, command, cwd, extra_env, run_context=run_context) + + def _start_locked(self, job_id, command, cwd, extra_env, *, run_context): """ Startet einen Backup-Job als Subprozess. Gibt (True, None) bei Erfolg zurück, (False, Fehlermeldung) sonst. """ - job_key = _validate_job_key(job_key) - with self._lock: - state = self._states.get(job_key) - if state is not None and not state.finished: - return False, "Job is already running" + job_id = _validate_runtime_id(job_id) + state = self._states.get(job_id) + if state is not None and not state.finished: + return False, "Job is already running" + if job_id != "restore_test": + from job_runs import read_run_context + if not run_context: + raise ValueError("An immutable run context is required") + run_context = read_run_context(job_id, run_context["run_id"]) env = dict(os.environ) # Damit das Script seine lib/ findet env["BORG_SCRIPT_DIR"] = str(cwd) - run_id = f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:12]}" - env["BORG_UI_RUN_ID"] = run_id if extra_env: env.update(extra_env) + run_id = run_context["run_id"] if run_context else str(uuid.uuid4()) + env["BORG_UI_RUN_ID"] = run_id + if run_context: + env["BORG_UI_JOB_ID"] = job_id + env.pop("BORG_UI_JOB_KEY", None) + env["BORG_UI_FILE_ACTIVITY_RUN"] = "1" if run_context["context"]["job"].get("file_activity") else "0" + env["BORG_UI_ACTIVITY_LOG_DIR"] = str(Path(run_context["log_file"]).parent) log_file = None capture_record_file = None @@ -452,10 +414,10 @@ def start( from activity_log import activity_log_path from activity_log_capture import prepare_capture - log_file, capture_record_file = prepare_capture(job_key, run_id, Path(env["BORG_UI_ACTIVITY_LOG_DIR"])) + log_file, capture_record_file = prepare_capture(job_id, run_id, Path(env["BORG_UI_ACTIVITY_LOG_DIR"]), name=run_context["job_name_snapshot"]) log_handle = os.fdopen(os.open(log_file, os.O_WRONLY | os.O_NOFOLLOW), "wb") env["BORG_UI_CAPTURE_LOG"] = str(log_file) - env["BORG_UI_RETAINED_LOG"] = str(activity_log_path(Path(env["BORG_UI_ACTIVITY_LOG_DIR"]), job_key, run_id)) + env["BORG_UI_RETAINED_LOG"] = run_context["log_file"] command = [sys.executable, str(Path(__file__).with_name("activity_log_capture.py")), str(capture_record_file), *command] env["PYTHONUNBUFFERED"] = "1" env["PYTHONIOENCODING"] = "utf-8" @@ -475,19 +437,21 @@ def start( log_handle.close() new_state = _JobState(proc, datetime.now(), run_id, log_file, capture_record_file) - with self._lock: - self._states[job_key] = new_state + if run_context: + from job_runs import descriptors + new_state.run_snapshot = {**descriptors(run_context), "log_file": run_context["log_file"], "file_activity": False} + self._states[job_id] = new_state t = threading.Thread( target=self._reader, - args=(job_key, new_state), + args=(job_id, new_state), daemon=True, - name=f"job-reader-{job_key}", + name=f"job-reader-{job_id}", ) t.start() return True, None - def _reader(self, job_key: str, state: _JobState) -> None: + def _reader(self, job_id: str, state: _JobState) -> None: """Liest stdout des Subprozesses Zeile für Zeile in den Puffer.""" try: if state.log_file is None: @@ -527,15 +491,16 @@ def _reader(self, job_key: str, state: _JobState) -> None: # ── State-Abfrage ───────────────────────────────────────────────────────── - def get_state(self, job_key: str) -> dict: - job_key = _validate_job_key(job_key) + def get_state(self, job_id: str) -> dict: + job_id = _validate_runtime_id(job_id) with self._lock: - state = self._states.get(job_key) + state = self._states.get(job_id) if state is None: return {"running": False} if state.log_file is not None: with state._lock: return { + **state.run_snapshot, "running": not state.finished, "exit_code": state.exit_code, "line_count": state.line_count, @@ -548,6 +513,7 @@ def get_state(self, job_key: str) -> dict: } lines, finished, exit_code = state.snapshot() return { + **state.run_snapshot, "running": not finished, "exit_code": exit_code, "start_time": state.start_time.isoformat(), @@ -561,24 +527,24 @@ def get_all_states(self) -> dict: keys = list(self._states.keys()) return {k: self.get_state(k) for k in keys} - def is_running(self, job_key: str) -> bool: - job_key = _validate_job_key(job_key) + def is_running(self, job_id: str) -> bool: + job_id = _validate_runtime_id(job_id) with self._lock: - state = self._states.get(job_key) + state = self._states.get(job_id) return state is not None and not state.finished # ── SSE-Stream ──────────────────────────────────────────────────────────── - def stream_output(self, job_key: str) -> Generator[str, None, None]: + def stream_output(self, job_id: str, run_id: str = "") -> Generator[str, None, None]: """ SSE-Generator: liefert neue Log-Zeilen als 'data:' Events. Schließt mit einem 'done'-Event (Daten = Exit-Code). Bricht sofort ab wenn Job unbekannt ist. """ - job_key = _validate_job_key(job_key) + job_id = _validate_runtime_id(job_id) with self._lock: - state = self._states.get(job_key) - if state is None: + state = self._states.get(job_id) + if state is None or (run_id and state.run_id != run_id): yield "event: error\ndata: Job not found\n\n" return if state.log_file is not None: @@ -623,304 +589,114 @@ def stream_output(self, job_key: str) -> Generator[str, None, None]: time.sleep(0.1) -def _latest_job_exit_code(config: dict, job_key: str) -> Optional[int]: - try: - from status_api import get_status_data - for row in get_status_data(config).get("backups", []): - if str(row.get("key") or "") == job_key: - value = row.get("exit_code") - return int(value) if value is not None else None - except Exception: - pass - return None - - -def stream_job_output(config: dict, job_key: str) -> Generator[str, None, None]: - """Stream in-memory output or resume a live runner log discovered via locks.""" - key = _validate_job_key(job_key) +def stream_job_output(config: dict, job_id: str, run_id: str = "") -> Generator[str, None, None]: + """Stream one immutable run; a newer run never changes this stream's owner.""" + key = _validate_runtime_id(job_id) manager = JobManager.get() - if manager.is_running(key): + if key == "restore_test": yield from manager.stream_output(key) return - - durable = durable_running_states(config).get(key) - if not durable: - yield from manager.stream_output(key) + from job_runs import validate_run_id, find_run_status + from job_control import read_control_state + from activity_log import open_activity_file + validate_run_id(run_id) + memory = manager.get_state(key) + if memory.get("run_id") == run_id: + yield from manager.stream_output(key, run_id) return - log_file = Path(str(durable.get("log_file") or "")) - if not log_file.is_file(): - yield "event: error\ndata: Live log is not available for the recovered job run.\n\n" + durable = durable_running_states(config).get(key, {}) + state = durable if durable.get("run_id") == run_id else find_run_status(config, key, run_id) + if not state.get("log_file"): + yield "event: error\ndata: Log is not available for this run.\n\n" + return + from activity_log_capture import capture_record, open_capture_file + capture = capture_record(key, run_id) if state.get("file_activity") else {} + try: + binary = (open_capture_file(Path(capture["active_file"]).parent / "capture.json") + if capture else open_activity_file(Path(state["log_file"]))) + except OSError: + yield "event: error\ndata: Log could not be read.\n\n" return - yield ": heartbeat\n\n" - position = 0 - idle_after_finish = 0 last_heartbeat = time.monotonic() - while True: - emitted = False - try: - if durable.get("file_activity"): - from activity_log_capture import capture_record, open_capture_file - capture = capture_record(key, str(durable.get("run_id") or "")) - else: - capture = {} - if capture: - binary = open_capture_file(Path(capture["active_file"]).parent / "capture.json") - else: - binary = log_file.open("rb") - with io.TextIOWrapper(binary, encoding="utf-8", errors="replace") as handle: - handle.seek(position) - for line in handle: - emitted = True - clean_line = line.rstrip("\r\n") - yield f"data: {clean_line}\n\n" - position = handle.tell() - if emitted: - last_heartbeat = time.monotonic() - except OSError: - yield "event: error\ndata: Live log could not be read.\n\n" - return - - if key not in durable_running_states(config): - idle_after_finish = 0 if emitted else idle_after_finish + 1 - if idle_after_finish >= 2: - exit_code = _latest_job_exit_code(config, key) - value = str(exit_code) if exit_code is not None else "?" - yield f"event: done\ndata: {value}\n\n" - return - else: - idle_after_finish = 0 - if not emitted and time.monotonic() - last_heartbeat >= SSE_HEARTBEAT_INTERVAL_SECONDS: - yield ": heartbeat\n\n" - last_heartbeat = time.monotonic() - time.sleep(0.5) + idle_after_finish = 0 + with io.TextIOWrapper(binary, encoding="utf-8", errors="replace") as handle: + while True: + line = handle.readline() + if line: + yield f"data: {line.rstrip(chr(10))}\n\n" + idle_after_finish = 0 + continue + current = durable_running_states(config).get(key, {}) + running = current.get("running") and current.get("run_id") == run_id + if not running: + idle_after_finish += 1 + if idle_after_finish >= 2: + control = read_control_state(run_id) + terminal = control if control.get("job_id") == key else find_run_status(config, key, run_id) + code = terminal.get("exit_code") + yield f"event: done\ndata: {code if code is not None else '?'}\n\n" + return + if time.monotonic() - last_heartbeat >= SSE_HEARTBEAT_INTERVAL_SECONDS: + yield ": heartbeat\n\n" + last_heartbeat = time.monotonic() + time.sleep(0.5) # ── Job-Erkennung ───────────────────────────────────────────────────────────── def _discover_jobs_uncached(scripts_dir: Path, data_root: Path | None = None) -> List[JobInfo]: - """ - Finds backup jobs from canonical JSON metadata. - """ - utility_types = {"restore_test"} - - def _make_job( - py_file: Optional[Path], - backup_type: str, - location: str, - *, - key: Optional[str] = None, - name: Optional[str] = None, - has_docker: Optional[bool] = None, - has_vm: Optional[bool] = None, - description: Optional[str] = None, - icon: Optional[str] = None, - icon_color: Optional[str] = None, - standard: str = "wizard", - enabled: bool = True, - compression: str = "", - retention_daily: str = "", - retention_weekly: str = "", - retention_monthly: str = "", - retention_yearly: str = "", - restore_test_policy_mode: str = "", - restore_test_interval_days: int = 30, - restore_test_validity_days: int = 30, - restore_test_level: int = 2, - restore_test_max_runtime_minutes: int = 0, - docker_control: Optional[dict] = None, - vm_control: Optional[dict] = None, - file_activity: bool = False, - ) -> JobInfo: - desc_file = py_file.with_suffix(".description") if py_file is not None else None - desc_text = ( - description - if description is not None - else ( - desc_file.read_text(encoding="utf-8").strip() - if desc_file is not None and desc_file.exists() - else "" - ) - ) - bt_lc = backup_type.lower() - default_docker_control = { - "mode": "all" if ((bt_lc == "appdata") if has_docker is None else bool(has_docker)) else "none", - "selected": [], - "ack_appdata_risk": False, - } - default_vm_control = { - "mode": "all" if ((bt_lc == "vms") if has_vm is None else bool(has_vm)) else "none", - "selected": [], - "ack_domains_risk": False, - } - return JobInfo( - key=key or f"{bt_lc}_{location}", - backup_type=backup_type, - location=location, - script_path=py_file, - name=(name or "").strip(), - has_docker=(bt_lc == "appdata") if has_docker is None else bool(has_docker), - has_vm=(bt_lc == "vms") if has_vm is None else bool(has_vm), - description=desc_text, - icon=(icon or "").strip().lower(), - icon_color=(icon_color or "").strip().lower(), - # Only explicit utility jobs should be filtered from normal - # backup selectors. Custom/unknown backup types are still jobs. - is_utility=bt_lc in utility_types, - standard=standard, - enabled=bool(enabled), - file_activity=file_activity, - compression=str(compression or "").strip(), - retention_daily=str(retention_daily or "").strip(), - retention_weekly=str(retention_weekly or "").strip(), - retention_monthly=str(retention_monthly or "").strip(), - retention_yearly=str(retention_yearly or "").strip(), - docker_control=docker_control or default_docker_control, - vm_control=vm_control or default_vm_control, - restore_test_policy_mode=str(restore_test_policy_mode or "").strip().lower(), - restore_test_interval_days=_safe_int(restore_test_interval_days, 30), - restore_test_validity_days=_safe_int(restore_test_validity_days, 30), - restore_test_level=_safe_int(restore_test_level, 2), - restore_test_max_runtime_minutes=_safe_int(restore_test_max_runtime_minutes, 0), - ) - - jobs_by_key: Dict[str, JobInfo] = {} + from job_store import read_jobs + from repository_context import load_repository_inventory, resolve_job_repository_context root = data_root if data_root is not None else (scripts_dir.parent if scripts_dir.name == "scripts" else scripts_dir) - # ── Wizard-Metadaten (prioritär) ────────────────────────────────────────── - meta_dirs = get_jobs_meta_dirs(scripts_dir, root) - for meta_dir in meta_dirs: - if not meta_dir.is_dir(): - continue - for meta_file in sorted(meta_dir.glob("*.json")): - try: - raw = json.loads(meta_file.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, UnicodeDecodeError): - continue - - # Pflichtfelder V1 - try: - key = str(raw["job_key"]).strip() - backup_type = str(raw["backup_type"]).strip() - location = str(raw["location"]).strip().lower() - script_name = str(raw.get("script") or "").strip() - except (KeyError, TypeError, ValueError): - continue - - if not key or not backup_type or not location: - continue - if location not in {"local", "usb", "smb", "storagebox", "custom"}: - continue - - script_path = (scripts_dir / script_name).resolve() - if script_name: - try: - script_path.relative_to(scripts_dir.resolve()) - except ValueError: - # Pfad außerhalb scripts_dir ignorieren - continue - else: - script_path = None - - features = raw.get("features") if isinstance(raw.get("features"), dict) else {} - retention = raw.get("retention") if isinstance(raw.get("retention"), dict) else {} - rt_policy = raw.get("restore_test_policy") if isinstance(raw.get("restore_test_policy"), dict) else {} - docker_control = _runtime_control_from_meta(raw, "docker") - vm_control = _runtime_control_from_meta(raw, "vm") - has_docker = bool(features.get("docker", False)) - has_vm = bool(features.get("vm", False)) - description = raw.get("description") - if description is not None: - description = str(description) - - # Preferred dir wins: only set if job key not seen yet. - jobs_by_key.setdefault(key, _make_job( - script_path, - backup_type, - location, - key=key, - name=str(raw.get("name") or "").strip(), - has_docker=has_docker, - has_vm=has_vm, - description=description, - icon=str(raw.get("icon") or "").strip().lower(), - icon_color=str(raw.get("icon_color") or "").strip().lower(), - standard="wizard", - enabled=bool(raw.get("enabled", True)), - file_activity=str(raw.get("file_activity", False)).strip().lower() in {"1", "true", "yes", "on"}, - compression=str(raw.get("compression") or "").strip(), - retention_daily=str(retention.get("daily") or "").strip(), - retention_weekly=str(retention.get("weekly") or "").strip(), - retention_monthly=str(retention.get("monthly") or "").strip(), - retention_yearly=str(retention.get("yearly") or "").strip(), - docker_control=docker_control, - vm_control=vm_control, - restore_test_policy_mode=str(rt_policy.get("mode") or "").strip().lower(), - restore_test_interval_days=_safe_int(rt_policy.get("interval_days"), 30), - restore_test_validity_days=_safe_int(rt_policy.get("validity_days") or rt_policy.get("interval_days"), 30), - restore_test_level=_safe_int(rt_policy.get("level"), 2), - restore_test_max_runtime_minutes=_safe_int(rt_policy.get("max_runtime_minutes"), 0), - )) - - return list(jobs_by_key.values()) - - -def _job_metadata_signature(meta_dir: Path, scripts_dir: Path, *, include_files: bool) -> tuple: - def _stat_signature(path: Path) -> tuple[int, int, int]: - try: - stat = path.stat() - return (int(stat.st_mtime_ns), int(stat.st_size), int(stat.st_ino)) - except OSError: - return (0, 0, 0) - - signature: list = [_stat_signature(meta_dir), _stat_signature(scripts_dir)] - if include_files and meta_dir.is_dir(): - signature.append(tuple( - (path.name, *_stat_signature(path)) - for path in sorted(meta_dir.glob("*.json")) + config = {"BACKUP_SCRIPTS_DIR": str(root)} + inventory = load_repository_inventory(config) + jobs = [] + for job_id, raw in read_jobs(get_jobs_meta_dir(scripts_dir, root)).items(): + context = resolve_job_repository_context(config, job_id, job=raw, + require_passphrase_file=False, inventory=inventory) + retention = raw.get("retention", {}) + policy = raw.get("restore_test_policy", {}) + docker = _runtime_control_from_meta(raw, "docker") + vm = _runtime_control_from_meta(raw, "vm") + jobs.append(JobInfo( + job_id=job_id, repository_key=raw["repository_key"], archive_prefixes=list(raw["archive_prefixes"]), + location=context["location"], name=raw["name"], description=raw.get("description", ""), + icon=raw.get("icon", ""), icon_color=raw.get("icon_color", ""), + enabled=raw.get("enabled", True), standard=raw.get("standard", "wizard"), + file_activity=raw.get("file_activity", False), compression=raw.get("compression", ""), + has_docker=docker["mode"] != "none", has_vm=vm["mode"] != "none", + docker_control=docker, vm_control=vm, + **{"retention_" + key: retention.get(key, "") for key in ("daily", "weekly", "monthly", "yearly")}, + restore_test_policy_mode=policy.get("mode", ""), + restore_test_interval_days=_safe_int(policy.get("interval_days"), 30), + restore_test_validity_days=_safe_int(policy.get("validity_days"), 30), + restore_test_level=_safe_int(policy.get("level"), 2), + restore_test_max_runtime_minutes=_safe_int(policy.get("max_runtime_minutes"), 0), )) - return tuple(signature) - - -def invalidate_job_discovery_cache() -> None: - """Invalidate cached static job metadata after an explicit metadata change.""" - with _job_discovery_cache_lock: - _job_discovery_cache.clear() + return jobs def discover_jobs(scripts_dir: Path, data_root: Path | None = None) -> List[JobInfo]: - """Read static job metadata once while keeping external changes observable.""" + """Read the validated canonical inventory; never migrate on discovery.""" + from inventory_store import inventory_lock root = data_root if data_root is not None else (scripts_dir.parent if scripts_dir.name == "scripts" else scripts_dir) - meta_dir = get_jobs_meta_dir(scripts_dir, root) - cache_key = f"{scripts_dir.resolve()}::{root.resolve()}" - with _job_discovery_cache_lock: - if cache_key not in _job_metadata_migrations: - migrate_jobs_metadata_dir(scripts_dir, root) - _job_metadata_migrations.add(cache_key) - - now = time.monotonic() - quick_signature = _job_metadata_signature(meta_dir, scripts_dir, include_files=False) - with _job_discovery_cache_lock: - cached = _job_discovery_cache.get(cache_key) - if cached and cached["quick_signature"] == quick_signature and now < cached["expires_at"]: - return copy.deepcopy(cached["jobs"]) - - full_signature = _job_metadata_signature(meta_dir, scripts_dir, include_files=True) - with _job_discovery_cache_lock: - cached = _job_discovery_cache.get(cache_key) - if cached and cached["full_signature"] == full_signature: - cached["quick_signature"] = quick_signature - cached["expires_at"] = now + _JOB_DISCOVERY_CACHE_TTL_SECONDS - return copy.deepcopy(cached["jobs"]) - - jobs = _discover_jobs_uncached(scripts_dir, root) - with _job_discovery_cache_lock: - _job_discovery_cache[cache_key] = { - "quick_signature": quick_signature, - "full_signature": full_signature, - "expires_at": now + _JOB_DISCOVERY_CACHE_TTL_SECONDS, - "jobs": copy.deepcopy(jobs), - } - return jobs + with inventory_lock(root / "config"): + return _discover_jobs_uncached(scripts_dir, root) + + +def latest_job_statuses(config: dict) -> dict: + """Job controls read status directly, without reporting/snapshot side effects.""" + from wizard_runner import _ensure_runtime_import_paths + _ensure_runtime_import_paths(resolve_data_root(config)) + from lib.status import StatusStore, time_ago + store = StatusStore(Path(config.get("STATUS_DIR") or "/mnt/user/backup-status")) + return {job_id: { + "job_id": job_id, "run_id": row.run_id, "status": row.status, + "timestamp": row.timestamp, "time_ago": time_ago(row.timestamp), + "exit_code": row.exit_code, "file_activity": row.file_activity, + "job_name_snapshot": row.job_name_snapshot, "log_file": row.log_file, + } for job_id, row in store.get_latest_per_key(store.load()).items()} def list_jobs(config: dict, latest_statuses: dict) -> List[dict]: @@ -931,38 +707,29 @@ def list_jobs(config: dict, latest_statuses: dict) -> List[dict]: scripts_dir = resolve_scripts_dir(config) data_root = resolve_data_root(config) runtime_states = get_all_runtime_states(config) - try: - from repository_context import load_repository_inventory - repository_inventory = load_repository_inventory(config) - except Exception: - repository_inventory = {"repositories": {}, "storages": {}} + from repository_context import load_repository_inventory, resolve_job_repository_context + repository_inventory = load_repository_inventory(config) result = [] for info in discover_jobs(scripts_dir, data_root): - last = latest_statuses.get(info.key) - run_state = runtime_states.get(info.key, {"running": False}) - try: - from repository_context import resolve_job_repository_context - repository_context = resolve_job_repository_context( - config, - info.key, - require_passphrase_file=False, - inventory=repository_inventory, - ) - repo_path = str(repository_context.get("repository_path") or "") - repository_key = str(repository_context.get("repository_key") or "") - repository = repository_context.get("repository") - repository_name = str( - repository.get("display_name") or repository.get("repository_name") or repository_key - ) if isinstance(repository, dict) else repository_key - except Exception: - repo_path = "" - repository_key = "" - repository_name = "" + last = latest_statuses.get(info.job_id) + run_state = runtime_states.get(info.job_id, {"running": False}) + if not run_state.get("run_id") and last and last.get("run_id"): + run_state = {"running": False, "run_id": last["run_id"], + "file_activity": last.get("file_activity", False), + "job_name_snapshot": last.get("job_name_snapshot", ""), + "log_available": bool(last.get("log_file"))} + repository_context = resolve_job_repository_context(config, info.job_id, + require_passphrase_file=False, inventory=repository_inventory) + repo_path = repository_context["repository_path"] + repository_key = repository_context["repository_key"] + repository = repository_context["repository"] + repository_name = repository.get("display_name") or repository.get("repository_name") or repository_key result.append( { - "key": info.key, - "backup_type": info.backup_type, + "job_id": info.job_id, + "archive_prefix": info.archive_prefixes[0], + "archive_prefixes": list(info.archive_prefixes), "location": info.location, "display_name": info.display_name, "name": info.name or info.display_name, @@ -1002,6 +769,7 @@ def list_jobs(config: dict, latest_statuses: dict) -> List[dict]: "run_log_available": run_state.get("log_available", True), "run_file_activity": run_state.get("file_activity", False), "run_id": run_state.get("run_id", ""), + "run_name_snapshot": run_state.get("job_name_snapshot", ""), } ) try: @@ -1011,7 +779,7 @@ def list_jobs(config: dict, latest_statuses: dict) -> List[dict]: verification = {} for job in result: - meta = verification.get(job["key"], {}) + meta = verification.get(job["job_id"], {}) job["restore_verification_status"] = meta.get("status", "never") job["restore_verification_reason"] = meta.get("reason", "") job["restore_verification_last_test_date"] = meta.get("last_test_date", "") diff --git a/api/legacy_job_transfer.py b/api/legacy_job_transfer.py new file mode 100644 index 00000000..c2ab43b9 --- /dev/null +++ b/api/legacy_job_transfer.py @@ -0,0 +1,74 @@ +"""Bounded v2 import boundary; no live inventory matching or writes (#478). + +Preview allocates temporary source selectors once. The client returns this map +for apply. Destination job IDs are independently allocated by the v3 importer. +The inactive migration's pure metadata projection is reused, never its planner, +snapshot, journal or installer. Source-host cache paths are not imported. +""" +from copy import deepcopy +from pathlib import Path +from uuid import uuid4 + +from job_model import validate_job_id, validate_job +from job_transfer import FORMAT, indexed, reference_map, validate_bundle, fail + + +def convert_legacy_bundle(config, bundle, source_ids=None): + if not isinstance(bundle, dict) or bundle.get('format') != 'bbui-job-bundle-v2': + return bundle, {} + from migrations.immutable_job_id_v1 import _validate_job, _operational_defaults, _prefixes + from config_api import read_expanded_conf + from repository_context import LEGACY_JOB_REPOSITORY_FIELDS + source = indexed(bundle.get('jobs'), 'job_key') + selectors = {key: str(uuid4()) for key in source} if source_ids is None else source_ids + if not isinstance(selectors, dict) or set(selectors) != set(source): + fail('invalid_legacy_transfer_selection', 'The legacy source selection map is incomplete') + for value in selectors.values(): validate_job_id(value) + if len(set(selectors.values())) != len(selectors): + fail('invalid_legacy_transfer_selection', 'Legacy source selectors must be unique') + converted = [] + defaults = read_expanded_conf(config) + for key, raw in source.items(): + if raw.get('schema_version') not in {1, 2, 3}: + fail('unsupported_legacy_bundle', 'Only known legacy job schemas can be converted') + label = '/bundle/' + key + '.json' + _validate_job(raw, label) + job = _operational_defaults(raw, defaults, label) + job['archive_prefixes'] = _prefixes(raw, True, label) + for field in {'job_key', 'backup_type', 'type_id', 'location', 'cache_reference', *LEGACY_JOB_REPOSITORY_FIELDS}: + job.pop(field, None) + job.update(schema_version=4, job_id=selectors[key], legacy_job_keys=[]) + validate_job(job) + converted.append(job) + repositories = indexed(bundle.get('repositories'), 'repository_key') + storages = indexed(bundle.get('storages'), 'storage_key') + selected_repos = {row['repository_key'] for row in converted} + if selected_repos - repositories.keys(): + fail('invalid_legacy_bundle', 'The legacy bundle lacks canonical repository objects') + repos = [] + for key in sorted(selected_repos): + row = repositories[key] + expected = {old for old, job in source.items() if job['repository_key'] == key} + for old, new in (('used_by', 'job_ids'), ('source_job_keys', 'source_job_ids')): + if old in row: + values = row.pop(old) + if not isinstance(values, list) or any(not isinstance(value, str) for value in values) or len(set(values)) != len(values) or set(values) & source.keys() != expected: + fail('invalid_legacy_references', 'Legacy repository assignments are inconsistent') + elif new in row: + fail('invalid_legacy_references', 'A v2 bundle must not mix canonical and legacy assignments') + row[new] = [selectors[value] for value in source if value in expected] + repos.append(row) + schedules = bundle.get('schedules', {}) + if not isinstance(schedules, dict) or set(schedules) - source.keys() - {'restore_test'}: + fail('invalid_legacy_references', 'A legacy schedule has no source job in the bundle') + storage_keys = {row['storage_key'] for row in repos} + if storage_keys - storages.keys(): fail('invalid_legacy_bundle', 'A legacy storage dependency is missing') + # Old v2 exports also contained unrelated profiles/service schedules. These + # are outside the explicitly selected job scope and never replace settings. + result = {'format': FORMAT, 'jobs': converted, 'repositories': repos, + 'storages': [row for key, row in storages.items() if key in storage_keys], + 'schedules': {selectors[key]: {**row, 'enabled': row.get('enabled', True)} for key, row in schedules.items() if key in source}, + 'passphrase_meta': {key: {'exists': bool(row.get('exists'))} for key, row in bundle.get('passphrase_meta', {}).items() if key in selected_repos}} + result['references'] = reference_map(result) + validate_bundle(result) + return result, deepcopy(selectors) diff --git a/api/migration_api.py b/api/migration_api.py index 42dd867d..13388fbf 100644 --- a/api/migration_api.py +++ b/api/migration_api.py @@ -224,6 +224,9 @@ def plan_migration_backup_cleanup(ui_config: dict, *, keep_per_active_id: int = for row in rows: migration_id = str(row.get("migration_id") or "") + if migration_id == "immutable_job_id_v1": + skipped.append({**row, "reason": "protected_identity_recovery"}) + continue if not row.get("recognized"): skipped.append({**row, "reason": "unrecognized_name"}) continue diff --git a/api/migration_barrier.py b/api/migration_barrier.py new file mode 100644 index 00000000..dd3473b5 --- /dev/null +++ b/api/migration_barrier.py @@ -0,0 +1,333 @@ +"""Cross-process admission and writer leases for the #479 migration. + +Admission defaults to denied, including before the first UI startup and after +a reboot. Only the startup coordinator may publish the versioned runtime ready +proof. The persistent inhibit is deliberately outside migrated owned stores. +Neither an admission check nor a PID list alone is a quiescence guarantee: +the coordinator must hold ``exclusive_migration`` throughout prepare/apply. +""" + +from contextlib import contextmanager +import fcntl +import hashlib +import json +import os +from pathlib import Path +import stat +import threading + + +PROTOCOL_VERSION = 1 +_local = threading.local() + + +class MigrationBlocked(RuntimeError): + api_status = 503 + api_code = "migration_maintenance" + + def __init__(self, reason="migration_maintenance"): + self.reason = reason + super().__init__(reason) + + +def data_root(config): + raw = str(config.get("BACKUP_SCRIPTS_DIR") or "/boot/config/borg-backup") + path = Path(os.path.abspath(raw)) + return path.parent if path.name == "scripts" else path + + +def _paths(config): + root = data_root(config) + namespace = hashlib.sha256(os.fsencode(root)).hexdigest() + runtime = Path(os.environ.get("BORG_UI_MIGRATION_GATE_ROOT") or "/run/borg-backup-ui/migration-gate") / namespace + return root / ".migration-gate" / "blocked.json", runtime + + +def _safe_directory(path, *, create=False): + path = Path(path) + if not path.is_absolute(): + raise MigrationBlocked("unsafe_gate_path") + current = Path(path.anchor) + for part in path.parts[1:]: + current /= part + try: + info = current.lstat() + except FileNotFoundError: + if not create: + raise MigrationBlocked("gate_storage_unavailable") from None + try: + current.mkdir(mode=0o700) + except FileExistsError: + pass + info = current.lstat() + if not stat.S_ISDIR(info.st_mode): + raise MigrationBlocked("unsafe_gate_path") + return path + + +def _mounted_root(root): + parts = root.parts + mount = None + if len(parts) > 2 and parts[1] == "mnt": + mount = Path(*parts[:4]) if parts[2] in {"disks", "remotes"} and len(parts) > 3 else Path(*parts[:3]) + elif len(parts) > 1 and parts[1] == "boot": + mount = Path("/boot") + if mount is not None and not mount.is_mount(): + raise MigrationBlocked("gate_storage_unavailable") + _safe_directory(root) + + +def _open_lock(path): + _safe_directory(path.parent, create=True) + directory_info = path.parent.stat() + if directory_info.st_uid != os.geteuid() or directory_info.st_mode & 0o022: + raise MigrationBlocked("unsafe_gate_lock") + fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.geteuid(): + os.close(fd) + raise MigrationBlocked("unsafe_gate_lock") + return fd + + +@contextmanager +def _admission(config): + _, runtime = _paths(config) + fd = _open_lock(runtime / "admission.lock") + try: + fcntl.flock(fd, fcntl.LOCK_EX) + yield runtime + finally: + os.close(fd) + + +def _proof(path): + try: + path.parent.lstat() + except FileNotFoundError: + return None + _safe_directory(path.parent) + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + except FileNotFoundError: + return None + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise MigrationBlocked("unsafe_gate_state") + if path.name == "ready.json" and (info.st_uid != os.geteuid() or info.st_mode & 0o022): + raise MigrationBlocked("unsafe_gate_state") + raw = os.read(fd, 4097) + if len(raw) > 4096: + raise MigrationBlocked("invalid_gate_state") + return json.loads(raw) + except (ValueError, UnicodeError): + raise MigrationBlocked("invalid_gate_state") from None + finally: + os.close(fd) + + +def _sync_dir(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _publish(path, value): + _safe_directory(path.parent, create=True) + # The admission lock serializes this fixed temporary name; an interrupted + # publish is never accepted as ready and can be safely replaced on restart. + temporary = path.with_name(path.name + ".pending") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.geteuid(): + raise MigrationBlocked("unsafe_gate_state") + os.ftruncate(fd, 0) + raw = (json.dumps(value, sort_keys=True) + "\n").encode() + with os.fdopen(fd, "wb", closefd=False) as handle: + handle.write(raw) + handle.flush() + os.fsync(fd) + finally: + os.close(fd) + os.replace(temporary, path) + _sync_dir(path.parent) + + +def _held(kind, key): + return getattr(_local, kind, {}).get(key, 0) > 0 + + +@contextmanager +def _activate(kind, key): + entries = getattr(_local, kind, None) + if entries is None: + entries = {} + setattr(_local, kind, entries) + entries[key] = entries.get(key, 0) + 1 + try: + yield + finally: + entries[key] -= 1 + if not entries[key]: + del entries[key] + + +class WriterLease: + """An acquired lease can be handed to an already-admitted worker thread.""" + + def __init__(self, fd, key): + self.fd, self.key = fd, key + + @contextmanager + def activate(self): + if self.fd is None: + raise MigrationBlocked("writer_lease_closed") + with _activate("writers", self.key): + yield self + + def close(self): + if self.fd is not None: + os.close(self.fd) + self.fd = None + + +def acquire_writer_lease(config): + """Acquire before creating a worker; close only after its final write.""" + marker, runtime = _paths(config) + key = str(runtime) + with _admission(config): + # Existing admitted work may finish, including nested finalization. + if not _held("writers", key): + _mounted_root(data_root(config)) + if _proof(marker) is not None: + raise MigrationBlocked() + if _proof(runtime / "ready.json") != {"protocol_version": PROTOCOL_VERSION, "ready": True}: + raise MigrationBlocked("startup_validation_required") + fd = _open_lock(runtime / "writers.lock") + try: + fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB) + except OSError: + os.close(fd) + raise MigrationBlocked("migration_in_progress") from None + return WriterLease(fd, key) + + +@contextmanager +def writer_lease(config): + lease = acquire_writer_lease(config) + try: + with lease.activate(): + yield lease + finally: + lease.close() + + +def block_writers(config): + """Close admission durably without waiting for or terminating live work.""" + marker, runtime = _paths(config) + with _admission(config): + (runtime / "ready.json").unlink(missing_ok=True) + _sync_dir(runtime) + _mounted_root(data_root(config)) + _publish(marker, {"protocol_version": PROTOCOL_VERSION, "blocked": True}) + + +def _process_kind(arguments): + names = {Path(arg).name for arg in arguments if arg and not arg.startswith("-")} + if "borg_backup_ui.py" in names: + return "previous_ui_process" + if "wizard_runner.py" in names: + return "backup_worker" + if "activity_log_capture.py" in names: + return "backup_capture_worker" + if "retention_runner.py" in names: + return "retention_worker" + if "factory_reset_worker.py" in names: + return "factory_reset_worker" + if "borg" in names or any(name.startswith("borg-linux-") for name in names): + return "borg_worker" + if names.intersection({"borg_restore_test.py", "borg_restore_test.sh"}): + return "restore_test_worker" + if any(name.startswith("borg_backup_") and name.endswith((".py", ".sh")) for name in names): + return "legacy_backup_worker" + if "-c" in arguments and any("from lib.notification_events import drain_notification_queue" in arg for arg in arguments): + return "notification_worker" + return "" + + +def blockers(config, *, proc_root=Path("/proc")): + """Bounded, read-only evidence for workers from a previously installed build. + + All known plugin workers on this host block, including legacy ones without + leases. No command line, job payload or environment is returned. + """ + rows = [] + try: + processes = list(Path(proc_root).iterdir()) + except OSError: + return [{"reason": "process_inspection_unavailable"}] + for process in processes: + if not process.name.isdigit() or int(process.name) == os.getpid(): + continue + try: + with (process / "cmdline").open("rb") as handle: + raw = handle.read(65537) + except FileNotFoundError: + continue # Process exited before inspection. + except OSError: + rows.append({"reason": "process_inspection_unavailable", "pid": int(process.name)}) + continue + if len(raw) > 65536: + rows.append({"reason": "process_inspection_incomplete", "pid": int(process.name)}) + continue + arguments = raw.decode("utf-8", errors="replace").split("\x00") + kind = _process_kind(arguments) + if kind: + rows.append({"reason": "worker_running", "kind": kind, "pid": int(process.name)}) + if len(rows) >= 100: + break + return rows + + +@contextmanager +def exclusive_migration(config): + """Try quiescence without waiting; retain exclusive ownership until exit.""" + marker, runtime = _paths(config) + fd = None + with _admission(config): + if _proof(marker) != {"protocol_version": PROTOCOL_VERSION, "blocked": True}: + raise MigrationBlocked("migration_gate_not_blocked") + fd = _open_lock(runtime / "writers.lock") + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + raise MigrationBlocked("writers_running") from None + try: + if blockers(config): + raise MigrationBlocked("legacy_workers_running") + with _activate("exclusive", str(runtime)): + yield + finally: + os.close(fd) + + +def clear_block(config): + """Only call after successful startup/integrity verification under exclusion.""" + marker, runtime = _paths(config) + if not _held("exclusive", str(runtime)): + raise MigrationBlocked("exclusive_migration_required") + with _admission(config): + _mounted_root(data_root(config)) + marker.unlink(missing_ok=True) + _sync_dir(marker.parent) + _publish(runtime / "ready.json", {"protocol_version": PROTOCOL_VERSION, "ready": True}) + + +def quiescence_held(config): + """A precondition callback must prove ownership, not just absence of PIDs.""" + return _held("exclusive", str(_paths(config)[1])) diff --git a/api/migrations/identity_apply.py b/api/migrations/identity_apply.py new file mode 100644 index 00000000..18c6f40b --- /dev/null +++ b/api/migrations/identity_apply.py @@ -0,0 +1,502 @@ +"""Explicit, crash-consistent identity cutover (#479). + +The coordinator owns authentication and holds writer exclusion for this entire +call. Neither importing this module nor reading its recovery state applies +anything. The original sealed plan and verified snapshot remain authoritative; +retries never allocate identities or capture already converted originals. +""" + +from __future__ import annotations + +from contextlib import contextmanager +import errno +import fcntl +import hashlib +import os +from pathlib import Path +import stat + +from . import identity_storage as storage +from . import immutable_job_id_v1 as planner + + +_BEGIN = "# --- BORG-BACKUP-UI BEGIN ---" +_END = "# --- BORG-BACKUP-UI END ---" +_WRITES = {"write_json", "write_bytes"} +_RETIRES = {"retire_source", "retire_auxiliary"} + + +def _fail(code): + raise storage.IdentityStorageError(code) from None + + +def _cron_parts(text): + if not isinstance(text, str): + _fail("invalid_plan") + if _BEGIN not in text and _END not in text: + return text, "", False + if text.count(_BEGIN) != 1 or text.count(_END) != 1: + _fail("invalid_plan") + begin, end = text.find(_BEGIN), text.find(_END) + if (end < begin or begin and text[begin - 1] != "\n" + or end and text[end - 1] != "\n"): + _fail("invalid_plan") + end += len(_END) + if end < len(text): + if text[end] != "\n": + _fail("invalid_plan") + end += 1 + if text[begin + len(_BEGIN):begin + len(_BEGIN) + 1] != "\n": + _fail("invalid_plan") + return text[:begin], text[end:], True + + +def replace_managed_cron(original, lines): + """Replace only the plugin block, retaining unrelated bytes verbatim.""" + if (not isinstance(lines, list) or any(not isinstance(line, str) or "\n" in line + or "\r" in line for line in lines)): + _fail("invalid_plan") + before, after, present = _cron_parts(original) + block = _BEGIN + "\n" + "\n".join(lines) + "\n" + _END + "\n" if lines else "" + # Prepending avoids changing a user's final line without a line terminator. + return before + block + after if present else block + original + + +def snapshot_handle(state_dir, plan, digest): + return {"path": str(Path(state_dir) / "snapshot"), + "plan_id": plan["plan_id"], "digest": digest} + + +def _content(action): + try: + if action["kind"] == "write_json": + raw = planner.encode_target_json(action["data"]) + else: + if not isinstance(action.get("text"), str): + _fail("invalid_plan") + raw = action["text"].encode("utf-8") + after = action["after"] + storage._valid_fingerprint(after) + if (not after["exists"] or after["size"] != len(raw) + or after["sha256"] != hashlib.sha256(raw).hexdigest()): + _fail("invalid_plan") + return raw + except (KeyError, ValueError, TypeError, UnicodeError): + _fail("invalid_plan") + + +def _actions(plan): + writes, retirements, derived = [], [], [] + targets, removed = {}, set() + for action in plan["actions"]: + kind = action.get("kind") + if kind not in _WRITES | _RETIRES | {"rebuild_derived"}: + _fail("invalid_plan") + if action.get("source") not in plan["inputs"] or action.get("target") not in plan["inputs"]: + _fail("invalid_plan") + if kind in _WRITES: + _content(action) + if action["target"] in targets: + _fail("invalid_plan") + targets[action["target"]] = action + writes.append(action) + else: + if action["source"] in removed: + _fail("invalid_plan") + removed.add(action["source"]) + if kind == "rebuild_derived": + if action["source"] != action["target"]: + _fail("invalid_plan") + derived.append(action) + else: + if action["source"] == action["target"]: + _fail("invalid_plan") + retirements.append(action) + if removed.intersection(targets): + _fail("invalid_plan") + for action in retirements: + if action["target"] not in targets: + _fail("invalid_plan") + return writes + retirements + derived, targets + + +def _quiescent(callback): + try: + quiet = callback() if callback else False + except Exception: + _fail("writers_active") + if quiet is not True: + _fail("writers_active") + + +@contextmanager +def _operation_lock(state_dir): + """The immutable plan inode is also the duplicate-operation lock.""" + with storage._directory(state_dir) as directory: + fd = os.open("plan.json", os.O_RDONLY | storage._NOFOLLOW, dir_fd=directory) + try: + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 + or info.st_uid != os.getuid() or info.st_nlink != 1): + _fail("unsafe_path") + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + _fail("writers_active") + yield + finally: + os.close(fd) + + +def _directory_identity(path, *, missing_ok=False): + with storage._directory(path, missing_ok=missing_ok) as directory: + if directory is None: + return None + info = os.fstat(directory) + return {"device": info.st_dev, "inode": info.st_ino, + "mode": stat.S_IMODE(info.st_mode), "owner": info.st_uid} + + +def _roots(plan, state_dir, *, started): + """Persist filesystem anchors before creating any destination directory.""" + path = Path(state_dir) / "apply-roots.json" + if storage.fingerprint_file(path)["exists"]: + value = storage._read_json(path) + if (not isinstance(value, dict) or set(value) != {"plan_id", "anchors", "create"} + or value["plan_id"] != plan["plan_id"] + or not isinstance(value["anchors"], dict) or not isinstance(value["create"], list)): + _fail("state_conflict") + return value + if started: + _fail("state_conflict") + anchors, missing = {}, set() + for filename in plan["inputs"]: + parent = Path(filename).parent + while True: + identity = _directory_identity(parent, missing_ok=True) + if identity is not None: + anchors[str(parent)] = identity + break + missing.add(str(parent)) + parent = parent.parent + value = {"plan_id": plan["plan_id"], "anchors": anchors, + "create": sorted(missing, key=lambda item: (len(Path(item).parts), item))} + storage._publish_once(path, storage._canonical(value)) + return value + + +def _directory_receipt(state_dir, path): + name = hashlib.sha256(str(path).encode("utf-8")).hexdigest() + return Path(state_dir) / ("directory-" + name + ".json") + + +def _verify_roots(roots, state_dir, plan): + for path, expected in roots["anchors"].items(): + planner._path(path) # Recheck required /mnt and /boot mounts too. + if _directory_identity(path, missing_ok=True) != expected: + _fail("inventory_changed") + for path in roots["create"]: + receipt = _directory_receipt(state_dir, path) + actual = _directory_identity(path, missing_ok=True) + if storage.fingerprint_file(receipt)["exists"]: + saved = storage._read_json(receipt) + if (not isinstance(saved, dict) or set(saved) != {"plan_id", "path", "identity"} + or saved["plan_id"] != plan["plan_id"] or saved["path"] != path): + _fail("state_conflict") + if actual is not None and actual != saved["identity"]: + _fail("inventory_changed") + elif actual is not None: + _fail("inventory_changed") + + +def _ensure_parent(path, roots, state_dir, plan): + needed = {str(p) for p in Path(path).parents} + for name in roots["create"]: + if name not in needed: + continue + directory = Path(name) + receipt = _directory_receipt(state_dir, name) + saved = storage._read_json(receipt) if storage.fingerprint_file(receipt)["exists"] else None + actual = _directory_identity(directory, missing_ok=True) + if actual is not None: + if saved is None or actual != saved["identity"]: + _fail("inventory_changed") + continue + # Persist the staged directory's inode BEFORE publication. This makes + # both sides of its rename independently verifiable after a crash. + stage = ".identity-dir-" + hashlib.sha256((plan["plan_id"] + name).encode()).hexdigest()[:32] + with storage._directory(directory.parent) as parent: + parent_mode = stat.S_IMODE(os.fstat(parent).st_mode) + try: + os.mkdir(stage, 0o700, dir_fd=parent) + os.fsync(parent) + except FileExistsError: + pass + identity = _directory_identity(directory.parent / stage) + with storage._directory(directory.parent / stage) as staged: + # The destination can be the Unraid FAT data filesystem, whose + # mount-defined mode cannot honor mkdir(0700). It may retain + # that existing parent's observed mode. This exception NEVER + # applies to the separate private recovery/snapshot directory. + if (identity["mode"] not in {0o700, parent_mode} or identity["owner"] != os.getuid() + or os.listdir(staged)): + _fail("state_conflict") + expected = {"plan_id": plan["plan_id"], "path": name, "identity": identity} + if saved is not None and saved != expected: + _fail("state_conflict") + storage._publish_once(receipt, storage._canonical(expected)) + # Destination absence was checked under the coordinator's writer + # lock. No unrelated files/directories may be replaced here. + if _directory_identity(directory, missing_ok=True) is not None: + _fail("inventory_changed") + os.rename(stage, directory.name, src_dir_fd=parent, dst_dir_fd=parent) + os.fsync(parent) + if _directory_identity(directory) != expected["identity"]: + _fail("verification_failed") + + +def _journal_state(records): + started, completed = set(), set() + for record in records: + if record["phase"] == "apply": + if record["status"] in {"pending", "applied"}: + started.update(record["action_ids"]) + if record["status"] == "applied": + completed.update(record["action_ids"]) + return started, completed + + +def _verify_footprint(config, plan, records, control_root): + started, completed = _journal_state(records) + actions, targets = _actions(plan) + removed = {a["source"]: a for a in actions if a["kind"] not in _WRITES} + actual = {path: storage.fingerprint_file(path) for path in plan["inputs"]} + for path, original in plan["inputs"].items(): + replacement, retirement = targets.get(path), removed.get(path) + if replacement and replacement["id"] in completed and actual[path] != replacement["after"]: + _fail("input_changed") + if retirement and retirement["id"] in completed and actual[path]["exists"]: + _fail("input_changed") + if actual[path] == original: + continue + if replacement and replacement["id"] in started and actual[path] == replacement["after"]: + continue + if retirement and retirement["id"] in started and not actual[path]["exists"]: + if (retirement["kind"] == "rebuild_derived" + or actual[retirement["target"]] == targets[retirement["target"]]["after"]): + continue + _fail("input_changed") + # The domain planner checks mounts, inventory membership, live owners and + # repository/secret references using the original saved allocation. + checked = planner.build_plan(config, journal_plan=plan, control_root=control_root) + if checked != plan: + _fail("inventory_changed") + + +def _publish_replacement(action, plan): + path = Path(action["target"]) + content = _content(action) + temporary = ".identity-" + plan["plan_id"][:16] + "-" + hashlib.sha256(action["id"].encode()).hexdigest()[:24] + ".stage" + with storage._directory(path.parent) as directory: + fd = os.open(temporary, os.O_RDONLY | os.O_CREAT | storage._NOFOLLOW | os.O_NONBLOCK, + 0o600, dir_fd=directory) + try: + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1 + or stat.S_IMODE(info.st_mode) not in {0o600, action["after"]["mode"]}): + _fail("unsafe_path") + existing = b"" + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + existing += chunk + # Only an exact prefix of this journaled action is a recoverable + # interrupted staging write. Unexpected bytes remain untouched. + if len(existing) > len(content) or not content.startswith(existing): + _fail("state_conflict") + # A crash after fchmod may leave an otherwise complete read-only + # stage. Restore owner access on this verified private inode only. + os.fchmod(fd, 0o600) + writable = os.open(temporary, os.O_RDWR | storage._NOFOLLOW | os.O_NONBLOCK, + dir_fd=directory) + reopened = os.fstat(writable) + if (info.st_dev, info.st_ino) != (reopened.st_dev, reopened.st_ino): + os.close(writable) + _fail("state_conflict") + os.close(fd) + fd = writable + free = os.fstatvfs(fd) + if free.f_bavail * free.f_frsize < len(content) - len(existing) + 4096: + _fail("insufficient_space") + os.lseek(fd, 0, os.SEEK_SET) + view = memoryview(content) + while view: + count = os.write(fd, view) + if count <= 0: + _fail("storage_unavailable") + view = view[count:] + os.ftruncate(fd, len(content)) + os.fchmod(fd, action["after"]["mode"]) + os.fsync(fd) + finally: + os.close(fd) + if storage.fingerprint_file(path.parent / temporary) != action["after"]: + _fail("verification_failed") + actual = storage.fingerprint_file(path) + if actual not in (plan["inputs"][str(path)], action["after"]): + _fail("input_changed") + os.replace(temporary, path.name, src_dir_fd=directory, dst_dir_fd=directory) + os.fsync(directory) + if storage.fingerprint_file(path) != action["after"]: + _fail("verification_failed") + + +def _retire(action, plan, targets): + path = Path(action["source"]) + if action["kind"] in _RETIRES: + target = targets[action["target"]] + if storage.fingerprint_file(target["target"]) != target["after"]: + _fail("verification_failed") + current = storage.fingerprint_file(path) + if not current["exists"]: + return + if current != plan["inputs"][str(path)]: + _fail("input_changed") + with storage._directory(path.parent) as directory: + os.unlink(path.name, dir_fd=directory) + os.fsync(directory) + if storage.fingerprint_file(path)["exists"]: + _fail("verification_failed") + + +def _read_cron(callback): + try: + text = callback() + if not isinstance(text, str): + _fail("input_changed") + text.encode("utf-8") + return text + except Exception: + _fail("input_changed") + + +def _cron_target(plan, state_dir, render_cron, *, started): + original = plan["external_inputs"]["managed_cron"]["text"] + path = Path(state_dir) / "apply-cron.json" + if storage.fingerprint_file(path)["exists"]: + value = storage._read_json(path) + if (not isinstance(value, dict) or set(value) != {"plan_id", "original", "target"} + or value["plan_id"] != plan["plan_id"] or value["original"] != original): + _fail("state_conflict") + else: + if started: + _fail("state_conflict") + try: + target = render_cron(original, plan) + except Exception: + _fail("invalid_plan") + value = {"plan_id": plan["plan_id"], "original": original, "target": target} + before, after, _ = _cron_parts(original) + new_before, new_after, _ = _cron_parts(value["target"]) + if before + after != new_before + new_after: + _fail("input_changed") + storage._publish_once(path, storage._canonical(value)) + return value["target"] + + +def apply_plan(config, state_dir, *, approval, quiescence_callback, read_cron, + write_cron, render_cron, control_root=None): + """Apply/resume only after an explicit authenticated, bound authorization. + + ``quiescence_callback`` must return exactly True while the caller holds + writer exclusion. ``read_cron()`` returns the actual full crontab text; + ``write_cron(text)`` installs it; ``render_cron(original, plan)`` is pure. + Filesystem and callback errors expose stable masked reason codes only. + """ + plan = None + active = [] + try: + plan = storage.load_plan(state_dir) + actions, targets = _actions(plan) + snapshot = snapshot_handle(state_dir, plan, approval.get("snapshot_digest") if isinstance(approval, dict) else None) + if (plan.get("classification") != "applicable" or plan.get("required") is not True + or plan.get("status") != "pending"): + _fail("invalid_plan") + if (not isinstance(approval, dict) or approval.get("approved") is not True + or approval.get("independent_backup_acknowledged") is not True + or approval.get("plan_id") != plan["plan_id"]): + _fail("approval_required") + with _operation_lock(state_dir): + _quiescent(quiescence_callback) + storage.verify_snapshot(plan, snapshot) + records = storage.read_journal(state_dir) + started = any(r["phase"] in {"apply", "commit"} and r["status"] in {"pending", "applied"} for r in records) + if not started: + storage.verify_preconditions(plan, snapshot, approval, + quiescence_check=quiescence_callback, + external_input_check=lambda: {"managed_cron": {"kind": "crontab", "text": _read_cron(read_cron)}}) + roots = _roots(plan, state_dir, started=started) + _verify_roots(roots, state_dir, plan) + _verify_footprint(config, plan, records, control_root) + cron = _cron_target(plan, state_dir, render_cron, started=started) + original_cron = plan["external_inputs"]["managed_cron"]["text"] + commit_started = any(r["phase"] == "commit" and r["status"] in {"pending", "applied"} for r in records) + if _read_cron(read_cron) not in ({original_cron, cron} if commit_started else {original_cron}): + _fail("input_changed") + already_applied = any(r["phase"] == "commit" and r["status"] == "applied" for r in records) + if not already_applied: + storage.append_journal(state_dir, plan, "pending", "resume" if started else "apply") + _, completed = _journal_state(records) + for action in actions: + if action["id"] in completed: + continue + active = [action["id"]] + _quiescent(quiescence_callback) + _verify_roots(roots, state_dir, plan) + # Writer exclusion spans the transaction. Check the file + # about to change here; scan the complete graph at entry + # and the final boundary, not once per historical record. + if action["kind"] in _WRITES and action["source"] != action["target"]: + if storage.fingerprint_file(action["source"]) != plan["inputs"][action["source"]]: + _fail("input_changed") + storage.append_journal(state_dir, plan, "pending", "apply", action_ids=active) + if action["kind"] in _WRITES: + _ensure_parent(action["target"], roots, state_dir, plan) + if storage.fingerprint_file(action["target"]) != action["after"]: + _publish_replacement(action, plan) + else: + _retire(action, plan, targets) + storage.append_journal(state_dir, plan, "applied", "apply", action_ids=active) + active = [] + _quiescent(quiescence_callback) + _verify_roots(roots, state_dir, plan) + _verify_footprint(config, plan, storage.read_journal(state_dir), control_root) + if planner.verify_target(config, control_root=control_root).get("valid") is not True: + _fail("verification_failed") + if not already_applied: + storage.append_journal(state_dir, plan, "pending", "commit") + if _read_cron(read_cron) not in {original_cron, cron}: + _fail("input_changed") + if _read_cron(read_cron) != cron: + write_cron(cron) + if _read_cron(read_cron) != cron: + _fail("verification_failed") + _quiescent(quiescence_callback) + if planner.verify_target(config, control_root=control_root).get("valid") is not True: + _fail("verification_failed") + if not already_applied: + storage.append_journal(state_dir, plan, "applied", "commit") + return {"migration_id": storage.MIGRATION_ID, "status": "applied", + "plan_id": plan["plan_id"], "snapshot_digest": snapshot["digest"], + "actions_completed": len(actions), "already_applied": already_applied} + except Exception as exc: + code = exc.code if isinstance(exc, storage.IdentityStorageError) else ( + "insufficient_space" if isinstance(exc, OSError) and exc.errno == errno.ENOSPC + else "storage_unavailable" if isinstance(exc, OSError) else "verification_failed") + if plan is not None and code not in {"approval_required", "writers_active", "invalid_journal"}: + try: + storage.append_journal(state_dir, plan, "failed", "apply", reason_code=code, action_ids=active) + except Exception: + pass # Preserve the original masked failure and all evidence. + _fail(code) diff --git a/api/migrations/identity_reasons.py b/api/migrations/identity_reasons.py new file mode 100644 index 00000000..8eed225e --- /dev/null +++ b/api/migrations/identity_reasons.py @@ -0,0 +1,124 @@ +"""Stable public reason vocabulary for identity planning (#479). + +Explicitly enumerated from planner, record projection and canonical job +validation. Runtime never scans source code or accepts arbitrary reason text. +""" + +PLANNING_REASON_CODES = frozenset({ + "ambiguous_archive_ownership", + "ambiguous_cache_reference", + "ambiguous_job_alias", + "ambiguous_retention_shape", + "conflicting_active_identity", + "conflicting_canonical_identity", + "conflicting_destination", + "conflicting_legacy_identity", + "conflicting_repository_assignments", + "dangling_repository", + "dangling_storage", + "destination_already_exists", + "duplicate_inventory_key", + "duplicate_job_id", + "duplicate_json_member", + "duplicate_legacy_alias", + "duplicate_legacy_identity", + "duplicate_repository_key", + "duplicate_restore_id", + "duplicate_schedule_identity", + "identity_cutover_incomplete", + "immutable_job_id", + "invalid_archive_prefix", + "invalid_cache_reference", + "invalid_configuration_encoding", + "invalid_identity_descriptor", + "invalid_identity_reference", + "invalid_inventory_entry", + "invalid_job_aliases", + "invalid_job_filename", + "invalid_job_id", + "invalid_job_name", + "invalid_job_repository", + "invalid_job_settings", + "invalid_job_shape", + "invalid_json", + "invalid_legacy_alias", + "invalid_migration_journal", + "invalid_owned_collection", + "invalid_owned_record", + "invalid_owned_state", + "invalid_owner_pid", + "invalid_recovery_targets", + "invalid_reminder_record", + "invalid_reminder_store", + "invalid_repository", + "invalid_repository_assignments", + "invalid_repository_references", + "invalid_repository_store", + "invalid_restore_id", + "invalid_restore_state", + "invalid_retention", + "invalid_run_context", + "invalid_run_id", + "invalid_runtime_control", + "invalid_runtime_pid", + "invalid_runtime_state", + "invalid_schedule", + "invalid_source_paths", + "invalid_status_identity", + "invalid_storage_store", + "invalid_store_shape", + "invalid_target_path", + "invalid_weekly_provenance", + "invalid_weekly_record", + "invalid_weekly_store", + "job_migration_required", + "legacy_weekly_store", + "missing_canonical_job_id", + "missing_restore_detail", + "missing_restore_index_entry", + "missing_secret_reference", + "mixed_repository_identity", + "mutable_active_reference", + "mutable_canonical_identity", + "mutable_job_identity", + "noncanonical_metadata_filename", + "noncanonical_source_paths", + "nonterminal_restore_history", + "orphan_active_notification", + "orphan_active_reference", + "orphan_active_restore", + "orphan_active_schedule", + "orphan_repository_reference", + "orphan_runtime_recovery", + "overlapping_owned_stores", + "owned_input_too_large", + "partial_migration_without_journal", + "reminder_identity_collision", + "repository_assignment_mismatch", + "required_mount_unavailable", + "restore_active_history_collision", + "restore_detail_filename_mismatch", + "restore_id_mismatch", + "restore_identity_mismatch", + "restore_snapshot_mismatch", + "restore_test_filename_mismatch", + "resume_existing_mapping", + "source_fingerprint_changed", + "target_collision", + "terminal_restore_in_active_store", + "unknown_control_file", + "unknown_recovery_state", + "unproven_legacy_alias", + "unsafe_path", + "unsafe_secret_reference", + "unsupported_record_kind", + "unsupported_record_schema", + "unsupported_schema", + "unsupported_store_schema", + "uuid_allocation_failed", + "weekly_destination_mismatch", + "weekly_projection_mismatch", + "weekly_value_conflict_preserved", + "widget_rebuild_required", + "writers_not_quiescent", +}) diff --git a/api/migrations/identity_records.py b/api/migrations/identity_records.py new file mode 100644 index 00000000..6fd01390 --- /dev/null +++ b/api/migrations/identity_records.py @@ -0,0 +1,588 @@ +"""Pure dependent-record projection for the inactive #447 identity migration. + +No file, process, Borg, scheduler or application helpers are called here. The +scanner supplies owned records as ``path -> {kind, data, ...}``; ``target_path`` +may be supplied for restore tests and a shared weekly destination. Results are +plans, never permission to write. ``sources`` identifies exact consumed files. + +Existing store schema versions are retained. ``identity_schema_version = 1`` +marks new enriched collections; canonical schema-1 records are also accepted. +Weekly observations have their own version-1 envelope with source provenance. +Historical descriptors are preserved, while active ``job_key`` references move +to ``legacy_job_key``. Later phases must implement readers for these shapes. +""" + +from __future__ import annotations + +from copy import deepcopy +import json +import re +from typing import Any +from uuid import UUID + + +_TERMINAL = {"done", "error", "aborted"} +_COLLECTIONS = { + "notification_queue": ("queue", True), + "notification_deliveries": ("deliveries", False), + "runtime_recovery": ("entries", True), + "restore_index": ("runs", False), +} +_DIRECT = {"status", "restore_test", "restore_detail", "control", "cancel_request", "resource_lock", "run_context"} +_STATUS_FILENAME = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}_(.+)\.status$") +_RESTORE_ID = re.compile(r"^[A-Za-z0-9._-]+$") +_RESTORE_SHARED_FIELDS = ( + "state", "archive", "started_at", "finished_at", "source_path", "target_dir", + "destination_path", "conflict_mode", "preserve_owner", "repository_key", + "repository_snapshot", "job_name_snapshot", "archive_prefix_snapshot", + "run_id", "repository_key_snapshot", "location_snapshot", "archive_prefixes_snapshot", +) + + +def _pointer(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _uuid(value: Any) -> bool: + if not isinstance(value, str): + return False + try: + parsed = UUID(value) + return str(parsed) == value and parsed.version == 4 and parsed.variant == "specified in RFC 4122" + except ValueError: + return False + + +def _json_key(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +class _Projection: + def __init__(self, records: dict, jobs: dict, aliases: dict, *, verify: bool = False): + self.input = records + self.jobs = jobs + self.aliases = aliases + self.verify = verify + self.records: dict = {} + self.bindings: list = [] + self.unassigned: list = [] + self.reasons: list = [] + self.restore_links: dict = {} + self.required = False + + def reason(self, code: str, source: str, locator: str = "", *, severity: str = "error") -> None: + item = {"code": code, "source": source, "locator": locator, "severity": severity} + if item not in self.reasons: + self.reasons.append(item) + + def schema(self, data: Any, source: str, locator: str = "") -> bool: + if not isinstance(data, dict): + self.reason("invalid_owned_record", source, locator) + return False + for field in ("schema_version", "identity_schema_version"): + if field in data and (type(data[field]) is not int or data[field] != 1): + self.reason("unsupported_record_schema", source, locator) + return False + return True + + def resolve(self, raw: Any, source: str, locator: str, *, active: bool, + code: str = "orphan_active_reference") -> str | None: + if not isinstance(raw, str) or not raw: + self.reason("invalid_identity_reference", source, locator) + return None + if raw in self.jobs and _uuid(raw): + return raw + if raw in self.aliases and self.aliases[raw] in self.jobs: + if active: + self.required = True + if self.verify and active: + self.reason("mutable_active_reference", source, locator) + return None + return self.aliases[raw] + if active: + self.reason(code, source, locator) + return None + + def bind(self, source: str, locator: str, job_id: str | None, legacy: str, + role: str, original: Any, reason: str = "no_configured_job") -> None: + self.bindings.append({"source": source, "locator": locator, "job_id": job_id, + "legacy_key": legacy, "role": role}) + if job_id is None and role != "system": + self.unassigned.append({"source": source, "locator": locator, + "reason": reason, "data": deepcopy(original)}) + + def restore_link(self, source: str, locator: str, job_id: str | None, + legacy: str, kind: str, row: dict) -> None: + restore_id = row.get("restore_id") + if (not isinstance(restore_id, str) or not _RESTORE_ID.fullmatch(restore_id) + or restore_id in {".", ".."}): + self.reason("invalid_restore_id", source, locator) + return + if kind == "restore_detail" and source.rsplit("/", 1)[-1] != restore_id + ".json": + self.reason("restore_detail_filename_mismatch", source, locator) + self.restore_links.setdefault(restore_id, []).append( + (source, locator, job_id, legacy, kind, row)) + + def row(self, data: Any, source: str, locator: str, kind: str, + *, active: bool = False, legacy: str = "") -> Any: + if not self.schema(data, source, locator): + return deepcopy(data) + original = deepcopy(data) + row = deepcopy(data) + if kind == "run_context": + try: + from job_runs import validate_run_context + validate_run_context(row, row.get("job_id"), row.get("run_id")) + if source.rsplit("/", 2)[-2] != row["run_id"]: + raise ValueError("Run directory does not match the payload") + except (ValueError, OSError, TypeError, KeyError, AttributeError): + self.reason("invalid_run_context", source, locator) + return row + if kind.startswith("restore_") and kind != "restore_test": + state = row.get("state") + if not isinstance(state, str) or state not in _TERMINAL | {"running"}: + self.reason("invalid_restore_state", source, locator) + active = state not in _TERMINAL + if kind in {"restore_index", "restore_detail"} and state not in _TERMINAL: + self.reason("nonterminal_restore_history", source, locator) + if kind == "restore_runs" and state in _TERMINAL: + self.reason("terminal_restore_in_active_store", source, locator) + if kind == "runtime_recovery": + if row.get("state") not in {"pending_restart", "restart_failed"}: + self.reason("unknown_recovery_state", source, locator) + if not isinstance(row.get("targets"), list) or not row["targets"]: + self.reason("invalid_recovery_targets", source, locator) + else: + target_ids = [] + for target in row["targets"]: + if (not isinstance(target, dict) or not isinstance(target.get("id"), str) + or not target["id"] or not isinstance(target.get("name"), str) or not target["name"]): + self.reason("invalid_recovery_targets", source, locator) + else: + target_ids.append(target["id"]) + if len(set(target_ids)) != len(target_ids): + self.reason("invalid_recovery_targets", source, locator) + if kind in {"control", "cancel_request", "resource_lock"}: + active = kind != "control" or row.get("finished") is not True + if ("run_id" in row and (not isinstance(row["run_id"], str) or not row["run_id"])): + self.reason("invalid_run_id", source, locator) + if "pid" in row and (type(row["pid"]) is not int or row["pid"] < 0): + self.reason("invalid_owner_pid", source, locator) + raw_key = row.get("job_key", row.get("legacy_job_key", "")) + if raw_key and not isinstance(raw_key, str): + self.reason("invalid_identity_reference", source, locator) + return row + payload_key = "" + type_field = "type" if kind == "restore_test" else "backup_type" + location_field = "backup_location" if kind == "runtime_recovery" else "location" + if type_field in row or location_field in row: + backup_type, location = row.get(type_field), row.get(location_field) + if not isinstance(backup_type, str) or not backup_type or not isinstance(location, str) or not location: + self.reason("invalid_identity_descriptor", source, locator) + return row + # Shipped restore proofs use type/location. Canonical writers keep + # these as report evidence (type is now an archive prefix), so only + # a legacy proof may use them to establish its former job alias. + if kind != "restore_test" or not row.get("job_id"): + payload_key = backup_type + "_" + location + candidates = [value for value in (raw_key, payload_key, legacy) if value] + is_system = (kind in {"notification_queue", "notification_deliveries"} + and not row.get("job_id") + and ((raw_key == "restore_test" and row.get("source") == "restore_test") + or (not raw_key and row.get("source") == "system"))) + if kind in {"notification_queue", "notification_deliveries"} and not row.get("job_id") and not raw_key: + is_system = row.get("service") in {"restore_test", "system"} + if kind == "resource_lock" and not row.get("job_id") and row.get("service") == "restore": + is_system = row.get("operation") == "restore" + if kind == "resource_lock" and not row.get("job_id") and raw_key == "restore_test": + is_system = row.get("operation") == "restore_test" + if is_system: + self.bind(source, locator, None, raw_key, "system", original) + return row + if not active and row.get("identity_state") == "unassigned": + self.bind(source, locator, None, raw_key or legacy, "history", original, + row.get("identity_reason", "no_configured_job")) + if kind in {"restore_runs", "restore_index", "restore_detail"}: + self.restore_link(source, locator, None, raw_key or legacy, kind, row) + return row + if kind == "status" and not payload_key and not row.get("job_id"): + self.reason("invalid_status_identity", source, locator) + return row + conflict = len(set(candidates)) > 1 + resolved = {self.aliases.get(value, value if value in self.jobs else None) for value in candidates} + if conflict and len(resolved) == 1 and None not in resolved: + conflict = False + supplied_id = row.get("job_id") + if supplied_id is not None and not _uuid(supplied_id): + self.reason("invalid_job_id", source, locator) + return row + if supplied_id and any(value is not None and value != supplied_id for value in resolved): + conflict = True + key = candidates[0] if candidates else "" + code = {"notification_queue": "orphan_active_notification", + "runtime_recovery": "orphan_runtime_recovery", + "restore_runs": "orphan_active_restore"}.get(kind, "orphan_active_reference") + if conflict: + if active: + self.reason("conflicting_active_identity", source, locator) + elif self.verify or supplied_id: + self.reason("conflicting_canonical_identity", source, locator) + job_id = None + elif supplied_id: + job_id = self.resolve(supplied_id, source, locator, active=active, code=code) + if self.verify and active and "job_key" in row: + self.reason("mutable_active_reference", source, locator) + elif key: + job_id = self.resolve(key, source, locator, active=active, code=code) + if self.verify and job_id is not None: + self.reason("missing_canonical_job_id", source, locator) + else: + if active: + self.reason(code, source, locator) + job_id = None + historical_reason = "conflicting_identity_evidence" if conflict else "no_configured_job" + if not active and supplied_id and job_id is None and not conflict: + historical_reason = "deleted_job" + elif not active and job_id is None and not conflict and payload_key: + # A former prefix can explain the diagnostic, never establish an + # alias. Keep the historical record unassigned even with one hint. + former_prefix = str(row.get(type_field)) + "-backup" + if any(former_prefix in job.get("archive_prefixes", []) for job in self.jobs.values()): + historical_reason = "no_authoritative_alias" + self.bind(source, locator, job_id, key, "active" if active else "history", original, historical_reason) + if job_id is not None: + row["job_id"] = job_id + if not supplied_id: + row.setdefault("schema_version", 1) + if active and "job_key" in row: + if "legacy_job_key" in row and row["legacy_job_key"] != row["job_key"]: + self.reason("conflicting_active_identity", source, locator) + row["legacy_job_key"] = row.pop("job_key") + else: + # Keep unknown historical IDs/descriptors, but never imply a live job. + row["identity_state"] = "unassigned" + row["identity_reason"] = historical_reason + row.setdefault("identity_schema_version", 1) + if kind in {"restore_runs", "restore_index", "restore_detail"}: + self.restore_link(source, locator, job_id, key, kind, row) + return row + + def emit(self, source: str, record: dict, data: Any, *, sources: list | None = None) -> None: + target = record.get("target_path", source) + if not isinstance(target, str) or not target.startswith("/"): + self.reason("invalid_target_path", source) + return + if target in self.records: + self.reason("target_collision", source) + return + self.records[target] = {**deepcopy(record), "data": data, + "sources": sources or [source], "target_path": target} + + def schedules(self, source: str, record: dict) -> None: + data = record["data"] + if not isinstance(data, dict): + self.reason("invalid_owned_record", source) + return + result = {} + for key, row in data.items(): + locator = "/" + _pointer(key) + if (not isinstance(row, dict) or not isinstance(row.get("cron"), str) + or ("enabled" in row and type(row["enabled"]) is not bool)): + self.reason("invalid_schedule", source, locator) + continue + cron = row["cron"] + if ((cron and (len(cron.split()) != 5 or any(not re.fullmatch(r"[\d*/,\-]+", part) for part in cron.split()))) + or (not cron and row.get("enabled", True))): + self.reason("invalid_schedule", source, locator) + continue + if key == "restore_test": + result[key] = deepcopy(row) + continue + job_id = self.resolve(key, source, locator, active=True, code="orphan_active_schedule") + if job_id is None: + continue + self.bind(source, locator, job_id, key, "active", row) + if job_id in result: + self.reason("duplicate_schedule_identity", source, locator) + else: + result[job_id] = deepcopy(row) + self.emit(source, record, result) + + def repositories(self, source: str, record: dict) -> None: + data = deepcopy(record["data"]) + if not self.schema(data, source) or not isinstance(data.get("repositories"), list): + self.reason("invalid_repository_store", source) + return + seen = set() + for index, row in enumerate(data["repositories"]): + locator = f"/repositories/{index}" + if not isinstance(row, dict) or not isinstance(row.get("repository_key"), str): + self.reason("invalid_repository", source, locator) + continue + repo = row["repository_key"] + if repo in seen: + self.reason("duplicate_repository_key", source, locator) + seen.add(repo) + expected = {job_id for job_id, job in self.jobs.items() if job.get("repository_key") == repo} + for old, new in (("used_by", "job_ids"), ("source_job_keys", "source_job_ids")): + if old in row and new in row: + self.reason("mixed_repository_identity", source, locator) + continue + field = old if old in row else new + if field == old: + self.required = True + values = row.get(field, []) + if not isinstance(values, list): + self.reason("invalid_repository_references", source, locator + "/" + field) + continue + converted = [] + for i, value in enumerate(values): + pointer = locator + f"/{field}/{i}" + if self.verify and field == old: + self.reason("mutable_active_reference", source, pointer) + job_id = self.resolve(value, source, pointer, active=True, code="orphan_repository_reference") + if job_id is not None: + self.bind(source, pointer, job_id, value, "active", value) + converted.append(job_id) + if len(set(converted)) != len(converted) or set(converted) != expected: + self.reason("repository_assignment_mismatch", source, locator + "/" + field) + row.pop(old, None) + row[new] = converted + self.emit(source, record, data) + + def reminders(self, source: str, record: dict) -> None: + data = deepcopy(record["data"]) + if not self.schema(data, source) or not isinstance(data.get("last_sent"), dict): + self.reason("invalid_reminder_store", source) + return + converted = {} + unassigned = deepcopy(data.get("unassigned", [])) + if not isinstance(unassigned, list): + self.reason("invalid_reminder_store", source) + return + for key, value in data["last_sent"].items(): + locator = "/last_sent/" + _pointer(key) + parts = key.split(":", 2) + if (len(parts) != 3 or not all(parts) or type(value) not in (int, float) + or value < 0): + self.reason("invalid_reminder_record", source, locator) + continue + event, job_key, due = parts + job_id = self.resolve(job_key, source, locator, active=False) + self.bind(source, locator, job_id, job_key, "history", value) + if job_id is None: + unassigned.append({"key": key, "value": value, "source": source, "locator": locator}) + continue + if self.verify and job_key != job_id: + self.reason("mutable_active_reference", source, locator) + if job_key != job_id: + self.required = True + new_key = f"{event}:{job_id}:{due}" + if new_key in converted and converted[new_key] != value: + self.reason("reminder_identity_collision", source, locator) + converted[new_key] = value + data["last_sent"] = converted + if unassigned: + data["unassigned"] = unassigned + self.emit(source, record, data) + + def weekly(self, sources: list[tuple[str, dict]]) -> None: + groups: dict = {} + targets = {record.get("target_path", source) for source, record in sources} + if len(targets) != 1: + self.reason("weekly_destination_mismatch", sources[0][0]) + return + canonical = all(isinstance(record["data"], dict) and "observations" in record["data"] + for _, record in sources) + if self.verify and (not canonical or len(sources) != 1): + self.reason("legacy_weekly_store", sources[0][0]) + for source, record in sources: + data = record["data"] + if not isinstance(data, dict): + self.reason("invalid_weekly_store", source) + continue + if "observations" in data: + if not self.schema(data, source) or not isinstance(data["observations"], list): + self.reason("invalid_weekly_store", source) + continue + entries = [(row.get("job_id") or row.get("legacy_job_key", ""), row, + f"/observations/{i}") for i, row in enumerate(data["observations"]) if isinstance(row, dict)] + if len(entries) != len(data["observations"]): + self.reason("invalid_weekly_record", source) + else: + entries = [] + for key, rows in data.items(): + if not isinstance(rows, list): + self.reason("invalid_weekly_store", source, "/" + _pointer(key)) + continue + entries.extend((key, row, f"/{_pointer(key)}/{i}") for i, row in enumerate(rows)) + for key, original, locator in entries: + if (not isinstance(original, dict) or not isinstance(original.get("week"), str) + or type(original.get("size")) is not int or original["size"] < 0): + self.reason("invalid_weekly_record", source, locator) + continue + supplied_id = original.get("job_id") + if supplied_id is not None and not _uuid(supplied_id): + self.reason("invalid_job_id", source, locator) + continue + legacy_id = self.aliases.get(original.get("legacy_job_key", "")) + if (supplied_id and legacy_id is not None and supplied_id != legacy_id + and original.get("identity_state") != "unassigned"): + self.reason("conflicting_canonical_identity", source, locator) + job_id = (None if original.get("identity_state") == "unassigned" + else self.resolve(key, source, locator, active=False)) + history_reason = "deleted_job" if supplied_id and job_id is None else "no_configured_job" + if original.get("identity_state") == "unassigned": + history_reason = original.get("identity_reason") or history_reason + self.bind(source, locator, job_id, key, "history", original, history_reason) + if self.verify and job_id and key != job_id: + self.reason("mutable_active_reference", source, locator) + row = deepcopy(original) + # A deleted job is absent from the active graph, not stripped + # of its former immutable identity in retained history. + row["job_id"] = supplied_id or job_id + if not canonical: + row["legacy_job_key"] = key + if job_id is None and (not self.verify or not supplied_id or "identity_state" in original): + row["identity_state"] = "unassigned" + if supplied_id: + row["identity_reason"] = history_reason + provenance = row.pop("source_records", [{"source": source, "locator": locator}]) + row.pop("conflict", None) + if not isinstance(provenance, list) or not provenance: + self.reason("invalid_weekly_provenance", source, locator) + continue + identity = _json_key(row) + if identity not in groups: + groups[identity] = {**row, "source_records": []} + for evidence in provenance: + if (not isinstance(evidence, dict) or not isinstance(evidence.get("source"), str) + or not isinstance(evidence.get("locator"), str)): + self.reason("invalid_weekly_provenance", source, locator) + elif evidence not in groups[identity]["source_records"]: + groups[identity]["source_records"].append(evidence) + observations = list(groups.values()) + by_week: dict = {} + for row in observations: + owner = row.get("job_id") or row.get("legacy_job_key") + by_week.setdefault((owner, row["week"]), set()).add(row["size"]) + for row in observations: + owner = row.get("job_id") or row.get("legacy_job_key") + if len(by_week[(owner, row["week"])]) > 1: + row["conflict"] = True + self.reason("weekly_value_conflict_preserved", sources[0][0], severity="warning") + payload = (deepcopy(sources[0][1]["data"]) if canonical and len(sources) == 1 else {}) + payload.update({"schema_version": 1, "identity_schema_version": 1, "observations": observations}) + source, record = sources[0] + if self.verify and canonical and len(sources) == 1 and payload != record["data"]: + self.reason("weekly_projection_mismatch", source) + self.emit(source, record, payload, sources=[path for path, _ in sources]) + + def run(self) -> dict: + weekly = [] + for source, record in sorted(self.input.items()): + if (not isinstance(source, str) or not source.startswith("/") or not isinstance(record, dict) + or "data" not in record or not isinstance(record.get("kind"), str)): + self.reason("invalid_owned_record", str(source)) + continue + kind, data = record["kind"], record["data"] + if kind == "weekly": + weekly.append((source, record)) + elif kind == "schedules": + self.schedules(source, record) + elif kind == "repositories": + self.repositories(source, record) + elif kind == "notification_state": + self.reminders(source, record) + elif kind in _DIRECT: + legacy = record.get("legacy_key", "") + if kind == "status" and isinstance(data, dict) and not data.get("job_id"): + match = _STATUS_FILENAME.fullmatch(source.rsplit("/", 1)[-1]) + legacy = match.group(1) if match else legacy + output = self.row(data, source, "", kind, legacy=legacy) + proof = data if self.verify else output + if (kind == "restore_test" and isinstance(proof, dict) + and proof.get("identity_state") != "unassigned" and _uuid(proof.get("job_id"))): + target = source if self.verify else record.get("target_path", source) + if isinstance(target, str) and target.rsplit("/", 1)[-1] != proof["job_id"] + ".test": + self.reason("restore_test_filename_mismatch", source) + self.emit(source, record, output) + elif kind in _COLLECTIONS: + field, active = _COLLECTIONS[kind] + if not self.schema(data, source) or not isinstance(data.get(field), list): + self.reason("invalid_owned_collection", source) + continue + output = deepcopy(data) + output[field] = [self.row(row, source, f"/{field}/{index}", kind, active=active) + for index, row in enumerate(data[field])] + self.emit(source, record, output) + elif kind == "restore_runs": + if not self.schema(data, source) or not isinstance(data.get("runs"), dict): + self.reason("invalid_owned_collection", source) + continue + output = deepcopy(data) + for restore_id, row in data["runs"].items(): + locator = "/runs/" + _pointer(restore_id) + if not isinstance(row, dict) or row.get("restore_id") != restore_id: + self.reason("restore_id_mismatch", source, locator) + output["runs"][restore_id] = self.row(row, source, locator, kind, active=True) + self.emit(source, record, output) + elif kind == "storages": + if self.schema(data, source) and isinstance(data.get("storages"), list): + self.emit(source, record, deepcopy(data)) + else: + self.reason("invalid_storage_store", source) + elif kind == "widget_cache": + # Derived caches cannot survive cutover as active legacy joins. + # Rebuild is owned by #476/#479 under the startup writer gate. + if self.schema(data, source): + self.reason("widget_rebuild_required", source, severity="warning") + else: + self.reason("unsupported_record_kind", source) + if weekly: + self.weekly(weekly) + for links in self.restore_links.values(): + assigned = {(entry[2], entry[5].get("job_id") or entry[3]) if entry[2] is None + else (entry[2], "") for entry in links} + if len(assigned) > 1: + for source, locator, *_ in links: + self.reason("restore_identity_mismatch", source, locator) + by_kind = {kind: [entry for entry in links if entry[4] == kind] + for kind in ("restore_index", "restore_detail", "restore_runs")} + for entries in by_kind.values(): + if len(entries) > 1: + for source, locator, *_ in entries: + self.reason("duplicate_restore_id", source, locator) + index, detail, active = (by_kind[kind] for kind in ("restore_index", "restore_detail", "restore_runs")) + if (index or detail) and active: + for source, locator, *_ in links: + self.reason("restore_active_history_collision", source, locator) + if index and not detail: + for source, locator, *_ in index: + self.reason("missing_restore_detail", source, locator) + if detail and not index: + for source, locator, *_ in detail: + self.reason("missing_restore_index_entry", source, locator) + if len(index) == len(detail) == 1: + summary, body = index[0][5], detail[0][5] + if any((field in summary) != (field in body) or summary.get(field) != body.get(field) + for field in _RESTORE_SHARED_FIELDS): + for source, locator, *_ in index + detail: + self.reason("restore_snapshot_mismatch", source, locator) + return {"records": self.records, "bindings": self.bindings, + "unassigned": self.unassigned, "reasons": self.reasons, + "required": self.required} + + +def project_records(records: dict, jobs: dict, aliases: dict) -> dict: + """Project validated owned stores without touching input objects or disk.""" + return _Projection(records, jobs, aliases).run() + + +def verify_records(records: dict, jobs: dict, aliases: dict | None = None) -> list[dict]: + """Check actual target records; active aliases are errors, never repaired. + + This is referential verification only. The scanner/journal verifier must + independently enforce filenames, ownership, completeness and exact bytes. + """ + return _Projection(records, jobs, aliases or {}, verify=True).run()["reasons"] diff --git a/api/migrations/identity_storage.py b/api/migrations/identity_storage.py new file mode 100644 index 00000000..9c9b77cc --- /dev/null +++ b/api/migrations/identity_storage.py @@ -0,0 +1,794 @@ +"""Private, inactive planning/snapshot primitives for #472. + +Nothing in this module rewrites installation data, invokes Borg, registers a +migration, or starts workers. The caller must supply an exact allowlisted plan +and a dedicated state directory on a persistent filesystem supporting private +0700/0600 permissions and hard links (normally not the Unraid FAT /boot USB). +A verified snapshot is not an apply engine, +a downloadable support bundle, or proof of an independent external backup. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from datetime import datetime, timezone +import errno +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import stat +from typing import Callable +import uuid + + +MIGRATION_ID = "immutable_job_id_v1" +STATUSES = frozenset({"pending", "applied", "skipped", "failed", "blocked", "not_applicable"}) +PHASES = frozenset({"detect", "plan", "snapshot", "verify", "confirm", "apply", "resume", "commit"}) +REASON_CODES = frozenset({ + "approval_required", "input_changed", "inventory_changed", "invalid_plan", + "invalid_snapshot", "snapshot_incomplete", "snapshot_changed", "unsafe_path", + "state_conflict", "invalid_journal", "storage_unavailable", "insufficient_space", + "writers_active", "verification_failed", "interrupted", "not_applicable", + "state_filesystem_unsupported", +}) +_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0) +_DIRECTORY = getattr(os, "O_DIRECTORY", 0) + + +class IdentityStorageError(RuntimeError): + """Stable error codes only: never expose filesystem exception/secret text.""" + + def __init__(self, code: str): + self.code = code if isinstance(code, str) and code in REASON_CODES else "verification_failed" + super().__init__(self.code) + + +def _fail(code: str): + raise IdentityStorageError(code) from None + + +def _canonical(value) -> bytes: + try: + return json.dumps(value, sort_keys=True, ensure_ascii=True, allow_nan=False, + separators=(",", ":")).encode("utf-8") + except (TypeError, ValueError, OverflowError, RecursionError): + _fail("invalid_plan") + + +def _digest(value) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _absolute(path) -> Path: + try: + raw = os.fspath(path) + except TypeError: + _fail("unsafe_path") + if not isinstance(raw, str) or not raw.startswith("/") or "\x00" in raw: + _fail("unsafe_path") + if raw != os.path.normpath(raw) or raw.startswith("//"): + _fail("unsafe_path") + try: + raw.encode("utf-8") + except UnicodeError: + _fail("unsafe_path") + return Path(raw) + + +def _os_error(exc: OSError): + _fail("insufficient_space" if exc.errno == errno.ENOSPC else + "unsafe_path" if exc.errno in {errno.ELOOP, errno.ENOTDIR} else "storage_unavailable") + + +@contextmanager +def _directory(path, *, missing_ok=False): + """Walk using directory FDs; never follow even an intermediate symlink.""" + path = _absolute(path) + fd = None + try: + fd = os.open("/", os.O_RDONLY | _DIRECTORY | _NOFOLLOW) + for component in path.parts[1:]: + try: + child = os.open(component, os.O_RDONLY | _DIRECTORY | _NOFOLLOW, dir_fd=fd) + except FileNotFoundError: + if missing_ok: + os.close(fd) + fd = None + yield None + return + raise + os.close(fd) + fd = child + yield fd + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) + + +def _read_file(path, *, private=False): + path = _absolute(path) + with _directory(path.parent, missing_ok=True) as directory: + if directory is None: + return {"exists": False}, None + fd = None + try: + try: + fd = os.open(path.name, os.O_RDONLY | _NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) + except FileNotFoundError: + return {"exists": False}, None + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode): + _fail("unsafe_path") + if private and (stat.S_IMODE(before.st_mode) != 0o600 + or before.st_uid != os.getuid() or before.st_nlink != 1): + _fail("unsafe_path") + chunks = [] + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + after = os.fstat(fd) + identity = lambda info: (info.st_dev, info.st_ino, info.st_size, + info.st_mtime_ns, info.st_ctime_ns, info.st_mode) + if identity(before) != identity(after): + _fail("input_changed") + content = b"".join(chunks) + if len(content) != before.st_size: + _fail("input_changed") + return {"exists": True, "size": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "mode": stat.S_IMODE(before.st_mode)}, content + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) + + +def fingerprint_file(path) -> dict: + """Read exact regular-file bytes; absent destinations are explicit.""" + return _read_file(path)[0] + + +def read_fingerprinted_file(path): + """Return one stable descriptor read: ``(fingerprint, bytes_or_None)``.""" + return _read_file(path) + + +def read_file(path) -> bytes: + """Read a required regular file without following any symlink component.""" + fingerprint, content = _read_file(path) + if not fingerprint["exists"]: + _fail("storage_unavailable") + return content + + +def inventory_group(path, suffixes) -> dict: + """Capture an immediate allowlist, including an absent directory itself. + + No recursive scan is performed. Matching non-files/symlinks are rejected, + not silently hidden. Directory identity detects replaced/missing mounts. + """ + path = _absolute(path) + if path == Path("/") or not isinstance(suffixes, (list, tuple)) or not suffixes: + _fail("unsafe_path") + if any(not isinstance(item, str) or not item.startswith(".") + or "/" in item or "\\" in item or len(item) > 32 for item in suffixes): + _fail("unsafe_path") + suffixes = sorted(set(suffixes)) + with _directory(path, missing_ok=True) as directory: + if directory is None: + return {"path": str(path), "suffixes": suffixes, "exists": False, "entries": []} + before = os.fstat(directory) + names = sorted(name for name in os.listdir(directory) if name.endswith(tuple(suffixes))) + for name in names: + info = os.stat(name, dir_fd=directory, follow_symlinks=False) + if not stat.S_ISREG(info.st_mode): + _fail("unsafe_path") + after = os.fstat(directory) + if (before.st_mtime_ns, before.st_ctime_ns) != (after.st_mtime_ns, after.st_ctime_ns): + _fail("inventory_changed") + return {"path": str(path), "suffixes": suffixes, "exists": True, + "device": before.st_dev, "inode": before.st_ino, "entries": names} + + +def inventory_directories(path) -> dict: + """Capture a bounded directory-only control root, not a recursive tree. + + Every child must be a real directory. Callers inventory the allowed files + inside each child separately, so an added run directory invalidates a plan. + """ + path = _absolute(path) + if path == Path("/"): + _fail("unsafe_path") + with _directory(path, missing_ok=True) as directory: + if directory is None: + return {"path": str(path), "kind": "directories", "exists": False, "entries": []} + before = os.fstat(directory) + names = sorted(os.listdir(directory)) + for name in names: + info = os.stat(name, dir_fd=directory, follow_symlinks=False) + if not stat.S_ISDIR(info.st_mode): + _fail("unsafe_path") + after = os.fstat(directory) + if (before.st_mtime_ns, before.st_ctime_ns) != (after.st_mtime_ns, after.st_ctime_ns): + _fail("inventory_changed") + return {"path": str(path), "kind": "directories", "exists": True, + "device": before.st_dev, "inode": before.st_ino, "entries": names} + + +def _valid_fingerprint(value): + if not isinstance(value, dict) or type(value.get("exists")) is not bool: + _fail("invalid_plan") + if not value["exists"]: + if value != {"exists": False}: + _fail("invalid_plan") + return + if set(value) != {"exists", "size", "sha256", "mode"}: + _fail("invalid_plan") + if type(value["size"]) is not int or value["size"] < 0: + _fail("invalid_plan") + if type(value["mode"]) is not int or not 0 <= value["mode"] <= 0o7777: + _fail("invalid_plan") + checksum = value["sha256"] + if not isinstance(checksum, str) or len(checksum) != 64 or any(c not in "0123456789abcdef" for c in checksum): + _fail("invalid_plan") + + +def seal_plan(plan: dict) -> dict: + """Validate structural storage boundaries and hash the complete plan. + + Domain/referential validation belongs to the planner. Multiple aliases may + deliberately map to the same UUID. No UUID is allocated by this module. + """ + if not isinstance(plan, dict): + _fail("invalid_plan") + result = json.loads(_canonical(plan)) + claimed = result.pop("plan_id", None) + if type(result.get("schema_version")) is not int or result["schema_version"] != 1: + _fail("invalid_plan") + if result.get("migration_id") != MIGRATION_ID or not isinstance(result.get("id_map"), dict): + _fail("invalid_plan") + for alias, identity in result["id_map"].items(): + if not isinstance(alias, str) or not alias or not isinstance(identity, str): + _fail("invalid_plan") + try: + value = uuid.UUID(identity) + except (ValueError, AttributeError): + _fail("invalid_plan") + if str(value) != identity or value.version != 4 or value.variant != uuid.RFC_4122: + _fail("invalid_plan") + inputs = result.get("inputs") + if not isinstance(inputs, dict): + _fail("invalid_plan") + for path, value in inputs.items(): + _absolute(path) + _valid_fingerprint(value) + external = result.get("external_inputs", {}) + if not isinstance(external, dict): + _fail("invalid_plan") + for name, item in external.items(): + if (not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", name) + or not isinstance(item, dict) or set(item) != {"text", "kind"} + or not isinstance(item["text"], str) or item["kind"] != "crontab"): + _fail("invalid_plan") + try: + item["text"].encode("utf-8") + except UnicodeError: + _fail("invalid_plan") + groups = result.get("inventory_groups", []) + if not isinstance(groups, list): + _fail("invalid_plan") + seen_roots = set() + for group in groups: + if not isinstance(group, dict) or not isinstance(group.get("entries"), list): + _fail("invalid_plan") + path = _absolute(group.get("path")) + suffixes = group.get("suffixes") + directory_group = group.get("kind") == "directories" + if (not directory_group and (not isinstance(suffixes, list) or not suffixes) + or type(group.get("exists")) is not bool): + _fail("invalid_plan") + if str(path) in seen_roots: + _fail("invalid_plan") + seen_roots.add(str(path)) + for name in group["entries"]: + if not isinstance(name, str) or not name or name in {".", ".."} or "/" in name or "\\" in name: + _fail("invalid_plan") + if not directory_group and str(path / name) not in inputs: + _fail("invalid_plan") + actions = result.get("actions", []) + if not isinstance(actions, list): + _fail("invalid_plan") + action_ids = set() + for action in actions: + if not isinstance(action, dict) or not isinstance(action.get("id"), str) or not action["id"]: + _fail("invalid_plan") + if action["id"] in action_ids: + _fail("invalid_plan") + action_ids.add(action["id"]) + for field in ("source", "target"): + if action.get(field) is not None and str(_absolute(action[field])) not in inputs: + _fail("invalid_plan") + digest = _digest(result) + if claimed is not None and claimed != digest: + _fail("invalid_plan") + result["plan_id"] = digest + return result + + +def _check_overlap(plan, state_dir): + state_dir = _absolute(state_dir) + for name in plan["inputs"]: + path = _absolute(name) + if path == state_dir or state_dir in path.parents or path in state_dir.parents: + _fail("unsafe_path") + for group in plan.get("inventory_groups", []): + path = _absolute(group["path"]) + # A dedicated migration directory can be below config, but never below + # one of the exact scanned jobs/status/restore-test directories. + if state_dir == path or path in state_dir.parents or state_dir in path.parents: + _fail("unsafe_path") + + +def _private_directory(path, *, create=False): + path = _absolute(path) + if create: + with _directory(path.parent) as parent: + try: + os.mkdir(path.name, mode=0o700, dir_fd=parent) + os.fsync(parent) + except FileExistsError: + pass + except OSError as exc: + _os_error(exc) + with _directory(path) as directory: + info = os.fstat(directory) + if stat.S_IMODE(info.st_mode) != 0o700 or info.st_uid != os.getuid(): + _fail("unsafe_path") + return path + + +def _publish_once(path: Path, content: bytes): + """Durable publication without overwriting an existing pathname.""" + with _directory(path.parent) as directory: + temporary = ".stage-" + uuid.uuid4().hex + fd = None + try: + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _NOFOLLOW, + 0o600, dir_fd=directory) + view = memoryview(content) + while view: + written = os.write(fd, view) + if written <= 0: + _fail("storage_unavailable") + view = view[written:] + os.fsync(fd) + os.close(fd) + fd = None + try: + os.link(temporary, path.name, src_dir_fd=directory, dst_dir_fd=directory, + follow_symlinks=False) + except FileExistsError: + existing, raw = _read_file(path, private=True) + if not existing["exists"] or raw != content: + _fail("state_conflict") + except OSError as exc: + if exc.errno in {errno.EOPNOTSUPP, errno.ENOSYS, errno.EXDEV, errno.EPERM}: + _fail("state_filesystem_unsupported") + raise + os.unlink(temporary, dir_fd=directory) + temporary = None + os.fsync(directory) + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) + if temporary is not None: + try: + os.unlink(temporary, dir_fd=directory) + except OSError: + pass + + +def _read_json(path): + exists, raw = _read_file(path, private=True) + if not exists["exists"]: + _fail("snapshot_incomplete") + try: + return json.loads(raw, object_pairs_hook=_unique_json_pairs) + except (ValueError, UnicodeError): + _fail("state_conflict") + + +def _unique_json_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def load_plan(state_dir) -> dict: + """Read/validate the persisted allocation before considering a retry.""" + state_dir = _private_directory(state_dir) + result = seal_plan(_read_json(state_dir / "plan.json")) + _check_overlap(result, state_dir) + return result + + +def persist_plan(plan: dict, state_dir) -> dict: + plan = seal_plan(plan) + _check_overlap(plan, state_dir) + state_dir = _private_directory(state_dir, create=True) + _publish_once(state_dir / "plan.json", _canonical(plan)) + persisted = load_plan(state_dir) + if persisted != plan: + _fail("state_conflict") + return persisted + + +def verify_inputs(plan: dict): + plan = seal_plan(plan) + for path, expected in plan["inputs"].items(): + if fingerprint_file(path) != expected: + _fail("input_changed") + for expected in plan.get("inventory_groups", []): + current = (inventory_directories(expected["path"]) if expected.get("kind") == "directories" + else inventory_group(expected["path"], expected["suffixes"])) + if current != expected: + _fail("inventory_changed") + return True + + +def _snapshot_metadata(plan, snapshot, *, create=False): + """Persist creation identity once, including across interrupted copies. + + The timestamp is separate from the pre-existing sealed plan: snapshot + creation never silently changes that plan's digest or UUID allocation. + """ + path = snapshot / "metadata.json" + fingerprint, _ = _read_file(path, private=True) + if not fingerprint["exists"]: + if not create: + _fail("snapshot_incomplete") + # Once copying/commit has begun, a missing header is lost evidence, + # not permission to assign a fresh creation time to existing bytes. + if fingerprint_file(snapshot / "manifest.json")["exists"]: + _fail("snapshot_incomplete") + with _directory(snapshot / "files") as directory: + if os.listdir(directory): + _fail("snapshot_incomplete") + metadata = {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], + "created_at": datetime.now(timezone.utc).isoformat()} + _publish_once(path, _canonical(metadata)) + metadata = _read_json(path) + if not isinstance(metadata, dict) or not isinstance(metadata.get("created_at"), str): + _fail("invalid_snapshot") + try: + timestamp = datetime.fromisoformat(metadata["created_at"]) + except (ValueError, TypeError): + _fail("invalid_snapshot") + if timestamp.tzinfo != timezone.utc or timestamp.isoformat() != metadata["created_at"]: + _fail("invalid_snapshot") + expected = {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], "created_at": metadata["created_at"]} + if _canonical(metadata) != _canonical(expected): + _fail("invalid_snapshot") + return metadata + + +def _snapshot_manifest(plan, metadata): + entries = {} + for path, expected in plan["inputs"].items(): + entries[path] = {"artifact_kind": "file", "original": expected, + "blob": hashlib.sha256(path.encode("utf-8")).hexdigest() + ".bin" + if expected["exists"] else None} + external = {} + for name, item in plan.get("external_inputs", {}).items(): + raw = item["text"].encode("utf-8") + external[name] = {"artifact_kind": "external", "kind": item["kind"], "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + "blob": "external-" + name + ".bin"} + return {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], "entries": entries, + "created_at": metadata["created_at"], "id_map": plan["id_map"], + # Deliberately exclude projected `data` and any other action + # payload: secrets/configuration bytes belong only in private + # original blobs and the protected plan, never duplicated here. + "actions": [{key: action[key] for key in ("id", "kind", "source", "target") + if key in action} for action in plan.get("actions", [])], + "external_inputs": external, + "inventory_groups": plan.get("inventory_groups", [])} + + +def _snapshot_requirements(plan): + prerequisites = plan.get("prerequisites", {}) + if not isinstance(prerequisites, dict): + _fail("invalid_plan") + if "managed_cron_captured" in prerequisites and prerequisites["managed_cron_captured"] is not True: + _fail("snapshot_incomplete") + if prerequisites.get("managed_cron_captured") is True and "managed_cron" not in plan.get("external_inputs", {}): + _fail("snapshot_incomplete") + + +def _snapshot_contents(snapshot, *, complete): + """Unknown interrupted staging files are evidence, never ignored inputs.""" + with _directory(snapshot) as directory: + names = set(os.listdir(directory)) + expected = {"metadata.json", "manifest.json", "files"} + if names - expected or complete and names != expected: + _fail("snapshot_incomplete") + + +def create_snapshot(plan: dict, state_dir) -> dict: + """Copy exact original inputs, including existing destination originals. + + A partially copied snapshot can be completed only with the same persisted + plan, unchanged inputs and already-valid blobs. Corrupt blobs are never + overwritten. There is deliberately no export or restoration endpoint. + """ + plan = seal_plan(plan) + _snapshot_requirements(plan) + plan = persist_plan(plan, state_dir) + verify_inputs(plan) + state_dir = _private_directory(state_dir) + snapshot = _private_directory(state_dir / "snapshot", create=True) + blobs = _private_directory(snapshot / "files", create=True) + _snapshot_contents(snapshot, complete=False) + with _directory(blobs) as directory: + free = os.fstatvfs(directory) + required = sum(item["size"] for item in plan["inputs"].values() if item["exists"]) + required += sum(len(item["text"].encode("utf-8")) for item in plan.get("external_inputs", {}).values()) + if free.f_bavail * free.f_frsize < required + 65536: + _fail("insufficient_space") + metadata = _snapshot_metadata(plan, snapshot, create=True) + manifest = _snapshot_manifest(plan, metadata) + for path, entry in manifest["entries"].items(): + actual, content = _read_file(path) + if actual != entry["original"]: + _fail("input_changed") + if entry["blob"] is not None: + _publish_once(blobs / entry["blob"], content) + for name, entry in manifest["external_inputs"].items(): + _publish_once(blobs / entry["blob"], plan["external_inputs"][name]["text"].encode("utf-8")) + verify_inputs(plan) + _publish_once(snapshot / "manifest.json", _canonical(manifest)) + handle = {"path": str(snapshot), "plan_id": plan["plan_id"], "digest": _digest(manifest)} + verify_snapshot(plan, handle) + return handle + + +def verify_snapshot(plan: dict, snapshot: dict) -> dict: + """Verify completeness, binding and each private stored byte independently.""" + plan = seal_plan(plan) + _snapshot_requirements(plan) + if not isinstance(snapshot, dict) or set(snapshot) != {"path", "plan_id", "digest"}: + _fail("invalid_snapshot") + path = _absolute(snapshot["path"]) + if path.name != "snapshot" or snapshot["plan_id"] != plan["plan_id"]: + _fail("invalid_snapshot") + _private_directory(path) + _snapshot_contents(path, complete=True) + if load_plan(path.parent) != plan: + _fail("invalid_snapshot") + metadata = _snapshot_metadata(plan, path) + manifest = _read_json(path / "manifest.json") + if _canonical(manifest) != _canonical(_snapshot_manifest(plan, metadata)) or _digest(manifest) != snapshot["digest"]: + _fail("invalid_snapshot") + blobs = _private_directory(path / "files") + expected_names = {entry["blob"] for entry in manifest["entries"].values() if entry["blob"]} + expected_names.update(entry["blob"] for entry in manifest["external_inputs"].values()) + with _directory(blobs) as directory: + # Interrupted private staging files are not recovery blobs. They carry + # no authority; committed snapshots must contain exactly their blobs. + if set(os.listdir(directory)) != expected_names: + _fail("snapshot_incomplete") + for entry in manifest["entries"].values(): + if entry["blob"]: + actual, _ = _read_file(blobs / entry["blob"], private=True) + original = entry["original"] + if not actual["exists"] or (actual["size"], actual["sha256"]) != (original["size"], original["sha256"]): + _fail("snapshot_changed") + for entry in manifest["external_inputs"].values(): + actual, _ = _read_file(blobs / entry["blob"], private=True) + if not actual["exists"] or (actual["size"], actual["sha256"]) != (entry["size"], entry["sha256"]): + _fail("snapshot_changed") + return manifest + + +def verify_preconditions(plan: dict, snapshot: dict, confirmation=None, *, + quiescence_check: Callable[[], bool] | None = None, + external_input_check: Callable[[], dict] | None = None) -> bool: + """Default-deny *library* gate; this neither applies data nor trusts a UI. + + Phase #479 must supply the real writer/maintenance check and authenticated + confirmation. Checking immediately here cannot replace locks held across + the eventual apply transaction. An acknowledgement is not external-copy + verification, and this function must never be described as such. + """ + plan = seal_plan(plan) + if (plan.get("classification") != "applicable" or plan.get("required") is not True + or plan.get("status") != "pending"): + _fail("invalid_plan") + prerequisites = plan.get("prerequisites") + if not isinstance(prerequisites, dict) or prerequisites.get("managed_cron_captured") is not True: + _fail("snapshot_incomplete") + _snapshot_requirements(plan) + if not isinstance(confirmation, dict) or confirmation.get("approved") is not True: + _fail("approval_required") + if (confirmation.get("independent_backup_acknowledged") is not True + or confirmation.get("plan_id") != plan["plan_id"] + or not isinstance(snapshot, dict) + or confirmation.get("snapshot_digest") != snapshot.get("digest")): + _fail("approval_required") + if quiescence_check is None: + _fail("writers_active") + try: + quiescent = quiescence_check() + except Exception: + _fail("writers_active") + if quiescent is not True: + _fail("writers_active") + verify_snapshot(plan, snapshot) + verify_inputs(plan) + if plan.get("external_inputs"): + if external_input_check is None: + _fail("input_changed") + try: + current = external_input_check() + except Exception: + _fail("input_changed") + if current != plan["external_inputs"]: + _fail("input_changed") + return True + + +def _journal_records(raw, plan): + records = [] + previous = None + if raw and not raw.endswith(b"\n"): + _fail("invalid_journal") + for line in raw.splitlines(): + try: + record = json.loads(line, object_pairs_hook=_unique_json_pairs) + except (ValueError, UnicodeError): + _fail("invalid_journal") + if not isinstance(record, dict): + _fail("invalid_journal") + expected = {"schema_version", "migration_id", "plan_id", "sequence", "timestamp", + "status", "phase", "reason_code", "action_ids", "previous", "digest"} + if (set(record) != expected or type(record["schema_version"]) is not int + or record["schema_version"] != 1 or record["migration_id"] != MIGRATION_ID): + _fail("invalid_journal") + if (record["plan_id"] != plan["plan_id"] or type(record["sequence"]) is not int + or record["sequence"] != len(records) + 1): + _fail("invalid_journal") + try: + timestamp = datetime.fromisoformat(record["timestamp"]) + if timestamp.utcoffset() is None: + _fail("invalid_journal") + except (ValueError, TypeError): + _fail("invalid_journal") + if (not isinstance(record["status"], str) or record["status"] not in STATUSES + or not isinstance(record["phase"], str) or record["phase"] not in PHASES): + _fail("invalid_journal") + if record["reason_code"] is not None and (not isinstance(record["reason_code"], str) or record["reason_code"] not in REASON_CODES): + _fail("invalid_journal") + known_actions = {action["id"] for action in plan.get("actions", [])} + if not isinstance(record["action_ids"], list) or any(not isinstance(item, str) or item not in known_actions for item in record["action_ids"]): + _fail("invalid_journal") + if record["previous"] != previous: + _fail("invalid_journal") + unsigned = dict(record) + claimed = unsigned.pop("digest") + if claimed != _digest(unsigned): + _fail("invalid_journal") + previous = claimed + records.append(record) + return records + + +def read_journal(state_dir) -> list: + state_dir = _private_directory(state_dir) + plan = load_plan(state_dir) + # append_journal may need multiple writes for one JSONL record. Status + # readers share its flock so a live append is never mistaken for corrupt + # recovery evidence. A genuinely torn record after process exit still + # fails validation; this does not repair or discard journal bytes. + with _directory(state_dir) as directory: + fd = None + try: + try: + fd = os.open("journal.jsonl", os.O_RDONLY | _NOFOLLOW | os.O_NONBLOCK, + dir_fd=directory) + except FileNotFoundError: + return [] + fcntl.flock(fd, fcntl.LOCK_SH) + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 + or info.st_uid != os.getuid() or info.st_nlink != 1): + _fail("unsafe_path") + chunks = [] + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + return _journal_records(b"".join(chunks), plan) + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) + + +def append_journal(state_dir, plan: dict, status: str, phase: str, *, + reason_code=None, action_ids=None) -> dict: + """Append a private, fsynced, hash-linked event without free-form errors.""" + plan = seal_plan(plan) + if (not isinstance(status, str) or status not in STATUSES + or not isinstance(phase, str) or phase not in PHASES + or reason_code is not None and (not isinstance(reason_code, str) or reason_code not in REASON_CODES)): + _fail("invalid_journal") + action_ids = [] if action_ids is None else action_ids + known_actions = {action["id"] for action in plan.get("actions", [])} + if not isinstance(action_ids, list) or any(not isinstance(item, str) or item not in known_actions for item in action_ids): + _fail("invalid_journal") + state_dir = _private_directory(state_dir) + if load_plan(state_dir) != plan: + _fail("state_conflict") + with _directory(state_dir) as directory: + fd = None + try: + fd = os.open("journal.jsonl", os.O_RDWR | os.O_APPEND | os.O_CREAT | _NOFOLLOW, + 0o600, dir_fd=directory) + fcntl.flock(fd, fcntl.LOCK_EX) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_uid != os.getuid() or info.st_nlink != 1: + _fail("unsafe_path") + raw = b"" + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + raw += chunk + records = _journal_records(raw, plan) + record = {"schema_version": 1, "migration_id": MIGRATION_ID, + "plan_id": plan["plan_id"], "sequence": len(records) + 1, + "timestamp": datetime.now(timezone.utc).isoformat(), + "status": status, "phase": phase, "reason_code": reason_code, + "action_ids": action_ids, + "previous": records[-1]["digest"] if records else None} + record["digest"] = _digest(record) + data = memoryview(_canonical(record) + b"\n") + while data: + written = os.write(fd, data) + if written <= 0: + _fail("storage_unavailable") + data = data[written:] + os.fsync(fd) + os.fsync(directory) + return record + except IdentityStorageError: + raise + except OSError as exc: + _os_error(exc) + finally: + if fd is not None: + os.close(fd) diff --git a/api/migrations/immutable_job_id_activation.py b/api/migrations/immutable_job_id_activation.py new file mode 100644 index 00000000..958116f8 --- /dev/null +++ b/api/migrations/immutable_job_id_activation.py @@ -0,0 +1,17 @@ +"""Startup detection is not consent to prepare or apply immutable IDs (#479).""" +MIGRATION_ID = "immutable_job_id_v1" +INTRODUCED_IN = "issue-447" +RECHECK_AFTER_FINAL = True +USER_INITIATED = True + + +def detect(config): + from identity_migration_api import get_assistant + return get_assistant(config).startup_detection() + + +def apply(config): + # The central runner can record the gate but cannot authorize conversion. + detected = detect(config) + return {"migration_id": MIGRATION_ID, "status": "blocked" if detected["status"] == "blocked" else "pending", + "details": {"reason": "Explicit migration preparation and separate approval required."}} diff --git a/api/migrations/immutable_job_id_v1.py b/api/migrations/immutable_job_id_v1.py new file mode 100644 index 00000000..57018aca --- /dev/null +++ b/api/migrations/immutable_job_id_v1.py @@ -0,0 +1,742 @@ +"""Inactive, read-only identity migration planner (#472). + +Deliberately not registered, imported by startup, exposed by HTTP, or given an +apply() entry point. Proposed JSON replacements are private planning data; +only the separate snapshot/journal utilities may write a dedicated state dir. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +import os +from pathlib import Path +import re +import stat +from uuid import UUID, uuid4 + +from . import identity_storage as storage +from .identity_records import project_records, verify_records + + +MIGRATION_ID = "immutable_job_id_v1" +INTRODUCED_IN = "pending-issue-447" +MAX_JSON_BYTES = 64 * 1024 * 1024 +_KEY = re.compile(r"^[A-Za-z0-9_.-]+$") +_TYPE = re.compile(r"^[a-z0-9_]+$") +_LOCATIONS = {"local", "usb", "smb", "storagebox", "custom"} +_LEGACY = {"job_key", "backup_type", "type_id", "location"} + + +class PlanningError(ValueError): + def __init__(self, code, source=""): + self.code, self.source = code, str(source) + super().__init__(code) # Never interpolate raw JSON, config or errors. + + +def _fail(code, source=""): + raise PlanningError(code, source) + + +def _uuid(value): + try: + parsed = UUID(value) if isinstance(value, str) else None + except ValueError: + parsed = None + return parsed is not None and parsed.version == 4 and str(parsed) == value + + +def _path(value): + if not isinstance(value, (str, Path)): + _fail("unsafe_path") + raw = str(value) + if not raw.startswith("/") or raw != os.path.normpath(raw) or raw.startswith("//"): + _fail("unsafe_path") + if any(c in raw for c in ("\x00", "\n", "\r", "$")) or raw == "/": + _fail("unsafe_path") + path = Path(raw) + parts = path.parts + mount = None + if len(parts) > 2 and parts[1] == "mnt": + mount = Path(*parts[:4]) if parts[2] in {"disks", "remotes"} and len(parts) > 3 else Path(*parts[:3]) + elif len(parts) > 1 and parts[1] == "boot": + mount = Path("/boot") + if mount is not None and not mount.is_mount(): + _fail("required_mount_unavailable", path) + return path + + +def _strict_json(raw, source): + def pairs(items): + result = {} + for key, value in items: + if key in result: + _fail("duplicate_json_member", source) + result[key] = value + return result + if len(raw) > MAX_JSON_BYTES: + _fail("owned_input_too_large", source) + try: + return json.loads(raw, object_pairs_hook=pairs, + parse_constant=lambda _: _fail("invalid_json", source)) + except (UnicodeError, json.JSONDecodeError, RecursionError): + _fail("invalid_json", source) + + +def _read_conf(raw): + """Same literal decoding/forward-reference semantics as status.load_config. + + No shell, environment expansion or import of lazy runtime helpers. Only + relevant non-secret values are copied into proposed canonical metadata. + """ + values = {} + try: + lines = raw.decode("utf-8").splitlines() + except UnicodeError: + _fail("invalid_configuration_encoding") + for line in lines: + line = line.strip().removeprefix("readonly ") + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key, value = key.strip(), value.strip() + if value.startswith('"'): + try: + decoded, end = json.JSONDecoder().raw_decode(value) + tail = value[end:].strip() + if isinstance(decoded, str) and (not tail or tail.startswith("#")): + value = decoded + except json.JSONDecodeError: + pass + elif value.startswith("'") and value.rfind("'") > 0: + end = value.rfind("'") + tail = value[end + 1:].strip() + if not tail or tail.startswith("#"): + value = value[1:end] + else: + value = value.split(" #", 1)[0].rstrip() + if value.startswith("("): + continue + values[key] = re.sub(r"\$\{([^}]+)\}", lambda m: values.get(m[1], m[0]), value) + return values + + +class Inventory: + def __init__(self): + self.inputs, self.groups, self.records, self.raw = {}, [], {}, {} + + def file(self, path, kind=None, **metadata): + path = _path(path) + name = str(path) + fingerprint, raw = storage.read_fingerprinted_file(path) + previous = self.inputs.get(name) + if previous is not None and previous != fingerprint: + _fail("source_fingerprint_changed", path) + self.inputs[name] = fingerprint + if raw is None: + return None + self.raw[name] = raw + if kind is not None: + value = _strict_json(raw, path) + if not isinstance(value, dict): + _fail("invalid_job_shape" if kind == "job" else "invalid_store_shape", path) + old = self.records.get(name) + if old and old["kind"] != kind: + _fail("overlapping_owned_stores", path) + self.records[name] = {"kind": kind, "data": value, **metadata} + return value + return raw + + def group(self, directory, suffix, kind, **metadata): + directory = _path(directory) + group = storage.inventory_group(directory, [suffix]) + if group not in self.groups: + self.groups.append(group) + for name in group["entries"]: + self.file(directory / name, kind, **metadata) + return group + + +def _inventory(config, control_root): + scan = Inventory() + data = _path(config.get("BACKUP_SCRIPTS_DIR", "/boot/config/borg-backup")) + if data.name == "scripts": + data = data.parent + conf = scan.file(data / "config/backup.conf") + expanded = _read_conf(conf) if conf is not None else {} + effective = dict(config) + for key in ("STATUS_DIR", "RESTORE_TEST_STATUS_DIR", "BORG_RESOURCE_LOCK_DIR", + "RUNTIME_RECOVERY_FILE", "UNRAID_DASHBOARD_WIDGET_FILE"): + if expanded.get(key): + effective[key] = expanded[key] + plugin = _path(effective.get("PLUGIN_DIR") or "/boot/config/plugins/borg-backup-ui") + jobs_dir = data / "config/jobs" + scan.group(jobs_dir, ".json", "job") + # Known lazy-migration locations only; never traverse runtime/vendor/recycle bins. + scripts = _path(effective.get("BORG_SCRIPTS_DIR") or str(data / "scripts")) + for legacy in sorted({scripts / "config/jobs", plugin / "runtime/config/jobs"} - {jobs_dir}): + scan.group(legacy, ".json", "job", legacy_directory=True) + singleton = { + "repositories.json": "repositories", "storages.json": "storages", + "schedules.json": "schedules", "restore-runs.json": "restore_runs", + "restore-history/index.json": "restore_index", + "notification-queue.json": "notification_queue", + "notification-deliveries.json": "notification_deliveries", + "notification-state.json": "notification_state", + } + for filename, kind in singleton.items(): + scan.file(data / "config" / filename, kind) + scan.file(_path(effective.get("RUNTIME_RECOVERY_FILE") or str(data / "config/runtime-recovery.json")), "runtime_recovery") + scan.group(data / "config/restore-history/runs", ".json", "restore_detail") + status_dir = _path(effective.get("STATUS_DIR") or "/mnt/user/backup-status") + scan.group(status_dir, ".status", "status") + weekly = _path(effective.get("SNAPSHOT_FILE") or str(status_dir.parent / "weekly-snapshots.json")) + for candidate in sorted({weekly, status_dir / "weekly-snapshots.json"}): + scan.file(candidate, "weekly", target_path=str(weekly)) + candidates = [] + if effective.get("RESTORE_TEST_STATUS_DIR"): + candidates.append(_path(effective["RESTORE_TEST_STATUS_DIR"])) + candidates += [status_dir.parent / "restore-status", status_dir / "restore-tests"] + # Scan each known location: hidden stale results are not silently abandoned. + for directory in dict.fromkeys(candidates): + scan.group(directory, ".test", "restore_test") + lock_dir = _path(effective.get("BORG_RESOURCE_LOCK_DIR") or str(data / "locks")) + scan.group(lock_dir, ".json", "resource_lock") + # This independent worker lock does not prove quiescence; preserve/capture it. + scan.file(data / "locks/notification-delivery.lock") + controls = _path(control_root or "/run/borg-backup-ui/jobs") + children = storage.inventory_directories(controls) + scan.groups.append(children) + for run in children["entries"]: + group = scan.group(controls / run, ".json", "control") + for name in group["entries"]: + path = str(controls / run / name) + if name == "cancel.request.json": + scan.records[path]["kind"] = "cancel_request" + elif name == "context.json": + scan.records[path]["kind"] = "run_context" + elif name != "state.json": + _fail("unknown_control_file", path) + widget = _path(effective.get("UNRAID_DASHBOARD_WIDGET_FILE") or str(plugin / "widget-status.json")) + scan.file(widget, "widget_cache") + return scan, data, jobs_dir, expanded + + +def _prefixes(meta, legacy, source): + values = meta.get("archive_prefixes", []) + if not isinstance(values, list) or any(not isinstance(v, str) or not _KEY.fullmatch(v) + or v in {".", ".."} for v in values): + _fail("invalid_archive_prefix", source) + if legacy and any(not re.fullmatch(r"[A-Za-z0-9_.-]+-backup", value) for value in values): + # The old reader silently ignored these. Adopting them would expand + # archive/prune ownership; dropping them would discard user data. + _fail("invalid_archive_prefix", source) + if not legacy and (not values or len(set(values)) != len(values)): + _fail("invalid_archive_prefix", source) + return list(dict.fromkeys(([meta["backup_type"] + "-backup"] if legacy else []) + values)) + + +def _validate_job(meta, source): + schema = meta.get("schema_version") + if type(schema) is not int or schema not in {1, 2, 3, 4}: + _fail("unsupported_schema", source) + if schema == 4: + if not _uuid(meta.get("job_id")): + _fail("invalid_job_id", source) + if _LEGACY.intersection(meta): + _fail("mutable_canonical_identity", source) + if "legacy_job_keys" not in meta: + _fail("invalid_legacy_alias", source) + else: + typ, location, key = meta.get("backup_type"), meta.get("location"), meta.get("job_key") + if not isinstance(typ, str) or not _TYPE.fullmatch(typ) or location not in _LOCATIONS: + _fail("conflicting_legacy_identity", source) + if key != f"{typ}_{location}" or Path(source).stem != key or "job_id" in meta: + _fail("conflicting_legacy_identity", source) + aliases = meta.get("legacy_job_keys", []) + if not isinstance(aliases, list) or any(not isinstance(a, str) or not _KEY.fullmatch(a) for a in aliases): + _fail("invalid_legacy_alias", source) + if len(set(aliases)) != len(aliases): + _fail("duplicate_legacy_alias", source) + _prefixes(meta, schema != 4, source) + if not isinstance(meta.get("name"), str) or not meta["name"].strip(): + _fail("invalid_job_name", source) + if not isinstance(meta.get("repository_key"), str) or not _KEY.fullmatch(meta["repository_key"]): + _fail("dangling_repository", source) + + +def _operational_defaults(meta, conf, source): + result = deepcopy(meta) + # Existing pure path converter does no writes. Ambiguous old strings may + # inspect source directories, but never silently split nonexistent paths. + try: + from ..job_source_paths import normalize_source_paths, upgrade_job_source_paths + except ImportError: + from job_source_paths import normalize_source_paths, upgrade_job_source_paths + try: + if meta["schema_version"] in {1, 2}: + result = upgrade_job_source_paths(result, job_key=meta["job_key"]) + elif normalize_source_paths(meta.get("source_paths")) != meta.get("source_paths"): + _fail("noncanonical_source_paths", source) + except ValueError: + _fail("invalid_source_paths", source) + if meta["schema_version"] == 4: + return result + from job_presentation import legacy_presentation_defaults + result.update(legacy_presentation_defaults(meta)) + tu = meta["backup_type"].upper() + # Preserve the runner's existing cache namespace and check marker. These + # paths are references, not identities, and are never renamed or deleted. + cache_dir = conf.get("BORG_CACHE_DIR") or str( + Path(conf.get("GLOBAL_BORG_CACHE_BASE") or "/mnt/cache/borg-cache") + / (meta["location"] + "_" + meta["backup_type"]) + ) + if "cache_reference" in result: + _fail("ambiguous_cache_reference", source) + result["cache_reference"] = { + "repository_key": meta["repository_key"], + "directory": cache_dir, + "check_flag_file": conf.get("BORG_CHECK_FLAG_FILE") or str(Path(cache_dir) / (".last_check_" + meta["backup_type"])), + } + # Env overrides unrelated to job metadata cannot be inferred safely. The + # startup coordinator must supply the actual expanded backup.conf, captured + # here, rather than invoking the runner which mutates process environment. + if not result.get("compression"): + result["compression"] = conf.get(f"COMPRESSION_{tu}", "lz4") + retention = result.get("retention", {}) + if not isinstance(retention, dict): + _fail("invalid_retention", source) + retention = deepcopy(retention) + if any(not isinstance(value, str) for value in retention.values()): + # Numeric zero follows a different truthiness path in the legacy + # runner. Neither silently filling defaults nor guessing intent is safe. + _fail("ambiguous_retention_shape", source) + for key, default in {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}.items(): + if not str(retention.get(key) or "").strip(): + retention[key] = conf.get(f"RETENTION_{tu}_{key.upper()}", default) + result["retention"] = retention + for kind in ("docker", "vm"): + key = kind + "_control" + if key not in result: + features = result.get("features", {}) + if not isinstance(features, dict) or type(features.get(kind, False)) is not bool: + _fail("invalid_runtime_control", source) + result[key] = {"mode": "all" if features.get(kind) else "none", "selected": [], + "ack_appdata_risk" if kind == "docker" else "ack_domains_risk": False} + elif not isinstance(result[key], dict): + _fail("invalid_runtime_control", source) + return result + + +def _plan_jobs(scan, jobs_dir, conf, allocator, journal): + jobs, aliases, sources, seen_legacy, canonical_ids = {}, {}, {}, set(), set() + rows = [(path, row["data"]) for path, row in sorted(scan.records.items()) if row["kind"] == "job"] + for source, meta in rows: + _validate_job(meta, source) + if meta["schema_version"] == 4: + job_id = meta["job_id"] + if job_id in canonical_ids: + _fail("duplicate_job_id", source) + canonical_ids.add(job_id) + for source, meta in rows: + legacy = meta["schema_version"] != 4 + if legacy: + key = meta["job_key"] + if any(alias != key for alias in meta.get("legacy_job_keys", [])) and journal is None: + _fail("unproven_legacy_alias", source) + if key in seen_legacy: + _fail("duplicate_legacy_identity", source) + seen_legacy.add(key) + proposed = journal.get("id_map", {}).get(key) if journal else None + if proposed is None: + try: + proposed = str(allocator()) + except Exception: + _fail("uuid_allocation_failed", source) + if not _uuid(proposed) or proposed in jobs or proposed in canonical_ids: + _fail("duplicate_job_id" if _uuid(proposed) else "invalid_job_id", source) + job_id = proposed + else: + job_id = meta["job_id"] + if Path(source) != jobs_dir / (job_id + ".json"): + _fail("noncanonical_metadata_filename", source) + target = _operational_defaults(meta, conf, source) + target["archive_prefixes"] = _prefixes(meta, legacy, source) + target["legacy_job_keys"] = list(dict.fromkeys(([meta["job_key"]] if legacy else []) + meta.get("legacy_job_keys", []))) + for alias in target["legacy_job_keys"]: + if alias in aliases: + _fail("duplicate_legacy_alias", source) + aliases[alias] = job_id + for key in _LEGACY: + target.pop(key, None) + target.update(schema_version=4, job_id=job_id) + # Share the target model's validation, but never its creation/write path. + try: + try: + from ..job_model import validate_job + except ImportError: + from job_model import validate_job + validate_job(target, filename=job_id + ".json") + except ValueError as exc: + _fail(getattr(exc, "api_code", "invalid_job_settings"), source) + jobs[job_id], sources[job_id] = target, source + if seen_legacy and canonical_ids: + # A mixed on-disk cutover is not a new installation to re-plan. The + # original persisted mapping/snapshot is required, even if aliases + # could currently resolve all remaining references. + _fail("partial_migration_without_journal") + return jobs, aliases, sources, bool(seen_legacy) + + +def _check_repositories(scan, jobs, aliases): + def collection(kind, name, key): + selected = [r["data"] for r in scan.records.values() if r["kind"] == kind] + if not selected: + return {} + raw = selected[0] + if type(raw.get("schema_version")) is not int or raw["schema_version"] != 1 or not isinstance(raw.get(name), list): + _fail("unsupported_store_schema") + result = {} + for row in raw[name]: + if not isinstance(row, dict) or not isinstance(row.get(key), str) or not _KEY.fullmatch(row[key]): + _fail("invalid_inventory_entry") + if row[key] in result: + _fail("duplicate_inventory_key") + result[row[key]] = row + return result + repos = collection("repositories", "repositories", "repository_key") + storages = collection("storages", "storages", "storage_key") + for job in jobs.values(): + if job["repository_key"] not in repos: + _fail("dangling_repository") + for key, repo in repos.items(): + if repo.get("storage_key") not in storages: + _fail("dangling_storage") + expected = {job_id for job_id, job in jobs.items() if job["repository_key"] == key} + for field in ("used_by", "source_job_keys", "job_ids", "source_job_ids"): + if field not in repo: + continue + values = repo[field] + if not isinstance(values, list) or any(not isinstance(v, str) for v in values): + _fail("invalid_repository_assignments") + mapped = [value if value in jobs else aliases.get(value) for value in values] + if None in mapped or set(mapped) != expected or len(set(mapped)) != len(mapped): + _fail("conflicting_repository_assignments") + for field in ("passphrase_ref", "keyfile_ref"): + if repo.get(field): + path = _path(repo[field]) + # Existence/type only; never read secret contents into the plan. + for parent in [*reversed(path.parents), path]: + if parent.is_symlink(): + _fail("unsafe_secret_reference") + try: + info = path.stat() + except OSError: + _fail("missing_secret_reference") + if not stat.S_ISREG(info.st_mode): + _fail("unsafe_secret_reference") + # Delimiter overlap also matters (p-* includes p-long-*). + ownership = [] + for job_id, job in jobs.items(): + for prefix in job["archive_prefixes"]: + for other_repo, other_prefix, other_id in ownership: + if other_repo == job["repository_key"] and other_id != job_id and ( + prefix == other_prefix or prefix.startswith(other_prefix + "-") or other_prefix.startswith(prefix + "-") + ): + _fail("ambiguous_archive_ownership") + ownership.append((job["repository_key"], prefix, job_id)) + + +def _check_live_owners(scan): + for source, record in scan.records.items(): + data = record["data"] + rows = data.get("entries", []) if record["kind"] == "runtime_recovery" else [data] + if record["kind"] not in {"runtime_recovery", "resource_lock", "control"}: + continue + if not isinstance(rows, list): + _fail("invalid_runtime_state", source) + for row in rows: + if not isinstance(row, dict): + _fail("invalid_runtime_state", source) + if row.get("finished") is True: + continue + pid = row.get("pid") + if pid is None: + continue # Shape/identity validation belongs to the projector. + if type(pid) is not int or pid <= 0: + _fail("invalid_runtime_pid", source) + try: + os.kill(pid, 0) + except ProcessLookupError: + continue + except (PermissionError, OSError): + _fail("writers_not_quiescent", source) + else: + _fail("writers_not_quiescent", source) + + +def _changed_active(scan, projected): + for record in scan.records.values(): + kind, data = record["kind"], record["data"] + if kind == "schedules" and any(key != "restore_test" and not _uuid(key) for key in data): + return True + if kind == "repositories" and any("used_by" in r or "source_job_keys" in r for r in data.get("repositories", [])): + return True + if projected.get("required"): + return True + for binding in projected["bindings"]: + if binding["job_id"] is None: + continue + record = scan.records[binding["source"]] + if record["kind"] in {"schedules", "repositories", "notification_state"}: + continue + row = record["data"] + for part in binding["locator"].split("/")[1:]: + part = part.replace("~1", "/").replace("~0", "~") + row = row[int(part)] if isinstance(row, list) else row[part] + if not isinstance(row, dict) or row.get("job_id") != binding["job_id"]: + return True + if binding["role"] == "active" and "job_key" in row: + return True + if record["kind"] == "restore_test" and Path(binding["source"]).stem != binding["job_id"]: + return True + return False + + +def encode_target_json(value): + """Frozen planned JSON encoding; the future applier must use these bytes.""" + return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=True, allow_nan=False) + "\n").encode("utf-8") + + +def _resume_check(scan, journal): + if journal is None: + return None + try: + saved = storage.seal_plan(journal) + if saved.get("plan_id") != journal.get("plan_id") or saved.get("classification") != "applicable": + _fail("invalid_migration_journal") + # Check the COMPLETE saved footprint, not just still-discoverable files. + # A vanished source is only a valid retirement when its exact canonical + # replacement is present. JSON-equivalent edits/chmod are not accepted. + replacements = {a["target"]: a.get("after") for a in saved.get("actions", []) + if a.get("kind") in {"write_json", "write_bytes"}} + retired = {a["source"]: a["target"] for a in saved.get("actions", []) + if a.get("kind") in {"retire_source", "retire_auxiliary"}} + derived = {a["source"] for a in saved.get("actions", []) + if a.get("kind") == "rebuild_derived" and a.get("source") == a.get("target")} + actual_inputs = {path: storage.fingerprint_file(path) for path in saved["inputs"]} + if set(scan.inputs) - set(saved["inputs"]): + _fail("source_fingerprint_changed") + for path, actual in actual_inputs.items(): + if actual == saved["inputs"][path]: + continue + if (not actual["exists"] and path in retired + and actual_inputs.get(retired[path]) == replacements.get(retired[path]) + and replacements.get(retired[path]) is not None): + continue + if path in replacements and actual == replacements[path]: + continue + if path in derived and not actual["exists"]: + continue + _fail("source_fingerprint_changed", path) + current_groups = {g["path"]: g for g in scan.groups} + if set(current_groups) != {g["path"] for g in saved["inventory_groups"]}: + _fail("source_fingerprint_changed") + for group in saved["inventory_groups"]: + current = current_groups[group["path"]] + if group.get("kind") == "directories": + if current != group: + _fail("source_fingerprint_changed", group["path"]) + continue + # The explicit apply engine may publish a previously absent jobs + # directory from a legacy-only metadata location. Its private + # directory receipts verify the original/new filesystem identity; + # this read-only check additionally requires exact planned members. + created_target_group = (not group["exists"] and current["exists"] + and any(str(Path(path).parent) == group["path"] + for path in replacements)) + if (not created_target_group + and {k: v for k, v in current.items() if k != "entries"} + != {k: v for k, v in group.items() if k != "entries"}): + _fail("source_fingerprint_changed", group["path"]) + expected_names = {Path(path).name for path, fp in actual_inputs.items() + if str(Path(path).parent) == group["path"] and fp["exists"] + and Path(path).name.endswith(tuple(group["suffixes"]))} + if set(current["entries"]) != expected_names: + _fail("source_fingerprint_changed", group["path"]) + return saved + except storage.IdentityStorageError: + _fail("invalid_migration_journal") + + +def build_plan(config, *, uuid_factory=uuid4, journal_plan=None, control_root=None, cron_text=None): + """Return a proposed complete mapping without changing any installation file. + + Pass a plan loaded/validated from the private journal to reuse allocated + IDs. Fresh dry runs propose IDs; only persist_plan makes that mapping + durable. No caller-supplied boolean can authorize application here. + """ + scan = None + try: + scan, data, jobs_dir, conf = _inventory(config, control_root) + journal = _resume_check(scan, journal_plan) + if journal is not None: + # Secret contents are not migration inputs, but their referenced + # existence/type must still be checked on a resumed plan. + _check_repositories(scan, journal["jobs"], journal["aliases"]) + _check_live_owners(scan) + # Keep the original plan ID, complete original snapshot footprint, + # and UUID map. Do not re-plan from half-converted stores or produce + # a new snapshot of already converted data after an interruption. + return journal + jobs, aliases, sources, legacy = _plan_jobs(scan, jobs_dir, conf, uuid_factory, journal) + _check_repositories(scan, jobs, aliases) + _check_live_owners(scan) + records = {p: deepcopy(r) for p, r in scan.records.items() if r["kind"] not in {"job", "storages"}} + for path, row in records.items(): + if row["kind"] == "restore_test": + key = Path(path).stem + row["legacy_key"] = key + job_id = key if key in jobs else aliases.get(key) + if job_id: + row["target_path"] = str(Path(path).with_name(job_id + ".test")) + projected = project_records(records, jobs, aliases) + reasons = projected.get("reasons", []) + fatal = [r for r in reasons if r.get("severity") != "warning" and r["code"] != "weekly_value_conflict_preserved"] + if fatal: + return _blocked(fatal, scan) + mutable_active = _changed_active(scan, projected) + if jobs and not legacy and mutable_active: + _fail("partial_migration_without_journal") + required = legacy or mutable_active or bool(journal) + actions = [] + destinations = {} + def write(source, target, payload): + target = str(target) + existing = destinations.get(target) + if existing is not None and existing != payload: + _fail("conflicting_destination", target) + destinations[target] = payload + scan.file(target) + if source != target and scan.inputs[target]["exists"]: + if target not in scan.records or scan.records[target]["data"] != payload: + _fail("destination_already_exists", target) + if source == target and source in scan.records and scan.records[source]["data"] == payload: + return + encoded = encode_target_json(payload) + mode = scan.inputs[target].get("mode", scan.inputs[source].get("mode", 0o600)) + after = {"exists": True, "size": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(), "mode": mode} + actions.append({"kind": "write_json", "source": source, "target": target, + "data": payload, "after": after}) + if required: + for job_id, job in jobs.items(): + source = sources[job_id] + target = str(jobs_dir / (job_id + ".json")) + write(source, target, job) + if source != target: + actions.append({"kind": "retire_source", "source": source, "target": target}) + for target, row in projected["records"].items(): + origins = row.get("sources", [target]) + # Projector retains explicit source metadata on renamed records. + source = target if target in origins else row.get("source", origins[0]) + if source not in scan.inputs: + source = next((p for p, r in records.items() if r.get("target_path") == target), target) + if row["kind"] == "widget_cache": + actions.append({"kind": "rebuild_derived", "source": source, "target": target}) + continue + write(source, target, row["data"]) + for old in origins: + if old != target and old in scan.inputs: + actions.append({"kind": "retire_source", "source": old, "target": target}) + if source != target and source not in origins: + actions.append({"kind": "retire_source", "source": source, "target": target}) + # Derived caches deliberately have no projected payload: rebuilding + # must use the final verified graph, not copy old display keys. + for path, row in records.items(): + if row["kind"] == "widget_cache" and path not in projected["records"]: + actions.append({"kind": "rebuild_derived", "source": path, "target": path}) + for action in actions: + action["id"] = hashlib.sha256(json.dumps(action, sort_keys=True).encode()).hexdigest() + plan = { + "schema_version": 1, "migration_id": MIGRATION_ID, + "classification": "applicable" if required else "not_applicable", + "status": "pending" if required else "not_applicable", "required": bool(required), + "jobs": jobs, "aliases": aliases, "id_map": aliases, "job_sources": sources, + "inputs": scan.inputs, "inventory_groups": scan.groups, + "actions": actions, "records": projected["records"], + "bindings": projected.get("bindings", []), "unassigned": projected.get("unassigned", []), + "reasons": reasons + ([{"code": "resume_existing_mapping", "severity": "warning", "source": "", "locator": ""}] if journal else []), + "prerequisites": {"managed_cron_captured": isinstance(cron_text, str)}, + "external_inputs": {"managed_cron": {"kind": "crontab", "text": cron_text}} if isinstance(cron_text, str) else {}, + "activation_allowed": False, + } + # End-of-scan revalidation catches concurrent edits/additions, including + # destinations that were previously absent. Still no writer permission. + plan = storage.seal_plan(plan) + storage.verify_inputs(plan) + return plan + except PlanningError as exc: + return _blocked([{"code": exc.code, "source": exc.source, "locator": ""}], scan) + except storage.IdentityStorageError as exc: + return _blocked([{"code": exc.code, "source": "", "locator": ""}], scan) + except (OSError, ValueError, TypeError, KeyError, OverflowError, RecursionError): + return _blocked([{"code": "invalid_owned_state", "source": "", "locator": ""}], scan) + + +def _blocked(reasons, scan=None): + return {"migration_id": MIGRATION_ID, "required": True, "classification": "blocked", + "status": "blocked", "jobs": {}, "aliases": {}, "id_map": {}, "actions": [], + "records": {}, "bindings": [], "unassigned": [], "reasons": reasons, + "activation_allowed": False} + + +def detect(config, *, control_root=None): + """Runner-shaped read-only summary. Blocked never means required=False.""" + plan = build_plan(config, control_root=control_root) + return {key: plan[key] for key in ("migration_id", "required", "classification", "status", "reasons")} + + +def verify_target(config, *, control_root=None): + """Read actual target files, never authorize services from a proposed plan.""" + return _verify_target(config, control_root=control_root, allow_derived_rebuild=False) + + +def verify_active_target(config, *, control_root=None): + """Verify actual startup identity references, permitting widget recreation. + + Only the derived ``widget_rebuild_required`` warning is allowed beyond the + strict cutover check. The startup coordinator still owns writer admission + and the gated widget refresh; this read-only result never enables either. + """ + return _verify_target(config, control_root=control_root, allow_derived_rebuild=True) + + +def _verify_target(config, *, control_root, allow_derived_rebuild): + plan = build_plan(config, control_root=control_root) + reasons = list(plan.get("reasons", [])) + if plan["classification"] != "not_applicable": + reasons.append({"code": "identity_cutover_incomplete", "source": "", "locator": ""}) + else: + try: + # Verify source records, not replacements that could conceal a + # stale FK. Both scans must represent the same complete graph. + scan, _, _, _ = _inventory(config, control_root) + if scan.inputs != plan["inputs"] or scan.groups != plan["inventory_groups"]: + _fail("source_fingerprint_changed") + records = {p: r for p, r in scan.records.items() if r["kind"] not in {"job", "storages"}} + for path, row in records.items(): + if row["kind"] == "restore_test": + row["legacy_key"] = Path(path).stem + reasons.extend(verify_records(records, plan["jobs"], plan["aliases"])) + storage.verify_inputs(plan) + except PlanningError as exc: + reasons.append({"code": exc.code, "source": exc.source, "locator": ""}) + except storage.IdentityStorageError as exc: + reasons.append({"code": exc.code, "source": "", "locator": ""}) + except (OSError, ValueError, TypeError, KeyError, OverflowError, RecursionError): + reasons.append({"code": "invalid_owned_state", "source": "", "locator": ""}) + fatal = [r for r in reasons if r.get("severity") != "warning" + or r["code"] == "widget_rebuild_required" and not allow_derived_rebuild] + return {"valid": not fatal, "reasons": reasons, "writable_services_allowed": False, + "activation_allowed": False, "migration_id": MIGRATION_ID} diff --git a/api/migrations/job_presentation_v1.py b/api/migrations/job_presentation_v1.py new file mode 100644 index 00000000..2066ed85 --- /dev/null +++ b/api/migrations/job_presentation_v1.py @@ -0,0 +1,170 @@ +"""Repair proven, untouched presentation after immutable-ID migration (#479). + +The original approved plan and snapshot remain read-only. A cosmetic repair +never guesses from a current name/prefix or requires removed recovery material. +""" +from copy import deepcopy +import hashlib + +from inventory_store import atomic_write_bytes, inventory_lock +from job_model import validate_job, validate_job_id +from job_presentation import legacy_presentation_defaults +from migration_barrier import quiescence_held + +from . import identity_storage as storage, immutable_job_id_v1 as identity +from .audit import append_event, config_dir + +MIGRATION_ID = 'job_presentation_v1' +INTRODUCED_IN = 'issue-447-479' + + +def _fingerprint(raw, mode): + return {'exists': True, 'size': len(raw), 'sha256': hashlib.sha256(raw).hexdigest(), 'mode': mode} + + +def _evidence(config): + from identity_migration_api import IdentityMigrationAssistant + assistant = IdentityMigrationAssistant(config) + state = assistant._state_dir() + if state is None: + return None + assistant._validate_state_layout(state) + meta = assistant._meta(state) + if not meta or meta['stage'] != 'complete' or meta['status'] != 'applied' or not meta['acknowledged']: + return None + plan = storage.load_plan(state) + journal = storage.read_journal(state) + if (not journal or journal[-1]['phase'] != 'commit' or journal[-1]['status'] != 'applied' + or meta.get('plan_id') != plan['plan_id']): + return None + handle, manifest = assistant._snapshot(state, plan) + if handle['digest'] != meta.get('snapshot_digest'): + return None + recovery = assistant._validate_location(state.parent / ('.job-presentation-v1-' + plan['plan_id'][:16])) + return state, plan, manifest, recovery + + +def _candidates(config, evidence): + state, plan, manifest, recovery = evidence + targets = {row['target']: row for row in plan['actions'] if row.get('kind') == 'write_json'} + jobs_dir = config_dir(config) / 'jobs' + candidates, skipped = [], [] + for job_id, planned in sorted(plan['jobs'].items()): + validate_job_id(job_id) + source = plan['job_sources'][job_id] + original_entry = manifest['entries'][source] + _, raw = storage._read_file(state / 'snapshot/files' / original_entry['blob'], private=True) + if _fingerprint(raw, original_entry['original']['mode']) != original_entry['original']: + storage._fail('snapshot_changed') + original = identity._strict_json(raw, source) + defaults = legacy_presentation_defaults(original) if original.get('schema_version') in {1, 2, 3} else {} + updates = {key: value for key, value in defaults.items() if not str(planned.get(key) or '').strip()} + path = jobs_dir / (job_id + '.json') + reason = 'explicit_or_unchanged_presentation' if not defaults else 'already_preserved' + if updates: + action = targets.get(str(path)) + if not action or action.get('source') != source or action.get('data') != planned: + storage._fail('invalid_plan') + before = identity.encode_target_json(planned) + if _fingerprint(before, action['after']['mode']) != action['after']: + storage._fail('invalid_plan') + corrected = {**deepcopy(planned), **updates} + validate_job(corrected, filename=path.name) + after = identity.encode_target_json(corrected) + candidates.append({'job_id': job_id, 'path': path, 'updates': updates, + 'before': before, 'after': after, 'before_fp': action['after'], + 'after_fp': _fingerprint(after, action['after']['mode'])}) + continue + skipped.append({'job_id': job_id, 'reason': reason}) + return candidates, skipped + + +def detect(config): + # Only the registry writes audit/state. No recovery paths are created here. + from identity_migration_api import IdentityMigrationAssistant + exists = storage.fingerprint_file(IdentityMigrationAssistant(config).selector)['exists'] + return {'required': exists, 'reason': 'inspect_original_presentation' if exists else 'no_identity_migration'} + + +def _result(config, status, *, actions=None, skipped=None, **details): + details = {'actions': actions or [], 'skipped': skipped or [], **details} + append_event(config, {'event': 'migration_' + status, 'migration_id': MIGRATION_ID, + 'status': status, 'details': details}) + return {'migration_id': MIGRATION_ID, 'status': status, 'details': details} + + +def apply(config): + if not quiescence_held(config): + return {'migration_id': MIGRATION_ID, 'status': 'failed', 'details': { + 'error_type': 'PresentationRepairError', 'error': 'exclusive_migration_required', 'failed_phase': 'apply'}} + with inventory_lock(config_dir(config)): + # Recovery evidence is optional for cosmetic repair. Missing/corrupt + # evidence must never block an otherwise healthy installation. + try: + evidence = _evidence(config) + if evidence is None: + return _result(config, 'skipped', reason='original_recovery_unavailable') + candidates, skipped = _candidates(config, evidence) + except Exception: + return _result(config, 'skipped', reason='original_recovery_unavailable') + recovery = evidence[3] + actions, changed = [], [] + try: + for item in candidates: + current = storage.fingerprint_file(item['path']) + backup = recovery / (item['job_id'] + '.json') + if current == item['after_fp']: + # Only our exact verified before-copy establishes a prior + # interrupted repair; otherwise this is a user's change. + try: + _, saved = storage._read_file(backup, private=True) + except Exception: + saved = None + if saved == item['before']: + actions.append({'job_id': item['job_id'], 'action': 'already_applied'}) + continue + if current != item['before_fp']: + skipped.append({'job_id': item['job_id'], 'reason': 'changed_since_identity_migration'}) + continue + # Dedicated private persistent storage, never the Unraid FAT + # config volume and never a new file in the approved snapshot. + try: + storage._private_directory(recovery, create=True) + storage._publish_once(backup, item['before']) + _, saved = storage._read_file(backup, private=True) + if saved != item['before']: + storage._fail('snapshot_changed') + except Exception: + skipped.append({'job_id': item['job_id'], 'reason': 'private_recovery_unavailable'}) + continue + event = {'job_id': item['job_id'], 'action': 'materialize_automatic_presentation', + 'appearance': item['updates'], 'path': str(item['path']), 'backup': str(backup), + 'before_sha256': item['before_fp']['sha256'], 'after_sha256': item['after_fp']['sha256']} + append_event(config, {'event': 'migration_job_pending', 'migration_id': MIGRATION_ID, + 'status': 'pending', 'details': event}) + if storage.fingerprint_file(item['path']) != item['before_fp']: + storage._fail('input_changed') + changed.append(item) + atomic_write_bytes(item['path'], item['after'], mode=item['after_fp']['mode']) + if storage.fingerprint_file(item['path']) != item['after_fp']: + storage._fail('verification_failed') + actions.append(event) + return _result(config, 'applied' if actions else 'skipped', actions=actions, skipped=skipped) + except Exception as exc: + rollback = [] + for item in reversed(changed): + try: + current = storage.fingerprint_file(item['path']) + if current == item['after_fp']: + _, saved = storage._read_file(recovery / (item['job_id'] + '.json'), private=True) + if saved != item['before']: + storage._fail('snapshot_changed') + atomic_write_bytes(item['path'], saved, mode=item['before_fp']['mode']) + if storage.fingerprint_file(item['path']) != item['before_fp']: + storage._fail('input_changed') + rollback.append({'job_id': item['job_id'], 'status': 'restored'}) + except Exception: + rollback.append({'job_id': item['job_id'], 'status': 'failed'}) + return _result(config, 'failed', actions=actions, skipped=skipped, + error_type=type(exc).__name__, error='job_presentation_write_failed', + failed_phase='apply', rollback=rollback) diff --git a/api/migrations/registry.py b/api/migrations/registry.py index 87b4d5d2..037a5930 100644 --- a/api/migrations/registry.py +++ b/api/migrations/registry.py @@ -6,7 +6,7 @@ from datetime import datetime from typing import Any -from . import canonical_backup_conf_v1 +from . import canonical_backup_conf_v1, immutable_job_id_activation, job_presentation_v1 from .audit import ( append_event, config_dir as audit_config_dir, @@ -20,11 +20,13 @@ from security_utils import mask_secrets MIGRATIONS = [ + immutable_job_id_activation, + job_presentation_v1, canonical_backup_conf_v1, ] FINAL_STATES = {"applied", "not_required", "not_applicable", "skipped"} -APPLY_STATES = FINAL_STATES | {"failed"} +APPLY_STATES = FINAL_STATES | {"failed", "pending", "blocked"} PROVEN_APPLIED_EVENTS = {"migration_applied", "migration_completed"} @@ -296,6 +298,7 @@ def run_startup_migrations(config: dict) -> dict[str, Any]: skipped = [] failed = [] blocked = [] + pending = [] messages = [] blocked_by = "" @@ -342,12 +345,14 @@ def run_startup_migrations(config: dict) -> dict[str, Any]: messages.append(f"{migration_id}=failed") continue if not bool(detected.get("required")): - skipped.append(migration_id) + proven_applied = bool(getattr(migration, "USER_INITIATED", False) and detected.get("status") == "applied" + and (previous_state != "applied" or not _is_central_registry_result(previous))) + (applied if proven_applied else skipped).append(migration_id) results[migration_id] = { "migration_id": migration_id, "introduced_in": str(migration.INTRODUCED_IN), "runner": "central_migration_registry", - "status": "not_required", + "status": "applied" if proven_applied else "not_required", "details": detected, } messages.append(f"{migration_id}=not_required") @@ -362,6 +367,9 @@ def run_startup_migrations(config: dict) -> dict[str, Any]: if status == "failed": failed.append(migration_id) blocked_by = migration_id + elif status in {"pending", "blocked"}: + (pending if status == "pending" else blocked).append(migration_id) + blocked_by = migration_id elif status == "applied": applied.append(migration_id) else: @@ -369,13 +377,16 @@ def run_startup_migrations(config: dict) -> dict[str, Any]: messages.append(f"{migration_id}={status}") summary = { - "status": "failed" if failed else "ok", + "status": "failed" if failed else "pending" if pending else "blocked" if blocked else "ok", "applied": applied, "skipped": skipped, "failed": failed, "blocked": blocked, + "pending": pending, "messages": messages, "results": results, } - _write_state_and_log(config, summary) + # Pending detection is read-only: no state file or previous migration rewrites. + if not pending and not blocked or failed: + _write_state_and_log(config, summary) return summary diff --git a/api/notification_reminder_api.py b/api/notification_reminder_api.py index a69245a8..9383227d 100644 --- a/api/notification_reminder_api.py +++ b/api/notification_reminder_api.py @@ -66,9 +66,9 @@ def _active_channels(event_type: str) -> list[str]: if backup_channels: schedules = get_schedules(effective) jobs = { - str(job.get("key") or "").strip(): job + str(job.get("job_id") or "").strip(): job for job in list_jobs(effective, {}) - if isinstance(job, dict) and str(job.get("key") or "").strip() + if isinstance(job, dict) and str(job.get("job_id") or "").strip() } status = get_status_data(effective) latest = _latest_backup_status_by_key(status.get("backups") or []) @@ -137,21 +137,21 @@ def run_due_notification_reminders(config: dict) -> dict[str, Any]: continue checked += 1 - job_key = str(row.get("job_key") or "").strip() - if not job_key: + job_id = str(row.get("job_id") or "").strip() + if not job_id: continue due_marker = str(row.get("next_due_at") or "").strip() if not due_marker: skipped += 1 - rows.append({"job_key": job_key, "sent": False, "reason": "missing_due_marker"}) + rows.append({"job_id": job_id, "sent": False, "reason": "missing_due_marker"}) continue - key = reminder_key("restore_test_overdue", job_key, due_marker) + key = reminder_key("restore_test_overdue", job_id, due_marker) if not reminder_allowed(effective, key): skipped += 1 - rows.append({"job_key": job_key, "sent": False, "reason": "interval_not_elapsed"}) + rows.append({"job_id": job_id, "sent": False, "reason": "interval_not_elapsed"}) continue - display_name = str(row.get("display_name") or job_key) + display_name = str(row.get("display_name") or job_id) message = build_restore_test_notification_message( job_name=display_name, status="Overdue", @@ -166,7 +166,7 @@ def run_due_notification_reminders(config: dict) -> dict[str, Any]: message=message, severity="warning", job_name=f"Borg Backup UI ({display_name})", - job_key=job_key, + job_id=job_id, status="overdue", source="scheduled_reminder", extra={"due_marker": due_marker}, @@ -175,10 +175,10 @@ def run_due_notification_reminders(config: dict) -> dict[str, Any]: if any(results.values()): mark_reminder_sent(effective, key) sent += 1 - rows.append({"job_key": job_key, "sent": True, "channels": results}) + rows.append({"job_id": job_id, "sent": True, "channels": results}) else: skipped += 1 - rows.append({"job_key": job_key, "sent": False, "reason": "no_channel_sent", "channels": results}) + rows.append({"job_id": job_id, "sent": False, "reason": "no_channel_sent", "channels": results}) return { "checked": checked, @@ -209,9 +209,9 @@ def _send_backup_overdue_reminders(effective: dict, mail_config) -> dict[str, An return {"checked": 0, "sent": 0, "skipped": 0, "rows": []} jobs = { - str(job.get("key") or "").strip(): job + str(job.get("job_id") or "").strip(): job for job in list_jobs(effective, {}) - if isinstance(job, dict) and str(job.get("key") or "").strip() + if isinstance(job, dict) and str(job.get("job_id") or "").strip() } status = get_status_data(effective) latest = _latest_backup_status_by_key(status.get("backups") or []) @@ -225,17 +225,17 @@ def _send_backup_overdue_reminders(effective: dict, mail_config) -> dict[str, An sent_state = state.get("last_sent") if isinstance(state.get("last_sent"), dict) else {} interval_hours = reminder_interval_hours(effective) tolerance_hours = _backup_overdue_tolerance_hours(effective) - for job_key, sched in schedules.items(): - if job_key == "restore_test" or not isinstance(sched, dict) or not bool(sched.get("enabled", True)): + for job_id, sched in schedules.items(): + if job_id == "restore_test" or not isinstance(sched, dict) or not bool(sched.get("enabled", True)): continue - job = jobs.get(str(job_key)) + job = jobs.get(str(job_id)) if not job or job.get("enabled") is False: continue item = _backup_overdue_item( - str(job_key), + str(job_id), sched, job, - latest.get(str(job_key)) or {}, + latest.get(str(job_id)) or {}, sent_state, now, interval_hours, @@ -243,23 +243,23 @@ def _send_backup_overdue_reminders(effective: dict, mail_config) -> dict[str, An ) if item["state"] == "unsupported": skipped += 1 - rows.append({"job_key": job_key, "event": "backup_overdue", "sent": False, "reason": "unsupported_cron"}) + rows.append({"job_id": job_id, "event": "backup_overdue", "sent": False, "reason": "unsupported_cron"}) continue checked += 1 if item["state"] == "current": - clear_reminder_prefix(effective, f"backup_overdue:{job_key}:") + clear_reminder_prefix(effective, f"backup_overdue:{job_id}:") continue if item["state"] == "overdue_waiting": skipped += 1 - rows.append({"job_key": job_key, "event": "backup_overdue", "sent": False, "reason": "interval_not_elapsed"}) + rows.append({"job_id": job_id, "event": "backup_overdue", "sent": False, "reason": "interval_not_elapsed"}) continue if item["state"] != "overdue_ready": skipped += 1 - rows.append({"job_key": job_key, "event": "backup_overdue", "sent": False, "reason": item.get("reason") or "not_overdue"}) + rows.append({"job_id": job_id, "event": "backup_overdue", "sent": False, "reason": item.get("reason") or "not_overdue"}) continue - display_name = str(job.get("display_name") or job.get("name") or job_key) + display_name = str(job.get("display_name") or job.get("name") or job_id) due_marker = str(item.get("expected_run_marker") or "") message = build_backup_notification_message( job_name=display_name, @@ -275,7 +275,7 @@ def _send_backup_overdue_reminders(effective: dict, mail_config) -> dict[str, An message=message, severity="warning", job_name=f"Borg Backup ({display_name})", - job_key=str(job_key), + job_id=str(job_id), status="overdue", repository=str(job.get("repo_path") or ""), source="scheduled_reminder", @@ -283,18 +283,18 @@ def _send_backup_overdue_reminders(effective: dict, mail_config) -> dict[str, An ) results = send_event(effective, event, mail_config=mail_config) if any(results.values()): - mark_reminder_sent(effective, str(item.get("reminder_key") or reminder_key("backup_overdue", str(job_key), due_marker))) + mark_reminder_sent(effective, str(item.get("reminder_key") or reminder_key("backup_overdue", str(job_id), due_marker))) sent += 1 - rows.append({"job_key": job_key, "event": "backup_overdue", "sent": True, "channels": results}) + rows.append({"job_id": job_id, "event": "backup_overdue", "sent": True, "channels": results}) else: skipped += 1 - rows.append({"job_key": job_key, "event": "backup_overdue", "sent": False, "reason": "no_channel_sent", "channels": results}) + rows.append({"job_id": job_id, "event": "backup_overdue", "sent": False, "reason": "no_channel_sent", "channels": results}) return {"checked": checked, "sent": sent, "skipped": skipped, "rows": rows} def _backup_overdue_item( - job_key: str, + job_id: str, sched: dict, job: dict, last: dict, @@ -308,11 +308,11 @@ def _backup_overdue_item( cron = str(sched.get("cron") or "").strip() latest_expected_run = _latest_expected_run(cron, now) next_scheduled_run = _next_expected_run(cron, now) - display_name = str(job.get("display_name") or job.get("name") or job_key) + display_name = str(job.get("display_name") or job.get("name") or job_id) if latest_expected_run is None: return { "type": "backup_overdue", - "job_key": str(job_key), + "job_id": str(job_id), "display_name": display_name, "cron": cron, "state": "unsupported", @@ -323,7 +323,7 @@ def _backup_overdue_item( if last_ts is None: return { "type": "backup_overdue", - "job_key": str(job_key), + "job_id": str(job_id), "display_name": display_name, "cron": cron, "state": "missing_status", @@ -347,7 +347,7 @@ def _backup_overdue_item( overdue_after = expected_run + timedelta(hours=backup_tolerance_hours) overdue = now > overdue_after and (last_ts is None or last_ts < expected_run) expected_marker = expected_run.strftime("%Y-%m-%d %H:%M:%S") - key = reminder_key("backup_overdue", str(job_key), expected_marker) + key = reminder_key("backup_overdue", str(job_id), expected_marker) reminder = ( {"sent": False, "sent_at": "", "next_allowed_at": "", "allowed": True} if last_ts is not None and last_ts >= expected_run @@ -357,7 +357,7 @@ def _backup_overdue_item( reason = "ready_to_send" if state == "overdue_ready" else ("interval_not_elapsed" if state == "overdue_waiting" else "not_overdue") return { "type": "backup_overdue", - "job_key": str(job_key), + "job_id": str(job_id), "display_name": display_name, "cron": cron, "state": state, @@ -387,17 +387,17 @@ def _backup_overdue_diagnostics( if not isinstance(schedules, dict): return [] items: list[dict[str, Any]] = [] - for job_key, sched in sorted(schedules.items()): - if job_key == "restore_test" or not isinstance(sched, dict) or not bool(sched.get("enabled", True)): + for job_id, sched in sorted(schedules.items()): + if job_id == "restore_test" or not isinstance(sched, dict) or not bool(sched.get("enabled", True)): continue - job = jobs.get(str(job_key)) + job = jobs.get(str(job_id)) if not job or job.get("enabled") is False: continue item = _backup_overdue_item( - str(job_key), + str(job_id), sched, job, - latest.get(str(job_key)) or {}, + latest.get(str(job_id)) or {}, sent, now, interval_hours, @@ -421,11 +421,11 @@ def _restore_test_overdue_diagnostics(plan: dict, sent: dict, now: datetime, int continue if row.get("enabled") is False: continue - job_key = str(row.get("job_key") or "").strip() - if not job_key: + job_id = str(row.get("job_id") or "").strip() + if not job_id: continue due_marker = str(row.get("next_due_at") or "").strip() - key = reminder_key("restore_test_overdue", job_key, due_marker) if due_marker else "" + key = reminder_key("restore_test_overdue", job_id, due_marker) if due_marker else "" reminder = _reminder_state_for_key(sent, key, now, interval_hours) if key else { "sent": False, "sent_at": "", @@ -442,8 +442,8 @@ def _restore_test_overdue_diagnostics(plan: dict, sent: dict, now: datetime, int }.get(state, "unknown") items.append({ "type": "restore_test_overdue", - "job_key": job_key, - "display_name": str(row.get("display_name") or job_key), + "job_id": job_id, + "display_name": str(row.get("display_name") or job_id), "state": state, "reason": reason, "next_due_at": due_marker, @@ -487,18 +487,17 @@ def _latest_backup_status_by_key(rows: list) -> dict[str, dict]: for row in rows or []: if not isinstance(row, dict): continue - keys = [] - explicit_key = str(row.get("key") or "").strip() - if explicit_key: - keys.append(explicit_key) - backup_type = str(row.get("backup_type") or row.get("type") or "").strip().lower() - location = str(row.get("location") or "").strip().lower() - if backup_type and location: - keys.append(f"{backup_type}_{location}") - for key in keys: - current = latest.get(key) - if current is None or _status_is_newer(row, current): - latest[key] = row + from job_model import validate_job_id + key = row.get("job_id") + try: + validate_job_id(key) + except ValueError: + continue + if row.get("identity_state") == "unassigned": + continue + current = latest.get(key) + if current is None or _status_is_newer(row, current): + latest[key] = row return latest diff --git a/api/report_mail_api.py b/api/report_mail_api.py index dfc1036f..bd2aa9c2 100644 --- a/api/report_mail_api.py +++ b/api/report_mail_api.py @@ -160,7 +160,7 @@ def _login_if_needed(smtp_obj): # ── HTML-Report-Generator ────────────────────────────────────────────────────── def _build_html_report(config: dict, now: Optional[datetime] = None) -> str: - from status import StatusStore, format_bytes, format_duration + from status import StatusStore, BackupStatus, format_bytes, format_duration status_dir = Path(config["STATUS_DIR"]) store = StatusStore(status_dir) @@ -175,7 +175,8 @@ def _build_html_report(config: dict, now: Optional[datetime] = None) -> str: rows = [] grouped_rows: dict[str, list[str]] = {} group_stats: dict[str, dict[str, int]] = {} - job_meta = _job_metadata_by_key(config) + job_meta = {job_id: row for job_id, row in _job_metadata_by_key(config).items() if row.get('enabled') is not False} + latest = {job_id: latest.get(job_id) or BackupStatus(job_id=job_id, identity_state='assigned', status='unknown') for job_id in job_meta} schedules = _report_schedules(config) planned_job_keys = _planned_job_keys_for_period( set(latest.keys()) | set(job_meta.keys()), @@ -192,11 +193,13 @@ def _build_html_report(config: dict, now: Optional[datetime] = None) -> str: log_notes = [] for key, st in sorted(latest.items(), key=lambda item: _status_sort_key(item[1], item[0])): - location_key = _location_key(st) meta = job_meta.get(key, {}) + location_key = _report_location(key, st, meta) job_label = _job_label(key, st, meta) archive_fmt = st.archive_name or "—" secondary = _job_secondary_line(key, archive_fmt) + if st.job_name_snapshot and st.job_name_snapshot != meta.get('name'): + secondary += ' | Run name: ' + st.job_name_snapshot status_color = { "success": "#22c55e", "skipped": "#f59e0b", @@ -279,7 +282,7 @@ def _build_html_report(config: dict, now: Optional[datetime] = None) -> str: total = len(latest) error_total = sum(1 for st in latest.values() if st.status == "error") - warn_total = sum(1 for st in latest.values() if st.status in {"warning", "skipped"}) + warn_total = sum(1 for st in latest.values() if st.status in {"warning", "skipped", "unknown", "cancelled"}) summary_color = "#22c55e" if error_total == 0 and warn_total == 0 else ("#f59e0b" if error_total == 0 else "#ef4444") summary_text = "All backups OK" if error_total == 0 and warn_total == 0 else ( f"{error_total} errors, {warn_total} warnings" @@ -364,32 +367,9 @@ def _he(s: str) -> str: return str(s).replace("&", "&").replace("<", "<").replace(">", ">") -def _job_metadata_by_key(config: dict) -> dict[str, dict[str, str]]: - if not config.get("BACKUP_SCRIPTS_DIR"): - return {} - try: - from jobs_api import discover_jobs, resolve_data_root, resolve_scripts_dir - - data_root = resolve_data_root(config) - scripts_dir = resolve_scripts_dir(config) - jobs = discover_jobs(scripts_dir, data_root) - except Exception: - return {} - - result: dict[str, dict[str, str]] = {} - for info in jobs: - key = str(getattr(info, "key", "") or "").strip() - if not key: - continue - result[key] = { - "name": str(getattr(info, "name", "") or "").strip(), - "display_name": str(getattr(info, "display_name", "") or "").strip(), - "description": str(getattr(info, "description", "") or "").strip(), - "backup_type": str(getattr(info, "backup_type", "") or "").strip(), - "location": str(getattr(info, "location", "") or "").strip().lower(), - "enabled": bool(getattr(info, "enabled", True)), - } - return result +def _job_metadata_by_key(config: dict) -> dict: + from status_read_model import configured_jobs + return configured_jobs(config) def _report_schedules(config: dict) -> dict: @@ -434,6 +414,8 @@ def _job_label(key: str, st: Any, meta: dict[str, str]) -> str: def _derived_job_label(st: Any) -> str: + if getattr(st, "job_name_snapshot", ""): + return st.job_name_snapshot backup_type = str(getattr(st, "backup_type", "") or "").strip() location = str(getattr(st, "location", "") or "").strip() if backup_type and backup_type != "unknown": @@ -735,13 +717,13 @@ def _repo_growth_7d(statuses: list, key: str, period_start: datetime, period_end if size <= 0: continue if st.timestamp_dt < period_start: - baseline = size + baseline = (size, getattr(st, "repository_snapshot", "")) continue if st.timestamp_dt <= period_end: - current = size + current = (size, getattr(st, "repository_snapshot", "")) if baseline is None or current is None: return None - return current - baseline + return current[0] - baseline[0] if baseline[1] and baseline[1] == current[1] else None def _cron_expected_dates(cron: str, start: datetime, end: datetime) -> set | None: @@ -836,11 +818,9 @@ def _report_key_sort(key: str, latest: dict[str, Any], job_meta: dict[str, dict[ def _report_location(key: str, st: Any, meta: dict[str, Any]) -> str: - value = str(meta.get("location") or getattr(st, "location", "") or "").strip().lower() + value = str(meta.get("location") or getattr(st, "location_snapshot", "") or "").strip().lower() if value: return value - if "_" in key: - return key.rsplit("_", 1)[1].lower() return "unknown" @@ -849,7 +829,7 @@ def _report_job_label(key: str, st: Any, meta: dict[str, Any]) -> str: meta.get("name"), meta.get("display_name"), _derived_job_label(st) if st is not None else "", - key.replace("_", " ").title(), + key, ): text = str(value or "").strip() if text: diff --git a/api/reports_api.py b/api/reports_api.py index 31068478..81b18430 100644 --- a/api/reports_api.py +++ b/api/reports_api.py @@ -1,7 +1,5 @@ """api/reports_api.py – Berichte: historische Auswertung aus .status-Dateien""" -import json -from pathlib import Path from typing import List @@ -28,71 +26,32 @@ def _fmt_duration(secs): return f"{s}s" -def _parse_job_key(job_key: str): - """Split 'appdata_local' → ('appdata', 'local'). Handles multi-underscore locations.""" - known_locations = ("local", "usb", "smb", "storagebox") - for loc in known_locations: - if job_key.endswith("_" + loc): - btype = job_key[: -(len(loc) + 1)] - return btype, loc - parts = job_key.rsplit("_", 1) - return (parts[0], parts[1]) if len(parts) == 2 else (job_key, "") - - -def _parse_status_file_stem(stem: str): - """Split '_