diff --git a/api/activity_log.py b/api/activity_log.py index 7cdb9d87..891a01f6 100644 --- a/api/activity_log.py +++ b/api/activity_log.py @@ -7,10 +7,12 @@ from __future__ import annotations import codecs +import json import os import re import stat from contextlib import contextmanager +from functools import lru_cache from pathlib import Path WINDOW_BYTES = 65536 @@ -19,9 +21,12 @@ _RUN = re.compile(r"^[A-Za-z0-9_.-]{8,96}$") -def activity_log_path(directory: Path, job_key: str, run_id: str) -> Path: +def activity_log_path(directory: Path, job_key: str, run_id: str, *, job_name: str = "", location: str = "") -> Path: if not _KEY.fullmatch(job_key) or not _RUN.fullmatch(run_id): raise ValueError("Invalid activity log identity") + if job_name or location: + from job_identity import job_log_filename, job_run_date_tag + return directory / job_log_filename(job_name, location, job_key, job_run_date_tag(run_id)) return directory / f"Borg-Backup_{job_key}--activity-{run_id}.log" @@ -33,6 +38,53 @@ def open_activity_file(path: Path): return os.fdopen(fd, "rb") +def _saved_activity_run(config: dict, job_key: str, run_id: str) -> tuple[Path, dict] | None: + """Resolve a saved run by its status metadata after RAM state is gone.""" + from jobs_api import _runtime_log_dir + status_dir = str(config.get("STATUS_DIR") or "") + archive_dir = str(config.get("STATUS_ARCHIVE_DIR") or (Path(status_dir) / "archive" if status_dir else "")) + try: + return _find_saved_activity_run(status_dir, archive_dir, _runtime_log_dir(config), job_key, run_id) + except FileNotFoundError: + return None + + +@lru_cache(maxsize=128) +def _find_saved_activity_run(status_dir: str, archive_dir: str, log_dir: Path, + job_key: str, run_id: str) -> tuple[Path, dict]: + # Completed run mappings are immutable. Cache only successful lookups so + # each bounded log window does not reread all historical status files. + matches = {} + for directory in {status_dir, archive_dir} - {""}: + for status_file in Path(directory).glob(f"*_{job_key}.status"): + try: + data = json.loads(status_file.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(data, dict) or data.get("job_id") != job_key or data.get("run_id") != run_id: + continue + if data.get("file_activity") is not True or not isinstance(data.get("log_file"), str) or not data["log_file"]: + continue + matches[Path(data["log_file"])] = data + if not matches: + # A runner can fail before writing status (for example while loading + # its job). Its complete retained log must still reopen after reboot. + from job_identity import job_log_paths + marker = f"INFO File activity run: job_id={job_key} run_id={run_id}\n".encode() + for candidate in job_log_paths(log_dir, job_key): + try: + with open_activity_file(candidate) as handle: + if handle.readline(256) == marker: + matches[candidate] = {} + except (OSError, ValueError): + continue + if len(matches) > 1: + raise ValueError("Multiple logs found for this job run") + if not matches: + raise FileNotFoundError("No saved file-activity log found for this job run") + return next(iter(matches.items())) + + def resolve_activity_run(config: dict, job_key: str, run_id: str = "") -> tuple[Path, dict]: from jobs_api import JobManager, durable_running_states, _runtime_log_dir from job_control import read_control_state @@ -53,6 +105,18 @@ def resolve_activity_run(config: dict, job_key: str, run_id: str = "") -> tuple[ # Exact run filenames allow reconnecting after completion or a UI # restart, without accepting arbitrary filesystem paths from clients. state = {"running": False, "exit_code": None, "run_id": run_id} + from job_identity import job_log_paths + matches = [candidate for candidate in job_log_paths(path.parent, job_key) + if candidate.name.endswith(f"--activity-{run_id}.log")] + if len(matches) > 1: + raise ValueError("Multiple logs found for this job run") + if matches: + path = matches[0] + else: + saved = _saved_activity_run(config, job_key, run_id) + if saved: + path, status = saved + state["exit_code"] = status.get("exit_code") control = read_control_state(run_id) if control.get("job_key") == job_key: state["phase"] = control.get("phase", "") diff --git a/api/activity_log_capture.py b/api/activity_log_capture.py index 7918c180..ca5e259b 100644 --- a/api/activity_log_capture.py +++ b/api/activity_log_capture.py @@ -63,10 +63,10 @@ def open_capture_file(record_path: Path): raise -def prepare_capture(job_key: str, run_id: str, destination: Path) -> tuple[Path, Path]: +def prepare_capture(job_key: str, run_id: str, destination: Path, *, job_name: str = "", location: str = "") -> tuple[Path, Path]: from activity_log import activity_log_path - retained = activity_log_path(destination, job_key, run_id) + retained = activity_log_path(destination, job_key, run_id, job_name=job_name, location=location) active = activity_log_path(CAPTURE_ROOT / run_id, job_key, run_id) active.parent.mkdir(parents=True, mode=0o700) with os.fdopen(os.open(active, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as handle: @@ -157,6 +157,10 @@ def supervise(record_path: Path, command: list[str]) -> int: record["pid"] = os.getpid() record["process_start"] = process_token(os.getpid()) write_record(record_path, record) + if Path(record["retained_file"]).name.startswith("BBUI-"): + # The visible filename is the same as for a normal run. Keep the exact + # run identity in the log even if startup fails before status is saved. + print(f"INFO File activity run: job_id={record['job_key']} run_id={record['run_id']}", flush=True) try: process = subprocess.Popen(command) code = process.wait() diff --git a/api/archive_browser.py b/api/archive_browser.py index 5d5eb5b5..5d0075cc 100644 --- a/api/archive_browser.py +++ b/api/archive_browser.py @@ -17,6 +17,13 @@ _CACHE_LOCK = threading.Lock() +class ArchiveNotFoundError(ValueError): + """The archive selection no longer exists in the current repository.""" + + api_code = "restore_archive_unavailable" + api_status = 404 + + def _prune_expired_cache_entries(now: float) -> None: expired = [key for key, value in _CACHE.items() if float(value.get("expires", 0)) <= now] for key in expired: @@ -46,6 +53,10 @@ def build_archive_index(repo: str, archive: str, env: dict[str, str]) -> dict[st except subprocess.TimeoutExpired as exc: raise TimeoutError("borg archive listing timed out") from exc if result.returncode != 0: + # Borg's C-locale error identifies an absent archive, not an unavailable + # repository or another list failure. Keep the original detail for logs. + if f"Archive {archive} does not exist" in result.stderr.splitlines(): + raise ArchiveNotFoundError(f"borg list failed: {result.stderr.strip()}") raise RuntimeError(f"borg list failed: {result.stderr.strip()}") index: dict[str, dict[str, dict[str, Any]]] = {} diff --git a/api/archive_prefix.py b/api/archive_prefix.py index 95f8a123..266ad344 100644 --- a/api/archive_prefix.py +++ b/api/archive_prefix.py @@ -4,7 +4,42 @@ from typing import Iterable -_ARCHIVE_PREFIX_RX = re.compile(r"^[A-Za-z0-9_.-]+-backup$") +_ARCHIVE_PREFIX_RX = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def validate_archive_prefix(value: object) -> str: + prefix = str(value or "").strip() + if not prefix or not _ARCHIVE_PREFIX_RX.fullmatch(prefix): + raise ValueError("Archive prefix may contain only letters, digits, dots, underscores and hyphens") + return prefix + + +def archive_prefix_from_metadata(meta: dict) -> str: + return validate_archive_prefix(meta.get("archive_prefix")) + + +def job_archive_prefixes(meta: dict) -> list[str]: + return list(dict.fromkeys([ + archive_prefix_from_metadata(meta), + *(validate_archive_prefix(value) for value in meta.get("archive_prefixes", [])), + ])) + + +def validate_prefix_ownership(candidate: dict, other_jobs: Iterable[dict]) -> None: + """A Borg '-*' selection belongs to one job per repository.""" + prefixes = job_archive_prefixes(candidate) + for other in other_jobs: + if other.get("job_id") == candidate.get("job_id"): + continue + if other.get("repository_key") != candidate.get("repository_key"): + continue + for prefix in prefixes: + for owned in job_archive_prefixes(other): + if prefix == owned or prefix.startswith(owned + "-") or owned.startswith(prefix + "-"): + raise ValueError( + f"Archive prefix '{prefix}' overlaps with '{owned}' of job " + f"'{other.get('name') or other.get('job_id')}' in the selected repository" + ) def archive_prefix_from_backup_type(backup_type: str) -> str: diff --git a/api/borg_ssh.py b/api/borg_ssh.py index 2bf5ae30..e9cee877 100644 --- a/api/borg_ssh.py +++ b/api/borg_ssh.py @@ -25,7 +25,7 @@ ("LogLevel", "ERROR"), ("WarnWeakCrypto", "no"), ) -_MANAGED_OPTION_NAMES = {name.lower() for name, _ in _SSH_OPTIONS} +_MANAGED_OPTION_NAMES = {name.lower() for name, _ in _SSH_OPTIONS} | {"ignoreunknown"} _SSH_INTERRUPTION_MARKERS = ( "connection reset by peer", "broken pipe", @@ -38,7 +38,8 @@ def _option_name(value: str) -> str: - return str(value or "").split("=", 1)[0].strip().lower() + parts = str(value or "").replace("=", " ", 1).split(maxsplit=1) + return parts[0].lower() if parts else "" def build_borg_rsh(existing: str = "", identity_file: str = "") -> str: @@ -52,21 +53,32 @@ def build_borg_rsh(existing: str = "", identity_file: str = "") -> str: identity = str(identity_file or "").strip() cleaned: list[str] = [] + ignore_unknown_seen = False index = 0 while index < len(tokens): token = tokens[index] + option = None + option_length = 1 if token == "-o" and index + 1 < len(tokens): - value = tokens[index + 1] - if _option_name(value) in _MANAGED_OPTION_NAMES: - index += 2 - continue - cleaned.extend((token, value)) - index += 2 + option = tokens[index + 1] + option_length = 2 + elif token.startswith("-o") and len(token) > 2: + option = token[2:] + if option is not None: + name = _option_name(option) + if name == "ignoreunknown" and not ignore_unknown_seen: + # SSH uses the first list. Extend it in place so any custom + # options following it still have their original exemption. + parts = option.replace("=", " ", 1).split(maxsplit=1) + patterns = parts[1].split(",") if len(parts) > 1 else [] + if "warnweakcrypto" not in {pattern.lower() for pattern in patterns}: + patterns.append("WarnWeakCrypto") + cleaned.extend(("-o", "IgnoreUnknown=" + ",".join(patterns))) + ignore_unknown_seen = True + elif name not in _MANAGED_OPTION_NAMES: + cleaned.extend(tokens[index:index + option_length]) + index += option_length continue - if token.startswith("-o") and len(token) > 2: - if _option_name(token[2:]) in _MANAGED_OPTION_NAMES: - index += 1 - continue if identity and token == "-i" and index + 1 < len(tokens): index += 2 continue @@ -76,6 +88,9 @@ def build_borg_rsh(existing: str = "", identity_file: str = "") -> str: cleaned.append(token) index += 1 + # Older clients must see this exemption before WarnWeakCrypto is parsed. + if not ignore_unknown_seen: + cleaned.extend(("-o", "IgnoreUnknown=WarnWeakCrypto")) if identity: cleaned.extend(("-i", identity)) for name, value in _SSH_OPTIONS: diff --git a/api/check_api.py b/api/check_api.py index ee458c7a..f75874f6 100644 --- a/api/check_api.py +++ b/api/check_api.py @@ -208,7 +208,9 @@ def _repository_command( if not selected_job_key: raise ValueError("Prune requires a backup job with a retention policy") retention = self._job_retention(config, selected_job_key) - archive_prefix = _archive_prefix_from_job_key(selected_job_key) + from archive_prefix import archive_prefix_from_metadata + from repository_context import load_job_metadata + archive_prefix = archive_prefix_from_metadata(load_job_metadata(config, selected_job_key)) cmd = [ "borg", "prune", "--lock-wait", self._LOCK_WAIT_SECONDS, "--list", "--progress", diff --git a/api/config_api.py b/api/config_api.py index 6acaeeb6..0f3b6c46 100644 --- a/api/config_api.py +++ b/api/config_api.py @@ -795,7 +795,6 @@ def get_settings_data(ui_config: dict, include_storagebox_setup: bool = True) -> "RESTORE_TEST_LEVEL": conf.get("RESTORE_TEST_LEVEL", "2"), "RESTORE_TEST_INTERVAL_DAYS": conf.get("RESTORE_TEST_INTERVAL_DAYS", "30"), "RESTORE_TEST_LOCATION": conf.get("RESTORE_TEST_LOCATION", "local"), - "RESTORE_TEST_FORCE_CHUNK_TYPES": conf.get("RESTORE_TEST_FORCE_CHUNK_TYPES", "vms,photos"), "RESTORE_TEST_FULL_DRYRUN_MAX_ARCHIVE_GB": conf.get("RESTORE_TEST_FULL_DRYRUN_MAX_ARCHIVE_GB", "500"), "RESTORE_TEST_MIN_COVERAGE": conf.get("RESTORE_TEST_MIN_COVERAGE", "5"), "RESTORE_TEST_MAX_ENTRIES": conf.get("RESTORE_TEST_MAX_ENTRIES", "1000"), @@ -1088,7 +1087,12 @@ def _is_required_storage_mount_available(mount_path: Path) -> bool: return False -def ensure_data_dirs(global_data_dir: str) -> dict: +def ensure_data_dirs(global_data_dir: str, *, read_only: bool = False) -> dict: + """Check storage, with a read-only mode for routine status requests. + + Routine status reads inspect existing directories and access permissions. + Setup and runtime writers retain the actual write probe by default. + """ root = (global_data_dir or "").strip() if not root: raise ValueError("GLOBAL_DATA_DIR is not set") @@ -1103,8 +1107,16 @@ def ensure_data_dirs(global_data_dir: str) -> dict: created = [] for key in ("base", "logs", "status", "restore_status", "cache", "remotes"): p = Path(paths[key]) - p.mkdir(parents=True, exist_ok=True) - created.append(str(p)) + if read_only: + if not p.is_dir(): + raise RuntimeError(f"Required data directory is missing or not a directory: {p}") + if not os.access(p, os.W_OK | os.X_OK): + raise RuntimeError(f"Required data directory is not writable or accessible: {p}") + else: + p.mkdir(parents=True, exist_ok=True) + created.append(str(p)) + if read_only: + return {"ok": True, "paths": paths, "created": created} # write test in status dir probe = Path(paths["status"]) / ".borg-ui-write-test" probe.write_text("ok\n", encoding="utf-8") @@ -1167,7 +1179,7 @@ def validate_runtime_config(ui_config: dict) -> dict: }) else: try: - ensure_data_dirs(data_dir) + ensure_data_dirs(data_dir, read_only=True) except Exception as exc: errors.append({ "key": "GLOBAL_DATA_DIR", diff --git a/api/history_api.py b/api/history_api.py index 0e576051..b361c427 100644 --- a/api/history_api.py +++ b/api/history_api.py @@ -42,24 +42,33 @@ def get_history_data(config: dict, filters: dict | None = None) -> dict: per_page = 20 entries = [] + from jobs_api import discover_jobs, resolve_data_root, resolve_scripts_dir + jobs = {job.key: job for job in discover_jobs(resolve_scripts_dir(config), resolve_data_root(config))} if config.get("BACKUP_SCRIPTS_DIR") else {} location_counts = {location: 0 for location in ("storagebox", "usb", "smb", "local")} known_types = {"flash", "appdata", "photos", "vms", "sonstiges"} for f in sorted(status_dir.glob("*.status"), reverse=True): # Filename: YYYY-MM-DD_HH-MM-SS_type_location.status stem = f.stem - parts = stem.split("_") - if len(parts) < 4: + parts = stem.split("_", 2) + if len(parts) < 3: continue date_part = parts[0] # 2026-03-01 time_part = parts[1] # 02-15-43 - backup_type = parts[2] # flash / appdata / … - location = "_".join(parts[3:]) # local / usb / storagebox try: raw = json.loads(f.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + backup_type = str(raw.get("backup_type") or "unknown") + location = str(raw.get("location") or "unknown") + job_id = str(raw.get("job_id") or "") + job = jobs.get(job_id) + if job is None: + continue + if filters.get("job_key") and filters["job_key"] != job_id: + continue + status = raw.get("status", "unknown") exit_code = raw.get("borg_exit_code", raw.get("exit_code")) @@ -83,6 +92,9 @@ def get_history_data(config: dict, filters: dict | None = None) -> dict: entries.append({ "entry_kind": "backup_run", + "job_id": job_id, + "job_key": job_id, + "job_name": (job.name or job.display_name) if job else backup_type, "filename": f.name, "date": date_part, "time": time_part.replace("-", ":"), @@ -129,6 +141,7 @@ def _ts_key(entry: dict): end = start + per_page return { + "jobs": [{"job_id": job.key, "name": job.name or job.display_name} for job in jobs.values()], "entries": entries[start:end], "total": total, "page": page, diff --git a/api/homepage_widget_api.py b/api/homepage_widget_api.py index bbb1106c..2f03a76b 100644 --- a/api/homepage_widget_api.py +++ b/api/homepage_widget_api.py @@ -28,12 +28,16 @@ def _read_jobs(config: dict) -> list[dict]: continue if not isinstance(raw, dict): continue - key = str(raw.get("job_key") or path.stem).strip() + from job_identity import metadata_job_id + try: + key = metadata_job_id(raw) + except ValueError: + continue if not key: continue policy = raw.get("restore_test_policy") if isinstance(raw.get("restore_test_policy"), dict) else {} location = str(raw.get("location") or "").strip().lower() - name = str(raw.get("name") or raw.get("backup_type") or key).strip() + name = str(raw.get("name") or "Backup").strip() location_label = { "local": "Local", "usb": "USB", @@ -49,7 +53,7 @@ def _read_jobs(config: dict) -> list[dict]: "is_utility": bool(raw.get("is_utility", False)), "restore_test_policy": policy, }) - return rows + return sorted(rows, key=lambda row: row["name"].casefold()) def _read_latest_backup_rows(config: dict) -> list[dict]: @@ -57,7 +61,8 @@ def _read_latest_backup_rows(config: dict) -> list[dict]: status_dir = Path(str(config.get("STATUS_DIR") or "/mnt/user/backup-status")) store = StatusStore(status_dir) - latest = store.get_latest_per_key(store.load()) + job_ids = {job["key"] for job in _read_jobs(config)} + latest = store.get_latest_per_key([status for status in store.load() if status.key in job_ids]) rows: list[dict] = [] for key, status in latest.items(): rows.append({ diff --git a/api/inventory_store.py b/api/inventory_store.py index 14b1346c..a034a833 100644 --- a/api/inventory_store.py +++ b/api/inventory_store.py @@ -6,6 +6,7 @@ import fcntl import json import os +import stat import tempfile import threading from contextlib import contextmanager @@ -95,11 +96,15 @@ def inventory_lock(config_dir: Path) -> Iterator[None]: process_lock = _process_lock(lock_path) with process_lock: + fd = None try: fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600) - os.fchmod(fd, 0o600) + if stat.S_IMODE(os.fstat(fd).st_mode) != 0o600: + os.fchmod(fd, 0o600) fcntl.flock(fd, fcntl.LOCK_EX) except OSError as exc: + if fd is not None: + os.close(fd) raise InventoryAccessError(f"Cannot acquire inventory lock: {lock_path}") from exc state[key] = {"depth": 1, "fd": fd} try: diff --git a/api/job_identity.py b/api/job_identity.py new file mode 100644 index 00000000..d019f4da --- /dev/null +++ b/api/job_identity.py @@ -0,0 +1,100 @@ +"""Permanent job identity, independent of display and archive naming (#486).""" + +from __future__ import annotations + +import uuid +import re +from pathlib import Path +from datetime import datetime, timezone + +JOB_SCHEMA_VERSION = 4 + + +class JobIdConflictError(ValueError): + api_code = "job_id_exists" + api_status = 409 + + +def new_job_id(jobs_dir: Path | None = None) -> str: + for _ in range(100): + candidate = str(uuid.uuid4()) + if jobs_dir is None or not (jobs_dir / f"{candidate}.json").exists(): + return candidate + raise JobIdConflictError("Could not generate an unused job ID. Please try again.") + + +def validate_job_id(value: object) -> str: + value = str(value or "").strip() + try: + parsed = uuid.UUID(value) + except ValueError as exc: + raise ValueError("Job ID must be a UUID") from exc + if str(parsed) != value or parsed.int == 0: + raise ValueError("Job ID must be a canonical, nonempty UUID") + return value + + +def metadata_job_id(meta: dict) -> str: + job_id = validate_job_id(meta.get("job_id")) + if meta.get("job_key") != job_id: + raise ValueError("Job ID and job key disagree") + return job_id + + +def active_job_ids(config: dict) -> set[str]: + from jobs_api import discover_jobs, resolve_data_root, resolve_scripts_dir + if not config.get("BACKUP_SCRIPTS_DIR"): + return set() + return {job.key for job in discover_jobs(resolve_scripts_dir(config), resolve_data_root(config))} + + +def historical_job_id(value: object) -> str: + """An unresolved historical record has no job identity (#495).""" + try: + return validate_job_id(value) + except (ValueError, TypeError): + return "" + + +def _filename_label(value: str, max_bytes: int, fallback: str) -> str: + """Readable path component, bounded in UTF-8 without splitting a character.""" + value = re.sub(r"[^\w.-]+", "_", str(value or ""), flags=re.UNICODE) + # Keep the '--' separator unambiguous even for user-provided job names. + value = re.sub(r"[-_]{2,}", "_", value).strip("._-") + return value.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore").rstrip("._-") or fallback + + +def job_file_component(job_name: str, location: str, job_id: str, *, name_bytes: int = 100) -> str: + identity = validate_job_id(job_id) + place = _filename_label(location, 10, "unknown") + name = _filename_label(job_name, min(100, name_bytes), "Job") + return f"{name}_{place}_{identity}" + + +def job_log_filename(job_name: str, location: str, job_id: str, run_label: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,120}", run_label): + raise ValueError("Invalid log run label") + # Longer activity run identifiers also fit the shared 255-byte limit. + fixed = f"BBUI-_{_filename_label(location, 10, 'unknown')}_{validate_job_id(job_id)}--{run_label}.log" + component = job_file_component(job_name, location, job_id, name_bytes=255 - len(fixed.encode("utf-8"))) + return f"BBUI-{component}--{run_label}.log" + + +def job_run_date_tag(run_id: str = "") -> str: + """Use the managed run's start time for logs with or without a file list.""" + if re.fullmatch(r"\d{8}T\d{6}Z-[A-Za-z0-9]+", run_id): + started = datetime.strptime(run_id.split("-", 1)[0], "%Y%m%dT%H%M%SZ") + return started.replace(tzinfo=timezone.utc).astimezone().strftime("%Y-%m-%d_%H-%M-%S") + return datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + + +def job_log_paths(directory: Path, job_key: str): + """Find this job's logs across name changes and previous filename formats.""" + for path in directory.glob("*.log"): + name = path.name + if name.startswith(f"Borg-Backup_{job_key}--"): + yield path + elif name.startswith("BBUI-"): + component, separator, _run = name.partition("--") + if separator and component.endswith(f"_{job_key}"): + yield path diff --git a/api/job_settings.py b/api/job_settings.py new file mode 100644 index 00000000..5a83ffcc --- /dev/null +++ b/api/job_settings.py @@ -0,0 +1,30 @@ +"""Explicit job settings, independent of the former backup type (#495).""" + +import re + +JOB_SETTINGS_SCHEMA = 5 +DEFAULT_COMPRESSION = "lz4" +DEFAULT_RETENTION = {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"} +DEFAULT_ICON = "archive" + + +class JobSettingsError(ValueError): + api_code = "job_settings_invalid" + + +def explicit_job_settings(meta: dict) -> tuple[str, dict[str, str]]: + compression = str(meta.get("compression") or "").strip() + if not compression: + raise JobSettingsError("Job compression is missing. Edit and save the job settings.") + source = meta.get("retention") + if not isinstance(source, dict): + raise JobSettingsError("Job retention is missing. Edit and save the job settings.") + retention = {} + for period in DEFAULT_RETENTION: + value = str(source.get(period, "")).strip() + if not re.fullmatch(r"\d+", value): + raise JobSettingsError(f"Job retention for {period} must be a non-negative whole number.") + retention[period] = value + if not any(int(value) for value in retention.values()): + raise JobSettingsError("At least one job retention rule must be greater than zero.") + return compression, retention diff --git a/api/jobs_api.py b/api/jobs_api.py index 4681e320..631b93c5 100644 --- a/api/jobs_api.py +++ b/api/jobs_api.py @@ -210,8 +210,9 @@ def _fallback_runtime_log(config: dict, job_key: str, started_at: str) -> str: if not log_dir.is_dir(): return "" try: + from job_identity import job_log_paths candidates = sorted( - log_dir.glob(f"Borg-Backup_{job_key}--*.log"), + job_log_paths(log_dir, job_key), key=lambda path: path.stat().st_mtime, reverse=True, ) @@ -364,13 +365,14 @@ class JobInfo: restore_test_level: int = 2 restore_test_max_runtime_minutes: int = 0 file_activity: bool = False + archive_prefix: str = "" @property def display_name(self) -> str: loc_label = {"local": "Lokal", "usb": "USB", "smb": "SMB", "storagebox": "Storagebox"}.get( self.location, self.location ) - return f"{self.backup_type.capitalize()} – {loc_label}" + return f"{self.name or 'Backup'} – {loc_label}" class _JobState: @@ -449,13 +451,15 @@ def start( log_handle = None try: if env.get("BORG_UI_FILE_ACTIVITY_RUN") == "1": - from activity_log import activity_log_path - from activity_log_capture import prepare_capture + from activity_log_capture import prepare_capture, read_record - log_file, capture_record_file = prepare_capture(job_key, run_id, Path(env["BORG_UI_ACTIVITY_LOG_DIR"])) + log_file, capture_record_file = prepare_capture( + job_key, run_id, Path(env["BORG_UI_ACTIVITY_LOG_DIR"]), + job_name=env.get("BORG_UI_JOB_NAME", ""), location=env.get("BORG_UI_JOB_LOCATION", ""), + ) log_handle = os.fdopen(os.open(log_file, os.O_WRONLY | os.O_NOFOLLOW), "wb") env["BORG_UI_CAPTURE_LOG"] = str(log_file) - env["BORG_UI_RETAINED_LOG"] = str(activity_log_path(Path(env["BORG_UI_ACTIVITY_LOG_DIR"]), job_key, run_id)) + env["BORG_UI_RETAINED_LOG"] = read_record(capture_record_file)["retained_file"] command = [sys.executable, str(Path(__file__).with_name("activity_log_capture.py")), str(capture_record_file), *command] env["PYTHONUNBUFFERED"] = "1" env["PYTHONIOENCODING"] = "utf-8" @@ -702,7 +706,6 @@ def _discover_jobs_uncached(scripts_dir: Path, data_root: Path | None = None) -> """ Finds backup jobs from canonical JSON metadata. """ - utility_types = {"restore_test"} def _make_job( py_file: Optional[Path], @@ -731,6 +734,7 @@ def _make_job( docker_control: Optional[dict] = None, vm_control: Optional[dict] = None, file_activity: bool = False, + archive_prefix: str = "", ) -> JobInfo: desc_file = py_file.with_suffix(".description") if py_file is not None else None desc_text = ( @@ -742,34 +746,34 @@ def _make_job( else "" ) ) - bt_lc = backup_type.lower() default_docker_control = { - "mode": "all" if ((bt_lc == "appdata") if has_docker is None else bool(has_docker)) else "none", + "mode": "all" if has_docker else "none", "selected": [], "ack_appdata_risk": False, } default_vm_control = { - "mode": "all" if ((bt_lc == "vms") if has_vm is None else bool(has_vm)) else "none", + "mode": "all" if has_vm else "none", "selected": [], "ack_domains_risk": False, } return JobInfo( - key=key or f"{bt_lc}_{location}", + key=key, backup_type=backup_type, location=location, script_path=py_file, name=(name or "").strip(), - has_docker=(bt_lc == "appdata") if has_docker is None else bool(has_docker), - has_vm=(bt_lc == "vms") if has_vm is None else bool(has_vm), + has_docker=bool(has_docker), + has_vm=bool(has_vm), description=desc_text, icon=(icon or "").strip().lower(), icon_color=(icon_color or "").strip().lower(), # Only explicit utility jobs should be filtered from normal # backup selectors. Custom/unknown backup types are still jobs. - is_utility=bt_lc in utility_types, + is_utility=False, standard=standard, enabled=bool(enabled), file_activity=file_activity, + archive_prefix=archive_prefix, compression=str(compression or "").strip(), retention_daily=str(retention_daily or "").strip(), retention_weekly=str(retention_weekly or "").strip(), @@ -799,14 +803,15 @@ def _make_job( # Pflichtfelder V1 try: - key = str(raw["job_key"]).strip() - backup_type = str(raw["backup_type"]).strip() + from job_identity import metadata_job_id + key = metadata_job_id(raw) + backup_type = "" location = str(raw["location"]).strip().lower() script_name = str(raw.get("script") or "").strip() except (KeyError, TypeError, ValueError): continue - if not key or not backup_type or not location: + if not key or not location: continue if location not in {"local", "usb", "smb", "storagebox", "custom"}: continue @@ -838,6 +843,7 @@ def _make_job( backup_type, location, key=key, + archive_prefix=str(raw.get("archive_prefix") or ""), name=str(raw.get("name") or "").strip(), has_docker=has_docker, has_vm=has_vm, @@ -861,7 +867,7 @@ def _make_job( restore_test_max_runtime_minutes=_safe_int(rt_policy.get("max_runtime_minutes"), 0), )) - return list(jobs_by_key.values()) + return sorted(jobs_by_key.values(), key=lambda job: (job.name or job.display_name).casefold()) def _job_metadata_signature(meta_dir: Path, scripts_dir: Path, *, include_files: bool) -> tuple: @@ -892,11 +898,6 @@ def discover_jobs(scripts_dir: Path, data_root: Path | None = None) -> List[JobI root = data_root if data_root is not None else (scripts_dir.parent if scripts_dir.name == "scripts" else scripts_dir) meta_dir = get_jobs_meta_dir(scripts_dir, root) cache_key = f"{scripts_dir.resolve()}::{root.resolve()}" - with _job_discovery_cache_lock: - if cache_key not in _job_metadata_migrations: - migrate_jobs_metadata_dir(scripts_dir, root) - _job_metadata_migrations.add(cache_key) - now = time.monotonic() quick_signature = _job_metadata_signature(meta_dir, scripts_dir, include_files=False) with _job_discovery_cache_lock: @@ -962,6 +963,8 @@ def list_jobs(config: dict, latest_statuses: dict) -> List[dict]: result.append( { "key": info.key, + "job_id": info.key, + "archive_prefix": info.archive_prefix, "backup_type": info.backup_type, "location": info.location, "display_name": info.display_name, diff --git a/api/migrations/job_ids_v1.py b/api/migrations/job_ids_v1.py new file mode 100644 index 00000000..26bc7ee8 --- /dev/null +++ b/api/migrations/job_ids_v1.py @@ -0,0 +1,405 @@ +"""Enrich Main records with permanent job IDs, preserving their payloads (#486).""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import re +from datetime import datetime +from pathlib import Path +from time import monotonic + +from inventory_store import atomic_write_bytes, atomic_write_json, inventory_lock +from job_identity import metadata_job_id, new_job_id, validate_job_id +from security_utils import mask_secrets + +from .audit import append_event, config_dir, now, write_pending_state + +MIGRATION_ID = "job_ids_v1" +INTRODUCED_IN = "2026.09.07.1400" +RECHECK_AFTER_FINAL = True + + +class _Progress: + """Bounded console progress; existing JSONL remains the durable audit.""" + + def __init__(self): + self.started = monotonic() + self.last_report = self.started + self.phase = "" + + def report(self, phase: str, done: int | None = None, total: int | None = None): + tick = monotonic() + if phase == self.phase and done != total and tick - self.last_report < 5: + return + self.phase = phase + self.last_report = tick + counter = f" {done}/{total} files" if total is not None else "" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"[{timestamp}] Migration {MIGRATION_ID}: {phase}{counter}; " + f"elapsed={tick - self.started:.1f}s", flush=True) + + +def _read(path: Path): + if path.is_symlink(): + raise ValueError(f"Migration input must not be a symlink: {path}") + try: + return json.loads(path.read_bytes()) + except (ValueError, UnicodeError) as exc: + raise ValueError(f"Invalid migration JSON: {path}") from exc + + +def _journal(config: dict) -> Path: + return config_dir(config) / "job-id-migration.json" + + +def _jobs(config: dict) -> list[tuple[Path, dict]]: + rows = [] + seen = set() + for path in sorted((config_dir(config) / "jobs").glob("*.json")): + data = _read(path) + if not isinstance(data, dict): + raise ValueError(f"Job metadata must be an object: {path}") + key = str(data.get("job_key") or "") + if not key or key != path.stem or key in seen: + raise ValueError(f"Job filename/key conflict: {path}") + if data.get("job_id"): + metadata_job_id(data) + elif (key != f"{data.get('backup_type', '')}_{data.get('location', '')}" + or not re.fullmatch(r"[A-Za-z0-9_]+", str(data.get("backup_type") or ""))): + raise ValueError(f"Ambiguous legacy job identity: {path}") + seen.add(key) + rows.append((path, data)) + return rows + + +def detect(config: dict) -> dict: + journal = _journal(config) + pending = journal.is_file() and _read(journal).get("status") != "applied" + required = pending or any(not data.get("job_id") for _, data in _jobs(config)) + return {"required": required, "migration_id": MIGRATION_ID} + + +def _paths(config: dict) -> dict: + # Read canonical runtime paths before normal startup applies them to the UI. + from config_api import read_expanded_conf + effective = dict(config) + canonical = read_expanded_conf(config) + for key in ("STATUS_DIR", "RESTORE_TEST_STATUS_DIR", "GLOBAL_BORG_CACHE_BASE"): + if canonical.get(key): + effective[key] = canonical[key] + from restore_tests_api import resolve_restore_test_dir + status = Path(effective.get("STATUS_DIR", "/mnt/user/backup-status")) + return { + "status": status, + "archive": Path(effective.get("STATUS_ARCHIVE_DIR") or status / "archive"), + "restore": resolve_restore_test_dir(effective), + "weekly": Path(effective.get("SNAPSHOT_FILE") or status.parent / "weekly-snapshots.json"), + "legacy_weekly": status / "weekly-snapshots.json", + "cache": Path(effective.get("GLOBAL_BORG_CACHE_BASE") or "/mnt/cache/borg-cache"), + } + + +def _preconditions(config: dict, paths: dict) -> None: + from jobs_api import active_resource_locks, durable_running_states + from status import status_storage_unavailable_reason + if active_resource_locks(config) or durable_running_states(config): + raise RuntimeError("Job ID migration requires backup and restore workers to finish; restart the plugin afterwards") + for path in paths.values(): + reason = status_storage_unavailable_reason(path) + if reason: + raise RuntimeError(f"Job ID migration storage unavailable: {reason}") + if not path.is_absolute(): + raise ValueError("Job ID migration requires absolute configured paths") + + +def _plan(config: dict, paths: dict, progress: _Progress) -> dict: + progress.report("Inspecting jobs and references") + rows = _jobs(config) + assignment = {data["job_key"]: data.get("job_id") or new_job_id() for _, data in rows} + if len(set(assignment.values())) != len(assignment): + raise ValueError("Duplicate active job IDs") + for job_id in assignment.values(): + validate_job_id(job_id) + root = config_dir(config) + run_id = new_job_id() + snapshot = root / "migration-backups" / f"{MIGRATION_ID}-{run_id}" + operations = [] + unresolved = [] + + def resolve(key, path, *, strict=False): + if key in assignment: + return assignment[key] + if key in assignment.values(): + return key + if key: + if strict: + raise ValueError(f"Unresolved active job reference in {path}") + unresolved.append({"file": str(path), "code": "unresolved_historical_job", + "reference": mask_secrets(str(key))}) + return key + + def add(path, data, target=None): + before = path.read_bytes() + after = (json.dumps(data, ensure_ascii=False, indent=2) + "\n").encode() + target = target or path + if _read(path) == data and path == target: + return + if target != path and target.exists(): + raise ValueError(f"Migration target already exists: {target}") + stat = path.stat() + operations.append((path, target, before, after, stat.st_atime_ns, stat.st_mtime_ns)) + + def refs(value, path): + if isinstance(value, list): + return [refs(item, path) for item in value] + if not isinstance(value, dict): + return value + out = {key: refs(item, path) for key, item in value.items()} + if out.get("job_key"): + out["job_key"] = resolve(out["job_key"], path) + return out + + # Preserve every job field. backup_type remains an operational/default value. + migrated_jobs = [] + for path, data in rows: + migrated = copy.deepcopy(data) + job_id = assignment[data["job_key"]] + migrated.update(job_id=job_id, job_key=job_id) + migrated["schema_version"] = 4 + migrated.setdefault("cache_subdir", f"{data['location']}_{str(data['backup_type']).lower()}") + migrated.setdefault("check_flag_name", f".last_check_{str(data['backup_type']).lower()}") + migrated.setdefault("archive_prefix", f"{str(data['backup_type']).lower()}-backup") + if not isinstance(migrated.get("archive_prefixes", []), list): + raise ValueError(f"Invalid archive prefix list: {path}") + migrated["archive_prefixes"] = list(dict.fromkeys([ + migrated["archive_prefix"], *migrated.get("archive_prefixes", []), + ])) + add(path, migrated, path.with_name(f"{job_id}.json")) + migrated_jobs.append(migrated) + from archive_prefix import validate_prefix_ownership + for migrated in migrated_jobs: + validate_prefix_ownership(migrated, migrated_jobs) + + path = root / "schedules.json" + if path.is_file(): + data = _read(path) + out = {} + for key, value in data.items(): + new_key = key if key == "restore_test" else resolve(key, path, strict=True) + if new_key in out: + raise ValueError("Conflicting job schedules") + out[new_key] = value + add(path, out) + + path = root / "repositories.json" + if rows and not path.is_file(): + raise ValueError("Active jobs require the canonical repository inventory") + if path.is_file(): + data = _read(path) + repositories = {repo["repository_key"]: repo for repo in data["repositories"]} + job_repositories = {job["job_key"]: job.get("repository_key") for _, job in rows} + if any(key not in repositories for key in job_repositories.values()): + raise ValueError("Active job references an unknown repository") + for repo in data["repositories"]: + if any(key in job_repositories and job_repositories[key] != repo["repository_key"] + for key in repo.get("used_by", [])): + raise ValueError("Conflicting active repository/job reference") + for field in ("used_by", "source_job_keys"): + if field in repo: + repo[field] = [resolve(key, path, strict=field == "used_by") for key in repo[field]] + add(path, data) + + def historical_identity(data, path, primary): + # Conflicting evidence must not attach old results to the wrong job. + owner = assignment.get(primary, primary) + hints = [data.get("job_key"), data.get("job_id")] + conflict = any(assignment.get(hint, hint) != owner for hint in hints if hint) + active = next((job for _, job in rows if assignment[job["job_key"]] == owner), None) + if active: + backup_type = data.get("backup_type") or data.get("type") + conflict = conflict or bool(backup_type and backup_type != active.get("backup_type")) + conflict = conflict or bool(data.get("location") and data["location"] != active.get("location")) + if conflict: + unresolved.append({"file": str(path), "code": "conflicting_historical_identity"}) + return "" + return resolve(primary, path) + + for directory in set([paths["status"], paths["archive"]]): + for path in sorted(directory.glob("*.status")): + data = _read(path) + old = str(data.get("job_key") or f"{data.get('backup_type', '')}_{data.get('location', '')}") + job_id = historical_identity(data, path, old) + if job_id in assignment.values(): + if data.get("job_id") and data["job_id"] != job_id: + raise ValueError(f"Conflicting status job ID: {path}") + data["job_id"] = job_id + if "job_key" in data: + data["job_key"] = job_id + add(path, data) + + for path in sorted(paths["restore"].glob("*.test")): + data = _read(path) + job_id = historical_identity(data, path, path.stem) + if job_id in assignment.values(): + if data.get("job_id") and data["job_id"] != job_id: + raise ValueError(f"Conflicting restore-test job ID: {path}") + data["job_id"] = job_id + if "job_key" in data: + data["job_key"] = job_id + add(path, data, path.with_name(f"{job_id}.test")) + + for path in set([paths["weekly"], paths["legacy_weekly"]]): + if path.is_file(): + data = _read(path) + out = {} + for key, value in data.items(): + new_key = resolve(key, path) + if new_key in out: + raise ValueError(f"Conflicting weekly observations: {path}") + out[new_key] = value + add(path, out) + + for path in [root / name for name in ( + "restore-runs.json", "notification-queue.json", "notification-deliveries.json", + "restore-history/index.json", + )] + sorted((root / "restore-history/runs").glob("*.json")): + if path.is_file(): + add(path, refs(_read(path), path)) + + path = root / "runtime-recovery.json" + if path.is_file(): + data = refs(_read(path), path) + for entry in data.get("entries", []): + old = str(entry.get("job_key") or f"{entry.get('backup_type', '')}_{entry.get('backup_location', '')}") + job_id = resolve(old, path) + if job_id in assignment.values(): + entry["job_id"] = job_id + add(path, data) + + path = root / "notification-state.json" + if path.is_file(): + data = _read(path) + sent = {} + for key, value in data.get("last_sent", {}).items(): + parts = key.split(":", 2) + if len(parts) == 3: + parts[1] = resolve(parts[1], path) + new_key = ":".join(parts) + if new_key in sent: + raise ValueError("Conflicting reminder state") + sent[new_key] = value + data["last_sent"] = sent + add(path, data) + + # Stage complete originals and proposed bytes before publishing the plan. + # No affected input is changed until this durable plan owns the UUIDs. + progress.report("Saving recovery copies", 0, len(operations)) + snapshot.mkdir(parents=True, mode=0o700) + entries = [] + for index, (source, target, before, after, atime, mtime) in enumerate(operations): + before_file = snapshot / f"{index}.before" + after_file = snapshot / f"{index}.after" + atomic_write_bytes(before_file, before) + atomic_write_bytes(after_file, after) + entries.append({ + "source": str(source), "target": str(target), + "before": str(before_file), "after": str(after_file), + "before_sha256": hashlib.sha256(before).hexdigest(), + "after_sha256": hashlib.sha256(after).hexdigest(), + "atime_ns": atime, "mtime_ns": mtime, + }) + progress.report("Saving recovery copies", index + 1, len(operations)) + return {"migration_id": MIGRATION_ID, "status": "pending", "run_id": run_id, + "timestamp": now(), "assignment": assignment, "operations": entries, + "unresolved": unresolved, "backup_directory": str(snapshot)} + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else "" + + +def _apply_operation(op: dict) -> None: + source, target = Path(op["source"]), Path(op["target"]) + before, after = op["before_sha256"], op["after_sha256"] + staged = Path(op["after"]) + if _digest(staged) != after or _digest(Path(op["before"])) != before: + raise ValueError("Migration snapshot checksum mismatch") + source_hash, target_hash = _digest(source), _digest(target) + if source == target: + if source_hash not in (before, after): + raise ValueError(f"Migration input changed since snapshot: {source}") + elif source_hash not in ("", before) or target_hash not in ("", after) or not (source_hash or target_hash): + raise ValueError(f"Migration rename conflicts with changed data: {source}") + if target_hash != after: + atomic_write_bytes(target, staged.read_bytes()) + os.utime(target, ns=(op["atime_ns"], op["mtime_ns"])) + if source != target and source.exists(): + source.unlink() + fd = os.open(source.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _apply_with_progress(config: dict, progress: _Progress) -> dict: + with inventory_lock(config_dir(config)): + paths = _paths(config) + _preconditions(config, paths) + journal = _journal(config) + plan = _read(journal) if journal.is_file() else None + if plan and plan.get("status") == "applied": + if any(not data.get("job_id") for _, data in _jobs(config)): + raise ValueError("Unmigrated job added after migration; use the job import API") + return {"status": "not_required"} + if not plan: + plan = _plan(config, paths, progress) + atomic_write_json(journal, plan) + else: + progress.report("Resuming saved migration") + write_pending_state(config, migration_id=MIGRATION_ID, introduced_in=INTRODUCED_IN, + run_id=plan["run_id"], source_classification="main_job_metadata") + append_event(config, {"event": "migration_started", "migration_id": MIGRATION_ID, + "run_id": plan["run_id"], "backup_directory": plan["backup_directory"]}) + try: + total = len(plan["operations"]) + progress.report("Updating job references", 0, total) + for index, op in enumerate(plan["operations"], 1): + _apply_operation(op) + append_event(config, {"event": "migration_file_applied", "migration_id": MIGRATION_ID, + "source": op["source"], "target": op["target"], + "action": "enrich_job_identity"}) + progress.report("Updating job references", index, total) + progress.report("Verifying migrated files", 0, total) + for index, op in enumerate(plan["operations"], 1): + if _digest(Path(op["target"])) != op["after_sha256"]: + raise ValueError("Job ID migration verification failed") + progress.report("Verifying migrated files", index, total) + _jobs(config) + plan.update(status="applied", applied_at=now()) + atomic_write_json(journal, plan) + except Exception as exc: + append_event(config, {"event": "migration_failed", "migration_id": MIGRATION_ID, + "error_type": type(exc).__name__, "error": mask_secrets(str(exc))}) + raise + details = {"affected_files": [op["target"] for op in plan["operations"]], + "backup_directory": plan["backup_directory"], "job_count": len(plan["assignment"]), + "unresolved_history": plan["unresolved"], + "actions": ["Preserved originals", "Assigned permanent job IDs", "Updated job references"]} + append_event(config, {"event": "migration_applied", "migration_id": MIGRATION_ID, **details}) + return {"status": "applied", "details": details} + + +def apply(config: dict) -> dict: + progress = _Progress() + progress.report("Starting; web server waits for completion") + try: + result = _apply_with_progress(config, progress) + except Exception as exc: + progress.report(f"Failed during {progress.phase}: {type(exc).__name__}: {mask_secrets(str(exc))}") + raise + progress.report("Completed successfully" if result["status"] == "applied" else "Already applied; no changes") + return result diff --git a/api/migrations/job_settings_v1.py b/api/migrations/job_settings_v1.py new file mode 100644 index 00000000..89b4c17e --- /dev/null +++ b/api/migrations/job_settings_v1.py @@ -0,0 +1,125 @@ +"""Materialize effective settings before removing type dependencies (#495).""" + +import copy +import hashlib +from pathlib import Path + +from config_api import read_expanded_conf +from inventory_store import atomic_write_bytes, atomic_write_json, inventory_lock +from job_identity import metadata_job_id, new_job_id +from job_settings import DEFAULT_RETENTION, JOB_SETTINGS_SCHEMA, explicit_job_settings +from security_utils import mask_secrets + +from .audit import append_event, config_dir, now, write_pending_state +from .job_ids_v1 import _apply_operation, _digest, _jobs, _paths, _preconditions, _read + +MIGRATION_ID = "job_settings_v1" +INTRODUCED_IN = "2026.09.07.2355" +RECHECK_AFTER_FINAL = True + +_ICONS = set("flash appdata photos vms sonstiges docker folder cloud archive database server home music video documents code camera usb shield".split()) +_COLORS = set("blue indigo purple pink green lime violet amber orange red rose teal cyan gray".split()) +_TYPE_COLORS = {"flash": "theme-blue", "appdata": "theme-orange", "photos": "theme-purple", "vms": "theme-green"} + + +def _journal(config): + return config_dir(config) / "job-settings-migration.json" + + +def _needs_migration(meta): + return int(meta.get("schema_version") or 0) < JOB_SETTINGS_SCHEMA + + +def detect(config): + journal = _journal(config) + pending = journal.is_file() and _read(journal).get("status") != "applied" + return {"required": pending or any(_needs_migration(meta) for _, meta in _jobs(config))} + + +def materialize(meta, conf): + result = copy.deepcopy(meta) + metadata_job_id(result) + backup_type = str(result.get("backup_type") or "").strip().lower() + type_suffix = "".join(c if c.isalnum() else "_" for c in backup_type.upper()) + result["compression"] = str(result.get("compression") or "").strip() or conf.get(f"COMPRESSION_{type_suffix}", "lz4") + retention = result.get("retention") if isinstance(result.get("retention"), dict) else {} + result["retention"] = { + **retention, + **{period: str(retention.get(period) if retention.get(period) is not None else "").strip() or str(conf.get(f"RETENTION_{type_suffix}_{period.upper()}", default)) + for period, default in DEFAULT_RETENTION.items()}, + } + if str(result.get("icon") or "").lower() not in _ICONS: + result["icon"] = backup_type if backup_type in _ICONS else "sonstiges" + if str(result.get("icon_color") or "").lower() not in _COLORS: + result["icon_color"] = _TYPE_COLORS.get(backup_type, "") + result.pop("backup_type", None) + result.pop("type_id", None) + result["schema_version"] = JOB_SETTINGS_SCHEMA + explicit_job_settings(result) + return result + + +def _plan(config): + conf = read_expanded_conf(config) + proposed = [(path, materialize(meta, conf)) for path, meta in _jobs(config) if _needs_migration(meta)] + run_id = new_job_id() + backup = config_dir(config) / "migration-backups" / f"{MIGRATION_ID}-{run_id}" + backup.mkdir(parents=True, mode=0o700) + operations = [] + import json + for index, (path, meta) in enumerate(proposed): + before, after = path.read_bytes(), (json.dumps(meta, ensure_ascii=False, indent=2) + "\n").encode() + before_file, after_file = backup / f"{index}.before", backup / f"{index}.after" + stat = path.stat() + atomic_write_bytes(before_file, before) + atomic_write_bytes(after_file, after) + operations.append({ + "source": str(path), "target": str(path), "before": str(before_file), "after": str(after_file), + "before_sha256": hashlib.sha256(before).hexdigest(), "after_sha256": hashlib.sha256(after).hexdigest(), + "atime_ns": stat.st_atime_ns, "mtime_ns": stat.st_mtime_ns, + }) + return {"migration_id": MIGRATION_ID, "status": "pending", "run_id": run_id, + "timestamp": now(), "backup_directory": str(backup), "operations": operations} + + +def apply(config): + with inventory_lock(config_dir(config)): + _preconditions(config, _paths(config)) + journal = _journal(config) + plan = _read(journal) if journal.is_file() else None + if plan and plan.get("status") == "applied": + if any(_needs_migration(meta) for _, meta in _jobs(config)): + raise ValueError("Unsupported old job added after migration; create a new configuration export after upgrading the source installation") + return {"status": "not_required"} + if plan is None: + if not detect(config)["required"]: + return {"status": "not_required"} + plan = _plan(config) + atomic_write_json(journal, plan) + write_pending_state(config, migration_id=MIGRATION_ID, introduced_in=INTRODUCED_IN, + run_id=plan["run_id"], source_classification="job_metadata") + append_event(config, {"event": "migration_started", "migration_id": MIGRATION_ID, + "run_id": plan["run_id"], "backup_directory": plan["backup_directory"]}) + try: + total = len(plan["operations"]) + for index, op in enumerate(plan["operations"], 1): + print(f"[{now()}] Migration {MIGRATION_ID}: saving explicit job settings {index}/{total}", flush=True) + _apply_operation(op) + append_event(config, {"event": "migration_file_applied", "migration_id": MIGRATION_ID, + "source": op["source"], "target": op["target"], + "action": "materialize_settings_and_appearance"}) + for op in plan["operations"]: + if _digest(Path(op["target"])) != op["after_sha256"]: + raise ValueError("Job settings migration verification failed") + explicit_job_settings(_read(Path(op["target"]))) + plan.update(status="applied", applied_at=now()) + atomic_write_json(journal, plan) + except Exception as exc: + append_event(config, {"event": "migration_failed", "migration_id": MIGRATION_ID, + "error_type": type(exc).__name__, "error": mask_secrets(str(exc))}) + raise + details = {"affected_files": [op["target"] for op in plan["operations"]], + "backup_directory": plan["backup_directory"], + "actions": ["Preserved original job metadata", "Saved effective settings, icons and colors", "Removed obsolete job type fields"]} + append_event(config, {"event": "migration_applied", "migration_id": MIGRATION_ID, **details}) + return {"status": "applied", "details": details} diff --git a/api/migrations/registry.py b/api/migrations/registry.py index 87b4d5d2..24aeabf8 100644 --- a/api/migrations/registry.py +++ b/api/migrations/registry.py @@ -6,7 +6,7 @@ from datetime import datetime from typing import Any -from . import canonical_backup_conf_v1 +from . import canonical_backup_conf_v1, job_ids_v1, job_settings_v1 from .audit import ( append_event, config_dir as audit_config_dir, @@ -21,6 +21,8 @@ MIGRATIONS = [ canonical_backup_conf_v1, + job_ids_v1, + job_settings_v1, ] FINAL_STATES = {"applied", "not_required", "not_applicable", "skipped"} diff --git a/api/notification_reminder_api.py b/api/notification_reminder_api.py index a69245a8..ec38d0e0 100644 --- a/api/notification_reminder_api.py +++ b/api/notification_reminder_api.py @@ -491,10 +491,6 @@ def _latest_backup_status_by_key(rows: list) -> dict[str, dict]: explicit_key = str(row.get("key") or "").strip() if explicit_key: keys.append(explicit_key) - backup_type = str(row.get("backup_type") or row.get("type") or "").strip().lower() - location = str(row.get("location") or "").strip().lower() - if backup_type and location: - keys.append(f"{backup_type}_{location}") for key in keys: current = latest.get(key) if current is None or _status_is_newer(row, current): diff --git a/api/report_mail_api.py b/api/report_mail_api.py index dfc1036f..e3782775 100644 --- a/api/report_mail_api.py +++ b/api/report_mail_api.py @@ -164,7 +164,9 @@ def _build_html_report(config: dict, now: Optional[datetime] = None) -> str: status_dir = Path(config["STATUS_DIR"]) store = StatusStore(status_dir) - all_statuses = store.load() + job_meta = _job_metadata_by_key(config) + job_ids = set(job_meta) + all_statuses = [status for status in store.load() if status.key in job_ids] latest = store.get_latest_per_key(all_statuses) generated_at = now or datetime.now() period_start_dt, period_end_dt = _weekly_report_period(generated_at) @@ -175,7 +177,6 @@ def _build_html_report(config: dict, now: Optional[datetime] = None) -> str: rows = [] grouped_rows: dict[str, list[str]] = {} group_stats: dict[str, dict[str, int]] = {} - job_meta = _job_metadata_by_key(config) schedules = _report_schedules(config) planned_job_keys = _planned_job_keys_for_period( set(latest.keys()) | set(job_meta.keys()), @@ -191,7 +192,7 @@ def _build_html_report(config: dict, now: Optional[datetime] = None) -> str: issues = [] log_notes = [] - for key, st in sorted(latest.items(), key=lambda item: _status_sort_key(item[1], item[0])): + for key, st in sorted(latest.items(), key=lambda item: _report_key_sort(item[0], latest, job_meta)): location_key = _location_key(st) meta = job_meta.get(key, {}) job_label = _job_label(key, st, meta) @@ -831,16 +832,13 @@ def _report_key_sort(key: str, latest: dict[str, Any], job_meta: dict[str, dict[ meta = job_meta.get(key, {}) st = latest.get(key) location = _report_location(key, st, meta) - backup_type = str(getattr(st, "backup_type", "") or meta.get("backup_type") or "unknown") - return (*_location_order(location), backup_type.lower(), _report_job_label(key, st, meta).lower(), key.lower()) + return (*_location_order(location), _report_job_label(key, st, meta).casefold()) def _report_location(key: str, st: Any, meta: dict[str, Any]) -> str: value = str(meta.get("location") or getattr(st, "location", "") or "").strip().lower() if value: return value - if "_" in key: - return key.rsplit("_", 1)[1].lower() return "unknown" @@ -871,23 +869,8 @@ def _app_icon_img_html() -> str: def _status_sort_key(st, fallback_key: str) -> tuple: - backup_type_order = { - "appdata": 0, - "flash": 1, - "photos": 2, - "vms": 3, - "VMs": 3, - "sonstiges": 4, - "unknown": 9, - } location = str(getattr(st, "location", "") or "unknown") - backup_type = str(getattr(st, "backup_type", "") or "unknown") - return ( - *_location_order(location), - backup_type_order.get(backup_type, backup_type_order.get(backup_type.lower(), 8)), - backup_type.lower(), - fallback_key.lower(), - ) + return (*_location_order(location), fallback_key.casefold()) def _time_ago(timestamp_str: str, reference: datetime) -> str: diff --git a/api/reports_api.py b/api/reports_api.py index 31068478..7acf3103 100644 --- a/api/reports_api.py +++ b/api/reports_api.py @@ -49,38 +49,47 @@ def _parse_status_file_stem(stem: str): def get_report_jobs(config: dict) -> List[dict]: """Returns all unique jobs found in status files.""" - status_dir = Path(config["STATUS_DIR"]) + from jobs_api import discover_jobs, resolve_scripts_dir, resolve_data_root + metadata = {j.key: j for j in discover_jobs(resolve_scripts_dir(config), resolve_data_root(config))} if config.get("BACKUP_SCRIPTS_DIR") else {} seen = {} - for f in sorted(status_dir.glob("*.status")): - backup_type, location = _parse_status_file_stem(f.stem) - if not backup_type or not location: + for path in sorted(Path(config["STATUS_DIR"]).glob("*.status")): + try: + record = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): continue - key = f"{backup_type}_{location}" - if key not in seen: - seen[key] = { - "key": key, - "backup_type": backup_type, - "location": location, - "display_name": f"{backup_type.capitalize()} ({location})", - } - return list(seen.values()) + job_id = str(record.get("job_id") or "") + backup_type, location = record.get("backup_type", "unknown"), record.get("location", "unknown") + key = job_id + job = metadata.get(key) + if job is None: + continue + seen[key] = { + "key": key, "job_id": job_id, + "backup_type": backup_type, "location": location, + "display_name": (job.name or job.display_name) if job else f"{backup_type.capitalize()} ({location})", + } + return sorted(seen.values(), key=lambda row: row["display_name"].casefold()) def get_report_data(config: dict, job_key: str) -> dict: """Returns full time-series report for a job from its .status files.""" - backup_type, location = _parse_job_key(job_key) + from job_identity import active_job_ids + if job_key not in active_job_ids(config): + raise ValueError("Job no longer exists") + backup_type, location = "", "" status_dir = Path(config["STATUS_DIR"]) runs = [] for f in sorted(status_dir.glob("*.status")): - ftype, floc = _parse_status_file_stem(f.stem) - if ftype != backup_type or floc != location: - continue try: raw = json.loads(f.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue + key = str(raw.get("job_id") or "") + if key != job_key: + continue + backup_type, location = raw.get("backup_type", ""), raw.get("location", "") ts = raw.get("timestamp", "") runs.append({ "timestamp": ts, diff --git a/api/repositories_api.py b/api/repositories_api.py index 3161986a..1f88080e 100644 --- a/api/repositories_api.py +++ b/api/repositories_api.py @@ -2653,6 +2653,7 @@ def save_job_repository_transaction( previous_repository_key: str = "", previous_job_key: str = "", previous_metadata_path: Path | None = None, + create_only: bool = False, ) -> None: """Persist job metadata and both repository link lists as one recoverable unit.""" repo_path = repositories_file(config) @@ -2660,6 +2661,15 @@ def save_job_repository_transaction( previous = Path(previous_metadata_path) if previous_metadata_path else None try: with inventory_lock(repo_path.parent): + from archive_prefix import validate_prefix_ownership + from job_identity import JobIdConflictError, metadata_job_id + if create_only and target.exists(): + raise JobIdConflictError("New job ID already exists") + if metadata_job_id(metadata) != job_key: + raise ValueError("Job identity does not match the repository assignment") + other_jobs = [json.loads(path.read_text(encoding="utf-8")) + for path in target.parent.glob("*.json") if path != target] + validate_prefix_ownership(metadata, other_jobs) old_target = target.read_bytes() if target.exists() else None old_previous = None if previous is not None and previous != target and previous.exists(): diff --git a/api/repository_context.py b/api/repository_context.py index e70d98e2..c2c92126 100644 --- a/api/repository_context.py +++ b/api/repository_context.py @@ -35,7 +35,8 @@ def jobs_dir(config: dict) -> Path: def load_job_metadata(config: dict, job_key: str) -> dict[str, Any]: - key = str(job_key or "").strip() + from job_identity import validate_job_id, metadata_job_id + key = validate_job_id(job_key) if not key: raise RepositoryContextError("Job key is missing") path = jobs_dir(config) / f"{key}.json" @@ -47,6 +48,8 @@ def load_job_metadata(config: dict, job_key: str) -> dict[str, Any]: raise RepositoryContextError(f"Job metadata is not readable: {key}") from exc if not isinstance(payload, dict): raise RepositoryContextError(f"Job metadata is invalid: {key}") + if metadata_job_id(payload) != key: + raise RepositoryContextError(f"Job metadata ID does not match filename: {key}") return payload @@ -152,6 +155,9 @@ def resolve_job_repository_context( raise RepositoryContextError( f"Job '{resolved_job_key}' awaits repository migration ({details})" ) + from job_identity import metadata_job_id + if metadata_job_id(metadata) != resolved_job_key: + raise RepositoryContextError("Job ID does not match repository context") repository_key = str(metadata.get("repository_key") or "").strip() source = inventory if isinstance(inventory, dict) else load_repository_inventory(config) repository = repository_by_key(config, repository_key, inventory=source) diff --git a/api/restore_api.py b/api/restore_api.py index ed0f10e3..8f600aee 100644 --- a/api/restore_api.py +++ b/api/restore_api.py @@ -453,14 +453,10 @@ def acquire_restore_repository_lock(config: dict, info: dict, job_key: str, rest def _archive_filter_rows_for_restore_job(job_key: str, info: dict) -> list[dict]: + from archive_prefix import archive_prefix_from_metadata, job_archive_prefixes job = info.get("job") if isinstance(info.get("job"), dict) else {} - current_prefix = archive_prefix_from_backup_type(job.get("backup_type") if isinstance(job, dict) else "") - stored = job.get("archive_prefixes") if isinstance(job.get("archive_prefixes"), list) else [] - prefixes = normalize_archive_prefixes([ - current_prefix, - *stored, - archive_prefix_from_job_key(job_key), - ]) + current_prefix = archive_prefix_from_metadata(job) + prefixes = job_archive_prefixes(job) return [ { "prefix": prefix, diff --git a/api/restore_tests_api.py b/api/restore_tests_api.py index 62d32d66..c61be398 100644 --- a/api/restore_tests_api.py +++ b/api/restore_tests_api.py @@ -28,6 +28,8 @@ def list_restore_tests(config: dict) -> List[dict]: if not test_dir.exists(): return [] + from job_identity import active_job_ids + job_ids = active_job_ids(config) results = [] for test_file in sorted(test_dir.glob("*.test")): try: @@ -35,9 +37,10 @@ def list_restore_tests(config: dict) -> List[dict]: except (json.JSONDecodeError, OSError): continue - stem = test_file.stem - data["job_key"] = stem - data["key"] = stem or f"{data.get('type', '?')}_{data.get('location', '?')}" + data["job_key"] = str(data.get("job_id") or "") + if data["job_key"] not in job_ids: + continue + data["key"] = data["job_key"] data["time_ago"] = _time_ago(data.get("test_date", "")) data["duration_formatted"] = _fmt_duration(data.get("test_duration_seconds", 0)) data["report_schema_version"] = _safe_int(data.get("report_schema_version"), 0) @@ -123,6 +126,7 @@ def list_restore_test_plan(config: dict) -> dict: "location": job.get("location") or "", "enabled": bool(job.get("enabled", True)), "backup_type": job.get("backup_type") or "", + "archive_prefix": job.get("archive_prefix") or "", "icon": job.get("icon") or "", "icon_color": job.get("icon_color") or "", "is_utility": bool(job.get("is_utility", False)), @@ -137,7 +141,7 @@ def list_restore_test_plan(config: dict) -> dict: "job_meta_file": str((data_root / "config" / "jobs" / f"{key}.json")), }) - rows.sort(key=lambda r: str(r.get("display_name") or "").lower()) + rows.sort(key=lambda r: str(r.get("name") or r.get("display_name") or "").casefold()) return { "defaults": { "interval_days": interval_default, @@ -160,7 +164,7 @@ def update_restore_test_policy(config: dict, job_key: str, policy_raw: dict) -> key = str(job_key or "").strip() if not key: raise ValueError("job_key is missing") - if not re.fullmatch(r"[A-Za-z0-9_]+", key): + if not re.fullmatch(r"[A-Za-z0-9_-]+", key): raise ValueError("Invalid job_key") if not isinstance(policy_raw, dict): raise ValueError("policy must be an object") diff --git a/api/settings_transfer_api.py b/api/settings_transfer_api.py index 4128a5b7..c5b8dcc1 100644 --- a/api/settings_transfer_api.py +++ b/api/settings_transfer_api.py @@ -19,7 +19,6 @@ from typing import Dict, List, Tuple from config_api import get_smb_profile_job_refs -from job_source_paths import SourcePathValidationError, upgrade_job_source_paths from jobs_api import get_jobs_meta_dir, resolve_data_root, resolve_scripts_dir from schedule_api import get_schedules, write_schedules @@ -44,6 +43,40 @@ def __init__(self, api_code: str, message: str): self.api_code = str(api_code or "encrypted_export_invalid") +class ConfigurationExportError(ValueError): + api_code = "configuration_export_unsupported" + + def __init__(self): + super().__init__("This configuration package uses a format that is no longer supported. The import was aborted and no changes were made. Update and migrate the source installation, then create a new configuration export.") + + +def _validate_jobs_bundle(bundle: dict) -> None: + from job_identity import metadata_job_id + from job_settings import JOB_SETTINGS_SCHEMA, explicit_job_settings + from archive_prefix import job_archive_prefixes + from job_source_paths import normalize_source_paths + if not isinstance(bundle, dict) or bundle.get("format") != "bbui-job-bundle-v3" or not isinstance(bundle.get("jobs"), list): + raise ConfigurationExportError() + for job in bundle["jobs"]: + if not isinstance(job, dict) or job.get("schema_version") != JOB_SETTINGS_SCHEMA: + raise ConfigurationExportError() + try: + metadata_job_id(job) + explicit_job_settings(job) + job_archive_prefixes(job) + normalize_source_paths(job.get("source_paths")) + except (ValueError, TypeError, KeyError) as exc: + raise ConfigurationExportError() from exc + + +def _validate_profile_export(payload: dict) -> None: + if (not isinstance(payload, dict) or payload.get("format") != "bbui-profile-secrets-v2" + or not isinstance(payload.get("manifest"), list) + or not isinstance(payload.get("files"), list) + or not isinstance(payload.get("settings_payload"), dict)): + raise ConfigurationExportError() + + def _canonical_profile_payload(config: dict) -> dict: from storage_objects_api import settings_profiles_from_storages profiles = settings_profiles_from_storages(config) @@ -121,7 +154,7 @@ def export_jobs_bundle(config: dict, selected_keys: List[str] | None = None) -> else: passphrase_meta[repository_key] = {"path": pp_path, "exists": False} bundle = { - "format": "bbui-job-bundle-v2", + "format": "bbui-job-bundle-v3", "exported_at": datetime.now(timezone.utc).isoformat(), "jobs": jobs, "repositories": repositories, @@ -232,49 +265,36 @@ def _collect_repository_key_exports(config: dict, bundle: dict) -> dict[str, dic def _normalize_job_key(base: str) -> str: - out = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(base or "").strip()) - while "__" in out: - out = out.replace("__", "_") - return out.strip("_") + value = str(base or "").strip() + return value if value and all(ch.isalnum() or ch in "_-" for ch in value) else "" + + +def _import_identity(config: dict, raw: dict) -> str: + from job_identity import metadata_job_id + return metadata_job_id(raw) def _resolve_import_key(existing: set[str], desired: str, mode: str) -> Tuple[str | None, str]: + from job_identity import new_job_id, validate_job_id key = _normalize_job_key(desired) if not key: return None, "invalid" + validate_job_id(key) + if mode == "rename": + return new_job_id(), "renamed" if key not in existing: return key, "new" if mode == "skip": return None, "skipped_exists" if mode == "overwrite": return key, "overwrite" - if mode == "rename": - idx = 2 - while f"{key}_{idx}" in existing: - idx += 1 - return f"{key}_{idx}", "renamed" return None, "skipped_exists" def _canonical_import_jobs(jobs: list, selected: set[str] | None = None) -> list: - """Upgrade old bundle jobs at the import boundary, never during runtime.""" - normalized: list = [] - for raw in jobs: - if not isinstance(raw, dict): - normalized.append(raw) - continue - source_key = str(raw.get("job_key") or "").strip() - if selected and source_key not in selected: - normalized.append(dict(raw)) - continue - label = source_key or "" - try: - normalized.append(upgrade_job_source_paths(raw, job_key=label)) - except SourcePathValidationError as exc: - raise ValueError( - f"Imported job '{label}' cannot be converted to structured source paths: {exc}" - ) from exc - return normalized + """Read supported jobs without converting old configuration packages.""" + _validate_jobs_bundle({"format": "bbui-job-bundle-v3", "jobs": jobs}) + return [dict(job) for job in jobs] def _job_preview_rows(config: dict, bundle: dict) -> list[dict]: @@ -300,7 +320,7 @@ def _job_preview_rows(config: dict, bundle: dict) -> list[dict]: if not isinstance(raw, dict): continue src_key = str(raw.get("job_key") or "").strip() - key_norm = _normalize_job_key(src_key) + key_norm = _import_identity(config, raw) if _normalize_job_key(src_key) else "" conflict = "new" if not key_norm: conflict = "invalid" @@ -340,7 +360,7 @@ def _job_preview_rows(config: dict, bundle: dict) -> list[dict]: rows.append({ "job_key": src_key, "name": str(raw.get("name") or src_key), - "backup_type": str(raw.get("backup_type") or ""), + "archive_prefix": str(raw.get("archive_prefix") or ""), "location": str(raw.get("location") or ""), "repository_key": repository_key, "repository": { @@ -356,14 +376,11 @@ def _job_preview_rows(config: dict, bundle: dict) -> list[dict]: "suggested_mode": "overwrite" if conflict == "exists" else "skip", "passphrase": {"status": pp_status, "bundle": pp, "local": pp_local}, }) - return rows + return sorted(rows, key=lambda row: row["name"].casefold()) def preview_jobs_bundle(config: dict, bundle: dict) -> dict: - if not isinstance(bundle, dict): - raise ValueError("Invalid bundle") - if bundle.get("format") != "bbui-job-bundle-v2": - raise ValueError("Unknown bundle format") + _validate_jobs_bundle(bundle) normalized_bundle = dict(bundle) normalized_bundle["jobs"] = _canonical_import_jobs( bundle.get("jobs") if isinstance(bundle.get("jobs"), list) else [] @@ -614,7 +631,7 @@ def _apply_repository_inventory(config: dict, bundle: dict, jobs: list[dict], dr } -def import_jobs_bundle( +def _import_jobs_bundle_locked( config: dict, bundle: dict, mode: str = "skip", @@ -626,10 +643,7 @@ def import_jobs_bundle( ) -> dict: if mode not in {"skip", "overwrite", "rename"}: raise ValueError("Invalid import mode") - if not isinstance(bundle, dict): - raise ValueError("Invalid bundle") - if bundle.get("format") != "bbui-job-bundle-v2": - raise ValueError("Unknown bundle format") + _validate_jobs_bundle(bundle) if settings_mode not in {"ignore", "merge", "replace"}: raise ValueError("Invalid settings import mode") @@ -644,7 +658,7 @@ def import_jobs_bundle( row for row in jobs if isinstance(row, dict) and (not selected_set or str(row.get("job_key") or "").strip() in selected_set) ] - inventory_report = _apply_repository_inventory(config, bundle, inventory_jobs, bool(dry_run)) + inventory_report = _apply_repository_inventory(config, bundle, inventory_jobs, True) jobs_dir = _jobs_dir(config) existing_files = {p.stem for p in jobs_dir.glob("*.json")} @@ -652,6 +666,10 @@ def import_jobs_bundle( report: List[dict] = [] applied_jobs: List[Tuple[str, dict]] = [] schedule_updates: Dict[str, dict] = {} + current_jobs = {p.stem: json.loads(p.read_text(encoding="utf-8")) for p in jobs_dir.glob("*.json")} + source_keys = [str(row.get("job_key") or "") for row in inventory_jobs] + if len(set(source_keys)) != len(source_keys): + raise ValueError("Duplicate job identity in import bundle") selected = selected_set per_mode = per_job_mode if isinstance(per_job_mode, dict) else {} @@ -666,27 +684,42 @@ def import_jobs_bundle( mode_job = str(per_mode.get(src_key, mode)).strip().lower() if mode_job not in {"skip", "overwrite", "rename"}: mode_job = mode - final_key, action = _resolve_import_key(existing, src_key, mode_job) + if not _normalize_job_key(src_key): + raise ValueError("Invalid source job identity in import bundle") + final_key, action = _resolve_import_key(existing, _import_identity(config, raw), mode_job) if not final_key: report.append({"job_key": src_key, "status": action}) continue patched = dict(raw) patched["job_key"] = final_key - if final_key != src_key: - name = str(patched.get("name") or final_key) - if f"({src_key})" not in name and src_key: - patched["name"] = f"{name} ({final_key})" + patched["job_id"] = final_key + from job_settings import JOB_SETTINGS_SCHEMA + patched["schema_version"] = JOB_SETTINGS_SCHEMA + from archive_prefix import job_archive_prefixes + old = current_jobs.get(final_key) + patched["cache_subdir"] = old["cache_subdir"] if old else final_key + patched["check_flag_name"] = old["check_flag_name"] if old else ".last_check" + patched["archive_prefixes"] = list(dict.fromkeys([ + *job_archive_prefixes(patched), *(job_archive_prefixes(old) if old else []), + ])) applied_jobs.append((final_key, patched)) existing.add(final_key) if src_key in schedules: schedule_updates[final_key] = schedules[src_key] report.append({"job_key": src_key, "new_job_key": final_key, "status": action, "mode": mode_job}) + from archive_prefix import validate_prefix_ownership + final_jobs = {**current_jobs, **dict(applied_jobs)} + for _, candidate in applied_jobs: + validate_prefix_ownership(candidate, final_jobs.values()) + settings_applied = False settings_report = {"mode": settings_mode, "applied": 0, "conflicts": 0} settings_backup = None settings_payload = bundle.get("settings_payload") if not dry_run: + # Prefix validation and all identity decisions precede inventory writes. + inventory_report = _apply_repository_inventory(config, bundle, inventory_jobs, False) settings_applied, settings_report, settings_backup = _apply_settings_payload( config, settings_payload, @@ -695,13 +728,18 @@ def import_jobs_bundle( ) if not dry_run: + from inventory_store import atomic_write_json for key, raw in applied_jobs: target = jobs_dir / f"{key}.json" - target.write_text(json.dumps(raw, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + atomic_write_json(target, raw) # merge schedules merged = get_schedules(config) merged.update(schedule_updates) write_schedules(config, merged) + from repositories_api import reconcile_repository_usage + from jobs_api import invalidate_job_discovery_cache + reconcile_repository_usage(config) + invalidate_job_discovery_cache() return { "dry_run": bool(dry_run), @@ -716,6 +754,18 @@ def import_jobs_bundle( } +def import_jobs_bundle( + config: dict, bundle: dict, mode: str = "skip", dry_run: bool = True, + selected_jobs: list[str] | None = None, per_job_mode: dict | None = None, + settings_mode: str = "merge", per_profile_mode: dict | None = None, +) -> dict: + _validate_jobs_bundle(bundle) + from inventory_store import inventory_lock + with inventory_lock(_jobs_dir(config).parent): + return _import_jobs_bundle_locked(config, bundle, mode, dry_run, selected_jobs, + per_job_mode, settings_mode, per_profile_mode) + + def _secrets_dir() -> Path: p = Path("/boot/config/borg-backup/secrets") p.mkdir(parents=True, exist_ok=True) @@ -1205,7 +1255,7 @@ def export_jobs_bundle_encrypted(config: dict, password: str, selected_keys: lis passphrase_files = _collect_job_passphrase_files(bundle) key_files = _collect_job_key_files(config, bundle, include_content=True) payload = { - "format": "bbui-job-bundle-secure-v2", + "format": "bbui-job-bundle-secure-v3", "created_at": datetime.now(timezone.utc).isoformat(), "bundle": bundle, "passphrase_files": passphrase_files, @@ -1227,8 +1277,8 @@ def preview_jobs_bundle_encrypted(config: dict, password: str, payload_b64: str) enc = _decode_encrypted_export_payload(payload_b64) plaintext, encryption_format = _decrypt_encrypted_export(enc, str(password or "")) payload = _decode_encrypted_json_payload(plaintext) - if payload.get("format") != "bbui-job-bundle-secure-v2": - raise ValueError("Unknown encrypted jobs format") + if payload.get("format") != "bbui-job-bundle-secure-v3": + raise ConfigurationExportError() bundle = payload.get("bundle") bundle = dict(bundle) if isinstance(bundle, dict) else {} bundle.pop("settings_payload", None) @@ -1395,11 +1445,10 @@ def import_jobs_bundle_encrypted( enc = _decode_encrypted_export_payload(payload_b64) plaintext, encryption_format = _decrypt_encrypted_export(enc, str(password or "")) payload = _decode_encrypted_json_payload(plaintext) - if payload.get("format") != "bbui-job-bundle-secure-v2": - raise ValueError("Unknown encrypted jobs format") + if payload.get("format") != "bbui-job-bundle-secure-v3": + raise ConfigurationExportError() bundle = payload.get("bundle") - if not isinstance(bundle, dict): - raise ValueError("Invalid bundle") + _validate_jobs_bundle(bundle) passphrase_files = payload.get("passphrase_files") if isinstance(payload.get("passphrase_files"), dict) else {} key_files = payload.get("key_files") if isinstance(payload.get("key_files"), dict) else {} borg_key_exports = payload.get("borg_key_exports") if isinstance(payload.get("borg_key_exports"), dict) else {} @@ -1556,7 +1605,7 @@ def export_profile_secrets_backup(config: dict, password: str) -> dict: settings_payload = _canonical_profile_payload(config) entries = _collect_profile_secrets(settings_payload) payload = { - "format": "bbui-profile-secrets-v1", + "format": "bbui-profile-secrets-v2", "created_at": datetime.now(timezone.utc).isoformat(), "settings_payload": { "smb_profiles": settings_payload.get("smb_profiles") if isinstance(settings_payload.get("smb_profiles"), list) else [], @@ -1597,8 +1646,7 @@ def preview_profile_secrets_backup(config: dict, password: str, payload_b64: str enc = _decode_encrypted_export_payload(payload_b64) plaintext, encryption_format = _decrypt_encrypted_export(enc, str(password or "")) payload = _decode_encrypted_json_payload(plaintext) - if payload.get("format") != "bbui-profile-secrets-v1": - raise ValueError("Invalid profile secrets format") + _validate_profile_export(payload) manifest = payload.get("manifest") if isinstance(payload.get("manifest"), list) else [] incoming_settings_payload = payload.get("settings_payload") if isinstance(payload.get("settings_payload"), dict) else None settings_payload = _canonical_profile_payload(config) @@ -1665,8 +1713,7 @@ def import_profile_secrets_backup( enc = _decode_encrypted_export_payload(payload_b64) plaintext, encryption_format = _decrypt_encrypted_export(enc, str(password or "")) payload = _decode_encrypted_json_payload(plaintext) - if payload.get("format") != "bbui-profile-secrets-v1": - raise ValueError("Invalid profile secrets format") + _validate_profile_export(payload) manifest = payload.get("manifest") if isinstance(payload.get("manifest"), list) else [] files = payload.get("files") if isinstance(payload.get("files"), list) else [] diff --git a/api/status_api.py b/api/status_api.py index 5e01c593..9fb971b7 100644 --- a/api/status_api.py +++ b/api/status_api.py @@ -38,7 +38,9 @@ def get_status_data(config: dict, force_snapshot_write: bool = False) -> Dict[st _import_legacy_snapshot_if_needed(snapshot_file, legacy_snapshot_file) store = StatusStore(status_dir) - all_statuses = store.load() + from job_identity import active_job_ids + job_ids = active_job_ids(config) + all_statuses = [status for status in store.load() if status.key in job_ids] latest = store.get_latest_per_key(all_statuses) _auto_write_weekly_snapshot(snapshot_file, latest, force_write=force_snapshot_write) @@ -72,6 +74,7 @@ def get_status_data(config: dict, force_snapshot_write: bool = False) -> Dict[st backups.append( { "key": key, + "job_id": st.job_id, "backup_type": st.backup_type, "location": st.location, "status": st.status, @@ -114,7 +117,8 @@ def get_status_data(config: dict, force_snapshot_write: bool = False) -> Dict[st scripts_dir = resolve_scripts_dir(config) data_root = resolve_data_root(config) jobs = [] - for j in discover_jobs(scripts_dir, data_root): + job_info = {j.key: j for j in discover_jobs(scripts_dir, data_root)} + for j in job_info.values(): jobs.append( { "key": j.key, @@ -131,8 +135,12 @@ def get_status_data(config: dict, force_snapshot_write: bool = False) -> Dict[st verification = build_restore_verification_map(config, jobs) except Exception: verification = {} + job_info = {} for b in backups: + info = job_info.get(b["key"]) + if info: + b["name"] = info.name or info.display_name meta = verification.get(str(b.get("key") or ""), {}) b["restore_verification_status"] = meta.get("status", "never") b["restore_verification_reason"] = meta.get("reason", "") @@ -146,6 +154,7 @@ def get_status_data(config: dict, force_snapshot_write: bool = False) -> Dict[st b["restore_test_policy"] = meta.get("policy") _apply_backup_overdue_metadata(config, backups) + backups.sort(key=lambda row: str(row.get("name") or row.get("backup_type") or "").casefold()) total = len(backups) success = sum(1 for b in backups if b["status"] == "success" and not bool(b.get("backup_overdue"))) @@ -153,7 +162,7 @@ def get_status_data(config: dict, force_snapshot_write: bool = False) -> Dict[st skipped = sum(1 for b in backups if b["status"] == "skipped") error = sum(1 for b in backups if b["status"] == "error") - snapshots = _load_all_snapshots(snapshot_file) + snapshots = {key: values for key, values in _load_all_snapshots(snapshot_file).items() if key in job_ids} check_interval_days = int(config.get("GLOBAL_BORG_CHECK_INTERVAL_DAYS", "30") or "30") @@ -340,7 +349,7 @@ def _status_key(st: Any) -> str: key = getattr(st, "key", None) if key: return str(key) - return f"{getattr(st, 'backup_type', 'unknown')}_{getattr(st, 'location', 'unknown')}" + return str(getattr(st, "job_id", "") or "") def _load_previous_status_sizes(all_statuses: List[Any], latest_per_key: Dict[str, Any]) -> Dict[str, int]: diff --git a/api/unraid_dashboard_widget.py b/api/unraid_dashboard_widget.py index a98a97f6..38619a12 100644 --- a/api/unraid_dashboard_widget.py +++ b/api/unraid_dashboard_widget.py @@ -237,7 +237,9 @@ def _read_status_file_data(config: dict) -> dict[str, Any]: status_dir = Path(str(config.get("STATUS_DIR") or "")) store = StatusStore(status_dir) - latest = store.get_latest_per_key(store.load()) + from job_identity import active_job_ids + job_ids = active_job_ids(config) + latest = store.get_latest_per_key([status for status in store.load() if status.key in job_ids]) except Exception: latest = {} @@ -538,19 +540,7 @@ def _read_jobs(config: dict, backups: list[dict[str, Any]]) -> list[dict[str, An return [row for row in list_jobs(config, latest) if isinstance(row, dict)] except Exception: - return [ - { - "key": str(row.get("key") or "").strip(), - "display_name": _display_job_name(row), - "name": _display_job_name(row), - "enabled": True, - "running": False, - "restore_verification_status": row.get("restore_verification_status") or "never", - "restore_verification_is_overdue": bool(row.get("restore_verification_is_overdue", False)), - } - for row in backups - if str(row.get("key") or "").strip() - ] + return _read_static_jobs(config) def _read_static_jobs(config: dict) -> list[dict[str, Any]]: diff --git a/api/wizard_api.py b/api/wizard_api.py index 021a368f..6f099d00 100644 --- a/api/wizard_api.py +++ b/api/wizard_api.py @@ -5,6 +5,7 @@ scriptless wizard runner. """ +import hashlib import json import os import re @@ -12,12 +13,9 @@ from pathlib import Path from typing import Optional -from job_source_paths import JOB_SCHEMA_VERSION, SourcePathValidationError, normalize_source_paths +from job_source_paths import SourcePathValidationError, normalize_source_paths -def _type_upper(type_id: str) -> str: - return re.sub(r"[^A-Z0-9]", "_", type_id.upper()) - _RUNTIME_MODES = {"all", "selected", "none"} _DOCKER_RUNTIME_MODES = _RUNTIME_MODES | {"except_selected"} @@ -204,6 +202,15 @@ def _source_contains_path_component(raw_sources: list[str], component: str) -> b } +class JobNameValidationError(ValueError): + api_code = "job_name_too_long" + + +def _validate_job_name_length(name: str) -> None: + if len(name.strip()) > 100: + raise JobNameValidationError("Job name must not exceed 100 characters") + + class RetentionValidationError(ValueError): """Expose a stable API code for localized wizard retention errors.""" @@ -241,13 +248,17 @@ def validate_params( require_runtime_ack: bool = True, ) -> None: """Wirft ValueError bei ungültigen Parametern.""" - type_id = params.get("type_id", "").strip() - if not type_id: - raise ValueError("Type ID must not be empty") - if not re.fullmatch(r"[a-z0-9_]+", type_id): - raise ValueError("Type ID may contain only lowercase letters, digits, and underscores") + from archive_prefix import validate_archive_prefix + from job_identity import validate_job_id + params["archive_prefix"] = validate_archive_prefix(params.get("archive_prefix")) + existing_key = str(params.get("existing_job_key") or "").strip() + if allow_existing: + validate_job_id(existing_key) + elif existing_key: + raise ValueError("An existing job ID requires edit mode") if not params.get("job_name", "").strip(): raise ValueError("Job name must not be empty") + _validate_job_name_length(params["job_name"]) retention = _retention_from_params(params) params["file_activity"] = _bool_value(params.get("file_activity"), default=False) for period, value in retention.items(): @@ -303,10 +314,10 @@ def validate_params( raise ValueError("VM domain backup risk must be acknowledged when not shutting down all VMs") from jobs_api import get_jobs_meta_dir - job_key = f"{type_id}_{location}" - meta_target = get_jobs_meta_dir(scripts_dir, data_root) / f"{job_key}.json" - if meta_target.exists() and not allow_existing: - raise FileExistsError(f"Job already exists: {type_id}_{location}") + if allow_existing: + meta_target = get_jobs_meta_dir(scripts_dir, data_root) / f"{existing_key}.json" + if not meta_target.is_file(): + raise ValueError("The job being edited no longer exists") def _repository_from_params(params: dict, ui_config: Optional[dict]) -> Optional[dict]: @@ -344,9 +355,8 @@ def _repository_encryption(repo: Optional[dict], fallback: str = "repokey-blake2 def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dict: - from archive_prefix import archive_prefix_from_backup_type, normalize_archive_prefixes + from archive_prefix import archive_prefix_from_metadata, job_archive_prefixes from jobs_api import discover_jobs, get_jobs_meta_dirs, resolve_data_root - from config_api import read_expanded_conf data_root = resolve_data_root(ui_config) jobs = {j.key: j for j in discover_jobs(scripts_dir, data_root)} @@ -354,19 +364,12 @@ def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dic raise ValueError(f"Unknown job: {job_key}") info = jobs[job_key] - conf = read_expanded_conf(ui_config) - type_id = str(info.backup_type or "").lower() location = str(info.location or "local").lower() # Prefer explicit wizard metadata values if available. meta_source_paths: list[str] = [] meta_exclude_paths: list[str] = [] - meta_compression = "" meta_file_activity = False - meta_keep_daily = "" - meta_keep_weekly = "" - meta_keep_monthly = "" - meta_keep_yearly = "" meta_repository_key = "" meta_mount_before_run = True meta_unmount_after_run = True @@ -396,20 +399,11 @@ def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dic try: meta = candidate meta_exclude_paths = _exclude_paths(meta.get("exclude_paths", [])) - meta_compression = str(meta.get("compression") or "").strip() meta_file_activity = _bool_value(meta.get("file_activity"), default=False) - meta_ret = meta.get("retention") if isinstance(meta.get("retention"), dict) else {} - meta_keep_daily = str(meta_ret.get("daily") or "").strip() - meta_keep_weekly = str(meta_ret.get("weekly") or "").strip() - meta_keep_monthly = str(meta_ret.get("monthly") or "").strip() - meta_keep_yearly = str(meta_ret.get("yearly") or "").strip() meta_repository_key = str(meta.get("repository_key") or "").strip() meta_mount_before_run = bool(meta.get("mount_before_run", True)) meta_unmount_after_run = bool(meta.get("unmount_after_run", True)) - meta_archive_prefixes = normalize_archive_prefixes([ - archive_prefix_from_backup_type(type_id), - *(meta.get("archive_prefixes") if isinstance(meta.get("archive_prefixes"), list) else []), - ]) + meta_archive_prefixes = job_archive_prefixes(meta) meta_docker_control = _runtime_control_from_meta(meta, "docker") meta_vm_control = _runtime_control_from_meta(meta, "vm") break @@ -432,7 +426,8 @@ def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dic repository_context = {} repo_path = "" assignment_error = str(exc) - compression = meta_compression or conf.get(f"COMPRESSION_{_type_upper(type_id)}", "lz4") + from job_settings import explicit_job_settings + compression, effective_retention = explicit_job_settings(meta) # Prefer explicit job metadata name (JSON) over display label with location suffix. # This keeps edited names stable (e.g. "Flash" stays "Flash", not "Flash - Lokal"). @@ -444,7 +439,8 @@ def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dic params = { "job_key": job_key, - "type_id": type_id, + "job_id": job_key, + "archive_prefix": archive_prefix_from_metadata(meta), "job_name": (info.name or "").strip() or info.display_name or job_key, "description": info.description or "", "icon": str(getattr(info, "icon", "") or "").strip().lower(), @@ -465,14 +461,12 @@ def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dic "file_activity": meta_file_activity, "encryption": str(repository_context.get("encryption") or ""), "passphrase": "", - "keep_daily": meta_keep_daily or conf.get(f"RETENTION_{_type_upper(type_id)}_DAILY", "7"), - "keep_weekly": meta_keep_weekly or conf.get(f"RETENTION_{_type_upper(type_id)}_WEEKLY", "4"), - "keep_monthly": meta_keep_monthly or conf.get(f"RETENTION_{_type_upper(type_id)}_MONTHLY", "6"), - "keep_yearly": meta_keep_yearly or conf.get(f"RETENTION_{_type_upper(type_id)}_YEARLY", "3"), + "keep_daily": effective_retention["daily"], + "keep_weekly": effective_retention["weekly"], + "keep_monthly": effective_retention["monthly"], + "keep_yearly": effective_retention["yearly"], "standard": info.standard, - "archive_prefixes": meta_archive_prefixes or normalize_archive_prefixes([ - archive_prefix_from_backup_type(type_id), - ]), + "archive_prefixes": meta_archive_prefixes, "schedule": { "cron": str(schedule.get("cron") or "").strip(), "enabled": bool(schedule.get("enabled", True)), @@ -483,7 +477,6 @@ def load_job_for_wizard(job_key: str, scripts_dir: Path, ui_config: dict) -> dic def generate_flow_preview(params: dict, ui_config: Optional[dict] = None, scripts_dir: Optional[Path] = None) -> dict: """Erzeugt eine textuelle Backup-Flow-Vorschau fuer den Wizard.""" - type_id = params["type_id"].strip() location = params.get("location", "local") source_paths = normalize_source_paths(params.get("source_paths")) exclude_paths = _exclude_paths(params.get("exclude_paths", [])) @@ -543,7 +536,7 @@ def add_step(code: str, message: str, **params) -> None: } if location == "storagebox" else {"checked": False, "exists": False, "needs_init_confirm": False, "message": ""} return { "runner": "scriptless-wizard-runner", - "job_key": f"{type_id}_{location}", + "job_key": str(params.get("existing_job_key") or ""), "summary": { "location": location, "repo": repo_path, @@ -569,10 +562,28 @@ def add_step(code: str, message: str, **params) -> None: def save_job(params: dict, scripts_dir: Path, data_root: Optional[Path] = None, ui_config: Optional[dict] = None) -> dict: + """Keep new-job input intact if its proposed ID was taken before saving.""" + from job_identity import JobIdConflictError, new_job_id + from jobs_api import get_jobs_meta_dir + attempt = dict(params) + for _ in range(10): + try: + return _save_job(attempt, scripts_dir, data_root, ui_config) + except JobIdConflictError: + if str(attempt.get("existing_job_key") or "").strip(): + raise + # Both conflict checks run before any job or repository writes. + attempt["job_id"] = new_job_id(get_jobs_meta_dir(scripts_dir, data_root)) + raise JobIdConflictError("Could not save with an unused job ID. Please try saving again.") + + +def _save_job(params: dict, scripts_dir: Path, data_root: Optional[Path] = None, ui_config: Optional[dict] = None) -> dict: """Speichert Job-eigene Wizard-Metadaten mit kanonischer Repository-Referenz.""" - from archive_prefix import archive_prefix_from_backup_type, normalize_archive_prefixes + from archive_prefix import job_archive_prefixes, validate_archive_prefix + from job_identity import JobIdConflictError, new_job_id, metadata_job_id, validate_job_id from jobs_api import get_jobs_meta_dir - type_id = params["type_id"].strip() + _validate_job_name_length(params.get("job_name", "")) + archive_prefix = validate_archive_prefix(params.get("archive_prefix")) location = params.get("location", "local") description = params.get("description", "").strip() icon = str(params.get("icon", "")).strip().lower() @@ -588,46 +599,46 @@ def save_job(params: dict, scripts_dir: Path, data_root: Optional[Path] = None, existing_job_key = str(params.get("existing_job_key", "")).strip() # ── Wizard-Metadaten schreiben (Phase 2) ───────────────────────────────── - job_key = f"{type_id}_{location}" + requested_id = validate_job_id(params["job_id"]) if "job_id" in params else "" + job_key = validate_job_id(existing_job_key) if existing_job_key else (requested_id or new_job_id(get_jobs_meta_dir(scripts_dir, data_root))) + if params.get("job_id") and params["job_id"] != job_key: + raise ValueError("The permanent job ID cannot be changed") now_iso = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") jobs_meta_dir = get_jobs_meta_dir(scripts_dir, data_root) jobs_meta_dir.mkdir(parents=True, exist_ok=True) meta_path = jobs_meta_dir / f"{job_key}.json" existing = {} - if meta_path.exists(): - try: - existing = json.loads(meta_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, UnicodeDecodeError): - existing = {} - elif existing_job_key and existing_job_key != job_key: - old_meta_path = jobs_meta_dir / f"{existing_job_key}.json" - if old_meta_path.exists(): - try: - existing = json.loads(old_meta_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, UnicodeDecodeError): - existing = {} + if existing_job_key: + existing = json.loads(meta_path.read_text(encoding="utf-8")) + if metadata_job_id(existing) != job_key: + raise ValueError("The permanent job ID cannot be changed") + elif meta_path.exists(): + raise JobIdConflictError("New job ID already exists") + from job_settings import JOB_SETTINGS_SCHEMA, explicit_job_settings mount_before_run = bool(params.get("mount_before_run", existing.get("mount_before_run", True))) unmount_after_run = bool(params.get("unmount_after_run", existing.get("unmount_after_run", True))) docker_control = _runtime_control_from_params(params, "docker", existing) vm_control = _runtime_control_from_params(params, "vm", existing) - archive_prefixes = normalize_archive_prefixes([ - archive_prefix_from_backup_type(type_id), - archive_prefix_from_backup_type(existing.get("backup_type")), - *(existing.get("archive_prefixes") if isinstance(existing.get("archive_prefixes"), list) else []), - ]) + archive_prefixes = list(dict.fromkeys([ + archive_prefix, *(job_archive_prefixes(existing) if existing else []), + ])) metadata = { - "schema_version": JOB_SCHEMA_VERSION, + **existing, + "schema_version": JOB_SETTINGS_SCHEMA, "job_key": job_key, + "job_id": job_key, + "cache_subdir": existing.get("cache_subdir", job_key), + "check_flag_name": existing.get("check_flag_name", ".last_check"), "name": params.get("job_name", "").strip() or job_key, "description": description, "icon": icon, "icon_color": icon_color, "enabled": bool(existing.get("enabled", True)), "standard": "wizard", - "backup_type": type_id, + "archive_prefix": archive_prefix, "archive_prefixes": archive_prefixes, "location": location, "mount_before_run": mount_before_run if location == "smb" else True, @@ -637,18 +648,27 @@ def save_job(params: dict, scripts_dir: Path, data_root: Optional[Path] = None, "source_paths": normalize_source_paths(params.get("source_paths")), "exclude_paths": _exclude_paths(params.get("exclude_paths", [])), "features": { + **existing.get("features", {}), "docker": docker_control["mode"] != "none", "vm": vm_control["mode"] != "none", }, - "docker_control": docker_control, - "vm_control": vm_control, + "docker_control": {**existing.get("docker_control", {}), **docker_control}, + "vm_control": {**existing.get("vm_control", {}), **vm_control}, "compression": str(params.get("compression", "lz4")).strip() or "lz4", "file_activity": file_activity, - "retention": retention, + "retention": {**existing.get("retention", {}), **retention}, "created_at": existing.get("created_at", now_iso), "updated_at": now_iso, } + if not existing or str(existing.get("repository_key") or "") != selected_repository_key: + # Check results belong to a repository, even when the job ID/cache stays + # the same. Preserve legacy markers until the job actually changes repo. + repository_digest = hashlib.sha256(selected_repository_key.encode("utf-8")).hexdigest() + metadata["check_flag_name"] = f".last_check-{repository_digest}" metadata["repository_key"] = selected_repository_key + metadata.pop("backup_type", None) + metadata.pop("type_id", None) + explicit_job_settings(metadata) if isinstance(existing.get("restore_test_policy"), dict): metadata["restore_test_policy"] = dict(existing["restore_test_policy"]) @@ -666,9 +686,12 @@ def save_job(params: dict, scripts_dir: Path, data_root: Optional[Path] = None, previous_repository_key=str(existing.get("repository_key") or ""), previous_job_key=existing_job_key or job_key, previous_metadata_path=previous_meta_path, + create_only=not bool(existing_job_key), ) return { + "job_id": job_key, + "job_key": job_key, "filename": "", "path": "", "script": "", diff --git a/api/wizard_runner.py b/api/wizard_runner.py index 5db6f407..227b1045 100644 --- a/api/wizard_runner.py +++ b/api/wizard_runner.py @@ -40,9 +40,6 @@ def _ensure_runtime_import_paths(backup_scripts_dir: Path) -> None: sys.path.insert(0, raw) -def _type_upper(type_id: str) -> str: - return "".join(c if c.isalnum() else "_" for c in type_id.upper()) - def _env_flag(value: object, default: bool = False) -> bool: if value is None: @@ -327,6 +324,8 @@ def cleanup(self) -> None: def _load_env_from_job(job_key: str, borg_scripts_dir: Path, backup_scripts_dir: Path) -> tuple[dict, dict]: + from job_identity import validate_job_id + job_key = validate_job_id(job_key) _ensure_runtime_import_paths(backup_scripts_dir) from lib.status import load_config # type: ignore @@ -362,10 +361,8 @@ def _load_env_from_job(job_key: str, borg_scripts_dir: Path, backup_scripts_dir: if conf_file.is_file(): env.update(load_config(conf_file)) - type_id = str(meta.get("backup_type") or "").strip().lower() + env["BORG_UI_JOB_KEY"] = job_key location = str(repository_context.get("location") or meta.get("location") or "local").strip().lower() - if not type_id: - raise ValueError("backup_type is missing from job metadata") if location not in {"local", "usb", "smb", "storagebox", "custom"}: raise ValueError(f"invalid location in job metadata: {location}") if location == "storagebox": @@ -374,35 +371,41 @@ def _load_env_from_job(job_key: str, borg_scripts_dir: Path, backup_scripts_dir: env["STORAGEBOX_USER"] = str(storage.get("user", "")).strip() env["STORAGEBOX_BASE_PATH"] = str(storage.get("base_path", "/./backup")).strip() or "/./backup" - tu = _type_upper(type_id) cache_base = env.get("GLOBAL_BORG_CACHE_BASE", "/mnt/cache/borg-cache") - cache_dir = f"{cache_base}/{location}_{type_id}" - date_tag = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + cache_subdir = str(meta["cache_subdir"]) + check_flag_name = str(meta["check_flag_name"]) + if any(not name or Path(name).name != name or name in {".", ".."} for name in (cache_subdir, check_flag_name)): + raise ValueError("Invalid job cache reference") + cache_dir = f"{cache_base}/{cache_subdir}" + from job_identity import job_run_date_tag + date_tag = job_run_date_tag(os.environ.get("BORG_UI_RUN_ID", "")) log_dir = env.get("GLOBAL_LOG_DIR", "/mnt/user/Logs") from job_source_paths import normalize_source_paths source_paths = normalize_source_paths(meta.get("source_paths"), field=f"Job '{job_key}' source_paths") exclude_paths = meta.get("exclude_paths") if isinstance(meta.get("exclude_paths"), list) else [] - meta_compression = str(meta.get("compression") or "").strip() + from job_settings import explicit_job_settings + meta_compression, meta_ret = explicit_job_settings(meta) meta_file_activity = _env_flag(meta.get("file_activity"), default=False) # A managed run keeps its start-time option even if the job is edited. if os.environ.get("BORG_UI_FILE_ACTIVITY_RUN") in {"0", "1"}: meta_file_activity = os.environ["BORG_UI_FILE_ACTIVITY_RUN"] == "1" - meta_ret = meta.get("retention") if isinstance(meta.get("retention"), dict) else {} meta_keep_daily = str(meta_ret.get("daily") or "").strip() meta_keep_weekly = str(meta_ret.get("weekly") or "").strip() meta_keep_monthly = str(meta_ret.get("monthly") or "").strip() meta_keep_yearly = str(meta_ret.get("yearly") or "").strip() - env.setdefault("JOB_NAME", str(meta.get("name") or job_key)) + env["JOB_NAME"] = os.environ.get("BORG_UI_JOB_NAME") or str(meta.get("name") or job_key) env.setdefault("BACKUP_SCRIPTS_DIR", str(backup_scripts_dir)) - env.setdefault("BACKUP_TYPE", type_id) - env.setdefault("BACKUP_LOCATION", location) + env.pop("BACKUP_TYPE", None) + env["BACKUP_LOCATION"] = os.environ.get("BORG_UI_JOB_LOCATION") or location env.setdefault("DATE_TAG", date_tag) env.setdefault("LOG_DIR", log_dir) - # Use job_key for log filename so variants like flash_local/flash_usb are separated. - env.setdefault("LOG_FILE", f"{log_dir}/Borg-Backup_{job_key}--{date_tag}.log") + from job_identity import job_log_filename + env.setdefault("LOG_FILE", str(Path(log_dir) / job_log_filename( + env["JOB_NAME"], env["BACKUP_LOCATION"], job_key, date_tag, + ))) if meta_file_activity and os.environ.get("BORG_UI_CAPTURE_LOG"): env["LOG_FILE"] = os.environ["BORG_UI_CAPTURE_LOG"] # Retention still applies to saved logs; only this run's writes use RAM. @@ -418,17 +421,17 @@ def _load_env_from_job(job_key: str, borg_scripts_dir: Path, backup_scripts_dir: if "@" not in netloc and netloc: env["BORG_REPO"] = urlunsplit((parts.scheme, f"{storagebox_user}@{netloc}", parts.path, parts.query, parts.fragment)) logging.info("Storage Box repository URI has no user; using STORAGEBOX_USER=%s", storagebox_user) - env.setdefault("BORG_COMPRESSION", meta_compression or env.get(f"COMPRESSION_{tu}", "lz4")) + env["BORG_COMPRESSION"] = meta_compression env["BORG_FILE_ACTIVITY"] = "1" if meta_file_activity else "0" env.setdefault("BORG_CHECKPOINT_INTERVAL", env.get("GLOBAL_BORG_CHECKPOINT_INTERVAL", "1800")) - env.setdefault("BORG_CACHE_DIR", cache_dir) + env["BORG_CACHE_DIR"] = cache_dir env.setdefault("BORG_CHECK_INTERVAL_DAYS", env.get("GLOBAL_BORG_CHECK_INTERVAL_DAYS", "30")) - env.setdefault("BORG_CHECK_FLAG_FILE", f"{cache_dir}/.last_check_{type_id}") - env.setdefault("BORG_KEEP_DAILY", meta_keep_daily or env.get(f"RETENTION_{tu}_DAILY", "7")) - env.setdefault("BORG_KEEP_WEEKLY", meta_keep_weekly or env.get(f"RETENTION_{tu}_WEEKLY", "4")) - env.setdefault("BORG_KEEP_MONTHLY", meta_keep_monthly or env.get(f"RETENTION_{tu}_MONTHLY", "6")) - env.setdefault("BORG_KEEP_YEARLY", meta_keep_yearly or env.get(f"RETENTION_{tu}_YEARLY", "3")) - env.setdefault("LOCK_FILE", f"{env.get('LOCK_FILE_DIR', '/var/run')}/borg-backup-{type_id}.lock") + env["BORG_CHECK_FLAG_FILE"] = f"{cache_dir}/{check_flag_name}" + env["BORG_KEEP_DAILY"] = meta_keep_daily + env["BORG_KEEP_WEEKLY"] = meta_keep_weekly + env["BORG_KEEP_MONTHLY"] = meta_keep_monthly + env["BORG_KEEP_YEARLY"] = meta_keep_yearly + env.setdefault("LOCK_FILE", f"{env.get('LOCK_FILE_DIR', '/var/run')}/borg-backup-{job_key}.lock") env["BACKUP_PATHS_JSON"] = json.dumps(source_paths, ensure_ascii=False) env["BACKUP_EXCLUDE_PATHS_JSON"] = json.dumps( [str(path).strip() for path in exclude_paths if str(path).strip()], @@ -636,6 +639,7 @@ def set_phase(phase: str) -> None: BackupJob, BackupJobConfig, RequiredSourcePathsMissing, + UsbMountAccessError, ) from lib.borg_runner import BorgConfig, BorgRunner, parse_borg_stats # type: ignore from lib.notifications import MailConfig # type: ignore @@ -717,7 +721,8 @@ def set_phase(phase: str) -> None: if vm_control["mode"] != "none": vm_mgr = VmManager(VmConfig.from_config(env)) - archive_prefix = f"{env.get('BACKUP_TYPE', 'job')}-backup" + from archive_prefix import archive_prefix_from_metadata + archive_prefix = archive_prefix_from_metadata(meta) abort_on_parity = _env_flag(env.get("ABORT_ON_PARITY_CHECK"), default=True) with BackupJob( job_config, @@ -799,7 +804,7 @@ def set_phase(phase: str) -> None: job.set_result(exit_code, parse_borg_stats(job_config.log_file)) result_code = exit_code return result_code - except RequiredSourcePathsMissing: + except (RequiredSourcePathsMissing, UsbMountAccessError): result_code = 2 return 2 except Exception: diff --git a/borg_backup_ui.py b/borg_backup_ui.py index bb7e614a..63ccaf67 100644 --- a/borg_backup_ui.py +++ b/borg_backup_ui.py @@ -449,7 +449,7 @@ def _security_audit( f"target={tgt} detail={det}" ) - def _require_data_dir_ready(self) -> None: + def _require_data_dir_ready(self, *, read_only: bool = False) -> None: from config_api import read_expanded_conf, ensure_data_dirs conf = read_expanded_conf(self.config) data_dir = str(conf.get("GLOBAL_DATA_DIR", "")).strip() @@ -457,7 +457,7 @@ def _require_data_dir_ready(self) -> None: raise RuntimeError( "GLOBAL_DATA_DIR is not set. Configure a primary data directory in Settings first." ) - ensure_data_dirs(data_dir) + ensure_data_dirs(data_dir, read_only=read_only) def _get_api_token(self) -> str: return _load_or_create_api_token(self.config) @@ -987,6 +987,7 @@ def do_GET(self): "/api/history/log": lambda: self._get_log_file(parsed.query), "/api/jobs/log/window": lambda: self._get_activity_log(parsed.query), "/api/wizard/job": lambda: self._get_wizard_job(parsed.query), + "/api/wizard/new-job-id": self._get_wizard_new_job_id, "/api/wizard/source-dirs": lambda: self._get_wizard_source_dirs(parsed.query), "/api/wizard/runtime-inventory": self._get_wizard_runtime_inventory, "/api/storage/check/jobs": self._get_check_jobs, @@ -1747,6 +1748,20 @@ def _delete_job(self) -> dict: raise RuntimeError("The job is currently running; wait for it to finish") info = jobs[job_key] + passphrase_path = None + if body.get("delete_passphrase"): + from repository_context import resolve_job_repository_context + from repositories_api import read_repository_store + context = resolve_job_repository_context(self.config, job_key, require_passphrase_file=False) + reference = str(context.get("passphrase_ref") or "") + if reference: + for repository in read_repository_store(self.config)["repositories"]: + if str(repository.get("passphrase_ref") or "") == reference: + if repository.get("repository_key") != context["repository_key"] or any( + key != job_key for key in repository.get("used_by", []) + ): + raise ValueError("Passphrase is still referenced by another job or repository") + passphrase_path = Path(reference) conf = read_expanded_conf(self.config) status_dir = Path(self.config.get("STATUS_DIR", "/mnt/user/backup-status")) log_dir = Path(conf.get("GLOBAL_LOG_DIR", "/mnt/user/Logs")) @@ -1777,14 +1792,21 @@ def _delete_job(self) -> dict: delete_artifacts = bool(body.get("delete_artifacts", False)) - # Status-Dateien: *_{backup_type}_{location}.status + # Historical filenames remain unchanged; ownership lives in the payload. deleted_status = 0 + owned_logs = set() if delete_artifacts: - for f in status_dir.glob(f"*_{info.backup_type}_{info.location}.status"): + for f in status_dir.glob("*.status"): try: + record = json.loads(f.read_text(encoding="utf-8")) + if record.get("job_id") != job_key: + continue + log = Path(str(record.get("log_file") or "")) + if log.is_absolute() and log.resolve().parent == log_dir.resolve(): + owned_logs.add(log) f.unlink() deleted_status += 1 - except OSError: + except (OSError, ValueError): pass deleted_restore_test = False @@ -1797,36 +1819,28 @@ def _delete_job(self) -> dict: except OSError: pass - # Log-Dateien: Borg-Backup[_-]{backup_type}--*.log + # New logs use the ID; old logs are selected through owned status records. deleted_logs = 0 if delete_artifacts: - for pattern in ( - f"Borg-Backup_{info.backup_type}--*.log", - f"Borg-Backup-{info.backup_type}--*.log", - ): - for f in log_dir.glob(pattern): - try: - f.unlink() - deleted_logs += 1 - except OSError: - pass - - # Passphrase-Datei (optional) - deleted_passphrase = False - if body.get("delete_passphrase"): - suffix = f"{info.backup_type}_{info.location}".lower() - candidates = [ - Path(f"/boot/config/borg-backup/secrets/.borg-passphrase-{suffix}"), - Path(f"/boot/config/borg-backup/secrets/.borg-passphrase-{info.backup_type}".lower()), - ] - for p in candidates: + from job_identity import job_log_paths + owned_logs.update(job_log_paths(log_dir, job_key)) + for f in owned_logs: try: - if p.is_symlink() or p.exists(): - p.unlink() - deleted_passphrase = True + f.unlink() + deleted_logs += 1 except OSError: pass + # Passphrase-Datei (optional) + deleted_passphrase = False + if passphrase_path is not None: + try: + if passphrase_path.is_symlink() or passphrase_path.exists(): + passphrase_path.unlink() + deleted_passphrase = True + except OSError: + pass + # Schedule-Eintrag immer mit aufräumen (idempotent), # damit keine verwaisten Cron-Trigger für gelöschte Jobs bleiben. delete_schedule(self.config, job_key) @@ -2127,6 +2141,7 @@ def _get_history(self, query_string: str) -> dict: from urllib.parse import parse_qs qs = parse_qs(query_string) filters = { + "job_key": (qs.get("job_key") or [""])[0] or None, "type": (qs.get("type") or [""])[0].lower() or None, "location": (qs.get("location") or [""])[0].lower() or None, "status": (qs.get("status") or [""])[0].lower() or None, @@ -2147,6 +2162,12 @@ def _get_rt_running(self) -> dict: from jobs_api import JobManager return JobManager.get().get_state("restore_test") + def _get_wizard_new_job_id(self) -> dict: + from job_identity import new_job_id + from jobs_api import get_jobs_meta_dir, resolve_data_root, resolve_scripts_dir + jobs_dir = get_jobs_meta_dir(resolve_scripts_dir(self.config), resolve_data_root(self.config)) + return {"job_id": new_job_id(jobs_dir)} + def _get_wizard_job(self, qs: str) -> dict: from urllib.parse import parse_qs as _pqs from wizard_api import load_job_for_wizard @@ -2186,7 +2207,7 @@ def _get_wizard_runtime_inventory(self) -> dict: } def _get_restore_archives(self, qs_str: str) -> dict: - self._require_data_dir_ready() + self._require_data_dir_ready(read_only=True) from restore_api import list_archives_with_context from urllib.parse import parse_qs qs = parse_qs(qs_str) @@ -2196,7 +2217,7 @@ def _get_restore_archives(self, qs_str: str) -> dict: return list_archives_with_context(self.config, job_key) def _get_restore_files(self, qs_str: str) -> dict: - self._require_data_dir_ready() + self._require_data_dir_ready(read_only=True) from restore_api import list_files from urllib.parse import parse_qs, unquote qs = parse_qs(qs_str) @@ -2221,7 +2242,7 @@ def _get_report_data(self, qs_str: str) -> dict: return get_report_data(self.config, job_key) def _get_repo_stats(self, qs_str: str) -> dict: - self._require_data_dir_ready() + self._require_data_dir_ready(read_only=True) from restore_api import get_repo_stats from urllib.parse import parse_qs qs = parse_qs(qs_str) @@ -2231,7 +2252,7 @@ def _get_repo_stats(self, qs_str: str) -> dict: return get_repo_stats(self.config, job_key) def _get_restore_target_dirs(self, qs_str: str) -> dict: - self._require_data_dir_ready() + self._require_data_dir_ready(read_only=True) from restore_api import list_allowed_target_roots, list_target_dirs_with_config from urllib.parse import parse_qs, unquote qs = parse_qs(qs_str) @@ -2247,7 +2268,7 @@ def _get_restore_target_dirs(self, qs_str: str) -> dict: } def _get_restore_state(self, qs_str: str) -> dict: - self._require_data_dir_ready() + self._require_data_dir_ready(read_only=True) from restore_api import get_restore_state from urllib.parse import parse_qs qs = parse_qs(qs_str) @@ -3337,6 +3358,8 @@ def _post_run_job(self) -> dict: extra_env = { "BORG_UI_BORG_SCRIPTS_DIR": str(borg_scripts_dir), "BORG_UI_JOB_KEY": job_key, + "BORG_UI_JOB_NAME": info.name or info.display_name, + "BORG_UI_JOB_LOCATION": info.location, "BORG_UI_APP_VERSION": APP_VERSION, "BORG_UI_REQUEST_ID": request_id, "BORG_UI_REQUEST_SOURCE": source, @@ -3818,7 +3841,7 @@ def _handle_api(self, fn): self.send_header("Content-Length", str(len(content))) cache_control = ( "no-store" - if path in {"/api/widget/summary", "/api/settings/homepage-widget-token", "/api/repositories/key-export"} + if path in {"/api/widget/summary", "/api/settings/homepage-widget-token", "/api/repositories/key-export", "/api/wizard/new-job-id"} else "no-cache" ) self.send_header("Cache-Control", cache_control) diff --git a/docs/changelog.md b/docs/changelog.md index 7d8a727a..5f9e2702 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,6 +6,43 @@ Das Plugin-Manifest `borg-backup-ui.plg` enthaelt nur noch eine kurze nutzerrele ## Unreleased +### Issue #502 - USB mount preflight and access failures +- Require a real mount point as well as a directory and write access before a USB backup proceeds. An existing unmounted directory now follows the existing USB-not-mounted skip path; missing and non-writable targets keep their skipped outcome. +- Inspect the path with `stat()` so I/O errors are retained. Report USB access failures as `usb_mount_access_failed`, with the path and OS error in the existing log, status, lifecycle event and failure notification. The scriptless runner exits cleanly with code 2 before Docker/VM changes and Borg create/maintenance. +- Avoid repository-size and check-state queries during finalization of this USB preflight failure. Keep status persistence and lock cleanup; do not add write probes, automatic mounts, device monitoring or migrations. +- Add focused tests for missing/unmounted/read-only targets, EIO/ENODEV at each preflight stage, passing mounted targets, failure reporting and runner resource release. Actual Unraid hardware verification remains pending. + +### Issues #499, #500 and #501 - external Job-ID tester follow-up +- #499: Refresh the selected job and archive list when returning to Browse & Restore. Discard stale file selections, confirmations and precheck results on source changes; ignore late responses from the previous source. +- #500: Report a missing Borg archive as an actionable API error with German/English guidance. Clear failed file-list loading states and invalid selections, including network and malformed-response failures, so users can reselect or retry without restarting the service. +- #501: Use a repository-qualified check marker for new jobs and when an existing job changes repository. Preserve existing markers for unchanged jobs and renames. Keep the cache directory and old markers intact; returning to a legacy repository can require one fresh check before its qualified marker is available. +- Continue in PR #494 with separate issue-linked commits. No new migration, archive movement, retention change or repository-switch confirmation is introduced. External tester verification remains required before general release. + +### Issue #496 - compatible SSH warning suppression on older Unraid versions +- Add `IgnoreUnknown=WarnWeakCrypto` before `WarnWeakCrypto=no` in the shared Borg SSH transport. Older clients can ignore this specific unsupported setting; newer clients retain warning suppression. +- Preserve existing command-line ignore lists and their position when normalizing custom SSH commands. Keep identity selection, keepalives and other transport settings unchanged; do not introduce an Unraid-version branch or a blanket unknown-option exemption. +- Lower the Community Apps minimum version to Unraid 6.12.5 and align the German/English requirements. Python 3.10 or newer remains required through the separate Python plugin. +- Include the fix on the existing #486 branch and PR #494 at the maintainer's request. SSH-profile backup verification on older and current Unraid installations remains required before general release. + +### Issue #497 - reduce unnecessary idle and navigation writes +- Keep unchanged notification queues untouched during background checks, including missing/empty queues and retries that are not yet due. Preserve locked claims, enqueueing, delivery results and retry persistence. +- Inspect inventory-lock permissions before changing them. Existing private locks no longer receive redundant chmod calls; incorrect permissions are still corrected and failed acquisitions close their file descriptor. +- Make setup-status validation inspect existing data directories, permissions and backing mounts without creating directories or write-test files. Setup and runtime initialization retain their actual write probe; missing or unavailable storage still prevents readiness. +- Extend the read-only storage check to restore archive/file lists, repository statistics, target-directory browsing and restore-state queries. Backup, restore, check and download actions keep their actual write probe. +- Route service Python bytecode and inherited child-process caches to the private RAM directory `/run/borg-backup-ui/pycache`. Reuse the cache across service restarts; let Unraid clear it at reboot. If the optional directory cannot be prepared, disable bytecode writes for that run without falling back to USB. No fixed RAM allocation or job-data relocation is introduced. +- Maintainer captures on 2026-09-09 confirmed zero idle/UI queue replacements and zero redundant inventory chmod events in five-minute/ten-minute measurements. The ten remaining restore probes and ten one-off Python cache writes motivated the focused follow-up above; repeat those checks with the updated candidate. +- Continue on the #486 branch and PR #494 as requested by the maintainer, with a dedicated #497 commit and release-note fragment. Notification/authentication storage locations and login/session behavior remain unchanged. +- Verify idle writes again on Unraid using the existing five-minute idle and ten-minute UI capture procedure. Local regression checks do not replace measurement on the boot USB device. + +### Issue #495 - explicit job settings and supported export formats +- Continue with #486 in PR #494 as explicitly approved by the maintainer; #447 remains frozen. +- Register `job_settings_v1` after `job_ids_v1`. Snapshot original job metadata, materialize effective compression/retention and automatic appearance, preserve other fields, and remove obsolete job type fields in schema 5. Audit and resume interrupted writes without changing explicit values. +- Read saved settings in the Wizard and runner. Use a neutral archive icon for new jobs without an explicit selection; retain migrated theme colors exactly. +- Accept job bundle v3, encrypted job bundle v3 and profile export v2. Check inner formats and required job fields before preview/import writes; retain the existing authenticated encryption envelope. Old exports are deliberately not converted during import. +- Keep unresolved historical files without synthetic job identities, reassignment or cleanup. Current job calculations use only matching UUIDs. +- Remove the restore-test type rule and its settings input. Use the existing archive-size threshold; sample regular files across the archive, cap the target coverage by the configured file limits, and report file-based coverage. A successful full-archive dry-run reports full file coverage. +- Document the need for fresh job AND profile exports in German and English, including a forum announcement draft; Borg archives are unaffected. + ### Issue #463 - file activity log performance - Capture stdout and stderr directly in a RAM-backed runtime file only for jobs with file activity enabled, avoiding an unbounded API-process line buffer and writes into backed-up log directories during the run. - Retain the complete log only after the runner exits, using an independent supervisor that survives a WebUI restart. Preserve cursor identity across the copy, release RAM after successful persistence, and keep a downloadable RAM copy with a visible error if saving fails. History/status references point at the final path. diff --git a/docs/maintainer/attachments/bbui-io-watch.py b/docs/maintainer/attachments/bbui-io-watch.py new file mode 100644 index 00000000..79fb404a --- /dev/null +++ b/docs/maintainer/attachments/bbui-io-watch.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Temporary Linux file-event diagnostic; no plugin imports or file-content reads. + +Outputs JSONL to stdout. Redirect/tee outside the watched trees (on Unraid: /tmp). +READ is an observed file-access event; WRITE_CLOSE only means a writable handle +was closed. Event counts are not byte counts or physical device I/O counts. +Directory discovery reads metadata once; directory-access events are omitted. +""" +import argparse +from collections import Counter +import ctypes +from datetime import datetime +import json +import os +from pathlib import Path +import select +import signal +import struct +import sys +import time + + +EVENTS = { + 0x0001: "READ", 0x0002: "WRITE", 0x0004: "METADATA", + 0x0008: "WRITE_CLOSE", 0x0040: "MOVE_FROM", 0x0080: "MOVE_TO", + 0x0100: "CREATE", 0x0200: "DELETE", 0x0400: "WATCH_DELETED", + 0x0800: "WATCH_MOVED", 0x2000: "UNMOUNT", +} +ISDIR, IGNORED, OVERFLOW = 0x40000000, 0x8000, 0x4000 +HEADER = struct.Struct("iIII") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--phase", required=True, help="Measurement label, e.g. idle or ui") + parser.add_argument("--seconds", type=int, default=300) + parser.add_argument("--data-root", help="Actual plugin GLOBAL_DATA_DIR; no automatic mount or directory creation") + parser.add_argument("--root", action="append", help="Override default flash roots; recursively watch this path") + args = parser.parse_args() + if args.seconds < 1: + parser.error("--seconds must be positive") + if not sys.platform.startswith("linux"): + parser.error("Linux is required") + + def emit(kind, **fields): + print(json.dumps({"time": datetime.now().astimezone().isoformat(timespec="milliseconds"), + "phase": args.phase, "kind": kind, **fields}, ensure_ascii=True), flush=True) + + libc = ctypes.CDLL(None, use_errno=True) + libc.inotify_init1.argtypes = [ctypes.c_int] + libc.inotify_init1.restype = ctypes.c_int + libc.inotify_add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32] + libc.inotify_add_watch.restype = ctypes.c_int + fd = libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC) + if fd < 0: + raise OSError(ctypes.get_errno(), "inotify_init1 failed") + watches, counters, read_batch = {}, Counter(), Counter() + incomplete = False + stopped = False + + def stop(_signum, _frame): + nonlocal stopped + stopped = True + signal.signal(signal.SIGINT, stop) + signal.signal(signal.SIGTERM, stop) + + def warn(message, **details): + nonlocal incomplete + incomplete = True + emit("warning", message=message, **details) + + mask = sum(EVENTS) | 0x01000000 | 0x02000000 # ONLYDIR, DONT_FOLLOW + + def watch_tree(raw, recursive): + path = Path(os.path.abspath(raw)) + if path.is_symlink() or not path.is_dir(): + warn("Directory missing or symlink; not monitored", path=str(path)) + return + wd = libc.inotify_add_watch(fd, os.fsencode(path), mask) + if wd < 0: + warn("Cannot add directory watch", path=str(path), errno=ctypes.get_errno()) + return + previous = watches.get(wd) + if previous: + recursive = recursive or previous[1] + watches[wd] = (path, recursive) + if recursive: + try: + with os.scandir(path) as children: + subdirs = [entry.path for entry in children if entry.is_dir(follow_symlinks=False)] + for child in subdirs: + watch_tree(child, True) + except OSError as exc: + warn("Cannot enumerate directory", path=str(path), errno=exc.errno) + + def flush_reads(): + for path, count in sorted(read_batch.items()): + emit("reads", path=path, events=count) + read_batch.clear() + + try: + roots = args.root or ["/boot/config/borg-backup", "/boot/config/plugins/borg-backup-ui"] + for root in roots: + watch_tree(root, True) + if args.data_root: + base = Path(args.data_root) + # Include root-level stores, but do not recurse into Borg caches or mounted repositories. + watch_tree(base, False) + for child in ("logs", "status", "restore-status"): + watch_tree(base / child, True) + if not watches: + emit("error", message="No directories could be watched") + return 1 + emit("ready", directories=len(watches), seconds=args.seconds, + note="Only paths/events are recorded. Read events are grouped every 10 seconds; no process attribution or physical I/O measurement.") + start = last_reads = time.monotonic() + while not stopped and time.monotonic() - start < args.seconds: + ready, _, _ = select.select([fd], [], [], min(1.0, max(0, args.seconds - (time.monotonic() - start)))) + if ready: + raw = os.read(fd, 1024 * 1024) + offset = 0 + while offset + HEADER.size <= len(raw): + wd, event_mask, cookie, length = HEADER.unpack_from(raw, offset) + offset += HEADER.size + name = os.fsdecode(raw[offset:offset + length].split(b"\0", 1)[0]) + offset += length + if event_mask & OVERFLOW: + warn("Kernel event queue overflow; measurement has gaps") + continue + watched = watches.get(wd) + if watched is None: + continue + parent, recursive = watched + path = str(parent / name) if name else str(parent) + actions = [label for bit, label in EVENTS.items() if event_mask & bit] + if event_mask & ISDIR: + actions = [label for label in actions if label != "READ"] + if "READ" in actions: + read_batch[path] += 1 + for action in actions: + counters[(path, action)] += 1 + mutations = [action for action in actions if action != "READ"] + if mutations: + emit("event", path=path, events=mutations, cookie=cookie) + if event_mask & ISDIR and event_mask & (0x0100 | 0x0080) and recursive: + watch_tree(path, True) + if event_mask & (0x0800 | 0x2000): + warn("Watched directory moved or unmounted; restart measurement", path=path) + stopped = True + if event_mask & IGNORED: + watches.pop(wd, None) + if time.monotonic() - last_reads >= 10: + flush_reads() + last_reads = time.monotonic() + flush_reads() + totals = {} + for (path, action), count in sorted(counters.items()): + totals.setdefault(path, {})[action] = count + for path, counts in totals.items(): + emit("summary", path=path, events=counts) + emit("finished", elapsed_seconds=round(time.monotonic() - start, 1), + incomplete=incomplete, observed_events=sum(counters.values())) + return 1 if incomplete else 0 + finally: + os.close(fd) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/maintainer/issue-486-job-identity-analysis.md b/docs/maintainer/issue-486-job-identity-analysis.md new file mode 100644 index 00000000..14a2ef0a --- /dev/null +++ b/docs/maintainer/issue-486-job-identity-analysis.md @@ -0,0 +1,468 @@ +# Job identity dependency analysis (#486) + +Status: implementation on the isolated #486 branch; awaiting the maintainer test. +No live Unraid data has been migrated by the development agent. + +## Baseline and use of #447 + +- Main reviewed: `3f4058ca4d8a9b947c46467706e4bbd0d4c31de5`, version `2026.09.07.0935`. +- Feature branch: `codex/issue-486-job-ids`. Keep one implementation PR unmerged + until the maintainer completes the feature test. +- Reuse the original dependency inventory from #447/#471 at commit `9ac8715` + (`docs/maintainer/identity-dependencies.json`) as a discovery checklist. +- Cross-check the frozen #447 inventory at `6e0261a` for subsequently discovered + dependencies. Its implementation-specific modules and target behaviors are + not requirements for #486. +- Source of authority: [#486](https://github.com/borgforge/borg-backup-ui/issues/486) + and the maintainer's subsequent approvals. #447 remains frozen. Its code is not + merged or cherry-picked into this branch. + +A static scan of Main's application, API, runtime, UI and plugin sources finds +44 files containing the old identity tokens (`job_key`, `type_id`, `backup_type` +and their runtime/JavaScript variants). This is a candidate list, not a count of +files that must change. Some matches only describe a job or are unused helpers. +Activity-log capture and lookup are present on Main but were not in the original +#471 inventory; the later frozen inventory includes them. + +## Starting records and path rules + +### Readable run filenames (approved maintainer follow-up) + +New runs write `BBUI-__--.log` +and `___.status`. The saved log uses +the same local start-time format with or without the file-list option. Internal +RAM capture paths remain tied to the run ID. Status metadata records the run ID, +file-list option and saved log path so reopening a completed file list does not +depend on the current job name or RAM state after a reboot. Previous activity +filenames remain readable. A run marker at the start of a captured log also +preserves lookup if startup fails before a status file is written. Additional +skip logs retain their reason suffix +(`SKIPPED_PARITY`, `USB_NOT_MOUNTED`, or `USB_NOT_WRITABLE`); the status payload +records the corresponding reason and points to the complete run log. +Existing files are not renamed and do not require another migration. + +The filename label records the name at run start. Unsafe filename characters +are replaced, and the label is limited to 100 UTF-8 bytes (or less if needed +to keep a longer activity filename within 255 bytes). This does not truncate +the stored job name. The Wizard input and save API limit job names to 100 +characters. New status payloads include the full job name and ID; log headers +include both as well. + +Log lookup, retention and optional job-artifact deletion match the full ID in +both old and new filenames. Activity-log lookup after a restart matches the +full job ID and run ID independently of the current job name. Status readers +continue to use the payload ID and preserve the date-first filename layout. + +The canonical starting objects are the existing JSON files under +`/config/jobs/`, resolved by `jobs_api.get_jobs_meta_dir()` and +`repository_context.jobs_dir()`. Their current `job_key` and filename are mapping +evidence. `backup_type` plus location is secondary evidence to validate, not a +reason to guess that two records belong to the same job. Names alone are never +identity evidence. + +For an installation whose data root is `/mnt/user/borg_backup_ui`, the canonical +job directory is `/mnt/user/borg_backup_ui/config/jobs/`. This is a path example, +not a read of the maintainer's current Unraid installation. + +Resolve status, restore-test, weekly-snapshot, cache and log paths from the actual +configuration. They can live outside the data root. In particular, Main supports +both the configured/current weekly snapshot location and a legacy copy under +the status directory. Inventory both before normal readers can import or write +them. Preserve application-owned data and unrelated fields in every affected file. + +## Persistent files: readers, writers and required identity work + +Names below refer to existing Main functions. A function listed as a reader may +also cause a write through a helper; notable examples are called out explicitly. + +| Existing store | Main writers | Main readers | Required work in #486 | +| --- | --- | --- | --- | +| `config/jobs/.json` | `wizard_api.save_job()` through `repositories_api.save_job_repository_transaction()`; `BackupUIHandler._put_job_enabled()`; `restore_tests_api.update_restore_test_policy()`; `settings_transfer_api.import_jobs_bundle()` | `jobs_api._discover_jobs_uncached()` / `discover_jobs()` / `list_jobs()`; `wizard_api.load_job_for_wizard()`; `repository_context.load_job_metadata()`; direct readers listed below | Persist one UUID per job; address that job by UUID across reads and edits. Preserve all settings and unknown JSON fields, including policies, icons/colors and original timestamps. Persist the full current archive prefix and retain all prior prefixes. | +| `config/schedules.json` and managed root crontab block | `schedule_api.save_schedule()`, `write_schedules()`, `delete_schedule()`, `apply_all_schedules()`; import; existing orphan cleanup | `get_schedules()`; Jobs, Dashboard, notifications, widgets, report mail, wizard | Convert backup-job map keys and generated run requests to IDs. Keep expressions/enabled flags. The service key `restore_test` remains a service key. Finish conversion before orphan cleanup can inspect the new inventory. | +| `config/repositories.json`: `used_by`, `source_job_keys` | `save_job_repository_transaction()`, `_link_repository_to_job_locked()`, `_unlink_job_from_repositories_locked()`, `reconcile_repository_usage()`, import | `repository_assignment_report()`, `resolve_job_repository_context()`, `CheckManager._repository_command()`, repository/storage UI and health | Convert contained job references consistently. Preserve repository IDs, storage references, credentials, paths and statistics. Do not introduce a new repository model or rename fields solely for cleanup. | +| `STATUS_DIR/*.status`, and archived status files where configured/present | `BackupJob._save_status()`, `_save_skip_status()` -> `status.BackupStatus.save()`; `StatusStore.load(move_to_archive=True)` can move files | `BackupStatus.from_file()`, `StatusStore.get_latest_per_key()`; `status_api.get_status_data()`; `history_api.get_history_data()`; `reports_api`; report mail and widgets | Add IDs to unambiguous historical records, preserve their evidence and filenames, and write IDs on future runs. Select/group by payload ID rather than parsing names. Inventory archived copies without changing Main's retention or display coverage. | +| `weekly-snapshots.json` at configured/current and legacy paths | `status_api._auto_write_weekly_snapshot()`; `_import_legacy_snapshot_if_needed()` copies a legacy file | `_load_last_week_sizes()`, `_load_all_snapshots()`, `_load_previous_status_sizes()` | Remap the existing job-keyed arrays to IDs; preserve week/size observations and both original inputs. Handle ambiguous keys without silently merging or discarding values. No new weekly-observation schema. | +| `/.test` | `runtime/scripts/borg_restore_test.py: RestoreTest._write()`; explicit delete handler | `restore_tests_api.list_restore_tests()`, `list_restore_test_plan()`, `build_restore_verification_map()`, `_load_test_file()`; runtime test reader; widget readers | Preserve complete result payloads, test dates, validity, level, report details and archive references. Use IDs for lookup and ownership, including any necessary filename change. Policies remain in their existing job JSON. | +| `config/restore-runs.json` | `restore_api._persist_restore_runs()` from the existing async lifecycle | `_ensure_restore_runs_loaded()`, `list_restore_runs()`, `get_restore_state()` | Enrich existing run records with the job ID. Preserve the independent `restore_id`, selected archive, paths, state and recovery details. Do not treat an active operation as safe to rewrite while it is running. | +| `config/restore-history/index.json` and `runs/.json` | `_record_restore_history()`, `_history_summary_from_run()`, `_history_detail_from_run()`, `_write_history_index()` | `_read_history_index()`, `list_restore_history()`, `get_restore_history_detail()` | Keep the existing index/detail structure and restore IDs; update job references consistently in both. Preserve historical names, repositories, results and chronological order. | +| `config/notification-state.json` | `write_notification_state()`, `mark_reminder_sent()`, `clear_reminder_prefix()`, `cleanup_reminder_state()` | `read_notification_state()`, `reminder_allowed()`, reminder diagnostics | Remap only job identity in existing reminder keys. Preserve sent timestamps and due markers so migration does not reset reminders. | +| `config/notification-queue.json`, `config/notification-deliveries.json` | `enqueue_event_apprise()`, `_append_queue_item()`, `drain_notification_queue()`, `_record_delivery_status_unlocked()` | Queue drain, `read_notification_delivery_status()`, system health | Enrich/remap job references in pending events and delivery records where resolvable. Preserve event IDs, retry state, messages and delivery history. Do not send notifications during migration. | +| `config/runtime-recovery.json` | `record_runtime_stopped()`, `mark_runtime_restarted()`, `acknowledge_runtime_recovery()` | `read_runtime_recovery_state()`, `pending_runtime_recovery_entries()`, `summarize_runtime_recovery()` | Carry stable job attribution alongside existing recovery entries. Their entry IDs and Docker/VM targets are separate identities and remain unchanged. No new recovery workflow. | +| Widget cache, normally `/boot/config/plugins/borg-backup-ui/widget-status.json` | `write_unraid_dashboard_widget_cache()`, startup/status-file cache writers | Cache readers, `plugin/widget-status.php`, Unraid widget | Rebuild derived job references from migrated data after startup is safe. Preserve widget structure, labels and existing counters. This cache is not authoritative job/history evidence. | + +Backup History does not have a separate canonical history database on Main: +`history_api` and `reports_api` read `.status` files. Restore History has the +separate index/detail files shown above. Updating only `config/jobs` leaves both +sets of consumers disconnected. + +`status.SnapshotManager`, `status.RestoreTest` and `StatusStore.aggregate_by_key()` +contain older key-based helpers. The production-tree search found no external +call sites for these helpers. They are not a reason to add another live migration +path or modernize unused code. Verify call sites again when implementation changes +their callers. + +## Functions that construct or decode the mutable identity + +| Main function | Existing dependency | Required separation | +| --- | --- | --- | +| `wizard_api.validate_params()`, `generate_flow_preview()`, `save_job()` | Construct `type_id + '_' + location`; filename doubles as conflict/identity check | UUID owns the job. Validate the complete prefix independently, including the approved repository-scoped overlap rule. | +| `jobs_api._discover_jobs_uncached()` and its `_make_job()` helper | Read `job_key`; can synthesize `backup_type + '_' + location` | Read the persisted ID; preserve descriptive/default inputs independently. | +| `status.BackupStatus.key`, `status_api._status_key()` | Return/reconstruct `backup_type + '_' + location` | Read the ID from the record; do not recreate a second active identity after migration. | +| `history_api.get_history_data()` | Splits status filename into type and location; its `type` filter uses those parts | Read job ownership from payload. Preserve run descriptors and time ordering. Adapt actual job selectors to ID and name. | +| `reports_api._parse_job_key()`, `_parse_status_file_stem()`, `get_report_jobs()`, `get_report_data()` | Parse keys/filenames, reconstruct keys and filter timeseries by type/location | Group/select the same logical job by ID across renames and prefix edits. Preserve report calculations. | +| `BackupJob._send_notification_event()`, `_save_status()` | Build notification/reminder attribution from type/location; status save has a runtime-key fallback | Carry the ID from the admitted job run through status and notification writes. | +| `notification_reminder_api._latest_backup_status_by_key()` | Falls back to a constructed type/location key | Use persisted IDs for schedule/status/proof joins. | +| `archive_prefix.archive_prefix_from_job_key()` and `archive_prefix_from_backup_type()` | Turn mutable identity inputs into an archive prefix | Operational callers read the full prefix from job metadata. A migration-only derivation of the old actual prefix is allowed. | +| `restore_api._archive_filter_rows_for_restore_job()` | Combines type-derived, recorded and key-derived prefixes | Use the retained full current/previous prefixes of the ID-selected job. | +| `check_api.CheckManager._repository_command()` | Derives a prune prefix from the selected job key | Resolve the selected ID to its explicit current prefix and unchanged retention settings. Keep native Borg prune. | +| `wizard_runner._load_env_from_job()` | Uses type/location for cache directory, check flag, defaults, log name and lock name | Separate job/run ownership from operational data. Preserve existing cache/check references and effective settings across migration and prefix edits. | +| `BackupUIHandler._delete_job()` | Uses type/location file globs for optional status/log deletion and legacy secret guesses | Preserve confirmation and deletion scope; select owned job artifacts by ID/evidence. Do not broaden deletion or delete repositories. | +| `ui/js/pages/wizard.js: saveWizardJob()` | Reconstructs the key for a separate schedule request after saving the job | Use the ID returned by the save operation. | + +## Other active consumers to update, not replace + +- HTTP boundary in `borg_backup_ui.py`: job run/cancel/enabled/delete, schedules, + wizard edit, report selection, repository maintenance, restore browse/precheck/ + execution, test policy/run selection, activity/live-log requests, and request + context logging. Trace request JSON, query parameters and responses together. +- `JobManager.start()`, `get_state()`, `is_running()`, `stream_output()`; + `active_resource_locks()`, `durable_running_states()`, `stream_job_output()`. +- `wizard_runner.main()`, `ResourceLockSet`, `job_control.JobControl` and + `request_cancel()`: job ownership moves to IDs; resource IDs and independent + run IDs retain their current roles. +- `activity_log.resolve_activity_run()` / `activity_log_path()`; + `activity_log_capture.prepare_capture()`, `capture_record()`, `running_captures()` + and `retain_capture()`. RAM state lives under `/run/borg-backup-ui/jobs/` and + `/run/borg-backup-ui/activity-logs/`; retained logs use the configured log path. + Do not rewrite live ownership files under running workers. Preserve readable + retained logs and their recorded references. +- `repository_context.resolve_job_repository_context()`, + `smb_mount._job_smb_meta()` / `ensure_smb_mount_for_job()`: resolve the same + repository/storage by job ID without changing mount behavior. +- `runtime/scripts/borg_restore_test.py: discover_repos()`, `RestoreTest.test_repo()` + and `_notify_event()`; API plan, verification, run and policy handlers. Preserve + current test behavior; repairing the missing cron trigger belongs to #493. +- `status_api.get_status_data()`, `jobs_api.list_jobs()`, + `report_mail_api._job_metadata_by_key()`, `_planned_job_keys_for_period()`, + `_statuses_for_key_in_window()` and `_repo_growth_7d()`; reminder joins; + `homepage_widget_api._read_jobs()` / `_read_latest_backup_rows()` and Unraid + widget `_backup_rows_by_key()`, `_job_cache_items()`, `_read_static_jobs()`. +- Import/export: `export_jobs_bundle()`, `_job_preview_rows()`, + `_resolve_import_key()`, `import_jobs_bundle()` and encrypted wrappers. Preserve + existing export content and modes. Importing as a copy creates a new ID; + updating an existing job keeps its ID. Old bundles without IDs need a bounded + conversion at the existing import boundary, not a second permanent identity. +- `system_health_api._collect_job_health()`, + `factory_reset_api._active_operation_blockers()`, and + `support_bundle_api.create_support_bundle()`: adapt actual references/checks; + retain existing reset semantics and full sanitized diagnostic content. +- UI consumers: `ui/js/core/app-core.js` and the `jobs`, `wizard`, `dashboard`, + `history`, `reports`, `restore`, `restore-tests`, `storage` and `settings` pages. + Inspect action attributes, selection values, lookup maps, import selections + and name labels together. UUID values must not become visible sorting labels. + +## Uses that must not be blindly replaced + +- `backup_type` currently supplies fallback icons/colors, display descriptions, + type-specific compression/retention defaults and some Docker/VM defaults. + Preserve the effective values when removing it from identity. Do not put a + UUID in these fields or infer their meaning from a UUID. +- Repository/storage/profile keys and Borg repository IDs identify different + objects. Keep them and existing credential references unchanged. +- `runtime/lib/borg_runner.py: BorgRunner.create()` and `prune()` already accept + an archive prefix. Change the value supplied by the job layer as needed; retain + native `borg prune --verbose --list --show-rc` and current-prefix retention. +- `config_api._scan_per_repo_passphrases()` calls a descriptive filename fragment + `type_id`; it is not an active job lookup. Preserve secret-file references. +- `storage_profiles_api.build_storage_repo_uri()` has no production-tree caller + in the reviewed Main. Its argument name alone does not justify changing it. +- No source-manifest feature, replacement status/report schema, shortened support + bundle, combined-prefix retention, migration assistant, or new page redesign + is authorized by this analysis. + +## Reuse Main's migration infrastructure + +Use `api/migrations/registry.py: run_startup_migrations()` and the existing +`detect(config)` / `apply(config)` contract. Add one job-ID migration module to +that registry; do not import the migration subsystem from #447. + +Reuse `api/migrations/audit.py` for state and JSONL audit, `inventory_store` for +existing atomic-write/lock primitives, and the snapshot pattern demonstrated by +`canonical_backup_conf_v1`. A multi-file migration still needs its own durable +old-key-to-ID assignment and progress information; a single atomic file write +does not make a set of writes atomic. + +The existing startup sequence evaluates migrations before enabling normal +services. Reuse `_evaluate_startup_migrations()`, `startup_state`, and +`_activate_runtime_services()` for failure reporting and blocking. Ensure all +configured affected storage paths are ready before planning, and establish that +old backup/restore workers are no longer writing affected records. If existing +startup/process checks cannot provide this prerequisite, report that specific +gap before proposing any broader mechanism. + +For the new migration: + +1. Read source JSON directly, validate exact old keys and references, and resolve + configured paths. Avoid normal readers that write, such as status snapshot + generation, job-directory migration or repository reconciliation. +2. Preserve affected originals and persist the ID assignment before changing + jobs or dependent records. An interrupted retry reuses the assigned IDs. +3. Enrich records and remap only necessary identity references. Preserve fields, + existing prefix lists, evidence, timestamps and unknown historical data. +4. Verify active references. Preserve/report unresolved historical records without + guessing or deleting them. Never declare success after a partial conversion. +5. Let normal startup regenerate cron and derived caches only after success. + Existing orphan cleanup must not delete schedules mid-conversion. + +## Focused checks to prepare + +- Representative Main jobs: built-in/custom types, underscores in old keys, + explicit and automatic icons/colors, multiple locations, multiple recorded + prefixes, restore policies, enabled/disabled schedules and unknown fields. +- Linked `.status`, weekly observations, `.test`, restore index/detail records, + repository references and notification state. Compare all original evidence. +- First migration, repeat invocation, interrupted write/retry, ambiguous history, + duplicate active IDs and unavailable configured storage. +- Rename and prefix edit keep identity, schedules, history and test/check proof; + future archive names use the full prefix. Exercise approved prefix conflicts. +- Use UUID/file order different from name order. Verify alphabetical job names + within existing groups in every list/selector; preserve run chronology. +- Preserve native prune output, import/export behavior and support-bundle scope. +- Keep focus on #486: no implementation of #493 and no adoption of #447 extras. + +The original analysis below establishes the baseline. Implementation and +copy-based migration validation are documented in the final section; the live +Unraid installation test remains the maintainer's next step. + +## Supplied production copy: 2026.09.07.0935 + +On 2026-09-07 the maintainer supplied an unmodified production-data copy for +#486. The copied application's `APP_VERSION` confirms `2026.09.07.0935`. +Inspection used direct, read-only JSON parsing, not copied application code or +normal application readers that could write. No migration was executed during +this initial inventory; subsequent copy-only migration results are listed below. +Production payloads, configuration secrets and authentication data are not +included in this branch. A local ignored fingerprint manifest records the 671 +JSON records read for the identity audit. + +The actual job root in this copy is `/boot/config/borg-backup/config/jobs/`, +represented by `borg-backup/config/jobs/` in the supplied directory. Runtime +status, restore-test, log and cache paths are under `/mnt/user/borg_backup_ui`, +represented by `borg_backup_ui/`. Do not mistake `GLOBAL_DATA_DIR` for the +canonical job root. Main's `_apply_runtime_dirs_from_conf()` applies the paths +in canonical `config/backup.conf`; the older `STATUS_DIR` in the UI bootstrap +configuration is not the effective status path for these records. + +| Store in the supplied copy | Observed baseline | Required preservation check | +| --- | --- | --- | +| Job metadata | 14 schema-v3 jobs, no `job_id`; unique keys; filenames, `job_key` and type/location pairs agree | Assign exactly 14 stable IDs; preserve all original settings and fields | +| Schedules | 11 backup schedules, 10 enabled; all keys resolve | Preserve expressions and enabled flags; the absent `restore_test` trigger belongs to #493 | +| Repositories | 13 repositories for 14 jobs; both reverse-reference lists match job assignments | Preserve the shared repository and all 14 assignments | +| Regular backup status/history | 568 `.status` files; all resolve to existing jobs and agree with their filenames | Preserve all results, check fields, timestamps, statistics and log references | +| Current weekly snapshots | 14 keys, 28 week/size observations; all resolve | Remap keys without changing the observations | +| Legacy weekly snapshots under `status/` | 17 keys, 107 observations; six keys with eight observations have no current job | Preserve unresolved history and both input files; do not silently merge or replace the current snapshot | +| Restore-test results | Six `.test` files; filenames and payload type/location agree with existing jobs; dates/results present | Preserve full reports and the existing proof, independent of whether the current policy is enabled | +| Notification delivery history | 200 records; 196 resolve, four refer to two absent jobs | Preserve all delivery records; do not guess the four historical owners | +| Reminder state | Three job-specific reminder keys | Preserve event names, due markers and sent timestamps | +| Cache/check markers | All 14 current jobs have their expected existing `.last_check_` file | Continue to reference existing markers and caches after introducing IDs | + +Important cases already represented by this copy: + +- Eleven jobs have empty explicit icon and color fields. Their effective display + depends on Main's type defaults; empty fields must not become a visual change + when identity changes. Three jobs carry explicit icon/color selections. +- Ten jobs have no explicit prefix list, three have one entry, and one has two + current/historical prefixes. Retain both entries of the latter. Two jobs share + a repository; their current and recorded prefixes do not overlap. Identical + prefixes across different repositories are present and remain permitted. +- The two weekly files have 11 shared job keys, with different arrays for all + 11. Main imports the legacy file only when the current file is absent; this + migration must not introduce a new merge policy. +- The six unmatched legacy weekly keys and two unmatched notification keys do + not exist in the current job inventory. Their eight weekly observations and + four deliveries account for the previously reported unresolved-history + diagnostics by store and count. That is historical missing ownership, not + evidence that an active job is missing its new ID. +- Existing backup outcomes are 536 success, 19 skipped, nine error, three + warning and one cancelled. Check status is already `unknown` in 19 records + and `ok` in 549. Preservation tests must compare these original values rather + than manufacture success or known check results. +- The status recycle directory also contains 69 `.status` files and four old + restore-test files. Two status records and one test do not exactly match a + current job key. Keep recycle contents separate from active history; do not + revive them or infer ownership by case folding/name similarity. + +Coverage gaps for synthetic fixtures, not missing production files: + +- Restore runs, Restore History, the notification queue and runtime recovery + are present but empty. Add synthetic nonempty examples from their existing + Main schemas to test identity references without creating real operations. +- Add controlled prefix conflicts, ambiguous references, duplicate IDs, + unknown fields and interrupted/repeated migration cases. The real copy does + not demonstrate those failure/retry behaviors. +- Use deliberately different UUID and name orders for UI checks. Do not rely + on the order of the current filenames as a sufficient sorting test. + +This provided the input for representative migration fixtures. The implementation +validation below records the executable before/after checks; private source files +are not committed as test fixtures. + +## Discarding the experiment and Unraid limitation + +An unmerged feature PR can be closed without changing Main. This says nothing +about reverting an already installed test plugin or migrated persistent data. + +Unraid does not provide the plugin downgrade assumed in the earlier planning +text. A migration snapshot protects data; it is not a plugin downgrade. Do not +promise reinstalling the older Main package as the normal return path, restore +old migration-state files as a substitute, or invent a rollback feature in #486. +Corrections to an installed test version use a corrected version and documented +data repair where necessary. Any separate installation recovery would require +its own concrete plan and authorization. Test data copies first. + +## Implementation and test candidate (#486) + +The implementation uses one new registered migration, `job_ids_v1`, after the +existing canonical inventory migrations. It assigns each job a UUID, stores +`job_id` and the existing API field `job_key` with that UUID, and renames job +metadata to `.json`. Main's public API structures and page layouts are +retained. The full editable current prefix is `archive_prefix`; existing +`archive_prefixes` remains the prefix history used by Browse & Restore. +The initial #486 candidate retained descriptive `backup_type` values. The +maintainer subsequently approved implementing #495 in this same branch/PR: +`job_settings_v1` now saves effective settings and appearance directly, then removes +obsolete type fields. Existing installations pass through both registered migrations; +already migrated #486 installations only need the explicit-settings migration. + +The supported configuration boundary is job schema 5, job bundle v3, encrypted job +bundle v3 and profile export v2. Previous job AND profile exports are rejected before +import writes. Create fresh exports after successful migration. Current calculations +exclude historical records without a matching existing UUID; their files are retained. + +The maintainer approved removing the restore-test type rule instead of adding a +per-job chunk option. The existing archive-size threshold selects chunk mode. +Coverage and sample limits now count regular files, excluding directories. Default +5% coverage is capped at 1,000 files; the actual achieved percentage is reported. +See `issue-495-publication-notice.md` for the required bilingual release/forum notice. + +Migration preserves unknown job fields, existing status filenames, status and +check values, restore-test results/report IDs, weekly observations, timestamps, +and repository assignments. Restore-test files become `.test`; +notification/restore stores and schedule keys are converted without discarding +unresolved historical entries. Conflicting historical evidence is retained and +recorded for review rather than claimed by an active job. Invalid active +references block startup before the first data change. + +`cache_subdir` and `check_flag_name` preserve the exact previous cache/check +location for migrated jobs. New jobs receive an ID-specific cache location. +These references stay stable when names and prefixes change. Borg still handles +pruning once for the current prefix, with its existing verbose output; older +prefixes remain available for restore, as on Main. + +### Data preservation and recovery + +Before replacing any input, migration stages complete originals and proposed +files under `/config/migration-backups/job_ids_v1-/` and +persists `config/job-id-migration.json`. The journal records the UUID assignment, +source/target filenames, before/after SHA-256 checksums, timestamps and actions. +The existing migration JSONL log and central failure gate are reused. + +If interrupted, startup resumes the same journal and IDs. A file modified since +the snapshot causes a failure rather than being overwritten. Do not delete or +edit the journal to force another migration. Preserve it, the matching snapshot +and the migration audit log when diagnosing a failure. Restore an individual +original only with the plugin stopped and after checking its journal entry and +checksum; record the exact files and reason. A partial/manual restore is not a +completed migration and must not be used to bypass the startup gate. Use a +corrected plugin version for code fixes. The snapshots do not downgrade Unraid +or the installed plugin. + +### Validation performed during implementation + +- Migration tests cover field preservation, repeat execution, interrupted + replacement/rename, changed input on retry, unavailable storage, active + workers, conflicting active references and ambiguous history. +- Integration tests exercise name/full-prefix editing, unchanged IDs/schedules, + cache/check references, history/report/restore joins, alphabetical job lists, + complete support-bundle job records, old/new exports and import conflicts. +- Prefix tests include historical prefixes and overlapping Borg selections in + one repository, allowed reuse by the same job, and separate repositories. +- Runtime, native prune, notification, repository and UI tests retain their + existing behavioral assertions with canonical UUID fixtures. +- Local browser review uses synthetic data and real API readers; it does not + start backups or install anything on the maintainer's server. +- A private copy of 598 relevant supplied files was migrated: 14 jobs and 594 + converted files. Settings, status/check/restore payloads and weekly values + were compared against the originals. A repeat invocation made no changes. + The 598 input files on the supplied share were verified unchanged afterwards. + Ten unresolved historical references remain: four notification deliveries + and six weekly keys (containing eight measurements), as identified above. + +The remaining maintainer acceptance checks on Unraid are still required. Keep PR #494 +unmerged until that test is accepted. Stable release promotion remains separate. + +### Maintainer findings on test version 2026.09.07.1400 + +On Unraid the job-ID migration completed successfully at 14:14:16, following +startup at 14:10:34. Repository assignments had no errors, schedules were applied, +and the web server started at 14:14:17. This confirms the live migration step; +the remaining acceptance checks below are still pending. The 3m42s startup wait +had no progress messages. The maintainer approved a limited start-log addition: +announce migration before snapshot writes, report phase/file counts approximately +every five seconds during saving, conversion and verification, then log elapsed +time and success or a masked failure. Resume uses the existing journal/IDs and +reports that it is resuming. No additional audit files or browser startup mode +are introduced; the web server starts after the existing migration gate. + +The repository maintenance confirmation still derived its displayed archive +filter from the UUID. The backend already used the stored full archive prefix. +The correction reads `archive_prefix` in the dialog, prefers the selected job's +current name, sorts retention sources by name, and removes the former type/location +fallback for job ownership. Browser-logic tests cover German and English, selected +source changes and UUID submission. A local browser check confirmed the dialog +and filter changes without executing maintenance. + +Focused progress tests verify that the first message precedes snapshot writes, +slow writes produce bounded intermediate counters, success follows verification, +and an interrupted run reports failure before resuming with the original IDs. + +### Maintainer test on Unraid + +For a repeated test from the supplied pre-migration state (`2026.09.07.0935`), +first let backup/restore workers finish and stop the plugin service with +`/etc/rc.d/rc.borg_backup_ui stop`. Preserve the current test data separately, +then restore `/mnt/user/borg_backup_ui` and `/boot/config/borg-backup` together +from the same original backup. Replace the directory contents completely; +merging copies can leave UUID job files from the previous attempt behind. +The migration state, ID assignment journal and audit are under +`/boot/config/borg-backup/config`, so they must belong to the restored state +as well. Install the new test package before starting the plugin again. +A fresh migration can generate different UUIDs than the discarded test; +subsequent starts of that newly migrated state must preserve its UUIDs. + +1. Keep the supplied original data copy; install the verified test-channel + package on the existing Main data with the array/pools available and no + backup/restore worker running. +2. Confirm the existing migration status reports success and all 14 jobs remain + present, with their icons, colors, schedules, histories and restore/check + evidence. Check the complete job JSON in a new support bundle. +3. Restart the plugin once: the same IDs must remain, without a second conversion. +4. Rename a selected test job and change its full prefix. Its ID, old history, + restore evidence, schedule and cache/check reference must remain attached. +5. Run that test job: inspect the new archive prefix and native verbose prune + output. The new status/history entry must use the same job ID. +6. Check prefix conflicts in a shared repository and allowed equal prefixes in + separate repositories. Verify alphabetical names in existing job groups, + and the ID in Edit Job and expanded History details. +7. Check the 100-character job-name limit, including umlauts. Run a test job + with and without the file-list option: saved logs use the same readable + filename format, with the name at run start, location and full job ID. + Status files retain the job ID and point to the correct complete log. +8. Where available, inspect runs skipped by parity or unavailable/read-only + USB storage: additional information logs keep their reason suffix and + the status contains the matching skip reason. Reopen a saved file list + after restarting the plugin. +9. Check the compact retention-source selector and the retention table in + Repository Maintenance. Changing the source updates the current archive + filter and retention values. Browse & Restore lists earlier filters under + one shared heading. diff --git a/docs/maintainer/issue-495-publication-notice.md b/docs/maintainer/issue-495-publication-notice.md new file mode 100644 index 00000000..e385ab76 --- /dev/null +++ b/docs/maintainer/issue-495-publication-notice.md @@ -0,0 +1,35 @@ +# Issue #495: publication notice draft + +Publish these notices with the eventual stable release and in the forum announcement. +This document is a draft; it does not authorize stable promotion or post to the forum. + +## Deutsch + +**Bitte neue Konfigurationssicherungen erstellen:** Mit dieser Version erhalten Jobs +dauerhafte IDs und speichern ihre Einstellungen direkt. Nach dem Update und der +erfolgreichen Migration bitte neue Job- und Profilexporte erstellen. Die bisherigen +Job- und Profilexporte werden beim Import abgelehnt, bevor Daten geschrieben werden. +Vorhandene Borg-Backup-Archive bleiben verwendbar und müssen nicht neu erstellt werden. + +Restore-Tests verwenden für große Archive weiterhin eine Dateistichprobe in Gruppen. +Die Umschaltung richtet sich jetzt ausschließlich nach der eingestellten Archivgröße +(standardmäßig 500 GB). Die bisherige Sonderregel für die Typen `photos` und `vms` +entfällt; bei kleineren Archiven wird dadurch künftig der vollständige Dry-Run verwendet. +Die angestrebte Abdeckung beträgt standardmäßig 5 %, mit höchstens 1.000 regulären +Dateien. Verzeichnisse zählen nicht mit. Bei 100.000 Dateien entsprechen 1.000 +geprüfte Dateien einer Abdeckung von 1 %. + +## English + +**Create fresh configuration backups:** This version gives jobs permanent IDs and +stores their effective settings directly. After updating and successfully completing +the migration, create new job and profile exports. Previous job and profile exports +are rejected before any import data is written. Existing Borg backup archives remain +usable and do not need to be recreated. + +Restore tests continue to use file samples in groups for large archives. Switching +now depends only on the configured archive size (500 GB by default). The former +`photos` and `vms` type rule is removed, so smaller archives now use the full dry-run. +The default target coverage is 5%, capped at 1,000 regular files. Directories do not +count towards the sample or coverage. For 100,000 files, testing 1,000 files gives +1% coverage. diff --git a/docs/maintainer/issue-497-io-verification.md b/docs/maintainer/issue-497-io-verification.md new file mode 100644 index 00000000..b6f95244 --- /dev/null +++ b/docs/maintainer/issue-497-io-verification.md @@ -0,0 +1,259 @@ +# Issue #497: idle and navigation write verification + +This fix is tested on top of the permanent-job-ID branch at the maintainer's +request. An already migrated test-channel installation can update directly; +restoring pre-migration data is not required. This test does not cover #496 or +#498, and does not change authentication or move configuration stores. + +## Automated checks + +Focused tests cover unchanged/missing queues, delayed and exhausted retries, +persisted claims and enqueueing during delivery, unchanged lock metadata, +permission correction, thread/process serialization, and read-only setup +status with unavailable, missing or unwritable data storage. Explicit setup +still creates directories and performs an actual write test. + +The follow-up covers the actual restore GET handlers as well as setup status: +archive/file lists, repository statistics, target-directory browsing and restore +state. Missing/unmounted/unwritable storage still blocks these requests; they +do not repair missing directories. Actual backup, restore, check and download +actions retain their write probe. A read-only access check does not guarantee +that later writes succeed (for example when storage fills up). + +Service-launcher tests use an isolated payload and cache path to verify real +Python imports in the service and a child process, cache reuse on the next +start, and successful execution without bytecode writes when the optional +cache directory cannot be prepared. + +## First maintainer measurements on 2026-09-09 + +The first #497 candidate was `2026.09.08.2345`. Both new captures completed +without reported gaps and monitored 180 directories: + +- `bbui-io-idle-497.jsonl`: 00:00:25 to 00:05:25, 300 seconds. No content writes, + replacements, create/delete events or metadata changes; 20 writable-handle + closes on `notification-delivery.lock` without content WRITE events. +- `bbui-io-ui-497.jsonl`: 00:06:12 to 00:16:13, 600 seconds. No queue saves and no + inventory-lock permission updates. Ten remaining data-directory write probes + and ten individual Python bytecode cache writes on the boot USB prompted the + follow-up. Nine of the probes occurred during the final minute. +- `users.json` was saved once at 00:06:30; `sessions.json` twice at 00:06:24 and + 00:06:30, with no further session saves during navigation. The paired saves + fit sign-in; the capture alone does not identify the earlier session action. + +The maintainer accepted the queue, lock and setup-status changes. The restore +GET correction and RAM bytecode cache were subsequently tested in the second +round below. Raw file captures are not published in the repository. + +## Follow-up maintainer measurements on 2026-09-09 + +Candidate: `2026.09.09.0916`, built from +`9be8b322b269cc0f73b95279e9012c04525b0887` on `codex/issue-486-job-ids`. +The maintainer previously reported Unraid `7.4.0 beta 2`; the monitor does not +record the OS or plugin version itself. Configured data root: +`/mnt/user/borg_backup_ui`. All times below are CEST (`UTC+02:00`). + +| Capture | Start | End | Duration | Watched directories | Reported gaps | +| --- | --- | --- | ---: | ---: | --- | +| `bbui-io-idle-497.jsonl` | 09:23:04.770 | 09:28:04.776 | 300 s | 180 | None | +| `bbui-io-ui-497.jsonl` | 09:30:37.854 | 09:40:37.864 | 600 s | 180 | None | + +The individual event records agree with the per-path summaries and final event +totals in both files. Both report `incomplete: false`, with no warning/error +records. The filenames were reused for this round; use timestamps and hashes +to distinguish them from the earlier captures. + +| Observed operation | Idle, 5 min | UI, 10 min | +| --- | ---: | ---: | +| Replace `notification-queue.json` | 0 | 0 | +| Change `.inventory.lock` permissions/metadata | 0 | 0 | +| Create/write/delete `.borg-ui-write-test` | 0 | 0 | +| Write Python `.pyc` files on the boot USB | 0 | 0 | +| Save `users.json` or `sessions.json` | 0 | 0 | +| Save `repository-info-refresh-state.json` | 1 | 0 | + +During UI navigation, no content writes, replacements, creations/deletions or +metadata changes were recorded anywhere in the watched trees. The 67,760 total +events consist of 67,684 READ events and 76 WRITE_CLOSE events on existing lock +files: 7 on `.inventory.lock` and 69 on `notification-delivery.lock`. No content +WRITE events accompanied those lock closes. The zero session saves do not +prove that sign-in persistence changed; authentication was not modified. + +The idle capture contains 758 events: 730 READ, 22 WRITE_CLOSE, one CREATE, +one WRITE, one MOVE_FROM, one MOVE_TO and two METADATA events. The only save +occurred at 09:25:23: a temporary file was written and renamed over +`config/repository-info-refresh-state.json`. One of the WRITE_CLOSE events +belongs to that save; the other 21 belong to the two lock files. + +The repository-info scheduler waits 300 seconds after service startup and +persists its planning state. This single save is consistent with that behavior, +but neither exact startup timing nor changes beyond timestamps can be proven +from the file-event capture. It is separate from the removed recurring queue +saves, restore probes and Python USB cache writes. + +The maintainer additionally reported: + +```text +root@TheTwist:~# du -sh /run/borg-backup-ui/pycache +13M /run/borg-backup-ui/pycache +``` + +The measured cache allocation is approximately 13 MiB, within the initial +10-20 MiB estimate. The RAM directory is outside the original monitor roots. +The cache-size measurement and absence of USB `.pyc` writes support the +intended placement; these captures do not measure total process RAM or device +write commands. + +Result: the targeted #497 write sources did not recur in the observed idle/UI +windows. The ten remaining restore probes and ten distinct USB Python-cache +writes from the first candidate were absent. Service-restart reuse, rebuilding +after an Unraid reboot, and real notification delivery are not demonstrated by +these captures and remain separate manual checks. No stable release approval +or permission to merge is implied by this record. + +SHA-256 of the supplied follow-up evidence files: + +```text +0c9a11a28549b3c91f4abb88ea81c29f1fa81e5ed182292a340b991341dc1d0b bbui-io-idle-497.jsonl +e19aa35fb666bbdb7a391f6b01c30a670927af04cbbfc4e6b276ad6bd744bc26 bbui-io-ui-497.jsonl +``` + +## Attached diagnostic script and invocation + +Attachment: [bbui-io-watch.py](attachments/bbui-io-watch.py). +This is the unchanged script used for the captures, retained as a versioned +attachment. Its SHA-256 is +`a01135a0d51873770f3046854f46d073ba8dd4e28b5c01d7ec0a48ac986ee34c`. +It is a standalone Linux/Python 3 diagnostic using only the standard library; +it is not installed or started by the plugin. + +Save the attached file as `/tmp/bbui-io-watch.py` on Unraid. Record the installed +plugin version and `cat /etc/unraid-version` separately. Wait until startup and +migration have finished. Keep outputs in RAM, outside the watched paths. +`/tmp` contents disappear at reboot, so copy the results out before rebooting. + +Idle capture: close the plugin UI and observe five minutes. + +```bash +python3 /tmp/bbui-io-watch.py \ + --phase idle --seconds 300 \ + --data-root /mnt/user/borg_backup_ui \ + | tee /tmp/bbui-io-idle-497.jsonl +``` + +UI capture: observe ten minutes, visit each page without changing settings, +include Browse & Restore, and use repeated rapid navigation in the last minute. +Record whether sign-in/out or another deliberate action occurred. + +```bash +python3 /tmp/bbui-io-watch.py \ + --phase ui --seconds 600 \ + --data-root /mnt/user/borg_backup_ui \ + | tee /tmp/bbui-io-ui-497.jsonl +``` + +Use the installation's actual data root. The monitor automatically watches +`/boot/config/borg-backup` and `/boot/config/plugins/borg-backup-ui` recursively. +`--data-root` adds the root itself and recursively watches its `logs`, `status` +and `restore-status` directories. It deliberately excludes Borg cache and +remote-mount trees. It does not follow directory symlinks or read file contents. +Filenames and paths can nevertheless contain private information. + +Optional capture including the RAM cache: `--root` replaces the default boot +roots, so list them explicitly when adding the RAM path. + +```bash +python3 /tmp/bbui-io-watch.py \ + --phase ui-ram --seconds 600 \ + --root /boot/config/borg-backup \ + --root /boot/config/plugins/borg-backup-ui \ + --root /run/borg-backup-ui/pycache \ + --data-root /mnt/user/borg_backup_ui \ + | tee /tmp/bbui-io-ui-497-ram.jsonl +``` + +Read events are grouped every ten seconds; other events are emitted immediately. +The final per-path summaries repeat the already reported counts: do not add +summaries and individual/batched records together. Check `ready`, any warnings, +and `finished.elapsed_seconds` plus `finished.incomplete`. An early manual stop +can have `incomplete: false` but a shorter duration. `WRITE_CLOSE` alone is not +a content write, and event counts are not byte counts, process attribution, +physical device I/O or a USB-lifespan measurement. The script is an event +recorder; interpretation and comparison are documented above. + +## Python bytecode cache + +The service launcher sets `PYTHONPYCACHEPREFIX=/run/borg-backup-ui/pycache` +before starting Python, with a private `0700` cache directory. Python child +processes inherit it. Current cron entries call the API, so scheduled jobs use +the same cache as manually started jobs. Direct developer invocations outside +the service launcher do not receive this setting automatically. + +On Unraid this path is in RAM and disappears at reboot. It is shared across +processes and retained across plugin-service restarts. It is unrelated to Borg +repository caches, job counts or backup sizes. There is no fixed allocation or +32 MiB cap. The initial 10-20 MiB estimate (32 MiB planning allowance) is not a +measured limit; Python versions and loaded modules affect the size. + +If preparing this optional directory fails, the launcher sets +`PYTHONDONTWRITEBYTECODE=1` and logs the condition. The service continues without +bytecode writes; it does not fall back to writing caches alongside USB source +files. New imports may take longer without a usable bytecode cache. + +## Unraid acceptance + +Record the test-channel version, Unraid version and configured data directory. +Let startup/migration finish before starting the measurements. Use the same +inotify monitor and watched paths as the original #497 captures; keep its +output in RAM, outside the watched directories. + +1. Close Borg Backup UI tabs and capture five minutes of idle activity. +2. Capture ten minutes of UI activity: sign in, visit each page without saving + configuration, then navigate repeatedly during the final minute. Record + the start of rapid navigation so unequal periods can be compared correctly. +3. Compare file-content writes, replacements, create/delete events and metadata + changes. READ events and closing a writable handle alone do not prove a + file-content write. Record any monitor overflows or gaps. + +Expected results: + +- `config/notification-queue.json`: no saves when missing, empty or unchanged, + including retry entries whose next attempt is still in the future. Actual + enqueueing, due claims, retries and delivery-status changes still write. +- `config/.inventory.lock`: no repeated permission updates while its existing + permissions are `0600`. Creation or correction of permissions is expected + when needed; keep locking enabled. +- `status/.borg-ui-write-test`: no create/write/delete events from ordinary + page-status requests. The probe remains expected during explicit data-dir + setup and runtime initialization, so measure after startup has finished. +- Normal sign-in can still save `users.json` and `sessions.json`. The fix makes + no claim about session writes that were not reproduced in the original test. +- Setup readiness still reports missing or unavailable directories. It no + longer silently creates or repairs directories during a GET request. +- Service and child-process `.pyc` files appear under + `/run/borg-backup-ui/pycache`, with none created or replaced alongside the + plugin's Python sources on `/boot`. Visit all pages once to populate lazy + imports, then repeat navigation and compare warm-cache events. + +Measure the actual cache allocation after navigation and a backup/restore-test +run (adjust the command only if the implementation path changes): + +```bash +du -sh /run/borg-backup-ui/pycache +stat -c '%a %U:%G %n' /run/borg-backup-ui/pycache +``` + +After all active jobs have finished, verify that a plugin-service restart +reuses the cache. On a later normal Unraid reboot verify that the cache is +recreated in RAM and the application starts normally. The original inotify +monitor's boot/data roots do not include this RAM directory; use `--root` +explicitly when capturing its events. + +Verify a notification using the maintainer's selected test destination and +confirm delivery status; do not send messages to other recipients as part of +an unattended check. Keep successful delivery and failure/retry behavior in +the acceptance record. + +Record the two capture summaries and any remaining unexpected writes before +stable approval. Local filesystem tests demonstrate application behavior; +they do not measure physical USB I/O or predict USB lifespan. diff --git a/docs/user-manual/de/user-manual.md b/docs/user-manual/de/user-manual.md index 3fd9a721..37a6b8dc 100644 --- a/docs/user-manual/de/user-manual.md +++ b/docs/user-manual/de/user-manual.md @@ -57,7 +57,7 @@ Die Anwendung kennt die Rollen `admin`, `operator` und `viewer`: ### 1.3 Installation und Erstkonfiguration -Borg Backup UI befindet sich in der **Public Beta** und wird über **Unraid Community Apps** installiert. Voraussetzungen sind **Unraid 7.2.0 oder neuer** sowie **Python 3.10 oder neuer**. Installieren Sie das separate Plugin **Python 3 for Unraid** zuerst über Community Apps. BorgBackup selbst ist im Borg-Backup-UI-Paket enthalten; eine zusätzliche Borg- oder pip-Installation ist nicht erforderlich. +Borg Backup UI befindet sich in der **Public Beta** und wird über **Unraid Community Apps** installiert. Voraussetzungen sind **Unraid 6.12.5 oder neuer** sowie **Python 3.10 oder neuer**. Installieren Sie das separate Plugin **Python 3 for Unraid** zuerst über Community Apps. BorgBackup selbst ist im Borg-Backup-UI-Paket enthalten; eine zusätzliche Borg- oder pip-Installation ist nicht erforderlich. 1. Öffnen Sie in Unraid **Apps**. 2. Installieren Sie **Python 3 for Unraid**, falls es noch nicht vorhanden ist. @@ -700,6 +700,10 @@ Typische Statuswerte: - **Fehlgeschlagen** - **Nicht verfügbar** +Bei Restore-Tests ab Level 2 entscheidet die eingestellte Archivgröße über den Dry-Run: Unterhalb des Schwellwerts wird das vollständige Archiv geprüft, ab dem Schwellwert eine Dateistichprobe in Gruppen. Standardmäßig beträgt der Schwellwert 500 GB; 0 deaktiviert die Umschaltung. Die frühere Sonderregel für die Typen `photos` und `vms` entfällt. + +Die Stichprobe zielt standardmäßig auf 5 % der regulären Dateien, begrenzt durch die eingestellten Dateilimits (standardmäßig 1.000). Verzeichnisse zählen weder als Dateien noch zur Abdeckung. Beispiel: Bei 10.000 Dateien werden 500 geprüft; bei 100.000 Dateien greift die Grenze von 1.000, entsprechend 1 % Abdeckung. Der Bericht zeigt die erreichte Datei-Abdeckung. Die zusätzliche Level-3-Prüfung verwendet weiterhin ihre eigene Stichprobengröße. + ### 8.6 Best Practices - Planen Sie Restore Tests für wichtige Jobs regelmäßig. @@ -952,7 +956,9 @@ Importstrategien können je nach Importtyp vorhandene Einträge behalten, ersetz > **Warnung:** Bewahren Sie Export-Passwörter sicher auf. Ohne passendes Passwort können verschlüsselte Exporte nicht wiederhergestellt werden. -Neue verschlüsselte Exporte verwenden eine versionierte, authentifizierte Hülle. Ein falsches Passwort sowie beschädigte, abgeschnittene oder manipulierte Dateien werden geprüft, bevor Importdaten geschrieben werden. Ältere AES-CBC-Exporte bleiben importierbar, erscheinen in der Vorschau jedoch mit einem Legacy-Hinweis. Erstellen Sie nach einem Legacy-Import einen neuen Export im aktuellen Format. +Neue verschlüsselte Exporte verwenden eine versionierte, authentifizierte Hülle. Ein falsches Passwort sowie beschädigte, abgeschnittene oder manipulierte Dateien werden geprüft, bevor Importdaten geschrieben werden. Zusätzlich muss das enthaltene Konfigurationsformat unterstützt werden; Dateiname und Exportdatum sind dafür nicht entscheidend. + +> **Nach dem Update neue Exporte erstellen:** Job- und Profilexporte im bisherigen Format sind nicht mehr importierbar. Erstellen Sie nach erfolgreicher Migration neue Job- und Profilexporte. Alte Formate werden vor Änderungen an Jobs, Einstellungen oder Secrets abgelehnt. Vorhandene Borg-Backup-Archive bleiben für Wiederherstellungen verwendbar und müssen nicht neu erstellt werden. ### 9.11 Erweitert diff --git a/docs/user-manual/en/user-manual.md b/docs/user-manual/en/user-manual.md index 89f20636..431f1fbe 100644 --- a/docs/user-manual/en/user-manual.md +++ b/docs/user-manual/en/user-manual.md @@ -57,7 +57,7 @@ The application supports the `admin`, `operator`, and `viewer` roles: ### 1.3 Installation and Initial Setup -Borg Backup UI is in **public beta** and is installed through **Unraid Community Apps**. It requires **Unraid 7.2.0 or newer** and **Python 3.10 or newer**. Install the separate **Python 3 for Unraid** plugin from Community Apps first. BorgBackup itself is bundled with Borg Backup UI; no separate Borg or pip installation is required. +Borg Backup UI is in **public beta** and is installed through **Unraid Community Apps**. It requires **Unraid 6.12.5 or newer** and **Python 3.10 or newer**. Install the separate **Python 3 for Unraid** plugin from Community Apps first. BorgBackup itself is bundled with Borg Backup UI; no separate Borg or pip installation is required. 1. Open **Apps** in Unraid. 2. Install **Python 3 for Unraid** if it is not already present. @@ -700,6 +700,10 @@ Typical status values: - **Failed** - **Not available** +For restore tests at level 2 or above, the configured archive size selects the dry-run mode: below the threshold, the full archive is checked; at or above it, a file sample is checked in groups. The default threshold is 500 GB; 0 disables switching. The former special rule for the `photos` and `vms` types is removed. + +The sample targets 5% of regular files by default, capped by the configured file limits (1,000 by default). Directories do not count as files or towards coverage. For example, 10,000 files give a 500-file sample; with 100,000 files, the 1,000-file cap gives 1% coverage. The report shows achieved file coverage. The additional level 3 check continues to use its separate sample size. + ### 8.6 Best Practices - Schedule restore tests regularly for important jobs. @@ -951,7 +955,9 @@ Import strategies can keep, replace, or rename existing entries depending on the > **Warning:** Store export passwords securely. Encrypted exports cannot be restored without the matching password. -New encrypted exports use a versioned, authenticated envelope. Wrong passwords and damaged, truncated, or manipulated files are checked before import data is written. Older AES-CBC exports remain importable but show a legacy warning in the preview. Create a new export in the current format after a legacy import. +New encrypted exports use a versioned, authenticated envelope. Wrong passwords and damaged, truncated, or manipulated files are checked before import data is written. The enclosed configuration format must also be supported; the filename and export date do not determine compatibility. + +> **Create fresh exports after updating:** Previous-format job and profile exports can no longer be imported. After successful migration, create new job and profile exports. Old formats are rejected before changing jobs, settings, or secrets. Existing Borg backup archives remain usable for restoring data and do not need to be recreated. ### 9.11 Advanced diff --git a/plugin/rc.borg_backup_ui b/plugin/rc.borg_backup_ui index 9325443c..ab88743f 100644 --- a/plugin/rc.borg_backup_ui +++ b/plugin/rc.borg_backup_ui @@ -6,6 +6,7 @@ PLUGIN_DIR="/boot/config/plugins/borg-backup-ui" PIDFILE="/var/run/borg_backup_ui.pid" WAIT_PIDFILE="/var/run/borg_backup_ui_start_wait.pid" LOGFILE="/var/log/borg_backup_ui.log" +PYTHON_CACHE_DIR="/run/borg-backup-ui/pycache" BORG_BUNDLE_DIR="${PLUGIN_DIR}/runtime/bin/borg" BORG_BUNDLE_VERSIONED="${BORG_BUNDLE_DIR}/borg-linux-glibc231-x86_64-1.4.5" BORG_BUNDLE_PLAIN="${BORG_BUNDLE_DIR}/borg" @@ -131,6 +132,14 @@ start() { echo "WARNING: bundled Borg binary not found, using system PATH." fi echo "Starting Borg Backup UI..." + # Keep bytecode off the boot USB stick. API-started jobs (including cron + # requests) inherit this cache, which survives service restarts until reboot. + export PYTHONPYCACHEPREFIX="${PYTHON_CACHE_DIR}" + if ! install -d -m 0700 "${PYTHON_CACHE_DIR}"; then + # An optional cache must not prevent startup or fall back to USB writes. + export PYTHONDONTWRITEBYTECODE=1 + echo "WARNING: Python RAM cache is unavailable; bytecode writes are disabled for this run." >> "$LOGFILE" + fi PY_BIN="$(python_runtime_status)" if [ -z "$PY_BIN" ]; then [ "${BBUI_DEFERRED_START:-0}" = "1" ] && rm -f "$WAIT_PIDFILE" diff --git a/plugins/borg-backup-ui.xml b/plugins/borg-backup-ui.xml index 28010c46..28f97257 100644 --- a/plugins/borg-backup-ui.xml +++ b/plugins/borg-backup-ui.xml @@ -7,7 +7,7 @@ https://github.com/borgforge/borg-backup-ui https://raw.githubusercontent.com/borgforge/borg-backup-ui/main/README.md Backup: Tools:System Plugins: - 7.2.0 + 6.12.5 true https://raw.githubusercontent.com/borgforge/borg-backup-ui/main/ui/assets/app-icon.png MIT diff --git a/release-notes/pending/486.md b/release-notes/pending/486.md new file mode 100644 index 00000000..23b1fc53 --- /dev/null +++ b/release-notes/pending/486.md @@ -0,0 +1,9 @@ +- #486: Jobs receive permanent IDs. Names and full archive prefixes can be edited while schedules, history, restore results and check/cache references stay attached to the same job. +- #486: Existing Main data is migrated once with complete affected-file snapshots and resumable ID assignments. Jobs sort by name; their ID is visible in Edit Job and expanded History details. +- #486: Reject overlapping current or previous archive prefixes of different jobs in the same repository. Existing Borg prune behavior and verbose output are preserved. +- #486: Repository maintenance confirmation shows the selected job's current name and full archive filter, with retention sources sorted by job name. The compact selector shows only the job name and archive filter; a table lists the retention values and per-period limits and updates with the selection. The dialog clarifies that recorded previous prefixes are not additionally included in prune. +- #486: Startup logs show job-ID migration phases, file progress and elapsed time while the web server waits for migration to finish. +- #486: Browse & Restore groups previous archive filters under one heading in the filter history popover. +- #486: New backup logs and status files include the job name, location and full job ID. Saved logs use the same date format with or without the file-list option; additional parity/USB skip logs retain their reason suffix. Existing files remain readable after renaming a job; the Wizard limits job names to 100 characters, while filename labels are safely shortened for UTF-8 byte limits. +- #486: Compact the Wizard's Basics step by removing the prefix explanation, reducing inner spacing and placing the name limit beside its label. All steps use the same larger dialog height, and the archive-pattern popover groups previous patterns under one heading. +- #486: New Job displays its permanent job ID before saving, after checking that it is unused. Collisions automatically generate another ID; a collision during saving also preserves all entries and retries with an unused ID. Canceling creates no job. Saved jobs keep their IDs. diff --git a/release-notes/pending/495.md b/release-notes/pending/495.md new file mode 100644 index 00000000..38499bf2 --- /dev/null +++ b/release-notes/pending/495.md @@ -0,0 +1,5 @@ +- #495: Jobs store their effective compression and retention directly. A one-time audited migration preserves existing settings, icons, colors and cache/check references; new jobs use the archive icon with a neutral default color. +- #495: Old job and profile configuration exports are rejected before import writes. After updating and completing migration, create fresh job and profile exports. Existing Borg backup archives remain usable and do not need to be recreated. +- #495 (DE): Alte Job- und Profil-Konfigurationsexporte sind nicht mehr importierbar. Nach Update und erfolgreicher Migration neue Job- und Profilexporte erstellen. Vorhandene Borg-Backup-Archive bleiben nutzbar und muessen nicht neu erstellt werden. +- #495: Historical files without a matching job UUID remain on disk but are excluded from current job associations and calculations. They are not reassigned or deleted. +- #495: Restore-test chunk mode is selected by archive size, without the former photos/vms type rule. Coverage and sample limits count regular files, not directories. The default target is 5 percent, capped at 1,000 files; reports show the achieved file coverage. diff --git a/release-notes/pending/496.md b/release-notes/pending/496.md new file mode 100644 index 00000000..36436f83 --- /dev/null +++ b/release-notes/pending/496.md @@ -0,0 +1,2 @@ +- #496: Fix SSH backup failures caused by the unsupported WarnWeakCrypto option on older OpenSSH clients while retaining warning suppression on newer clients. The minimum supported Unraid version is 6.12.5; the separate Python 3 for Unraid plugin with Python 3.10 or newer is still required. +- #496 (DE): SSH-Backups brechen mit aelteren OpenSSH-Clients nicht mehr wegen der unbekannten Option WarnWeakCrypto ab. Neuere Clients unterdruecken die Warnung weiterhin. Die unterstuetzte Mindestversion ist Unraid 6.12.5; das separate Plugin Python 3 for Unraid mit Python 3.10 oder neuer bleibt erforderlich. diff --git a/release-notes/pending/497.md b/release-notes/pending/497.md new file mode 100644 index 00000000..244b84e1 --- /dev/null +++ b/release-notes/pending/497.md @@ -0,0 +1,2 @@ +- #497: Reduce unnecessary boot USB writes by leaving unchanged notification queues untouched, avoiding redundant permission updates on inventory locks, and keeping the service's Python bytecode cache in RAM. Routine page-status and restore-browsing requests no longer create write-test files in the data directory; setup and actual write operations retain their storage checks. +- #497 (DE): Weniger unnoetige Schreibzugriffe auf den Unraid-Bootstick: Unveraenderte Benachrichtigungswarteschlangen werden nicht erneut gespeichert, korrekte Rechte von Inventarsperren nicht erneut gesetzt und der Python-Bytecode-Cache des Dienstes liegt im RAM. Normale Statusabfragen und das Durchsuchen im Restore-Bereich erzeugen keine Schreibtest-Dateien mehr im Datenverzeichnis; Einrichtung und Schreiboperationen behalten ihre Speicherpruefungen. diff --git a/release-notes/pending/499.md b/release-notes/pending/499.md new file mode 100644 index 00000000..957e6b6c --- /dev/null +++ b/release-notes/pending/499.md @@ -0,0 +1,2 @@ +- #499: Browse & Restore refreshes archives when reopening or reselecting a job and clears stale selections after repository changes. New backups become visible without restarting the plugin. +- #499 (DE): Browse & Restore aktualisiert die Archive beim erneuten Oeffnen oder Auswaehlen eines Jobs und verwirft veraltete Auswahlen nach Repository-Wechseln. Neue Backups werden ohne Plugin-Neustart sichtbar. diff --git a/release-notes/pending/500.md b/release-notes/pending/500.md new file mode 100644 index 00000000..9bc3a688 --- /dev/null +++ b/release-notes/pending/500.md @@ -0,0 +1,2 @@ +- #500: Missing restore archives produce an actionable message instead of a generic internal error. Failed file-list requests clear the loading indicator and stale restore selections. +- #500 (DE): Fehlende Restore-Archive werden mit einem verstaendlichen Hinweis statt eines allgemeinen internen Fehlers gemeldet. Fehlgeschlagene Dateilisten-Abfragen beenden die Ladeanzeige und verwerfen ungueltige Restore-Auswahlen. diff --git a/release-notes/pending/501.md b/release-notes/pending/501.md new file mode 100644 index 00000000..d3509d9f --- /dev/null +++ b/release-notes/pending/501.md @@ -0,0 +1,2 @@ +- #501: Changing a job's repository no longer reuses the previous repository's successful integrity check. Existing jobs keep their check schedule until their repository changes. +- #501 (DE): Beim Repository-Wechsel eines Jobs wird die erfolgreiche Integritaetspruefung des vorherigen Repositorys nicht mehr uebernommen. Unveraenderte Jobs behalten ihren bisherigen Pruefplan. diff --git a/release-notes/pending/502.md b/release-notes/pending/502.md new file mode 100644 index 00000000..a4d7af68 --- /dev/null +++ b/release-notes/pending/502.md @@ -0,0 +1,2 @@ +- #502: USB backups now verify the mount before starting. Unmounted targets are skipped; USB access errors stop the job with a clear reason in the log, status and notification, without starting Borg. +- #502 (DE): USB-Backups pruefen jetzt vor dem Start den Mount. Nicht eingehangene Ziele werden uebersprungen; USB-Zugriffsfehler beenden den Job mit einem klaren Grund in Log, Status und Benachrichtigung, ohne Borg zu starten. diff --git a/runtime/config/backup.conf.example b/runtime/config/backup.conf.example index 9956a6fc..5cb70d12 100644 --- a/runtime/config/backup.conf.example +++ b/runtime/config/backup.conf.example @@ -80,7 +80,6 @@ UI_SESSION_TIMEOUT_MINUTES="30" RESTORE_TEST_LEVEL="2" RESTORE_TEST_INTERVAL_DAYS="30" RESTORE_TEST_LOCATION="local" -RESTORE_TEST_FORCE_CHUNK_TYPES="vms,photos" RESTORE_TEST_FULL_DRYRUN_MAX_ARCHIVE_GB="500" RESTORE_TEST_MIN_COVERAGE="5" RESTORE_TEST_MAX_ENTRIES="1000" diff --git a/runtime/lib/backup_job.py b/runtime/lib/backup_job.py index 6a3dd780..7bbbdae2 100644 --- a/runtime/lib/backup_job.py +++ b/runtime/lib/backup_job.py @@ -29,6 +29,7 @@ import os import re import shutil +import stat import subprocess import sys import time @@ -49,6 +50,7 @@ REQUIRED_SOURCE_PATHS_MISSING = "required_source_paths_missing" RUNTIME_RECOVERY_FAILED = "runtime_recovery_failed" USER_CANCELLED = "user_cancelled" +USB_MOUNT_ACCESS_FAILED = "usb_mount_access_failed" def _path_uses_symlink(path: Path) -> bool: @@ -102,6 +104,18 @@ def __init__(self, missing_paths: List[Path]) -> None: ) +class UsbMountAccessError(RuntimeError): + """USB preflight could not inspect the target; Borg must not be started.""" + + failure_code = USB_MOUNT_ACCESS_FAILED + + def __init__(self, mount_path: Path, cause: OSError) -> None: + super().__init__( + f"USB drive is not accessible at {mount_path}: {cause}. " + "Backup was not started. Check the USB connection and mount state." + ) + + def _log_section(title: str) -> None: logger.info(_HR) logger.info(" %s", title) @@ -149,10 +163,13 @@ class BackupJobConfig: borg_keep_yearly: int = 3 retained_log_file: Optional[Path] = None + job_id: str = "" @classmethod def from_config(cls, env: dict) -> "BackupJobConfig": """Liest Konfiguration aus Umgebungsvariablen.""" + from job_identity import validate_job_id + job_id = validate_job_id(env.get("BORG_UI_JOB_KEY")) try: raw_paths = json.loads(env.get("BACKUP_PATHS_JSON", "") or "") except (json.JSONDecodeError, TypeError, ValueError) as exc: @@ -208,8 +225,9 @@ def from_config(cls, env: dict) -> "BackupJobConfig": ) return cls( + job_id=job_id, job_name=env.get("JOB_NAME", "Borg Backup"), - backup_type=env.get("BACKUP_TYPE", "unknown"), + backup_type="", backup_location=env.get("BACKUP_LOCATION") or env.get("LOCATION", "unknown"), lock_file=Path(env.get("LOCK_FILE", "/tmp/borg-backup.lock")), log_dir=Path(env.get("LOG_DIR", "/tmp")), @@ -301,6 +319,8 @@ def _log_startup_banner(self) -> None: cfg = self.config _log_section("BACKUP START") logger.info("Job: %s", cfg.job_name) + if cfg.job_id: + logger.info("Job ID: %s", cfg.job_id) logger.info("Date: %s", cfg.date_tag) logger.info("Log: %s", cfg.log_file) logger.info("") @@ -346,7 +366,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): str(path) for path in exc_val.missing_paths ] self._final_msg = str(exc_val) - logger.error("Job aborted by exception: %s", exc_val) + if isinstance(exc_val, UsbMountAccessError): + self._failure_code = exc_val.failure_code + self._final_msg = str(exc_val) + logger.error("%s", exc_val) + else: + logger.error("Job aborted by exception: %s", exc_val) if not self._skip_finish: _log_section("PHASE 5: CLEANUP & COMPLETION") @@ -545,18 +570,30 @@ def start_vms(self) -> None: def check_usb_mount(self, mount_path: Path) -> None: """ - Prüft ob USB-Laufwerk verfügbar und beschreibbar ist. + Prüft Mount, Verzeichnis und Schreibrechte vor dem Backup. - Sendet Notification und löst SystemExit(0) aus wenn nicht verfügbar. - Wird von USB-Backup-Skripten explizit aufgerufen. + Fehlende/unbeschreibbare Mounts werden übersprungen; Zugriffsfehler + brechen den Lauf mit UsbMountAccessError ab. """ - if not mount_path.is_dir(): + try: + # stat() preserves I/O errors even on Python versions whose is_dir() + # and is_mount() turn some filesystem errors into False. + try: + is_directory = stat.S_ISDIR(mount_path.stat().st_mode) + except (FileNotFoundError, NotADirectoryError): + is_directory = False + mounted = is_directory and mount_path.is_mount() + writable = mounted and os.access(mount_path, os.W_OK) + except OSError as exc: + raise UsbMountAccessError(mount_path, exc) from exc + + if not mounted: self._write_mini_log( "USB_NOT_MOUNTED", [ - f"Borg Backup ({self.config.backup_type}) - Skipped because the USB drive is missing", + f"Borg Backup ({self.config.job_name}) - Skipped because the USB drive is missing", f"Mount path: {mount_path}", - "Status: directory does not exist", + "Status: path is not a mounted directory", "Reason: USB drive is not connected or mounted", ], ) @@ -564,11 +601,11 @@ def check_usb_mount(self, mount_path: Path) -> None: self._persist_skip_status_once() raise SystemExit(0) - if not os.access(mount_path, os.W_OK): + if not writable: self._write_mini_log( "USB_NOT_WRITABLE", [ - f"Borg Backup ({self.config.backup_type}) - Skipped because the USB drive is read-only", + f"Borg Backup ({self.config.job_name}) - Skipped because the USB drive is read-only", f"Mount path: {mount_path}", "Status: not writable", "Reason: USB drive is read-only or lacks write permissions", @@ -622,7 +659,7 @@ def _get_field(key: str) -> str: self._write_mini_log( "SKIPPED_PARITY", [ - f"Borg Backup ({self.config.backup_type}) - Skipped because a parity operation is running", + f"Borg Backup ({self.config.job_name}) - Skipped because a parity operation is running", f"Operation: {resync_action}", f"Progress: {progress}% ({resync_pos}/{resync_size})", "Reason: Preserve system performance during the parity operation", @@ -685,8 +722,12 @@ def cleanup_old_logs(self) -> None: "Removing logs older than %d days...", self.config.log_retention_days ) cutoff = time.time() - (self.config.log_retention_days * 86400) - pattern = f"Borg-Backup_{self.config.backup_type}--*.log" - for log_path in self.config.log_dir.glob(pattern): + if self.config.job_id: + from job_identity import job_log_paths + paths = job_log_paths(self.config.log_dir, self.config.job_id) + else: + paths = self.config.log_dir.glob(f"Borg-Backup_{self.config.backup_type}--*.log") + for log_path in paths: try: if log_path.stat().st_mtime < cutoff: log_path.unlink() @@ -786,6 +827,7 @@ def _record_docker_recovery_state(self) -> None: kind="docker", targets=targets, job_name=self.config.job_name, + job_id=self.config.job_id, backup_type=self.config.backup_type, backup_location=self.config.backup_location, log_file=str(self.config.log_file), @@ -807,6 +849,7 @@ def _record_vm_recovery_state(self) -> None: kind="vm", targets=[{"id": name, "name": name} for name in result.stopped_vms], job_name=self.config.job_name, + job_id=self.config.job_id, backup_type=self.config.backup_type, backup_location=self.config.backup_location, log_file=str(self.config.log_file), @@ -882,7 +925,10 @@ def _do_finish(self) -> None: exit_code, ) else: - logger.info("Borg backup failed (exit %d)", exit_code) + if self._failure_code == USB_MOUNT_ACCESS_FAILED: + logger.info("Backup aborted during USB preflight (exit %d); Borg backup was not started", exit_code) + else: + logger.info("Borg backup failed (exit %d)", exit_code) self._send_notification_event( "backup_failed", "Borg Backup UI: Backup failed", @@ -989,6 +1035,10 @@ def _save_skip_status(self) -> None: reason_code = "skipped" logger.info("Saving skipped status: %s", reason) bs = BackupStatus( + job_id=self.config.job_id, + job_name=self.config.job_name, + run_id=os.environ.get("BORG_UI_RUN_ID", ""), + file_activity=bool(self.config.retained_log_file), backup_type=self.config.backup_type, location=self.config.backup_location, timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), @@ -1070,8 +1120,8 @@ def _send_notification_event( title=title, message=message, severity=severity, - job_name=f"Borg Backup ({self.config.backup_type})", - job_key=f"{self.config.backup_type}_{self.config.backup_location}", + job_name=self.config.job_name, + job_key=self.config.job_id, status=event_type, duration_seconds=duration, repository=self.config.borg_repo or os.environ.get("BORG_REPO", ""), @@ -1101,7 +1151,7 @@ def _save_status(self, duration: int) -> Path | None: status_str = "error" stats = self._borg_stats - if self._failure_code == REQUIRED_SOURCE_PATHS_MISSING: + if self._failure_code in {REQUIRED_SOURCE_PATHS_MISSING, USB_MOUNT_ACCESS_FAILED}: repo_size = 0 repo_check_date, repo_check_status, repo_next_check = ( "unknown", @@ -1118,6 +1168,10 @@ def _save_status(self, duration: int) -> Path | None: transfer_speed = stats.deduplicated_size // duration bs = BackupStatus( + job_id=self.config.job_id, + job_name=self.config.job_name, + run_id=os.environ.get("BORG_UI_RUN_ID", ""), + file_activity=bool(self.config.retained_log_file), backup_type=self.config.backup_type, location=self.config.backup_location, timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), @@ -1165,7 +1219,7 @@ def _emit_lifecycle_finished( request_id=os.environ.get("BORG_UI_REQUEST_ID", ""), source=os.environ.get("BORG_UI_REQUEST_SOURCE", "backup_job"), actor=os.environ.get("BORG_UI_REQUEST_ACTOR", ""), - job_key=os.environ.get("BORG_UI_JOB_KEY", f"{self.config.backup_type}_{self.config.backup_location}"), + job_key=self.config.job_id, run_id=os.environ.get("BORG_UI_RUN_ID", ""), status=status, exit_code=exit_code, @@ -1255,11 +1309,16 @@ def _write_mini_log(self, suffix: str, lines: List[str]) -> None: """Schreibt einen kleinen Informations-Log für Skip-Szenarien.""" try: self.config.log_dir.mkdir(parents=True, exist_ok=True) - mini_log = ( - self.config.log_dir - / f"Borg-Backup_{self.config.backup_type}--{self.config.date_tag}_{suffix}.log" - ) + if self.config.job_id: + from job_identity import job_log_filename + filename = job_log_filename(self.config.job_name, self.config.backup_location, + self.config.job_id, f"{self.config.date_tag}_{suffix}") + else: + filename = f"Borg-Backup_{self.config.backup_type}--{self.config.date_tag}_{suffix}.log" + mini_log = self.config.log_dir / filename ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + if self.config.job_id: + lines = [f"Job: {self.config.job_name}", f"Job ID: {self.config.job_id}", *lines] content = "\n".join(f"[{ts}] {line}" for line in lines) + "\n" mini_log.write_text(content, encoding="utf-8") except OSError as exc: @@ -1299,7 +1358,7 @@ def _write_mini_log(self, suffix: str, lines: List[str]) -> None: if args.command == "info": print(f"job_name: {cfg.job_name}") - print(f"backup_type: {cfg.backup_type}") + print(f"job_id: {cfg.job_id}") print(f"backup_location: {cfg.backup_location}") print(f"lock_file: {cfg.lock_file}") print(f"log_dir: {cfg.log_dir}") diff --git a/runtime/lib/notification_events.py b/runtime/lib/notification_events.py index 81bc6f23..85c23706 100644 --- a/runtime/lib/notification_events.py +++ b/runtime/lib/notification_events.py @@ -254,8 +254,11 @@ def drain_notification_queue(config: dict, *, max_items: int = 20) -> dict[str, due.append(row) else: pending.append(row) - store["queue"] = pending - _write_json(_queue_path(config), store) + # Idle checks and retries that are not due must not rewrite the queue + # on the boot device. Keep selection and persistence under the lock. + if pending != rows: + store["queue"] = pending + _write_json(_queue_path(config), store) delivered = 0 failed = 0 diff --git a/runtime/lib/runtime_recovery.py b/runtime/lib/runtime_recovery.py index 01539ee2..7311eb1a 100644 --- a/runtime/lib/runtime_recovery.py +++ b/runtime/lib/runtime_recovery.py @@ -62,6 +62,7 @@ def record_runtime_stopped( backup_type: str, backup_location: str, log_file: str, + job_id: str = "", ) -> str: normalized_targets = _normalize_targets(targets) if not normalized_targets: @@ -72,6 +73,7 @@ def record_runtime_stopped( "state": "pending_restart", "kind": str(kind or "").strip(), "job_name": str(job_name or "").strip(), + "job_id": str(job_id or ""), "backup_type": str(backup_type or "").strip(), "backup_location": str(backup_location or "").strip(), "log_file": str(log_file or "").strip(), diff --git a/runtime/lib/status.py b/runtime/lib/status.py index 32288dfc..c909596f 100644 --- a/runtime/lib/status.py +++ b/runtime/lib/status.py @@ -118,6 +118,10 @@ class BackupStatus: repository_check_date: str = "" repository_check_status: str = "unknown" # ok | overdue | unknown repository_next_check: str = "" + job_id: str = "" + job_name: str = "" + run_id: str = "" + file_activity: bool = False # Pfad der Quelldatei (nicht serialisiert) source_path: Optional[Path] = field(default=None, repr=False, compare=False) @@ -132,6 +136,10 @@ def from_file(cls, path: Path) -> "BackupStatus": return cls(source_path=path) obj = cls(source_path=path) + obj.job_id = str(data.get("job_id") or "") + obj.job_name = str(data.get("job_name") or "") + obj.run_id = str(data.get("run_id") or "") + obj.file_activity = data.get("file_activity") is True obj.backup_type = str(data.get("backup_type", "unknown")) obj.location = str(data.get("location", "unknown")) obj.timestamp = str(data.get("timestamp", "")) @@ -177,8 +185,9 @@ def from_file(cls, path: Path) -> "BackupStatus": @property def key(self) -> str: - """Eindeutiger Schlüssel: backup_type_location.""" - return f"{self.backup_type}_{self.location}" + """Use the permanent ID; unresolved history cannot own an active job.""" + from job_identity import historical_job_id + return historical_job_id(self.job_id) @property def timestamp_dt(self) -> Optional[datetime]: @@ -202,8 +211,10 @@ def save(self, status_dir: Path) -> Path: """Schreibt Status als JSON-Datei in status_dir. Gibt den Dateipfad zurück.""" ensure_status_storage_directory(status_dir) timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - path = status_dir / f"{timestamp}_{self.backup_type}_{self.location}.status" - data = {k: v for k, v in asdict(self).items() if k != "source_path"} + from job_identity import job_file_component + identity = job_file_component(self.job_name, self.location, self.job_id) + path = status_dir / f"{timestamp}_{identity}.status" + data = {k: v for k, v in asdict(self).items() if k not in {"source_path", "backup_type"}} path.write_text(json.dumps(data, indent=2), encoding="utf-8") logger.info("Saved backup status: %s", path) return path @@ -213,6 +224,7 @@ def save(self, status_dir: Path) -> Path: class RestoreTest: """Repräsentiert den Inhalt einer .test Datei.""" + job_id: str = "" test_date: str = "" test_result: str = "unknown" # success | failed | unavailable test_level: int = 0 @@ -273,7 +285,9 @@ def from_file(cls, path: Path) -> "RestoreTest": backup_type = parts[0] if len(parts) > 0 else "unknown" location = parts[1] if len(parts) > 1 else "unknown" - obj = cls(source_path=path, backup_type=backup_type, location=location) + obj = cls(source_path=path, job_id=str(data.get("job_id") or ""), + backup_type=str(data.get("backup_type") or data.get("type") or backup_type), + location=str(data.get("location") or location)) obj.test_date = str(data.get("test_date", "") or "") obj.test_result = str(data.get("test_result", "unknown") or "unknown") obj.test_level = int(data.get("test_level", 0) or 0) @@ -328,7 +342,8 @@ def from_file(cls, path: Path) -> "RestoreTest": @property def key(self) -> str: - return f"{self.backup_type}_{self.location}" + from job_identity import historical_job_id + return historical_job_id(self.job_id) @property def test_date_dt(self) -> Optional[datetime]: @@ -423,11 +438,13 @@ def load(self, move_to_archive: bool = False) -> List[BackupStatus]: def get_latest_per_key( self, statuses: Optional[List[BackupStatus]] = None ) -> Dict[str, BackupStatus]: - """Gibt pro (backup_type_location) den neuesten Status zurück.""" + """Return the latest status per UUID; unresolved records have no key.""" data = statuses if statuses is not None else self._statuses latest: Dict[str, BackupStatus] = {} for st in data: key = st.key + if not key: + continue existing = latest.get(key) if existing is None: latest[key] = st @@ -440,11 +457,13 @@ def get_latest_per_key( def aggregate_by_key( self, statuses: Optional[List[BackupStatus]] = None ) -> Dict[str, "_BackupAggregate"]: - """Aggregiert Statistiken pro (backup_type_location).""" + """Aggregate statistics per UUID.""" data = statuses if statuses is not None else self._statuses result: Dict[str, _BackupAggregate] = {} for st in data: key = st.key + if not key: + continue if key not in result: result[key] = _BackupAggregate(key=key) result[key].add(st) @@ -495,11 +514,11 @@ def success_rate(self) -> float: @property def backup_type(self) -> str: - return self.key.rsplit("_", 1)[0] if "_" in self.key else self.key + return self.latest.backup_type if self.latest else "" @property def location(self) -> str: - return self.key.rsplit("_", 1)[1] if "_" in self.key else "" + return self.latest.location if self.latest else "" # --------------------------------------------------------------------------- diff --git a/runtime/scripts/borg_restore_test.py b/runtime/scripts/borg_restore_test.py index 21ed9c7a..51ce56f3 100644 --- a/runtime/scripts/borg_restore_test.py +++ b/runtime/scripts/borg_restore_test.py @@ -191,9 +191,12 @@ def discover_repos(conf: dict) -> list: continue if str(raw.get("runner", "")).strip() != "scriptless-wizard-runner": continue - btype = str(raw.get("backup_type", "")).strip() - job_key = str(raw.get("job_key") or jf.stem).strip() - if not btype or not job_key: + from job_identity import metadata_job_id + try: + job_key = metadata_job_id(raw) + except ValueError: + continue + if not job_key: continue try: context = resolve_job_repository_context(config, job_key, job=raw, inventory=inventory) @@ -208,7 +211,8 @@ def discover_repos(conf: dict) -> list: seen.add(key) repos.append({ "job_key": job_key, - "type": btype, + "type": "", + "name": str(raw.get("name") or "Backup"), "location": location, "path": repo_path, "encryption": str(context.get("encryption") or "").strip().lower(), @@ -241,16 +245,14 @@ def __init__(self, conf: dict, args: argparse.Namespace): self.test_interval = int(conf.get("RESTORE_TEST_INTERVAL_DAYS", 30)) self.status_dir = _resolve_restore_test_dir(conf) self.min_coverage = int(conf.get("RESTORE_TEST_MIN_COVERAGE", 5)) - self.max_entries = int(conf.get("RESTORE_TEST_MAX_ENTRIES", 10000)) + self.max_entries = int(conf.get("RESTORE_TEST_MAX_ENTRIES", 1000)) self.sample_size = int(conf.get("RESTORE_TEST_SAMPLE_SIZE", 5)) self.borg_timeout = int(conf.get("RESTORE_TEST_BORG_TIMEOUT", 180)) self.dryrun_timeout = int(conf.get("RESTORE_TEST_DRY_RUN_TIMEOUT", 0)) # 0 = no timeout self.dryrun_chunk = int(conf.get("RESTORE_TEST_DRY_RUN_CHUNK_SIZE", 200)) - self.dryrun_max_files = int(conf.get("RESTORE_TEST_DRY_RUN_MAX_FILES", 1500)) + self.dryrun_max_files = int(conf.get("RESTORE_TEST_DRY_RUN_MAX_FILES", 1000)) self.level3_legacy_sampling = str(conf.get("RESTORE_TEST_LEVEL3_LEGACY_SAMPLING", "false")).strip().lower() == "true" - force_types_raw = str(conf.get("RESTORE_TEST_FORCE_CHUNK_TYPES", "vms")) - self.force_chunk_types = {x.strip().lower() for x in force_types_raw.split(",") if x.strip()} - self.full_dryrun_max_archive_gb = int(conf.get("RESTORE_TEST_FULL_DRYRUN_MAX_ARCHIVE_GB", 200)) + self.full_dryrun_max_archive_gb = int(conf.get("RESTORE_TEST_FULL_DRYRUN_MAX_ARCHIVE_GB", 500)) self.log_dir = Path(conf.get("GLOBAL_LOG_DIR", "/mnt/user/Logs")) self.log_dir.mkdir(parents=True, exist_ok=True) @@ -560,15 +562,14 @@ def _mark_not_tested(self, steps: list, after_step: str) -> None: def test_repo(self, repo: dict) -> int: """0=OK, 1=Fehler, 2=übersprungen, 3=unavailable""" - btype = repo["type"] location = repo["location"] path = repo["path"] encryption = str(repo.get("encryption") or "").strip().lower() pp_file = repo["passphrase_file"] - key = str(repo.get("job_key") or f"{btype}_{location}") + key = str(repo["job_key"]) self.log(f"{'─'*60}") - self.log(f"TEST: {btype} ({location})") + self.log(f"TEST: {repo.get('name') or key} ({location})") self.log(f" Repository: {path}") if self.args.dry_run: @@ -756,52 +757,56 @@ def test_repo(self, repo: dict) -> int: self.log("Level 2: Extract Dry-Run") s_probe = time.time() - r_count = self._borg(["list", "--short", f"{path}::{last_archive}"], env, timeout=300) - full_count = len(r_count.stdout.splitlines()) if r_count.returncode == 0 else 0 - test_count = max(100, full_count * self.min_coverage // 100) if full_count else 100 - test_count = min(test_count, self.max_entries) - self.log(f" Testing {test_count} of {full_count} entries") - r_list = self._borg(["list", "--json-lines", f"{path}::{last_archive}"], env, timeout=300) - tested_entries: list = [] - tested_files = tested_folders = 0 - if r_list.returncode == 0: - for line in r_list.stdout.splitlines()[:test_count]: - try: - e = json.loads(line) - except json.JSONDecodeError: - continue - etype = e.get("type", "?") - epath = e.get("path", "") - if etype == "d": - tested_entries.append(f"d {epath}") - tested_folders += 1 - elif etype == "-": - tested_entries.append(f"- {epath}") - tested_files += 1 - else: - tested_entries.append(f"{etype} {epath}") - tested_total = len(tested_entries) + if r_list.returncode != 0: + err = self._analyze_error(r_list.stderr) + code = self._failure_code_from_category(err["category"]) + steps.append({"step_id": "restore_probe", "status": "failed", + "duration_ms": int((time.time() - s_probe) * 1000), + "message": "Could not list archive files for the restore probe", + "command": "borg list --json-lines", "error_code": code}) + self._mark_not_tested(steps, "restore_probe") + self._write(key, repo, "failed", int(time.time()-t0), 0, 0, 0, "unknown", + last_archive, archive_stats, [], exit_code=1, steps=steps, + error_category=err["category"], error_details=err["details"], + failure_code=code, failure_hint=err["details"]) + return 1 + file_paths = [] + for line in r_list.stdout.splitlines(): + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if entry.get("type") == "-" and entry.get("path"): + file_paths.append(entry["path"]) + full_count = len(file_paths) + archive_stats["files_count"] = full_count + # Coverage and both existing limits refer to regular files, not directories (#495). + target_count = max(1, (full_count * self.min_coverage + 99) // 100) + test_count = min(full_count, target_count, max(1, self.max_entries), max(1, self.dryrun_max_files)) + tested_paths = random.sample(file_paths, test_count) + tested_entries = [f"- {path}" for path in tested_paths] + tested_files = tested_total = len(tested_paths) + tested_folders = 0 archive_size_gb = int(archive_stats.get("original_size", 0)) / (1024**3) if archive_stats else 0 - force_chunk = btype.strip().lower() in self.force_chunk_types - if self.full_dryrun_max_archive_gb > 0 and archive_size_gb >= self.full_dryrun_max_archive_gb: - force_chunk = True + force_chunk = self.full_dryrun_max_archive_gb > 0 and archive_size_gb >= self.full_dryrun_max_archive_gb - if force_chunk and tested_entries: - reason = f"type rule ({btype})" if btype.strip().lower() in self.force_chunk_types else f"archive size {archive_size_gb:.1f} GB" - self.log(f" Chunk mode enabled ({reason})") + if force_chunk and tested_paths: + self.log(f" Chunk mode enabled (archive size {archive_size_gb:.1f} GB)") + self.log(f" Testing {test_count} of {full_count} files (target {self.min_coverage}%, file limits applied)") failed_chunk = None - tested_files_only = [e[2:] for e in tested_entries if e.startswith("- ")] - random.shuffle(tested_files_only) - tested_paths = tested_files_only[:max(1, self.dryrun_max_files)] + completed_paths = [] for i in range(0, len(tested_paths), max(1, self.dryrun_chunk)): chunk = tested_paths[i:i + max(1, self.dryrun_chunk)] r_chunk = self._borg(["extract", "--dry-run", f"{path}::{last_archive}", *chunk], env, timeout=self.dryrun_timeout) if r_chunk.returncode != 0: failed_chunk = r_chunk break + completed_paths.extend(chunk) if failed_chunk is not None: + tested_files = tested_total = len(completed_paths) + tested_entries = [f"- {path}" for path in completed_paths] err = self._analyze_error(failed_chunk.stderr) code = self._failure_code_from_category(err["category"]) steps.append({ @@ -821,6 +826,7 @@ def test_repo(self, repo: dict) -> int: steps=steps, failure_code=code, failure_hint=err["details"]) return 1 else: + self.log(f" Testing the complete archive ({full_count} files)") r_dry = self._borg(["extract", "--dry-run", f"{path}::{last_archive}"], env, timeout=self.dryrun_timeout) if r_dry.returncode != 0 and not (r_dry.returncode == 124 and tested_entries): err = self._analyze_error(r_dry.stderr) @@ -841,6 +847,8 @@ def test_repo(self, repo: dict) -> int: error_category=err["category"], error_details=err["details"], error_output=r_dry.stderr[:500], steps=steps, failure_code=code, failure_hint=err["details"]) return 1 + if r_dry.returncode == 0: + tested_files = tested_total = full_count steps.append({ "step_id": "restore_probe", "status": "passed", @@ -945,6 +953,8 @@ def _write(self, key: str, repo: dict, result_str: str, duration: int, l3_details: dict = None, error_category: str = "none", error_details: str = "", error_output: str = "", reason: str = "", steps: list | None = None, failure_code: str = "", failure_hint: str = "") -> None: + from job_identity import validate_job_id + key = validate_job_id(key) _ensure_status_storage_directory(self.status_dir) test_file = self.status_dir / f"{key}.test" now = datetime.now() @@ -955,10 +965,11 @@ def _write(self, key: str, repo: dict, result_str: str, duration: int, ) data = { + "job_id": key, "report_schema_version": 1, "report_id": f"RT-{now.strftime('%Y%m%d-%H%M%S')}-{key}", "repository": repo["path"], - "type": repo["type"], + "job_name": repo.get("name") or key, "location": repo["location"], "test_level": self.test_level, "test_date": now.strftime("%Y-%m-%d %H:%M:%S"), @@ -975,7 +986,7 @@ def _write(self, key: str, repo: dict, result_str: str, duration: int, "test_coverage": coverage, "test_coverage_percentage": test_coverage_pct, "coverage_percent": test_coverage_pct, - "coverage_basis": f"{tested_total}/{stats.get('files_count', 0) if isinstance(stats, dict) else 0}", + "coverage_basis": f"{tested_files}/{stats.get('files_count', 0) if isinstance(stats, dict) else 0}", "tested_archive": archive, "tested_entries": entries, "overall_status": overall_status, diff --git a/tests/job_fixtures.py b/tests/job_fixtures.py new file mode 100644 index 00000000..1043cafa --- /dev/null +++ b/tests/job_fixtures.py @@ -0,0 +1,41 @@ +"""Canonical test jobs with readable labels and deterministic UUIDs (#486). + +Legacy migration inputs deliberately do not use this helper. +""" + +import uuid + + +def job_id(label: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_URL, "https://example.invalid/test-jobs/" + label)) + + +def identified_job(metadata: dict) -> dict: + result = dict(metadata) + key = str(result["job_key"]) + try: + uuid.UUID(key) + except ValueError: + key = job_id(key) + result.update(job_id=key, job_key=key, schema_version=5) + result.setdefault("archive_prefix", result["backup_type"] + "-backup") + result.setdefault("archive_prefixes", [result["archive_prefix"]]) + result.setdefault("cache_subdir", result["location"] + "_" + result["backup_type"].lower()) + result.setdefault("check_flag_name", ".last_check_" + result["backup_type"].lower()) + result.setdefault("compression", "lz4") + result.setdefault("retention", {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}) + return result + + +def write_job(root, label, **values): + """Give status/report fixtures an existing job rather than synthesizing one from history.""" + import json + from pathlib import Path + backup_type, location = label.rsplit("_", 1) + meta = identified_job({"job_key": label, "backup_type": backup_type, "location": location, + "name": backup_type, "enabled": True, **values}) + target = Path(root) / "config" / "jobs" / f"{meta['job_id']}.json" + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists(): + target.write_text(json.dumps(meta), encoding="utf-8") + return meta diff --git a/tests/job_identity_ui.cjs b/tests/job_identity_ui.cjs new file mode 100644 index 00000000..e3ccf27a --- /dev/null +++ b/tests/job_identity_ui.cjs @@ -0,0 +1,244 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const test = require('node:test'); + +for (const language of ['de', 'en']) { + test(`maintenance confirmation uses the selected job's name and full prefix (${language})`, async () => { + const labels = JSON.parse(fs.readFileSync(`ui/i18n/${language}.json`, 'utf8')); + const elements = new Map(); + const context = vm.createContext({ + window: {BBUI: {components: {i18n: {t(key, params = {}) { + const label = key.split('.').reduce((value, part) => value?.[part], labels) || key; + return label.replace(/\{(\w+)\}/g, (_, name) => params[name] ?? ''); + }}}}, addEventListener() {}}, + document: {getElementById(key) { + if (!elements.has(key)) elements.set(key, {value: '', innerHTML: '', + classList: {add() {}, remove() {}, contains() {return false;}}}); + return elements.get(key); + }}, + escHtml: value => String(value), + }); + vm.runInContext(fs.readFileSync('ui/js/pages/storage.js', 'utf8'), context); + const alpha = {key: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', name: 'Zulu', + archive_prefix: 'Flash-Config', backup_type: 'old_type', location: 'local', + retention: {daily: '14', weekly: '4', monthly: '6', yearly: '3'}}; + const zulu = {key: 'ffffffff-ffff-4fff-8fff-ffffffffffff', name: 'Alpha', + archive_prefix: 'testdata-backup', backup_type: 'old_type', location: 'local', + retention: {daily: '7', weekly: '4', monthly: '6', yearly: '3'}}; + const repo = {repository_key: 'shared', display_name: 'Repository title', + job_name: 'Old repository job label', used_by: [alpha.key, zulu.key]}; + const state = context.window.BBUI.storageState; + state.data = {groups: {local: [repo]}}; + state.jobs = [alpha, zulu]; + const confirmation = vm.runInContext("openStorageMaintenanceConfirm('shared', 'prune', 'quick')", context); + const html = elements.get('storage-maintenance-confirm-info').innerHTML; + assert.ok(html.includes('testdata-backup-*')); + assert.ok(html.includes('Flash-Config-*')); + assert.ok(html.includes('Repository title')); + assert.ok(!html.includes('Old repository job label')); + assert.ok(!html.includes(alpha.key + '-backup')); + assert.ok(!html.includes(zulu.key + '-backup')); + assert.ok(html.indexOf(`value="${zulu.key}"`) < html.indexOf(`value="${alpha.key}"`)); + context.document.getElementById('storage-maintenance-retention-job').value = alpha.key; + vm.runInContext('updateStorageMaintenanceRetentionPreview()', context); + const preview = elements.get('storage-maintenance-retention-preview').innerHTML; + assert.ok(preview.includes('Zulu')); + assert.ok(preview.includes('Flash-Config-*')); + assert.ok(preview.includes('14')); + assert.ok(!preview.includes('testdata-backup-*')); + vm.runInContext('closeStorageMaintenanceConfirm(true)', context); + const result = await confirmation; + assert.equal(result.jobKey, alpha.key); + assert.equal(result.confirmed, true); + // An unlinked job with the same former type/location must not become a source. + context.unlinked = {backup_type: 'old_type', location: 'local'}; + assert.equal(vm.runInContext('storageJobsForRepository(unlinked).length', context), 0); + context.missingPrefix = {key: alpha.key}; + assert.equal(vm.runInContext('storageArchiveFilterFromJob(missingPrefix)', context), ''); + }); +} + +for (const language of ['de', 'en']) { + test(`wizard accepts 100 characters and rejects longer names (${language})`, () => { + const labels = JSON.parse(fs.readFileSync(`ui/i18n/${language}.json`, 'utf8')); + const context = vm.createContext({ + window: {BBUI: {components: {i18n: {t(key) { + return key.split('.').reduce((value, part) => value?.[part], labels) || key; + }}}}, addEventListener() {}}, + document: {}, input: {job_id: '645de013-df1e-49e3-89f0-39c9bb3e299b', job_name: 'ä'.repeat(100), archive_prefix: 'test-backup'}, error: '', + }); + vm.runInContext(fs.readFileSync('ui/js/pages/wizard.js', 'utf8'), context); + vm.runInContext(` + wizardClearError = () => { error = ''; }; + _wizardCollectParams = () => input; + _wizardShowError = (step, message) => { error = message; }; + `, context); + assert.equal(vm.runInContext('_wizardValidate(1)', context), true); + context.input.job_name += 'ä'; + assert.equal(vm.runInContext('_wizardValidate(1)', context), false); + assert.equal(context.error, labels.wizard.validationJobNameLength); + assert.equal(vm.runInContext("wizardApiErrorMessage({code:'job_name_too_long'})", context), context.error); + }); +} + +test('wizard schedules the saved UUID and retries without creating another job', async () => { + const id = '645de013-df1e-49e3-89f0-39c9bb3e299b'; + const proposedId = 'bc198590-b17b-4a30-a5c4-f721c45cdaea'; + const elements = new Map(); + const requests = []; + let failSchedule = true; + let closed = false; + const context = vm.createContext({ + window: {BBUI: {core: {getSchedulesData: () => ({}), setSchedulesData() {}}}, addEventListener() {}}, + document: {getElementById(key) { + if (!elements.has(key)) elements.set(key, {checked: true, classList: {add() {}, remove() {}}}); + return elements.get(key); + }}, + jobsState: {}, + apiErrorMessage: data => data.error || '', + refreshJobs: async () => {}, showMsg() {}, + fetch: async (url, options) => { + requests.push({url, body: JSON.parse(options.body)}); + return url === '/api/wizard/save' + ? {ok: true, json: async () => ({job_id: id, job_key: id})} + : {ok: !failSchedule, status: failSchedule ? 500 : 200, + json: async () => failSchedule ? {error: 'crontab failed'} : {saved: true}}; + }, + closePreview: () => {closed = true;}, + }); + vm.runInContext(fs.readFileSync('ui/js/pages/wizard.js', 'utf8'), context); + context.window.BBUI.wizardState.jobId = proposedId; + vm.runInContext(` + _wizardValidate = () => true; + _wizardCollectParams = () => ({job_name: 'My job', archive_prefix: 'flash-config', + job_id: wizardState.jobId, existing_job_key: wizardState.existingJobKey, location: 'local'}); + _wizardBuildCron = () => '0 9 * * *'; + closeWizard = closePreview; + `, context); + assert.equal(vm.runInContext("_wizardArchivePrefix('flash-config')", context), 'flash-config'); + assert.equal(vm.runInContext("_wizardArchivePrefix('Flash-Config')", context), 'Flash-Config'); + await vm.runInContext('saveWizardJob()', context); + assert.equal(requests[0].body.job_id, proposedId); + assert.equal(requests[0].body.existing_job_key, ''); + assert.equal(requests[1].body.job_key, id); + assert.equal(closed, false); + assert.equal(context.window.BBUI.wizardState.existingJobKey, id); + assert.equal(context.window.BBUI.wizardState.jobId, id); + assert.equal(elements.get('wiz-job-id').value, id); + failSchedule = false; + await vm.runInContext('saveWizardJob()', context); + assert.equal(requests[2].body.job_id, id); + assert.equal(requests[2].body.job_name, requests[0].body.job_name); + assert.equal(requests[2].body.archive_prefix, requests[0].body.archive_prefix); + assert.equal(requests[2].body.existing_job_key, id); + assert.equal(requests[3].body.job_key, id); + assert.equal(closed, true); +}); + +function newJobWizardContext(language = 'en') { + const labels = JSON.parse(fs.readFileSync(`ui/i18n/${language}.json`, 'utf8')); + const elements = new Map(); + const requests = []; + function element(key) { + if (!elements.has(key)) { + const classes = new Set(); + elements.set(key, {id: key, value: '', style: {}, dataset: {}, checked: false, + classList: {add: key => classes.add(key), remove: key => classes.delete(key), + contains: key => classes.has(key), toggle(key, active) {active ? classes.add(key) : classes.delete(key);}}, + setAttribute() {}, removeAttribute() {}, addEventListener() {}, closest: () => null, + querySelectorAll: () => [...elements.values()], + }); + } + return elements.get(key); + } + const context = vm.createContext({ + window: {BBUI: {components: {i18n: {t(key, params = {}) { + const value = key.split('.').reduce((value, part) => value?.[part], labels) || key; + return value.replace(/\{(\w+)\}/g, (_, name) => params[name] ?? ''); + }}}}, addEventListener() {}}, + document: {getElementById: element, body: element('body')}, + fetch(url, options) {return new Promise(resolve => requests.push({url, options, resolve}));}, + apiErrorMessage: data => data.message || 'Request failed', + }); + vm.runInContext(fs.readFileSync('ui/js/pages/wizard.js', 'utf8'), context); + // Keep identity lifecycle, form collection and navigation real; omit unrelated UI rendering. + for (const name of ['wizardBindRuntimeControls', '_wizardSyncRiskAcknowledgement', + 'wizardCancelSourceSuggestRequest', 'wizardRenderSourcePaths', 'wizardCancelExcludeSuggestRequest', + 'wizardRenderExcludePaths', 'wizardUpdateRetentionManualLink', '_wizardScheduleApplyUI', + 'wizardSchedulePreview', 'wizardUpdateIconPreview', 'wizardRenderArchivePrefixSummary', + 'wizardAutoFill', 'wizardRenderRuntimeControls', 'wizardUpdateFinalRiskAcknowledgements']) { + vm.runInContext(`${name} = () => {};`, context); + } + vm.runInContext(` + wizardLoadStorageTargets = wizardLoadRepositories = wizardLoadRuntimeInventory = async () => {}; + wizardSelectedStorage = wizardSelectedRepository = () => ({}); + _wizardRuntimeMode = () => 'none'; + _wizardRiskAcknowledged = () => false; + `, context); + return {context, elements, requests, state: context.window.BBUI.wizardState}; +} + +test('new job shows one ID, preserves it during navigation, and discards it on cancel', async () => { + const {context, elements, requests, state} = newJobWizardContext(); + const displayed = '645de013-df1e-49e3-89f0-39c9bb3e299b'; + vm.runInContext("openWizard({type: 'click'})", context); + assert.equal(elements.get('wiz-job-id-group').hidden, false); + assert.equal(elements.get('wizard-next-btn').disabled, true); + assert.equal(requests.length, 1); + assert.equal(requests[0].url, '/api/wizard/new-job-id'); + assert.equal(requests[0].options.cache, 'no-store'); + requests[0].resolve({ok: true, json: async () => ({job_id: displayed})}); + await state.loadingPromise; + assert.equal(elements.get('wiz-job-id').value, displayed); + assert.equal(elements.get('wizard-next-btn').disabled, false); + elements.get('wiz-job-name').value = 'New job'; + elements.get('wiz-archive-prefix').value = 'new-job-backup'; + await vm.runInContext('wizardNext()', context); + assert.equal(state.step, 2); + vm.runInContext('wizardBack()', context); + assert.equal(state.jobId, displayed); + assert.equal(vm.runInContext('_wizardCollectParams().job_id', context), displayed); + vm.runInContext('closeWizard({force:true})', context); + assert.equal(state.jobId, ''); + assert.equal(requests.length, 1); // Cancel sends no write or deletion request. +}); + +test('late ID replies cannot change a reopened wizard or an existing job', async () => { + const {context, elements, requests, state} = newJobWizardContext(); + const oldId = '645de013-df1e-49e3-89f0-39c9bb3e299b'; + const newId = 'bc198590-b17b-4a30-a5c4-f721c45cdaea'; + vm.runInContext('openWizard()', context); + const abandoned = state.loadingPromise; + vm.runInContext('closeWizard({force:true}); openWizard()', context); + requests[1].resolve({ok: true, json: async () => ({job_id: newId})}); + await state.loadingPromise; + requests[0].resolve({ok: true, json: async () => ({job_id: oldId})}); + await abandoned; + assert.equal(elements.get('wiz-job-id').value, newId); + vm.runInContext('closeWizard({force:true}); openWizard()', context); + const replaced = state.loadingPromise; + vm.runInContext(`openWizard('${oldId}')`, context); + assert.equal(requests.length, 3); // Edit initialization never generates a new ID. + requests[2].resolve({ok: true, json: async () => ({job_id: newId})}); + await replaced; + assert.equal(state.jobId, oldId); + assert.equal(elements.get('wiz-job-id').value, oldId); +}); + +for (const language of ['de', 'en']) { + test(`failed or invalid ID responses keep the wizard from continuing (${language})`, async () => { + for (const response of [{ok:false, message:'Unavailable'}, {ok:true, job_id:'not-a-uuid'}, + {ok:true}, {ok:true, job_id:['645de013-df1e-49e3-89f0-39c9bb3e299b']}]) { + const {context, elements, requests, state} = newJobWizardContext(language); + vm.runInContext('openWizard()', context); + requests[0].resolve({ok: response.ok, status:503, json: async () => response}); + await state.loadingPromise; + assert.equal(state.jobId, ''); + assert.equal(elements.get('wizard-next-btn').disabled, true); + assert.equal(elements.get('wizard-error-1').classList.contains('hidden'), false); + assert.ok(elements.get('wizard-error-1').textContent.includes(language === 'de' ? 'erneut öffnen' : 'reopen')); + assert.equal(vm.runInContext('_wizardValidate(9)', context), false); + } + }); +} diff --git a/tests/restore_browse_state.cjs b/tests/restore_browse_state.cjs new file mode 100644 index 00000000..bbd3b32e --- /dev/null +++ b/tests/restore_browse_state.cjs @@ -0,0 +1,205 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const test = require('node:test'); + +function deferred() { + let resolve; + const promise = new Promise(done => { resolve = done; }); + return {promise, resolve}; +} + +const response = (data, status = 200) => ({ok: status < 400, status, json: async () => data}); + +function page(language = 'en') { + const labels = JSON.parse(fs.readFileSync(`ui/i18n/${language}.json`, 'utf8')); + const elements = new Map(); + const get = id => { + if (!elements.has(id)) { + elements.set(id, { + value: '', checked: false, disabled: false, textContent: '', style: {}, html: '', + get innerHTML() { return this.html; }, + set innerHTML(value) { this.html = value; if (id.endsWith('-sel')) this.value = ''; }, + appendChild() {}, classList: {add() {}, remove() {}, toggle() {}}, + }); + } + return elements.get(id); + }; + const context = vm.createContext({ + window: {BBUI: {components: {i18n: {t(key, params = {}) { + const label = key.split('.').reduce((value, part) => value?.[part], labels) || key; + return label.replace(/\{(\w+)\}/g, (_, name) => params[name] ?? ''); + }}}}, addEventListener() {}}, + document: {getElementById: get, createElement: () => ({})}, + messages: [], hideEl() {}, showMsg() {}, escHtml: value => String(value), + apiErrorMessage: data => labels.api.errors[data.code] || data.message || 'API error', + }); + vm.runInContext(fs.readFileSync('ui/js/pages/restore.js', 'utf8'), context); + for (const name of ['restoreSetLiveMode', 'restoreSwitchView', 'restoreSetStep', + '_restoreBindTargetAutocomplete', '_restoreRenderSelectionSummary', '_restoreRenderSelectedBox', + 'restoreLoadRuns', 'restoreLoadHistory', 'renderRestoreJobSidebar', 'renderRestoreSelectedJob', + 'renderRestoreSourceContext', 'renderRestoreArchiveList', 'renderRestorePrecheck', + '_restoreRenderBreadcrumb', '_setRestoreAssistBusy', 'restoreUpdateConfirmState']) context[name] = () => {}; + context.restoreLoadAllowedTargetRoots = async () => {}; + context._restorePrimaryAllowedRoot = () => '/mnt/user'; + context._isAllowedRestoreTarget = () => true; + context._restoreMsg = (message, error) => context.messages.push({message, error}); + context._restoreRenderFiles = files => { get('restore-filelist').innerHTML = JSON.stringify(files); }; + return {context, get, state: context.window.BBUI.restoreState, labels}; +} + +test('returning after backups, repository switches and rename refreshes the same job', async () => { + const {context, get, state} = page(); + const calls = []; + let repository = 'repo1', name = 'Test'; + const archives = {repo1: [{name: 'first'}], repo2: [{name: 'other-repository'}]}; + context.fetch = async url => { + calls.push(url); + return response(url === '/api/jobs' + ? {jobs: [{key: 'job-id', name, repository_key: repository}]} + : {archives: archives[repository], archive_filters: []}); + }; + state.job = 'job-id'; + for (const stage of ['initial', 'backup', 'repo2', 'repo1', 'rename']) { + if (stage === 'backup') archives.repo1.push({name: 'scheduled'}); + if (stage === 'repo2' || stage === 'repo1') repository = stage; + if (stage === 'rename') name = 'Renamed'; + state.archive = 'stale'; state.files = [{name: 'old-file'}]; + state.selectedPath = 'old-file'; state.precheck = {ok: true}; + get('restore-source-path').value = 'old-file'; + get('restore-confirm-check').checked = true; + await context.restoreInit(); + assert.equal(state.job, 'job-id'); + assert.equal(state.jobs[0].repository_key, repository); + assert.equal(state.jobs[0].name, name); + assert.equal(JSON.stringify(state.archives), JSON.stringify(archives[repository])); + assert.equal(state.archive, ''); + assert.equal(state.selectedPath, ''); + assert.equal(state.precheck, null); + assert.equal(state.files.length, 0); + assert.equal(get('restore-source-path').value, ''); + assert.equal(get('restore-confirm-check').checked, false); + } + assert.equal(calls.filter(url => url.startsWith('/api/restore/archives')).length, 5); +}); + +test('late archive-list responses cannot restore the previous job selection', async () => { + const {context, get, state} = page(); + const first = deferred(); + context.fetch = url => url.endsWith('job=first') ? first.promise : Promise.resolve(response({archives: [{name: 'second'}]})); + get('restore-job-sel').value = 'first'; + const pending = context.restoreLoadArchives(); + get('restore-job-sel').value = 'second'; + await context.restoreLoadArchives(); + first.resolve(response({archives: [{name: 'obsolete'}]})); + await pending; + assert.equal(state.job, 'second'); + assert.equal(state.archives[0].name, 'second'); +}); + +test('source changes discard in-flight file listings and precheck results', async () => { + const {context, get, state} = page(); + state.job = 'job-id'; + get('restore-archive-sel').value = 'old'; + const files = deferred(); + context.fetch = () => files.promise; + const browse = context.restoreBrowse(''); + get('restore-job-sel').value = 'job-id'; + context.fetch = async () => response({archives: [{name: 'new'}]}); + await context.restoreLoadArchives(); + files.resolve(response({files: [{name: 'obsolete'}]})); + await browse; + assert.equal(state.files.length, 0); + assert.equal(get('restore-filelist').innerHTML, ''); + + const check = deferred(); + state.archive = 'new'; state.selectedPath = 'data'; + get('restore-target-path').value = '/mnt/user/test'; + context.fetch = () => check.promise; + const precheck = context.restoreRunPrecheck(); + context.fetch = async () => response({archives: []}); + await context.restoreLoadArchives(); + check.resolve(response({ok: true})); + await precheck; + assert.equal(state.precheck, null); + assert.equal(state.selectedPath, ''); +}); + +test('a removed job or failed archive refresh leaves no old selectable archive', async () => { + const {context, get, state} = page(); + state.job = 'removed'; state.archive = 'old'; state.archives = [{name: 'old'}]; + context.fetch = async () => response({jobs: [{key: 'remaining'}]}); + await context.restoreInit(); + assert.equal(state.job, ''); + assert.equal(state.archives.length, 0); + get('restore-job-sel').value = 'remaining'; + context.fetch = async () => response({code: 'internal_error'}, 500); + await context.restoreLoadArchives(); + assert.equal(state.archive, ''); + assert.equal(state.archives.length, 0); +}); + +for (const language of ['de', 'en']) test(`missing archives stop loading and allow reselection (${language})`, async () => { + const {context, get, state, labels} = page(language); + state.job = 'job-id'; state.archive = 'missing'; state.archives = [{name: 'missing'}]; + state.selectedPath = 'stale'; state.precheck = {ok: true}; + get('restore-archive-sel').value = 'missing'; + get('restore-source-path').value = 'stale'; get('restore-confirm-check').checked = true; + context.fetch = async () => response({code: 'restore_archive_unavailable', error: 'Archive missing does not exist'}, 404); + await context.restoreBrowse(''); + assert.equal(state.archive, ''); + assert.equal(state.archives.length, 0); + assert.equal(state.files.length, 0); + assert.equal(state.selectedPath, ''); + assert.equal(state.precheck, null); + assert.equal(get('restore-confirm-check').checked, false); + assert.ok(get('restore-filelist').innerHTML.includes(labels.api.errors.restore_archive_unavailable)); + assert.ok(get('restore-filelist').innerHTML.includes('role="alert"')); + + get('restore-job-sel').value = 'job-id'; + context.fetch = async () => response({archives: [{name: 'available'}]}); + await context.restoreLoadArchives(); + get('restore-archive-sel').value = 'available'; + context.fetch = async () => response({files: [{name: 'recovered.txt'}]}); + await context.restoreBrowse(''); + assert.equal(state.files[0].name, 'recovered.txt'); + assert.ok(get('restore-filelist').innerHTML.includes('recovered.txt')); +}); + +for (const failure of ['server', 'network', 'json', 'invalid-data']) test(`${failure} errors clear loading and allow retry`, async () => { + const {context, get, state} = page(); + state.job = 'job-id'; state.archive = 'archive'; + state.selectedPath = 'stale'; state.precheck = {ok: true}; + get('restore-archive-sel').value = 'archive'; + context.fetch = async () => { + if (failure === 'network') throw new Error('Failed to fetch'); + if (failure === 'json') return {ok: true, json: async () => { throw new Error('Invalid JSON'); }}; + return failure === 'server' ? response({code: 'internal_error'}, 500) : response(null); + }; + await context.restoreBrowse(''); + assert.equal(state.selectedPath, ''); + assert.equal(state.precheck, null); + assert.ok(get('restore-filelist').innerHTML.includes('role="alert"')); + assert.equal(state.archive, 'archive'); + context.fetch = async () => response({files: [{name: 'retry.txt'}]}); + await context.restoreBrowse(''); + assert.equal(state.files[0].name, 'retry.txt'); +}); + +test('a late missing-archive error cannot clear a newer successful selection', async () => { + const {context, get, state} = page(); + state.job = 'job-id'; + const old = deferred(); + get('restore-archive-sel').value = 'old'; + context.fetch = () => old.promise; + const pending = context.restoreBrowse(''); + get('restore-archive-sel').value = 'new'; + context.fetch = async () => response({files: [{name: 'current.txt'}]}); + await context.restoreBrowse(''); + const messageCount = context.messages.length; + old.resolve(response({code: 'restore_archive_unavailable'}, 404)); + await pending; + assert.equal(state.archive, 'new'); + assert.equal(state.files[0].name, 'current.txt'); + assert.equal(context.messages.length, messageCount); +}); diff --git a/tests/security/test_authenticated_secret_exports.py b/tests/security/test_authenticated_secret_exports.py index f28f84d1..810b3d2f 100644 --- a/tests/security/test_authenticated_secret_exports.py +++ b/tests/security/test_authenticated_secret_exports.py @@ -176,7 +176,7 @@ def test_every_secret_bearing_export_entry_point_uses_authenticated_format(tmp_p secrets_dir.mkdir() monkeypatch.setattr(transfer, "_secrets_dir", lambda: secrets_dir) monkeypatch.setattr(transfer, "export_jobs_bundle", lambda config, selected_keys=None: { - "bundle": {"format": "bbui-job-bundle-v2", "jobs": []}, + "bundle": {"format": "bbui-job-bundle-v3", "jobs": []}, "job_count": 0, }) monkeypatch.setattr(transfer, "_collect_job_passphrase_files", lambda bundle: {}) diff --git a/tests/security/test_borg_keyfile_persistence.py b/tests/security/test_borg_keyfile_persistence.py index 23bdc757..53082175 100644 --- a/tests/security/test_borg_keyfile_persistence.py +++ b/tests/security/test_borg_keyfile_persistence.py @@ -13,6 +13,8 @@ ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tests")) +from job_fixtures import identified_job, job_id API_ROOT = ROOT / "api" if str(API_ROOT) not in sys.path: sys.path.insert(0, str(API_ROOT)) @@ -131,13 +133,15 @@ def test_encrypted_job_transfer_restores_keyfile_to_target_store(tmp_path: Path) source_config = {"BACKUP_SCRIPTS_DIR": str(source)} jobs = source / "config" / "jobs" jobs.mkdir(parents=True) - (jobs / "flash_local.json").write_text(json.dumps({ - "schema_version": 3, + (jobs / f"{job_id('flash_local')}.json").write_text(json.dumps(identified_job({ + "schema_version": 5, + "backup_type": "flash", + "location": "local", "job_key": "flash_local", "name": "Flash", "repository_key": "repo_flash", "source_paths": ["/boot"], - }), encoding="utf-8") + })), encoding="utf-8") write_storage_store(source_config, {"storages": [{ "storage_key": "storage_local", "display_name": "Local", diff --git a/tests/test_activity_log.py b/tests/test_activity_log.py index 10f9c09e..2942a64b 100644 --- a/tests/test_activity_log.py +++ b/tests/test_activity_log.py @@ -1,3 +1,4 @@ +from job_fixtures import job_id import hashlib import io import json @@ -361,7 +362,7 @@ def test_saved_history_references_retained_log_while_active_reads_use_ram(tmp_pa active.parent.mkdir() active.write_text('WARNING source changed\n') cfg = BackupJobConfig( - job_name='Files', backup_type='files', backup_location='local', + job_name='Files', job_id=job_id('files_local'), backup_type='files', backup_location='local', lock_file=tmp_path / 'job.lock', log_dir=retained.parent, log_file=active, backup_paths=[tmp_path], borg_cache_dir=tmp_path / 'cache', date_tag='2026-09-06', status_dir=tmp_path / 'status', retained_log_file=retained, diff --git a/tests/test_backup_conf_canonical.py b/tests/test_backup_conf_canonical.py index de3bf1e0..c563a39d 100644 --- a/tests/test_backup_conf_canonical.py +++ b/tests/test_backup_conf_canonical.py @@ -158,6 +158,7 @@ def test_setup_status_tracks_optional_first_run_milestones_and_dismissal(tmp_pat data_dir = tmp_path / "runtime-data" write_conf(config, {"GLOBAL_DATA_DIR": str(data_dir)}) + ensure_data_dirs(str(data_dir)) setup = get_setup_status(config) assert setup["global_data_dir_set"] is True @@ -174,3 +175,72 @@ def test_setup_status_tracks_optional_first_run_milestones_and_dismissal(tmp_pat setup_after_dismiss = get_setup_status(config) assert setup_after_dismiss["setup"]["optional_dismissed"] is True assert setup_after_dismiss["setup"]["show_optional_wizard"] is False + + +def test_setup_status_does_not_create_or_probe_data_directories(tmp_path, monkeypatch): + config = _config(tmp_path) + data_dir = tmp_path / "runtime-data" + write_conf(config, {"GLOBAL_DATA_DIR": str(data_dir)}) + ensure_data_dirs(str(data_dir)) + before = {p: (p.stat().st_mtime_ns, p.stat().st_ctime_ns) for p in [data_dir, *data_dir.iterdir()]} + original_mkdir = Path.mkdir + + def guarded_mkdir(path, *args, **kwargs): + assert not path.is_relative_to(data_dir), "Status must not create data directories" + return original_mkdir(path, *args, **kwargs) + + monkeypatch.setattr(Path, "mkdir", guarded_mkdir) + for _ in range(10): + assert get_setup_status(config)["ready"] is True + assert {p: (p.stat().st_mtime_ns, p.stat().st_ctime_ns) for p in before} == before + assert not list(data_dir.rglob(".borg-ui-write-test")) + + +@pytest.mark.parametrize("problem", ["missing", "file", "unwritable"]) +def test_setup_status_reports_unusable_directory_without_repairing_it(tmp_path, monkeypatch, problem): + import config_api + + config = _config(tmp_path) + data_dir = tmp_path / "runtime-data" + write_conf(config, {"GLOBAL_DATA_DIR": str(data_dir)}) + ensure_data_dirs(str(data_dir)) + status_dir = data_dir / "status" + if problem in {"missing", "file"}: + status_dir.rmdir() + if problem == "file": + status_dir.write_text("existing file") + else: + original_access = config_api.os.access + monkeypatch.setattr(config_api.os, "access", lambda path, mode: False if Path(path) == status_dir else original_access(path, mode)) + result = get_setup_status(config) + assert result["ready"] is False + assert result["validation"]["errors"][0]["message_code"] == "config_data_dir_unusable" + if problem == "missing": + assert not status_dir.exists() + elif problem == "file": + assert status_dir.read_text() == "existing file" + + +@pytest.mark.parametrize("root", ["/mnt/user/borg-backup-ui", "/mnt/datapool2/borg-backup-ui", "/mnt/disks/USB-A/borg-backup-ui"]) +def test_setup_status_still_rejects_unavailable_mounts(tmp_path, monkeypatch, root): + import config_api + + config = _config(tmp_path) + write_conf(config, {"GLOBAL_DATA_DIR": root}) + monkeypatch.setattr(config_api, "_is_required_storage_mount_available", lambda _path: False) + result = get_setup_status(config) + assert result["ready"] is False + assert "unavailable" in result["validation"]["errors"][0]["message"] + + +def test_setup_write_probe_still_reports_actual_write_failure(tmp_path, monkeypatch): + original_write = Path.write_text + + def fail_probe(path, *args, **kwargs): + if path.name == ".borg-ui-write-test": + raise OSError("injected read-only filesystem") + return original_write(path, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", fail_probe) + with pytest.raises(OSError, match="read-only filesystem"): + ensure_data_dirs(str(tmp_path / "runtime-data")) diff --git a/tests/test_backup_exclusions.py b/tests/test_backup_exclusions.py index f6329f3d..70eea60e 100644 --- a/tests/test_backup_exclusions.py +++ b/tests/test_backup_exclusions.py @@ -1,3 +1,4 @@ +from job_fixtures import job_id import io import json import logging @@ -35,6 +36,7 @@ def test_exclusion_must_exist_below_a_source(tmp_path: Path): def test_backup_job_config_reads_exclusions_as_json(tmp_path: Path): cfg = BackupJobConfig.from_config({ + "BORG_UI_JOB_KEY": job_id("sources_local"), "BACKUP_PATHS_JSON": json.dumps([str(tmp_path / "source")]), "BACKUP_EXCLUDE_PATHS_JSON": json.dumps([str(tmp_path / "source" / "cache")]), }) @@ -47,6 +49,7 @@ def test_backup_job_config_preserves_multiple_source_paths_with_spaces(tmp_path: second = tmp_path / "Second source" cfg = BackupJobConfig.from_config({ + "BORG_UI_JOB_KEY": job_id("sources_local"), "BACKUP_PATHS_JSON": json.dumps([str(first), str(second)]), "BACKUP_EXCLUDE_PATHS_JSON": "[]", }) @@ -63,6 +66,7 @@ def test_backup_job_config_resolves_symlinked_source_root_for_borg(tmp_path: Pat visible_root.symlink_to(real_root, target_is_directory=True) cfg = BackupJobConfig.from_config({ + "BORG_UI_JOB_KEY": job_id("sources_local"), "BACKUP_PATHS_JSON": json.dumps([str(visible_root)]), "BACKUP_EXCLUDE_PATHS_JSON": json.dumps([str(visible_root / "cache")]), }) @@ -81,6 +85,7 @@ def test_backup_job_config_resolves_nested_path_below_symlinked_share_for_borg(t visible_root.symlink_to(real_root, target_is_directory=True) cfg = BackupJobConfig.from_config({ + "BORG_UI_JOB_KEY": job_id("sources_local"), "BACKUP_PATHS_JSON": json.dumps([str(visible_root / "adguard")]), "BACKUP_EXCLUDE_PATHS_JSON": json.dumps([str(visible_root / "adguard" / "cache")]), }) diff --git a/tests/test_borg_ssh_transport.py b/tests/test_borg_ssh_transport.py index 47cae093..fe0e0f8c 100644 --- a/tests/test_borg_ssh_transport.py +++ b/tests/test_borg_ssh_transport.py @@ -1,7 +1,11 @@ import shlex +import shutil +import subprocess import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] API_ROOT = ROOT / "api" @@ -41,10 +45,86 @@ def test_borg_ssh_keeps_custom_options_and_applies_managed_keepalives(): assert options["ServerAliveCountMax"] == "10" assert options["TCPKeepAlive"] == "yes" assert options["ControlPersist"] == "600" + assert options["IgnoreUnknown"] == "WarnWeakCrypto" assert options["WarnWeakCrypto"] == "no" + assert tokens.index("IgnoreUnknown=WarnWeakCrypto") < tokens.index("WarnWeakCrypto=no") assert "ServerAliveCountMax=3" not in tokens +@pytest.mark.parametrize("existing", [ + "ssh -o WarnWeakCrypto=yes", + "ssh -oWarnWeakCrypto=yes -oIgnoreUnknown=WarnWeakCrypto", + "ssh -o 'WarnWeakCrypto yes' -o 'IgnoreUnknown WarnWeakCrypto'", + "ssh -o warnweakcrypto=yes -o ignoreunknown=warnweakcrypto", +]) +def test_borg_ssh_normalization_keeps_one_ordered_compatibility_pair(existing): + command = build_borg_rsh(existing, "/root/.ssh/storage key") + tokens = shlex.split(command) + options = _ssh_options(command) + + assert options["IgnoreUnknown"].lower() == "warnweakcrypto" + assert options["WarnWeakCrypto"] == "no" + assert sum(token.lower().startswith("ignoreunknown=") for token in tokens) == 1 + assert sum(token.lower().startswith("warnweakcrypto=") for token in tokens) == 1 + assert tokens.index("IgnoreUnknown=" + options["IgnoreUnknown"]) < tokens.index("WarnWeakCrypto=no") + assert build_borg_rsh(command, "/root/.ssh/storage key") == command + + +@pytest.mark.parametrize("option", [ + "-o IgnoreUnknown=BBUIOptionalTestOption", + "-oIgnoreUnknown=BBUIOptionalTestOption", + "-o 'IgnoreUnknown BBUIOptionalTestOption'", +]) +def test_borg_ssh_preserves_existing_ignore_list_and_its_position(option): + command = build_borg_rsh( + f"ssh {option} -o BBUIOptionalTestOption=yes " + "-o IgnoreUnknown=UnusedLaterList -o ProxyJump=gateway -i /old/key", + "/new/key", + ) + tokens = shlex.split(command) + options = _ssh_options(command) + + assert options["IgnoreUnknown"] == "BBUIOptionalTestOption,WarnWeakCrypto" + assert tokens.index("IgnoreUnknown=" + options["IgnoreUnknown"]) < tokens.index("BBUIOptionalTestOption=yes") + assert options["ProxyJump"] == "gateway" + assert "IgnoreUnknown=UnusedLaterList" not in tokens + assert "/old/key" not in tokens + assert tokens[tokens.index("-i") + 1] == "/new/key" + assert build_borg_rsh(command, "/new/key") == command + + +@pytest.mark.skipif(shutil.which("ssh") is None, reason="OpenSSH client is unavailable") +@pytest.mark.parametrize("existing", [ + "ssh", + "ssh -o IgnoreUnknown=BBUIOptionalTestOption -o BBUIOptionalTestOption=yes", +]) +def test_borg_ssh_options_are_accepted_by_real_client_without_connecting(existing): + result = subprocess.run( + shlex.split(build_borg_rsh(existing)) + ["-F", "/dev/null", "-G", "127.0.0.1"], + capture_output=True, text=True, timeout=10, + ) + + assert result.returncode == 0, result.stderr + effective = dict(line.split(" ", 1) for line in result.stdout.splitlines() if " " in line) + assert effective["serveraliveinterval"] == "30" + assert effective["serveralivecountmax"] == "10" + # New clients expose the setting; older clients must accept its absence. + if "warnweakcrypto" in effective: + assert effective["warnweakcrypto"] == "no" + + +@pytest.mark.skipif(shutil.which("ssh") is None, reason="OpenSSH client is unavailable") +def test_borg_ssh_does_not_ignore_unrelated_unknown_options(): + result = subprocess.run( + shlex.split(build_borg_rsh()) + + ["-F", "/dev/null", "-G", "-o", "BBUIUnsupportedTestOption=yes", "127.0.0.1"], + capture_output=True, text=True, timeout=10, + ) + + assert result.returncode != 0 + assert "bad configuration option: bbuiunsupportedtestoption" in result.stderr.lower() + + def test_borg_ssh_is_only_configured_for_ssh_targets(): local_env = {"BORG_RSH": "custom-command"} configure_borg_ssh(local_env, {"storage_type": "local"}, "/mnt/backup/repo") diff --git a/tests/test_history_api.py b/tests/test_history_api.py index 77738345..f914e4c3 100644 --- a/tests/test_history_api.py +++ b/tests/test_history_api.py @@ -1,3 +1,4 @@ +from job_fixtures import job_id, write_job import json from pathlib import Path @@ -5,10 +6,12 @@ def _write_status(root: Path, timestamp: str, backup_type: str, location: str, status: str = "success") -> None: + write_job(root, backup_type + "_" + location) date, time = timestamp.split(" ") path = root / f"{date}_{time.replace(':', '-')}_{backup_type}_{location}.status" path.write_text(json.dumps({ "timestamp": timestamp, + "job_id": job_id(backup_type + "_" + location), "backup_type": backup_type, "location": location, "status": status, @@ -22,7 +25,7 @@ def test_location_counts_cover_filtered_history_before_pagination(tmp_path: Path _write_status(tmp_path, "2026-06-21 09:00:00", "appdata", "local") _write_status(tmp_path, "2026-06-21 08:00:00", "photos", "smb") - result = get_history_data({"STATUS_DIR": str(tmp_path)}, {"page": 1, "per_page": 1}) + result = get_history_data({"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(tmp_path)}, {"page": 1, "per_page": 1}) assert len(result["entries"]) == 1 assert result["total"] == 5 @@ -35,7 +38,7 @@ def test_location_filter_keeps_complete_sidebar_counts(tmp_path: Path) -> None: _write_status(tmp_path, "2026-06-21 11:00:00", "appdata", "usb") _write_status(tmp_path, "2026-06-21 10:00:00", "appdata", "local") - result = get_history_data({"STATUS_DIR": str(tmp_path)}, { + result = get_history_data({"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(tmp_path)}, { "type": "appdata", "location": "usb", "page": 1, diff --git a/tests/test_homepage_widget.py b/tests/test_homepage_widget.py index 89727366..a06f0802 100644 --- a/tests/test_homepage_widget.py +++ b/tests/test_homepage_widget.py @@ -1,4 +1,5 @@ from __future__ import annotations +from job_fixtures import identified_job, job_id, write_job import json import os @@ -92,12 +93,12 @@ def test_general_api_token_cannot_replace_widget_token(tmp_path: Path): def test_homepage_widget_summary_is_stable_and_redacted(monkeypatch): jobs = [ - {"key": "flash_local", "name": "Flash", "display_name": "Flash - Local", "enabled": True}, + {"key": job_id('flash_local'), "name": "Flash", "display_name": "Flash - Local", "enabled": True}, {"key": "appdata_usb", "name": "Appdata", "display_name": "Appdata - USB", "enabled": True}, {"key": "photos_local", "name": "Photos", "display_name": "Photos - Local", "enabled": False}, ] latest = [ - {"key": "flash_local", "status": "success", "timestamp": "2026-07-14 09:00:00"}, + {"key": job_id('flash_local'), "status": "success", "timestamp": "2026-07-14 09:00:00"}, {"key": "appdata_usb", "status": "warning", "timestamp": "2026-07-14 10:00:00"}, ] monkeypatch.setattr(homepage_widget_api, "_read_jobs", lambda _config: jobs) @@ -111,7 +112,7 @@ def test_homepage_widget_summary_is_stable_and_redacted(monkeypatch): monkeypatch.setattr( jobs_api, "get_all_runtime_states", - lambda _config: {"flash_local": {"running": True, "log_file": "/secret/job.log"}}, + lambda _config: {job_id('flash_local'): {"running": True, "log_file": "/secret/job.log"}}, ) result = homepage_widget_api.build_homepage_widget_summary( @@ -171,7 +172,7 @@ def test_unraid_dashboard_widget_cache_is_flash_safe_and_redacted(tmp_path: Path status = { "summary": {"success": 1, "warning": 1, "skipped": 0, "error": 0}, "backups": [ - { + {"job_id": job_id('appdata_local'), "key": "appdata_local", "backup_type": "appdata", "location": "local", @@ -260,14 +261,16 @@ def test_unraid_dashboard_widget_status_file_cache_marks_overdue_jobs(tmp_path: status_dir = tmp_path / "status" status_dir.mkdir() snapshot_file = tmp_path / "weekly-snapshots.json" + write_job(tmp_path, "flash_local") config = { + "BACKUP_SCRIPTS_DIR": str(tmp_path), "UNRAID_DASHBOARD_WIDGET_FILE": str(cache_file), "STATUS_DIR": str(status_dir), "SNAPSHOT_FILE": str(snapshot_file), "NOTIFY_BACKUP_OVERDUE_TOLERANCE_HOURS": "1", } (status_dir / "2026-08-09_12-00-00_flash_local.status").write_text( - json.dumps({ + json.dumps({"job_id": job_id('flash_local'), "backup_type": "flash", "location": "local", "timestamp": "2026-08-09 12:00:00", @@ -278,14 +281,14 @@ def test_unraid_dashboard_widget_status_file_cache_marks_overdue_jobs(tmp_path: encoding="utf-8", ) monkeypatch.setattr("schedule_api.get_schedules", lambda _config: { - "flash_local": {"enabled": True, "cron": "0 12 * * *"}, + job_id('flash_local'): {"enabled": True, "cron": "0 12 * * *"}, }) monkeypatch.setattr( unraid_dashboard_widget, "_read_jobs", lambda _config, _backups: [ { - "key": "flash_local", + "key": job_id('flash_local'), "display_name": "Flash - Lokal", "enabled": True, "running": False, @@ -297,7 +300,7 @@ def test_unraid_dashboard_widget_status_file_cache_marks_overdue_jobs(tmp_path: "jobs_api.list_jobs", lambda _config, _context: [ { - "key": "flash_local", + "key": job_id('flash_local'), "display_name": "Flash - Lokal", "enabled": True, } @@ -395,12 +398,14 @@ def test_unraid_dashboard_widget_status_file_cache_clears_finished_running_lock( cache_file = tmp_path / "widget-status.json" status_dir = tmp_path / "status" status_dir.mkdir() + write_job(tmp_path, "sonstiges_usb") config = { + "BACKUP_SCRIPTS_DIR": str(tmp_path), "UNRAID_DASHBOARD_WIDGET_FILE": str(cache_file), "STATUS_DIR": str(status_dir), } (status_dir / "2026-08-29_22-20-06_sonstiges_usb.status").write_text( - json.dumps({ + json.dumps({"job_id": job_id('sonstiges_usb'), "backup_type": "sonstiges", "location": "usb", "timestamp": "2026-08-29 22:20:06", @@ -415,7 +420,7 @@ def test_unraid_dashboard_widget_status_file_cache_clears_finished_running_lock( "_read_jobs", lambda _config, _backups: [ { - "key": "sonstiges_usb", + "key": job_id('sonstiges_usb'), "display_name": "Sonstiges - USB", "enabled": True, "running": True, @@ -449,12 +454,14 @@ def test_unraid_dashboard_widget_status_file_cache_keeps_newer_running_job(tmp_p cache_file = tmp_path / "widget-status.json" status_dir = tmp_path / "status" status_dir.mkdir() + write_job(tmp_path, "sonstiges_usb") config = { + "BACKUP_SCRIPTS_DIR": str(tmp_path), "UNRAID_DASHBOARD_WIDGET_FILE": str(cache_file), "STATUS_DIR": str(status_dir), } (status_dir / "2026-08-29_22-20-06_sonstiges_usb.status").write_text( - json.dumps({ + json.dumps({"job_id": job_id('sonstiges_usb'), "backup_type": "sonstiges", "location": "usb", "timestamp": "2026-08-29 22:20:06", @@ -469,7 +476,7 @@ def test_unraid_dashboard_widget_status_file_cache_keeps_newer_running_job(tmp_p "_read_jobs", lambda _config, _backups: [ { - "key": "sonstiges_usb", + "key": job_id('sonstiges_usb'), "display_name": "Sonstiges - USB", "enabled": True, "running": True, @@ -510,7 +517,7 @@ def test_unraid_dashboard_widget_startup_cache_is_written_without_backup_status( "discover_jobs", lambda _scripts_dir, _data_root: [ SimpleNamespace( - key="flash_local", + key=job_id('flash_local'), name="Flash", display_name="Flash - Lokal", enabled=True, @@ -565,7 +572,7 @@ def test_unraid_dashboard_widget_startup_cache_preserves_existing_fresh_cache(tm "warnings": 0, "failed": 0, "running": 0, - "items": [{"key": "flash_local", "last_status": "success"}], + "items": [{"key": job_id('flash_local'), "last_status": "success"}], }, } cache_file.write_text(json.dumps(existing), encoding="utf-8") @@ -589,7 +596,9 @@ def test_unraid_dashboard_widget_startup_cache_rebuilds_running_only_fresh_cache cache_file = tmp_path / "widget-status.json" status_dir = tmp_path / "status" status_dir.mkdir() + write_job(tmp_path, "flash_local") config = { + "BACKUP_SCRIPTS_DIR": str(tmp_path), "UNRAID_DASHBOARD_WIDGET_FILE": str(cache_file), "STATUS_DIR": str(status_dir), } @@ -607,7 +616,7 @@ def test_unraid_dashboard_widget_startup_cache_rebuilds_running_only_fresh_cache "running": 1, "items": [ { - "key": "flash_local", + "key": job_id('flash_local'), "enabled": True, "last_status": "", "last_timestamp": "", @@ -618,7 +627,7 @@ def test_unraid_dashboard_widget_startup_cache_rebuilds_running_only_fresh_cache encoding="utf-8", ) (status_dir / "2026-08-28_12-30-00_flash_local.status").write_text( - json.dumps({ + json.dumps({"job_id": job_id('flash_local'), "backup_type": "flash", "location": "local", "timestamp": "2026-08-28 12:30:00", @@ -633,7 +642,7 @@ def test_unraid_dashboard_widget_startup_cache_rebuilds_running_only_fresh_cache "_read_jobs", lambda _config, _backups: [ { - "key": "flash_local", + "key": job_id('flash_local'), "display_name": "Flash - Lokal", "enabled": True, "running": False, @@ -685,14 +694,14 @@ def test_unraid_dashboard_widget_startup_cache_rejects_empty_fresh_status_scan(t "warnings": 0, "failed": 0, "running": 0, - "items": [{"key": "flash_local", "enabled": True, "last_status": "", "last_timestamp": ""}], + "items": [{"key": job_id('flash_local'), "enabled": True, "last_status": "", "last_timestamp": ""}], }, "status": {"state": "ok"}, }), encoding="utf-8", ) job = { - "key": "flash_local", + "key": job_id('flash_local'), "display_name": "Flash - Lokal", "enabled": True, "running": False, @@ -706,7 +715,7 @@ def test_unraid_dashboard_widget_startup_cache_rejects_empty_fresh_status_scan(t "discover_jobs", lambda _scripts_dir, _data_root: [ SimpleNamespace( - key="flash_local", + key=job_id('flash_local'), name="Flash", display_name="Flash - Lokal", enabled=True, @@ -745,7 +754,7 @@ def test_unraid_dashboard_widget_startup_cache_import_is_one_time(tmp_path: Path "state": "skipped", "reason": "no_backup_status_rows", }, - "jobs": {"enabled": 1, "items": [{"key": "flash_local"}]}, + "jobs": {"enabled": 1, "items": [{"key": job_id('flash_local')}]}, } cache_file.write_text(json.dumps(existing), encoding="utf-8") monkeypatch.setattr( @@ -814,7 +823,7 @@ def test_unraid_dashboard_widget_startup_cache_rebuilds_old_fresh_cache_without_ "schema_version": 1, "cache_state": "fresh", "generated_at": "2026-08-10T12:00:00Z", - "jobs": {"items": [{"key": "flash_local"}]}, + "jobs": {"items": [{"key": job_id('flash_local')}]}, } monkeypatch.setattr( unraid_dashboard_widget, diff --git a/tests/test_i18n_resources.py b/tests/test_i18n_resources.py index b9c4465d..3a025bfe 100644 --- a/tests/test_i18n_resources.py +++ b/tests/test_i18n_resources.py @@ -222,7 +222,7 @@ def test_user_manuals_cover_current_stable_safety_and_runtime_guidance(): required_shared = ( "2026.08.31.0907", - "Unraid 7.2.0", + "Unraid 6.12.5", "Python 3 for Unraid", "backup.start.priority=1", "Borg Server", diff --git a/tests/test_inventory_consistency.py b/tests/test_inventory_consistency.py index 93d6a488..01f6c5fe 100644 --- a/tests/test_inventory_consistency.py +++ b/tests/test_inventory_consistency.py @@ -1,8 +1,11 @@ from __future__ import annotations +import json +from job_fixtures import identified_job, job_id import sys import threading import multiprocessing +import os import time from pathlib import Path @@ -93,6 +96,51 @@ def test_inventory_files_keep_restrictive_permissions(tmp_path: Path) -> None: assert storages_file(config).stat().st_mode & 0o777 == 0o600 +def test_inventory_lock_does_not_reapply_existing_private_permissions(tmp_path, monkeypatch): + with inventory_store.inventory_lock(tmp_path): + pass + path = tmp_path / ".inventory.lock" + before = path.stat() + + def unexpected_chmod(*_args): + pytest.fail("Existing private lock permissions must not be reapplied") + + monkeypatch.setattr(inventory_store.os, "fchmod", unexpected_chmod) + for _ in range(10): + with inventory_store.inventory_lock(tmp_path): + with inventory_store.inventory_lock(tmp_path): + assert path.stat().st_mode & 0o7777 == 0o600 + after = path.stat() + assert (after.st_ino, after.st_mtime_ns, after.st_ctime_ns) == (before.st_ino, before.st_mtime_ns, before.st_ctime_ns) + + +@pytest.mark.parametrize("mode", [0o644, 0o700, 0o2600]) +def test_inventory_lock_still_corrects_changed_permissions(tmp_path, mode): + path = tmp_path / ".inventory.lock" + path.touch() + path.chmod(mode) + with inventory_store.inventory_lock(tmp_path): + assert path.stat().st_mode & 0o7777 == 0o600 + + +def test_inventory_lock_closes_descriptor_when_permission_update_fails(tmp_path, monkeypatch): + path = tmp_path / ".inventory.lock" + path.touch() + path.chmod(0o644) + descriptors = [] + + def fail_chmod(fd, _mode): + descriptors.append(fd) + raise PermissionError("injected permission failure") + + monkeypatch.setattr(inventory_store.os, "fchmod", fail_chmod) + with pytest.raises(inventory_store.InventoryAccessError): + with inventory_store.inventory_lock(tmp_path): + pytest.fail("The transaction must not start without required permissions") + with pytest.raises(OSError): + os.fstat(descriptors[0]) + + def test_repository_inventory_cache_avoids_reparse_and_detects_external_change( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -187,7 +235,7 @@ def test_job_link_transaction_rolls_back_when_job_write_fails( config = _config(tmp_path) repository = _repository("repo_target") write_repository_store(config, {"repositories": [repository]}) - metadata_path = tmp_path / "config" / "jobs" / "appdata_local.json" + metadata_path = tmp_path / "config" / "jobs" / (job_id('appdata_local') + ".json") def fail_job_write(_path, _payload, **_kwargs): raise inventory_store.InventoryAccessError("injected job write failure") @@ -197,9 +245,9 @@ def fail_job_write(_path, _payload, **_kwargs): repositories_api.save_job_repository_transaction( config, metadata_path, - {"job_key": "appdata_local", "repository_key": "repo_target"}, + {"job_id": job_id('appdata_local'), "archive_prefix": "appdata-backup", "job_key": job_id('appdata_local'), "repository_key": "repo_target"}, "repo_target", - "appdata_local", + job_id('appdata_local'), ) assert not metadata_path.exists() restored = read_repository_store(config)["repositories"][0] @@ -323,9 +371,9 @@ def test_job_delete_transaction_rolls_back_when_metadata_delete_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: config = _config(tmp_path) - repository = {**_repository("repo_target"), "used_by": ["appdata_local"], "source_job_keys": ["appdata_local"]} + repository = {**_repository("repo_target"), "used_by": [job_id('appdata_local')], "source_job_keys": [job_id('appdata_local')]} write_repository_store(config, {"repositories": [repository]}) - metadata_path = tmp_path / "config" / "jobs" / "appdata_local.json" + metadata_path = tmp_path / "config" / "jobs" / (job_id('appdata_local') + ".json") metadata_path.parent.mkdir(parents=True) metadata_path.write_text('{"job_key":"appdata_local","repository_key":"repo_target"}\n', encoding="utf-8") original_unlink = Path.unlink @@ -337,11 +385,11 @@ def fail_metadata_unlink(path: Path, *args, **kwargs): monkeypatch.setattr(Path, "unlink", fail_metadata_unlink) with pytest.raises(OSError, match="injected"): - repositories_api.delete_job_metadata_transaction(config, [metadata_path], "appdata_local") + repositories_api.delete_job_metadata_transaction(config, [metadata_path], job_id('appdata_local')) assert metadata_path.exists() restored = read_repository_store(config)["repositories"][0] - assert restored["used_by"] == ["appdata_local"] - assert restored["source_job_keys"] == ["appdata_local"] + assert restored["used_by"] == [job_id('appdata_local')] + assert restored["source_job_keys"] == [job_id('appdata_local')] def test_repository_usage_is_rebuilt_from_authoritative_job_assignments(tmp_path: Path) -> None: @@ -359,23 +407,24 @@ def test_repository_usage_is_rebuilt_from_authoritative_job_assignments(tmp_path "used_by": ["stale_job"], "source_job_keys": ["stale_job"], }]}) - metadata_path = tmp_path / "config" / "jobs" / "appdata_local.json" + metadata_path = tmp_path / "config" / "jobs" / (job_id('appdata_local') + ".json") metadata_path.parent.mkdir(parents=True) metadata_path.write_text( - '{"schema_version":2,"job_key":"appdata_local","repository_key":"repo_target"}\n', + json.dumps({"schema_version": 4, "job_key": job_id("appdata_local"), + "job_id": job_id("appdata_local"), "repository_key": "repo_target"}), encoding="utf-8", ) before = repositories_api.repository_assignment_report(config) assert before["ok"] is False - assert before["usage_mismatches"][0]["expected_job_keys"] == ["appdata_local"] + assert before["usage_mismatches"][0]["expected_job_keys"] == [job_id('appdata_local')] after = repositories_api.reconcile_repository_usage(config) assert after["ok"] is True assert after["reconciled_repository_keys"] == ["repo_target"] repository = read_repository_store(config)["repositories"][0] - assert repository["used_by"] == ["appdata_local"] - assert repository["source_job_keys"] == ["appdata_local"] + assert repository["used_by"] == [job_id('appdata_local')] + assert repository["source_job_keys"] == [job_id('appdata_local')] def test_repository_assignment_report_guides_job_wizard_repair(tmp_path: Path) -> None: diff --git a/tests/test_issue_205_wizard_repository_feedback.py b/tests/test_issue_205_wizard_repository_feedback.py index 930e7c98..e53c3a03 100644 --- a/tests/test_issue_205_wizard_repository_feedback.py +++ b/tests/test_issue_205_wizard_repository_feedback.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id import json import subprocess import sys @@ -119,9 +120,9 @@ def test_edit_wizard_loads_existing_weekly_schedule(tmp_path: Path, monkeypatch: jobs_dir = data_root / "config" / "jobs" scripts_dir.mkdir(parents=True) jobs_dir.mkdir(parents=True) - (jobs_dir / "flash_local.json").write_text(json.dumps({ + (jobs_dir / (job_id('flash_local') + ".json")).write_text(json.dumps(identified_job({ "schema_version": 3, - "job_key": "flash_local", + "job_key": job_id('flash_local'), "backup_type": "flash", "location": "local", "name": "Flash", @@ -129,9 +130,9 @@ def test_edit_wizard_loads_existing_weekly_schedule(tmp_path: Path, monkeypatch: "runner": "scriptless-wizard-runner", "repository_key": "repo_flash_local_test", "source_paths": ["/boot"], - }), encoding="utf-8") + })), encoding="utf-8") (data_root / "config" / "schedules.json").write_text(json.dumps({ - "flash_local": {"cron": "10 6 * * 2", "enabled": True}, + job_id('flash_local'): {"cron": "10 6 * * 2", "enabled": True}, }), encoding="utf-8") config = {"BACKUP_SCRIPTS_DIR": str(data_root)} write_storage_store(config, {"storages": [{ @@ -152,7 +153,7 @@ def test_edit_wizard_loads_existing_weekly_schedule(tmp_path: Path, monkeypatch: }]}) monkeypatch.setattr("config_api.read_expanded_conf", lambda _config: {}) - loaded = load_job_for_wizard("flash_local", scripts_dir, config) + loaded = load_job_for_wizard(job_id('flash_local'), scripts_dir, config) assert loaded["schedule"] == {"cron": "10 6 * * 2", "enabled": True} @@ -178,18 +179,18 @@ def test_edit_wizard_preserves_schedule_inventory_values( jobs_dir = data_root / "config" / "jobs" scripts_dir.mkdir(parents=True) jobs_dir.mkdir(parents=True) - (jobs_dir / "flash_local.json").write_text(json.dumps({ + (jobs_dir / (job_id('flash_local') + ".json")).write_text(json.dumps(identified_job({ "schema_version": 3, - "job_key": "flash_local", + "job_key": job_id('flash_local'), "backup_type": "flash", "location": "local", "name": "Flash", "runner": "scriptless-wizard-runner", "repository_key": "repo_flash_local_test", "source_paths": ["/boot"], - }), encoding="utf-8") + })), encoding="utf-8") (data_root / "config" / "schedules.json").write_text(json.dumps({ - "flash_local": {"cron": cron, "enabled": enabled}, + job_id('flash_local'): {"cron": cron, "enabled": enabled}, }), encoding="utf-8") config = {"BACKUP_SCRIPTS_DIR": str(data_root)} write_storage_store(config, {"storages": [{ @@ -210,7 +211,7 @@ def test_edit_wizard_preserves_schedule_inventory_values( }]}) monkeypatch.setattr("config_api.read_expanded_conf", lambda _config: {}) - loaded = load_job_for_wizard("flash_local", scripts_dir, config) + loaded = load_job_for_wizard(job_id('flash_local'), scripts_dir, config) assert loaded["schedule"] == {"cron": cron, "enabled": enabled} diff --git a/tests/test_issue_458_retention_help.py b/tests/test_issue_458_retention_help.py index 051a6421..b9831da0 100644 --- a/tests/test_issue_458_retention_help.py +++ b/tests/test_issue_458_retention_help.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id import json import sys from pathlib import Path @@ -82,8 +83,8 @@ def test_manual_repository_prune_blocks_all_zero_policy(tmp_path: Path) -> None: config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs = tmp_path / "config" / "jobs" jobs.mkdir(parents=True) - (jobs / "appdata_local.json").write_text(json.dumps({ - "job_key": "appdata_local", + (jobs / (job_id('appdata_local') + ".json")).write_text(json.dumps({"job_id": job_id('appdata_local'), "archive_prefix": "appdata-backup", + "job_key": job_id('appdata_local'), "repository_key": "repo_appdata", "retention": {"daily": "0", "weekly": "0", "monthly": "0", "yearly": "0"}, }), encoding="utf-8") @@ -91,7 +92,7 @@ def test_manual_repository_prune_blocks_all_zero_policy(tmp_path: Path) -> None: with pytest.raises(ValueError, match="At least one retention value"): CheckManager()._repository_command( config, - {"repository_key": "repo_appdata", "used_by": ["appdata_local"]}, + {"repository_key": "repo_appdata", "used_by": [job_id('appdata_local')]}, "/mnt/backup/appdata", "prune", "quick", @@ -142,8 +143,8 @@ def test_retention_step_explains_periods_and_blocks_all_zero_in_both_languages() assert "time periods" in en["wizard"]["retentionExplanation"] assert "größer als 0" in de["wizard"]["validationRetentionRequired"] assert "greater than 0" in en["wizard"]["validationRetentionRequired"] - assert "max. 1/Tag" in de["storage"]["repositoryRetentionDaily"] - assert "max. 1/day" in en["storage"]["repositoryRetentionDaily"] + assert "1 pro Tag" in de["storage"]["repositoryRetentionDailyLimit"] + assert "1 per day" in en["storage"]["repositoryRetentionDailyLimit"] def test_quick_help_and_manuals_use_the_same_retention_semantics() -> None: diff --git a/tests/test_issue_463_file_activity_log.py b/tests/test_issue_463_file_activity_log.py index 2ba9cb07..49e641d9 100644 --- a/tests/test_issue_463_file_activity_log.py +++ b/tests/test_issue_463_file_activity_log.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id import io import json import logging @@ -120,7 +121,7 @@ def test_job_metadata_round_trip_and_runner_environment(tmp_path: Path, monkeypa scripts_dir = tmp_path / "scripts" source = tmp_path / "source" source.mkdir() - params = { + params = {"archive_prefix": 'files-backup', "type_id": "files", "job_name": "Files", "location": "local", @@ -134,10 +135,10 @@ def test_job_metadata_round_trip_and_runner_environment(tmp_path: Path, monkeypa assert metadata["file_activity"] is True monkeypatch.setattr("config_api.read_expanded_conf", lambda _config: {}) - loaded = load_job_for_wizard("files_local", scripts_dir, config) + loaded = load_job_for_wizard(result['job_id'], scripts_dir, config) assert loaded["file_activity"] is True - env, _ = wizard_runner._load_env_from_job("files_local", scripts_dir, tmp_path) + env, _ = wizard_runner._load_env_from_job(result['job_id'], scripts_dir, tmp_path) assert env["BORG_FILE_ACTIVITY"] == "1" @@ -146,21 +147,21 @@ def test_missing_job_field_is_disabled_and_preview_exposes_setting(tmp_path: Pat scripts_dir = tmp_path / "scripts" jobs_dir = tmp_path / "config" / "jobs" jobs_dir.mkdir(parents=True, exist_ok=True) - (jobs_dir / "files_local.json").write_text(json.dumps({ + (jobs_dir / (job_id('files_local') + ".json")).write_text(json.dumps(identified_job({ "schema_version": 3, - "job_key": "files_local", + "job_key": job_id('files_local'), "name": "Files", "backup_type": "files", "location": "local", "repository_key": "repo_files_test", "source_paths": [str(tmp_path / "source")], "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, - }) + "\n", encoding="utf-8") + })) + "\n", encoding="utf-8") monkeypatch.setattr("config_api.read_expanded_conf", lambda _config: {}) - loaded = load_job_for_wizard("files_local", scripts_dir, config) + loaded = load_job_for_wizard(job_id('files_local'), scripts_dir, config) monkeypatch.setenv("BORG_FILE_ACTIVITY", "1") - env, _ = wizard_runner._load_env_from_job("files_local", scripts_dir, tmp_path) + env, _ = wizard_runner._load_env_from_job(job_id('files_local'), scripts_dir, tmp_path) preview = generate_flow_preview({ "type_id": "files", "location": "local", @@ -196,7 +197,7 @@ def test_wizard_and_manuals_explain_file_activity_and_privacy() -> None: assert "white-space: nowrap" in styles assert "#wizard-modal .modal-wizard" in styles assert "max-height: calc(100vh - 32px)" in styles - assert "flex: 1 1 448px" in styles + assert "flex: 1 1 480px" in styles assert "file_activity: !!document.getElementById('wiz-file-activity').checked" in script assert "wizard.previewFileActivity" in script assert "Support-Paketen" in de["wizard"]["fileActivityPrivacy"] @@ -212,15 +213,15 @@ def test_managed_run_preserves_start_time_option_and_capture_path(tmp_path, monk _local_repository_config(tmp_path) jobs = tmp_path / 'config' / 'jobs' jobs.mkdir() - meta = { - 'schema_version': 3, 'job_key': 'files_local', 'backup_type': 'files', 'location': 'local', + meta = identified_job({ + 'schema_version': 3, 'job_key': job_id('files_local'), 'backup_type': 'files', 'location': 'local', 'repository_key': 'repo_files_test', 'source_paths': [str(tmp_path / 'source')], 'file_activity': not started_enabled, - } - (jobs / 'files_local.json').write_text(json.dumps(meta)) + }) + (jobs / (job_id('files_local') + ".json")).write_text(json.dumps(meta)) capture = tmp_path / 'logs' / 'Borg-Backup_files_local--activity-test.log' monkeypatch.setenv('BORG_UI_FILE_ACTIVITY_RUN', '1' if started_enabled else '0') monkeypatch.setenv('BORG_UI_CAPTURE_LOG', str(capture)) - env, _ = wizard_runner._load_env_from_job('files_local', tmp_path / 'scripts', tmp_path) + env, _ = wizard_runner._load_env_from_job(job_id('files_local'), tmp_path / 'scripts', tmp_path) assert env['BORG_FILE_ACTIVITY'] == ('1' if started_enabled else '0') assert (env['LOG_FILE'] == str(capture)) == started_enabled diff --git a/tests/test_job_file_names.py b/tests/test_job_file_names.py new file mode 100644 index 00000000..c3090b07 --- /dev/null +++ b/tests/test_job_file_names.py @@ -0,0 +1,271 @@ +"""Readable run files retain permanent ownership and safe filename sizes (#486).""" + +import json +import logging +import os +import sys +import time +from types import SimpleNamespace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +for folder in (ROOT, ROOT / "api", ROOT / "runtime", ROOT / "runtime/lib"): + sys.path.insert(0, str(folder)) + +from job_fixtures import job_id +from job_identity import job_file_component, job_log_filename, job_log_paths, job_run_date_tag +from lib.backup_job import BackupJob, BackupJobConfig +from lib.status import BackupStatus +from jobs_api import _fallback_runtime_log + + +def make_config(tmp_path, key, name="TestJobNeuerName"): + return BackupJobConfig( + job_id=key, job_name=name, backup_type="old_type", backup_location="storagebox", + lock_file=tmp_path / "job.lock", log_dir=tmp_path, log_file=tmp_path / "run.log", + backup_paths=[], borg_cache_dir=tmp_path / "cache", date_tag="2026-09-07_15-00-01", + status_dir=tmp_path / "status", + ) + + +@pytest.mark.parametrize("name", ["TestJobNeuerName", "ä" * 100, "資料" * 50, + "../My Job/USB:*?\\\nName", "🌍" * 100]) +def test_run_files_fit_utf8_filename_limit_and_preserve_full_identity(tmp_path, name): + key = job_id("readable") + labels = ["2026-09-07_15-00-01", "2026-09-07_15-00-01_USB_NOT_MOUNTED", + "activity-" + "a" * 96] + for label in labels: + filename = job_log_filename(name, "storagebox", key, label) + assert filename.startswith("BBUI-") + assert filename.endswith(f"_storagebox_{key}--{label}.log") + assert len(filename.encode("utf-8")) <= 255 + assert Path(filename).name == filename + assert not any(char in filename for char in '\\/:*?\n') + (tmp_path / filename).write_text("complete log") + status = BackupStatus(job_id=key, job_name=name, backup_type="old_type", location="storagebox") + path = status.save(tmp_path) + assert len(path.name.encode("utf-8")) <= 255 + assert path.name.endswith(f"_{job_file_component(name, 'storagebox', key)}.status") + payload = json.loads(path.read_text()) + assert payload["job_name"] == name + assert payload["job_id"] == key + assert BackupStatus.from_file(path).key == key + + +def test_readable_names_and_log_header_match_the_job_at_run_time(tmp_path, caplog, monkeypatch): + key = job_id("readable") + cfg = make_config(tmp_path, key) + assert job_log_filename(cfg.job_name, cfg.backup_location, key, cfg.date_tag) == ( + f"BBUI-TestJobNeuerName_storagebox_{key}--2026-09-07_15-00-01.log" + ) + job = BackupJob(cfg) + with caplog.at_level(logging.INFO): + job._log_startup_banner() + assert "TestJobNeuerName" in caplog.text and key in caplog.text + job._write_mini_log("USB_NOT_MOUNTED", ["Skipped"]) + mini = next(tmp_path.glob("*_USB_NOT_MOUNTED.log")) + assert f"TestJobNeuerName_storagebox_{key}--" in mini.name + assert f"Job ID: {key}" in mini.read_text() + monkeypatch.setattr(job, "_emit_lifecycle_finished", lambda **_: None) + job._skip_reason = "USB is not mounted" + job._save_skip_status() + skipped = json.loads(next(cfg.status_dir.glob("*.status")).read_text()) + assert skipped["job_name"] == cfg.job_name and skipped["job_id"] == key + + +def test_log_lookup_and_retention_follow_id_across_renames_without_touching_other_jobs(tmp_path): + selected, other = job_id("selected"), job_id("other") + old = tmp_path / job_log_filename("Previous name", "local", selected, "2026-09-01_10-00-00") + legacy = tmp_path / f"Borg-Backup_{selected}--2026-09-01_10-00-00.log" + fresh = tmp_path / job_log_filename("Renamed", "usb", selected, "2026-09-07_10-00-00") + foreign = tmp_path / job_log_filename("Previous name", "local", other, "2026-09-01_10-00-00") + # A job name that contains another ID must not trick ownership matching. + confusing = tmp_path / job_log_filename(f"Fake_local_{selected}--suffix", "local", other, "2026-09-01_10-00-00") + for path in (old, legacy, fresh, foreign, confusing): + path.write_text("preserve ownership") + os.utime(path, (1, 1)) + os.utime(fresh, None) + assert set(job_log_paths(tmp_path, selected)) == {old, legacy, fresh} + assert _fallback_runtime_log({"GLOBAL_LOG_DIR": str(tmp_path)}, selected, "") == str(fresh) + BackupJob(make_config(tmp_path, selected, "Another rename")).cleanup_old_logs() + assert not old.exists() and not legacy.exists() + assert fresh.exists() and foreign.exists() and confusing.exists() + + +@pytest.mark.parametrize("archived", [False, True]) +def test_saved_activity_log_is_found_after_capture_state_and_job_name_change(tmp_path, monkeypatch, archived): + import activity_log + import activity_log_capture + import jobs_api + monkeypatch.setattr(activity_log_capture, "CAPTURE_ROOT", tmp_path / "ram") + monkeypatch.setattr(jobs_api.JobManager, "get", classmethod(lambda cls: jobs_api.JobManager())) + monkeypatch.setattr(jobs_api, "durable_running_states", lambda _: {}) + monkeypatch.setattr("job_control.read_control_state", lambda _: {}) + key, run = job_id("activity"), "20260907T130001Z-123456abcdef" + logs = tmp_path / "logs" + active, record_path = activity_log_capture.prepare_capture( + key, run, logs, job_name="Name at start", location="usb", + ) + active.write_text("A changed-Größe.stl\n") + assert activity_log_capture.retain_capture(record_path, 0) + record = activity_log_capture.read_record(record_path) + retained = Path(record["retained_file"]) + assert retained.name == f"BBUI-Name_at_start_usb_{key}--{job_run_date_tag(run)}.log" + cfg = make_config(tmp_path, key, "Name at start") + cfg.retained_log_file = retained + cfg.backup_location = "usb" + monkeypatch.setenv("BORG_UI_RUN_ID", run) + job = BackupJob(cfg) + job.set_result(0) + status_path = job._save_status(10) + saved = BackupStatus.from_file(status_path) + assert saved.run_id == run and saved.file_activity is True + if archived: + archive_dir = cfg.status_dir / "archive" + archive_dir.mkdir() + status_path.rename(archive_dir / status_path.name) + record_path.unlink() # Reboot loses the RAM capture record. + config = {"GLOBAL_LOG_DIR": str(logs), "STATUS_DIR": str(cfg.status_dir)} + resolved, _ = activity_log.resolve_activity_run(config, key, run) + assert resolved == retained + result = activity_log.get_activity_window(config, {"job": [key], "run": [run]}) + assert result["text"] == "A changed-Größe.stl\n" + assert result["exit_code"] == 0 + # A different run of the same job must not read this run's file. + with pytest.raises(FileNotFoundError): + activity_log.get_activity_window(config, {"job": [key], "run": [run + "0"]}) + + +def test_file_list_option_uses_the_same_retained_filename_as_normal_runner(tmp_path, monkeypatch): + import activity_log_capture + import wizard_runner + from test_job_identity_integration import migrated + + config, jobs, ids, root = migrated(tmp_path) + key = ids[jobs[0]["job_key"]] + monkeypatch.setattr(activity_log_capture, "CAPTURE_ROOT", tmp_path / "ram") + monkeypatch.setenv("BORG_UI_RUN_ID", "20260907T130001Z-123456abcdef") + monkeypatch.setenv("BORG_UI_JOB_NAME", "Name at start") + monkeypatch.setenv("BORG_UI_JOB_LOCATION", "local") + monkeypatch.setenv("BORG_UI_FILE_ACTIVITY_RUN", "0") + for name in ("LOG_FILE", "BORG_UI_CAPTURE_LOG", "BORG_UI_RETAINED_LOG"): + monkeypatch.delenv(name, raising=False) + normal, _ = wizard_runner._load_env_from_job(key, root / "scripts", root) + expected = Path(normal["LOG_FILE"]) + active, record = activity_log_capture.prepare_capture( + key, os.environ["BORG_UI_RUN_ID"], expected.parent, + job_name="Name at start", location="local", + ) + retained = activity_log_capture.read_record(record)["retained_file"] + monkeypatch.setenv("BORG_UI_FILE_ACTIVITY_RUN", "1") + monkeypatch.setenv("BORG_UI_CAPTURE_LOG", str(active)) + monkeypatch.setenv("BORG_UI_RETAINED_LOG", retained) + enabled, _ = wizard_runner._load_env_from_job(key, root / "scripts", root) + assert Path(enabled["BORG_UI_RETAINED_LOG"]) == expected + assert enabled["DATE_TAG"] == normal["DATE_TAG"] + assert enabled["LOG_FILE"] == str(active) + assert "--activity-" not in expected.name + + +def test_failed_startup_log_reopens_without_status_or_ram_after_restart(tmp_path, monkeypatch): + import activity_log + import activity_log_capture + import jobs_api + + key = job_id("startup-failure") + logs = tmp_path / "logs" + logs.mkdir() + monkeypatch.setattr(activity_log_capture, "CAPTURE_ROOT", tmp_path / "ram") + manager = jobs_api.JobManager() + monkeypatch.setattr(jobs_api.JobManager, "get", classmethod(lambda cls: manager)) + monkeypatch.setattr(jobs_api, "durable_running_states", lambda _: {}) + monkeypatch.setattr("job_control.read_control_state", lambda _: {}) + assert manager.start(key, [sys.executable, "-c", "print('ERROR simulated startup failure'); raise SystemExit(2)"], logs, { + "BORG_UI_FILE_ACTIVITY_RUN": "1", "BORG_UI_ACTIVITY_LOG_DIR": str(logs), + "BORG_UI_JOB_NAME": "Photos", "BORG_UI_JOB_LOCATION": "local", + }) == (True, None) + state = manager._states[key] + assert state.proc.wait(timeout=10) == 2 + deadline = time.monotonic() + 5 + while not state.finished and time.monotonic() < deadline: + time.sleep(0.01) + assert state.finished + retained = Path(activity_log_capture.read_record(state.capture_record_file)["retained_file"]) + assert retained.name == f"BBUI-Photos_local_{key}--{job_run_date_tag(state.run_id)}.log" + assert f"job_id={key} run_id={state.run_id}" in retained.read_text().splitlines()[0] + state.capture_record_file.unlink() + manager._states.clear() + config = {"GLOBAL_LOG_DIR": str(logs), "STATUS_DIR": str(tmp_path / "missing-status")} + params = {"job": [key], "run": [state.run_id]} + result = activity_log.get_activity_window(config, params) + assert "ERROR simulated startup failure" in result["text"] + assert result["running"] is False + # Subsequent windows only open the requested log, not its identification + # header or unrelated history again. + opened = [] + original_open = activity_log.open_activity_file + def track_open(path): + opened.append(path) + return original_open(path) + monkeypatch.setattr(activity_log, "open_activity_file", track_open) + activity_log.get_activity_window(config, params) + assert opened == [retained] + + +@pytest.mark.parametrize("file_activity", [False, True]) +@pytest.mark.parametrize("reason,suffix", [ + ("parity_active", "SKIPPED_PARITY"), + ("usb_not_mounted", "USB_NOT_MOUNTED"), + ("usb_not_writable", "USB_NOT_WRITABLE"), +]) +def test_skipped_runs_keep_readable_log_names_and_status_links(tmp_path, monkeypatch, file_activity, reason, suffix): + import activity_log_capture + + key, run = job_id("skip"), "20260907T130001Z-123456abcdef" + monkeypatch.setenv("BORG_UI_RUN_ID", run) + monkeypatch.setattr(activity_log_capture, "CAPTURE_ROOT", tmp_path / "ram") + cfg = make_config(tmp_path, key, "Photos") + cfg.backup_location = "usb" + cfg.date_tag = job_run_date_tag(run) + retained = cfg.log_dir / job_log_filename(cfg.job_name, cfg.backup_location, key, cfg.date_tag) + record = None + if file_activity: + cfg.log_file, record = activity_log_capture.prepare_capture( + key, run, cfg.log_dir, job_name=cfg.job_name, location=cfg.backup_location, + ) + cfg.retained_log_file = retained + else: + cfg.log_file = retained + cfg.log_file.write_text("Backup run started\n") + job = BackupJob(cfg) + monkeypatch.setattr(job, "_emit_lifecycle_finished", lambda **_: None) + monkeypatch.setattr(job, "_send_notification_event", lambda *args, **kwargs: None) + if reason == "parity_active": + monkeypatch.setattr("lib.backup_job.shutil.which", lambda _: "/fake/mdcmd") + monkeypatch.setattr("lib.backup_job.subprocess.run", lambda *args, **kwargs: SimpleNamespace( + stdout="mdResyncAction=check\nmdResyncPos=50\nmdResyncSize=100\n", + )) + check = job.check_parity + else: + mount = tmp_path / "usb-mount" + if reason == "usb_not_writable": + mount.mkdir() + monkeypatch.setattr(Path, "is_mount", lambda path: path == mount) + monkeypatch.setattr("lib.backup_job.os.access", lambda *args: False) + check = lambda: job.check_usb_mount(mount) + with pytest.raises(SystemExit) as stopped: + check() + assert stopped.value.code == 0 + mini = cfg.log_dir / job_log_filename(cfg.job_name, cfg.backup_location, key, f"{cfg.date_tag}_{suffix}") + assert mini.is_file() and key in mini.read_text() + status_path = next(cfg.status_dir.glob("*.status")) + data = json.loads(status_path.read_text()) + assert data["status"] == "skipped" and data["skip_reason_code"] == reason + assert data["log_file"] == str(retained) + assert data["run_id"] == run and data["file_activity"] is file_activity + assert status_path.name.endswith(f"_Photos_usb_{key}.status") + if record: + assert activity_log_capture.retain_capture(record, 0) + assert Path(data["log_file"]).is_file() diff --git a/tests/test_job_id_migration.py b/tests/test_job_id_migration.py new file mode 100644 index 00000000..727f3e3b --- /dev/null +++ b/tests/test_job_id_migration.py @@ -0,0 +1,268 @@ +"""Main-format data preservation and interrupted job-ID conversion (#486).""" + +import copy +import json +import os +from pathlib import Path +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +for folder in (ROOT / "api", ROOT / "runtime", ROOT / "runtime/lib"): + sys.path.insert(0, str(folder)) + +from migrations import job_ids_v1 as migration +from job_identity import validate_job_id + + +def write(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data)) + + +def main_fixture(tmp_path): + root = tmp_path / "plugin" + status = tmp_path / "array/status" + restore = tmp_path / "array/restore-status" + status.mkdir(parents=True) + restore.mkdir(parents=True) + config = {"BACKUP_SCRIPTS_DIR": str(root), "BORG_SCRIPTS_DIR": str(ROOT / "runtime/scripts"), + "STATUS_DIR": str(status), "RESTORE_TEST_STATUS_DIR": str(restore)} + cfg = root / "config" + jobs = [] + for key, name in [("custom_type_local", "Zulu"), ("other_local", "Alpha")]: + job = {"schema_version": 3, "job_key": key, "backup_type": key.removesuffix("_local"), + "location": "local", "name": name, "repository_key": "shared", + "archive_prefixes": [key.removesuffix("_local") + "-backup", name + "-prior-backup"], + "source_paths": [str(tmp_path / "source")], "icon": "", "icon_color": "", + "created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-02-01T00:00:00Z", + "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, + "restore_test_policy": {"mode": "scheduled", "validity_days": 30, "level": 3}, + "extension": {"must_survive": [1, "example"]}} + jobs.append(job) + write(cfg / "jobs" / (key + ".json"), job) + write(status / f"2026-09-01_12-00-00_{key}.status", { + "backup_type": job["backup_type"], "location": "local", "timestamp": "2026-09-01 12:00:00", + "status": "success", "repository_check_status": "ok", "repository_check_date": "2026-08-30", + "archive_name": job["archive_prefixes"][0] + "-2026-09-01_12-00-00", + "original_size": 1048576, "log_file": "/example/logs/old.log", "unknown": {"x": 1}, + }) + write(restore / (key + ".test"), {"type": job["backup_type"], "location": "local", + "test_date": "2026-08-30 10:00:00", "test_result": "success", "steps": [{"evidence": 1}], + "report_id": "keep-independent-report-id"}) + cfg.joinpath("backup.conf").write_text(f'STATUS_DIR="{status}"\nRESTORE_TEST_STATUS_DIR="{restore}"\n') + with cfg.joinpath("backup.conf").open("a") as handle: + handle.write(f'GLOBAL_BORG_CACHE_BASE="{tmp_path / "cache"}"\n') + keys = [j["job_key"] for j in jobs] + write(cfg / "repositories.json", {"schema_version": 1, "repositories": [ + {"repository_key": "shared", "used_by": keys, "source_job_keys": keys, "keep": {"stats": 3}}, + ]}) + write(cfg / "schedules.json", {keys[0]: {"cron": "0 9 * * *", "enabled": True}, + keys[1]: {"cron": "0 10 * * *", "enabled": False}, "restore_test": {"cron": "0 8 * * *"}}) + for path, week in [(status.parent / "weekly-snapshots.json", "2026-W36"), + (status / "weekly-snapshots.json", "2026-W30")]: + write(path, {keys[0]: [{"week": week, "size": 1234}], "deleted_usb": [{"week": week, "size": 50}]}) + run = {"restore_id": "keep-run-id", "job_key": keys[0], "state": "completed", "evidence": {"x": 1}} + write(cfg / "restore-runs.json", {"runs": {"keep-run-id": run}}) + write(cfg / "restore-history/index.json", {"runs": [run]}) + write(cfg / "restore-history/runs/keep-run-id.json", run) + write(cfg / "notification-queue.json", {"queue": [{"id": "event-id", "job_key": keys[0], "attempts": 2}]}) + write(cfg / "notification-deliveries.json", {"deliveries": [ + {"job_key": keys[1], "id": "delivered"}, {"job_key": "deleted_usb", "id": "historical"}]}) + write(cfg / "notification-state.json", {"last_sent": {f"backup_overdue:{keys[0]}:2026-09-01 10:00:00": 1234}}) + write(cfg / "runtime-recovery.json", {"entries": [ + {"id": "recovery-id", "backup_type": jobs[0]["backup_type"], "backup_location": "local", "state": "recovered"}]}) + return config, jobs + + +def json_files(root): + return {str(p): p.read_bytes() for p in root.rglob("*") + if p.is_file() and p.suffix in (".json", ".status", ".test")} + + +def test_preserves_main_payloads_and_references_and_repeats_without_changes(tmp_path): + config, jobs = main_fixture(tmp_path) + before = json_files(tmp_path) + mtimes = {path: Path(path).stat().st_mtime_ns for path in before} + assert migration.detect(config)["required"] is True + result = migration.apply(config) + assert result["status"] == "applied" + plan = json.loads(migration._journal(config).read_text()) + assignment = plan["assignment"] + assert len(set(assignment.values())) == 2 + for job in jobs: + job_id = validate_job_id(assignment[job["job_key"]]) + source = Path(config["BACKUP_SCRIPTS_DIR"]) / "config/jobs" / (job["job_key"] + ".json") + target = source.with_name(job_id + ".json") + actual = json.loads(target.read_text()) + expected = {**job, "job_id": job_id, "job_key": job_id, "schema_version": 4, + "cache_subdir": "local_" + job["backup_type"], + "check_flag_name": ".last_check_" + job["backup_type"], + "archive_prefix": job["backup_type"] + "-backup"} + assert actual == expected + assert not source.exists() + for op in plan["operations"]: + assert Path(op["before"]).read_bytes() == before[op["source"]] + assert Path(op["target"]).stat().st_mtime_ns == mtimes[op["source"]] + old, new = json.loads(before[op["source"]]), json.loads(Path(op["target"]).read_bytes()) + if op["source"].endswith((".status", ".test")): + assert {k: v for k, v in new.items() if k != "job_id"} == old + if op["source"].endswith("weekly-snapshots.json"): + assert new == {assignment.get(k, k): v for k, v in old.items()} + cfg = Path(config["BACKUP_SCRIPTS_DIR"]) / "config" + assert json.loads((cfg / "notification-deliveries.json").read_text())["deliveries"][1]["job_key"] == "deleted_usb" + assert json.loads((cfg / "schedules.json").read_text())["restore_test"] == {"cron": "0 8 * * *"} + assert json.loads((cfg / "restore-runs.json").read_text())["runs"]["keep-run-id"]["job_key"] == assignment[jobs[0]["job_key"]] + after = json_files(tmp_path) + assert migration.detect(config)["required"] is False + assert migration.apply(config)["status"] == "not_required" + assert json_files(tmp_path) == after + + +def test_interrupted_write_reuses_saved_ids_and_originals(monkeypatch, tmp_path, capsys): + config, _ = main_fixture(tmp_path) + original = migration._apply_operation + count = 0 + + def interrupt(op): + nonlocal count + original(op) + count += 1 + if count == 2: + raise OSError("simulated interruption after two converted files") + + monkeypatch.setattr(migration, "_apply_operation", interrupt) + with pytest.raises(OSError): + migration.apply(config) + failure_log = capsys.readouterr().out + assert 'Failed during Updating job references' in failure_log + assert 'Completed successfully' not in failure_log + plan = json.loads(migration._journal(config).read_text()) + assert migration.detect(config)["required"] is True + monkeypatch.setattr(migration, "_apply_operation", original) + assert migration.apply(config)["status"] == "applied" + retry_log = capsys.readouterr().out + assert 'Resuming saved migration' in retry_log + assert 'Saving recovery copies' not in retry_log + assert 'Completed successfully' in retry_log + assert json.loads(migration._journal(config).read_text())["assignment"] == plan["assignment"] + + +def test_progress_starts_before_snapshot_and_reports_bounded_file_counts(tmp_path, monkeypatch, capsys): + config, _ = main_fixture(tmp_path) + tick = [0.0] + monkeypatch.setattr(migration, 'monotonic', lambda: tick[0]) + original = migration.atomic_write_bytes + initial_output = [] + + def delayed_write(path, content, **kwargs): + if not initial_output: + initial_output.append(capsys.readouterr().out) + assert 'Starting; web server waits for completion' in initial_output[0] + assert 'Saving recovery copies 0/' in initial_output[0] + original(path, content, **kwargs) + tick[0] += 0.5 + + monkeypatch.setattr(migration, 'atomic_write_bytes', delayed_write) + result = migration.apply(config) + output = initial_output[0] + capsys.readouterr().out + total = len(result['details']['affected_files']) + for phase in ('Saving recovery copies', 'Updating job references', 'Verifying migrated files'): + assert f'{phase} 0/{total} files' in output + assert f'{phase} {total}/{total} files' in output + saving = [line for line in output.splitlines() if 'Saving recovery copies' in line] + assert 2 < len(saving) < total # Progress during slow I/O, without one line per file. + assert f'Saving recovery copies 5/{total} files; elapsed=5.0s' in output + assert output.index('Verifying migrated files') < output.index('Completed successfully') + assert 'elapsed=' in output.splitlines()[-1] + + +def test_refuses_changed_source_on_retry(monkeypatch, tmp_path): + config, _ = main_fixture(tmp_path) + original = migration._apply_operation + monkeypatch.setattr(migration, "_apply_operation", lambda op: (_ for _ in ()).throw(OSError("interrupt"))) + with pytest.raises(OSError): + migration.apply(config) + plan = json.loads(migration._journal(config).read_text()) + source = Path(plan["operations"][0]["source"]) + changed = json.loads(source.read_text()) + changed["name"] = "Edited outside migration" + write(source, changed) + monkeypatch.setattr(migration, "_apply_operation", original) + with pytest.raises(ValueError, match="changed data"): + migration.apply(config) + assert json.loads(source.read_text()) == changed + + +def test_active_reference_conflict_changes_no_job_files(tmp_path): + config, _ = main_fixture(tmp_path) + schedules = Path(config["BACKUP_SCRIPTS_DIR"]) / "config/schedules.json" + write(schedules, {"missing_job": {"cron": "0 9 * * *"}}) + before = json_files(tmp_path) + with pytest.raises(ValueError, match="Unresolved active"): + migration.apply(config) + assert json_files(tmp_path) == before + + +def test_storage_unavailable_and_live_worker_block_before_writes(monkeypatch, tmp_path): + import jobs_api + import status + config, _ = main_fixture(tmp_path) + before = json_files(tmp_path) + monkeypatch.setattr(status, "status_storage_unavailable_reason", lambda path: "array not mounted") + with pytest.raises(RuntimeError, match="storage unavailable"): + migration.apply(config) + assert json_files(tmp_path) == before + monkeypatch.setattr(status, "status_storage_unavailable_reason", lambda path: "") + monkeypatch.setattr(jobs_api, "active_resource_locks", lambda config: [{"pid": os.getpid()}]) + with pytest.raises(RuntimeError, match="workers to finish"): + migration.apply(config) + assert json_files(tmp_path) == before + + +def test_conflicting_history_is_retained_without_claiming_an_owner(tmp_path): + config, jobs = main_fixture(tmp_path) + path = Path(config['STATUS_DIR']) / '2026-09-02_12-00-00_conflict.status' + payload = {'job_key': jobs[0]['job_key'], 'backup_type': jobs[1]['backup_type'], + 'location': 'local', 'timestamp': '2026-09-02 12:00:00', 'status': 'success'} + write(path, payload) + restore = Path(config['RESTORE_TEST_STATUS_DIR']) / (jobs[0]['job_key'] + '.test') + proof = json.loads(restore.read_text()) + proof['type'] = jobs[1]['backup_type'] + write(restore, proof) + result = migration.apply(config) + assert json.loads(path.read_text()) == payload + assert json.loads(restore.read_text()) == proof + assert len([r for r in result['details']['unresolved_history'] + if r['code'] == 'conflicting_historical_identity']) == 2 + + +def test_retry_finishes_rename_interrupted_between_new_file_and_old_file_removal(tmp_path, monkeypatch): + config, _ = main_fixture(tmp_path) + original = migration._apply_operation + def interrupted(op): + migration.atomic_write_bytes(Path(op['target']), Path(op['after']).read_bytes()) + raise OSError('interrupted before source unlink') + monkeypatch.setattr(migration, '_apply_operation', interrupted) + with pytest.raises(OSError): + migration.apply(config) + plan = json.loads(migration._journal(config).read_text()) + assert Path(plan['operations'][0]['source']).exists() + assert Path(plan['operations'][0]['target']).exists() + assert migration.detect(config)['required'] is True + monkeypatch.setattr(migration, '_apply_operation', original) + assert migration.apply(config)['status'] == 'applied' + assert not Path(plan['operations'][0]['source']).exists() + + +def test_migration_rejects_repository_reference_conflict_before_changes(tmp_path): + config, _ = main_fixture(tmp_path) + path = Path(config['BACKUP_SCRIPTS_DIR']) / 'config/repositories.json' + data = json.loads(path.read_text()) + data['repositories'].append({'repository_key': 'another', 'used_by': ['custom_type_local']}) + write(path, data) + before = json_files(tmp_path) + with pytest.raises(ValueError, match='Conflicting active repository'): + migration.apply(config) + assert json_files(tmp_path) == before diff --git a/tests/test_job_identity_integration.py b/tests/test_job_identity_integration.py new file mode 100644 index 00000000..891afb46 --- /dev/null +++ b/tests/test_job_identity_integration.py @@ -0,0 +1,426 @@ +"""Identity survives the normal Main workflows without changing their results (#486).""" +import copy +import json +from pathlib import Path +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +for folder in (ROOT / 'api', ROOT / 'runtime', ROOT / 'runtime/lib'): + sys.path.insert(0, str(folder)) + +from test_job_id_migration import main_fixture, write +from job_fixtures import job_id +from migrations import job_ids_v1 as migration +from archive_prefix import validate_prefix_ownership, validate_archive_prefix +from wizard_api import load_job_for_wizard, save_job +from jobs_api import discover_jobs, list_jobs +from repositories_api import write_repository_store, read_repository_store +from storage_objects_api import write_storage_store +from schedule_api import get_schedules +from status_api import get_status_data +from history_api import get_history_data +from reports_api import get_report_jobs, get_report_data +from restore_tests_api import list_restore_tests, list_restore_test_plan +from settings_transfer_api import export_jobs_bundle, import_jobs_bundle +import wizard_runner + + +def migrated(tmp_path): + config, jobs = main_fixture(tmp_path) + root = Path(config['BACKUP_SCRIPTS_DIR']) + (tmp_path / 'source').mkdir() + write_storage_store(config, {'storages': [{ + 'storage_key': 'local-test', 'display_name': 'Local', 'storage_type': 'local', + 'location': 'local', 'identity': 'local:' + str(tmp_path / 'repos'), + 'base_path': str(tmp_path / 'repos'), + }]}) + write_repository_store(config, {'repositories': [ + {'repository_key': key, 'display_name': key, 'storage_key': 'local-test', + 'relative_path': key, 'encryption': 'none', + 'used_by': [j['job_key'] for j in jobs] if key == 'shared' else []} + for key in ('shared', 'separate') + ]}) + assert migration.apply(config)['status'] == 'applied' + from migrations import job_settings_v1 + assert job_settings_v1.apply(config)['status'] == 'applied' + plan = json.loads(migration._journal(config).read_text()) + return config, jobs, plan['assignment'], root + + +def test_new_wizard_id_is_stateless_and_uses_the_admin_api_route(tmp_path): + from borg_backup_ui import BackupUIHandler + from job_identity import validate_job_id + handler = BackupUIHandler.__new__(BackupUIHandler) + handler.config = {'BACKUP_SCRIPTS_DIR': str(tmp_path)} + handler.path = '/api/wizard/new-job-id' + handler.command = 'GET' + replies = [] + handler._handle_api = lambda fn: replies.append(fn()) + handler.do_GET() + handler.do_GET() + first, second = [validate_job_id(reply['job_id']) for reply in replies] + assert first != second + assert handler._required_role_for_request(handler.path, 'GET') == 'admin' + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize('scripts_setting', [False, True]) +def test_new_wizard_id_skips_existing_ids_before_display(tmp_path, monkeypatch, scripts_setting): + import job_identity + from borg_backup_ui import BackupUIHandler + occupied = job_id('already-saved') + unused = job_id('not-yet-saved') + metadata = tmp_path / 'config/jobs' / (occupied + '.json') + write(metadata, {'job_id': occupied, 'name': 'Existing job'}) + before = metadata.read_bytes() + candidates = iter([occupied, occupied, unused]) + monkeypatch.setattr(job_identity.uuid, 'uuid4', lambda: next(candidates)) + handler = BackupUIHandler.__new__(BackupUIHandler) + handler.config = {'BACKUP_SCRIPTS_DIR': str(tmp_path / 'scripts' if scripts_setting else tmp_path)} + assert handler._get_wizard_new_job_id() == {'job_id': unused} + assert metadata.read_bytes() == before + assert list(metadata.parent.iterdir()) == [metadata] + + +def test_unusable_id_generator_stops_without_writing_jobs(tmp_path, monkeypatch): + import job_identity + occupied = job_id('already-saved') + target = tmp_path / (occupied + '.json') + target.write_text('existing') + monkeypatch.setattr(job_identity.uuid, 'uuid4', lambda: occupied) + with pytest.raises(job_identity.JobIdConflictError): + job_identity.new_job_id(tmp_path) + assert target.read_text() == 'existing' + assert list(tmp_path.iterdir()) == [target] + + +def test_new_wizard_saves_displayed_id_after_failed_write_without_overwriting(tmp_path, monkeypatch): + import repositories_api + from job_identity import new_job_id + config, jobs, ids, root = migrated(tmp_path) + params = load_job_for_wizard(ids[jobs[0]['job_key']], root / 'scripts', config) + displayed = new_job_id() + params.update(job_id=displayed, job_name='New job', repository_key='separate') + target = root / 'config/jobs' / (displayed + '.json') + original_repo = (root / 'config/repositories.json').read_bytes() + original_write = repositories_api.atomic_write_json + + def failed_write(*args, **kwargs): + raise OSError('simulated write failure') + + monkeypatch.setattr(repositories_api, 'atomic_write_json', failed_write) + with pytest.raises(OSError, match='simulated write failure'): + save_job(params, root / 'scripts', root, config) + assert not target.exists() + assert (root / 'config/repositories.json').read_bytes() == original_repo + + monkeypatch.setattr(repositories_api, 'atomic_write_json', original_write) + result = save_job(params, root / 'scripts', root, config) + assert result['job_id'] == result['job_key'] == displayed + saved = target.read_bytes() + assert json.loads(saved)['job_id'] == displayed + params.update(job_name='Another job with all inputs retained', archive_prefix='another-job') + params.update(description='Keep this description', icon='flash', icon_color='blue', + compression='zstd,3', keep_daily='11', file_activity=True) + replacement = save_job(params, root / 'scripts', root, config) + assert replacement['job_id'] != displayed + assert target.read_bytes() == saved + new_job = json.loads(Path(replacement['metadata_path']).read_text()) + for field, expected in {'name': params['job_name'], 'description': params['description'], + 'icon': 'flash', 'icon_color': 'blue', 'compression': 'zstd,3', + 'file_activity': True, 'archive_prefix': 'another-job', + 'source_paths': params['source_paths'], 'repository_key': 'separate'}.items(): + assert new_job[field] == expected + assert new_job['retention']['daily'] == '11' + params['job_id'] = '../invalid-id' + with pytest.raises(ValueError, match='UUID'): + save_job(params, root / 'scripts', root, config) + + +def test_simultaneous_creates_with_the_same_displayed_id_cannot_overwrite(tmp_path, monkeypatch): + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + import repositories_api + from job_identity import new_job_id + config, jobs, ids, root = migrated(tmp_path) + params = load_job_for_wizard(ids[jobs[0]['job_key']], root / 'scripts', config) + displayed = new_job_id() + params.update(job_id=displayed, repository_key='separate') + transaction = repositories_api.save_job_repository_transaction + barrier = Barrier(2) + + def concurrent_transaction(*args, **kwargs): + # Both requests have passed the existence check outside the inventory lock. + if args[4] == displayed: + barrier.wait(timeout=10) + return transaction(*args, **kwargs) + + monkeypatch.setattr(repositories_api, 'save_job_repository_transaction', concurrent_transaction) + + def create(name): + result = save_job({**params, 'job_name': name, 'archive_prefix': name.replace(' ', '-')}, + root / 'scripts', root, config) + return name, result['job_id'] + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(create, ['First request', 'Second request'])) + created_ids = {saved_id for _, saved_id in results} + assert len(created_ids) == 2 + assert displayed in created_ids + for name, saved_id in results: + stored = json.loads((root / 'config/jobs' / (saved_id + '.json')).read_text()) + assert stored['name'] == name + repository = next(row for row in read_repository_store(config)['repositories'] if row['repository_key'] == 'separate') + assert set(repository['used_by']) == created_ids + + +def test_name_and_full_prefix_edit_keeps_every_job_relationship(tmp_path, monkeypatch): + config, jobs, ids, root = migrated(tmp_path) + key = ids[jobs[0]['job_key']] + meta_path = root / 'config/jobs' / (key + '.json') + before = json.loads(meta_path.read_text()) + schedules = get_schedules(config) + results = list_restore_tests(config) + initial = {row['key']: row for row in get_status_data(config)['backups']}[key] + monkeypatch.setattr(wizard_runner.os, 'environ', dict(wizard_runner.os.environ)) + env_before, _ = wizard_runner._load_env_from_job(key, root / 'scripts', root) + params = load_job_for_wizard(key, root / 'scripts', config) + params.update(existing_job_key=key, job_name='A renamed job', archive_prefix='flash-config') + result = save_job(params, root / 'scripts', root, config) + after = json.loads(meta_path.read_text()) + assert result['job_id'] == result['job_key'] == key + assert 'backup_type' not in after + for field in ('icon', 'icon_color', 'cache_subdir', 'check_flag_name', + 'source_paths', 'retention', 'restore_test_policy', 'extension', 'created_at'): + assert after[field] == before[field] + assert after['archive_prefixes'] == ['flash-config', *before['archive_prefixes']] + assert get_schedules(config) == schedules + assert list_restore_tests(config) == results + env_after, _ = wizard_runner._load_env_from_job(key, root / 'scripts', root) + for field in ('BORG_UI_JOB_KEY', 'BORG_CACHE_DIR', 'BORG_CHECK_FLAG_FILE', 'BORG_REPO'): + assert env_after[field] == env_before[field] + statuses = get_status_data(config)['backups'] + row = next(r for r in statuses if r['key'] == key) + for field in ('repository_check_status', 'repository_check_date', 'status', 'archive_name', + 'original_size', 'restore_verification_last_test_date', 'restore_verification_status'): + assert row.get(field) == initial.get(field) + assert row['name'] == 'A renamed job' + assert [row['name'] for row in statuses] == ['A renamed job', 'Alpha'] + assert [job.name for job in discover_jobs(root / 'scripts', root)] == ['A renamed job', 'Alpha'] + history = get_history_data(config, {'job_key': key}) + assert len(history['entries']) == 1 + assert history['entries'][0]['job_id'] == key + assert history['entries'][0]['job_name'] == 'A renamed job' + assert get_report_data(config, key)['run_count'] == 1 + assert get_report_jobs(config)[0]['display_name'] == 'A renamed job' + assert list_restore_test_plan(config)['jobs'][0]['job_key'] == key + exported = export_jobs_bundle(config, [key])['bundle'] + assert exported['jobs'] == [after] + assert exported['schedules'] == {key: schedules[key]} + # Reusing a prefix owned by this same job is allowed. + params['archive_prefix'] = before['archive_prefix'] + assert save_job(params, root / 'scripts', root, config)['job_id'] == key + params['job_id'] = job_id('different') + with pytest.raises(ValueError, match='cannot be changed'): + save_job(params, root / 'scripts', root, config) + + +def test_two_jobs_can_share_a_prefix_only_in_different_repositories(tmp_path, monkeypatch): + config, jobs, ids, root = migrated(tmp_path) + key = ids[jobs[0]['job_key']] + params = load_job_for_wizard(key, root / 'scripts', config) + params.pop('job_id', None) + params.update(job_name='New independent job', repository_key='separate') + created = save_job(params, root / 'scripts', root, config) + assert created['job_id'] not in ids.values() + assert len(discover_jobs(root / 'scripts', root)) == 3 + monkeypatch.setattr(wizard_runner.os, 'environ', dict(wizard_runner.os.environ)) + original, _ = wizard_runner._load_env_from_job(key, root / 'scripts', root) + independent, _ = wizard_runner._load_env_from_job(created['job_id'], root / 'scripts', root) + assert independent['BORG_CACHE_DIR'] != original['BORG_CACHE_DIR'] + assert independent['BORG_CHECK_FLAG_FILE'] != original['BORG_CHECK_FLAG_FILE'] + assert independent['LOCK_FILE'] != original['LOCK_FILE'] + params['repository_key'] = 'shared' + with pytest.raises(ValueError, match='overlaps'): + save_job(params, root / 'scripts', root, config) + assert len(discover_jobs(root / 'scripts', root)) == 3 + + +@pytest.mark.parametrize('prefix', ['flash', 'flash-config', 'old-flash']) +def test_prefix_ownership_includes_overlap_and_previous_prefixes(prefix): + owner = {'job_id': job_id('first'), 'name': 'First', 'repository_key': 'shared', + 'archive_prefix': 'flash', 'archive_prefixes': ['old-flash']} + candidate = {**owner, 'job_id': job_id('second'), 'archive_prefix': prefix, 'archive_prefixes': []} + with pytest.raises(ValueError, match='overlaps'): + validate_prefix_ownership(candidate, [owner]) + validate_prefix_ownership({**candidate, 'repository_key': 'other'}, [owner]) + validate_prefix_ownership(owner, [owner]) + assert validate_archive_prefix('flash-config') == 'flash-config' + + +def test_supported_imports_preserve_identity_and_old_imports_are_rejected(tmp_path): + config, jobs, ids, root = migrated(tmp_path) + key = ids[jobs[0]['job_key']] + bundle = export_jobs_bundle(config, [key])['bundle'] + bundle['jobs'][0]['name'] = 'Updated from export' + report = import_jobs_bundle(config, bundle, mode='overwrite', dry_run=False) + assert report['report'][0]['new_job_key'] == key + assert json.loads((root / 'config/jobs' / (key + '.json')).read_text())['extension'] == jobs[0]['extension'] + # An old export is rejected even when the migration journal knows its old key. + legacy = copy.deepcopy(bundle) + legacy['jobs'] = [copy.deepcopy(jobs[0])] + legacy['jobs'][0]['backup_type'] = jobs[0]['backup_type'].upper() + legacy['schedules'] = {jobs[0]['job_key']: {'cron': '5 9 * * *', 'enabled': True}} + before_legacy = {p: p.read_bytes() for p in (root / 'config').rglob('*') if p.is_file()} + from settings_transfer_api import ConfigurationExportError + with pytest.raises(ConfigurationExportError): + import_jobs_bundle(config, legacy, mode='overwrite', dry_run=False) + assert {p: p.read_bytes() for p in before_legacy} == before_legacy + assert import_jobs_bundle(config, bundle, mode='skip', dry_run=False)['imported_count'] == 0 + before = {p.name: p.read_bytes() for p in (root / 'config/jobs').glob('*.json')} + with pytest.raises(ValueError, match='overlaps'): + import_jobs_bundle(config, bundle, mode='rename', dry_run=False) + assert {p.name: p.read_bytes() for p in (root / 'config/jobs').glob('*.json')} == before + # Importing a copy into another repository allocates an independent ID. + copied = copy.deepcopy(bundle) + copied['jobs'][0]['repository_key'] = 'separate' + report = import_jobs_bundle(config, copied, mode='rename', dry_run=False) + copied_id = report['report'][0]['new_job_key'] + assert copied_id != key + assert get_schedules(config)[copied_id] == bundle['schedules'][key] + repositories = {r['repository_key']: r for r in read_repository_store(config)['repositories']} + assert repositories['separate']['used_by'] == [copied_id] + assert set(repositories['shared']['used_by']) == set(ids.values()) + + +def test_migrated_jobs_remain_complete_in_support_bundle(tmp_path, monkeypatch): + import base64 + import io + import zipfile + import support_bundle_api + import system_health_api + config, jobs, ids, root = migrated(tmp_path) + monkeypatch.setattr(system_health_api, 'get_system_health_data', lambda _: {}) + payload = support_bundle_api.create_support_bundle(config, app_version='test-486') + with zipfile.ZipFile(io.BytesIO(base64.b64decode(payload['payload_b64']))) as bundle: + for key in ids.values(): + expected = json.loads((root / 'config/jobs' / (key + '.json')).read_text()) + assert json.loads(bundle.read('jobs/' + key + '.json')) == expected + + +@pytest.mark.parametrize('character', ['a', 'ä', '資']) +def test_wizard_limits_job_name_characters_without_truncating_saved_metadata(tmp_path, character): + from wizard_api import JobNameValidationError + config, jobs, ids, root = migrated(tmp_path) + key = ids[jobs[0]['job_key']] + params = load_job_for_wizard(key, root / 'scripts', config) + params.update(existing_job_key=key, job_name=character * 100) + assert save_job(params, root / 'scripts', root, config)['job_id'] == key + path = root / 'config/jobs' / (key + '.json') + before = path.read_bytes() + assert json.loads(before)['name'] == character * 100 + params['job_name'] += character + with pytest.raises(JobNameValidationError) as error: + save_job(params, root / 'scripts', root, config) + assert error.value.api_code == 'job_name_too_long' + assert path.read_bytes() == before + + +def test_delete_optional_artifacts_selects_only_the_requested_id(tmp_path, monkeypatch): + from borg_backup_ui import BackupUIHandler + import schedule_api + config, jobs, ids, root = migrated(tmp_path) + selected, retained = [ids[j['job_key']] for j in jobs] + log_dir = tmp_path / 'logs' + log_dir.mkdir() + with (root / 'config/backup.conf').open('a') as handle: + handle.write(f'GLOBAL_LOG_DIR="{log_dir}"\n') + owned = log_dir / 'old-custom-type.log' + other = log_dir / 'other.log' + owned.write_text('selected') + other.write_text('retained') + from job_identity import job_log_filename + new_owned = log_dir / job_log_filename('Earlier name', 'local', selected, '2026-09-07_15-00-01') + new_other = log_dir / job_log_filename('Earlier name', 'local', retained, '2026-09-07_15-00-01') + new_owned.write_text('orphaned selected log') + new_other.write_text('other job log') + for path in Path(config['STATUS_DIR']).glob('*.status'): + payload = json.loads(path.read_text()) + payload['log_file'] = str(owned if payload['job_id'] == selected else other) + write(path, payload) + monkeypatch.setattr(schedule_api, 'delete_schedule', lambda *args: {}) + handler = BackupUIHandler.__new__(BackupUIHandler) + handler.config = config + handler._read_json_body = lambda: {'job_key': selected, 'delete_artifacts': True} + assert handler._delete_job()['deleted'] is True + assert [j.key for j in discover_jobs(root / 'scripts', root)] == [retained] + assert len(list(Path(config['STATUS_DIR']).glob('*.status'))) == 1 + assert not owned.exists() + assert not new_owned.exists() + assert new_other.read_text() == 'other job log' + assert other.read_text() == 'retained' + assert [r['job_key'] for r in list_restore_tests(config)] == [retained] + + +def test_repository_switch_uses_its_own_check_marker_and_status(tmp_path, monkeypatch): + from types import SimpleNamespace + from lib import borg_runner + from lib.backup_job import BackupJob + + config, jobs, ids, root = migrated(tmp_path) + key = ids[jobs[0]['job_key']] + monkeypatch.setattr(wizard_runner.os, 'environ', dict(wizard_runner.os.environ)) + env, _ = wizard_runner._load_env_from_job(key, root / 'scripts', root) + legacy_flag = Path(env['BORG_CHECK_FLAG_FILE']) + legacy_flag.parent.mkdir(parents=True, exist_ok=True) + legacy_flag.write_text('original repository check') + legacy_before = (legacy_flag.read_bytes(), legacy_flag.stat().st_mtime_ns) + original_cache = env['BORG_CACHE_DIR'] + calls = [] + monkeypatch.setattr(borg_runner, '_run_borg', lambda command, *_args: calls.append(command) or 0) + flags = {} + for repository, expect_check in [('separate', True), ('shared', True), ('separate', False), ('shared', False)]: + params = load_job_for_wizard(key, root / 'scripts', config) + params.update(existing_job_key=key, repository_key=repository) + assert save_job(params, root / 'scripts', root, config)['job_id'] == key + env, _ = wizard_runner._load_env_from_job(key, root / 'scripts', root) + flag = Path(env['BORG_CHECK_FLAG_FILE']) + assert env['BORG_CACHE_DIR'] == original_cache + assert flag != legacy_flag + if repository in flags: + assert flag == flags[repository] + flags[repository] = flag + runtime = BackupJob.__new__(BackupJob) + runtime.config = SimpleNamespace(borg_check_flag_file=flag, borg_check_interval_days=30) + if expect_check: + assert runtime._get_repo_check_info() == ('unknown', 'unknown', 'unknown') + else: + assert runtime._get_repo_check_info()[1] == 'ok' + before_calls = len(calls) + runner = borg_runner.BorgRunner(borg_runner.BorgConfig( + repo=env['BORG_REPO'], check_flag_file=flag, check_interval_days=30)) + assert runner.check() == 0 + assert len(calls) == before_calls + int(expect_check) + if expect_check: + assert calls[-1][-1] == env['BORG_REPO'] + assert runtime._get_repo_check_info()[1] == 'ok' + assert (legacy_flag.read_bytes(), legacy_flag.stat().st_mtime_ns) == legacy_before + assert flags['shared'] != flags['separate'] + + +def test_new_job_reuses_original_repository_check_after_switching_back(tmp_path, monkeypatch): + config, jobs, ids, root = migrated(tmp_path) + monkeypatch.setattr(wizard_runner.os, 'environ', dict(wizard_runner.os.environ)) + params = load_job_for_wizard(ids[jobs[0]['job_key']], root / 'scripts', config) + params.update(job_id=job_id('new-check-job'), archive_prefix='new-check', repository_key='shared') + key = save_job(params, root / 'scripts', root, config)['job_id'] + env, _ = wizard_runner._load_env_from_job(key, root / 'scripts', root) + original_flag = env['BORG_CHECK_FLAG_FILE'] + for repository in ['separate', 'shared']: + params = load_job_for_wizard(key, root / 'scripts', config) + params.update(existing_job_key=key, repository_key=repository) + save_job(params, root / 'scripts', root, config) + env, _ = wizard_runner._load_env_from_job(key, root / 'scripts', root) + assert env['BORG_CHECK_FLAG_FILE'] == original_flag diff --git a/tests/test_job_identity_ui.py b/tests/test_job_identity_ui.py new file mode 100644 index 00000000..b22d6ff4 --- /dev/null +++ b/tests/test_job_identity_ui.py @@ -0,0 +1,11 @@ +import shutil +import subprocess +from pathlib import Path +import pytest + + +def test_job_identity_browser_logic(): + node = shutil.which('node') + if not node: + pytest.skip('Node.js is required for the job identity UI tests') + subprocess.run([node, 'tests/job_identity_ui.cjs'], cwd=Path(__file__).resolve().parents[1], check=True) diff --git a/tests/test_job_runtime_detection.py b/tests/test_job_runtime_detection.py index f617ffd5..af24a1af 100644 --- a/tests/test_job_runtime_detection.py +++ b/tests/test_job_runtime_detection.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id import json import os import sys @@ -84,13 +85,13 @@ def test_dead_resource_lock_is_not_reported_as_running(tmp_path: Path): def test_job_discovery_cache_reuses_metadata_and_detects_atomic_update(tmp_path: Path, monkeypatch): jobs_dir = tmp_path / "config" / "jobs" jobs_dir.mkdir(parents=True) - metadata = jobs_dir / "flash_local.json" - payload = { - "job_key": "flash_local", + metadata = jobs_dir / (job_id('flash_local') + ".json") + payload = identified_job({ + "job_key": job_id('flash_local'), "name": "Flash", "backup_type": "flash", "location": "local", - } + }) metadata.write_text(json.dumps(payload), encoding="utf-8") jobs_api.invalidate_job_discovery_cache() original = jobs_api._discover_jobs_uncached diff --git a/tests/test_job_settings_migration.py b/tests/test_job_settings_migration.py new file mode 100644 index 00000000..af6a53af --- /dev/null +++ b/tests/test_job_settings_migration.py @@ -0,0 +1,153 @@ +"""Type-free jobs preserve effective settings and reject old exports (#495).""" + +import base64 +import json +from pathlib import Path + +import pytest + +from test_job_id_migration import main_fixture, write +from migrations import job_ids_v1, job_settings_v1 +from job_identity import new_job_id +from job_settings import explicit_job_settings +from settings_transfer_api import ( + ConfigurationExportError, _encrypt_authenticated_export, + import_jobs_bundle, preview_jobs_bundle, import_jobs_bundle_encrypted, + preview_jobs_bundle_encrypted, import_profile_secrets_backup, preview_profile_secrets_backup, +) + + +def _identified(config): + job_ids_v1.apply(config) + return sorted((Path(config['BACKUP_SCRIPTS_DIR']) / 'config/jobs').glob('*.json')) + + +def test_settings_migration_preserves_values_and_repeats_without_changes(tmp_path): + config, _ = main_fixture(tmp_path) + paths = _identified(config) + original = json.loads(paths[0].read_text()) + original.update(backup_type='flash', compression='', icon='', icon_color='') + original['retention'].pop('daily') + original['retention']['yearly'] = 0 + write(paths[0], original) + conf = Path(config['BACKUP_SCRIPTS_DIR']) / 'config/backup.conf' + with conf.open('a') as out: + out.write('COMPRESSION_FLASH="zstd,7"\nRETENTION_FLASH_DAILY="12"\nRETENTION_FLASH_WEEKLY="99"\n') + result = job_settings_v1.apply(config) + assert result['status'] == 'applied' + saved = json.loads(paths[0].read_text()) + assert saved['compression'] == 'zstd,7' + assert saved['retention'] == {**original['retention'], 'daily': '12', 'yearly': '0'} + assert saved['icon'] == 'flash' + assert saved['icon_color'] == 'theme-blue' + assert 'backup_type' not in saved and 'type_id' not in saved + for key in ('job_id', 'job_key', 'name', 'archive_prefixes', 'cache_subdir', 'check_flag_name', 'extension', 'created_at', 'updated_at'): + assert saved[key] == original[key] + backups = [json.loads(path.read_text()) for path in Path(result['details']['backup_directory']).glob('*.before')] + assert original in backups + before = {p: p.read_bytes() for p in paths} + assert job_settings_v1.detect(config)['required'] is False + assert job_settings_v1.apply(config)['status'] == 'not_required' + assert before == {p: p.read_bytes() for p in paths} + + +def test_settings_migration_resumes_after_interrupted_write(tmp_path, monkeypatch): + config, _ = main_fixture(tmp_path) + paths = _identified(config) + real_apply = job_settings_v1._apply_operation + count = 0 + def fail_second(op): + nonlocal count + count += 1 + if count == 2: + raise OSError('simulated write interruption') + real_apply(op) + monkeypatch.setattr(job_settings_v1, '_apply_operation', fail_second) + with pytest.raises(OSError, match='interruption'): + job_settings_v1.apply(config) + pending = json.loads(job_settings_v1._journal(config).read_text()) + assert pending['status'] == 'pending' + assert job_settings_v1.detect(config)['required'] is True + monkeypatch.setattr(job_settings_v1, '_apply_operation', real_apply) + assert job_settings_v1.apply(config)['status'] == 'applied' + applied = json.loads(job_settings_v1._journal(config).read_text()) + assert applied['backup_directory'] == pending['backup_directory'] + for path in paths: + saved = json.loads(path.read_text()) + assert saved['schema_version'] == 5 + explicit_job_settings(saved) + + +def test_invalid_old_settings_do_not_partially_migrate_jobs(tmp_path): + config, _ = main_fixture(tmp_path) + paths = _identified(config) + invalid = json.loads(paths[-1].read_text()) + invalid['retention']['daily'] = 'invalid' + write(paths[-1], invalid) + before = {path: path.read_bytes() for path in paths} + with pytest.raises(ValueError, match='retention'): + job_settings_v1.apply(config) + assert {path: path.read_bytes() for path in paths} == before + + +def _encrypted(payload): + return base64.b64encode(_encrypt_authenticated_export(json.dumps(payload).encode(), 'synthetic-test-password')).decode() + + +@pytest.mark.parametrize('kind', ['jobs', 'profiles']) +@pytest.mark.parametrize('preview', [False, True]) +def test_old_encrypted_packages_are_rejected_without_writing(tmp_path, kind, preview): + target = tmp_path / 'not-created' + config = {'BACKUP_SCRIPTS_DIR': str(target)} + payload = {'format': 'bbui-job-bundle-secure-v2', 'bundle': {'format': 'bbui-job-bundle-v2', 'jobs': []}} if kind == 'jobs' else {'format': 'bbui-profile-secrets-v1', 'manifest': [], 'files': []} + function = (preview_jobs_bundle_encrypted if preview else import_jobs_bundle_encrypted) if kind == 'jobs' else (preview_profile_secrets_backup if preview else import_profile_secrets_backup) + with pytest.raises(ConfigurationExportError): + function(config, 'synthetic-test-password', _encrypted(payload)) + assert not target.exists() + + +@pytest.mark.parametrize('preview', [False, True]) +def test_new_format_marker_does_not_make_old_job_metadata_supported(tmp_path, preview): + target = tmp_path / 'not-created' + bundle = {'format': 'bbui-job-bundle-v3', 'jobs': [{'schema_version': 3, 'job_key': 'flash_local', 'backup_type': 'flash'}]} + with pytest.raises(ConfigurationExportError): + (preview_jobs_bundle if preview else import_jobs_bundle)({'BACKUP_SCRIPTS_DIR': str(target)}, bundle) + assert not target.exists() + + +def test_orphan_status_and_restore_files_remain_without_becoming_jobs(tmp_path): + from test_job_identity_integration import migrated + from status_api import get_status_data + from reports_api import get_report_jobs + from history_api import get_history_data + from restore_tests_api import list_restore_tests + config, _, ids, _ = migrated(tmp_path) + status = Path(config['STATUS_DIR']) + orphan = status / '2026-09-07_13-00-00_deleted_local.status' + write(orphan, {'backup_type': 'deleted', 'location': 'local', 'status': 'success', 'timestamp': '2026-09-07 13:00:00'}) + removed = status / '2026-09-07_14-00-00_deleted_uuid.status' + write(removed, {'job_id': new_job_id(), 'status': 'success', 'timestamp': '2026-09-07 14:00:00'}) + restore = Path(config['RESTORE_TEST_STATUS_DIR']) / 'deleted_local.test' + write(restore, {'test_result': 'success', 'type': 'deleted', 'location': 'local'}) + before = {path: path.read_bytes() for path in (orphan, removed, restore)} + assert {row['key'] for row in get_status_data(config)['backups']} == set(ids.values()) + assert {row['key'] for row in get_report_jobs(config)} == set(ids.values()) + assert {row['job_id'] for row in get_history_data(config)['entries']} == set(ids.values()) + assert {row['job_key'] for row in list_restore_tests(config)} == set(ids.values()) + assert {path: path.read_bytes() for path in before} == before + + +def test_new_backup_run_without_uuid_is_rejected(): + from runtime.lib.backup_job import BackupJobConfig + with pytest.raises(ValueError, match="Job ID must be a UUID"): + BackupJobConfig.from_config({"BACKUP_TYPE": "flash", "BACKUP_LOCATION": "local"}) + + +def test_new_restore_status_without_uuid_is_rejected_before_writing(tmp_path): + from test_restore_test_runner_profiles import _load_restore_runner + runner = _load_restore_runner() + instance = object.__new__(runner.RestoreTest) + instance.status_dir = tmp_path / "not-created" + with pytest.raises(ValueError, match="Job ID must be a UUID"): + instance._write("flash_local", {}, "success", 1, 0, 0, 0, "unknown", "", {}, []) + assert not instance.status_dir.exists() diff --git a/tests/test_job_source_paths.py b/tests/test_job_source_paths.py index 909e0627..76d3e8bd 100644 --- a/tests/test_job_source_paths.py +++ b/tests/test_job_source_paths.py @@ -17,10 +17,10 @@ convert_legacy_source_paths, normalize_source_paths, ) -from settings_transfer_api import _canonical_import_jobs # noqa: E402 +from settings_transfer_api import ConfigurationExportError, _canonical_import_jobs # noqa: E402 -def test_old_job_bundle_is_upgraded_only_at_import_boundary(tmp_path: Path) -> None: +def test_old_job_bundle_is_rejected_at_import_boundary(tmp_path: Path) -> None: source = tmp_path / "Source with spaces" source.mkdir() jobs = [{ @@ -29,11 +29,9 @@ def test_old_job_bundle_is_upgraded_only_at_import_boundary(tmp_path: Path) -> N "paths": {"default": str(source)}, }] - upgraded = _canonical_import_jobs(jobs) + with pytest.raises(ConfigurationExportError): + _canonical_import_jobs(jobs) - assert upgraded[0]["schema_version"] == 3 - assert upgraded[0]["source_paths"] == [str(source)] - assert "paths" not in upgraded[0] assert "paths" in jobs[0] @@ -46,7 +44,7 @@ def test_old_job_bundle_reports_ambiguous_paths_clearly(tmp_path: Path) -> None: "paths": {"default": f"{existing} {tmp_path / 'missing'}"}, }] - with pytest.raises(ValueError, match="Imported job 'data_local'.*cannot be migrated unambiguously"): + with pytest.raises(ConfigurationExportError): _canonical_import_jobs(jobs) diff --git a/tests/test_lifecycle_log.py b/tests/test_lifecycle_log.py index 16aadf19..cc8e5b1d 100644 --- a/tests/test_lifecycle_log.py +++ b/tests/test_lifecycle_log.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id import os from pathlib import Path import sys @@ -23,6 +24,7 @@ def _backup_job_config(tmp_path: Path) -> BackupJobConfig: job_name="Flash", backup_type="flash", backup_location="local", + job_id=job_id("flash_local"), lock_file=tmp_path / "job.lock", log_dir=tmp_path / "logs", log_file=tmp_path / "logs" / "backup.log", @@ -75,7 +77,7 @@ def test_backup_finish_emits_lifecycle_summary(tmp_path: Path, monkeypatch): text = log_file.read_text(encoding="utf-8") assert "JOB finished" in text assert "request_id=req-123" in text - assert "job_key=flash_local" in text + assert f"job_key={job_id('flash_local')}" in text assert "run_id=run-123" in text assert "status=success" in text assert "exit_code=0" in text @@ -129,7 +131,7 @@ def test_notification_event_emits_lifecycle_summary(tmp_path: Path, monkeypatch) event_type="backup_success", title="Backup OK", message="done", - job_key="flash_local", + job_key=job_id("flash_local"), status="success", duration_seconds=45, exit_code=0, @@ -140,7 +142,7 @@ def test_notification_event_emits_lifecycle_summary(tmp_path: Path, monkeypatch) assert result["unraid"] is True text = log_file.read_text(encoding="utf-8") assert "JOB notification" in text - assert "job_key=flash_local" in text + assert f"job_key={job_id('flash_local')}" in text assert "event=backup_success" in text assert "apprise_mode=queued" in text or "apprise_mode=sync" in text @@ -176,6 +178,7 @@ def get_state(self, job_key): monkeypatch.setattr(jobs_api, "discover_jobs", lambda _scripts, _data: [ SimpleNamespace( key="flash_local", + name="Flash configuration", enabled=True, standard="wizard", backup_type="flash", @@ -190,6 +193,8 @@ def get_state(self, job_key): assert captured["extra_env"]["BORG_UI_REQUEST_ID"] == "req-scheduled" assert captured["extra_env"]["BORG_UI_REQUEST_SOURCE"] == "schedule" assert captured["extra_env"]["BORG_UI_REQUEST_ACTOR"] == "scheduler" + assert captured["extra_env"]["BORG_UI_JOB_NAME"] == "Flash configuration" + assert captured["extra_env"]["BORG_UI_JOB_LOCATION"] == "local" def test_restore_test_script_contains_lifecycle_summary_hooks(): diff --git a/tests/test_navigation_io.py b/tests/test_navigation_io.py new file mode 100644 index 00000000..ed8cb173 --- /dev/null +++ b/tests/test_navigation_io.py @@ -0,0 +1,112 @@ +"""Restore navigation must not write storage probes (#497).""" + +import sys +from pathlib import Path +from unittest.mock import Mock + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +for directory in (ROOT, ROOT / "api", ROOT / "runtime", ROOT / "runtime/lib"): + if str(directory) not in sys.path: + sys.path.insert(0, str(directory)) + +import config_api +import restore_api +from borg_backup_ui import BackupUIHandler + + +READ_REQUESTS = [ + ("_get_restore_archives", "job=example", "list_archives_with_context", ("example",), {"archives": []}), + ("_get_restore_files", "job=example&archive=backup&path=docs", "list_files", ("example", "backup", "docs"), []), + ("_get_repo_stats", "job=example", "get_repo_stats", ("example",), {"size": 123}), + ("_get_restore_target_dirs", "prefix=%2Fmnt%2Fuser&limit=12", "list_target_dirs_with_config", ("/mnt/user", 12), []), + ("_get_restore_state", "restore_id=example", "get_restore_state", ("example",), {"status": "idle"}), +] + + +@pytest.fixture +def handler_and_data(tmp_path): + config = { + "BACKUP_SCRIPTS_DIR": str(tmp_path / "config-root"), + "BACKUP_CONF_SCHEMA_FILE": str(ROOT / "runtime/config/backup.conf.example"), + } + data = tmp_path / "data" + config_api.write_conf(config, {"GLOBAL_DATA_DIR": str(data)}) + config_api.ensure_data_dirs(str(data)) + handler = object.__new__(BackupUIHandler) + handler.config = config + return handler, data + + +@pytest.mark.parametrize("method,query,dependency,args,result", READ_REQUESTS) +def test_restore_navigation_reads_without_changing_data_directories( + handler_and_data, monkeypatch, method, query, dependency, args, result +): + handler, data = handler_and_data + downstream = Mock(return_value=result) + monkeypatch.setattr(restore_api, dependency, downstream) + monkeypatch.setattr(restore_api, "list_allowed_target_roots", lambda _config: ["/mnt/user"]) + paths = [data, *data.rglob("*")] + before = {p: (p.stat().st_mtime_ns, p.stat().st_ctime_ns) for p in paths} + original_write = Path.write_text + + def no_probe(path, *a, **kw): + assert path.name != ".borg-ui-write-test", "Navigation attempted a storage write" + return original_write(path, *a, **kw) + + monkeypatch.setattr(Path, "write_text", no_probe) + expected = {"files": result} if method == "_get_restore_files" else result + if method == "_get_restore_target_dirs": + expected = {"dirs": result, "allowed_roots": ["/mnt/user"]} + for _ in range(3): + assert getattr(handler, method)(query) == expected + downstream.assert_called_with(handler.config, *args) + assert set(data.rglob("*")) == set(paths) - {data} + assert {p: (p.stat().st_mtime_ns, p.stat().st_ctime_ns) for p in paths} == before + + +@pytest.mark.parametrize("method,query,dependency,args,result", READ_REQUESTS) +@pytest.mark.parametrize("problem", ["missing", "unwritable", "unmounted"]) +def test_restore_navigation_rejects_unavailable_storage_without_repair( + handler_and_data, monkeypatch, method, query, dependency, args, result, problem +): + handler, data = handler_and_data + if problem == "missing": + (data / "status").rmdir() + elif problem == "unwritable": + monkeypatch.setattr(config_api.os, "access", lambda *_: False) + else: + config_api.write_conf(handler.config, {"GLOBAL_DATA_DIR": "/mnt/user/unavailable"}) + monkeypatch.setattr(config_api, "_is_required_storage_mount_available", lambda _: False) + downstream = Mock(side_effect=AssertionError("Unavailable storage must block the operation")) + monkeypatch.setattr(restore_api, dependency, downstream) + with pytest.raises(RuntimeError): + getattr(handler, method)(query) + downstream.assert_not_called() + if problem == "missing": + assert not (data / "status").exists() + + +@pytest.mark.parametrize("method", [ + "_post_run_job", "_post_run_check", "_post_restore_start", "_post_restore_precheck", + "_start_restore_test_from_body", "_handle_restore_download", +]) +def test_actual_actions_still_fail_when_storage_write_probe_fails(handler_and_data, monkeypatch, method): + handler, _data = handler_and_data + original_write = Path.write_text + + def fail_probe(path, *args, **kwargs): + if path.name == ".borg-ui-write-test": + raise OSError("simulated storage write failure") + return original_write(path, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", fail_probe) + if method == "_handle_restore_download": + handler.send_error = Mock() + handler._handle_restore_download(None) + handler.send_error.assert_called_once_with(500, "simulated storage write failure") + else: + with pytest.raises(OSError, match="simulated storage write failure"): + args = ({},) if method == "_start_restore_test_from_body" else () + getattr(handler, method)(*args) diff --git a/tests/test_notification_events.py b/tests/test_notification_events.py index 851f5768..8a67f878 100644 --- a/tests/test_notification_events.py +++ b/tests/test_notification_events.py @@ -7,6 +7,8 @@ import sys from types import SimpleNamespace +import pytest + ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "runtime")) sys.path.insert(0, str(ROOT / "api")) @@ -341,6 +343,68 @@ def test_queued_apprise_delivery_retries_without_sleeping(monkeypatch, tmp_path) assert status["deliveries"][-1]["status"] == "retrying" assert status["deliveries"][-1]["message"] == "provider unavailable" + queue_path = tmp_path / "config" / "notification-queue.json" + before = (queue_path.read_bytes(), queue_path.stat().st_mtime_ns, queue_path.stat().st_ino) + assert drain_notification_queue({"BACKUP_SCRIPTS_DIR": str(tmp_path)})["checked"] == 0 + assert (queue_path.read_bytes(), queue_path.stat().st_mtime_ns, queue_path.stat().st_ino) == before + + monkeypatch.setattr("lib.notification_events.time.time", lambda: queue["queue"][0]["next_attempt_at"] + 1) + exhausted = drain_notification_queue({"BACKUP_SCRIPTS_DIR": str(tmp_path)}) + assert exhausted == {"checked": 1, "delivered": 0, "failed": 1, "retrying": 0, "remaining": 0} + assert read_notification_delivery_status({"BACKUP_SCRIPTS_DIR": str(tmp_path)})["deliveries"][-1]["status"] == "failed" + + +@pytest.mark.parametrize("rows", [None, [], [{"id": "later", "next_attempt_at": 2000, "attempts_made": 1}]]) +def test_idle_queue_checks_do_not_save_unchanged_or_missing_queue(tmp_path, monkeypatch, rows): + from lib import notification_events + + config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} + path = tmp_path / "config" / "notification-queue.json" + if rows is not None: + path.parent.mkdir() + path.write_text(json.dumps({"schema_version": 1, "updated_at": "unchanged", "queue": rows})) + before = (path.read_bytes(), path.stat().st_mtime_ns, path.stat().st_ino) + monkeypatch.setattr(notification_events.time, "time", lambda: 1000) + + def unexpected_save(*_args): + pytest.fail("An idle queue check must not write JSON") + + monkeypatch.setattr(notification_events, "_write_json", unexpected_save) + for _ in range(10): + assert drain_notification_queue(config) == { + "checked": 0, "delivered": 0, "failed": 0, "retrying": 0, "remaining": len(rows or []), + } + if rows is None: + assert not path.exists() + else: + assert (path.read_bytes(), path.stat().st_mtime_ns, path.stat().st_ino) == before + + +def test_queue_claim_preserves_future_and_newly_enqueued_notifications(tmp_path, monkeypatch): + from lib import notification_events + + config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} + monkeypatch.setattr(notification_events.time, "time", lambda: 1000) + for name, due in (("first", 0), ("second", 0), ("later", 2000)): + notification_events._append_queue_item(config, {"id": name, "next_attempt_at": due}) + delivered = [] + + def deliver(_config, item): + # The claim must be persisted before provider I/O. Enqueueing during + # delivery must not be lost when this drain completes. + remaining = notification_events._read_queue_store(config)["queue"] + assert item["id"] not in [row["id"] for row in remaining] + if item["id"] == "first": + notification_events._append_queue_item(config, {"id": "new", "next_attempt_at": 2000}) + delivered.append(item["id"]) + return "delivered" + + monkeypatch.setattr(notification_events, "_deliver_queue_item", deliver) + assert drain_notification_queue(config, max_items=1)["remaining"] == 3 + assert drain_notification_queue(config, max_items=1)["remaining"] == 2 + assert delivered == ["first", "second"] + assert [row["id"] for row in notification_events._read_queue_store(config)["queue"]] == ["later", "new"] + def test_apprise_queue_records_dropped_entries_when_full(tmp_path): store = tmp_path / "config" / "apprise-profiles.json" @@ -659,6 +723,7 @@ def test_backup_overdue_uses_type_location_status_when_key_is_missing(monkeypatc monkeypatch.setattr("schedule_api.get_schedules", lambda cfg: {"appdata_usb": {"enabled": True, "cron": "0 10 * * *"}}) monkeypatch.setattr("jobs_api.list_jobs", lambda cfg, opts: [{"key": "appdata_usb", "display_name": "Appdata", "enabled": True, "repo_path": "/repo"}]) monkeypatch.setattr("status_api.get_status_data", lambda cfg: {"backups": [{ + "key": "appdata_usb", "backup_type": "appdata", "location": "usb", "timestamp": "2026-07-01 10:04:45", @@ -716,8 +781,8 @@ def test_backup_overdue_sender_matches_diagnostics_and_sends_only_ready_jobs(mon {"key": "sonstiges_usb", "display_name": "Sonstiges - USB", "enabled": True, "repo_path": "/repo/sonstiges"}, ]) monkeypatch.setattr("status_api.get_status_data", lambda cfg: {"backups": [ - {"backup_type": "appdata", "location": "usb", "timestamp": "2026-07-02 12:10:27", "status": "success"}, - {"backup_type": "sonstiges", "location": "usb", "timestamp": "2026-07-01 15:00:01", "status": "success"}, + {"key": "appdata_usb", "backup_type": "appdata", "location": "usb", "timestamp": "2026-07-02 12:10:27", "status": "success"}, + {"key": "sonstiges_usb", "backup_type": "sonstiges", "location": "usb", "timestamp": "2026-07-01 15:00:01", "status": "success"}, ]}) stale_appdata = "backup_overdue:appdata_usb:2026-07-02 10:00:00" mark_reminder_sent({"BACKUP_SCRIPTS_DIR": str(tmp_path)}, stale_appdata, now=datetime(2026, 7, 2, 8, 0, 0).timestamp()) @@ -744,6 +809,7 @@ def test_notification_reminder_diagnostics_reports_backup_overdue_window(monkeyp monkeypatch.setattr("schedule_api.get_schedules", lambda cfg: {"appdata_usb": {"enabled": True, "cron": "0 10 * * *"}}) monkeypatch.setattr("jobs_api.list_jobs", lambda cfg, opts: [{"key": "appdata_usb", "display_name": "Appdata", "enabled": True, "repo_path": "/repo"}]) monkeypatch.setattr("status_api.get_status_data", lambda cfg: {"backups": [{ + "key": "appdata_usb", "backup_type": "appdata", "location": "usb", "timestamp": "2026-07-02 10:04:45", @@ -815,6 +881,7 @@ def test_notification_reminder_diagnostics_distinguishes_missed_and_next_backup_ monkeypatch.setattr("schedule_api.get_schedules", lambda cfg: {"photos_usb": {"enabled": True, "cron": "0 14 * * 0"}}) monkeypatch.setattr("jobs_api.list_jobs", lambda cfg, opts: [{"key": "photos_usb", "display_name": "Photos - USB", "enabled": True, "repo_path": "/repo"}]) monkeypatch.setattr("status_api.get_status_data", lambda cfg: {"backups": [{ + "key": "photos_usb", "backup_type": "photos", "location": "usb", "timestamp": "2026-07-01 07:58:54", diff --git a/tests/test_plugin_manifest.py b/tests/test_plugin_manifest.py index 45f02a72..10047bcd 100644 --- a/tests/test_plugin_manifest.py +++ b/tests/test_plugin_manifest.py @@ -54,12 +54,12 @@ def test_community_apps_metadata_mentions_python_runtime_requirement() -> None: assert "Runtime requirement" in profile_text -def test_community_apps_metadata_requires_supported_openssh_runtime() -> None: +def test_community_apps_metadata_supports_unraid_6_12_5() -> None: plugin = ROOT / "plugins" / "borg-backup-ui.xml" plugin_root = ET.parse(plugin).getroot() - assert plugin_root.findtext("MinVer") == "7.2.0" + assert plugin_root.findtext("MinVer") == "6.12.5" def test_community_apps_metadata_uses_unraid_forum_support_thread() -> None: diff --git a/tests/test_python_ram_cache.py b/tests/test_python_ram_cache.py new file mode 100644 index 00000000..2b3c4390 --- /dev/null +++ b/tests/test_python_ram_cache.py @@ -0,0 +1,90 @@ +"""Exercise the service launcher with an isolated payload and RAM-cache path (#497).""" + +import json +import os +from pathlib import Path +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[1] + + +def _launcher(tmp_path): + plugin = tmp_path / "plugin" + plugin.mkdir() + cache = tmp_path / "ram/pycache" + (plugin / "bbui_cache_example.py").write_text("VALUE = 497\n") + (plugin / "borg_backup_ui.py").write_text(''' +import json, os, pathlib, subprocess, sys +import bbui_cache_example +root = pathlib.Path(__file__).parent +child = subprocess.check_output([ + sys.executable, "-c", + "import bbui_cache_example, json, sys; print(json.dumps({'prefix': sys.pycache_prefix, 'value': bbui_cache_example.VALUE}))" +], cwd=root, text=True) +report = {'prefix': sys.pycache_prefix, 'cached': bbui_cache_example.__cached__, + 'child': json.loads(child), 'value': bbui_cache_example.VALUE} +temporary = root / 'result.tmp' +temporary.write_text(json.dumps(report)) +temporary.replace(root / 'result.json') +''') + script = (ROOT / "plugin/rc.borg_backup_ui").read_text() + replacements = { + "/boot/config/plugins/borg-backup-ui": str(plugin), + "/run/borg-backup-ui/pycache": str(cache), + "/var/run/borg_backup_ui.pid": str(tmp_path / "service.pid"), + "/var/run/borg_backup_ui_start_wait.pid": str(tmp_path / "wait.pid"), + "/var/log/borg_backup_ui.log": str(tmp_path / "service.log"), + } + for original, replacement in replacements.items(): + assert original in script + script = script.replace(original, replacement) + launcher = tmp_path / "rc.borg_backup_ui" + launcher.write_text(script) + return launcher, plugin, cache + + +def _start(launcher, plugin, tmp_path): + report = plugin / "result.json" + report.unlink(missing_ok=True) + # Isolate the launcher from developer Python settings and an earlier exited + # test payload's PID. The production stop/start lifecycle is unchanged. + (tmp_path / "service.pid").unlink(missing_ok=True) + env = {k: v for k, v in os.environ.items() if not k.startswith("PYTHON")} + env["PYTHONPYCACHEPREFIX"] = str(tmp_path / "wrong-cache") + run = subprocess.run(["bash", str(launcher), "start"], env=env, capture_output=True, text=True, timeout=15) + assert run.returncode == 0, run.stderr + deadline = time.monotonic() + 10 + while not report.exists() and time.monotonic() < deadline: + time.sleep(0.02) + assert report.exists(), (tmp_path / "service.log").read_text() + return json.loads(report.read_text()) + + +def test_service_and_child_processes_reuse_cache_outside_plugin(tmp_path): + launcher, plugin, cache = _launcher(tmp_path) + first = _start(launcher, plugin, tmp_path) + cached = Path(first["cached"]) + assert first["prefix"] == str(cache) + assert first["child"] == {"prefix": str(cache), "value": 497} + assert cached.is_relative_to(cache) and cached.is_file() + assert cache.stat().st_mode & 0o777 == 0o700 + before = (cached.stat().st_ino, cached.stat().st_mtime_ns) + second = _start(launcher, plugin, tmp_path) + assert second == first + assert (cached.stat().st_ino, cached.stat().st_mtime_ns) == before + assert not list(plugin.rglob("*.pyc")) + assert not (tmp_path / "wrong-cache").exists() + + +def test_unavailable_optional_cache_does_not_block_service_or_write_to_plugin(tmp_path): + launcher, plugin, cache = _launcher(tmp_path) + cache.parent.mkdir() + cache.write_text("not a directory") + report = _start(launcher, plugin, tmp_path) + assert report["value"] == report["child"]["value"] == 497 + assert report["prefix"] == report["child"]["prefix"] == str(cache) + assert not list(plugin.rglob("*.pyc")) + assert not (tmp_path / "wrong-cache").exists() + assert cache.read_text() == "not a directory" + assert "bytecode writes are disabled" in (tmp_path / "service.log").read_text() diff --git a/tests/test_remaining_ui_redesign.py b/tests/test_remaining_ui_redesign.py index 4037c5a2..442a2e31 100644 --- a/tests/test_remaining_ui_redesign.py +++ b/tests/test_remaining_ui_redesign.py @@ -67,7 +67,7 @@ def test_storage_prune_confirmation_shows_archive_filter() -> None: assert "function storageJobsForRepository(repo)" in script assert "function storageArchivePrefixFromJob(job)" in script assert "function storageArchiveFilterFromJob(job)" in script - assert "function storageRetentionSummary(job)" in script + assert "function storageRetentionTableHtml(job)" in script assert "function storageMaintenancePruneDetailsHtml(repo, job)" in script assert "function updateStorageMaintenanceRetentionPreview()" in script assert 'id="storage-maintenance-retention-job"' in script @@ -80,10 +80,10 @@ def test_storage_prune_confirmation_shows_archive_filter() -> None: assert "storage.repositoryMaintenanceMultipleJobsHint" in script assert '"repositoryMaintenanceRetentionSource": "Retention-Quelle: {job}"' in de assert '"repositoryMaintenanceArchiveFilter": "Archivfilter: {filter}"' in de - assert '"repositoryMaintenanceRetention": "Retention: {retention}"' in de + assert '"repositoryMaintenanceRetention": "Aufbewahrung"' in de assert '"repositoryMaintenanceRetentionSource": "Retention source: {job}"' in en assert '"repositoryMaintenanceArchiveFilter": "Archive filter: {filter}"' in en - assert '"repositoryMaintenanceRetention": "Retention: {retention}"' in en + assert '"repositoryMaintenanceRetention": "Retention"' in en def test_repository_information_has_a_background_refresh_loop() -> None: diff --git a/tests/test_report_mail_api.py b/tests/test_report_mail_api.py index 6dee394d..eb34ceff 100644 --- a/tests/test_report_mail_api.py +++ b/tests/test_report_mail_api.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id, write_job import json from datetime import datetime from pathlib import Path @@ -20,6 +21,8 @@ def _write_status(status_dir: Path, name: str, data: dict) -> None: + data = {**data, "job_id": job_id(data["backup_type"] + "_" + data["location"])} + write_job(status_dir.parent, data["backup_type"] + "_" + data["location"]) path = status_dir / name path.write_text(json.dumps(data), encoding="utf-8") @@ -29,7 +32,7 @@ def _write_job_meta(root: Path, key: str, *, name: str, backup_type: str, locati jobs_dir = root / "config" / "jobs" scripts_dir.mkdir(parents=True, exist_ok=True) jobs_dir.mkdir(parents=True, exist_ok=True) - (jobs_dir / f"{key}.json").write_text(json.dumps({ + (jobs_dir / f"{job_id(key)}.json").write_text(json.dumps(identified_job({ "schema_version": 3, "job_key": key, "name": name, @@ -37,10 +40,11 @@ def _write_job_meta(root: Path, key: str, *, name: str, backup_type: str, locati "location": location, "repository_key": f"repo_{key}", "source_paths": ["/mnt/user/appdata"], - }), encoding="utf-8") + })), encoding="utf-8") def _write_schedules(root: Path, schedules: dict) -> None: + schedules = {job_id(k): v for k, v in schedules.items()} config_dir = root / "config" config_dir.mkdir(parents=True, exist_ok=True) (config_dir / "schedules.json").write_text(json.dumps(schedules), encoding="utf-8") @@ -175,7 +179,7 @@ def test_weekly_report_job_details_show_repository_growth(tmp_path: Path): "repository_size": 2 * 1024 ** 3, }) - html = _build_html_report({"STATUS_DIR": str(status_dir)}, now=REPORT_NOW) + html = _build_html_report({"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(status_dir)}, now=REPORT_NOW) assert "Growth 7d" in html assert "+1.0 GB" in html @@ -292,12 +296,12 @@ def test_weekly_report_sorts_jobs_by_location(tmp_path: Path): "status": "success", }) - html = _build_html_report({"STATUS_DIR": str(status_dir)}, now=REPORT_NOW) + html = _build_html_report({"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(status_dir)}, now=REPORT_NOW) assert html.index(">Local<") < html.index(">USB<") assert html.index(">USB<") < html.index(">Storagebox<") - assert html.index("Photos - Local") < html.index("Flash - USB") - assert html.index("Flash - USB") < html.index("Appdata - Storagebox") + assert html.index(">photos") < html.index(">flash") + assert html.index(">flash") < html.index(">appdata") def test_weekly_report_ignores_non_error_log_hints(tmp_path: Path): @@ -318,7 +322,7 @@ def test_weekly_report_ignores_non_error_log_hints(tmp_path: Path): "log_file": str(log_file), }) - html = _build_html_report({"STATUS_DIR": str(status_dir)}, now=REPORT_NOW) + html = _build_html_report({"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(status_dir)}, now=REPORT_NOW) assert "Log Details" not in html assert "Kein Mail-Versand" not in html diff --git a/tests/test_reports_api.py b/tests/test_reports_api.py index 75721afc..76af7087 100644 --- a/tests/test_reports_api.py +++ b/tests/test_reports_api.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id, write_job import json import sys from pathlib import Path @@ -23,6 +24,7 @@ def _write_status(path: Path, **overrides) -> None: "files_count": 10, } payload.update(overrides) + write_job(path.parent.parent, payload["backup_type"] + "_" + payload["location"]) path.write_text(json.dumps(payload), encoding="utf-8") @@ -30,17 +32,18 @@ def test_reports_parse_status_files_for_multi_underscore_job_keys(tmp_path: Path status_dir = tmp_path / "status" status_dir.mkdir() _write_status( - status_dir / "2026-08-28_09-00-00_borg_backup_taeglich_backuppf1_local.status" + status_dir / "2026-08-28_09-00-00_borg_backup_taeglich_backuppf1_local.status", + job_id=job_id("borg_backup_taeglich_backuppf1_local"), backup_type="borg_backup_taeglich_backuppf1", location="local" ) - config = {"STATUS_DIR": str(status_dir)} + config = {"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(status_dir)} jobs = get_report_jobs(config) - assert [job["key"] for job in jobs] == ["borg_backup_taeglich_backuppf1_local"] + assert [job["key"] for job in jobs] == [job_id("borg_backup_taeglich_backuppf1_local")] assert jobs[0]["backup_type"] == "borg_backup_taeglich_backuppf1" assert jobs[0]["location"] == "local" - data = get_report_data(config, "borg_backup_taeglich_backuppf1_local") + data = get_report_data(config, job_id("borg_backup_taeglich_backuppf1_local")) assert data["run_count"] == 1 assert data["success_count"] == 1 assert data["monthly_status"] == [ @@ -52,16 +55,17 @@ def test_reports_parse_smb_status_files(tmp_path: Path) -> None: status_dir = tmp_path / "status" status_dir.mkdir() _write_status( - status_dir / "2026-08-28_09-00-00_methusalix_backup_taeglich_smb.status" + status_dir / "2026-08-28_09-00-00_methusalix_backup_taeglich_smb.status", + job_id=job_id("methusalix_backup_taeglich_smb"), backup_type="methusalix_backup_taeglich", location="smb" ) - config = {"STATUS_DIR": str(status_dir)} + config = {"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(status_dir)} jobs = get_report_jobs(config) - assert [job["key"] for job in jobs] == ["methusalix_backup_taeglich_smb"] + assert [job["key"] for job in jobs] == [job_id("methusalix_backup_taeglich_smb")] assert jobs[0]["backup_type"] == "methusalix_backup_taeglich" assert jobs[0]["location"] == "smb" - data = get_report_data(config, "methusalix_backup_taeglich_smb") + data = get_report_data(config, job_id("methusalix_backup_taeglich_smb")) assert data["run_count"] == 1 assert data["success_count"] == 1 diff --git a/tests/test_repository_archive_browser.py b/tests/test_repository_archive_browser.py index b2d89198..cecfd278 100644 --- a/tests/test_repository_archive_browser.py +++ b/tests/test_repository_archive_browser.py @@ -168,3 +168,30 @@ def test_repository_archive_browser_frontend_is_read_only_and_localized(): assert ' None: "storage.repositoryPassphrase", "storage.repositoryKeyExport", "wizard.jobName", - "wizard.typeId", + "wizard.archivePrefix", "wizard.sourcePaths", "wizard.storageTarget", "wizard.repositorySelect", diff --git a/tests/test_required_source_paths.py b/tests/test_required_source_paths.py index 14100e27..1144467f 100644 --- a/tests/test_required_source_paths.py +++ b/tests/test_required_source_paths.py @@ -1,3 +1,4 @@ +from job_fixtures import job_id import json import logging from pathlib import Path @@ -22,6 +23,7 @@ def _config(tmp_path: Path, source_paths: list[Path]) -> BackupJobConfig: return BackupJobConfig( job_name="Required sources", + job_id=job_id("data_local"), backup_type="data", backup_location="local", lock_file=tmp_path / "job.lock", diff --git a/tests/test_restore_archive_filter.py b/tests/test_restore_archive_filter.py index ca340f18..f68ec83a 100644 --- a/tests/test_restore_archive_filter.py +++ b/tests/test_restore_archive_filter.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id from pathlib import Path from types import SimpleNamespace import json @@ -34,7 +35,7 @@ def test_browse_restore_filters_archives_by_job_prefix_history(tmp_path: Path, m "storage_key": "local", "storage": {}, "job": { - "job_key": "testdaten_local", + "job_key": job_id("testdaten_local"), "job_id": job_id("testdaten_local"), "archive_prefix": "testdaten-backup", "backup_type": "testdaten", "archive_prefixes": ["oldtestdaten-backup"], }, @@ -116,9 +117,9 @@ def test_save_job_preserves_previous_archive_prefixes(tmp_path: Path, monkeypatc scripts_dir = tmp_path / "scripts" jobs_dir = tmp_path / "config" / "jobs" jobs_dir.mkdir(parents=True) - (jobs_dir / "oldtype_local.json").write_text(json.dumps({ + (jobs_dir / (job_id("oldtype_local") + ".json")).write_text(json.dumps({ "schema_version": 2, - "job_key": "oldtype_local", + "job_key": job_id("oldtype_local"), "job_id": job_id("oldtype_local"), "archive_prefix": "oldtype-backup", "name": "Old type", "backup_type": "oldtype", "archive_prefixes": ["oldertype-backup"], @@ -139,14 +140,16 @@ def fake_transaction(config, metadata_path, metadata, repository_key, job_key, * monkeypatch.setattr("repositories_api.save_job_repository_transaction", fake_transaction) wizard_api.save_job({ - "existing_job_key": "oldtype_local", + "existing_job_key": job_id("oldtype_local"), + "archive_prefix": "newtype-backup", "type_id": "newtype", "location": "local", "repository_key": "repo-shared", "source_paths": ["/mnt/user/appdata"], }, scripts_dir, tmp_path, {"BACKUP_SCRIPTS_DIR": str(tmp_path)}) - assert captured["job_key"] == "newtype_local" + assert captured["job_key"] == job_id("oldtype_local") + assert "backup_type" not in captured["metadata"] assert captured["metadata"]["archive_prefixes"] == [ "newtype-backup", "oldtype-backup", diff --git a/tests/test_restore_browse_state.py b/tests/test_restore_browse_state.py new file mode 100644 index 00000000..3038706f --- /dev/null +++ b/tests/test_restore_browse_state.py @@ -0,0 +1,15 @@ +"""Run the real restore-page JavaScript against controlled API responses.""" + +from pathlib import Path +import shutil +import subprocess + +import pytest + + +def test_restore_browse_state(): + node = shutil.which("node") + if not node: + pytest.skip("Node.js is required for restore UI tests") + subprocess.run([node, "tests/restore_browse_state.cjs"], + cwd=Path(__file__).resolve().parents[1], check=True) diff --git a/tests/test_restore_path_safety.py b/tests/test_restore_path_safety.py index a14738c4..2cdc4fa0 100644 --- a/tests/test_restore_path_safety.py +++ b/tests/test_restore_path_safety.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id from pathlib import Path import io import sys @@ -88,7 +89,7 @@ def wait(self): with pytest.raises(ValueError, match="outside"): restore_api.start_restore( cfg, - "appdata_local", + job_id("appdata_local"), "archive-1", "foo", str(target), diff --git a/tests/test_restore_test_runner_profiles.py b/tests/test_restore_test_runner_profiles.py index d88c314a..f6fe7fcb 100644 --- a/tests/test_restore_test_runner_profiles.py +++ b/tests/test_restore_test_runner_profiles.py @@ -1,5 +1,7 @@ +from job_fixtures import identified_job, job_id import importlib.util import json +import pytest from pathlib import Path from types import SimpleNamespace @@ -43,16 +45,16 @@ def test_restore_runner_discovers_usb_profile_repository(tmp_path, monkeypatch) monkeypatch.setattr(runner, "SCRIPT_DIR", script_dir) monkeypatch.setenv("BORG_UI_DATA_ROOT", str(tmp_path / "runtime")) - (jobs_dir / "testjob_usb.json").write_text( - json.dumps({ + (jobs_dir / (job_id('testjob_usb') + ".json")).write_text( + json.dumps(identified_job({ "schema_version": 2, "enabled": True, "runner": "scriptless-wizard-runner", - "job_key": "testjob_usb", + "job_key": job_id('testjob_usb'), "backup_type": "testjob", "location": "usb", "repository_key": "repo_testjob_usb", - }), + })), encoding="utf-8", ) (config_dir / "storages.json").write_text(json.dumps({ @@ -86,8 +88,8 @@ def test_restore_runner_discovers_usb_profile_repository(tmp_path, monkeypatch) "job_key", "type", "location", "path", "encryption", "passphrase_file", "profile_key", "mount_before_run", "unmount_after_run", )} for row in repos] == [{ - "job_key": "testjob_usb", - "type": "testjob", + "job_key": job_id('testjob_usb'), + "type": "", "location": "usb", "path": "/mnt/disks/WCJ54TRQ/borg-backup-testjob", "encryption": "none", @@ -196,16 +198,16 @@ def test_restore_runner_discovers_smb_profile_repository(tmp_path, monkeypatch) monkeypatch.setattr(runner, "SCRIPT_DIR", script_dir) monkeypatch.setenv("BORG_UI_DATA_ROOT", str(tmp_path / "runtime")) - (jobs_dir / "photos_smb.json").write_text( - json.dumps({ + (jobs_dir / (job_id('photos_smb') + ".json")).write_text( + json.dumps(identified_job({ "schema_version": 2, "enabled": True, "runner": "scriptless-wizard-runner", - "job_key": "photos_smb", + "job_key": job_id('photos_smb'), "backup_type": "photos", "location": "smb", "repository_key": "repo_photos_smb", - }), + })), encoding="utf-8", ) (config_dir / "storages.json").write_text(json.dumps({ @@ -269,3 +271,60 @@ def fake_run(command, **_kwargs): assert mounted is True assert error == "" + + +@pytest.mark.parametrize("size_gb,file_count,directory_count,expected,chunked", [ + (600, 10_000, 5_000, 500, True), + (600, 100_000, 5_000, 1_000, True), + (500, 20_000, 25_000, 1_000, True), + (500, 30, 3_000, 2, True), + (499, 10_000, 5_000, 10_000, False), +]) +def test_restore_probe_uses_size_and_regular_file_limits(tmp_path, monkeypatch, size_gb, file_count, directory_count, expected, chunked): + runner = _load_restore_runner() + instance = _restore_test_instance(runner, monkeypatch) + instance.test_level = 2 + instance.min_coverage = 5 + instance.max_entries = 1000 + instance.dryrun_max_files = 1000 + instance.dryrun_chunk = 100 + instance.dryrun_timeout = 0 + instance.sample_size = 5 + instance.full_dryrun_max_archive_gb = 500 + # An old configuration entry must no longer force sampling below the size threshold. + instance.conf = {"RESTORE_TEST_FORCE_CHUNK_TYPES": "photos,vms"} + rows = [{"type": "d", "path": f"folder-{i}"} for i in range(directory_count)] + rows += [{"type": "-", "path": f"folder-{i % 10}/file-{i}"} for i in range(file_count)] + rows += [{"type": "l", "path": "symbolic-link"}] + calls, results = [], [] + def fake_borg(args, _env, timeout=None): + if args[:3] == ["list", "--short", "--last"]: + output = "archive-1\n" + elif args[:2] == ["info", "--json"]: + output = json.dumps({"archives": [{"stats": {"original_size": size_gb * 1024**3}}]}) + elif args[:2] == ["list", "--json-lines"]: + output = "\n".join(json.dumps(row) for row in rows) + elif args[:2] == ["extract", "--dry-run"]: + calls.append(args) + output = "" + else: + raise AssertionError(args) + return SimpleNamespace(returncode=0, stdout=output, stderr="") + monkeypatch.setattr(instance, "_env", lambda *_args: {}) + monkeypatch.setattr(instance, "_borg", fake_borg) + monkeypatch.setattr(instance, "_write", lambda *args, **kwargs: results.append((args, kwargs))) + result = instance.test_repo({"job_key": job_id("photos_local"), "type": "photos", "name": "Renamed job", + "location": "local", "path": str(tmp_path), "encryption": "none", "passphrase_file": ""}) + assert result == 0 + args, details = results[-1] + assert args[4:7] == (expected, 0, expected) + assert details["test_coverage_pct"] == round(expected / file_count * 100, 1) + assert args[9]["files_count"] == file_count + if chunked: + selected = [path for call in calls for path in call[3:]] + assert len(selected) == len(set(selected)) == expected + assert all("/file-" in path for path in selected) + assert all(0 < len(call[3:]) <= 100 for call in calls) + assert len(args[10]) == expected + else: + assert len(calls) == 1 and len(calls[0]) == 3 diff --git a/tests/test_restore_tests_policy_contract.py b/tests/test_restore_tests_policy_contract.py index 7f18a3fd..f15bcb4a 100644 --- a/tests/test_restore_tests_policy_contract.py +++ b/tests/test_restore_tests_policy_contract.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id import json from datetime import datetime, timedelta from pathlib import Path @@ -15,10 +16,10 @@ from api.restore_tests_api import list_restore_test_plan, update_restore_test_policy -def _make_job(root: Path, key: str = "flash_local") -> None: +def _make_job(root: Path, key: str = job_id('flash_local')) -> None: jobs_dir = root / "config" / "jobs" jobs_dir.mkdir(parents=True, exist_ok=True) - data = { + data = identified_job({ "key": key, "job_key": key, "name": "Flash", @@ -28,28 +29,29 @@ def _make_job(root: Path, key: str = "flash_local") -> None: "repository_key": "repo_flash_local", "enabled": True, "restore_test_policy": {"mode": "scheduled", "interval_days": 30, "level": 2}, - } - (jobs_dir / f"{key}.json").write_text(json.dumps(data, indent=2), encoding="utf-8") + }) + data["key"] = data["job_key"] + (jobs_dir / f"{data['job_key']}.json").write_text(json.dumps(data, indent=2), encoding="utf-8") def test_policy_contract_rejects_bad_interval(tmp_path: Path): _make_job(tmp_path) cfg = {"BACKUP_SCRIPTS_DIR": str(tmp_path), "RESTORE_TEST_INTERVAL_DAYS": "30"} with pytest.raises(ValueError, match="interval_days"): - update_restore_test_policy(cfg, "flash_local", {"mode": "scheduled", "interval_days": 0, "level": 2}) + update_restore_test_policy(cfg, job_id('flash_local'), {"mode": "scheduled", "interval_days": 0, "level": 2}) def test_policy_contract_rejects_bad_level(tmp_path: Path): _make_job(tmp_path) cfg = {"BACKUP_SCRIPTS_DIR": str(tmp_path), "RESTORE_TEST_INTERVAL_DAYS": "30"} with pytest.raises(ValueError, match="level"): - update_restore_test_policy(cfg, "flash_local", {"mode": "scheduled", "interval_days": 7, "level": 9}) + update_restore_test_policy(cfg, job_id('flash_local'), {"mode": "scheduled", "interval_days": 7, "level": 9}) def test_policy_contract_accepts_valid_payload(tmp_path: Path): _make_job(tmp_path) cfg = {"BACKUP_SCRIPTS_DIR": str(tmp_path), "RESTORE_TEST_INTERVAL_DAYS": "30"} - out = update_restore_test_policy(cfg, "flash_local", {"mode": "manual_only", "interval_days": 7, "level": 1}) + out = update_restore_test_policy(cfg, job_id('flash_local'), {"mode": "manual_only", "interval_days": 7, "level": 1}) assert out["saved"] is True assert out["policy"]["mode"] == "manual_only" assert out["policy"]["interval_days"] == 7 @@ -61,7 +63,8 @@ def test_scheduled_plan_keeps_failed_manual_result_due(tmp_path: Path): restore_dir = tmp_path / "restore-status" restore_dir.mkdir(parents=True) recent = datetime.now() - timedelta(hours=2) - (restore_dir / "flash_local.test").write_text(json.dumps({ + (restore_dir / (job_id('flash_local') + ".test")).write_text(json.dumps({ + "job_id": job_id("flash_local"), "test_result": "failed", "test_date": recent.strftime("%Y-%m-%d %H:%M:%S"), }), encoding="utf-8") @@ -75,7 +78,7 @@ def test_scheduled_plan_keeps_failed_manual_result_due(tmp_path: Path): plan = list_restore_test_plan(cfg) row = plan["jobs"][0] - assert row["job_key"] == "flash_local" + assert row["job_key"] == job_id('flash_local') assert row["last_test_result"] == "failed" assert row["is_overdue"] is True assert row["next_due_at"] == recent.strftime("%Y-%m-%d %H:%M:%S") diff --git a/tests/test_settings_transfer_repository_model.py b/tests/test_settings_transfer_repository_model.py index e0203300..37a20cb8 100644 --- a/tests/test_settings_transfer_repository_model.py +++ b/tests/test_settings_transfer_repository_model.py @@ -1,4 +1,5 @@ from __future__ import annotations +from job_fixtures import identified_job, job_id import json import sys @@ -23,15 +24,15 @@ def _canonical_source(root: Path) -> tuple[dict, Path]: secret.write_text("secret\n", encoding="utf-8") jobs_dir = root / "config" / "jobs" jobs_dir.mkdir(parents=True) - (jobs_dir / "appdata_local.json").write_text(json.dumps({ + (jobs_dir / (job_id('appdata_local') + ".json")).write_text(json.dumps(identified_job({ "schema_version": 3, - "job_key": "appdata_local", + "job_key": job_id('appdata_local'), "name": "Appdata", "backup_type": "appdata", "location": "local", "repository_key": "repo_appdata", "source_paths": ["/mnt/user/appdata"], - }) + "\n", encoding="utf-8") + })) + "\n", encoding="utf-8") write_storage_store(config, {"storages": [{ "storage_key": "storage_local", "display_name": "Local", @@ -58,7 +59,7 @@ def test_job_export_contains_canonical_repository_inventory(tmp_path: Path): result = export_jobs_bundle(config) bundle = result["bundle"] - assert bundle["format"] == "bbui-job-bundle-v2" + assert bundle["format"] == "bbui-job-bundle-v3" assert bundle["jobs"][0]["repository_key"] == "repo_appdata" assert "repo" not in bundle["jobs"][0] assert bundle["repositories"][0]["storage_key"] == "storage_local" @@ -96,15 +97,15 @@ def test_job_import_restores_repository_and_storage_before_job(tmp_path: Path): assert result["imported_count"] == 1 assert result["repository_inventory"] == {"repositories": 1, "storages": 1} imported_job = json.loads( - (tmp_path / "target" / "config" / "jobs" / "appdata_local.json").read_text(encoding="utf-8") + (tmp_path / "target" / "config" / "jobs" / (job_id('appdata_local') + ".json")).read_text(encoding="utf-8") ) - assert imported_job["schema_version"] == 3 + assert imported_job["schema_version"] == 5 assert imported_job["source_paths"] == ["/mnt/user/appdata"] assert read_repository_store(target_config)["repositories"][0]["repository_key"] == "repo_appdata" assert read_storage_store(target_config)["storages"][0]["storage_key"] == "storage_local" context = resolve_job_repository_context( target_config, - "appdata_local", + job_id('appdata_local'), require_passphrase_file=False, ) assert context["repository_path"] == "/mnt/backup/borg-backup-appdata" diff --git a/tests/test_smb_profiles_api.py b/tests/test_smb_profiles_api.py index 93f07c48..c9e7e681 100644 --- a/tests/test_smb_profiles_api.py +++ b/tests/test_smb_profiles_api.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id from pathlib import Path import json import sys @@ -213,10 +214,10 @@ def test_smb_profile_usage_blocks_delete_when_job_references_profile(tmp_path: P data_root = tmp_path / "data" meta_dir = data_root / "config" / "jobs" meta_dir.mkdir(parents=True) - (meta_dir / "job1.json").write_text( - json.dumps({ + (meta_dir / (job_id('job1') + ".json")).write_text( + json.dumps({"job_id": job_id('job1'), "archive_prefix": "job1-backup", "schema_version": 2, - "job_key": "job1", + "job_key": job_id('job1'), "name": "Job 1", "location": "smb", "repository_key": "repo_job1", diff --git a/tests/test_status_snapshots.py b/tests/test_status_snapshots.py index 3a665fc7..ea4047e6 100644 --- a/tests/test_status_snapshots.py +++ b/tests/test_status_snapshots.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id, write_job import json from pathlib import Path from types import SimpleNamespace @@ -25,22 +26,22 @@ def test_weekly_snapshots_import_legacy_once_and_write_only_canonical(tmp_path: legacy_snapshot_file = status_dir / "weekly-snapshots.json" legacy_snapshot_file.parent.mkdir(parents=True) legacy_snapshot_file.write_text( - json.dumps({"appdata_local": [{"week": "2026-06-22", "size": 100}]}), + json.dumps({job_id('appdata_local'): [{"week": "2026-06-22", "size": 100}]}), encoding="utf-8", ) _import_legacy_snapshot_if_needed(snapshot_file, legacy_snapshot_file) _auto_write_weekly_snapshot( snapshot_file, - {"appdata_local": SimpleNamespace(repository_size=200)}, + {job_id('appdata_local'): SimpleNamespace(repository_size=200)}, force_write=True, ) canonical = json.loads(snapshot_file.read_text(encoding="utf-8")) legacy = json.loads(legacy_snapshot_file.read_text(encoding="utf-8")) - assert canonical["appdata_local"][-1]["size"] == 200 - assert legacy == {"appdata_local": [{"week": "2026-06-22", "size": 100}]} + assert canonical[job_id('appdata_local')][-1]["size"] == 200 + assert legacy == {job_id('appdata_local'): [{"week": "2026-06-22", "size": 100}]} def test_weekly_snapshot_does_not_create_path_below_unmounted_user_share(monkeypatch): @@ -52,7 +53,7 @@ def test_weekly_snapshot_does_not_create_path_below_unmounted_user_share(monkeyp ): _auto_write_weekly_snapshot( Path("/mnt/user/borg_backup_ui/weekly-snapshots.json"), - {"appdata_local": SimpleNamespace(repository_size=200)}, + {job_id('appdata_local'): SimpleNamespace(repository_size=200)}, force_write=True, ) @@ -62,7 +63,8 @@ def test_weekly_snapshot_does_not_create_path_below_unmounted_user_share(monkeyp def _write_status(status_dir: Path, name: str, payload: dict) -> None: status_dir.mkdir(parents=True, exist_ok=True) - base = { + write_job(status_dir.parent, "appdata_local") + base = {"job_id": job_id('appdata_local'), "backup_type": "appdata", "location": "local", "timestamp": "2026-07-01 10:00:00", @@ -87,10 +89,10 @@ def test_dashboard_growth_falls_back_to_previous_status_when_snapshot_baseline_m "repository_size": 150, }) - data = get_status_data({"STATUS_DIR": str(status_dir), "SNAPSHOT_FILE": str(snapshot_file)}) + data = get_status_data({"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(status_dir), "SNAPSHOT_FILE": str(snapshot_file)}) row = data["backups"][0] - assert row["key"] == "appdata_local" + assert row["key"] == job_id('appdata_local') assert row["growth_bytes"] == 50 assert row["growth_formatted"] == "+50 B" @@ -99,7 +101,7 @@ def test_dashboard_growth_prefers_weekly_snapshot_over_previous_status(tmp_path: status_dir = tmp_path / "status" snapshot_file = tmp_path / "weekly-snapshots.json" snapshot_file.write_text( - json.dumps({"appdata_local": [ + json.dumps({job_id('appdata_local'): [ {"week": "2026-06-22", "size": 120}, {"week": "2026-06-29", "size": 140}, ]}), @@ -114,7 +116,7 @@ def test_dashboard_growth_prefers_weekly_snapshot_over_previous_status(tmp_path: "repository_size": 150, }) - data = get_status_data({"STATUS_DIR": str(status_dir), "SNAPSHOT_FILE": str(snapshot_file)}) + data = get_status_data({"BACKUP_SCRIPTS_DIR": str(tmp_path), "STATUS_DIR": str(status_dir), "SNAPSHOT_FILE": str(snapshot_file)}) row = data["backups"][0] assert row["growth_bytes"] == 10 @@ -122,8 +124,8 @@ def test_dashboard_growth_prefers_weekly_snapshot_over_previous_status(tmp_path: def test_dashboard_marks_missed_scheduled_backup_overdue(monkeypatch): - backups = [{ - "key": "appdata_local", + backups = [{"job_id": job_id('appdata_local'), + "key": job_id('appdata_local'), "backup_type": "appdata", "location": "local", "status": "success", @@ -131,10 +133,10 @@ def test_dashboard_marks_missed_scheduled_backup_overdue(monkeypatch): }] monkeypatch.setattr("schedule_api.get_schedules", lambda cfg: { - "appdata_local": {"enabled": True, "cron": "0 14 * * *"}, + job_id('appdata_local'): {"enabled": True, "cron": "0 14 * * *"}, }) monkeypatch.setattr("jobs_api.list_jobs", lambda cfg, ctx: [{ - "key": "appdata_local", + "key": job_id('appdata_local'), "name": "Appdata", "enabled": True, }]) @@ -152,8 +154,8 @@ def test_dashboard_marks_missed_scheduled_backup_overdue(monkeypatch): def test_dashboard_keeps_current_scheduled_backup_success(monkeypatch): - backups = [{ - "key": "appdata_local", + backups = [{"job_id": job_id('appdata_local'), + "key": job_id('appdata_local'), "backup_type": "appdata", "location": "local", "status": "success", @@ -161,10 +163,10 @@ def test_dashboard_keeps_current_scheduled_backup_success(monkeypatch): }] monkeypatch.setattr("schedule_api.get_schedules", lambda cfg: { - "appdata_local": {"enabled": True, "cron": "0 14 * * *"}, + job_id('appdata_local'): {"enabled": True, "cron": "0 14 * * *"}, }) monkeypatch.setattr("jobs_api.list_jobs", lambda cfg, ctx: [{ - "key": "appdata_local", + "key": job_id('appdata_local'), "name": "Appdata", "enabled": True, }]) diff --git a/tests/test_storage_profiles.py b/tests/test_storage_profiles.py index 7cd89c34..e2e2a73e 100644 --- a/tests/test_storage_profiles.py +++ b/tests/test_storage_profiles.py @@ -1,3 +1,5 @@ +from job_fixtures import identified_job, job_id +import json from pathlib import Path import sys @@ -26,9 +28,9 @@ def _write_storagebox_reference(data_root: Path) -> dict: config = {"BACKUP_SCRIPTS_DIR": str(data_root)} meta_dir = data_root / "config" / "jobs" meta_dir.mkdir(parents=True, exist_ok=True) - (meta_dir / "job1.json").write_text( - '{"schema_version":2,"job_key":"job1","name":"Job 1",' - '"location":"storagebox","repository_key":"repo_job1"}\n', + (meta_dir / (job_id("job1") + ".json")).write_text( + json.dumps(identified_job({"job_key": "job1", "name": "Job 1", "backup_type": "job1", + "location": "storagebox", "repository_key": "repo_job1"})), encoding="utf-8", ) write_storage_store(config, {"storages": [{ diff --git a/tests/test_storage_repository_manager.py b/tests/test_storage_repository_manager.py index 0087c810..9c78f59e 100644 --- a/tests/test_storage_repository_manager.py +++ b/tests/test_storage_repository_manager.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id import json import subprocess import sys @@ -103,12 +104,12 @@ def test_repository_maintenance_commands_use_repository_and_job_retention(tmp_pa config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs = tmp_path / "config" / "jobs" jobs.mkdir(parents=True) - (jobs / "photos_local.json").write_text(json.dumps({ - "job_key": "photos_local", + (jobs / (job_id('photos_local') + ".json")).write_text(json.dumps({"job_id": job_id('photos_local'), "archive_prefix": "photos-backup", + "job_key": job_id('photos_local'), "repository_key": "repo_photos", "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, }), encoding="utf-8") - repository = {"repository_key": "repo_photos", "used_by": ["photos_local"]} + repository = {"repository_key": "repo_photos", "used_by": [job_id('photos_local')]} manager = CheckManager() assert manager._repository_command(config, repository, "/mnt/backup/photos", "check", "quick") == [ @@ -139,7 +140,7 @@ def test_repository_prune_requires_explicit_retention_source_for_shared_reposito config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs = tmp_path / "config" / "jobs" jobs.mkdir(parents=True) - for job_key in ("photos_local", "appdata_local"): + for job_key in (job_id('photos_local'), job_id('appdata_local')): (jobs / f"{job_key}.json").write_text(json.dumps({ "job_key": job_key, "repository_key": "repo_shared", @@ -147,7 +148,7 @@ def test_repository_prune_requires_explicit_retention_source_for_shared_reposito }), encoding="utf-8") manager = CheckManager() - repository = {"repository_key": "repo_shared", "used_by": ["photos_local", "appdata_local"]} + repository = {"repository_key": "repo_shared", "used_by": [job_id('photos_local'), job_id('appdata_local')]} with pytest.raises(ValueError, match="select a retention source job"): manager._repository_command(config, repository, "/mnt/backup/shared", "prune", "quick") @@ -157,29 +158,29 @@ def test_repository_prune_uses_selected_job_retention_source(tmp_path: Path): config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs = tmp_path / "config" / "jobs" jobs.mkdir(parents=True) - (jobs / "photos_local.json").write_text(json.dumps({ - "job_key": "photos_local", + (jobs / (job_id('photos_local') + ".json")).write_text(json.dumps({"job_id": job_id('photos_local'), "archive_prefix": "photos-backup", + "job_key": job_id('photos_local'), "repository_key": "repo_shared", "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, }), encoding="utf-8") - (jobs / "appdata_local.json").write_text(json.dumps({ - "job_key": "appdata_local", + (jobs / (job_id('appdata_local') + ".json")).write_text(json.dumps({"job_id": job_id('appdata_local'), "archive_prefix": "Appdata-Config", + "job_key": job_id('appdata_local'), "repository_key": "repo_shared", "retention": {"daily": "14", "weekly": "8", "monthly": "3", "yearly": "1"}, }), encoding="utf-8") command = CheckManager()._repository_command( config, - {"repository_key": "repo_shared", "used_by": ["photos_local", "appdata_local"]}, + {"repository_key": "repo_shared", "used_by": [job_id('photos_local'), job_id('appdata_local')]}, "/mnt/backup/shared", "prune", "quick", - job_key="appdata_local", + job_key=job_id('appdata_local'), ) assert command == [ "borg", "prune", "--lock-wait", "30", "--list", "--progress", - "--glob-archives", "appdata-backup-*", + "--glob-archives", "Appdata-Config-*", "--keep-daily", "14", "--keep-weekly", "8", "--keep-monthly", "3", "--keep-yearly", "1", "/mnt/backup/shared", ] @@ -189,13 +190,13 @@ def test_repository_prune_rejects_retention_source_from_other_repository(tmp_pat config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs = tmp_path / "config" / "jobs" jobs.mkdir(parents=True) - (jobs / "photos_local.json").write_text(json.dumps({ - "job_key": "photos_local", + (jobs / (job_id('photos_local') + ".json")).write_text(json.dumps({"job_id": job_id('photos_local'), "archive_prefix": "photos-backup", + "job_key": job_id('photos_local'), "repository_key": "repo_photos", "retention": {"daily": "7"}, }), encoding="utf-8") - (jobs / "appdata_local.json").write_text(json.dumps({ - "job_key": "appdata_local", + (jobs / (job_id('appdata_local') + ".json")).write_text(json.dumps({"job_id": job_id('appdata_local'), "archive_prefix": "appdata-backup", + "job_key": job_id('appdata_local'), "repository_key": "repo_appdata", "retention": {"daily": "14"}, }), encoding="utf-8") @@ -203,11 +204,11 @@ def test_repository_prune_rejects_retention_source_from_other_repository(tmp_pat with pytest.raises(ValueError, match="does not use this repository"): CheckManager()._repository_command( config, - {"repository_key": "repo_photos", "used_by": ["photos_local"]}, + {"repository_key": "repo_photos", "used_by": [job_id('photos_local')]}, "/mnt/backup/photos", "prune", "quick", - job_key="appdata_local", + job_key=job_id('appdata_local'), ) @@ -541,16 +542,16 @@ def test_repository_lifecycle_blocks_live_job_reference(tmp_path: Path): config, repository_path, _secret = _write_lifecycle_repository(tmp_path) jobs = tmp_path / "config" / "jobs" jobs.mkdir(parents=True) - (jobs / "photos_local.json").write_text(json.dumps({ + (jobs / (job_id('photos_local') + ".json")).write_text(json.dumps({"job_id": job_id('photos_local'), "archive_prefix": "photos-backup", "schema_version": 2, - "job_key": "photos_local", + "job_key": job_id('photos_local'), "repository_key": "repo_photos", }), encoding="utf-8") preview = prepare_repository_lifecycle(config, "repo_photos", "remove") assert preview["allowed"] is False - assert preview["job_keys"] == ["photos_local"] + assert preview["job_keys"] == [job_id('photos_local')] assert "jobs_linked" in preview["blockers"] with pytest.raises(RepositoryLifecycleConflict, match="jobs or operations"): apply_repository_lifecycle(config, { @@ -562,9 +563,9 @@ def test_repository_lifecycle_blocks_live_job_reference(tmp_path: Path): def test_deleted_job_is_unlinked_from_repository_inventory(tmp_path: Path): - config, _repository_path, _secret = _write_lifecycle_repository(tmp_path, used_by=["photos_local"]) + config, _repository_path, _secret = _write_lifecycle_repository(tmp_path, used_by=[job_id('photos_local')]) - unlink_job_from_repositories(config, "photos_local") + unlink_job_from_repositories(config, job_id('photos_local')) repository = read_repository_store(config)["repositories"][0] assert repository["used_by"] == [] diff --git a/tests/test_usb_mount_preflight.py b/tests/test_usb_mount_preflight.py new file mode 100644 index 00000000..cc74f80b --- /dev/null +++ b/tests/test_usb_mount_preflight.py @@ -0,0 +1,181 @@ +"""Reject unavailable USB targets before runtime changes or Borg access (#502).""" + +import errno +import json +import logging +from pathlib import Path +from types import SimpleNamespace +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +for folder in (ROOT, ROOT / 'api', ROOT / 'runtime', ROOT / 'runtime/lib'): + sys.path.insert(0, str(folder)) + +from job_fixtures import job_id +from lib import backup_job +from lib.backup_job import BackupJob, BackupJobConfig, UsbMountAccessError, USB_MOUNT_ACCESS_FAILED + + +@pytest.fixture +def job(tmp_path, monkeypatch): + cfg = BackupJobConfig( + job_id=job_id('usb-preflight'), job_name='USB test', backup_type='', backup_location='usb', + lock_file=tmp_path / 'job.lock', log_dir=tmp_path, log_file=tmp_path / 'backup.log', + backup_paths=[tmp_path / 'source'], borg_cache_dir=tmp_path / 'cache', + borg_repo=str(tmp_path / 'drive/repository'), date_tag='2026-09-10_15-00-01', + status_dir=tmp_path / 'status', borg_check_flag_file=tmp_path / 'old-check', + ) + cfg.backup_paths[0].mkdir() + cfg.borg_check_flag_file.touch() + instance = BackupJob(cfg) + instance.notifications = [] + instance.events = [] + monkeypatch.setattr(instance, '_send_notification_event', lambda *args: instance.notifications.append(args)) + import lifecycle_log + monkeypatch.setattr(lifecycle_log, 'emit_lifecycle', lambda *_args, **kwargs: instance.events.append(kwargs)) + monkeypatch.setattr(instance, '_refresh_unraid_dashboard_widget_cache', lambda *_args: None) + return instance + + +def _reject_borg(monkeypatch): + def forbidden(*_args, **_kwargs): + pytest.fail('USB rejection must not execute any Borg command, including completion info') + monkeypatch.setattr(backup_job.subprocess, 'run', forbidden) + + +def _status(job): + return json.loads(next(job.config.status_dir.glob('*.status')).read_text()) + + +@pytest.mark.parametrize('kind,reason', [ + ('missing', 'usb_not_mounted'), ('directory', 'usb_not_mounted'), + ('file', 'usb_not_mounted'), ('readonly', 'usb_not_writable'), +]) +def test_unmounted_or_readonly_usb_is_skipped_without_target_writes(job, tmp_path, monkeypatch, kind, reason): + mount = tmp_path / 'drive' + if kind in {'directory', 'readonly'}: + mount.mkdir() + elif kind == 'file': + mount.write_text('not a mount') + if kind == 'readonly': + monkeypatch.setattr(Path, 'is_mount', lambda path: path == mount) + access = backup_job.os.access + monkeypatch.setattr(backup_job.os, 'access', lambda path, mode: False if path == mount else access(path, mode)) + _reject_borg(monkeypatch) + with pytest.raises(SystemExit) as stopped: + with job: + job.check_usb_mount(mount) + pytest.fail('Docker/VM stop and backup must not be reached') + assert stopped.value.code == 0 + data = _status(job) + assert (data['status'], data['exit_code'], data['skip_reason_code']) == ('skipped', 0, reason) + assert not job.config.lock_file.exists() + assert len(job.notifications) == 1 and job.notifications[0][0] == 'backup_skipped' + assert not (mount / 'repository').exists() + if kind in {'directory', 'readonly'}: + assert list(mount.iterdir()) == [] + + +@pytest.mark.parametrize('stage', ['stat', 'mount', 'access']) +@pytest.mark.parametrize('error_number', [errno.EIO, errno.ENODEV]) +def test_mount_access_errors_have_specific_failure_and_never_query_borg( + job, tmp_path, monkeypatch, caplog, stage, error_number, +): + mount = tmp_path / 'drive' + mount.mkdir() + original_stat = Path.stat + def failing_stat(path, *args, **kwargs): + if path == mount and stage == 'stat': + raise OSError(error_number, 'simulated USB access failure', str(mount)) + return original_stat(path, *args, **kwargs) + def mounted(path): + if path == mount and stage == 'mount': + raise OSError(error_number, 'simulated USB access failure', str(mount)) + return path == mount + original_access = backup_job.os.access + def access(path, mode): + if path == mount and stage == 'access': + raise OSError(error_number, 'simulated USB access failure', str(mount)) + return original_access(path, mode) + monkeypatch.setattr(Path, 'stat', failing_stat) + monkeypatch.setattr(Path, 'is_mount', mounted) + monkeypatch.setattr(backup_job.os, 'access', access) + _reject_borg(monkeypatch) + with caplog.at_level(logging.INFO), pytest.raises(UsbMountAccessError): + with job: + job.check_usb_mount(mount) + pytest.fail('Docker/VM stop and backup must not be reached') + data = _status(job) + assert (data['status'], data['exit_code']) == ('error', 2) + assert data['failure_code'] == USB_MOUNT_ACCESS_FAILED + assert 'USB drive is not accessible' in data['error_message'] + assert 'Backup was not started' in data['error_message'] + assert str(mount) in data['error_message'] and f'[Errno {error_number}]' in data['error_message'] + assert data['repository_check_status'] == 'unknown' and data['repository_size'] == 0 + assert not job.config.lock_file.exists() + assert len(job.notifications) == 1 and job.notifications[0][0] == 'backup_failed' + assert 'USB drive is not accessible' in job.notifications[0][2] + assert job.events[0]['failure_code'] == USB_MOUNT_ACCESS_FAILED + assert 'USB preflight' in caplog.text + assert 'Borg backup failed' not in caplog.text + assert list(mount.iterdir()) == [] + + +def test_mounted_accessible_usb_passes_without_creating_probe_files(job, tmp_path, monkeypatch): + mount = tmp_path / 'drive' + mount.mkdir() + monkeypatch.setattr(Path, 'is_mount', lambda path: path == mount) + _reject_borg(monkeypatch) + assert job.check_usb_mount(mount) is None + assert not job.config.status_dir.exists() + assert list(mount.iterdir()) == [] + assert job.notifications == [] + + +def test_real_runner_returns_usb_failure_and_releases_resources(job, tmp_path, monkeypatch): + import job_control + import lifecycle_log + import wizard_runner + from lib.borg_runner import BorgRunner + + mount = tmp_path / 'drive' + mount.mkdir() + phases, resources, actions = [], [], [] + control = SimpleNamespace(update_phase=lambda phase, **kwargs: phases.append((phase, kwargs)), + is_cancel_requested=lambda: False) + monkeypatch.setattr(job_control, 'JobControl', lambda *_args: control) + monkeypatch.setattr(wizard_runner, 'ResourceLockSet', lambda **_kwargs: SimpleNamespace( + acquire=lambda _: (True, ''), release=lambda: resources.append('released'))) + monkeypatch.setattr(wizard_runner, '_ensure_borg_available', lambda: 'borg') + monkeypatch.setattr(wizard_runner, '_setup_stdout_logging', lambda: None) + monkeypatch.setattr(wizard_runner, '_setup_full_logging', lambda _: None) + monkeypatch.setattr(wizard_runner, '_ensure_runtime_import_paths', lambda _: None) + monkeypatch.setattr(wizard_runner, '_load_env_from_job', lambda *_args: ( + {'BACKUP_SCRIPTS_DIR': str(tmp_path), 'ABORT_ON_PARITY_CHECK': 'false'}, + {'job_id': job.config.job_id, 'location': 'usb', 'archive_prefix': 'test-backup', + '_resolved_storage': {'mount_path': str(mount)}, + 'docker_control': {'mode': 'all'}, 'vm_control': {'mode': 'all'}})) + monkeypatch.setattr(BackupJobConfig, 'from_config', lambda _: job.config) + monkeypatch.setattr(BackupJob, '_send_notification_event', lambda *_args: None) + monkeypatch.setattr(BackupJob, '_refresh_unraid_dashboard_widget_cache', lambda *_args: None) + monkeypatch.setattr(BackupJob, 'stop_docker', lambda *_args, **_kwargs: actions.append('stop Docker')) + monkeypatch.setattr(BackupJob, 'shutdown_vms', lambda *_args, **_kwargs: actions.append('stop VMs')) + monkeypatch.setattr(BorgRunner, 'create', lambda *_args, **_kwargs: actions.append('create') or 0) + monkeypatch.setattr(BorgRunner, 'maintenance', lambda *_args, **_kwargs: actions.append('maintenance') or 0) + monkeypatch.setattr(lifecycle_log, 'emit_lifecycle', lambda *_args, **_kwargs: None) + for name, value in {'BORG_UI_JOB_KEY': job.config.job_id, 'BORG_UI_BORG_SCRIPTS_DIR': str(ROOT / 'runtime/scripts'), + 'BORG_SCRIPT_DIR': str(tmp_path), 'BORG_UI_RUN_ID': '20260910T150001Z-usbtest'}.items(): + monkeypatch.setenv(name, value) + original_stat = Path.stat + def failed(path, *args, **kwargs): + if path == mount: raise OSError(errno.EIO, 'Input/output error', str(mount)) + return original_stat(path, *args, **kwargs) + monkeypatch.setattr(Path, 'stat', failed) + _reject_borg(monkeypatch) + assert wizard_runner.main() == 2 + assert actions == [] and resources == ['released'] + assert not job.config.lock_file.exists() + assert phases[-1][0] == 'failed' and phases[-1][1]['exit_code'] == 2 + assert _status(job)['failure_code'] == USB_MOUNT_ACCESS_FAILED diff --git a/tests/test_usb_profiles_api.py b/tests/test_usb_profiles_api.py index 7e1b16b7..57fe6b2e 100644 --- a/tests/test_usb_profiles_api.py +++ b/tests/test_usb_profiles_api.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id from pathlib import Path import json import sys @@ -18,9 +19,9 @@ def _write_usb_reference(tmp_path: Path) -> dict: config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs_dir = tmp_path / "config" / "jobs" jobs_dir.mkdir(parents=True) - (jobs_dir / "photos-usb.json").write_text(json.dumps({ + (jobs_dir / (job_id('photos-usb') + ".json")).write_text(json.dumps({"job_id": job_id('photos-usb'), "archive_prefix": "photos-usb-backup", "schema_version": 2, - "job_key": "photos-usb", + "job_key": job_id('photos-usb'), "name": "Photos USB", "location": "usb", "repository_key": "repo_photos_usb", @@ -83,7 +84,7 @@ def test_get_usb_profile_job_refs_uses_canonical_storage_reference(tmp_path: Pat config = _write_usb_reference(tmp_path) assert usb_profiles_api.get_usb_profile_job_refs(config) == { - "usb-a": ["photos-usb (Photos USB)"] + "usb-a": [f"{job_id('photos-usb')} (Photos USB)"] } diff --git a/tests/test_wizard_remote_repo.py b/tests/test_wizard_remote_repo.py index c30d7c11..4217b07d 100644 --- a/tests/test_wizard_remote_repo.py +++ b/tests/test_wizard_remote_repo.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id from pathlib import Path import json import subprocess @@ -19,7 +20,7 @@ def _storagebox_params() -> dict: - return { + return {"archive_prefix": 'flash-backup', "type_id": "flash", "job_name": "Flash", "location": "storagebox", @@ -215,10 +216,10 @@ def test_edit_wizard_resolves_canonical_repository_object(tmp_path: Path, monkey jobs_dir = data_root / "config" / "jobs" scripts_dir.mkdir(parents=True) jobs_dir.mkdir(parents=True) - (jobs_dir / "vms_local.json").write_text( - json.dumps({ + (jobs_dir / (job_id('vms_local') + ".json")).write_text( + json.dumps(identified_job({ "schema_version": 3, - "job_key": "vms_local", + "job_key": job_id('vms_local'), "backup_type": "vms", "location": "local", "name": "VMs", @@ -227,7 +228,7 @@ def test_edit_wizard_resolves_canonical_repository_object(tmp_path: Path, monkey "repository_key": "repo_vms_local_test", "source_paths": ["/mnt/user/domains"], "archive_prefixes": ["oldvms-backup"], - }), + })), encoding="utf-8", ) config = {"BACKUP_SCRIPTS_DIR": str(data_root)} @@ -255,7 +256,7 @@ def test_edit_wizard_resolves_canonical_repository_object(tmp_path: Path, monkey ) loaded = load_job_for_wizard( - "vms_local", + job_id('vms_local'), scripts_dir, config, ) @@ -271,9 +272,9 @@ def test_edit_wizard_keeps_broken_assignment_repairable(tmp_path: Path, monkeypa jobs_dir = data_root / "config" / "jobs" scripts_dir.mkdir(parents=True) jobs_dir.mkdir(parents=True) - (jobs_dir / "photos_smb.json").write_text(json.dumps({ + (jobs_dir / (job_id('photos_smb') + ".json")).write_text(json.dumps(identified_job({ "schema_version": 3, - "job_key": "photos_smb", + "job_key": job_id('photos_smb'), "backup_type": "photos", "location": "smb", "name": "Photos", @@ -281,13 +282,13 @@ def test_edit_wizard_keeps_broken_assignment_repairable(tmp_path: Path, monkeypa "runner": "scriptless-wizard-runner", "repository_key": "repo_missing", "source_paths": ["/mnt/user/photos"], - }), encoding="utf-8") + })), encoding="utf-8") config = {"BACKUP_SCRIPTS_DIR": str(data_root)} write_storage_store(config, {"storages": []}) write_repository_store(config, {"repositories": []}) monkeypatch.setattr("config_api.read_expanded_conf", lambda _cfg: {}) - loaded = load_job_for_wizard("photos_smb", scripts_dir, config) + loaded = load_job_for_wizard(job_id('photos_smb'), scripts_dir, config) assert loaded["repository_key"] == "repo_missing" assert loaded["repo_path"] == "" diff --git a/tests/test_wizard_runner_runtime_path.py b/tests/test_wizard_runner_runtime_path.py index 1c50de5d..68a81f1e 100644 --- a/tests/test_wizard_runner_runtime_path.py +++ b/tests/test_wizard_runner_runtime_path.py @@ -1,3 +1,4 @@ +from job_fixtures import identified_job, job_id from pathlib import Path import json import shlex @@ -49,7 +50,7 @@ def test_wizard_runner_preserves_docker_exclusion_runtime_control(): def test_wizard_runner_passes_archive_prefix_to_maintenance() -> None: source = (ROOT / "api" / "wizard_runner.py").read_text(encoding="utf-8") - assert "archive_prefix = f\"{env.get('BACKUP_TYPE', 'job')}-backup\"" in source + assert "archive_prefix = archive_prefix_from_metadata(meta)" in source assert "runner.maintenance(archive_prefix=archive_prefix)" in source @@ -60,9 +61,9 @@ def test_wizard_runner_resolves_repository_and_secret_from_repository_object( config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs_dir = tmp_path / "config" / "jobs" jobs_dir.mkdir(parents=True) - (jobs_dir / "appdata_local.json").write_text(json.dumps({ + (jobs_dir / (job_id('appdata_local') + ".json")).write_text(json.dumps(identified_job({ "schema_version": 3, - "job_key": "appdata_local", + "job_key": job_id('appdata_local'), "name": "Appdata", "backup_type": "appdata", "location": "local", @@ -70,7 +71,7 @@ def test_wizard_runner_resolves_repository_and_secret_from_repository_object( "source_paths": ["/mnt/user/appdata"], "compression": "lz4", "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, - }) + "\n", encoding="utf-8") + })) + "\n", encoding="utf-8") secret = tmp_path / "secrets" / ".borg-passphrase-repo_appdata_test" secret.parent.mkdir() secret.write_text("secret\n", encoding="utf-8") @@ -97,7 +98,7 @@ def test_wizard_runner_resolves_repository_and_secret_from_repository_object( ) monkeypatch.delenv("BORG_PASSCOMMAND", raising=False) - env, metadata = wizard_runner._load_env_from_job("appdata_local", tmp_path / "scripts", tmp_path) + env, metadata = wizard_runner._load_env_from_job(job_id('appdata_local'), tmp_path / "scripts", tmp_path) assert env["BORG_REPO"] == "/mnt/backup/borg-backup-appdata" assert metadata["repository_key"] == "repo_appdata_test" @@ -114,9 +115,9 @@ def test_wizard_runner_keeps_ssh_identity_and_keepalive_options(tmp_path: Path, config = {"BACKUP_SCRIPTS_DIR": str(tmp_path)} jobs_dir = tmp_path / "config" / "jobs" jobs_dir.mkdir(parents=True) - (jobs_dir / "appdata_storagebox.json").write_text(json.dumps({ + (jobs_dir / (job_id('appdata_storagebox') + ".json")).write_text(json.dumps(identified_job({ "schema_version": 3, - "job_key": "appdata_storagebox", + "job_key": job_id('appdata_storagebox'), "name": "Appdata", "backup_type": "appdata", "location": "storagebox", @@ -124,7 +125,7 @@ def test_wizard_runner_keeps_ssh_identity_and_keepalive_options(tmp_path: Path, "source_paths": ["/mnt/user/appdata"], "compression": "lz4", "retention": {"daily": "7", "weekly": "4", "monthly": "6", "yearly": "3"}, - }) + "\n", encoding="utf-8") + })) + "\n", encoding="utf-8") secret = tmp_path / "secrets" / ".borg-passphrase-repo_appdata_storagebox" secret.parent.mkdir() secret.write_text("secret\n", encoding="utf-8") @@ -157,7 +158,7 @@ def test_wizard_runner_keeps_ssh_identity_and_keepalive_options(tmp_path: Path, ) monkeypatch.delenv("BORG_RSH", raising=False) - env, _metadata = wizard_runner._load_env_from_job("appdata_storagebox", tmp_path / "scripts", tmp_path) + env, _metadata = wizard_runner._load_env_from_job(job_id('appdata_storagebox'), tmp_path / "scripts", tmp_path) tokens = shlex.split(env["BORG_RSH"]) assert tokens[tokens.index("-i") + 1] == str(key_path) diff --git a/ui/i18n/de.json b/ui/i18n/de.json index 9d33cbed..fae9e93b 100644 --- a/ui/i18n/de.json +++ b/ui/i18n/de.json @@ -197,6 +197,7 @@ "storage_profile_in_use": "Storage-Profil kann nicht entfernt werden, weil noch Repositorys darauf verweisen: {repositories}", "repository_relocated": "Borg kennt dieses Repository noch unter einem früheren Pfad. Bitte das konfigurierte Speicherprofil mounten und die Repository-Aktion erneut ausführen.", "repository_busy": "Dieses Repository wird gerade von einer anderen Borg-Aktion verwendet.", + "restore_archive_unavailable": "Das ausgewählte Archiv ist im aktuellen Repository nicht mehr verfügbar. Wähle den Job erneut aus, um die Archivliste zu aktualisieren.", "repository_key_invalid": "Die ausgewählte Borg-Key-Datei ist leer, zu groß oder hat kein gültiges Borg-Key-Format.", "repository_key_mismatch": "Diese Borg-Key-Datei gehört zu einem anderen Repository.", "repository_key_unsupported": "Dieses Repository verwendet keinen exportierbaren Borg-Key.", @@ -208,7 +209,9 @@ "restore_test_already_running": "Ein Restore-Test läuft bereits.", "inventory_unavailable": "Repository- oder Storage-Daten sind beschädigt oder nicht lesbar. Bitte Systemzustand und Migrations-Backup prüfen.", "maintenance_mode": "Die Anwendung läuft wegen einer fehlgeschlagenen Startmigration im eingeschränkten Wartungsmodus. Bitte Systemzustand & Migration prüfen.", - "internal_error": "Interner Serverfehler. Bitte Protokoll und Request-ID prüfen." + "internal_error": "Interner Serverfehler. Bitte Protokoll und Request-ID prüfen.", + "configuration_export_unsupported": "Dieses Konfigurationspaket verwendet ein nicht mehr unterstütztes Format. Der Import wurde abgebrochen. Es wurden keine Änderungen vorgenommen. Erstelle nach dem Update und der Migration der Quellinstallation einen neuen Export.", + "job_settings_invalid": "Die Job-Einstellungen sind unvollständig oder ungültig. Prüfe Kompression und Aufbewahrung und speichere den Job erneut." }, "messages": { "smb_already_mounted": "SMB-Freigabe ist bereits eingehängt.", @@ -311,20 +314,26 @@ "repositoryMaintenanceConfirmAction": "Wartungsaktion: {action}", "repositoryMaintenanceRetentionSource": "Retention-Quelle: {job}", "repositoryMaintenanceArchiveFilter": "Archivfilter: {filter}", - "repositoryMaintenanceRetention": "Retention: {retention}", + "repositoryMaintenanceRetention": "Aufbewahrung", "repositoryMaintenanceSelectRetentionSource": "Retention-Quelle auswählen", "repositoryMaintenanceMultipleJobsHint": "Dieses Repository wird von mehreren Backup-Jobs genutzt. Prune verwendet nur den Archivfilter und die Retention des ausgewählten Jobs.", - "repositoryRetentionDaily": "{count} Tagesstände (max. 1/Tag)", - "repositoryRetentionWeekly": "{count} Wochenstände (max. 1/Woche)", - "repositoryRetentionMonthly": "{count} Monatsstände (max. 1/Monat)", - "repositoryRetentionYearly": "{count} Jahresstände (max. 1/Jahr)", + "repositoryRetentionPoints": "Stände", + "repositoryRetentionMaximum": "Maximum", + "repositoryRetentionDaily": "{count} Tagesstände", + "repositoryRetentionWeekly": "{count} Wochenstände", + "repositoryRetentionMonthly": "{count} Monatsstände", + "repositoryRetentionYearly": "{count} Jahresstände", + "repositoryRetentionDailyLimit": "1 pro Tag", + "repositoryRetentionWeeklyLimit": "1 pro Woche", + "repositoryRetentionMonthlyLimit": "1 pro Monat", + "repositoryRetentionYearlyLimit": "1 pro Jahr", "repositoryMaintenanceConfirmStart": "Jetzt starten", "repositoryCheck": "Check", "repositoryVerifyData": "Daten prüfen", "repositoryPrune": "Prune", "repositoryCompact": "Compact", "repositoryVerifyConfirm": "Die vollständige Datenprüfung kann lange dauern und hohe Last erzeugen. Jetzt starten?", - "repositoryPruneConfirm": "Prune entfernt Archive anhand der Aufbewahrungsrichtlinie des verknüpften Jobs. Jetzt starten?", + "repositoryPruneConfirm": "Die Aufbewahrungsregeln des ausgewählten Jobs gelten nur für Archive, die dem angezeigten aktuellen Archivfilter entsprechen. Gespeicherte frühere Präfixe werden nicht zusätzlich berücksichtigt. Jetzt starten?", "repositoryCompactConfirm": "Compact gibt nicht mehr verwendeten Speicherplatz in diesem Repository frei. Jetzt starten?", "repositoryRefreshInfo": "Info aktualisieren", "repositoryInfoUpdated": "Repository-Informationen wurden aktualisiert.", @@ -721,6 +730,13 @@ "deleteError": "Fehler beim Löschen: {message}" }, "wizard": { + "jobId": "Job-ID", + "jobIdInvalid": "Der Server hat keine gültige Job-ID geliefert.", + "jobIdLoadFailed": "Job-ID konnte nicht geladen werden. Bitte den Wizard schließen und erneut öffnen. {message}", + "jobIdNotReady": "Die Job-ID ist noch nicht verfügbar. Bitte warten oder den Wizard erneut öffnen.", + "jobIdExists": "Es konnte keine freie Job-ID vergeben werden. Bitte erneut speichern. Deine Eingaben bleiben erhalten.", + "archivePrefixPlaceholder": "z.B. meinedaten-backup", + "archivePrefix": "Archivpräfix", "newTitle": "Neuer Backup-Job", "editTitle": "Backup-Job bearbeiten", "loadFailed": "Wizard laden fehlgeschlagen: {message}", @@ -737,6 +753,7 @@ "jobName": "Job-Name", "displayNameHint": "(Anzeigename)", "jobNamePlaceholder": "z.B. Borg Backup (Meine Daten)", + "jobNameHint": "Maximal 100 Zeichen.", "typeId": "Typ-ID", "typeIdHint": "(nur a–z, 0–9, _)", "typeIdPlaceholder": "z.B. meinedaten", @@ -745,8 +762,8 @@ "archivePatternCurrentLabel": "Aktuelles Archivmuster", "archivePatternHistoryButton": "Archivmuster-Historie anzeigen", "archivePatternHistoryTitle": "Gespeicherte Archivmuster", - "archiveFilterCurrentBadge": "aktuell", - "archiveFilterPreviousBadge": "vorherig", + "archiveFilterCurrentBadge": "Aktuell", + "archiveFilterPreviousBadge": "Vorherige", "icon": "Icon", "automaticType": "Automatisch (Typ)", "photos": "Fotos", @@ -909,8 +926,9 @@ "removeExclude": "Ausschlusspfad entfernen", "removeSource": "Entfernen", "validationJobName": "Job-Name darf nicht leer sein.", - "validationTypeId": "Typ-ID darf nicht leer sein.", - "validationTypeFormat": "Typ-ID darf nur Kleinbuchstaben, Ziffern und _ enthalten.", + "validationJobNameLength": "Der Job-Name darf höchstens 100 Zeichen enthalten.", + "validationTypeId": "Archivpräfix darf nicht leer sein.", + "validationTypeFormat": "Archivpräfix darf nur Buchstaben, Ziffern, Punkt, _ und - enthalten.", "validationSource": "Bitte mindestens einen Ordner oder eine Datei auswählen, die gesichert werden soll.", "validationExcludeAbsolute": "Ausschlusspfade müssen absolute Pfade unter /mnt oder /boot sein.", "validationExcludeBelowSource": "Ein Ausschlusspfad muss unter einem ausgewählten zu sichernden Ordner liegen.", @@ -965,7 +983,10 @@ "previewError": "Vorschau-Fehler: {message}", "confirmRemoteRequired": "Bitte Remote-Repository-Anlage bestätigen.", "saved": "Job gespeichert: {key}", - "saveError": "Fehler: {message}" + "saveError": "Fehler: {message}", + "defaultIcon": "Standard (Archiv)", + "defaultColor": "Standard (Neutral)", + "savedThemeColor": "Bisherige Theme-Farbe" }, "dashboard": { "title": "Dashboard", @@ -1048,6 +1069,7 @@ "checkUnknown": "Check-Status unbekannt" }, "history": { + "jobId": "Job-ID", "title": "History", "subtitle": "Alle Backup-Läufe", "grouping": "Gruppierung", @@ -1059,7 +1081,7 @@ "locationLocal": "Lokal", "locationNetwork": "Netzwerk", "refresh": "Aktualisieren", - "allTypes": "Alle Typen", + "allTypes": "Alle Jobs", "other": "Sonstiges", "allLocations": "Alle Standorte", "allStatuses": "Alle Status", @@ -1074,7 +1096,7 @@ "entryCount": "{count} Einträge", "empty": "Keine Backup-Einträge gefunden.", "dateTime": "Datum / Zeit", - "type": "Typ", + "type": "Job", "location": "Ort", "duration": "Dauer", "originalSize": "Originalgröße", @@ -1255,8 +1277,8 @@ "archiveFiltersLabel": "Archivfilter", "archiveFilterHistoryButton": "Archivfilter-Historie anzeigen", "archiveFilterHistoryTitle": "Angewendete Archivfilter", - "archiveFilterCurrent": "aktuell", - "archiveFilterPrevious": "vorherig", + "archiveFilterCurrent": "Aktuell", + "archiveFilterPrevious": "Vorherige", "available": "Verfügbar", "noArchives": "Keine Archive gefunden", "selectElement": "Element auswählen", @@ -2222,7 +2244,8 @@ "secretImportSkipExisting": "Vorhandene Dateien überspringen", "secretImportOverwriteExisting": "Vorhandene Dateien ersetzen", "profilesImportOk": "Profile+Secrets Import OK: {count} Dateien", - "profilesImportFailed": "Profile+Secrets Import fehlgeschlagen: {message}" + "profilesImportFailed": "Profile+Secrets Import fehlgeschlagen: {message}", + "archivePrefix": "Archivpräfix" }, "forms": { "smtpTitle": "E-Mail (SMTP)", @@ -2294,12 +2317,11 @@ "intervalDays": "Intervall (Tage)", "defaultLocation": "Standard Location", "dryRunStrategy": "Strategie Dry-Run", - "forceChunkTypes": "Chunk-Modus erzwingen für Typen (CSV)", "chunkFromSize": "Ab Größe (GB) direkt Chunk-Modus", - "chunkHint": "Beispiel Typen: vms,photos. Wert 0 bei Größe deaktiviert die automatische Umschaltung.", + "chunkHint": "Ab dieser Archivgröße wird eine Dateistichprobe in Gruppen geprüft. 0 deaktiviert die Umschaltung. Dateigrenzen können die angestrebte Abdeckung begrenzen.", "limitsPerformance": "Limits & Performance", - "minimumCoverage": "Mindest-Coverage (%)", - "maxEntries": "Max. Einträge je Test", + "minimumCoverage": "Angestrebte Datei-Abdeckung (%)", + "maxEntries": "Maximale Dateien pro Stichprobe", "sampleFiles": "Level-3 Sample-Dateien", "borgTimeout": "Borg Standard Timeout (s)", "dryRunTimeout": "Dry-Run Timeout (s, 0 = aus)", diff --git a/ui/i18n/en.json b/ui/i18n/en.json index d6d57dfd..6dc40d7b 100644 --- a/ui/i18n/en.json +++ b/ui/i18n/en.json @@ -197,6 +197,7 @@ "storage_profile_in_use": "Storage profile cannot be removed because repositories still reference it: {repositories}", "repository_relocated": "Borg still knows this repository from an earlier path. Mount the configured storage profile and retry the repository action.", "repository_busy": "This repository is currently used by another Borg operation.", + "restore_archive_unavailable": "The selected archive is no longer available in the current repository. Select the job again to refresh the archive list.", "repository_key_invalid": "The selected Borg key file is empty, too large, or not a valid Borg key export.", "repository_key_mismatch": "This Borg key file belongs to a different repository.", "repository_key_unsupported": "This repository does not use an exportable Borg key.", @@ -208,7 +209,9 @@ "restore_test_already_running": "A restore test is already running.", "inventory_unavailable": "Repository or storage data is corrupt or unreadable. Check System Health and the migration backup.", "maintenance_mode": "The application is in restricted maintenance mode because a startup migration failed. Review System Health & Migration.", - "internal_error": "Internal server error. Check the log and request ID." + "internal_error": "Internal server error. Check the log and request ID.", + "configuration_export_unsupported": "This configuration package uses a format that is no longer supported. The import was aborted and no changes were made. Update and migrate the source installation, then create a new configuration export.", + "job_settings_invalid": "The job settings are incomplete or invalid. Check compression and retention, then save the job again." }, "messages": { "smb_already_mounted": "The SMB share is already mounted.", @@ -311,20 +314,26 @@ "repositoryMaintenanceConfirmAction": "Maintenance action: {action}", "repositoryMaintenanceRetentionSource": "Retention source: {job}", "repositoryMaintenanceArchiveFilter": "Archive filter: {filter}", - "repositoryMaintenanceRetention": "Retention: {retention}", + "repositoryMaintenanceRetention": "Retention", "repositoryMaintenanceSelectRetentionSource": "Select retention source", "repositoryMaintenanceMultipleJobsHint": "This repository is used by multiple backup jobs. Prune uses only the archive filter and retention policy of the selected job.", - "repositoryRetentionDaily": "{count} daily restore points (max. 1/day)", - "repositoryRetentionWeekly": "{count} weekly restore points (max. 1/week)", - "repositoryRetentionMonthly": "{count} monthly restore points (max. 1/month)", - "repositoryRetentionYearly": "{count} yearly restore points (max. 1/year)", + "repositoryRetentionPoints": "Restore points", + "repositoryRetentionMaximum": "Maximum", + "repositoryRetentionDaily": "{count} daily restore points", + "repositoryRetentionWeekly": "{count} weekly restore points", + "repositoryRetentionMonthly": "{count} monthly restore points", + "repositoryRetentionYearly": "{count} yearly restore points", + "repositoryRetentionDailyLimit": "1 per day", + "repositoryRetentionWeeklyLimit": "1 per week", + "repositoryRetentionMonthlyLimit": "1 per month", + "repositoryRetentionYearlyLimit": "1 per year", "repositoryMaintenanceConfirmStart": "Start now", "repositoryCheck": "Check", "repositoryVerifyData": "Verify data", "repositoryPrune": "Prune", "repositoryCompact": "Compact", "repositoryVerifyConfirm": "The full data verification can take a long time and create high load. Start now?", - "repositoryPruneConfirm": "Prune removes archives using the linked job's retention policy. Start now?", + "repositoryPruneConfirm": "The selected job's retention policy applies only to archives matching the displayed current archive filter. Recorded previous prefixes are not additionally included. Start now?", "repositoryCompactConfirm": "Compact reclaims unused space in this repository. Start now?", "repositoryRefreshInfo": "Refresh info", "repositoryInfoUpdated": "Repository information was updated.", @@ -721,6 +730,13 @@ "deleteError": "Error deleting schedule: {message}" }, "wizard": { + "jobId": "Job ID", + "jobIdInvalid": "The server did not return a valid job ID.", + "jobIdLoadFailed": "Could not load the job ID. Please close and reopen the Wizard. {message}", + "jobIdNotReady": "The job ID is not available yet. Please wait or reopen the Wizard.", + "jobIdExists": "An unused job ID could not be assigned. Please save again. Your entries are retained.", + "archivePrefixPlaceholder": "e.g. mydata-backup", + "archivePrefix": "Archive prefix", "newTitle": "New backup job", "editTitle": "Edit backup job", "loadFailed": "Could not load wizard: {message}", @@ -737,6 +753,7 @@ "jobName": "Job name", "displayNameHint": "(display name)", "jobNamePlaceholder": "e.g. Borg Backup (My Data)", + "jobNameHint": "Maximum 100 characters.", "typeId": "Type ID", "typeIdHint": "(a-z, 0-9, _ only)", "typeIdPlaceholder": "e.g. mydata", @@ -745,8 +762,8 @@ "archivePatternCurrentLabel": "Current archive pattern", "archivePatternHistoryButton": "Show archive name pattern history", "archivePatternHistoryTitle": "Stored archive name patterns", - "archiveFilterCurrentBadge": "current", - "archiveFilterPreviousBadge": "previous", + "archiveFilterCurrentBadge": "Current", + "archiveFilterPreviousBadge": "Previous", "icon": "Icon", "automaticType": "Automatic (type)", "photos": "Photos", @@ -909,8 +926,9 @@ "removeExclude": "Remove exclusion path", "removeSource": "Remove", "validationJobName": "Job name cannot be empty.", - "validationTypeId": "Type ID cannot be empty.", - "validationTypeFormat": "Type ID may contain lowercase letters, digits, and underscores only.", + "validationJobNameLength": "The job name must not exceed 100 characters.", + "validationTypeId": "Archive prefix cannot be empty.", + "validationTypeFormat": "Archive prefix may contain only letters, digits, dots, _ and -.", "validationSource": "Select at least one folder or file to back up.", "validationExcludeAbsolute": "Exclusion paths must be absolute paths below /mnt or /boot.", "validationExcludeBelowSource": "An exclusion path must be below a selected folder to back up.", @@ -965,7 +983,10 @@ "previewError": "Preview error: {message}", "confirmRemoteRequired": "Confirm remote repository creation.", "saved": "Job saved: {key}", - "saveError": "Error: {message}" + "saveError": "Error: {message}", + "defaultIcon": "Default (archive)", + "defaultColor": "Default (neutral)", + "savedThemeColor": "Preserved theme color" }, "dashboard": { "title": "Dashboard", @@ -1048,6 +1069,7 @@ "checkUnknown": "Check status unknown" }, "history": { + "jobId": "Job ID", "title": "History", "subtitle": "All backup runs", "grouping": "Grouping", @@ -1059,7 +1081,7 @@ "locationLocal": "Local", "locationNetwork": "Network", "refresh": "Refresh", - "allTypes": "All types", + "allTypes": "All jobs", "other": "Other", "allLocations": "All locations", "allStatuses": "All statuses", @@ -1074,7 +1096,7 @@ "entryCount": "{count} entries", "empty": "No backup entries found.", "dateTime": "Date / time", - "type": "Type", + "type": "Job", "location": "Location", "duration": "Duration", "originalSize": "Original size", @@ -1255,8 +1277,8 @@ "archiveFiltersLabel": "Archive filters", "archiveFilterHistoryButton": "Show archive filter history", "archiveFilterHistoryTitle": "Applied archive filters", - "archiveFilterCurrent": "current", - "archiveFilterPrevious": "previous", + "archiveFilterCurrent": "Current", + "archiveFilterPrevious": "Previous", "available": "Available", "noArchives": "No archives found", "selectElement": "Select an item", @@ -2222,7 +2244,8 @@ "secretImportSkipExisting": "Skip existing files", "secretImportOverwriteExisting": "Replace existing files", "profilesImportOk": "Profiles+secrets import OK: {count} files", - "profilesImportFailed": "Profiles+secrets import failed: {message}" + "profilesImportFailed": "Profiles+secrets import failed: {message}", + "archivePrefix": "Archive prefix" }, "forms": { "smtpTitle": "Email (SMTP)", @@ -2294,12 +2317,11 @@ "intervalDays": "Interval (days)", "defaultLocation": "Default location", "dryRunStrategy": "Dry-run strategy", - "forceChunkTypes": "Force chunk mode for types (CSV)", "chunkFromSize": "Use chunk mode from size (GB)", - "chunkHint": "Example types: vms,photos. A size value of 0 disables automatic switching.", + "chunkHint": "At this archive size, a file sample is checked in groups. 0 disables switching. File limits can reduce the target coverage.", "limitsPerformance": "Limits & performance", - "minimumCoverage": "Minimum coverage (%)", - "maxEntries": "Maximum entries per test", + "minimumCoverage": "Target file coverage (%)", + "maxEntries": "Maximum files per sample", "sampleFiles": "Level 3 sample files", "borgTimeout": "Borg default timeout (s)", "dryRunTimeout": "Dry-run timeout (s, 0 = off)", diff --git a/ui/index.html b/ui/index.html index a139ccb7..4156b418 100644 --- a/ui/index.html +++ b/ui/index.html @@ -413,13 +413,7 @@

History

Auswahl

Alle Standorte

Backup-Läufe und Restore-Testberichte
+
- - + + Maximal 100 Zeichen. +
+
- - + +
- + diff --git a/ui/js/components/app-bindings.js b/ui/js/components/app-bindings.js index 75963fb7..be2898d8 100644 --- a/ui/js/components/app-bindings.js +++ b/ui/js/components/app-bindings.js @@ -289,7 +289,7 @@ document.getElementById('wizard-next-btn')?.addEventListener('click', wizardNext); document.getElementById('wizard-save-btn')?.addEventListener('click', saveWizardJob); document.getElementById('wiz-job-name')?.addEventListener('input', () => wizardClearError(1)); - document.getElementById('wiz-type-id')?.addEventListener('input', () => { + document.getElementById('wiz-archive-prefix')?.addEventListener('input', () => { wizardAutoFill(); wizardRenderArchivePrefixSummary(); wizardClearError(1); diff --git a/ui/js/pages/dashboard.js b/ui/js/pages/dashboard.js index 4408fa89..d1c6fbdd 100644 --- a/ui/js/pages/dashboard.js +++ b/ui/js/pages/dashboard.js @@ -82,6 +82,7 @@ async function refreshStatus() { if (!job.is_utility && !knownKeys.has(String(job.key || '').toLowerCase())) { statusData.backups.push({ key: job.key, + archive_prefix: job.archive_prefix, backup_type: job.backup_type, location: job.location, display_name: job.display_name || job.name || '', @@ -100,6 +101,7 @@ async function refreshStatus() { b.enabled = job.enabled !== false; b.display_name = job.display_name || job.name || b.display_name || ''; b.name = job.name || job.display_name || b.name || ''; + b.archive_prefix = job.archive_prefix || ''; b.icon = job.icon || b.icon || ''; b.icon_color = job.icon_color || b.icon_color || ''; } @@ -229,14 +231,13 @@ function renderBackupGrid(backups) { return; } - const typeOrder = { flash: 0, appdata: 1, photos: 2, VMs: 3, vms: 3, sonstiges: 4 }; const visible = backups .filter((backup) => dashboardSelectedLocation === 'all' || dashboardLocationKey(backup) === dashboardSelectedLocation) .sort((a, b) => { const locationDelta = DASHBOARD_LOCATION_ORDER.indexOf(dashboardLocationKey(a)) - DASHBOARD_LOCATION_ORDER.indexOf(dashboardLocationKey(b)); if (locationDelta) return locationDelta; - return (typeOrder[a.backup_type] ?? 99) - (typeOrder[b.backup_type] ?? 99); + return String(a.name || a.display_name || '').localeCompare(String(b.name || b.display_name || '')); }); renderDashboardLocationSidebar(backups); @@ -479,11 +480,11 @@ function renderDashboardInventoryRow(backup) { let checkStatus = backup.repository_check_status; if (checkStatus === 'ok' && isStaleDate(backup.repository_check_date)) checkStatus = 'overdue'; const checkLabel = checkStatus ? repoCheckLabel({ ...backup, repository_check_status: checkStatus }) : dashboardT('dashboard.checkUnknown'); - const type = capitalize(backup.backup_type || '—'); - const iconKey = typeof resolveJobIcon === 'function' ? resolveJobIcon(backup) : (backup.icon || backup.backup_type); + const type = backup.name || backup.display_name || 'Backup'; + const iconKey = typeof resolveJobIcon === 'function' ? resolveJobIcon(backup) : (backup.icon || 'archive'); const iconColorKey = typeof resolveJobIconColor === 'function' ? resolveJobIconColor(backup) : ''; const iconColorClass = iconColorKey ? ` type-icon-color-${iconColorKey}` : ''; - const identityDetail = backup.archive_name || backup.key || dashboardT('dashboard.neverExecuted'); + const identityDetail = backup.archive_name || backup.archive_prefix || dashboardT('dashboard.neverExecuted'); const runTime = dashboardRelativeRunTime(backup.timestamp); const runDuration = dashboardRunDuration(backup.duration_seconds); const nextRun = dashboardNextRun(backup.key, backup.enabled); @@ -511,7 +512,7 @@ function renderDashboardInventoryRow(backup) { return `
- ${typeIcon(iconKey)} + ${typeIcon(iconKey)} ${escHtml(type)}${escHtml(identityDetail)}
diff --git a/ui/js/pages/history.js b/ui/js/pages/history.js index 1e3f224a..6a1a9bbf 100644 --- a/ui/js/pages/history.js +++ b/ui/js/pages/history.js @@ -22,7 +22,7 @@ async function refreshHistory() { const status = document.getElementById('history-filter-status')?.value || ''; const params = new URLSearchParams(); - if (type) params.set('type', type); + if (type) params.set('job_key', type); if (location) params.set('location', location); if (status) params.set('status', status); params.set('page', String(historyState.page || 1)); @@ -32,6 +32,11 @@ async function refreshHistory() { const res = await fetch('/api/history?' + params.toString(), { credentials: 'include' }); const data = await res.json(); if (!res.ok || data.error) throw new Error(apiErrorMessage(data, res.status)); + const filter = document.getElementById('history-filter-type'); + if (filter && Array.isArray(data.jobs)) { + filter.innerHTML = `` + data.jobs.map((job) => ``).join(''); + filter.value = type; + } historyState.data = data; historyState.loaded = true; renderHistory(data); @@ -130,7 +135,7 @@ function renderHistoryRow(e, idx) { const statusClass = e.status === 'cancelled' ? 'warning' : e.status; const statusBadge = `${historyStatusLabel(e.status)}`; const locClass = e.location || ''; - const typeLabel = historyTypeLabel(e.backup_type); + const typeLabel = e.job_name || historyTypeLabel(e.backup_type); const rowId = `hrow-${idx}`; const detailId = `hdetail-${idx}`; @@ -149,6 +154,7 @@ function renderHistoryRow(e, idx) { ${detailError ? renderHistoryError(detailError, e.status === 'skipped') : ''}
+ ${detailGroup(historyT('jobId'), e.job_id || '—', 'wide')} ${detailGroup(historyT('archive'), e.archive_name || '-', 'archive')} ${detailGroup(historyT('compressed'), e.compressed_size_fmt)} ${detailGroup(historyT('repositorySize'), e.repository_size_fmt)} diff --git a/ui/js/pages/jobs.js b/ui/js/pages/jobs.js index b0de7227..f2ec2e90 100644 --- a/ui/js/pages/jobs.js +++ b/ui/js/pages/jobs.js @@ -268,14 +268,13 @@ function renderJobsGrid(jobs) { } if (jobsNewBtn) jobsNewBtn.classList.remove('hidden'); - const typeOrder = { flash: 0, appdata: 1, photos: 2, VMs: 3, vms: 3, sonstiges: 4 }; const visible = jobs .filter((job) => jobsState.selectedLocation === 'all' || jobsLocationKey(job) === jobsState.selectedLocation) .sort((a, b) => { const locationDelta = JOBS_LOCATION_ORDER.indexOf(jobsLocationKey(a)) - JOBS_LOCATION_ORDER.indexOf(jobsLocationKey(b)); if (locationDelta) return locationDelta; - return (typeOrder[a.backup_type] ?? 99) - (typeOrder[b.backup_type] ?? 99); + return String(a.name || a.display_name || '').localeCompare(String(b.name || b.display_name || '')); }); renderJobsLocationSidebar(jobs); @@ -379,7 +378,7 @@ function renderJobsLocationGroup(location, jobs) { function renderJobCard(job) { const isRunning = job.running; - const titleName = job.name || job.display_name || capitalize(job.backup_type); + const titleName = job.name || job.display_name || 'Backup'; const iconKey = resolveJobIcon(job); const iconColorKey = resolveJobIconColor(job); const iconColorClass = iconColorKey ? ` type-icon-color-${iconColorKey}` : ''; @@ -470,8 +469,8 @@ function renderJobCard(job) { return `
-
${typeIcon(iconKey)}
-
${escHtml(titleName)}
${escHtml(job.key)}
+
${typeIcon(iconKey)}
+
${escHtml(titleName)}
${escHtml(job.archive_prefix || '')}
${job.description ? `
${renderDescriptionMarkdown(job.description)}
` : ''}
${features.join('')}
@@ -562,7 +561,7 @@ function resolveJobIcon(job) { ]); const icon = String(job?.icon || '').trim().toLowerCase(); if (icon && allowed.has(icon)) return icon; - return String(job?.backup_type || 'sonstiges').trim().toLowerCase() || 'sonstiges'; + return 'archive'; } function resolveJobIconColor(job) { @@ -571,7 +570,7 @@ function resolveJobIconColor(job) { 'green', 'lime', 'violet', 'amber', 'orange', 'red', 'rose', - 'teal', 'cyan', 'gray', + 'teal', 'cyan', 'gray', 'theme-blue', 'theme-orange', 'theme-purple', 'theme-green', ]); const color = String(job?.icon_color || '').trim().toLowerCase(); return allowed.has(color) ? color : ''; @@ -605,7 +604,7 @@ document.addEventListener('click', closeAllJobMenus); function _showDeleteJobModalForKey(jobKey) { const job = jobsState.jobs.find(j => j.key === jobKey); if (!job) return; - showDeleteJobModal(jobKey, job.display_name || job.name || job.key, job.backup_type || '', job.location || ''); + showDeleteJobModal(jobKey, job.display_name || job.name || job.key); } function onJobsGridClick(event) { @@ -673,7 +672,7 @@ function showStartModal(jobKey) { jobsState.pendingJobKey = jobKey; jobsState.confirmAction = 'start'; - const titleName = job.name || job.display_name || capitalize(job.backup_type); + const titleName = job.name || job.display_name || 'Backup'; document.getElementById('modal-title').textContent = jobsT('jobs.startTitle', { name: titleName }); document.getElementById('modal-description').textContent = jobsT('jobs.startDescription', { name: job.display_name || job.key }); @@ -782,7 +781,7 @@ function closeModal(options = {}) { if (pwPath) pwPath.textContent = ''; } -async function showDeleteJobModal(jobKey, displayName, typeId, location) { +async function showDeleteJobModal(jobKey, displayName) { jobsState.pendingDeleteJobKey = jobKey; jobsState.confirmAction = 'delete'; document.getElementById('modal-title').textContent = jobsT('jobs.deleteTitle'); @@ -807,9 +806,9 @@ async function showDeleteJobModal(jobKey, displayName, typeId, location) { const pwPath = document.getElementById('modal-delete-passphrase-path'); pwWrap.classList.add('hidden'); pwCb.checked = false; - if (typeId) { + if (jobKey) { try { - const res = await fetch(`/api/wizard/passphrase-check?type_id=${encodeURIComponent(typeId)}&location=${encodeURIComponent(location || '')}`); + const res = await fetch(`/api/wizard/passphrase-check?job_key=${encodeURIComponent(jobKey)}`); const data = await res.json(); if (data.exists) { pwPath.textContent = data.path; @@ -1170,7 +1169,7 @@ function _updateScheduleModalTitle(jobKey, displayName) { const job = jobsState.jobs.find(j => j.key === jobKey); if (job) { titleEl.textContent = jobsT('schedule.titleFor', { - name: `${capitalize(job.backup_type)} (${jobsLocationLabel(job.location)})`, + name: job.name || job.display_name, }); } else { titleEl.textContent = jobsT('schedule.titleFor', { name: jobKey }); diff --git a/ui/js/pages/reports.js b/ui/js/pages/reports.js index 16a678d8..2e6304c9 100644 --- a/ui/js/pages/reports.js +++ b/ui/js/pages/reports.js @@ -27,16 +27,16 @@ async function berichtInit() { const jobsData = jobsResponse.ok ? await jobsResponse.json() : { jobs: [] }; const configuredJobs = jobsData.jobs || []; reportState.jobs = (data.jobs || []).map((job) => { - const configured = configuredJobs.find((candidate) => String(candidate.key) === String(job.key)) - || configuredJobs.find((candidate) => String(candidate.backup_type || '').toLowerCase() === String(job.backup_type || '').toLowerCase() - && String(candidate.location || '').toLowerCase() === String(job.location || '').toLowerCase()); + const configured = configuredJobs.find((candidate) => String(candidate.key) === String(job.key)); return { ...job, - display_name: configured?.display_name || configured?.name || job.display_name, + display_name: configured?.name || configured?.display_name || job.display_name, + archive_prefix: configured?.archive_prefix || '', icon: configured?.icon || '', icon_color: configured?.icon_color || '', }; }); + reportState.jobs.sort((a, b) => String(a.display_name || '').localeCompare(String(b.display_name || ''))); for (const job of reportState.jobs) { const opt = document.createElement('option'); opt.value = job.key; @@ -105,7 +105,7 @@ function _berichtJobIcon(job) { const icon = resolveJobIcon(job); const color = resolveJobIconColor(job); const colorClass = color ? ` type-icon-color-${color}` : ''; - return `${typeIcon(icon)}`; + return `${typeIcon(icon)}`; } function _berichtRenderJobSidebar() { @@ -121,7 +121,7 @@ function _berichtRenderJobSidebar() { const order = ['local', 'usb', 'smb', 'storagebox']; const locations = [...new Set(jobs.map((job) => String(job.location || 'local').toLowerCase()))] .sort((a, b) => (order.indexOf(a) < 0 ? 99 : order.indexOf(a)) - (order.indexOf(b) < 0 ? 99 : order.indexOf(b))); - list.innerHTML = locations.map((location) => `

${escHtml(_berichtLocationLabel(location))}

${jobs.filter((job) => String(job.location || 'local').toLowerCase() === location).map((job) => ``).join('')}
`).join(''); + list.innerHTML = locations.map((location) => `

${escHtml(_berichtLocationLabel(location))}

${jobs.filter((job) => String(job.location || 'local').toLowerCase() === location).map((job) => ``).join('')}
`).join(''); list.querySelectorAll('[data-report-job]').forEach((button) => button.addEventListener('click', () => { const select = document.getElementById('bericht-job-sel'); if (select) select.value = button.dataset.reportJob || ''; diff --git a/ui/js/pages/restore-tests.js b/ui/js/pages/restore-tests.js index 0936a723..61e6ceb5 100644 --- a/ui/js/pages/restore-tests.js +++ b/ui/js/pages/restore-tests.js @@ -35,6 +35,11 @@ function restoreTestsStatusIcon(status) { return ``; } +function restoreTestJobName(result) { + const job = (restoreTestsState.jobs || []).find((item) => String(item.key) === String(result.job_key)); + return job?.name || job?.display_name || result.type || '—'; +} + function restoreTestsLocale() { return window.BBUI?.components?.i18n?.getLanguage?.() === 'en' ? 'en-US' : 'de-DE'; } @@ -72,7 +77,7 @@ function restoreTestsJobIcon(job) { const icon = resolveJobIcon(job); const color = resolveJobIconColor(job); const colorClass = color ? ` type-icon-color-${color}` : ''; - return `${typeIcon(icon)}`; + return `${typeIcon(icon)}`; } function renderRestoreTestsSidebar() { @@ -96,7 +101,7 @@ function renderRestoreTestsSidebar() { const configured = !!planJob && planJob.enabled !== false && String(planJob.policy?.mode || 'off') !== 'off'; const stateClass = planJob?.is_overdue ? 'warning' : configured ? 'success' : 'disabled'; const active = restoreTestsState.selectedJob === String(job.key); - return ``; + return ``; }).join('')}`; }).join(''); list.innerHTML = allEntry + groups; @@ -870,7 +875,7 @@ function renderRTReportRow(t, idx) { ${escHtml(dt)} RESTORE TEST - ${escHtml(t.job_key || t.type || '-')} + ${escHtml(restoreTestJobName(t))} ${escHtml(restoreTestsLocationLabel(t.location || ''))} ${escHtml(t.duration_formatted || '—')} ${escHtml(stats.original || '—')} @@ -883,7 +888,7 @@ function renderRTReportRow(t, idx) {
${escHtml(restoreTestsT('report'))}
-
${escHtml(t.job_key || t.type || 'Restore Test')}
+
${escHtml(restoreTestJobName(t))}
${escHtml(archive || restoreTestsT('noArchive'))}
diff --git a/ui/js/pages/restore.js b/ui/js/pages/restore.js index ffddce9d..0ccaa9ec 100644 --- a/ui/js/pages/restore.js +++ b/ui/js/pages/restore.js @@ -28,6 +28,8 @@ window.BBUI.restoreState = window.BBUI.restoreState || { jobs: [], archives: [], archiveFilters: [], + sourceRequest: 0, + filesRequest: 0, runs: [], history: [], historyTotal: 0, @@ -138,7 +140,7 @@ function restoreJobIcon(job) { const icon = resolveJobIcon(job); const color = resolveJobIconColor(job); const colorClass = color ? ` type-icon-color-${color}` : ''; - return `${typeIcon(icon)}`; + return `${typeIcon(icon)}`; } function renderRestoreJobSidebar() { @@ -158,11 +160,16 @@ function renderRestoreJobSidebar() { if (!locationJobs.length) return ''; return `
${escHtml(restoreLocationLabel(location))}${locationJobs.length}
${locationJobs.map((job) => { const active = String(job.key) === String(restoreState.job); - return ``; + return ``; }).join('')}
`; }).join(''); } +function restoreJobName(key) { + const job = (restoreState.jobs || []).find((item) => String(item.key) === String(key)); + return job?.name || job?.display_name || key || '—'; +} + function restoreLocationLabel(location) { const key = String(location || '').toLowerCase(); return ({ @@ -183,7 +190,7 @@ function renderRestoreSelectedJob() { if (badge) badge.textContent = ''; return; } - card.innerHTML = `${restoreJobIcon(job)}
${escHtml(restoreT('selectedJob'))}

${escHtml(job.display_name || job.name || job.key)}

${escHtml(job.key)} · ${escHtml(restoreLocationLabel(job.location))}
${escHtml(restoreT('ready'))}`; + card.innerHTML = `${restoreJobIcon(job)}
${escHtml(restoreT('selectedJob'))}

${escHtml(job.name || job.display_name || job.key)}

${escHtml(job.archive_prefix || '')} · ${escHtml(restoreLocationLabel(job.location))}
${escHtml(restoreT('ready'))}`; if (badge) badge.textContent = restoreT('jobSelected'); } @@ -220,11 +227,16 @@ function restoreArchiveFilterPopover(filters) { })) .filter((item) => item.filter); if (rows.length <= 1) return ''; + const groups = [true, false].map((current) => { + const group = rows.filter((row) => row.current === current); + if (!group.length) return ''; + return `${escHtml(restoreT(current ? 'archiveFilterCurrent' : 'archiveFilterPrevious'))}${group.map((row) => `${escHtml(row.filter)}`).join('')}`; + }).join(''); return ` ${escHtml(restoreT('archiveFilterHistoryTitle'))} - ${rows.map((row) => `${escHtml(restoreT(row.current ? 'archiveFilterCurrent' : 'archiveFilterPrevious'))}${escHtml(row.filter)}`).join('')} + ${groups} `; } @@ -427,7 +439,7 @@ function renderRestoreRuns(runs) { return `
${escHtml(restoreRunStateLabel(state))} - ${escHtml(run.job_key || '—')} + ${escHtml(run.job_name || restoreJobName(run.job_key))} ${escHtml(run.archive || '—')}
@@ -478,7 +490,7 @@ function renderRestoreHistory(payload) { const state = String(run.state || ''); return `
- ${escHtml(run.job_key || '—')} + ${escHtml(run.job_name || restoreJobName(run.job_key))} ${escHtml(run.archive || '—')}
@@ -518,7 +530,7 @@ async function restoreDeleteHistoryEntry(restoreId) { const id = String(restoreId || '').trim(); if (!id) return; const run = (restoreState.history || []).find((item) => String(item.restore_id || '') === id) || {}; - const label = run.job_key || id; + const label = run.job_name || restoreJobName(run.job_key) || id; const ok = await openRestoreHistoryDeleteConfirmModal(label, id); if (!ok) return; try { @@ -758,7 +770,48 @@ function _restoreBindTargetAutocomplete() { }); } +function restoreClearFileSelection() { + restoreState.filesRequest++; + restoreState.files = []; + restoreState.path = ''; + restoreState.selectedPath = ''; + restoreState.selectedName = ''; + restoreState.selectedType = ''; + restoreState.precheck = null; + restoreState.autoPrecheckKey = ''; + const source = document.getElementById('restore-source-path'); + if (source) source.value = ''; + const confirm = document.getElementById('restore-confirm-check'); + if (confirm) confirm.checked = false; + const output = document.getElementById('restore-precheck-output'); + if (output) output.textContent = ''; + const files = document.getElementById('restore-filelist'); + if (files) files.innerHTML = ''; + const breadcrumb = document.getElementById('restore-breadcrumb'); + if (breadcrumb) breadcrumb.innerHTML = ''; + renderRestorePrecheck(null); + _setRestoreAssistBusy(false); + restoreUpdateConfirmState(); + _restoreRenderSelectedBox(); +} + +function restoreClearArchives() { + restoreState.archive = ''; + restoreState.archives = []; + restoreState.archiveFilters = []; + restoreClearFileSelection(); + const select = document.getElementById('restore-archive-sel'); + if (select) select.innerHTML = ``; + renderRestoreArchiveList(); + renderRestoreSourceContext(); + _restoreRenderSelectionSummary(); +} + async function restoreInit() { + const selectedJob = restoreState.job; + const request = ++restoreState.sourceRequest; + restoreState.job = ''; + restoreClearArchives(); restoreState.completed = false; restoreSetLiveMode(false); const sel = document.getElementById('restore-job-sel'); @@ -773,6 +826,7 @@ async function restoreInit() { _restoreBindTargetAutocomplete(); const targetInput = document.getElementById('restore-target-path'); await restoreLoadAllowedTargetRoots(); + if (request !== restoreState.sourceRequest) return; if (targetInput && !targetInput.value.trim()) targetInput.value = `${_restorePrimaryAllowedRoot()}/`; _restoreRenderSelectionSummary(); _restoreRenderSelectedBox(); @@ -782,6 +836,8 @@ async function restoreInit() { try { const jobsRes = await fetch('/api/jobs', { credentials: 'include' }); const jobsData = await jobsRes.json(); + if (request !== restoreState.sourceRequest) return; + if (!jobsRes.ok) throw new Error(apiErrorMessage(jobsData, jobsRes.status)); const jobs = (jobsData.jobs || []).filter(j => !j.is_utility); restoreState.jobs = jobs; @@ -789,7 +845,7 @@ async function restoreInit() { if (job.is_utility) continue; const opt = document.createElement('option'); opt.value = job.key; - opt.textContent = job.display_name || job.name || job.key; + opt.textContent = job.name || job.display_name || job.key; sel.appendChild(opt); } @@ -797,6 +853,7 @@ async function restoreInit() { const checkRes = await fetch('/api/storage/check/jobs', { credentials: 'include' }); if (checkRes.ok) { const checkData = await checkRes.json(); + if (request !== restoreState.sourceRequest) return; for (const job of (checkData.jobs || [])) { const opt = document.createElement('option'); opt.value = job.key; @@ -806,28 +863,25 @@ async function restoreInit() { restoreState.jobs = (checkData.jobs || []).map((job) => ({ ...job, location: job.location || 'local' })); } } + if (restoreState.jobs.some(job => String(job.key) === String(selectedJob))) { + sel.value = selectedJob; + restoreState.job = selectedJob; + } renderRestoreJobSidebar(); renderRestoreSelectedJob(); renderRestoreSourceContext(); + if (restoreState.job) await restoreLoadArchives(); } catch (e) { + if (request !== restoreState.sourceRequest) return; _restoreMsg(restoreT('loadJobsError', { message: e.message }), true); } } async function restoreLoadArchives() { + const request = ++restoreState.sourceRequest; const jobKey = document.getElementById('restore-job-sel').value; restoreState.job = jobKey; - restoreState.archive = ''; - restoreState.path = ''; - restoreState.selectedPath = ''; - restoreState.selectedName = ''; - restoreState.selectedType = ''; - restoreState.autoPrecheckKey = ''; - restoreState.archives = []; - restoreState.archiveFilters = []; - renderRestoreSourceContext(); - const sel = document.getElementById('restore-archive-sel'); - if (sel) sel.innerHTML = ``; + restoreClearArchives(); _restoreMsg(''); if (!jobKey) { @@ -842,6 +896,7 @@ async function restoreLoadArchives() { try { const res = await fetch(`/api/restore/archives?job=${encodeURIComponent(jobKey)}`, { credentials: 'include' }); const data = await res.json(); + if (request !== restoreState.sourceRequest) return; if (!res.ok || data.error) { _restoreMsg(restoreT('error', { message: apiErrorMessage(data, res.status) }), true); return; } const sel = document.getElementById('restore-archive-sel'); @@ -859,6 +914,7 @@ async function restoreLoadArchives() { _restoreRenderSelectionSummary(); _restoreMsg(''); } catch (e) { + if (request !== restoreState.sourceRequest) return; _restoreMsg(restoreT('error', { message: e.message }), true); } } @@ -868,6 +924,10 @@ async function restoreBrowse(path) { const archive = document.getElementById('restore-archive-sel').value; if (!archive) return; + if (archive !== restoreState.archive) restoreClearFileSelection(); + const request = ++restoreState.filesRequest; + const sourceRequest = restoreState.sourceRequest; + const isCurrent = () => request === restoreState.filesRequest && sourceRequest === restoreState.sourceRequest; restoreState.archive = archive; restoreState.path = path; renderRestoreSourceContext(); @@ -889,13 +949,24 @@ async function restoreBrowse(path) { const url = `/api/restore/files?job=${encodeURIComponent(jobKey)}&archive=${encodeURIComponent(archive)}&path=${encodeURIComponent(path)}`; const res = await fetch(url, { credentials: 'include' }); const data = await res.json(); - if (!res.ok || data.error) { _restoreMsg(restoreT('error', { message: apiErrorMessage(data, res.status) }), true); return; } + if (!isCurrent()) return; + if (!res.ok || data?.error) { + const error = new Error(apiErrorMessage(data, res.status)); + error.archiveUnavailable = data?.code === 'restore_archive_unavailable'; + throw error; + } + if (!data || !Array.isArray(data.files)) throw new Error(apiErrorMessage({code: 'internal_error'})); _restoreMsg(''); _restoreRenderBreadcrumb(path); restoreState.files = data.files || []; _restoreRenderFiles(restoreState.files); } catch (e) { + if (!isCurrent()) return; + if (e.archiveUnavailable) restoreClearArchives(); + else restoreClearFileSelection(); + _restoreRenderSelectionSummary(); + if (filelist) filelist.innerHTML = ``; _restoreMsg(restoreT('error', { message: e.message }), true); } } @@ -1152,6 +1223,9 @@ function _setRestoreAssistBusy(busy) { } async function restoreRunPrecheck() { + const sourceRequest = restoreState.sourceRequest; + const filesRequest = restoreState.filesRequest; + const isCurrent = () => sourceRequest === restoreState.sourceRequest && filesRequest === restoreState.filesRequest; restoreSetLiveMode(false); hideEl('restore-assist-msg'); const source = restoreState.selectedPath || document.getElementById('restore-source-path')?.value || ''; @@ -1190,6 +1264,7 @@ async function restoreRunPrecheck() { }), }); const data = await res.json(); + if (!isCurrent()) return; if (!res.ok) throw new Error(restorePrecheckErrorMessage(data, res.status)); restoreState.precheck = data; renderRestorePrecheck(data); @@ -1214,13 +1289,16 @@ async function restoreRunPrecheck() { if (out) out.textContent = lines.join('\n'); showMsg('restore-assist-msg', data.ok ? 'success' : 'error', data.ok ? restoreT('precheckSuccess') : restoreT('precheckFailed')); } catch (err) { + if (!isCurrent()) return; restoreState.precheck = null; renderRestorePrecheck(null); if (out) out.textContent = ''; showMsg('restore-assist-msg', 'error', restoreT('precheckError', { message: err.message })); } finally { - _setRestoreAssistBusy(false); - restoreUpdateConfirmState(); + if (isCurrent()) { + _setRestoreAssistBusy(false); + restoreUpdateConfirmState(); + } } } diff --git a/ui/js/pages/settings.js b/ui/js/pages/settings.js index 94595a65..73be9618 100644 --- a/ui/js/pages/settings.js +++ b/ui/js/pages/settings.js @@ -2753,7 +2753,7 @@ function renderJobsImportPreview(d) { ` : ''; const jobTable = ` - + ${rows.map((r, idx) => { const feats = `${r?.features?.docker ? 'docker ' : ''}${r?.features?.vm ? 'vm' : ''}`.trim() || '—'; @@ -2768,7 +2768,7 @@ function renderJobsImportPreview(d) { return ` - + @@ -2884,7 +2884,7 @@ function renderProfileSecretsImportPreview(d) { ${settingsT('transfer.total', { count: stats.total })} · ${settingsT('transfer.presentCount', { count: stats.match })} · ${settingsT('transfer.differentCount', { count: stats.mismatch })} · ${settingsT('transfer.missingCount', { count: stats.missing })} · ${settingsT('transfer.profileMissingCount', { count: stats.profile_missing })}${stats.other ? ` · ${settingsT('transfer.other', { count: stats.other })}` : ''}
${settingsT('transfer.import')}${settingsT('transfer.name')}${settingsT('transfer.type')}${settingsT('transfer.location')}${settingsT('transfer.schedule')}${settingsT('transfer.features')}${settingsT('transfer.job')}${settingsT('transfer.passphrase')}${settingsT('transfer.mode')}
${settingsT('transfer.import')}${settingsT('transfer.name')}${settingsT('transfer.archivePrefix')}${settingsT('transfer.location')}${settingsT('transfer.schedule')}${settingsT('transfer.features')}${settingsT('transfer.job')}${settingsT('transfer.passphrase')}${settingsT('transfer.mode')}
${escHtml(r.name || r.job_key || '')}${escHtml(r.backup_type || '—')}${escHtml(r.archive_prefix || '—')} ${escHtml(r.location || '—')} ${escHtml(sch)} ${escHtml(feats)}
- + ${rows.map((r, idx) => { const pType = String(r.profile_type || '').toLowerCase(); @@ -3479,8 +3479,8 @@ function settingsTransferPassphraseStatusLabel(row) { function settingsTransferJobSubline(row) { const location = String(row?.location || '').trim(); - const type = String(row?.backup_type || '').trim(); - return [location, type && type !== location ? type : ''].filter(Boolean).join(' · ') || String(row?.job_key || ''); + const prefix = String(row?.archive_prefix || '').trim(); + return [location, prefix].filter(Boolean).join(' · ') || String(row?.job_key || ''); } function settingsTransferJobByKey(preview, key) { @@ -5732,7 +5732,6 @@ function renderSettingsRestoreTests(rt) {

${settingsT('forms.dryRunStrategy')}

- ${ftext('RESTORE_TEST_FORCE_CHUNK_TYPES', settingsT('forms.forceChunkTypes'), rt.RESTORE_TEST_FORCE_CHUNK_TYPES || 'vms,photos')} ${fnum('RESTORE_TEST_FULL_DRYRUN_MAX_ARCHIVE_GB', settingsT('forms.chunkFromSize'), rt.RESTORE_TEST_FULL_DRYRUN_MAX_ARCHIVE_GB || '500')}
diff --git a/ui/js/pages/storage.js b/ui/js/pages/storage.js index b0b36d4c..a1e8cd27 100644 --- a/ui/js/pages/storage.js +++ b/ui/js/pages/storage.js @@ -140,7 +140,7 @@ function storageRepositoryName(repo) { } function storageJobName(repo, job) { - return String(repo?.job_name || job?.name || repo?.display_name || repo?.used_by?.[0] || repo?.source_job_keys?.[0] || '').trim(); + return String(job?.name || job?.display_name || repo?.job_name || repo?.display_name || '').trim(); } function storageJobsForRepository(repo) { @@ -154,33 +154,17 @@ function storageJobsForRepository(repo) { const direct = jobs.find((job) => String(job.key || '') === jobKey); if (direct) matches.push(direct); } - if (!matches.length) { - const fallback = jobs.find((job) => - String(job.backup_type || '').toLowerCase() === String(repo?.backup_type || '').toLowerCase() - && String(job.location || '').toLowerCase() === String(repo?.location || '').toLowerCase() - ); - if (fallback) matches.push(fallback); - } const seen = new Set(); return matches.filter((job) => { const key = String(job?.key || ''); if (!key || seen.has(key)) return false; seen.add(key); return true; - }); + }).sort((a, b) => storageJobName(repo, a).localeCompare(storageJobName(repo, b))); } function storageArchivePrefixFromJob(job) { - const key = String(job?.key || job?.job_key || '').trim(); - for (const location of ['storagebox', 'local', 'usb', 'smb']) { - const suffix = `_${location}`; - if (key.endsWith(suffix)) { - const typeId = key.slice(0, -suffix.length); - return typeId ? `${typeId}-backup` : ''; - } - } - const typeId = key.includes('_') ? key.split('_').slice(0, -1).join('_') : key; - return typeId ? `${typeId}-backup` : ''; + return String(job?.archive_prefix || '').trim(); } function storageArchiveFilterFromJob(job) { @@ -198,15 +182,18 @@ function storageRetentionFromJob(job) { }; } -function storageRetentionSummary(job) { +function storageRetentionTableHtml(job) { const retention = storageRetentionFromJob(job); - const values = [ - storageT('storage.repositoryRetentionDaily', { count: retention.daily || '0' }), - storageT('storage.repositoryRetentionWeekly', { count: retention.weekly || '0' }), - storageT('storage.repositoryRetentionMonthly', { count: retention.monthly || '0' }), - storageT('storage.repositoryRetentionYearly', { count: retention.yearly || '0' }), - ]; - return values.join(', '); + const rows = ['Daily', 'Weekly', 'Monthly', 'Yearly'].map((period) => { + const value = storageT(`storage.repositoryRetention${period}`, { count: retention[period.toLowerCase()] || '0' }); + const limit = storageT(`storage.repositoryRetention${period}Limit`); + return `
`; + }).join(''); + return `
${settingsT('transfer.import')}${settingsT('transfer.type')}${settingsT('transfer.profile')}${settingsT('transfer.targetProfile')}${settingsT('transfer.secret')}${settingsT('transfer.status')}${settingsT('transfer.targetPath')}
${settingsT('transfer.import')}${settingsT('transfer.archivePrefix')}${settingsT('transfer.profile')}${settingsT('transfer.targetProfile')}${settingsT('transfer.secret')}${settingsT('transfer.status')}${settingsT('transfer.targetPath')}
${escHtml(value)}${escHtml(limit)}
+ + + ${rows} +
${escHtml(storageT('storage.repositoryMaintenanceRetention'))}
${escHtml(storageT('storage.repositoryRetentionPoints'))}${escHtml(storageT('storage.repositoryRetentionMaximum'))}
`; } function storageMaintenancePruneDetailsHtml(repo, job) { @@ -217,10 +204,7 @@ function storageMaintenancePruneDetailsHtml(repo, job) { const filter = storageT('storage.repositoryMaintenanceArchiveFilter', { filter: storageArchiveFilterFromJob(job) || '-', }); - const retention = storageT('storage.repositoryMaintenanceRetention', { - retention: storageRetentionSummary(job), - }); - return `
${escHtml(source)}
${escHtml(filter)}
${escHtml(retention)}`; + return `
${escHtml(source)}
${escHtml(filter)}
${storageRetentionTableHtml(job)}`; } function updateStorageMaintenanceRetentionPreview() { @@ -352,10 +336,10 @@ function storageRepositoryStatus(repo) { } function storageRepositoryIcon(repo, job, large = false) { - const icon = typeof resolveJobIcon === 'function' ? resolveJobIcon(job || repo) : repo?.backup_type; + const icon = typeof resolveJobIcon === 'function' ? resolveJobIcon(job || repo) : 'archive'; const color = typeof resolveJobIconColor === 'function' ? resolveJobIconColor(job || repo) : ''; const colorClass = color ? ` type-icon-color-${color}` : ''; - return `${typeIcon(icon)}`; + return `${typeIcon(icon)}`; } function storageGroupRows(data, repos) { @@ -1629,12 +1613,12 @@ function openStorageMaintenanceConfirm(repositoryKey, action, mode) { if (title) title.textContent = storageT('storage.repositoryMaintenanceConfirmTitle'); if (description) description.textContent = storageT(confirmKey); const pruneDetails = action === 'prune' - ? `${storageMaintenancePruneDetailsHtml(repo || {}, job)}` + ? `
${storageMaintenancePruneDetailsHtml(repo || {}, job)}
` : ''; const selector = action === 'prune' && jobs.length > 1 - ? `` + ? `` : ''; - if (info) info.innerHTML = `
${escHtml(storageRepositoryTitle(repo || {}, job))}
${escHtml(storageT('storage.repositoryMaintenanceConfirmAction', { action: storageMaintenanceTitle(resultKey) }))}${pruneDetails}${selector}
`; + if (info) info.innerHTML = `
${escHtml(storageRepositoryTitle(repo || {}, job))}
${escHtml(storageT('storage.repositoryMaintenanceConfirmAction', { action: storageMaintenanceTitle(resultKey) }))}${pruneDetails}${selector}
`; storageState.maintenanceConfirmation = { resolve, action, diff --git a/ui/js/pages/wizard.js b/ui/js/pages/wizard.js index edf5667b..cf57d3fa 100644 --- a/ui/js/pages/wizard.js +++ b/ui/js/pages/wizard.js @@ -6,6 +6,8 @@ window.BBUI.wizardState = window.BBUI.wizardState || { step: 1, mode: 'create', existingJobKey: '', + jobId: '', + jobIdRequest: 0, original: null, storages: [], selectedStorageKey: '', @@ -43,6 +45,9 @@ function wizardT(key, params = {}) { function wizardApiErrorMessage(payload, status = 0) { const data = payload && typeof payload === 'object' ? payload : {}; + if (data.code === 'job_settings_invalid') return apiErrorMessage(data, status); + if (data.code === 'job_name_too_long') return wizardT('wizard.validationJobNameLength'); + if (data.code === 'job_id_exists') return wizardT('wizard.jobIdExists'); if (data.code === 'retention_invalid') return wizardT('wizard.validationRetentionInvalid'); if (data.code === 'retention_all_zero') return wizardT('wizard.validationRetentionRequired'); for (const key of ['details', 'message', 'error']) { @@ -112,13 +117,13 @@ function _wizardUniqueList(values) { return out; } -function _wizardArchivePrefixFromTypeId(typeId) { - const clean = String(typeId || '').trim().toLowerCase(); - return /^[a-z0-9_]+$/.test(clean) ? `${clean}-backup` : ''; +function _wizardArchivePrefix(prefix) { + const clean = String(prefix || '').trim(); + return /^[A-Za-z0-9_.-]+$/.test(clean) ? clean : ''; } function wizardArchivePrefixRows() { - const currentPrefix = _wizardArchivePrefixFromTypeId(document.getElementById('wiz-type-id')?.value || ''); + const currentPrefix = _wizardArchivePrefix(document.getElementById('wiz-archive-prefix')?.value || ''); const prefixes = _wizardUniqueList([ currentPrefix, ...(Array.isArray(wizardState.archivePrefixes) ? wizardState.archivePrefixes : []), @@ -149,11 +154,16 @@ function wizardRenderArchivePrefixSummary() { function wizardArchivePrefixPopover(rows) { const cleanRows = (Array.isArray(rows) ? rows : []).filter((row) => String(row?.filter || '').trim()); if (cleanRows.length <= 1) return ''; + const groups = [true, false].map((current) => { + const group = cleanRows.filter((row) => !!row.current === current); + if (!group.length) return ''; + return `${escHtml(wizardT(current ? 'wizard.archiveFilterCurrentBadge' : 'wizard.archiveFilterPreviousBadge'))}${group.map((row) => `${escHtml(row.filter)}`).join('')}`; + }).join(''); return ` ${escHtml(wizardT('wizard.archivePatternHistoryTitle'))} - ${cleanRows.map((row) => `${escHtml(wizardT(row.current ? 'wizard.archiveFilterCurrentBadge' : 'wizard.archiveFilterPreviousBadge'))}${escHtml(row.filter)}`).join('')} + ${groups} `; } @@ -508,11 +518,41 @@ function _setWizardFormDisabled(disabled) { }); } -function openWizard() { +async function wizardLoadNewJobId(request) { + try { + const res = await fetch('/api/wizard/new-job-id', {cache: 'no-store'}); + const data = await res.json(); + if (request !== wizardState.jobIdRequest) return; + if (!res.ok) throw new Error(wizardApiErrorMessage(data, res.status)); + if (typeof data?.job_id !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(data.job_id)) { + throw new Error(wizardT('wizard.jobIdInvalid')); + } + wizardState.jobId = data.job_id; + document.getElementById('wiz-job-id').value = data.job_id; + } catch (err) { + if (request !== wizardState.jobIdRequest) return; + _wizardShowError(1, wizardT('wizard.jobIdLoadFailed', {message: err.message})); + } finally { + if (request === wizardState.jobIdRequest) { + _wizardUpdateStepNavigation(); + if (!wizardState.closeSnapshotTouched) _wizardCaptureCloseSnapshot(); + } + } +} + +function openWizard(existingJobKey = '') { + // The New Job button also calls this function directly as an event handler. + existingJobKey = typeof existingJobKey === 'string' ? existingJobKey : ''; wizardBindCloseDirtyTracking(); wizardBindRuntimeControls(); + _setWizardFormDisabled(false); wizardState.mode = 'create'; - wizardState.existingJobKey = ''; + wizardState.existingJobKey = existingJobKey; + wizardState.jobId = existingJobKey; + const request = ++wizardState.jobIdRequest; + document.getElementById('wiz-job-id-group').hidden = false; + document.getElementById('wiz-job-id-group').classList.remove('hidden'); + document.getElementById('wiz-job-id').value = existingJobKey; wizardState.original = null; wizardState.step = 1; wizardState.unlockedStep = 1; @@ -521,7 +561,7 @@ function openWizard() { const title = document.getElementById('wizard-modal-title'); if (title) title.textContent = wizardT('wizard.newTitle'); document.getElementById('wiz-job-name').value = ''; - document.getElementById('wiz-type-id').value = ''; + document.getElementById('wiz-archive-prefix').value = ''; document.getElementById('wiz-icon').value = ''; document.getElementById('wiz-icon-color').value = ''; document.getElementById('wiz-description').value = ''; @@ -578,10 +618,12 @@ function openWizard() { wizardUpdateIconPreview(); wizardRenderArchivePrefixSummary(); wizardState.loadingPromise = Promise.all([ + existingJobKey ? Promise.resolve() : wizardLoadNewJobId(request), wizardLoadStorageTargets(), wizardLoadRepositories(), wizardLoadRuntimeInventory(), ]).finally(() => { + if (request !== wizardState.jobIdRequest) return; wizardAutoFill(); if (!wizardState.closeSnapshotTouched) _wizardCaptureCloseSnapshot(); }); @@ -597,9 +639,23 @@ function _wizardFillFromJob(job) { wizardState.remoteRepoStatus = null; wizardState.archivePrefixes = _wizardUniqueList(Array.isArray(job.archive_prefixes) ? job.archive_prefixes : []); document.getElementById('wiz-job-name').value = job.job_name || ''; - document.getElementById('wiz-type-id').value = (job.type_id || '').toLowerCase(); + document.getElementById('wiz-archive-prefix').value = job.archive_prefix || ''; + document.getElementById('wiz-job-id-group').hidden = false; + document.getElementById('wiz-job-id-group').classList.remove('hidden'); + wizardState.jobId = job.job_id || ''; + document.getElementById('wiz-job-id').value = wizardState.jobId; document.getElementById('wiz-icon').value = (job.icon || '').toLowerCase(); - document.getElementById('wiz-icon-color').value = (job.icon_color || '').toLowerCase(); + const colorSelect = document.getElementById('wiz-icon-color'); + colorSelect.querySelectorAll('option[data-saved-theme]').forEach((option) => option.remove()); + const savedColor = (job.icon_color || '').toLowerCase(); + if (['theme-blue', 'theme-orange', 'theme-purple', 'theme-green'].includes(savedColor)) { + const option = document.createElement('option'); + option.value = savedColor; + option.textContent = wizardT('wizard.savedThemeColor'); + option.dataset.savedTheme = 'true'; + colorSelect.appendChild(option); + } + colorSelect.value = savedColor; document.getElementById('wiz-description').value = job.description || ''; document.getElementById('wiz-location').value = job.location || 'local'; _wizardSetRuntimeControl('docker', job.docker_control || { mode: job.use_docker ? 'all' : 'none' }); @@ -634,7 +690,8 @@ function _wizardFillFromJob(job) { } async function openWizardForJob(jobKey, mode = 'edit') { - openWizard(); + openWizard(jobKey); + const request = wizardState.jobIdRequest; wizardState.mode = mode; wizardState.existingJobKey = jobKey; const title = document.getElementById('wizard-modal-title'); @@ -642,13 +699,15 @@ async function openWizardForJob(jobKey, mode = 'edit') { _setWizardFormDisabled(true); try { await wizardState.loadingPromise; + if (request !== wizardState.jobIdRequest) return; const res = await fetch(`/api/wizard/job?job_key=${encodeURIComponent(jobKey)}`); const data = await res.json(); + if (request !== wizardState.jobIdRequest) return; if (!res.ok) throw new Error(wizardApiErrorMessage(data, res.status)); const job = data.job || {}; _wizardFillFromJob(job); wizardState.original = { - type_id: (job.type_id || '').toLowerCase(), + archive_prefix: job.archive_prefix || '', location: job.location || 'local', repository_key: String(job.repository_key || '').trim(), use_docker: !!job.use_docker, @@ -663,11 +722,14 @@ async function openWizardForJob(jobKey, mode = 'edit') { _renderWizardStep(wizardState.step); _wizardCaptureCloseSnapshot(); } catch (err) { + if (request !== wizardState.jobIdRequest) return; closeWizard({ force: true }); showMsg('jobs-message', 'error', wizardT('wizard.loadFailed', { message: err.message })); } finally { - _setWizardFormDisabled(false); - _wizardUpdateStepNavigation(); + if (request === wizardState.jobIdRequest) { + _setWizardFormDisabled(false); + _wizardUpdateStepNavigation(); + } } } @@ -675,7 +737,7 @@ function wizardNeedsScriptRegeneration(params) { if ((wizardState.mode || 'create') === 'create') return true; const orig = wizardState.original; if (!orig) return true; - if (params.type_id !== orig.type_id) return true; + if (params.archive_prefix !== orig.archive_prefix) return true; if (params.location !== orig.location) return true; if (params.repository_key !== orig.repository_key) return true; if (!!params.use_docker !== !!orig.use_docker) return true; @@ -689,6 +751,8 @@ function closeWizard(options = {}) { if (!options?.force && !_wizardConfirmDiscard()) return false; document.getElementById('wizard-modal').classList.add('hidden'); document.body.classList.remove('wizard-modal-open'); + wizardState.jobIdRequest++; + wizardState.jobId = ''; wizardState.closeSnapshot = ''; wizardState.closeSnapshotTouched = false; return true; @@ -718,11 +782,12 @@ function _renderWizardStep(n) { } function _wizardUpdateStepNavigation() { + document.getElementById('wizard-next-btn').disabled = !wizardState.jobId; [1,2,3,4,5,7,8,9].forEach((step) => { const dot = document.getElementById(`wstep-dot-${step}`); if (!dot) return; const skipped = !_wizardStepEnabled(step); - const locked = step > Number(wizardState.unlockedStep || 1); + const locked = !wizardState.jobId || step > Number(wizardState.unlockedStep || 1); dot.disabled = skipped || locked; dot.setAttribute('aria-disabled', String(skipped || locked)); dot.classList.toggle('wizard-step-skipped', skipped); @@ -755,8 +820,6 @@ function wizardAutoFill() { wizardSetStorageOptions(); const storage = wizardSelectedStorage(); wizardUpdateStorageTargetHint(storage); - // If icon not explicitly chosen, keep "auto" (empty) and let rendering - // derive it from backup_type/type_id. if (iconEl && iconEl.value === '') iconEl.value = ''; wizardUpdateIconPreview(); } @@ -769,8 +832,7 @@ function wizardIconMarkup(kind) { function wizardEffectiveIcon() { const chosen = (document.getElementById('wiz-icon')?.value || '').trim().toLowerCase(); if (chosen) return chosen; - const typeId = (document.getElementById('wiz-type-id')?.value || '').trim().toLowerCase(); - return typeId || 'sonstiges'; + return 'archive'; } function wizardUpdateIconPreview() { @@ -785,9 +847,9 @@ function wizardUpdateIconPreview() { 'green', 'lime', 'violet', 'amber', 'orange', 'red', 'rose', - 'teal', 'cyan', 'gray', + 'teal', 'cyan', 'gray', 'theme-blue', 'theme-orange', 'theme-purple', 'theme-green', ]); - box.className = `type-icon type-icon-${iconKey || 'sonstiges'}${knownColor.has(colorKey) ? ` type-icon-color-${colorKey}` : ''}`; + box.className = `type-icon${knownColor.has(colorKey) ? ` type-icon-color-${colorKey}` : ''}`; label.textContent = iconKey || wizardT('wizard.automatic'); } @@ -838,7 +900,8 @@ function _wizardCollectParams() { const dockerMode = _wizardRuntimeMode('docker'); const vmMode = _wizardRuntimeMode('vm'); return { - type_id: (document.getElementById('wiz-type-id').value || '').trim().toLowerCase(), + job_id: wizardState.jobId || '', + archive_prefix: (document.getElementById('wiz-archive-prefix').value || '').trim(), icon: (document.getElementById('wiz-icon').value || '').trim().toLowerCase(), icon_color: (document.getElementById('wiz-icon-color').value || '').trim().toLowerCase(), job_name: (document.getElementById('wiz-job-name').value || '').trim(), @@ -1174,10 +1237,12 @@ function wizardSourcePathsClick(event) { function _wizardValidate(step) { wizardClearError(step); const p = _wizardCollectParams(); + if (!p.job_id) { _wizardShowError(step, wizardT('wizard.jobIdNotReady')); return false; } if (step === 1) { if (!p.job_name) { _wizardShowError(1, wizardT('wizard.validationJobName')); return false; } - if (!p.type_id) { _wizardShowError(1, wizardT('wizard.validationTypeId')); return false; } - if (!/^[a-z0-9_]+$/.test(p.type_id)) { + if ([...p.job_name].length > 100) { _wizardShowError(1, wizardT('wizard.validationJobNameLength')); return false; } + if (!p.archive_prefix) { _wizardShowError(1, wizardT('wizard.validationTypeId')); return false; } + if (!/^[A-Za-z0-9_.-]+$/.test(p.archive_prefix)) { _wizardShowError(1, wizardT('wizard.validationTypeFormat')); return false; } @@ -1412,10 +1477,16 @@ async function saveWizardJob() { const data = await res.json(); if (!res.ok) throw new Error(wizardApiErrorMessage(data, res.status)); + // Retrying a failed schedule write edits the job that was already saved. + wizardState.existingJobKey = data.job_id; + wizardState.jobId = data.job_id; + document.getElementById('wiz-job-id').value = data.job_id; + wizardState.mode = 'edit'; + // Save schedule changes and surface crontab/application failures. const schedEnabled = document.getElementById('wiz-sched-enabled').checked; if (schedEnabled || wizardState.originalSchedule) { - const jobKey = `${params.type_id}_${params.location}`; + const jobKey = data.job_id; const cron = _wizardBuildCron(); const sRes = await fetch('/api/schedules', { method: 'PUT', @@ -1436,7 +1507,7 @@ async function saveWizardJob() { closeWizard({ force: true }); jobsState.loaded = false; await refreshJobs(); - showMsg('jobs-message', 'success', wizardT('wizard.saved', { key: `${params.type_id}_${params.location}` })); + showMsg('jobs-message', 'success', wizardT('wizard.saved', { key: params.job_name })); await window.BBUI?.setupWizard?.resumeAfterExternalSave?.('job'); } catch (err) { errEl.textContent = wizardT('wizard.saveError', { message: err.message }); diff --git a/ui/remaining-ui-redesign.css b/ui/remaining-ui-redesign.css index 2b9b0cc7..b759c9ac 100644 --- a/ui/remaining-ui-redesign.css +++ b/ui/remaining-ui-redesign.css @@ -845,6 +845,26 @@ gap: var(--ui-space-2); } +#storage-maintenance-confirm-info .modal-info-text { + width: 100%; + overflow-wrap: anywhere; +} + +.storage-maintenance-retention-table { + margin: 10px 0; +} + +.storage-maintenance-retention-table caption { + padding-bottom: 4px; + text-align: left; + font-weight: 600; +} + +.storage-maintenance-retention-table th, +.storage-maintenance-retention-table td { + padding: 6px 8px; +} + .storage-repository-management { display: grid; gap: var(--ui-space-4); diff --git a/ui/style.css b/ui/style.css index c2798ae8..60bef1e1 100644 --- a/ui/style.css +++ b/ui/style.css @@ -1194,10 +1194,10 @@ body.wizard-modal-open { height: 18px; } -.type-icon-flash { background: var(--state-info-dim); border-color: var(--state-info-border); color: var(--accent-hover); } -.type-icon-appdata { background: var(--loc-storagebox-dim); border-color: var(--state-warning-border); color: var(--loc-storagebox); } -.type-icon-photos { background: rgba(188, 123, 255, 0.12); border-color: rgba(188, 123, 255, 0.3); color: var(--violet); } -.type-icon-vms { background: var(--success-dim); border-color: var(--state-success-border); color: var(--success); } +.type-icon-color-theme-blue { background: var(--state-info-dim); border-color: var(--state-info-border); color: var(--accent-hover); } +.type-icon-color-theme-orange { background: var(--loc-storagebox-dim); border-color: var(--state-warning-border); color: var(--loc-storagebox); } +.type-icon-color-theme-purple { background: rgba(188, 123, 255, 0.12); border-color: rgba(188, 123, 255, 0.3); color: var(--violet); } +.type-icon-color-theme-green { background: var(--success-dim); border-color: var(--state-success-border); color: var(--success); } .type-icon-color-blue { background: rgba(59, 130, 246, 0.14) !important; border-color: rgba(59, 130, 246, 0.45) !important; color: #2563eb !important; } .type-icon-color-indigo { background: rgba(99, 102, 241, 0.14) !important; border-color: rgba(99, 102, 241, 0.45) !important; color: #4f46e5 !important; } .type-icon-color-purple { background: rgba(147, 51, 234, 0.14) !important; border-color: rgba(147, 51, 234, 0.45) !important; color: #9333ea !important; } @@ -5436,6 +5436,10 @@ body.wizard-modal-open { flex: 0 0 auto; } +#wizard-modal .modal-footer { + min-height: 65px; +} + .setup-wizard-modal { max-width: 760px; } .setup-wizard-body { @@ -5609,9 +5613,9 @@ body.wizard-modal-open { .wizard-body { padding: 20px 24px 8px; - height: 448px; + height: 480px; min-height: 0; - flex: 1 1 448px; + flex: 1 1 480px; overflow-y: auto; overscroll-behavior-y: contain; } @@ -5632,7 +5636,26 @@ body.wizard-modal-open { } #wizard-step-1 .wizard-step1-layout { - gap: 8px 10px; + padding: 0; + gap: 6px 10px; +} + +#wizard-modal .wizard-body:has(#wizard-step-1:not(.hidden)) { + padding-top: 12px; +} + +#wizard-step-1 .wizard-step1-layout > .form-group { + gap: 4px; +} + +#wizard-step-1 .wizard-label-inline { + flex-wrap: wrap; + gap: 2px 8px; +} + +#wiz-job-name-hint { + color: var(--text-muted); + font-size: 12px; } .wizard-archive-prefix-summary {