From 0adbb7a93c44d974376aa23ffdf05860cbd9d813 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 1 Sep 2026 09:13:16 +0200 Subject: [PATCH] create: rebuild the files cache from an archive of the same group When the local files cache is missing, borg rebuilds it by reading the archive this one continues from the repository. That archive was looked up by matching the series name only: archives = self.manifest.archives.list(match=[self.archive_name], ...) Archive series names are not unique across hosts, so in a repository shared by multiple machines or users this could pick a foreign archive: if host2 backed up its own "home" series after host1, host1 would rebuild its files cache from host2's archive. Almost nothing matches there, so borg reads and chunks everything again - the files cache silently stops working for everyone but the host that happened to write last. The lookup now matches the archive attributes given by the new --group-by option, defaulting to name,host. Valid keys are name, host and user; tags are not usable because a new archive is not known to belong to the tag group of an existing one, and an empty value is rejected because an archive must not continue an arbitrary unrelated archive. The host and user an archive gets stamped with now come from archive_hostname() / archive_username() in helpers, so the metadata written by create and the lookup done by the cache can not drift apart. Note that the local files cache file name is still derived from the series name alone. It lives on the client, so it is per host already, and keeping the name avoids invalidating everybody's files cache. --- src/borg/archive.py | 7 ++- src/borg/archiver/create_cmd.py | 36 +++++++++++++- src/borg/cache.py | 38 ++++++++++++-- src/borg/helpers/__init__.py | 3 +- src/borg/helpers/misc.py | 13 +++++ src/borg/helpers/parseformat.py | 21 ++++++-- .../testsuite/archiver/create_cmd_test.py | 49 +++++++++++++++++++ src/borg/testsuite/cache_test.py | 31 ++++++++++++ 8 files changed, 183 insertions(+), 15 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index bc45bdad41..a2bc0fd502 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -10,7 +10,6 @@ from contextlib import contextmanager from datetime import timedelta from functools import partial -from getpass import getuser from io import BytesIO from itertools import groupby, zip_longest from collections.abc import Iterator @@ -32,6 +31,7 @@ from .helpers import BackupSymlinkParentError, BackupPathTraversalError from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError, BackupTimeoutError from .helpers import HardLinkManager +from .helpers import archive_hostname, archive_username from .helpers import ChunkIteratorFileWrapper, open_item from .helpers import Error, IntegrityError, set_ec, sig_int from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns @@ -51,7 +51,6 @@ from .manifest import Manifest from .patterns import PathPrefixPattern, FnmatchPattern, IECommand from .item import Item, ArchiveItem, ItemDiff -from . import platform from .platform import acl_get, acl_set, set_flags, get_flags, set_times, swidth from .repository import Repository, NoManifestError from .repoobj import RepoObj @@ -770,8 +769,8 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m "item_ptrs": item_ptrs, # see #1473 "command_line": join_cmd(sys.argv), "cwd": self.cwd, - "hostname": os.environ.get("BORG_HOSTNAME") or platform.get_hostname(), - "username": os.environ.get("BORG_USERNAME") or getuser(), + "hostname": archive_hostname(), + "username": archive_username(), "time": nominal.isoformat(timespec="microseconds"), "start": start.isoformat(timespec="microseconds"), "end": end.isoformat(timespec="microseconds"), diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 3111d5da51..b18b4c24f7 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -17,6 +17,8 @@ from ..constants import * # NOQA from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec from ..helpers import archivename_validator, DigestAlgos, FilesCacheMode, files_cache_mode_no_ctime +from ..helpers import FilesCacheGroupBySpec +from ..helpers.parseformat import FILES_CACHE_GROUP_BY_KEYS from ..helpers import octal_int, nonnegative_seconds from ..helpers import read_input_map from ..helpers import eval_escapes @@ -313,7 +315,12 @@ def create_inner(archive, cache, fso): logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed)) if not dry_run: with Cache( - repository, manifest, progress=args.progress, cache_mode=args.files_cache_mode, archive_name=args.name + repository, + manifest, + progress=args.progress, + cache_mode=args.files_cache_mode, + archive_name=args.name, + archive_group_by=tuple(args.group_by.split(",")), ) as cache: archive = Archive( manifest, @@ -791,6 +798,21 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): done by comparing multiple file metadata values with previous values kept in the files cache. + The files cache is kept locally, one per archive series. If it is missing (e.g. on a + fresh machine or after the local cache was removed), borg rebuilds it by reading the + archive this one continues from the repository. That archive is the newest one having + the same archive attributes as given by ``--group-by``, by default the same series + name and the same host. Matching the series name alone would be wrong in a repository + shared by multiple machines or users, because they may use the same series name for + their own, unrelated data - borg would then rebuild the files cache from a foreign + archive, where almost nothing matches, and read and chunk everything again. + + Give ``--group-by name`` if the same series is written by different hosts on purpose + and they see the same files, or add ``user`` if one host backs up the same series as + different users. Beware of grouping by an attribute that is not stable over time: if + e.g. the hostname changes for every backup run (as it might for containers), borg will + never find an archive to rebuild the files cache from. + This comparison can operate in different modes as given by ``--files-cache``: - ctime,size,inode (default on POSIX systems) @@ -1221,6 +1243,18 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): help="operate files cache in MODE. default: %s (on Windows: %s, because ctime is " "file creation time there)." % (FILES_CACHE_MODE_UI_DEFAULT_POSIX, FILES_CACHE_MODE_UI_DEFAULT_WIN32), ) + fs_group.add_argument( + "--group-by", + metavar="KEYS", + dest="group_by", + action=Highlander, + type=FilesCacheGroupBySpec, + default="name,host", + help="comma-separated list of archive attributes identifying the archives this archive " + "belongs to; the newest of them is the archive the files cache is rebuilt from, if the " + "local files cache is missing. valid keys are: {}; default is: " + "name,host".format(", ".join(FILES_CACHE_GROUP_BY_KEYS)), + ) fs_group.add_argument( "--files-changed", metavar="MODE", diff --git a/src/borg/cache.py b/src/borg/cache.py index 475b0ad882..5cf26812ef 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -26,6 +26,7 @@ from .constants import CHUNKINDEX_SMALL_FRAGMENT_CAP, CHUNKINDEX_MERGE_ATTEMPTS, CHUNKINDEX_INVALID_SENTINEL from .hashindex import ChunkIndex, ChunkIndexEntry, ChunkIndexEntryFormat from .helpers import get_cache_dir +from .helpers import archive_hostname, archive_username from .helpers import chunkit from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list from .helpers import format_file_size, safe_encode @@ -58,6 +59,27 @@ def files_cache_name(archive_name, files_cache_name="files"): return files_cache_name + "." + suffix +def archive_group_patterns(archive_name, group_by): + """ + Build the match patterns selecting the archives belonging to the same group as a new archive. + + The new archive is named *archive_name* and gets stamped with this host and this user, so the + patterns describe the archives it continues, e.g. ["name:home", "host:myhost"] for the default + grouping. See "borg help match-archives" for the pattern syntax. + """ + patterns = [] + for group_by_key in group_by: + if group_by_key == "name": + patterns.append(f"name:{archive_name}") + elif group_by_key == "host": + patterns.append(f"host:{archive_hostname()}") + elif group_by_key == "user": + patterns.append(f"user:{archive_username()}") + else: + raise ValueError(f"invalid group-by key: {group_by_key}") + return patterns + + def discover_files_cache_names(path, files_cache_name="files"): """ Return a list of all files cache file names in the given directory. @@ -200,6 +222,7 @@ def __new__( progress=False, cache_mode=FILES_CACHE_MODE_DISABLED, archive_name=None, + archive_group_by=(), start_backup=None, ): return AdHocWithFilesCache( @@ -209,6 +232,7 @@ def __new__( progress=progress, cache_mode=cache_mode, archive_name=archive_name, + archive_group_by=archive_group_by, start_backup=start_backup, ) @@ -225,8 +249,9 @@ class FilesCacheMixin: FILES_CACHE_NAME = "files" - def __init__(self, cache_mode, archive_name=None, start_backup=None): + def __init__(self, cache_mode, archive_name=None, archive_group_by=(), start_backup=None): self.archive_name = archive_name # ideally a SERIES name + self.archive_group_by = archive_group_by # archive attributes identifying the previous archive assert not ("c" in cache_mode and "m" in cache_mode) assert "d" in cache_mode or "c" in cache_mode or "m" in cache_mode self.cache_mode = cache_mode @@ -294,9 +319,13 @@ def _build_files_cache(self): from .archive import Archive - # get the latest archive with the IDENTICAL name, supporting archive series: + # Get the latest archive of the same group, supporting archive series. Matching the name + # alone is not enough in a repository shared by multiple hosts or users, because they may + # use the same series name for their own, unrelated data - we would then build our files + # cache from a foreign archive, which just wastes time as almost nothing would match. + match = archive_group_patterns(self.archive_name, self.archive_group_by) try: - archives = self.manifest.archives.list(match=[self.archive_name], sort_by=["ts"], last=1) + archives = self.manifest.archives.list(match=match, sort_by=["ts"], last=1) except PermissionDenied: # maybe repo is in write-only mode? archives = None if not archives: @@ -1207,13 +1236,14 @@ def __init__( progress=False, cache_mode=FILES_CACHE_MODE_DISABLED, archive_name=None, + archive_group_by=(), start_backup=None, ): """ :param warn_if_unencrypted: print warning if accessing unknown unencrypted repository :param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison """ - FilesCacheMixin.__init__(self, cache_mode, archive_name, start_backup) + FilesCacheMixin.__init__(self, cache_mode, archive_name, archive_group_by, start_backup) ChunksMixin.__init__(self) assert isinstance(manifest, Manifest) self.manifest = manifest diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 96649b65c1..b3d5216d51 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -28,7 +28,7 @@ from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime from .fs import HardLinkManager -from .misc import sysinfo, log_multi, consume +from .misc import sysinfo, log_multi, consume, archive_hostname, archive_username from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper from .parseformat import octal_int, bin_to_hex, hex_to_bin, safe_encode, safe_decode from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd @@ -39,6 +39,7 @@ FilesystemDirSpec, SortBySpec, GroupBySpec, + FilesCacheGroupBySpec, CompressionSpec, ChunkerParams, DigestAlgos, diff --git a/src/borg/helpers/misc.py b/src/borg/helpers/misc.py index d6a027b632..0316835fe1 100644 --- a/src/borg/helpers/misc.py +++ b/src/borg/helpers/misc.py @@ -4,6 +4,7 @@ import platform # python stdlib import - if this fails, check that cwd != src/borg/ import sys from collections import deque +from getpass import getuser from itertools import islice from ..logger import create_logger @@ -15,6 +16,18 @@ from ..constants import ROBJ_FILE_STREAM +def archive_hostname(): + """Return the hostname a new archive is stamped with.""" + from ..platform import get_hostname + + return os.environ.get("BORG_HOSTNAME") or get_hostname() + + +def archive_username(): + """Return the username a new archive is stamped with.""" + return os.environ.get("BORG_USERNAME") or getuser() + + def sysinfo(): show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower() if show_sysinfo == "no": diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 67670c9bd2..75ee35eb34 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -582,24 +582,35 @@ def SortBySpec(text): return text.replace("timestamp", "ts").replace("archive", "name") -def GroupBySpec(text): +def GroupBySpec(text, valid_keys=None, allow_ungrouped=True): """Validate a comma-separated list of group-by keys. "" and "none" mean: do not group.""" from ..manifest import AI_GROUP_BY_KEYS + valid_keys = AI_GROUP_BY_KEYS if valid_keys is None else valid_keys if text in ("", "none"): + if not allow_ungrouped: + raise ArgumentTypeError("At least one group-by key is required (valid keys: %s)" % ", ".join(valid_keys)) return "" # idempotency: the normalized value must pass validation again seen = set() for group_key in text.split(","): - if group_key not in AI_GROUP_BY_KEYS: - raise ArgumentTypeError( - "Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(AI_GROUP_BY_KEYS)) - ) + if group_key not in valid_keys: + raise ArgumentTypeError("Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(valid_keys))) if group_key in seen: raise ArgumentTypeError("Duplicate group-by key: %s" % group_key) seen.add(group_key) return text +# A new archive is not known to belong to the tag group of an existing archive, and it must not +# continue an arbitrary unrelated archive, so this grouping is more restricted than prune's. +FILES_CACHE_GROUP_BY_KEYS = ["name", "host", "user"] + + +def FilesCacheGroupBySpec(text): + """Validate the group-by keys usable for finding the archive a new archive continues.""" + return GroupBySpec(text, valid_keys=FILES_CACHE_GROUP_BY_KEYS, allow_ungrouped=False) + + SIZE_UNITS = ("si", "iec", "raw") _warned_units: set[str] = set() # invalid BORG_UNITS values already complained about diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 3b44556d63..23718713e5 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -2048,6 +2048,48 @@ def _remove_files_cache(archiver, archive_name): cache_file.unlink() +def test_files_cache_rebuild_ignores_other_hosts(archivers, request, monkeypatch): + """The files cache must be rebuilt from an archive of the same host, not from a foreign one.""" + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + cmd(archiver, "repo-create", RK_ENCRYPTION) + + # host1 backs up its "home" series ... + monkeypatch.setenv("BORG_HOSTNAME", "host1") + cmd(archiver, "create", "home", "input") + host1_id = cmd(archiver, "repo-list", "--format={id}{NL}").strip() + + # ... and afterwards host2 backs up its own, unrelated "home" series into the same repository, + # so the newest archive named "home" is not host1's any more. + monkeypatch.setenv("BORG_HOSTNAME", "host2") + cmd(archiver, "create", "home", "input") + + # host1 lost its local files cache and has to rebuild it from the repository. + monkeypatch.setenv("BORG_HOSTNAME", "host1") + _remove_files_cache(archiver, "home") + output = cmd(archiver, "create", "--debug", "home", "input") + assert "Building files cache from" in output + assert host1_id in output # host2's archive would be useless here + + +def test_files_cache_rebuild_group_by_name_only(archivers, request, monkeypatch): + """--group-by name restores the previous behaviour of matching the series name only.""" + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + cmd(archiver, "repo-create", RK_ENCRYPTION) + + monkeypatch.setenv("BORG_HOSTNAME", "host1") + cmd(archiver, "create", "home", "input") + monkeypatch.setenv("BORG_HOSTNAME", "host2") + cmd(archiver, "create", "home", "input") + host2_id = cmd(archiver, "repo-list", "--format={id}{NL}", "--last", "1").strip() + + monkeypatch.setenv("BORG_HOSTNAME", "host1") + _remove_files_cache(archiver, "home") + output = cmd(archiver, "create", "--debug", "--group-by", "name", "home", "input") + assert host2_id in output # the newest archive of the series, whatever host made it + + def test_files_cache_rebuild_without_ctime(archivers, request): """Rebuilding from an archive that has no ctime must work - --noctime, and always on Windows.""" archiver = request.getfixturevalue(archivers) @@ -2057,3 +2099,10 @@ def test_files_cache_rebuild_without_ctime(archivers, request): _remove_files_cache(archiver, "home") output = cmd(archiver, "create", "--noctime", "--debug", "home", "input") assert "Building files cache from" in output + + +def test_files_cache_rebuild_group_by_invalid(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + output = cmd(archiver, "create", "--group-by", "", "home", "input", exit_code=2) + assert "At least one group-by key is required" in output diff --git a/src/borg/testsuite/cache_test.py b/src/borg/testsuite/cache_test.py index 24e64488d8..5577757a45 100644 --- a/src/borg/testsuite/cache_test.py +++ b/src/borg/testsuite/cache_test.py @@ -678,3 +678,34 @@ def test_files_cache_save_tolerates_missing_chunk(tmp_path, monkeypatch): finally: cache.close() repository.flush() + + +def test_archive_group_patterns(monkeypatch): + from ..cache import archive_group_patterns + + monkeypatch.setenv("BORG_HOSTNAME", "myhost") + monkeypatch.setenv("BORG_USERNAME", "myuser") + assert archive_group_patterns("home", ()) == [] + assert archive_group_patterns("home", ("name",)) == ["name:home"] + assert archive_group_patterns("home", ("name", "host")) == ["name:home", "host:myhost"] + assert archive_group_patterns("home", ("name", "host", "user")) == ["name:home", "host:myhost", "user:myuser"] + with pytest.raises(ValueError, match="invalid group-by key: tags"): + archive_group_patterns("home", ("tags",)) + + +def test_files_cache_group_by_spec(): + from argparse import ArgumentTypeError + + from ..helpers import FilesCacheGroupBySpec + + assert FilesCacheGroupBySpec("name,host") == "name,host" + assert FilesCacheGroupBySpec("name") == "name" + # the parsed value is fed through the spec again by the argument parser, so it must be stable: + assert FilesCacheGroupBySpec(FilesCacheGroupBySpec("name,host")) == FilesCacheGroupBySpec("name,host") + # tags are not usable here: a new archive is not known to belong to the tag group of an existing one. + with pytest.raises(ArgumentTypeError, match="Invalid group-by key: tags"): + FilesCacheGroupBySpec("name,tags") + # a new archive must not continue an arbitrary unrelated archive: + for text in ("", "none"): + with pytest.raises(ArgumentTypeError, match="At least one group-by key is required"): + FilesCacheGroupBySpec(text)