diff --git a/src/borg/archive.py b/src/borg/archive.py index bc45bdad41..a2bc0fd502 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -10,7 +10,6 @@ from contextlib import contextmanager from datetime import timedelta from functools import partial -from getpass import getuser from io import BytesIO from itertools import groupby, zip_longest from collections.abc import Iterator @@ -32,6 +31,7 @@ from .helpers import BackupSymlinkParentError, BackupPathTraversalError from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError, BackupTimeoutError from .helpers import HardLinkManager +from .helpers import archive_hostname, archive_username from .helpers import ChunkIteratorFileWrapper, open_item from .helpers import Error, IntegrityError, set_ec, sig_int from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns @@ -51,7 +51,6 @@ from .manifest import Manifest from .patterns import PathPrefixPattern, FnmatchPattern, IECommand from .item import Item, ArchiveItem, ItemDiff -from . import platform from .platform import acl_get, acl_set, set_flags, get_flags, set_times, swidth from .repository import Repository, NoManifestError from .repoobj import RepoObj @@ -770,8 +769,8 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m "item_ptrs": item_ptrs, # see #1473 "command_line": join_cmd(sys.argv), "cwd": self.cwd, - "hostname": os.environ.get("BORG_HOSTNAME") or platform.get_hostname(), - "username": os.environ.get("BORG_USERNAME") or getuser(), + "hostname": archive_hostname(), + "username": archive_username(), "time": nominal.isoformat(timespec="microseconds"), "start": start.isoformat(timespec="microseconds"), "end": end.isoformat(timespec="microseconds"), diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 3111d5da51..b18b4c24f7 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -17,6 +17,8 @@ from ..constants import * # NOQA from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec from ..helpers import archivename_validator, DigestAlgos, FilesCacheMode, files_cache_mode_no_ctime +from ..helpers import FilesCacheGroupBySpec +from ..helpers.parseformat import FILES_CACHE_GROUP_BY_KEYS from ..helpers import octal_int, nonnegative_seconds from ..helpers import read_input_map from ..helpers import eval_escapes @@ -313,7 +315,12 @@ def create_inner(archive, cache, fso): logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed)) if not dry_run: with Cache( - repository, manifest, progress=args.progress, cache_mode=args.files_cache_mode, archive_name=args.name + repository, + manifest, + progress=args.progress, + cache_mode=args.files_cache_mode, + archive_name=args.name, + archive_group_by=tuple(args.group_by.split(",")), ) as cache: archive = Archive( manifest, @@ -791,6 +798,21 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): done by comparing multiple file metadata values with previous values kept in the files cache. + The files cache is kept locally, one per archive series. If it is missing (e.g. on a + fresh machine or after the local cache was removed), borg rebuilds it by reading the + archive this one continues from the repository. That archive is the newest one having + the same archive attributes as given by ``--group-by``, by default the same series + name and the same host. Matching the series name alone would be wrong in a repository + shared by multiple machines or users, because they may use the same series name for + their own, unrelated data - borg would then rebuild the files cache from a foreign + archive, where almost nothing matches, and read and chunk everything again. + + Give ``--group-by name`` if the same series is written by different hosts on purpose + and they see the same files, or add ``user`` if one host backs up the same series as + different users. Beware of grouping by an attribute that is not stable over time: if + e.g. the hostname changes for every backup run (as it might for containers), borg will + never find an archive to rebuild the files cache from. + This comparison can operate in different modes as given by ``--files-cache``: - ctime,size,inode (default on POSIX systems) @@ -1221,6 +1243,18 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): help="operate files cache in MODE. default: %s (on Windows: %s, because ctime is " "file creation time there)." % (FILES_CACHE_MODE_UI_DEFAULT_POSIX, FILES_CACHE_MODE_UI_DEFAULT_WIN32), ) + fs_group.add_argument( + "--group-by", + metavar="KEYS", + dest="group_by", + action=Highlander, + type=FilesCacheGroupBySpec, + default="name,host", + help="comma-separated list of archive attributes identifying the archives this archive " + "belongs to; the newest of them is the archive the files cache is rebuilt from, if the " + "local files cache is missing. valid keys are: {}; default is: " + "name,host".format(", ".join(FILES_CACHE_GROUP_BY_KEYS)), + ) fs_group.add_argument( "--files-changed", metavar="MODE", diff --git a/src/borg/cache.py b/src/borg/cache.py index 475b0ad882..5cf26812ef 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -26,6 +26,7 @@ from .constants import CHUNKINDEX_SMALL_FRAGMENT_CAP, CHUNKINDEX_MERGE_ATTEMPTS, CHUNKINDEX_INVALID_SENTINEL from .hashindex import ChunkIndex, ChunkIndexEntry, ChunkIndexEntryFormat from .helpers import get_cache_dir +from .helpers import archive_hostname, archive_username from .helpers import chunkit from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list from .helpers import format_file_size, safe_encode @@ -58,6 +59,27 @@ def files_cache_name(archive_name, files_cache_name="files"): return files_cache_name + "." + suffix +def archive_group_patterns(archive_name, group_by): + """ + Build the match patterns selecting the archives belonging to the same group as a new archive. + + The new archive is named *archive_name* and gets stamped with this host and this user, so the + patterns describe the archives it continues, e.g. ["name:home", "host:myhost"] for the default + grouping. See "borg help match-archives" for the pattern syntax. + """ + patterns = [] + for group_by_key in group_by: + if group_by_key == "name": + patterns.append(f"name:{archive_name}") + elif group_by_key == "host": + patterns.append(f"host:{archive_hostname()}") + elif group_by_key == "user": + patterns.append(f"user:{archive_username()}") + else: + raise ValueError(f"invalid group-by key: {group_by_key}") + return patterns + + def discover_files_cache_names(path, files_cache_name="files"): """ Return a list of all files cache file names in the given directory. @@ -200,6 +222,7 @@ def __new__( progress=False, cache_mode=FILES_CACHE_MODE_DISABLED, archive_name=None, + archive_group_by=(), start_backup=None, ): return AdHocWithFilesCache( @@ -209,6 +232,7 @@ def __new__( progress=progress, cache_mode=cache_mode, archive_name=archive_name, + archive_group_by=archive_group_by, start_backup=start_backup, ) @@ -225,8 +249,9 @@ class FilesCacheMixin: FILES_CACHE_NAME = "files" - def __init__(self, cache_mode, archive_name=None, start_backup=None): + def __init__(self, cache_mode, archive_name=None, archive_group_by=(), start_backup=None): self.archive_name = archive_name # ideally a SERIES name + self.archive_group_by = archive_group_by # archive attributes identifying the previous archive assert not ("c" in cache_mode and "m" in cache_mode) assert "d" in cache_mode or "c" in cache_mode or "m" in cache_mode self.cache_mode = cache_mode @@ -294,9 +319,13 @@ def _build_files_cache(self): from .archive import Archive - # get the latest archive with the IDENTICAL name, supporting archive series: + # Get the latest archive of the same group, supporting archive series. Matching the name + # alone is not enough in a repository shared by multiple hosts or users, because they may + # use the same series name for their own, unrelated data - we would then build our files + # cache from a foreign archive, which just wastes time as almost nothing would match. + match = archive_group_patterns(self.archive_name, self.archive_group_by) try: - archives = self.manifest.archives.list(match=[self.archive_name], sort_by=["ts"], last=1) + archives = self.manifest.archives.list(match=match, sort_by=["ts"], last=1) except PermissionDenied: # maybe repo is in write-only mode? archives = None if not archives: @@ -1207,13 +1236,14 @@ def __init__( progress=False, cache_mode=FILES_CACHE_MODE_DISABLED, archive_name=None, + archive_group_by=(), start_backup=None, ): """ :param warn_if_unencrypted: print warning if accessing unknown unencrypted repository :param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison """ - FilesCacheMixin.__init__(self, cache_mode, archive_name, start_backup) + FilesCacheMixin.__init__(self, cache_mode, archive_name, archive_group_by, start_backup) ChunksMixin.__init__(self) assert isinstance(manifest, Manifest) self.manifest = manifest diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 96649b65c1..b3d5216d51 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -28,7 +28,7 @@ from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime from .fs import HardLinkManager -from .misc import sysinfo, log_multi, consume +from .misc import sysinfo, log_multi, consume, archive_hostname, archive_username from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper from .parseformat import octal_int, bin_to_hex, hex_to_bin, safe_encode, safe_decode from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd @@ -39,6 +39,7 @@ FilesystemDirSpec, SortBySpec, GroupBySpec, + FilesCacheGroupBySpec, CompressionSpec, ChunkerParams, DigestAlgos, diff --git a/src/borg/helpers/misc.py b/src/borg/helpers/misc.py index d6a027b632..0316835fe1 100644 --- a/src/borg/helpers/misc.py +++ b/src/borg/helpers/misc.py @@ -4,6 +4,7 @@ import platform # python stdlib import - if this fails, check that cwd != src/borg/ import sys from collections import deque +from getpass import getuser from itertools import islice from ..logger import create_logger @@ -15,6 +16,18 @@ from ..constants import ROBJ_FILE_STREAM +def archive_hostname(): + """Return the hostname a new archive is stamped with.""" + from ..platform import get_hostname + + return os.environ.get("BORG_HOSTNAME") or get_hostname() + + +def archive_username(): + """Return the username a new archive is stamped with.""" + return os.environ.get("BORG_USERNAME") or getuser() + + def sysinfo(): show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower() if show_sysinfo == "no": diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 67670c9bd2..75ee35eb34 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -582,24 +582,35 @@ def SortBySpec(text): return text.replace("timestamp", "ts").replace("archive", "name") -def GroupBySpec(text): +def GroupBySpec(text, valid_keys=None, allow_ungrouped=True): """Validate a comma-separated list of group-by keys. "" and "none" mean: do not group.""" from ..manifest import AI_GROUP_BY_KEYS + valid_keys = AI_GROUP_BY_KEYS if valid_keys is None else valid_keys if text in ("", "none"): + if not allow_ungrouped: + raise ArgumentTypeError("At least one group-by key is required (valid keys: %s)" % ", ".join(valid_keys)) return "" # idempotency: the normalized value must pass validation again seen = set() for group_key in text.split(","): - if group_key not in AI_GROUP_BY_KEYS: - raise ArgumentTypeError( - "Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(AI_GROUP_BY_KEYS)) - ) + if group_key not in valid_keys: + raise ArgumentTypeError("Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(valid_keys))) if group_key in seen: raise ArgumentTypeError("Duplicate group-by key: %s" % group_key) seen.add(group_key) return text +# A new archive is not known to belong to the tag group of an existing archive, and it must not +# continue an arbitrary unrelated archive, so this grouping is more restricted than prune's. +FILES_CACHE_GROUP_BY_KEYS = ["name", "host", "user"] + + +def FilesCacheGroupBySpec(text): + """Validate the group-by keys usable for finding the archive a new archive continues.""" + return GroupBySpec(text, valid_keys=FILES_CACHE_GROUP_BY_KEYS, allow_ungrouped=False) + + SIZE_UNITS = ("si", "iec", "raw") _warned_units: set[str] = set() # invalid BORG_UNITS values already complained about diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 3b44556d63..23718713e5 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -2048,6 +2048,48 @@ def _remove_files_cache(archiver, archive_name): cache_file.unlink() +def test_files_cache_rebuild_ignores_other_hosts(archivers, request, monkeypatch): + """The files cache must be rebuilt from an archive of the same host, not from a foreign one.""" + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + cmd(archiver, "repo-create", RK_ENCRYPTION) + + # host1 backs up its "home" series ... + monkeypatch.setenv("BORG_HOSTNAME", "host1") + cmd(archiver, "create", "home", "input") + host1_id = cmd(archiver, "repo-list", "--format={id}{NL}").strip() + + # ... and afterwards host2 backs up its own, unrelated "home" series into the same repository, + # so the newest archive named "home" is not host1's any more. + monkeypatch.setenv("BORG_HOSTNAME", "host2") + cmd(archiver, "create", "home", "input") + + # host1 lost its local files cache and has to rebuild it from the repository. + monkeypatch.setenv("BORG_HOSTNAME", "host1") + _remove_files_cache(archiver, "home") + output = cmd(archiver, "create", "--debug", "home", "input") + assert "Building files cache from" in output + assert host1_id in output # host2's archive would be useless here + + +def test_files_cache_rebuild_group_by_name_only(archivers, request, monkeypatch): + """--group-by name restores the previous behaviour of matching the series name only.""" + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + cmd(archiver, "repo-create", RK_ENCRYPTION) + + monkeypatch.setenv("BORG_HOSTNAME", "host1") + cmd(archiver, "create", "home", "input") + monkeypatch.setenv("BORG_HOSTNAME", "host2") + cmd(archiver, "create", "home", "input") + host2_id = cmd(archiver, "repo-list", "--format={id}{NL}", "--last", "1").strip() + + monkeypatch.setenv("BORG_HOSTNAME", "host1") + _remove_files_cache(archiver, "home") + output = cmd(archiver, "create", "--debug", "--group-by", "name", "home", "input") + assert host2_id in output # the newest archive of the series, whatever host made it + + def test_files_cache_rebuild_without_ctime(archivers, request): """Rebuilding from an archive that has no ctime must work - --noctime, and always on Windows.""" archiver = request.getfixturevalue(archivers) @@ -2057,3 +2099,10 @@ def test_files_cache_rebuild_without_ctime(archivers, request): _remove_files_cache(archiver, "home") output = cmd(archiver, "create", "--noctime", "--debug", "home", "input") assert "Building files cache from" in output + + +def test_files_cache_rebuild_group_by_invalid(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + output = cmd(archiver, "create", "--group-by", "", "home", "input", exit_code=2) + assert "At least one group-by key is required" in output diff --git a/src/borg/testsuite/cache_test.py b/src/borg/testsuite/cache_test.py index 24e64488d8..5577757a45 100644 --- a/src/borg/testsuite/cache_test.py +++ b/src/borg/testsuite/cache_test.py @@ -678,3 +678,34 @@ def test_files_cache_save_tolerates_missing_chunk(tmp_path, monkeypatch): finally: cache.close() repository.flush() + + +def test_archive_group_patterns(monkeypatch): + from ..cache import archive_group_patterns + + monkeypatch.setenv("BORG_HOSTNAME", "myhost") + monkeypatch.setenv("BORG_USERNAME", "myuser") + assert archive_group_patterns("home", ()) == [] + assert archive_group_patterns("home", ("name",)) == ["name:home"] + assert archive_group_patterns("home", ("name", "host")) == ["name:home", "host:myhost"] + assert archive_group_patterns("home", ("name", "host", "user")) == ["name:home", "host:myhost", "user:myuser"] + with pytest.raises(ValueError, match="invalid group-by key: tags"): + archive_group_patterns("home", ("tags",)) + + +def test_files_cache_group_by_spec(): + from argparse import ArgumentTypeError + + from ..helpers import FilesCacheGroupBySpec + + assert FilesCacheGroupBySpec("name,host") == "name,host" + assert FilesCacheGroupBySpec("name") == "name" + # the parsed value is fed through the spec again by the argument parser, so it must be stable: + assert FilesCacheGroupBySpec(FilesCacheGroupBySpec("name,host")) == FilesCacheGroupBySpec("name,host") + # tags are not usable here: a new archive is not known to belong to the tag group of an existing one. + with pytest.raises(ArgumentTypeError, match="Invalid group-by key: tags"): + FilesCacheGroupBySpec("name,tags") + # a new archive must not continue an arbitrary unrelated archive: + for text in ("", "none"): + with pytest.raises(ArgumentTypeError, match="At least one group-by key is required"): + FilesCacheGroupBySpec(text)