create: rebuild the files cache from an archive of the same group

src/borg/archive.py

@@ -10,7 +10,6 @@
 from contextlib import contextmanager
 from datetime import timedelta
 from functools import partial
-from getpass import getuser
 from io import BytesIO
 from itertools import groupby, zip_longest
 from collections.abc import Iterator
@@ -32,6 +31,7 @@
 from .helpers import BackupSymlinkParentError, BackupPathTraversalError
 from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError, BackupTimeoutError
 from .helpers import HardLinkManager
+from .helpers import archive_hostname, archive_username
 from .helpers import ChunkIteratorFileWrapper, open_item
 from .helpers import Error, IntegrityError, set_ec, sig_int
 from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns
@@ -51,7 +51,6 @@
 from .manifest import Manifest
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem, ItemDiff
-from . import platform
 from .platform import acl_get, acl_set, set_flags, get_flags, set_times, swidth
 from .repository import Repository, NoManifestError
 from .repoobj import RepoObj
@@ -770,8 +769,8 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m
             "item_ptrs": item_ptrs,  # see #1473
             "command_line": join_cmd(sys.argv),
             "cwd": self.cwd,
-            "hostname": os.environ.get("BORG_HOSTNAME") or platform.get_hostname(),
-            "username": os.environ.get("BORG_USERNAME") or getuser(),
+            "hostname": archive_hostname(),
+            "username": archive_username(),
             "time": nominal.isoformat(timespec="microseconds"),
             "start": start.isoformat(timespec="microseconds"),
             "end": end.isoformat(timespec="microseconds"),

src/borg/archiver/create_cmd.py

@@ -17,6 +17,8 @@
 from ..constants import *  # NOQA
 from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec
 from ..helpers import archivename_validator, DigestAlgos, FilesCacheMode, files_cache_mode_no_ctime
+from ..helpers import FilesCacheGroupBySpec
+from ..helpers.parseformat import FILES_CACHE_GROUP_BY_KEYS
 from ..helpers import octal_int, nonnegative_seconds
 from ..helpers import read_input_map
 from ..helpers import eval_escapes
@@ -313,7 +315,12 @@ def create_inner(archive, cache, fso):
         logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed))
         if not dry_run:
             with Cache(
-                repository, manifest, progress=args.progress, cache_mode=args.files_cache_mode, archive_name=args.name
+                repository,
+                manifest,
+                progress=args.progress,
+                cache_mode=args.files_cache_mode,
+                archive_name=args.name,
+                archive_group_by=tuple(args.group_by.split(",")),
             ) as cache:
                 archive = Archive(
                     manifest,
@@ -791,6 +798,21 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         done by comparing multiple file metadata values with previous values kept in
         the files cache.
 
+        The files cache is kept locally, one per archive series. If it is missing (e.g. on a
+        fresh machine or after the local cache was removed), borg rebuilds it by reading the
+        archive this one continues from the repository. That archive is the newest one having
+        the same archive attributes as given by ``--group-by``, by default the same series
+        name and the same host. Matching the series name alone would be wrong in a repository
+        shared by multiple machines or users, because they may use the same series name for
+        their own, unrelated data - borg would then rebuild the files cache from a foreign
+        archive, where almost nothing matches, and read and chunk everything again.
+
+        Give ``--group-by name`` if the same series is written by different hosts on purpose
+        and they see the same files, or add ``user`` if one host backs up the same series as
+        different users. Beware of grouping by an attribute that is not stable over time: if
+        e.g. the hostname changes for every backup run (as it might for containers), borg will
+        never find an archive to rebuild the files cache from.
+
         This comparison can operate in different modes as given by ``--files-cache``:
 
         - ctime,size,inode (default on POSIX systems)
@@ -1221,6 +1243,18 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
             help="operate files cache in MODE. default: %s (on Windows: %s, because ctime is "
             "file creation time there)." % (FILES_CACHE_MODE_UI_DEFAULT_POSIX, FILES_CACHE_MODE_UI_DEFAULT_WIN32),
         )
+        fs_group.add_argument(
+            "--group-by",
+            metavar="KEYS",
+            dest="group_by",
+            action=Highlander,
+            type=FilesCacheGroupBySpec,
+            default="name,host",
+            help="comma-separated list of archive attributes identifying the archives this archive "
+            "belongs to; the newest of them is the archive the files cache is rebuilt from, if the "
+            "local files cache is missing. valid keys are: {}; default is: "
+            "name,host".format(", ".join(FILES_CACHE_GROUP_BY_KEYS)),
+        )
         fs_group.add_argument(
             "--files-changed",
             metavar="MODE",

src/borg/cache.py

@@ -26,6 +26,7 @@
 from .constants import CHUNKINDEX_SMALL_FRAGMENT_CAP, CHUNKINDEX_MERGE_ATTEMPTS, CHUNKINDEX_INVALID_SENTINEL
 from .hashindex import ChunkIndex, ChunkIndexEntry, ChunkIndexEntryFormat
 from .helpers import get_cache_dir
+from .helpers import archive_hostname, archive_username
 from .helpers import chunkit
 from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list
 from .helpers import format_file_size, safe_encode
@@ -58,6 +59,27 @@ def files_cache_name(archive_name, files_cache_name="files"):
     return files_cache_name + "." + suffix
 
 
+def archive_group_patterns(archive_name, group_by):
+    """
+    Build the match patterns selecting the archives belonging to the same group as a new archive.
+
+    The new archive is named *archive_name* and gets stamped with this host and this user, so the
+    patterns describe the archives it continues, e.g. ["name:home", "host:myhost"] for the default
+    grouping. See "borg help match-archives" for the pattern syntax.
+    """
+    patterns = []
+    for group_by_key in group_by:
+        if group_by_key == "name":
+            patterns.append(f"name:{archive_name}")
+        elif group_by_key == "host":
+            patterns.append(f"host:{archive_hostname()}")
+        elif group_by_key == "user":
+            patterns.append(f"user:{archive_username()}")
+        else:
+            raise ValueError(f"invalid group-by key: {group_by_key}")
+    return patterns
+
+
 def discover_files_cache_names(path, files_cache_name="files"):
     """
     Return a list of all files cache file names in the given directory.
@@ -200,6 +222,7 @@ def __new__(
         progress=False,
         cache_mode=FILES_CACHE_MODE_DISABLED,
         archive_name=None,
+        archive_group_by=(),
         start_backup=None,
     ):
         return AdHocWithFilesCache(
@@ -209,6 +232,7 @@ def __new__(
             progress=progress,
             cache_mode=cache_mode,
             archive_name=archive_name,
+            archive_group_by=archive_group_by,
             start_backup=start_backup,
         )
 
@@ -225,8 +249,9 @@ class FilesCacheMixin:
 
     FILES_CACHE_NAME = "files"
 
-    def __init__(self, cache_mode, archive_name=None, start_backup=None):
+    def __init__(self, cache_mode, archive_name=None, archive_group_by=(), start_backup=None):
         self.archive_name = archive_name  # ideally a SERIES name
+        self.archive_group_by = archive_group_by  # archive attributes identifying the previous archive
         assert not ("c" in cache_mode and "m" in cache_mode)
         assert "d" in cache_mode or "c" in cache_mode or "m" in cache_mode
         self.cache_mode = cache_mode
@@ -294,9 +319,13 @@ def _build_files_cache(self):
 
         from .archive import Archive
 
-        # get the latest archive with the IDENTICAL name, supporting archive series:
+        # Get the latest archive of the same group, supporting archive series. Matching the name
+        # alone is not enough in a repository shared by multiple hosts or users, because they may
+        # use the same series name for their own, unrelated data - we would then build our files
+        # cache from a foreign archive, which just wastes time as almost nothing would match.
+        match = archive_group_patterns(self.archive_name, self.archive_group_by)
         try:
-            archives = self.manifest.archives.list(match=[self.archive_name], sort_by=["ts"], last=1)
+            archives = self.manifest.archives.list(match=match, sort_by=["ts"], last=1)
         except PermissionDenied:  # maybe repo is in write-only mode?
             archives = None
         if not archives:
@@ -1207,13 +1236,14 @@ def __init__(
         progress=False,
         cache_mode=FILES_CACHE_MODE_DISABLED,
         archive_name=None,
+        archive_group_by=(),
         start_backup=None,
     ):
         """
         :param warn_if_unencrypted: print warning if accessing unknown unencrypted repository
         :param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison
         """
-        FilesCacheMixin.__init__(self, cache_mode, archive_name, start_backup)
+        FilesCacheMixin.__init__(self, cache_mode, archive_name, archive_group_by, start_backup)
         ChunksMixin.__init__(self)
         assert isinstance(manifest, Manifest)
         self.manifest = manifest

src/borg/helpers/__init__.py

@@ -28,7 +28,7 @@
 from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special
 from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime
 from .fs import HardLinkManager
-from .misc import sysinfo, log_multi, consume
+from .misc import sysinfo, log_multi, consume, archive_hostname, archive_username
 from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
 from .parseformat import octal_int, bin_to_hex, hex_to_bin, safe_encode, safe_decode
 from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd
@@ -39,6 +39,7 @@
     FilesystemDirSpec,
     SortBySpec,
     GroupBySpec,
+    FilesCacheGroupBySpec,
     CompressionSpec,
     ChunkerParams,
     DigestAlgos,

src/borg/helpers/misc.py

@@ -4,6 +4,7 @@
 import platform  # python stdlib import - if this fails, check that cwd != src/borg/
 import sys
 from collections import deque
+from getpass import getuser
 from itertools import islice
 
 from ..logger import create_logger
@@ -15,6 +16,18 @@
 from ..constants import ROBJ_FILE_STREAM
 
 
+def archive_hostname():
+    """Return the hostname a new archive is stamped with."""
+    from ..platform import get_hostname
+
+    return os.environ.get("BORG_HOSTNAME") or get_hostname()
+
+
+def archive_username():
+    """Return the username a new archive is stamped with."""
+    return os.environ.get("BORG_USERNAME") or getuser()
+
+
 def sysinfo():
     show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower()
     if show_sysinfo == "no":

src/borg/helpers/parseformat.py

@@ -582,24 +582,35 @@ def SortBySpec(text):
     return text.replace("timestamp", "ts").replace("archive", "name")
 
 
-def GroupBySpec(text):
+def GroupBySpec(text, valid_keys=None, allow_ungrouped=True):
     """Validate a comma-separated list of group-by keys. "" and "none" mean: do not group."""
     from ..manifest import AI_GROUP_BY_KEYS
 
+    valid_keys = AI_GROUP_BY_KEYS if valid_keys is None else valid_keys
     if text in ("", "none"):
+        if not allow_ungrouped:
+            raise ArgumentTypeError("At least one group-by key is required (valid keys: %s)" % ", ".join(valid_keys))
         return ""  # idempotency: the normalized value must pass validation again
     seen = set()
     for group_key in text.split(","):
-        if group_key not in AI_GROUP_BY_KEYS:
-            raise ArgumentTypeError(
-                "Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(AI_GROUP_BY_KEYS))
-            )
+        if group_key not in valid_keys:
+            raise ArgumentTypeError("Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(valid_keys)))
         if group_key in seen:
             raise ArgumentTypeError("Duplicate group-by key: %s" % group_key)
         seen.add(group_key)
     return text
 
 
+# A new archive is not known to belong to the tag group of an existing archive, and it must not
+# continue an arbitrary unrelated archive, so this grouping is more restricted than prune's.
+FILES_CACHE_GROUP_BY_KEYS = ["name", "host", "user"]
+
+
+def FilesCacheGroupBySpec(text):
+    """Validate the group-by keys usable for finding the archive a new archive continues."""
+    return GroupBySpec(text, valid_keys=FILES_CACHE_GROUP_BY_KEYS, allow_ungrouped=False)
+
+
 SIZE_UNITS = ("si", "iec", "raw")
 
 _warned_units: set[str] = set()  # invalid BORG_UNITS values already complained about

src/borg/testsuite/archiver/create_cmd_test.py

@@ -2048,6 +2048,48 @@ def _remove_files_cache(archiver, archive_name):
     cache_file.unlink()
 
 
+def test_files_cache_rebuild_ignores_other_hosts(archivers, request, monkeypatch):
+    """The files cache must be rebuilt from an archive of the same host, not from a foreign one."""
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+
+    # host1 backs up its "home" series ...
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    cmd(archiver, "create", "home", "input")
+    host1_id = cmd(archiver, "repo-list", "--format={id}{NL}").strip()
+
+    # ... and afterwards host2 backs up its own, unrelated "home" series into the same repository,
+    # so the newest archive named "home" is not host1's any more.
+    monkeypatch.setenv("BORG_HOSTNAME", "host2")
+    cmd(archiver, "create", "home", "input")
+
+    # host1 lost its local files cache and has to rebuild it from the repository.
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    _remove_files_cache(archiver, "home")
+    output = cmd(archiver, "create", "--debug", "home", "input")
+    assert "Building files cache from" in output
+    assert host1_id in output  # host2's archive would be useless here
+
+
+def test_files_cache_rebuild_group_by_name_only(archivers, request, monkeypatch):
+    """--group-by name restores the previous behaviour of matching the series name only."""
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    cmd(archiver, "create", "home", "input")
+    monkeypatch.setenv("BORG_HOSTNAME", "host2")
+    cmd(archiver, "create", "home", "input")
+    host2_id = cmd(archiver, "repo-list", "--format={id}{NL}", "--last", "1").strip()
+
+    monkeypatch.setenv("BORG_HOSTNAME", "host1")
+    _remove_files_cache(archiver, "home")
+    output = cmd(archiver, "create", "--debug", "--group-by", "name", "home", "input")
+    assert host2_id in output  # the newest archive of the series, whatever host made it
+
+
 def test_files_cache_rebuild_without_ctime(archivers, request):
     """Rebuilding from an archive that has no ctime must work - --noctime, and always on Windows."""
     archiver = request.getfixturevalue(archivers)
@@ -2057,3 +2099,10 @@ def test_files_cache_rebuild_without_ctime(archivers, request):
     _remove_files_cache(archiver, "home")
     output = cmd(archiver, "create", "--noctime", "--debug", "home", "input")
     assert "Building files cache from" in output
+
+
+def test_files_cache_rebuild_group_by_invalid(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    output = cmd(archiver, "create", "--group-by", "", "home", "input", exit_code=2)
+    assert "At least one group-by key is required" in output

src/borg/testsuite/cache_test.py

@@ -678,3 +678,34 @@ def test_files_cache_save_tolerates_missing_chunk(tmp_path, monkeypatch):
         finally:
             cache.close()
         repository.flush()
+
+
+def test_archive_group_patterns(monkeypatch):
+    from ..cache import archive_group_patterns
+
+    monkeypatch.setenv("BORG_HOSTNAME", "myhost")
+    monkeypatch.setenv("BORG_USERNAME", "myuser")
+    assert archive_group_patterns("home", ()) == []
+    assert archive_group_patterns("home", ("name",)) == ["name:home"]
+    assert archive_group_patterns("home", ("name", "host")) == ["name:home", "host:myhost"]
+    assert archive_group_patterns("home", ("name", "host", "user")) == ["name:home", "host:myhost", "user:myuser"]
+    with pytest.raises(ValueError, match="invalid group-by key: tags"):
+        archive_group_patterns("home", ("tags",))
+
+
+def test_files_cache_group_by_spec():
+    from argparse import ArgumentTypeError
+
+    from ..helpers import FilesCacheGroupBySpec
+
+    assert FilesCacheGroupBySpec("name,host") == "name,host"
+    assert FilesCacheGroupBySpec("name") == "name"
+    # the parsed value is fed through the spec again by the argument parser, so it must be stable:
+    assert FilesCacheGroupBySpec(FilesCacheGroupBySpec("name,host")) == FilesCacheGroupBySpec("name,host")
+    # tags are not usable here: a new archive is not known to belong to the tag group of an existing one.
+    with pytest.raises(ArgumentTypeError, match="Invalid group-by key: tags"):
+        FilesCacheGroupBySpec("name,tags")
+    # a new archive must not continue an arbitrary unrelated archive:
+    for text in ("", "none"):
+        with pytest.raises(ArgumentTypeError, match="At least one group-by key is required"):
+            FilesCacheGroupBySpec(text)