From 7004e87c476f44273cf84977bd07e5c9c5eeaad5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 31 Aug 2026 05:39:00 +0200 Subject: [PATCH] prune: add --group-by, apply the retention rules per group prune applied the retention rules to one flat pool of all matching archives, so unrelated backup sets competed for the same retention slots: with a "home" and an "etc" series in one repository, --keep-daily 7 kept 7 daily archives in total, not 7 per series. The docs worked around this by telling users to run one prune call per series, and in a repository shared by several machines even that was not enough, because different hosts may use the same series name. --group-by KEYS groups the selected archives by the archive attributes name, host, user and/or tags, and applies the full rule set (--from prefilter and keep-oldest included) to each group separately. The default is --group-by name,host, so neither different series nor the same series name used by different hosts compete for retention slots. Archive series names are not unique in a shared repository, so grouping by name alone would still let one machine's archives push another machine's archives out of the retention slots. This can only ever keep more archives than before, never fewer. --group-by "" (or "none") restores the previous behaviour of treating all selected archives as one group, --group-by name groups by the series name only. Note the split between the two mechanisms: NAME / -a select which archives are considered at all, --group-by subdivides those into independent retention pools. Archives without host / user metadata (e.g. transferred from a borg 1.x repo) form their own group. Internal tags (starting with @) do not affect grouping. With more than one group, prune logs a summary line per group, and the JSON output gains a "group" object per archive. --- src/borg/archiver/prune_cmd.py | 207 ++++++++++++---- src/borg/helpers/__init__.py | 1 + src/borg/helpers/parseformat.py | 18 ++ src/borg/manifest.py | 4 + src/borg/testsuite/archiver/prune_cmd_test.py | 233 +++++++++++++++--- 5 files changed, 378 insertions(+), 85 deletions(-) diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index f1e9888609..daaf833f4e 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -1,3 +1,5 @@ +from collections import defaultdict +from collections.abc import Sequence from typing import Callable, NamedTuple from datetime import datetime, timedelta import logging @@ -9,9 +11,10 @@ from ..constants import * # NOQA from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error from ..helpers import archivename_validator, int_or_interval, sig_int, timestamp +from ..helpers import GroupBySpec from ..helpers import json_print, basic_json_data from ..helpers.argparsing import ArgumentParser -from ..manifest import ArchiveInfo, Manifest +from ..manifest import AI_GROUP_BY_KEYS, ArchiveInfo, Manifest from ..logger import create_logger @@ -187,6 +190,40 @@ def can_retain(a): return keep +def archive_group_key(archive_info: ArchiveInfo, group_by: Sequence[str]) -> tuple[str, ...]: + """Compute the grouping key of *archive_info* for the given *group_by* archive attributes.""" + key = [] + for group_by_key in group_by: + if group_by_key == "tags": + # internal tags (e.g. @PROT) say nothing about what an archive contains or where it + # came from, so they must not put an archive into a group of its own. + value = ",".join(tag for tag in archive_info.tags if not tag.startswith("@")) + else: + # host and user are empty for archives that do not have this metadata, e.g. archives + # transferred from a borg 1.x repo. they form their own group then. + value = getattr(archive_info, group_by_key) or "" + key.append(value) + return tuple(key) + + +def group_archives(archives: list[ArchiveInfo], group_by: Sequence[str]) -> dict[tuple[str, ...], list[ArchiveInfo]]: + """ + Group *archives* by the given *group_by* archive attributes, keeping their relative order. + + An empty *group_by* puts all archives into one group, so the retention rules are applied to + all given archives at once. + """ + groups: dict[tuple[str, ...], list[ArchiveInfo]] = defaultdict(list) + for archive_info in archives: + groups[archive_group_key(archive_info, group_by)].append(archive_info) + return groups + + +def format_group_key(key: tuple[str, ...], group_by: Sequence[str]) -> str: + """Format a grouping key for human consumption, e.g. \"name='home', host='myhost'\".""" + return ", ".join(f"{group_by_key}={value!r}" for group_by_key, value in zip(group_by, key)) + + class PruneMixIn: @with_repository(compatibility=(Manifest.Operation.DELETE,)) def do_prune(self, args, repository, manifest): @@ -197,42 +234,25 @@ def do_prune(self, args, repository, manifest): archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True) archives = [ai for ai in archives if "@PROT" not in ai.tags] - # Archives to keep along with the rule that ensured them being kept - keep = {} + # The retention rules are applied to each group of archives separately, so that unrelated + # backup sets (e.g. different archive series or different machines sharing a repository) + # do not compete for the same retention slots. + group_by = tuple(args.group_by.split(",")) if args.group_by else () + groups = group_archives(archives, group_by) + # All groups use the same reference timestamp, so interval based rules do not drift apart. from_timestamp = getattr(args, PRUNE_FROM.key) - candidate_archives = archives - - if from_timestamp is not None: - base_timestamp = from_timestamp + base_timestamp = from_timestamp if from_timestamp is not None else datetime.now().astimezone() - # `--from` is a prefilter: Archives made at or after this time are kept by default. They are not considered - # for pruning at all and thus won't falsely occupy an active retention period. - for archive in archives: - if archive.ts < from_timestamp: - break - keep[archive] = KeepResult(rule=PRUNE_FROM, idx=len(keep)) - candidate_archives = archives[len(keep) :] - else: - base_timestamp = datetime.now().astimezone() - - # Apply each retention rule to all candidate archives. The - # `previously_kept` parameter prevents later (coarser-grained) rules - # from double-counting archives already retained by earlier rules. - active_rules = [ - (rule, getattr(args, rule.key)) for rule in PRUNING_RULES if getattr(args, rule.key) is not None - ] - for rule, n_or_interval in active_rules: - keep |= prune( - archives=candidate_archives, - rule=rule, - n_or_interval=n_or_interval, - base_timestamp=base_timestamp, - keep_oldest=( - rule == active_rules[-1][0] - ), # Activate keep_oldest rule only for the largest active interval - previously_kept=frozenset(keep), - ) + # Archives to keep along with the rule that ensured them being kept + keep = {} + group_of = {} + group_summaries = [] + for key, group in groups.items(): + group_keep = self._compute_keep(group, args, base_timestamp) + keep |= group_keep + group_of.update(dict.fromkeys(group, key)) + group_summaries.append((key, len(group), len(group_keep))) archives_to_prune = set(archives) - set(keep) @@ -249,6 +269,16 @@ def do_prune(self, args, repository, manifest): else: logger.info("Repository contains %d archives.", manifest.archives.count()) logger.info("Applying rules to the matching %d archives...", len(archives)) + if len(group_summaries) > 1: + # with a single group, the totals below already say everything there is to say. + for key, group_total, group_kept in group_summaries: + logger.info( + "Group (%s): %d archives, keeping %d, pruning %d.", + format_group_key(key, group_by), + group_total, + group_kept, + group_total - group_kept, + ) logger.info("Keeping %d archives, pruning %d archives.", len(keep), len(archives_to_prune)) list_logger = logging.getLogger("borg.output.list") @@ -262,6 +292,7 @@ def do_prune(self, args, repository, manifest): # so we must call it before deleting the archive. if args.json: archive_data = formatter.get_item_data(archive_info, jsonline=True) + archive_data["group"] = dict(zip(group_by, group_of[archive_info])) else: archive_formatted = formatter.format_item(archive_info, jsonline=False) if archive_info in archives_to_prune: @@ -309,6 +340,46 @@ def do_prune(self, args, repository, manifest): if sig_int: raise Error("Got Ctrl-C / SIGINT.") + def _compute_keep(self, archives, args, base_timestamp): + """ + Apply the retention rules to *archives* (sorted by timestamp, newest first). + + Return the archives to keep, mapped to the rule that ensured them being kept. + """ + keep = {} + + from_timestamp = getattr(args, PRUNE_FROM.key) + candidate_archives = archives + + if from_timestamp is not None: + # `--from` is a prefilter: Archives made at or after this time are kept by default. They are not considered + # for pruning at all and thus won't falsely occupy an active retention period. + for archive in archives: + if archive.ts < from_timestamp: + break + keep[archive] = KeepResult(rule=PRUNE_FROM, idx=len(keep)) + candidate_archives = archives[len(keep) :] + + # Apply each retention rule to all candidate archives. The + # `previously_kept` parameter prevents later (coarser-grained) rules + # from double-counting archives already retained by earlier rules. + active_rules = [ + (rule, getattr(args, rule.key)) for rule in PRUNING_RULES if getattr(args, rule.key) is not None + ] + for rule, n_or_interval in active_rules: + keep |= prune( + archives=candidate_archives, + rule=rule, + n_or_interval=n_or_interval, + base_timestamp=base_timestamp, + keep_oldest=( + rule == active_rules[-1][0] + ), # Activate keep_oldest rule only for the largest active interval + previously_kept=frozenset(keep), + ) + + return keep + def _validate_prune_args(self, args): keep_args = {rule.key: getattr(args, rule.key) for rule in PRUNING_RULES if getattr(args, rule.key) is not None} @@ -367,32 +438,57 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): `GFS `_ (Grandfather-father-son) backup rotation scheme. - The recommended way to use prune is to give the archive series name to it via the - NAME argument (assuming you have the same name for all archives in a series). - Alternatively, you can also use --match-archives (-a), then only archives that - match the pattern are considered for deletion and only those archives count - towards the totals specified by the rules. - Otherwise, *all* archives in the repository are candidates for deletion! - There is no automatic distinction between archives representing different - contents. These need to be distinguished by specifying matching globs. + Two separate mechanisms decide what prune does: the archive *selection* determines + which archives are considered at all, and ``--group-by`` determines how the retention + rules subdivide the selected archives. - NAME is just another way of saying ``-a NAME``, so it can be combined with + Without NAME and without --match-archives (-a), *all* archives in the repository are + selected. Give the archive series name via the NAME argument (assuming you have the + same name for all archives in a series) or use --match-archives (-a) to only consider + a subset. NAME is just another way of saying ``-a NAME``, so it can be combined with --match-archives (-a). All given patterns must match (they are ANDed), thus giving both narrows down the selection. - If you have multiple series of archives with different data sets (e.g. - from different machines) in one shared repository, use one prune call per - series. In such a shared repository, the series name alone might not be - specific enough, because different machines or users may use the same series - name for their own, unrelated data. Additionally match on the archive metadata - that identifies the origin, e.g.:: + The retention rules are applied to each group of selected archives separately, so + that unrelated backup sets do not compete for the same retention slots. By default, + archives are grouped by their series name and the host they were made on + (``--group-by name,host``): ``--keep-daily 7`` keeps 7 daily archives *per series and + host*. Archive series names are not unique in a repository shared by multiple machines + or users - different machines may well use the same series name for their own, + unrelated data - so grouping by the name alone would let one machine's archives push + another machine's archives out of the retention slots. + + ``--group-by`` accepts a comma-separated list of the archive attributes ``name``, + ``host``, ``user`` and ``tags``. Archives without ``host`` / ``user`` metadata (e.g. + archives transferred from a borg 1.x repository) form their own group. Borg's internal + tags (starting with ``@``) do not affect grouping. Give ``--group-by ""`` to not group + at all and apply the rules to all selected archives at once. + + Use ``--group-by name`` if several machines back up the same data to the same archive + series on purpose and you want one retention policy for all of them. Add ``user`` if + one machine backs up the same series as different users and each of them should get + their own retention. + + So, to prune every series of every host in one call, the default is all you need:: + + borg prune --keep-daily 7 --keep-weekly 4 + + To only consider your own archives, select them - the grouping then applies to the + selection only:: - borg prune home -a host:myhost --keep-daily 7 --keep-weekly 4 + borg prune -a host:myhost --keep-daily 7 --keep-weekly 4 Note: the ``host:`` / ``user:`` metadata is what the client wrote into the archive; it is not a permission mechanism. Any client with delete permission for the repository can prune any archive in it. + Beware of grouping by an attribute that is not stable over time: if e.g. the hostname + changes for every backup run (as it might for containers), each archive ends up in a + group of its own and the retention rules will keep all of them. Set ``BORG_HOSTNAME`` + to a stable value in that case, or use ``--group-by name``. Running prune with + ``--info`` shows one summary line per group, which makes such a situation visible. + + The ``--keep`` option is the simplest way to specify a basic retention policy. It accepts a count or a time interval for retention (e.g. ``10`` or ``7d``, ``4w``). With a count it keeps at most that many @@ -514,6 +610,17 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): action=Highlander, help="only consider archives older than this for pruning", ) + subparser.add_argument( + "--group-by", + metavar="KEYS", + dest="group_by", + type=GroupBySpec, + default="name,host", + action=Highlander, + help="comma-separated list of archive attributes to group the archives by before " + "applying the retention rules to each group separately; valid keys are: {}; " + 'default is: name,host; use "" (or "none") to not group at all'.format(", ".join(AI_GROUP_BY_KEYS)), + ) subparser.add_argument( "--keep", dest=PRUNE_KEEP.key, diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index ff8a5a8a5a..96649b65c1 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -38,6 +38,7 @@ FilesystemPathSpec, FilesystemDirSpec, SortBySpec, + GroupBySpec, CompressionSpec, ChunkerParams, DigestAlgos, diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index a3fbeb90ce..67670c9bd2 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -582,6 +582,24 @@ def SortBySpec(text): return text.replace("timestamp", "ts").replace("archive", "name") +def GroupBySpec(text): + """Validate a comma-separated list of group-by keys. "" and "none" mean: do not group.""" + from ..manifest import AI_GROUP_BY_KEYS + + if text in ("", "none"): + return "" # idempotency: the normalized value must pass validation again + seen = set() + for group_key in text.split(","): + if group_key not in AI_GROUP_BY_KEYS: + raise ArgumentTypeError( + "Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(AI_GROUP_BY_KEYS)) + ) + if group_key in seen: + raise ArgumentTypeError("Duplicate group-by key: %s" % group_key) + seen.add(group_key) + return text + + SIZE_UNITS = ("si", "iec", "raw") _warned_units: set[str] = set() # invalid BORG_UNITS values already complained about diff --git a/src/borg/manifest.py b/src/borg/manifest.py index d25070d8f4..f121085bb1 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -47,6 +47,10 @@ class NoManifestError(Error): AI_HUMAN_SORT_KEYS = ["timestamp", "archive"] + list(ArchiveInfo._fields) AI_HUMAN_SORT_KEYS.remove("ts") +# archive attributes describing what an archive contains and where it came from, usable to group +# archives that belong together, e.g. for applying retention rules separately (see GroupBySpec). +AI_GROUP_BY_KEYS = ["name", "host", "user", "tags"] + def filter_archives_by_date(archives, older=None, newer=None, oldest=None, newest=None): def get_first_and_last_archive_ts(archives_list): diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index 4553a5fc9b..faa81a3fc3 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -15,8 +15,14 @@ PRUNE_SECONDLY, PRUNE_WEEKLY, PRUNE_YEARLY, + archive_group_key, + format_group_key, + group_archives, unique_period_func, ) +from ...helpers import GroupBySpec +from argparse import ArgumentTypeError + from ...helpers import CommandError from ...manifest import ArchiveInfo from . import cmd, RK_ENCRYPTION, generate_archiver_tests @@ -34,19 +40,31 @@ def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0, us= _create_archive_dt(archiver, backup_files, name, datetime(y, m, d, H, M, S, us, tzinfo=tzinfo)) +def prune_ungrouped(archiver, *args, **kwargs): + """ + Run prune with grouping switched off. + + The tests below give each archive an own name so they can be told apart in the prune output, + while testing the retention rules as if all of them belonged to one archive series. Grouping + by name (the default) would put each of these archives into a group of its own, so these + tests ask for a single group explicitly. Grouping itself is tested in test_prune_group_by_*. + """ + return cmd(archiver, "prune", "--group-by", "", *args, **kwargs) + + def test_prune_repository(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=1") assert re.search(r"Would prune:\s+test1", output) # Must keep the latest archive: assert re.search(r"Keeping archive \(rule: daily #1\):\s+test2", output) output = cmd(archiver, "repo-list") assert "test1" in output assert "test2" in output - output = cmd(archiver, "prune", "--list", "--keep-daily=1") + output = prune_ungrouped(archiver, "--list", "--keep-daily=1") assert re.search(r"Pruning archive \(1/1\):\s+test1", output) output = cmd(archiver, "repo-list") assert "test1" not in output @@ -90,7 +108,7 @@ def test_prune_repository_example(archivers, request, backup_files): _create_archive_ts(archiver, backup_files, "test23", 2015, 5, 31) # The next older daily backup _create_archive_ts(archiver, backup_files, "test24", 2015, 12, 16) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") # Prune second backup of the year assert re.search(r"Would prune:\s+test22", output) # Prune next older monthly and daily backups @@ -107,7 +125,7 @@ def test_prune_repository_example(archivers, request, backup_files): # Nothing pruned after dry run for i in range(1, 25): assert "test%02d" % i in output - cmd(archiver, "prune", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") + prune_ungrouped(archiver, "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") output = cmd(archiver, "repo-list") # All matching backups plus oldest kept for i in range(1, 22): @@ -150,9 +168,8 @@ def test_prune_repository_example_interval(archivers, request, backup_files): for (y, m, d), name in zip(archive_dates, names): _create_archive_ts(archiver, backup_files, name, y, m, d, H=16) - output = cmd( + output = prune_ungrouped( archiver, - "prune", "--list", "--dry-run", "--from=2026-06-04T16:00:00+00:00", @@ -250,7 +267,7 @@ def mk_name(tup): to_prune = list(set(test_dates) - set(to_keep)) # Use 99 instead of -1 to test that oldest backup is kept. - output = cmd(archiver, "prune", "--list", "--dry-run", f"--keep-{strat}=99") + output = prune_ungrouped(archiver, "--list", "--dry-run", f"--keep-{strat}=99") for a in map(mk_name, to_prune): assert re.search(rf"Would prune:\s+{a}", output) @@ -264,7 +281,7 @@ def mk_name(tup): for a in map(mk_name, test_dates): assert a in output - cmd(archiver, "prune", f"--keep-{strat}=99") + prune_ungrouped(archiver, f"--keep-{strat}=99") output = cmd(archiver, "repo-list") # All matching backups plus oldest kept for a in map(mk_name, to_keep): @@ -286,19 +303,19 @@ def test_prune_retain_and_expire_oldest(archivers, request, backup_files): # Archive and prune daily for 30 days for i in range(1, 31): _create_archive_ts(archiver, backup_files, "september%02d" % i, 2020, 9, i, 12) - cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") + prune_ungrouped(archiver, "--keep-daily=7", "--keep-monthly=1") # Archive and prune 6 days into the next month for i in range(1, 7): _create_archive_ts(archiver, backup_files, "october%02d" % i, 2020, 10, i, 12) - cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") + prune_ungrouped(archiver, "--keep-daily=7", "--keep-monthly=1") # Oldest backup is still retained - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") assert re.search(r"Keeping archive \(rule: monthly\[oldest\] #1" + r"\):\s+original_archive", output) # Archive one more day and prune. _create_archive_ts(archiver, backup_files, "october07", 2020, 10, 7, 12) - cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") + prune_ungrouped(archiver, "--keep-daily=7", "--keep-monthly=1") # Last day of previous month is retained as monthly, and oldest is expired. - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") assert re.search(r"Keeping archive \(rule: monthly #1\):\s+september30", output) assert "original_archive" not in output @@ -310,7 +327,7 @@ def test_prune_repository_prefix(archivers, request, backup_files): cmd(archiver, "create", "foo-2015-08-12-20:00", backup_files) cmd(archiver, "create", "bar-2015-08-12-10:00", backup_files) cmd(archiver, "create", "bar-2015-08-12-20:00", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:foo-*") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:foo-*") assert re.search(r"Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00", output) assert re.search(r"Would prune:\s+foo-2015-08-12-10:00", output) output = cmd(archiver, "repo-list") @@ -318,7 +335,7 @@ def test_prune_repository_prefix(archivers, request, backup_files): assert "foo-2015-08-12-20:00" in output assert "bar-2015-08-12-10:00" in output assert "bar-2015-08-12-20:00" in output - cmd(archiver, "prune", "--keep-daily=1", "--match-archives=sh:foo-*") + prune_ungrouped(archiver, "--keep-daily=1", "--match-archives=sh:foo-*") output = cmd(archiver, "repo-list") assert "foo-2015-08-12-10:00" not in output assert "foo-2015-08-12-20:00" in output @@ -333,7 +350,7 @@ def test_prune_repository_glob(archivers, request, backup_files): cmd(archiver, "create", "2015-08-12-20:00-foo", backup_files) cmd(archiver, "create", "2015-08-12-10:00-bar", backup_files) cmd(archiver, "create", "2015-08-12-20:00-bar", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:2015-*-foo") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:2015-*-foo") assert re.search(r"Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo", output) assert re.search(r"Would prune:\s+2015-08-12-10:00-foo", output) output = cmd(archiver, "repo-list") @@ -341,7 +358,7 @@ def test_prune_repository_glob(archivers, request, backup_files): assert "2015-08-12-20:00-foo" in output assert "2015-08-12-10:00-bar" in output assert "2015-08-12-20:00-bar" in output - cmd(archiver, "prune", "--keep-daily=1", "--match-archives=sh:2015-*-foo") + prune_ungrouped(archiver, "--keep-daily=1", "--match-archives=sh:2015-*-foo") output = cmd(archiver, "repo-list") assert "2015-08-12-10:00-foo" not in output assert "2015-08-12-20:00-foo" in output @@ -356,7 +373,7 @@ def test_prune_ignore_protected(archivers, request): cmd(archiver, "tag", "--set=@PROT", "archive1") # do not delete archive1! cmd(archiver, "create", "archive2", archiver.input_path) cmd(archiver, "create", "archive3", archiver.input_path) - output = cmd(archiver, "prune", "--list", "--keep=1", "--match-archives=sh:archive*") + output = prune_ungrouped(archiver, "--list", "--keep=1", "--match-archives=sh:archive*") assert "archive1" not in output # @PROT archives are completely ignored. assert re.search(r"Keeping archive \(rule: keep #1\):\s+archive3", output) assert re.search(r"Pruning archive \(.*?\):\s+archive2", output) @@ -465,7 +482,7 @@ def test_prune_list_with_metadata_format(archivers, request, backup_files): cmd(archiver, "create", "test2", backup_files) # {hostname} is a "call key" that triggers lazy loading of the archive from the repo. # With the buggy code this would raise Archive.DoesNotExist for the pruned archive. - output = cmd(archiver, "prune", "--list", "--keep-daily=1", "--format={name} {hostname}{NL}") + output = prune_ungrouped(archiver, "--list", "--keep-daily=1", "--format={name} {hostname}{NL}") assert "test1" in output assert "test2" in output @@ -475,7 +492,7 @@ def test_prune_json(archivers, request, backup_files): cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - prune_result = json.loads(cmd(archiver, "prune", "--json", "--dry-run", "--keep-daily=1")) + prune_result = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", "--keep-daily=1")) assert "repository" in prune_result assert "encryption" in prune_result assert len(prune_result["repository"]["id"]) == 64 @@ -506,7 +523,7 @@ def test_prune_json_list_pruned(archivers, request, backup_files): cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - prune_result = json.loads(cmd(archiver, "prune", "--json", "--dry-run", "--list-pruned", "--keep-daily=1")) + prune_result = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", "--list-pruned", "--keep-daily=1")) archives = prune_result["archives"] assert len(archives) == 1 assert archives[0]["name"] == "test1" @@ -519,7 +536,7 @@ def test_prune_keep_same_second(archivers, request, backup_files): cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep=2") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep=2") # Both archives are kept even though they have the same timestamp to the second. Would previously have failed with # old behavior of --keep-last. Archives sorted on seconds, order is undefined. assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test1", output) @@ -538,7 +555,7 @@ def test_prune_keep_int_or_interval(archivers, request, backup_files, keep_arg): ) # Would be pruned if `secondly`-rule was active. _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=1)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(seconds=1, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: skip #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-2", output) assert re.search(r"Keeping archive \(rule: keep #2\):\s+test-3", output) @@ -554,7 +571,7 @@ def test_prune_keep_secondly_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(seconds=1, microseconds=999999)) _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(seconds=2, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -571,7 +588,7 @@ def test_prune_keep_minutely_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(minutes=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(minutes=3)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(minutes=3, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -589,7 +606,7 @@ def test_prune_keep_hourly_int_or_interval(archivers, request, backup_files, kee _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(hours=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(hours=3)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(hours=3, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -607,7 +624,7 @@ def test_prune_keep_daily_int_or_interval(archivers, request, backup_files, keep _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=3)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=3, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -625,7 +642,7 @@ def test_prune_keep_weekly_int_or_interval(archivers, request, backup_files, kee _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=14)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=21)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=21, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -643,7 +660,7 @@ def test_prune_keep_monthly_int_or_interval(archivers, request, backup_files, ke _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=62)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=93)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=93, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -662,7 +679,7 @@ def test_prune_keep_13weekly_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=182)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=273)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=273, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -681,7 +698,7 @@ def test_prune_keep_3monthly_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=275)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=275, microseconds=1)) # 275d is the interval from dt to the oldest kept monthly archive - output = cmd(archiver, "prune", "--list", "--short", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--short", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -699,7 +716,7 @@ def test_prune_keep_yearly_int_or_interval(archivers, request, backup_files, kee _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=730)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=1095)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=1095, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -715,7 +732,7 @@ def test_prune_keep_daily_all(archivers, request, backup_files, keep_arg): _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=1)) _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(days=2)) _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=3)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-3", output) @@ -731,7 +748,7 @@ def test_prune_keep_flat_all(archivers, request, backup_files, keep_arg): _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(microseconds=2)) _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=3)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=3333)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: keep #2\):\s+test-2", output) assert re.search(r"Keeping archive \(rule: keep #3\):\s+test-3", output) @@ -834,7 +851,7 @@ def test_prune_from_prefiltered_archives_ignored_in_pruning(archivers, request, _create_archive_dt(archiver, backup_files, "test-b", dt - timedelta(hours=1)) _create_archive_dt(archiver, backup_files, "test-c", dt - timedelta(days=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), "--keep-daily=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), "--keep-daily=1") # 'test-b' is kept, meaning 'test-a' was entirely skipped for pruning consideration. # They would otherwise have occupied the same period. @@ -913,3 +930,149 @@ def test_unique_period_values_are_padded_and_ordered(): assert len(set(values)) == len(values) # each archive lands in a period of its own assert len({len(value) for value in values}) == 1 # uniform width, so ... assert values == sorted(values) # ... lexicographic ordering matches numeric ordering + + +def _grouped_archive(name="home", host="myhost", user="myuser", tags=()): + return ArchiveInfo(name=name, id=b"", ts=datetime(2024, 1, 1, tzinfo=timezone.utc), tags=tags, host=host, user=user) + + +def test_archive_group_key(): + archive_info = _grouped_archive(name="home", host="myhost", user="myuser", tags=("b", "a")) + assert archive_group_key(archive_info, ()) == () + assert archive_group_key(archive_info, ("name",)) == ("home",) + assert archive_group_key(archive_info, ("name", "host", "user")) == ("home", "myhost", "myuser") + assert archive_group_key(archive_info, ("host", "name")) == ("myhost", "home") # order matters + assert archive_group_key(archive_info, ("tags",)) == ("b,a",) + + +def test_archive_group_key_ignores_internal_tags(): + """Internal tags say nothing about an archive's contents, they must not create their own group.""" + tagged = _grouped_archive(tags=("@PROT", "important")) + untagged = _grouped_archive(tags=("important",)) + assert archive_group_key(tagged, ("tags",)) == archive_group_key(untagged, ("tags",)) == ("important",) + + +def test_archive_group_key_missing_metadata(): + """Archives without host / user metadata (e.g. transferred from borg 1.x) form their own group.""" + assert archive_group_key(_grouped_archive(host=None, user=None), ("host", "user")) == ("", "") + assert archive_group_key(_grouped_archive(host="", user=""), ("host", "user")) == ("", "") + + +def test_group_archives(): + home1, home2 = _grouped_archive(host="host1"), _grouped_archive(host="host2") + etc1 = _grouped_archive(name="etc", host="host1") + archives = [home1, etc1, home2] + + assert group_archives(archives, ()) == {(): archives} # no grouping: one group with all archives + assert group_archives(archives, ("name",)) == {("home",): [home1, home2], ("etc",): [etc1]} + assert group_archives(archives, ("name", "host")) == { + ("home", "host1"): [home1], + ("etc", "host1"): [etc1], + ("home", "host2"): [home2], + } + + +def test_format_group_key(): + assert format_group_key(("home", "host1"), ("name", "host")) == "name='home', host='host1'" + + +def test_group_by_spec(): + assert GroupBySpec("name") == "name" + assert GroupBySpec("name,host,user,tags") == "name,host,user,tags" + assert GroupBySpec("") == GroupBySpec("none") == "" + # the parsed value is fed through the spec again by the argument parser, so it must be stable: + assert GroupBySpec(GroupBySpec("name")) == GroupBySpec("name") + assert GroupBySpec(GroupBySpec("none")) == GroupBySpec("none") + with pytest.raises(ArgumentTypeError, match="Invalid group-by key: bogus"): + GroupBySpec("name,bogus") + with pytest.raises(ArgumentTypeError, match="Duplicate group-by key: name"): + GroupBySpec("name,name") + + +def _create_series(archiver, backup_files, name, hour): + """Create a two archive series called *name*, one archive on 2024-01-01, one on 2024-01-02.""" + _create_archive_ts(archiver, backup_files, name, 2024, 1, 1, H=hour) + _create_archive_ts(archiver, backup_files, name, 2024, 1, 2, H=hour) + + +def _create_shared_repo_series(archiver, backup_files, monkeypatch): + """host1 has a "home" and an "etc" series, host2 has an own, unrelated "home" series.""" + monkeypatch.setenv("BORG_HOSTNAME", "host1") + _create_series(archiver, backup_files, "home", hour=10) + _create_series(archiver, backup_files, "etc", hour=11) + monkeypatch.setenv("BORG_HOSTNAME", "host2") + _create_series(archiver, backup_files, "home", hour=12) + monkeypatch.delenv("BORG_HOSTNAME") + + +def test_prune_groups_by_name_and_host_by_default(archivers, request, backup_files, monkeypatch): + """Unrelated backup sets must not compete for the same retention slots.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_shared_repo_series(archiver, backup_files, monkeypatch) + # 3 groups: (home, host1), (etc, host1), (home, host2) - each keeps its own daily archive. + output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1") + assert output.count("Would prune:") == 3 + + +def test_prune_group_by_name_only(archivers, request, backup_files, monkeypatch): + """--group-by name pools the same series name of different hosts into one retention pool.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_shared_repo_series(archiver, backup_files, monkeypatch) + # 2 groups: "home" (of both hosts) and "etc". + output = cmd(archiver, "prune", "--list", "--dry-run", "--group-by", "name", "--keep-daily=1") + assert output.count("Would prune:") == 4 + + +def test_prune_group_by_none(archivers, request, backup_files): + """--group-by "" applies the rules to all selected archives at once.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_series(archiver, backup_files, "home", hour=10) + _create_series(archiver, backup_files, "etc", hour=11) + cmd(archiver, "prune", "--group-by", "", "--keep-daily=1") + output = cmd(archiver, "repo-list", "--format={name}{NL}") + assert output.split() == ["etc"] # only the single newest archive of all 4 survives + + +def test_prune_group_by_tags(archivers, request, backup_files): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_series(archiver, backup_files, "home", hour=10) + _create_series(archiver, backup_files, "etc", hour=11) + cmd(archiver, "tag", "--add=alice", "-a", "name:home") + cmd(archiver, "tag", "--add=bob", "-a", "name:etc") + output = cmd(archiver, "prune", "--list", "--dry-run", "--group-by", "tags", "--keep-daily=1") + assert output.count("Would prune:") == 2 # one archive pruned per tag group + + +def test_prune_group_by_logs_a_summary_per_group(archivers, request, backup_files, monkeypatch): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_shared_repo_series(archiver, backup_files, monkeypatch) + output = cmd(archiver, "prune", "--info", "--dry-run", "--keep-daily=1") + assert "Group (name='home', host='host1'): 2 archives, keeping 1, pruning 1." in output + assert "Group (name='etc', host='host1'): 2 archives, keeping 1, pruning 1." in output + assert "Group (name='home', host='host2'): 2 archives, keeping 1, pruning 1." in output + + +def test_prune_group_by_json(archivers, request, backup_files, monkeypatch): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + for host, hour in (("host1", 10), ("host2", 11)): + monkeypatch.setenv("BORG_HOSTNAME", host) + _create_series(archiver, backup_files, "home", hour=hour) + monkeypatch.delenv("BORG_HOSTNAME") + prune_result = json.loads( + cmd(archiver, "prune", "--json", "--dry-run", "--group-by", "name,host", "--keep-daily=1") + ) + groups = {tuple(sorted(archive["group"].items())) for archive in prune_result["archives"]} + assert groups == {(("host", "host1"), ("name", "home")), (("host", "host2"), ("name", "home"))} + + +def test_prune_group_by_invalid_key(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + output = cmd(archiver, "prune", "--group-by", "bogus", "--keep-daily=1", exit_code=2) + assert "Invalid group-by key: bogus" in output