diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index f1e9888609..daaf833f4e 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -1,3 +1,5 @@ +from collections import defaultdict +from collections.abc import Sequence from typing import Callable, NamedTuple from datetime import datetime, timedelta import logging @@ -9,9 +11,10 @@ from ..constants import * # NOQA from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error from ..helpers import archivename_validator, int_or_interval, sig_int, timestamp +from ..helpers import GroupBySpec from ..helpers import json_print, basic_json_data from ..helpers.argparsing import ArgumentParser -from ..manifest import ArchiveInfo, Manifest +from ..manifest import AI_GROUP_BY_KEYS, ArchiveInfo, Manifest from ..logger import create_logger @@ -187,6 +190,40 @@ def can_retain(a): return keep +def archive_group_key(archive_info: ArchiveInfo, group_by: Sequence[str]) -> tuple[str, ...]: + """Compute the grouping key of *archive_info* for the given *group_by* archive attributes.""" + key = [] + for group_by_key in group_by: + if group_by_key == "tags": + # internal tags (e.g. @PROT) say nothing about what an archive contains or where it + # came from, so they must not put an archive into a group of its own. + value = ",".join(tag for tag in archive_info.tags if not tag.startswith("@")) + else: + # host and user are empty for archives that do not have this metadata, e.g. archives + # transferred from a borg 1.x repo. they form their own group then. + value = getattr(archive_info, group_by_key) or "" + key.append(value) + return tuple(key) + + +def group_archives(archives: list[ArchiveInfo], group_by: Sequence[str]) -> dict[tuple[str, ...], list[ArchiveInfo]]: + """ + Group *archives* by the given *group_by* archive attributes, keeping their relative order. + + An empty *group_by* puts all archives into one group, so the retention rules are applied to + all given archives at once. + """ + groups: dict[tuple[str, ...], list[ArchiveInfo]] = defaultdict(list) + for archive_info in archives: + groups[archive_group_key(archive_info, group_by)].append(archive_info) + return groups + + +def format_group_key(key: tuple[str, ...], group_by: Sequence[str]) -> str: + """Format a grouping key for human consumption, e.g. \"name='home', host='myhost'\".""" + return ", ".join(f"{group_by_key}={value!r}" for group_by_key, value in zip(group_by, key)) + + class PruneMixIn: @with_repository(compatibility=(Manifest.Operation.DELETE,)) def do_prune(self, args, repository, manifest): @@ -197,42 +234,25 @@ def do_prune(self, args, repository, manifest): archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True) archives = [ai for ai in archives if "@PROT" not in ai.tags] - # Archives to keep along with the rule that ensured them being kept - keep = {} + # The retention rules are applied to each group of archives separately, so that unrelated + # backup sets (e.g. different archive series or different machines sharing a repository) + # do not compete for the same retention slots. + group_by = tuple(args.group_by.split(",")) if args.group_by else () + groups = group_archives(archives, group_by) + # All groups use the same reference timestamp, so interval based rules do not drift apart. from_timestamp = getattr(args, PRUNE_FROM.key) - candidate_archives = archives - - if from_timestamp is not None: - base_timestamp = from_timestamp + base_timestamp = from_timestamp if from_timestamp is not None else datetime.now().astimezone() - # `--from` is a prefilter: Archives made at or after this time are kept by default. They are not considered - # for pruning at all and thus won't falsely occupy an active retention period. - for archive in archives: - if archive.ts < from_timestamp: - break - keep[archive] = KeepResult(rule=PRUNE_FROM, idx=len(keep)) - candidate_archives = archives[len(keep) :] - else: - base_timestamp = datetime.now().astimezone() - - # Apply each retention rule to all candidate archives. The - # `previously_kept` parameter prevents later (coarser-grained) rules - # from double-counting archives already retained by earlier rules. - active_rules = [ - (rule, getattr(args, rule.key)) for rule in PRUNING_RULES if getattr(args, rule.key) is not None - ] - for rule, n_or_interval in active_rules: - keep |= prune( - archives=candidate_archives, - rule=rule, - n_or_interval=n_or_interval, - base_timestamp=base_timestamp, - keep_oldest=( - rule == active_rules[-1][0] - ), # Activate keep_oldest rule only for the largest active interval - previously_kept=frozenset(keep), - ) + # Archives to keep along with the rule that ensured them being kept + keep = {} + group_of = {} + group_summaries = [] + for key, group in groups.items(): + group_keep = self._compute_keep(group, args, base_timestamp) + keep |= group_keep + group_of.update(dict.fromkeys(group, key)) + group_summaries.append((key, len(group), len(group_keep))) archives_to_prune = set(archives) - set(keep) @@ -249,6 +269,16 @@ def do_prune(self, args, repository, manifest): else: logger.info("Repository contains %d archives.", manifest.archives.count()) logger.info("Applying rules to the matching %d archives...", len(archives)) + if len(group_summaries) > 1: + # with a single group, the totals below already say everything there is to say. + for key, group_total, group_kept in group_summaries: + logger.info( + "Group (%s): %d archives, keeping %d, pruning %d.", + format_group_key(key, group_by), + group_total, + group_kept, + group_total - group_kept, + ) logger.info("Keeping %d archives, pruning %d archives.", len(keep), len(archives_to_prune)) list_logger = logging.getLogger("borg.output.list") @@ -262,6 +292,7 @@ def do_prune(self, args, repository, manifest): # so we must call it before deleting the archive. if args.json: archive_data = formatter.get_item_data(archive_info, jsonline=True) + archive_data["group"] = dict(zip(group_by, group_of[archive_info])) else: archive_formatted = formatter.format_item(archive_info, jsonline=False) if archive_info in archives_to_prune: @@ -309,6 +340,46 @@ def do_prune(self, args, repository, manifest): if sig_int: raise Error("Got Ctrl-C / SIGINT.") + def _compute_keep(self, archives, args, base_timestamp): + """ + Apply the retention rules to *archives* (sorted by timestamp, newest first). + + Return the archives to keep, mapped to the rule that ensured them being kept. + """ + keep = {} + + from_timestamp = getattr(args, PRUNE_FROM.key) + candidate_archives = archives + + if from_timestamp is not None: + # `--from` is a prefilter: Archives made at or after this time are kept by default. They are not considered + # for pruning at all and thus won't falsely occupy an active retention period. + for archive in archives: + if archive.ts < from_timestamp: + break + keep[archive] = KeepResult(rule=PRUNE_FROM, idx=len(keep)) + candidate_archives = archives[len(keep) :] + + # Apply each retention rule to all candidate archives. The + # `previously_kept` parameter prevents later (coarser-grained) rules + # from double-counting archives already retained by earlier rules. + active_rules = [ + (rule, getattr(args, rule.key)) for rule in PRUNING_RULES if getattr(args, rule.key) is not None + ] + for rule, n_or_interval in active_rules: + keep |= prune( + archives=candidate_archives, + rule=rule, + n_or_interval=n_or_interval, + base_timestamp=base_timestamp, + keep_oldest=( + rule == active_rules[-1][0] + ), # Activate keep_oldest rule only for the largest active interval + previously_kept=frozenset(keep), + ) + + return keep + def _validate_prune_args(self, args): keep_args = {rule.key: getattr(args, rule.key) for rule in PRUNING_RULES if getattr(args, rule.key) is not None} @@ -367,32 +438,57 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): `GFS `_ (Grandfather-father-son) backup rotation scheme. - The recommended way to use prune is to give the archive series name to it via the - NAME argument (assuming you have the same name for all archives in a series). - Alternatively, you can also use --match-archives (-a), then only archives that - match the pattern are considered for deletion and only those archives count - towards the totals specified by the rules. - Otherwise, *all* archives in the repository are candidates for deletion! - There is no automatic distinction between archives representing different - contents. These need to be distinguished by specifying matching globs. + Two separate mechanisms decide what prune does: the archive *selection* determines + which archives are considered at all, and ``--group-by`` determines how the retention + rules subdivide the selected archives. - NAME is just another way of saying ``-a NAME``, so it can be combined with + Without NAME and without --match-archives (-a), *all* archives in the repository are + selected. Give the archive series name via the NAME argument (assuming you have the + same name for all archives in a series) or use --match-archives (-a) to only consider + a subset. NAME is just another way of saying ``-a NAME``, so it can be combined with --match-archives (-a). All given patterns must match (they are ANDed), thus giving both narrows down the selection. - If you have multiple series of archives with different data sets (e.g. - from different machines) in one shared repository, use one prune call per - series. In such a shared repository, the series name alone might not be - specific enough, because different machines or users may use the same series - name for their own, unrelated data. Additionally match on the archive metadata - that identifies the origin, e.g.:: + The retention rules are applied to each group of selected archives separately, so + that unrelated backup sets do not compete for the same retention slots. By default, + archives are grouped by their series name and the host they were made on + (``--group-by name,host``): ``--keep-daily 7`` keeps 7 daily archives *per series and + host*. Archive series names are not unique in a repository shared by multiple machines + or users - different machines may well use the same series name for their own, + unrelated data - so grouping by the name alone would let one machine's archives push + another machine's archives out of the retention slots. + + ``--group-by`` accepts a comma-separated list of the archive attributes ``name``, + ``host``, ``user`` and ``tags``. Archives without ``host`` / ``user`` metadata (e.g. + archives transferred from a borg 1.x repository) form their own group. Borg's internal + tags (starting with ``@``) do not affect grouping. Give ``--group-by ""`` to not group + at all and apply the rules to all selected archives at once. + + Use ``--group-by name`` if several machines back up the same data to the same archive + series on purpose and you want one retention policy for all of them. Add ``user`` if + one machine backs up the same series as different users and each of them should get + their own retention. + + So, to prune every series of every host in one call, the default is all you need:: + + borg prune --keep-daily 7 --keep-weekly 4 + + To only consider your own archives, select them - the grouping then applies to the + selection only:: - borg prune home -a host:myhost --keep-daily 7 --keep-weekly 4 + borg prune -a host:myhost --keep-daily 7 --keep-weekly 4 Note: the ``host:`` / ``user:`` metadata is what the client wrote into the archive; it is not a permission mechanism. Any client with delete permission for the repository can prune any archive in it. + Beware of grouping by an attribute that is not stable over time: if e.g. the hostname + changes for every backup run (as it might for containers), each archive ends up in a + group of its own and the retention rules will keep all of them. Set ``BORG_HOSTNAME`` + to a stable value in that case, or use ``--group-by name``. Running prune with + ``--info`` shows one summary line per group, which makes such a situation visible. + + The ``--keep`` option is the simplest way to specify a basic retention policy. It accepts a count or a time interval for retention (e.g. ``10`` or ``7d``, ``4w``). With a count it keeps at most that many @@ -514,6 +610,17 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): action=Highlander, help="only consider archives older than this for pruning", ) + subparser.add_argument( + "--group-by", + metavar="KEYS", + dest="group_by", + type=GroupBySpec, + default="name,host", + action=Highlander, + help="comma-separated list of archive attributes to group the archives by before " + "applying the retention rules to each group separately; valid keys are: {}; " + 'default is: name,host; use "" (or "none") to not group at all'.format(", ".join(AI_GROUP_BY_KEYS)), + ) subparser.add_argument( "--keep", dest=PRUNE_KEEP.key, diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index ff8a5a8a5a..96649b65c1 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -38,6 +38,7 @@ FilesystemPathSpec, FilesystemDirSpec, SortBySpec, + GroupBySpec, CompressionSpec, ChunkerParams, DigestAlgos, diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index a3fbeb90ce..67670c9bd2 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -582,6 +582,24 @@ def SortBySpec(text): return text.replace("timestamp", "ts").replace("archive", "name") +def GroupBySpec(text): + """Validate a comma-separated list of group-by keys. "" and "none" mean: do not group.""" + from ..manifest import AI_GROUP_BY_KEYS + + if text in ("", "none"): + return "" # idempotency: the normalized value must pass validation again + seen = set() + for group_key in text.split(","): + if group_key not in AI_GROUP_BY_KEYS: + raise ArgumentTypeError( + "Invalid group-by key: %s (valid keys: %s)" % (group_key, ", ".join(AI_GROUP_BY_KEYS)) + ) + if group_key in seen: + raise ArgumentTypeError("Duplicate group-by key: %s" % group_key) + seen.add(group_key) + return text + + SIZE_UNITS = ("si", "iec", "raw") _warned_units: set[str] = set() # invalid BORG_UNITS values already complained about diff --git a/src/borg/manifest.py b/src/borg/manifest.py index d25070d8f4..f121085bb1 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -47,6 +47,10 @@ class NoManifestError(Error): AI_HUMAN_SORT_KEYS = ["timestamp", "archive"] + list(ArchiveInfo._fields) AI_HUMAN_SORT_KEYS.remove("ts") +# archive attributes describing what an archive contains and where it came from, usable to group +# archives that belong together, e.g. for applying retention rules separately (see GroupBySpec). +AI_GROUP_BY_KEYS = ["name", "host", "user", "tags"] + def filter_archives_by_date(archives, older=None, newer=None, oldest=None, newest=None): def get_first_and_last_archive_ts(archives_list): diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index 4553a5fc9b..faa81a3fc3 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -15,8 +15,14 @@ PRUNE_SECONDLY, PRUNE_WEEKLY, PRUNE_YEARLY, + archive_group_key, + format_group_key, + group_archives, unique_period_func, ) +from ...helpers import GroupBySpec +from argparse import ArgumentTypeError + from ...helpers import CommandError from ...manifest import ArchiveInfo from . import cmd, RK_ENCRYPTION, generate_archiver_tests @@ -34,19 +40,31 @@ def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0, us= _create_archive_dt(archiver, backup_files, name, datetime(y, m, d, H, M, S, us, tzinfo=tzinfo)) +def prune_ungrouped(archiver, *args, **kwargs): + """ + Run prune with grouping switched off. + + The tests below give each archive an own name so they can be told apart in the prune output, + while testing the retention rules as if all of them belonged to one archive series. Grouping + by name (the default) would put each of these archives into a group of its own, so these + tests ask for a single group explicitly. Grouping itself is tested in test_prune_group_by_*. + """ + return cmd(archiver, "prune", "--group-by", "", *args, **kwargs) + + def test_prune_repository(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=1") assert re.search(r"Would prune:\s+test1", output) # Must keep the latest archive: assert re.search(r"Keeping archive \(rule: daily #1\):\s+test2", output) output = cmd(archiver, "repo-list") assert "test1" in output assert "test2" in output - output = cmd(archiver, "prune", "--list", "--keep-daily=1") + output = prune_ungrouped(archiver, "--list", "--keep-daily=1") assert re.search(r"Pruning archive \(1/1\):\s+test1", output) output = cmd(archiver, "repo-list") assert "test1" not in output @@ -90,7 +108,7 @@ def test_prune_repository_example(archivers, request, backup_files): _create_archive_ts(archiver, backup_files, "test23", 2015, 5, 31) # The next older daily backup _create_archive_ts(archiver, backup_files, "test24", 2015, 12, 16) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") # Prune second backup of the year assert re.search(r"Would prune:\s+test22", output) # Prune next older monthly and daily backups @@ -107,7 +125,7 @@ def test_prune_repository_example(archivers, request, backup_files): # Nothing pruned after dry run for i in range(1, 25): assert "test%02d" % i in output - cmd(archiver, "prune", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") + prune_ungrouped(archiver, "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1") output = cmd(archiver, "repo-list") # All matching backups plus oldest kept for i in range(1, 22): @@ -150,9 +168,8 @@ def test_prune_repository_example_interval(archivers, request, backup_files): for (y, m, d), name in zip(archive_dates, names): _create_archive_ts(archiver, backup_files, name, y, m, d, H=16) - output = cmd( + output = prune_ungrouped( archiver, - "prune", "--list", "--dry-run", "--from=2026-06-04T16:00:00+00:00", @@ -250,7 +267,7 @@ def mk_name(tup): to_prune = list(set(test_dates) - set(to_keep)) # Use 99 instead of -1 to test that oldest backup is kept. - output = cmd(archiver, "prune", "--list", "--dry-run", f"--keep-{strat}=99") + output = prune_ungrouped(archiver, "--list", "--dry-run", f"--keep-{strat}=99") for a in map(mk_name, to_prune): assert re.search(rf"Would prune:\s+{a}", output) @@ -264,7 +281,7 @@ def mk_name(tup): for a in map(mk_name, test_dates): assert a in output - cmd(archiver, "prune", f"--keep-{strat}=99") + prune_ungrouped(archiver, f"--keep-{strat}=99") output = cmd(archiver, "repo-list") # All matching backups plus oldest kept for a in map(mk_name, to_keep): @@ -286,19 +303,19 @@ def test_prune_retain_and_expire_oldest(archivers, request, backup_files): # Archive and prune daily for 30 days for i in range(1, 31): _create_archive_ts(archiver, backup_files, "september%02d" % i, 2020, 9, i, 12) - cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") + prune_ungrouped(archiver, "--keep-daily=7", "--keep-monthly=1") # Archive and prune 6 days into the next month for i in range(1, 7): _create_archive_ts(archiver, backup_files, "october%02d" % i, 2020, 10, i, 12) - cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") + prune_ungrouped(archiver, "--keep-daily=7", "--keep-monthly=1") # Oldest backup is still retained - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") assert re.search(r"Keeping archive \(rule: monthly\[oldest\] #1" + r"\):\s+original_archive", output) # Archive one more day and prune. _create_archive_ts(archiver, backup_files, "october07", 2020, 10, 7, 12) - cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1") + prune_ungrouped(archiver, "--keep-daily=7", "--keep-monthly=1") # Last day of previous month is retained as monthly, and oldest is expired. - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1") assert re.search(r"Keeping archive \(rule: monthly #1\):\s+september30", output) assert "original_archive" not in output @@ -310,7 +327,7 @@ def test_prune_repository_prefix(archivers, request, backup_files): cmd(archiver, "create", "foo-2015-08-12-20:00", backup_files) cmd(archiver, "create", "bar-2015-08-12-10:00", backup_files) cmd(archiver, "create", "bar-2015-08-12-20:00", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:foo-*") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:foo-*") assert re.search(r"Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00", output) assert re.search(r"Would prune:\s+foo-2015-08-12-10:00", output) output = cmd(archiver, "repo-list") @@ -318,7 +335,7 @@ def test_prune_repository_prefix(archivers, request, backup_files): assert "foo-2015-08-12-20:00" in output assert "bar-2015-08-12-10:00" in output assert "bar-2015-08-12-20:00" in output - cmd(archiver, "prune", "--keep-daily=1", "--match-archives=sh:foo-*") + prune_ungrouped(archiver, "--keep-daily=1", "--match-archives=sh:foo-*") output = cmd(archiver, "repo-list") assert "foo-2015-08-12-10:00" not in output assert "foo-2015-08-12-20:00" in output @@ -333,7 +350,7 @@ def test_prune_repository_glob(archivers, request, backup_files): cmd(archiver, "create", "2015-08-12-20:00-foo", backup_files) cmd(archiver, "create", "2015-08-12-10:00-bar", backup_files) cmd(archiver, "create", "2015-08-12-20:00-bar", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:2015-*-foo") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:2015-*-foo") assert re.search(r"Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo", output) assert re.search(r"Would prune:\s+2015-08-12-10:00-foo", output) output = cmd(archiver, "repo-list") @@ -341,7 +358,7 @@ def test_prune_repository_glob(archivers, request, backup_files): assert "2015-08-12-20:00-foo" in output assert "2015-08-12-10:00-bar" in output assert "2015-08-12-20:00-bar" in output - cmd(archiver, "prune", "--keep-daily=1", "--match-archives=sh:2015-*-foo") + prune_ungrouped(archiver, "--keep-daily=1", "--match-archives=sh:2015-*-foo") output = cmd(archiver, "repo-list") assert "2015-08-12-10:00-foo" not in output assert "2015-08-12-20:00-foo" in output @@ -356,7 +373,7 @@ def test_prune_ignore_protected(archivers, request): cmd(archiver, "tag", "--set=@PROT", "archive1") # do not delete archive1! cmd(archiver, "create", "archive2", archiver.input_path) cmd(archiver, "create", "archive3", archiver.input_path) - output = cmd(archiver, "prune", "--list", "--keep=1", "--match-archives=sh:archive*") + output = prune_ungrouped(archiver, "--list", "--keep=1", "--match-archives=sh:archive*") assert "archive1" not in output # @PROT archives are completely ignored. assert re.search(r"Keeping archive \(rule: keep #1\):\s+archive3", output) assert re.search(r"Pruning archive \(.*?\):\s+archive2", output) @@ -465,7 +482,7 @@ def test_prune_list_with_metadata_format(archivers, request, backup_files): cmd(archiver, "create", "test2", backup_files) # {hostname} is a "call key" that triggers lazy loading of the archive from the repo. # With the buggy code this would raise Archive.DoesNotExist for the pruned archive. - output = cmd(archiver, "prune", "--list", "--keep-daily=1", "--format={name} {hostname}{NL}") + output = prune_ungrouped(archiver, "--list", "--keep-daily=1", "--format={name} {hostname}{NL}") assert "test1" in output assert "test2" in output @@ -475,7 +492,7 @@ def test_prune_json(archivers, request, backup_files): cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - prune_result = json.loads(cmd(archiver, "prune", "--json", "--dry-run", "--keep-daily=1")) + prune_result = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", "--keep-daily=1")) assert "repository" in prune_result assert "encryption" in prune_result assert len(prune_result["repository"]["id"]) == 64 @@ -506,7 +523,7 @@ def test_prune_json_list_pruned(archivers, request, backup_files): cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - prune_result = json.loads(cmd(archiver, "prune", "--json", "--dry-run", "--list-pruned", "--keep-daily=1")) + prune_result = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", "--list-pruned", "--keep-daily=1")) archives = prune_result["archives"] assert len(archives) == 1 assert archives[0]["name"] == "test1" @@ -519,7 +536,7 @@ def test_prune_keep_same_second(archivers, request, backup_files): cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test1", backup_files) cmd(archiver, "create", "test2", backup_files) - output = cmd(archiver, "prune", "--list", "--dry-run", "--keep=2") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--keep=2") # Both archives are kept even though they have the same timestamp to the second. Would previously have failed with # old behavior of --keep-last. Archives sorted on seconds, order is undefined. assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test1", output) @@ -538,7 +555,7 @@ def test_prune_keep_int_or_interval(archivers, request, backup_files, keep_arg): ) # Would be pruned if `secondly`-rule was active. _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=1)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(seconds=1, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: skip #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-2", output) assert re.search(r"Keeping archive \(rule: keep #2\):\s+test-3", output) @@ -554,7 +571,7 @@ def test_prune_keep_secondly_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(seconds=1, microseconds=999999)) _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(seconds=2, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -571,7 +588,7 @@ def test_prune_keep_minutely_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(minutes=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(minutes=3)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(minutes=3, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -589,7 +606,7 @@ def test_prune_keep_hourly_int_or_interval(archivers, request, backup_files, kee _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(hours=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(hours=3)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(hours=3, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -607,7 +624,7 @@ def test_prune_keep_daily_int_or_interval(archivers, request, backup_files, keep _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=2)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=3)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=3, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -625,7 +642,7 @@ def test_prune_keep_weekly_int_or_interval(archivers, request, backup_files, kee _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=14)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=21)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=21, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -643,7 +660,7 @@ def test_prune_keep_monthly_int_or_interval(archivers, request, backup_files, ke _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=62)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=93)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=93, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -662,7 +679,7 @@ def test_prune_keep_13weekly_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=182)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=273)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=273, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -681,7 +698,7 @@ def test_prune_keep_3monthly_int_or_interval(archivers, request, backup_files, k _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=275)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=275, microseconds=1)) # 275d is the interval from dt to the oldest kept monthly archive - output = cmd(archiver, "prune", "--list", "--short", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--short", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -699,7 +716,7 @@ def test_prune_keep_yearly_int_or_interval(archivers, request, backup_files, kee _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=730)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=1095)) _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=1095, microseconds=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output) assert re.search(r"Would prune:\s+test-3", output) @@ -715,7 +732,7 @@ def test_prune_keep_daily_all(archivers, request, backup_files, keep_arg): _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=1)) _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(days=2)) _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=3)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-3", output) @@ -731,7 +748,7 @@ def test_prune_keep_flat_all(archivers, request, backup_files, keep_arg): _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(microseconds=2)) _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=3)) _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=3333)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: keep #2\):\s+test-2", output) assert re.search(r"Keeping archive \(rule: keep #3\):\s+test-3", output) @@ -834,7 +851,7 @@ def test_prune_from_prefiltered_archives_ignored_in_pruning(archivers, request, _create_archive_dt(archiver, backup_files, "test-b", dt - timedelta(hours=1)) _create_archive_dt(archiver, backup_files, "test-c", dt - timedelta(days=1)) - output = cmd(archiver, "prune", "--list", "--dry-run", "--from", dt.isoformat(), "--keep-daily=1") + output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), "--keep-daily=1") # 'test-b' is kept, meaning 'test-a' was entirely skipped for pruning consideration. # They would otherwise have occupied the same period. @@ -913,3 +930,149 @@ def test_unique_period_values_are_padded_and_ordered(): assert len(set(values)) == len(values) # each archive lands in a period of its own assert len({len(value) for value in values}) == 1 # uniform width, so ... assert values == sorted(values) # ... lexicographic ordering matches numeric ordering + + +def _grouped_archive(name="home", host="myhost", user="myuser", tags=()): + return ArchiveInfo(name=name, id=b"", ts=datetime(2024, 1, 1, tzinfo=timezone.utc), tags=tags, host=host, user=user) + + +def test_archive_group_key(): + archive_info = _grouped_archive(name="home", host="myhost", user="myuser", tags=("b", "a")) + assert archive_group_key(archive_info, ()) == () + assert archive_group_key(archive_info, ("name",)) == ("home",) + assert archive_group_key(archive_info, ("name", "host", "user")) == ("home", "myhost", "myuser") + assert archive_group_key(archive_info, ("host", "name")) == ("myhost", "home") # order matters + assert archive_group_key(archive_info, ("tags",)) == ("b,a",) + + +def test_archive_group_key_ignores_internal_tags(): + """Internal tags say nothing about an archive's contents, they must not create their own group.""" + tagged = _grouped_archive(tags=("@PROT", "important")) + untagged = _grouped_archive(tags=("important",)) + assert archive_group_key(tagged, ("tags",)) == archive_group_key(untagged, ("tags",)) == ("important",) + + +def test_archive_group_key_missing_metadata(): + """Archives without host / user metadata (e.g. transferred from borg 1.x) form their own group.""" + assert archive_group_key(_grouped_archive(host=None, user=None), ("host", "user")) == ("", "") + assert archive_group_key(_grouped_archive(host="", user=""), ("host", "user")) == ("", "") + + +def test_group_archives(): + home1, home2 = _grouped_archive(host="host1"), _grouped_archive(host="host2") + etc1 = _grouped_archive(name="etc", host="host1") + archives = [home1, etc1, home2] + + assert group_archives(archives, ()) == {(): archives} # no grouping: one group with all archives + assert group_archives(archives, ("name",)) == {("home",): [home1, home2], ("etc",): [etc1]} + assert group_archives(archives, ("name", "host")) == { + ("home", "host1"): [home1], + ("etc", "host1"): [etc1], + ("home", "host2"): [home2], + } + + +def test_format_group_key(): + assert format_group_key(("home", "host1"), ("name", "host")) == "name='home', host='host1'" + + +def test_group_by_spec(): + assert GroupBySpec("name") == "name" + assert GroupBySpec("name,host,user,tags") == "name,host,user,tags" + assert GroupBySpec("") == GroupBySpec("none") == "" + # the parsed value is fed through the spec again by the argument parser, so it must be stable: + assert GroupBySpec(GroupBySpec("name")) == GroupBySpec("name") + assert GroupBySpec(GroupBySpec("none")) == GroupBySpec("none") + with pytest.raises(ArgumentTypeError, match="Invalid group-by key: bogus"): + GroupBySpec("name,bogus") + with pytest.raises(ArgumentTypeError, match="Duplicate group-by key: name"): + GroupBySpec("name,name") + + +def _create_series(archiver, backup_files, name, hour): + """Create a two archive series called *name*, one archive on 2024-01-01, one on 2024-01-02.""" + _create_archive_ts(archiver, backup_files, name, 2024, 1, 1, H=hour) + _create_archive_ts(archiver, backup_files, name, 2024, 1, 2, H=hour) + + +def _create_shared_repo_series(archiver, backup_files, monkeypatch): + """host1 has a "home" and an "etc" series, host2 has an own, unrelated "home" series.""" + monkeypatch.setenv("BORG_HOSTNAME", "host1") + _create_series(archiver, backup_files, "home", hour=10) + _create_series(archiver, backup_files, "etc", hour=11) + monkeypatch.setenv("BORG_HOSTNAME", "host2") + _create_series(archiver, backup_files, "home", hour=12) + monkeypatch.delenv("BORG_HOSTNAME") + + +def test_prune_groups_by_name_and_host_by_default(archivers, request, backup_files, monkeypatch): + """Unrelated backup sets must not compete for the same retention slots.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_shared_repo_series(archiver, backup_files, monkeypatch) + # 3 groups: (home, host1), (etc, host1), (home, host2) - each keeps its own daily archive. + output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1") + assert output.count("Would prune:") == 3 + + +def test_prune_group_by_name_only(archivers, request, backup_files, monkeypatch): + """--group-by name pools the same series name of different hosts into one retention pool.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_shared_repo_series(archiver, backup_files, monkeypatch) + # 2 groups: "home" (of both hosts) and "etc". + output = cmd(archiver, "prune", "--list", "--dry-run", "--group-by", "name", "--keep-daily=1") + assert output.count("Would prune:") == 4 + + +def test_prune_group_by_none(archivers, request, backup_files): + """--group-by "" applies the rules to all selected archives at once.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_series(archiver, backup_files, "home", hour=10) + _create_series(archiver, backup_files, "etc", hour=11) + cmd(archiver, "prune", "--group-by", "", "--keep-daily=1") + output = cmd(archiver, "repo-list", "--format={name}{NL}") + assert output.split() == ["etc"] # only the single newest archive of all 4 survives + + +def test_prune_group_by_tags(archivers, request, backup_files): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_series(archiver, backup_files, "home", hour=10) + _create_series(archiver, backup_files, "etc", hour=11) + cmd(archiver, "tag", "--add=alice", "-a", "name:home") + cmd(archiver, "tag", "--add=bob", "-a", "name:etc") + output = cmd(archiver, "prune", "--list", "--dry-run", "--group-by", "tags", "--keep-daily=1") + assert output.count("Would prune:") == 2 # one archive pruned per tag group + + +def test_prune_group_by_logs_a_summary_per_group(archivers, request, backup_files, monkeypatch): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_shared_repo_series(archiver, backup_files, monkeypatch) + output = cmd(archiver, "prune", "--info", "--dry-run", "--keep-daily=1") + assert "Group (name='home', host='host1'): 2 archives, keeping 1, pruning 1." in output + assert "Group (name='etc', host='host1'): 2 archives, keeping 1, pruning 1." in output + assert "Group (name='home', host='host2'): 2 archives, keeping 1, pruning 1." in output + + +def test_prune_group_by_json(archivers, request, backup_files, monkeypatch): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + for host, hour in (("host1", 10), ("host2", 11)): + monkeypatch.setenv("BORG_HOSTNAME", host) + _create_series(archiver, backup_files, "home", hour=hour) + monkeypatch.delenv("BORG_HOSTNAME") + prune_result = json.loads( + cmd(archiver, "prune", "--json", "--dry-run", "--group-by", "name,host", "--keep-daily=1") + ) + groups = {tuple(sorted(archive["group"].items())) for archive in prune_result["archives"]} + assert groups == {(("host", "host1"), ("name", "home")), (("host", "host2"), ("name", "home"))} + + +def test_prune_group_by_invalid_key(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + output = cmd(archiver, "prune", "--group-by", "bogus", "--keep-daily=1", exit_code=2) + assert "Invalid group-by key: bogus" in output