diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 58503c7b89..e1af977078 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -414,6 +414,13 @@ def define_archive_filters_group( action="append", help='only consider archives matching all patterns. See "borg help match-archives".', ) + filters_group.add_argument( + "--exclude-archives", + metavar="PATTERN", + dest="exclude_archives", + action="append", + help='do not consider archives matching any pattern. See "borg help match-archives".', + ) if sort_by: sort_by_default = "timestamp" diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py index 0b4c766d08..c05b01fbd1 100644 --- a/src/borg/archiver/help_cmd.py +++ b/src/borg/archiver/help_cmd.py @@ -318,6 +318,12 @@ class HelpMixIn: selection, e.g. ``borg prune home -a host:myhost``. NAME accepts the same prefixes as the patterns below, e.g. ``borg info aid:1234abcd``. + The ``--exclude-archives`` option is the inverse: it accepts the same patterns, can also be + given multiple times, and an archive is skipped if it matches any of them. An archive is + therefore considered if it matches all ``--match-archives`` patterns and none of the + ``--exclude-archives`` patterns, e.g. + ``borg repo-list -a 'sh:myhost-*' --exclude-archives tags:@PROT``. + The patterns can have a prefix of: - name: pattern match on the archive name (default) diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 7c258d2220..a0723cebc2 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -195,7 +195,9 @@ def do_prune(self, args, repository, manifest): self._validate_prune_args(args) match = archive_match_patterns(args) - archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True) + archives = manifest.archives.list( + match=match, exclude=getattr(args, "exclude_archives", None), sort_by=["ts"], reverse=True + ) archives = [ai for ai in archives if "@PROT" not in ai.tags] # The retention rules are applied to each group of archives separately, so that unrelated diff --git a/src/borg/legacy/archives.py b/src/borg/legacy/archives.py index 5f216b333e..9e84f0d8f3 100644 --- a/src/borg/legacy/archives.py +++ b/src/borg/legacy/archives.py @@ -103,38 +103,49 @@ def _info_tuples(self, *, deleted=False): host=info["hostname"], ) - def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False): - archive_infos = list(self._info_tuples(deleted=deleted)) + def _matching_by_pattern(self, archive_infos, match, match_end): + # return the archives of archive_infos matching the single pattern match + if match.startswith("aid:"): + wanted_id = match.removeprefix("aid:") + archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] + if len(archive_infos) != 1: + raise CommandError("archive ID based match needs to match precisely one archive ID") + elif match.startswith("tags:"): + wanted_tags = match.removeprefix("tags:") + wanted_tags = [tag for tag in wanted_tags.split(",") if tag] + archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] + elif match.startswith("user:"): + wanted_user = match.removeprefix("user:") + archive_infos = [x for x in archive_infos if x.user == wanted_user] + elif match.startswith("host:"): + wanted_host = match.removeprefix("host:") + archive_infos = [x for x in archive_infos if x.host == wanted_host] + elif match.startswith("date:"): + wanted_date = match.removeprefix("date:") + try: + date_matches = compile_date_pattern(wanted_date) + except DatePatternError as exc: + raise CommandError(f"Invalid date pattern: {match} ({exc})") + archive_infos = [x for x in archive_infos if date_matches(x.ts)] + else: + match = match.removeprefix("name:") + regex = get_regex_from_pattern(match) + regex = re.compile(regex + match_end) + archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + return archive_infos + + def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False, exclude_patterns=None): + all_infos = list(self._info_tuples(deleted=deleted)) + archive_infos = all_infos if match_patterns: assert isinstance(match_patterns, list), f"match_pattern is a {type(match_patterns)}" - for match in match_patterns: - if match.startswith("aid:"): - wanted_id = match.removeprefix("aid:") - archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] - if len(archive_infos) != 1: - raise CommandError("archive ID based match needs to match precisely one archive ID") - elif match.startswith("tags:"): - wanted_tags = match.removeprefix("tags:") - wanted_tags = [tag for tag in wanted_tags.split(",") if tag] - archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] - elif match.startswith("user:"): - wanted_user = match.removeprefix("user:") - archive_infos = [x for x in archive_infos if x.user == wanted_user] - elif match.startswith("host:"): - wanted_host = match.removeprefix("host:") - archive_infos = [x for x in archive_infos if x.host == wanted_host] - elif match.startswith("date:"): - wanted_date = match.removeprefix("date:") - try: - date_matches = compile_date_pattern(wanted_date) - except DatePatternError as exc: - raise CommandError(f"Invalid date pattern: {match} ({exc})") - archive_infos = [x for x in archive_infos if date_matches(x.ts)] - else: - match = match.removeprefix("name:") - regex = get_regex_from_pattern(match) - regex = re.compile(regex + match_end) - archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + for match in match_patterns: # all patterns must match, so they are ANDed + archive_infos = self._matching_by_pattern(archive_infos, match, match_end) + if exclude_patterns: + assert isinstance(exclude_patterns, list), f"exclude_pattern is a {type(exclude_patterns)}" + for match in exclude_patterns: # any pattern excludes, so they are ORed + excluded_ids = {x.id for x in self._matching_by_pattern(all_infos, match, match_end)} + archive_infos = [x for x in archive_infos if x.id not in excluded_ids] return archive_infos def count(self): @@ -212,6 +223,7 @@ def list( *, match=None, match_end=r"\Z", + exclude=None, sort_by=(), reverse=False, first=None, @@ -229,7 +241,7 @@ def list( if isinstance(sort_by, (str, bytes)): raise TypeError("sort_by must be a sequence of str") - archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted) + archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted, exclude_patterns=exclude) if any([oldest, newest, older, newer]): archive_infos = filter_archives_by_date( @@ -255,6 +267,7 @@ def list_considering(self, args): return self.list( sort_by=args.sort_by.split(","), match=args.match_archives, + exclude=getattr(args, "exclude_archives", None), first=getattr(args, "first", None), last=getattr(args, "last", None), older=getattr(args, "older", None), diff --git a/src/borg/manifest.py b/src/borg/manifest.py index f3bb1246f0..8f01aef864 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -147,6 +147,7 @@ def list( *, match=None, match_end=r"\Z", + exclude=None, sort_by=(), reverse=False, first=None, @@ -254,38 +255,51 @@ def _info_tuples(self, *, deleted=False): host=info["hostname"], ) - def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False): - archive_infos = list(self._info_tuples(deleted=deleted)) + def _matching_by_pattern(self, archive_infos, match, match_end): + # return the archives of archive_infos matching the single pattern match + if match.startswith("aid:"): # do a match on the archive ID (prefix) + wanted_id = match.removeprefix("aid:") + archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] + if len(archive_infos) != 1: + raise CommandError("archive ID based match needs to match precisely one archive ID") + elif match.startswith("tags:"): + wanted_tags = match.removeprefix("tags:") + wanted_tags = [tag for tag in wanted_tags.split(",") if tag] # remove empty tags + archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] + elif match.startswith("user:"): + wanted_user = match.removeprefix("user:") + archive_infos = [x for x in archive_infos if x.user == wanted_user] + elif match.startswith("host:"): + wanted_host = match.removeprefix("host:") + archive_infos = [x for x in archive_infos if x.host == wanted_host] + elif match.startswith("date:"): + wanted_date = match.removeprefix("date:") + try: + date_matches = compile_date_pattern(wanted_date) + except DatePatternError as exc: + raise CommandError(f"Invalid date pattern: {match} ({exc})") + archive_infos = [x for x in archive_infos if date_matches(x.ts)] + else: # do a match on the name + match = match.removeprefix("name:") # accept optional name: prefix + regex = get_regex_from_pattern(match) + regex = re.compile(regex + match_end) + archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + return archive_infos + + def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False, exclude_patterns=None): + all_infos = list(self._info_tuples(deleted=deleted)) + archive_infos = all_infos if match_patterns: assert isinstance(match_patterns, list), f"match_pattern is a {type(match_patterns)}" - for match in match_patterns: - if match.startswith("aid:"): # do a match on the archive ID (prefix) - wanted_id = match.removeprefix("aid:") - archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] - if len(archive_infos) != 1: - raise CommandError("archive ID based match needs to match precisely one archive ID") - elif match.startswith("tags:"): - wanted_tags = match.removeprefix("tags:") - wanted_tags = [tag for tag in wanted_tags.split(",") if tag] # remove empty tags - archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] - elif match.startswith("user:"): - wanted_user = match.removeprefix("user:") - archive_infos = [x for x in archive_infos if x.user == wanted_user] - elif match.startswith("host:"): - wanted_host = match.removeprefix("host:") - archive_infos = [x for x in archive_infos if x.host == wanted_host] - elif match.startswith("date:"): - wanted_date = match.removeprefix("date:") - try: - date_matches = compile_date_pattern(wanted_date) - except DatePatternError as exc: - raise CommandError(f"Invalid date pattern: {match} ({exc})") - archive_infos = [x for x in archive_infos if date_matches(x.ts)] - else: # do a match on the name - match = match.removeprefix("name:") # accept optional name: prefix - regex = get_regex_from_pattern(match) - regex = re.compile(regex + match_end) - archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + for match in match_patterns: # all patterns must match, so they are ANDed + archive_infos = self._matching_by_pattern(archive_infos, match, match_end) + if exclude_patterns: + assert isinstance(exclude_patterns, list), f"exclude_pattern is a {type(exclude_patterns)}" + for match in exclude_patterns: # any pattern excludes, so they are ORed + # match against all archives, not only the remaining ones, so that an exclusion + # pattern does not depend on what the inclusion patterns happened to keep. + excluded_ids = {x.id for x in self._matching_by_pattern(all_infos, match, match_end)} + archive_infos = [x for x in archive_infos if x.id not in excluded_ids] return archive_infos def count(self): @@ -404,6 +418,7 @@ def list( *, match=None, match_end=r"\Z", + exclude=None, sort_by=(), reverse=False, first=None, @@ -434,7 +449,7 @@ def list( if isinstance(sort_by, (str, bytes)): raise TypeError("sort_by must be a sequence of str") - archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted) + archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted, exclude_patterns=exclude) if any([oldest, newest, older, newer]): archive_infos = filter_archives_by_date( @@ -463,6 +478,7 @@ def list_considering(self, args, *, reverse=False): sort_by=args.sort_by.split(","), reverse=reverse, match=args.match_archives, + exclude=getattr(args, "exclude_archives", None), first=getattr(args, "first", None), last=getattr(args, "last", None), older=getattr(args, "older", None), diff --git a/src/borg/testsuite/archiver/repo_list_cmd_test.py b/src/borg/testsuite/archiver/repo_list_cmd_test.py index 7022306208..677096b50c 100644 --- a/src/borg/testsuite/archiver/repo_list_cmd_test.py +++ b/src/borg/testsuite/archiver/repo_list_cmd_test.py @@ -23,6 +23,55 @@ def test_repo_list_glob(archivers, request, backup_files): assert "something-else" not in output +def test_repo_list_exclude_archives(archivers, request, backup_files): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + cmd(archiver, "create", "other-1", backup_files) + output = cmd(archiver, "repo-list", "--exclude-archives=sh:test-*") + assert "test-1" not in output + assert "test-2" not in output + assert "other-1" in output + + +def test_repo_list_exclude_archives_multiple(archivers, request, backup_files): + # several exclusion patterns are ORed: an archive matching any of them is skipped + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + cmd(archiver, "create", "test-3", backup_files) + output = cmd(archiver, "repo-list", "--exclude-archives=test-1", "--exclude-archives=test-3") + assert "test-1" not in output + assert "test-2" in output + assert "test-3" not in output + + +def test_repo_list_match_and_exclude_archives(archivers, request, backup_files): + # an archive is considered if it matches all --match-archives and none of the --exclude-archives + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + cmd(archiver, "create", "other-1", backup_files) + output = cmd(archiver, "repo-list", "--match-archives=sh:test-*", "--exclude-archives=test-2") + assert "test-1" in output + assert "test-2" not in output + assert "other-1" not in output + + +def test_repo_list_exclude_archives_by_tag(archivers, request, backup_files): + # exclusion accepts the same selector prefixes as --match-archives, not just names + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "--tags", "keepme", "--", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + output = cmd(archiver, "repo-list", "--exclude-archives=tags:keepme") + assert "test-1" not in output + assert "test-2" in output + + def test_archives_format(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) diff --git a/src/borg/testsuite/archives_test.py b/src/borg/testsuite/archives_test.py index 59c4d33896..27f3eb8fe8 100644 --- a/src/borg/testsuite/archives_test.py +++ b/src/borg/testsuite/archives_test.py @@ -53,7 +53,9 @@ def _archiveinfo(name, id_, ts=TS, *, username="", hostname="", tags=()): def _stub_matching_info_tuples(infos): ar, _, _ = _archives() - ar._matching_info_tuples = Mock(side_effect=lambda match_patterns, match_end, deleted=False: list(infos)) + ar._matching_info_tuples = Mock( + side_effect=lambda match_patterns, match_end, *, deleted=False, exclude_patterns=None: list(infos) + ) return ar