From 2c3b6e15dd4f3da737eb05f4247ebcc637f2c7ea Mon Sep 17 00:00:00 2001 From: m-altaifi Date: Tue, 1 Sep 2026 21:38:03 +0000 Subject: [PATCH 1/3] Add --exclude-archives, the inverse of --match-archives --match-archives can only narrow a selection: every pattern given must match, so there is no way to say "all archives except these". Doing that needed a shell loop around borg repo-list. Add --exclude-archives, taking the same patterns and selector prefixes. An archive is considered if it matches all --match-archives patterns and none of the --exclude-archives ones, so the new option ORs while the existing one ANDs. The per-pattern matching is factored out of _matching_info_tuples into _matching_by_pattern so both directions share one matcher and all prefixes (aid:, tags:, user:, host:, date:, name:) work for exclusion too. Exclusion patterns are evaluated against all archives rather than the already narrowed list, so an exclusion does not depend on what the inclusion patterns happened to keep. Wired at the shared archive filters group, so every command using define_archive_filters_group gets it. LegacyArchives is updated the same way so borg transfer does not silently ignore the option when reading a borg 1.x repository. --- src/borg/archiver/_common.py | 7 ++ src/borg/archiver/help_cmd.py | 6 ++ src/borg/archiver/prune_cmd.py | 4 +- src/borg/legacy/archives.py | 77 ++++++++++-------- src/borg/manifest.py | 78 +++++++++++-------- .../testsuite/archiver/repo_list_cmd_test.py | 49 ++++++++++++ 6 files changed, 157 insertions(+), 64 deletions(-) diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 58503c7b89..e1af977078 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -414,6 +414,13 @@ def define_archive_filters_group( action="append", help='only consider archives matching all patterns. See "borg help match-archives".', ) + filters_group.add_argument( + "--exclude-archives", + metavar="PATTERN", + dest="exclude_archives", + action="append", + help='do not consider archives matching any pattern. See "borg help match-archives".', + ) if sort_by: sort_by_default = "timestamp" diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py index 0b4c766d08..c05b01fbd1 100644 --- a/src/borg/archiver/help_cmd.py +++ b/src/borg/archiver/help_cmd.py @@ -318,6 +318,12 @@ class HelpMixIn: selection, e.g. ``borg prune home -a host:myhost``. NAME accepts the same prefixes as the patterns below, e.g. ``borg info aid:1234abcd``. + The ``--exclude-archives`` option is the inverse: it accepts the same patterns, can also be + given multiple times, and an archive is skipped if it matches any of them. An archive is + therefore considered if it matches all ``--match-archives`` patterns and none of the + ``--exclude-archives`` patterns, e.g. + ``borg repo-list -a 'sh:myhost-*' --exclude-archives tags:@PROT``. + The patterns can have a prefix of: - name: pattern match on the archive name (default) diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 7c258d2220..a0723cebc2 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -195,7 +195,9 @@ def do_prune(self, args, repository, manifest): self._validate_prune_args(args) match = archive_match_patterns(args) - archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True) + archives = manifest.archives.list( + match=match, exclude=getattr(args, "exclude_archives", None), sort_by=["ts"], reverse=True + ) archives = [ai for ai in archives if "@PROT" not in ai.tags] # The retention rules are applied to each group of archives separately, so that unrelated diff --git a/src/borg/legacy/archives.py b/src/borg/legacy/archives.py index 5f216b333e..36b502e5de 100644 --- a/src/borg/legacy/archives.py +++ b/src/borg/legacy/archives.py @@ -103,38 +103,49 @@ def _info_tuples(self, *, deleted=False): host=info["hostname"], ) - def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False): - archive_infos = list(self._info_tuples(deleted=deleted)) + def _matching_by_pattern(self, archive_infos, match, match_end): + # return the archives of archive_infos matching the single pattern match + if match.startswith("aid:"): + wanted_id = match.removeprefix("aid:") + archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] + if len(archive_infos) != 1: + raise CommandError("archive ID based match needs to match precisely one archive ID") + elif match.startswith("tags:"): + wanted_tags = match.removeprefix("tags:") + wanted_tags = [tag for tag in wanted_tags.split(",") if tag] + archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] + elif match.startswith("user:"): + wanted_user = match.removeprefix("user:") + archive_infos = [x for x in archive_infos if x.user == wanted_user] + elif match.startswith("host:"): + wanted_host = match.removeprefix("host:") + archive_infos = [x for x in archive_infos if x.host == wanted_host] + elif match.startswith("date:"): + wanted_date = match.removeprefix("date:") + try: + date_matches = compile_date_pattern(wanted_date) + except DatePatternError as exc: + raise CommandError(f"Invalid date pattern: {match} ({exc})") + archive_infos = [x for x in archive_infos if date_matches(x.ts)] + else: + match = match.removeprefix("name:") + regex = get_regex_from_pattern(match) + regex = re.compile(regex + match_end) + archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + return archive_infos + + def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False, exclude_patterns=None): + all_infos = list(self._info_tuples(deleted=deleted)) + archive_infos = all_infos if match_patterns: assert isinstance(match_patterns, list), f"match_pattern is a {type(match_patterns)}" - for match in match_patterns: - if match.startswith("aid:"): - wanted_id = match.removeprefix("aid:") - archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] - if len(archive_infos) != 1: - raise CommandError("archive ID based match needs to match precisely one archive ID") - elif match.startswith("tags:"): - wanted_tags = match.removeprefix("tags:") - wanted_tags = [tag for tag in wanted_tags.split(",") if tag] - archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] - elif match.startswith("user:"): - wanted_user = match.removeprefix("user:") - archive_infos = [x for x in archive_infos if x.user == wanted_user] - elif match.startswith("host:"): - wanted_host = match.removeprefix("host:") - archive_infos = [x for x in archive_infos if x.host == wanted_host] - elif match.startswith("date:"): - wanted_date = match.removeprefix("date:") - try: - date_matches = compile_date_pattern(wanted_date) - except DatePatternError as exc: - raise CommandError(f"Invalid date pattern: {match} ({exc})") - archive_infos = [x for x in archive_infos if date_matches(x.ts)] - else: - match = match.removeprefix("name:") - regex = get_regex_from_pattern(match) - regex = re.compile(regex + match_end) - archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + for match in match_patterns: # all patterns must match, so they are ANDed + archive_infos = self._matching_by_pattern(archive_infos, match, match_end) + if exclude_patterns: + assert isinstance(exclude_patterns, list), f"exclude_pattern is a {type(exclude_patterns)}" + for match in exclude_patterns: # any pattern excludes, so they are ORed + excluded_ids = {x.id for x in self._matching_by_pattern(all_infos, match, match_end)} + archive_infos = [x for x in archive_infos if x.id not in excluded_ids] return archive_infos def count(self): @@ -212,6 +223,7 @@ def list( *, match=None, match_end=r"\Z", + exclude=None, sort_by=(), reverse=False, first=None, @@ -229,7 +241,7 @@ def list( if isinstance(sort_by, (str, bytes)): raise TypeError("sort_by must be a sequence of str") - archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted) + archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted, exclude_patterns=exclude) if any([oldest, newest, older, newer]): archive_infos = filter_archives_by_date( @@ -255,6 +267,7 @@ def list_considering(self, args): return self.list( sort_by=args.sort_by.split(","), match=args.match_archives, + exclude=getattr(args, "exclude_archives", None), first=getattr(args, "first", None), last=getattr(args, "last", None), older=getattr(args, "older", None), @@ -267,7 +280,7 @@ def list_considering(self, args): def get_one(self, match, *, match_end=r"\Z", deleted=False): """Get exactly one archive matching .""" assert match is not None - archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted) + archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted, exclude_patterns=exclude) if len(archive_infos) != 1: raise CommandError(f"{match} needed to match precisely one archive, but matched {len(archive_infos)}.") return archive_infos[0] diff --git a/src/borg/manifest.py b/src/borg/manifest.py index f3bb1246f0..8f01aef864 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -147,6 +147,7 @@ def list( *, match=None, match_end=r"\Z", + exclude=None, sort_by=(), reverse=False, first=None, @@ -254,38 +255,51 @@ def _info_tuples(self, *, deleted=False): host=info["hostname"], ) - def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False): - archive_infos = list(self._info_tuples(deleted=deleted)) + def _matching_by_pattern(self, archive_infos, match, match_end): + # return the archives of archive_infos matching the single pattern match + if match.startswith("aid:"): # do a match on the archive ID (prefix) + wanted_id = match.removeprefix("aid:") + archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] + if len(archive_infos) != 1: + raise CommandError("archive ID based match needs to match precisely one archive ID") + elif match.startswith("tags:"): + wanted_tags = match.removeprefix("tags:") + wanted_tags = [tag for tag in wanted_tags.split(",") if tag] # remove empty tags + archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] + elif match.startswith("user:"): + wanted_user = match.removeprefix("user:") + archive_infos = [x for x in archive_infos if x.user == wanted_user] + elif match.startswith("host:"): + wanted_host = match.removeprefix("host:") + archive_infos = [x for x in archive_infos if x.host == wanted_host] + elif match.startswith("date:"): + wanted_date = match.removeprefix("date:") + try: + date_matches = compile_date_pattern(wanted_date) + except DatePatternError as exc: + raise CommandError(f"Invalid date pattern: {match} ({exc})") + archive_infos = [x for x in archive_infos if date_matches(x.ts)] + else: # do a match on the name + match = match.removeprefix("name:") # accept optional name: prefix + regex = get_regex_from_pattern(match) + regex = re.compile(regex + match_end) + archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + return archive_infos + + def _matching_info_tuples(self, match_patterns, match_end, *, deleted=False, exclude_patterns=None): + all_infos = list(self._info_tuples(deleted=deleted)) + archive_infos = all_infos if match_patterns: assert isinstance(match_patterns, list), f"match_pattern is a {type(match_patterns)}" - for match in match_patterns: - if match.startswith("aid:"): # do a match on the archive ID (prefix) - wanted_id = match.removeprefix("aid:") - archive_infos = [x for x in archive_infos if bin_to_hex(x.id).startswith(wanted_id)] - if len(archive_infos) != 1: - raise CommandError("archive ID based match needs to match precisely one archive ID") - elif match.startswith("tags:"): - wanted_tags = match.removeprefix("tags:") - wanted_tags = [tag for tag in wanted_tags.split(",") if tag] # remove empty tags - archive_infos = [x for x in archive_infos if set(x.tags) >= set(wanted_tags)] - elif match.startswith("user:"): - wanted_user = match.removeprefix("user:") - archive_infos = [x for x in archive_infos if x.user == wanted_user] - elif match.startswith("host:"): - wanted_host = match.removeprefix("host:") - archive_infos = [x for x in archive_infos if x.host == wanted_host] - elif match.startswith("date:"): - wanted_date = match.removeprefix("date:") - try: - date_matches = compile_date_pattern(wanted_date) - except DatePatternError as exc: - raise CommandError(f"Invalid date pattern: {match} ({exc})") - archive_infos = [x for x in archive_infos if date_matches(x.ts)] - else: # do a match on the name - match = match.removeprefix("name:") # accept optional name: prefix - regex = get_regex_from_pattern(match) - regex = re.compile(regex + match_end) - archive_infos = [x for x in archive_infos if regex.match(x.name) is not None] + for match in match_patterns: # all patterns must match, so they are ANDed + archive_infos = self._matching_by_pattern(archive_infos, match, match_end) + if exclude_patterns: + assert isinstance(exclude_patterns, list), f"exclude_pattern is a {type(exclude_patterns)}" + for match in exclude_patterns: # any pattern excludes, so they are ORed + # match against all archives, not only the remaining ones, so that an exclusion + # pattern does not depend on what the inclusion patterns happened to keep. + excluded_ids = {x.id for x in self._matching_by_pattern(all_infos, match, match_end)} + archive_infos = [x for x in archive_infos if x.id not in excluded_ids] return archive_infos def count(self): @@ -404,6 +418,7 @@ def list( *, match=None, match_end=r"\Z", + exclude=None, sort_by=(), reverse=False, first=None, @@ -434,7 +449,7 @@ def list( if isinstance(sort_by, (str, bytes)): raise TypeError("sort_by must be a sequence of str") - archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted) + archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted, exclude_patterns=exclude) if any([oldest, newest, older, newer]): archive_infos = filter_archives_by_date( @@ -463,6 +478,7 @@ def list_considering(self, args, *, reverse=False): sort_by=args.sort_by.split(","), reverse=reverse, match=args.match_archives, + exclude=getattr(args, "exclude_archives", None), first=getattr(args, "first", None), last=getattr(args, "last", None), older=getattr(args, "older", None), diff --git a/src/borg/testsuite/archiver/repo_list_cmd_test.py b/src/borg/testsuite/archiver/repo_list_cmd_test.py index 7022306208..677096b50c 100644 --- a/src/borg/testsuite/archiver/repo_list_cmd_test.py +++ b/src/borg/testsuite/archiver/repo_list_cmd_test.py @@ -23,6 +23,55 @@ def test_repo_list_glob(archivers, request, backup_files): assert "something-else" not in output +def test_repo_list_exclude_archives(archivers, request, backup_files): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + cmd(archiver, "create", "other-1", backup_files) + output = cmd(archiver, "repo-list", "--exclude-archives=sh:test-*") + assert "test-1" not in output + assert "test-2" not in output + assert "other-1" in output + + +def test_repo_list_exclude_archives_multiple(archivers, request, backup_files): + # several exclusion patterns are ORed: an archive matching any of them is skipped + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + cmd(archiver, "create", "test-3", backup_files) + output = cmd(archiver, "repo-list", "--exclude-archives=test-1", "--exclude-archives=test-3") + assert "test-1" not in output + assert "test-2" in output + assert "test-3" not in output + + +def test_repo_list_match_and_exclude_archives(archivers, request, backup_files): + # an archive is considered if it matches all --match-archives and none of the --exclude-archives + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + cmd(archiver, "create", "other-1", backup_files) + output = cmd(archiver, "repo-list", "--match-archives=sh:test-*", "--exclude-archives=test-2") + assert "test-1" in output + assert "test-2" not in output + assert "other-1" not in output + + +def test_repo_list_exclude_archives_by_tag(archivers, request, backup_files): + # exclusion accepts the same selector prefixes as --match-archives, not just names + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "--tags", "keepme", "--", "test-1", backup_files) + cmd(archiver, "create", "test-2", backup_files) + output = cmd(archiver, "repo-list", "--exclude-archives=tags:keepme") + assert "test-1" not in output + assert "test-2" in output + + def test_archives_format(archivers, request, backup_files): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) From 6117a54c390ca82f35c2050ffcb8546d1ca73d11 Mon Sep 17 00:00:00 2001 From: M Date: Thu, 3 Sep 2026 19:49:11 +0000 Subject: [PATCH 2/3] Fix legacy archive lookup without exclusions --- src/borg/legacy/archives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/legacy/archives.py b/src/borg/legacy/archives.py index 36b502e5de..9e84f0d8f3 100644 --- a/src/borg/legacy/archives.py +++ b/src/borg/legacy/archives.py @@ -280,7 +280,7 @@ def list_considering(self, args): def get_one(self, match, *, match_end=r"\Z", deleted=False): """Get exactly one archive matching .""" assert match is not None - archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted, exclude_patterns=exclude) + archive_infos = self._matching_info_tuples(match, match_end, deleted=deleted) if len(archive_infos) != 1: raise CommandError(f"{match} needed to match precisely one archive, but matched {len(archive_infos)}.") return archive_infos[0] From c0293cad394655084fde1c92f2dc941a78dcfd06 Mon Sep 17 00:00:00 2001 From: M Date: Sat, 5 Sep 2026 11:47:36 +0000 Subject: [PATCH 3/3] tests: accept archive exclusion matcher argument --- src/borg/testsuite/archives_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/borg/testsuite/archives_test.py b/src/borg/testsuite/archives_test.py index 59c4d33896..27f3eb8fe8 100644 --- a/src/borg/testsuite/archives_test.py +++ b/src/borg/testsuite/archives_test.py @@ -53,7 +53,9 @@ def _archiveinfo(name, id_, ts=TS, *, username="", hostname="", tags=()): def _stub_matching_info_tuples(infos): ar, _, _ = _archives() - ar._matching_info_tuples = Mock(side_effect=lambda match_patterns, match_end, deleted=False: list(infos)) + ar._matching_info_tuples = Mock( + side_effect=lambda match_patterns, match_end, *, deleted=False, exclude_patterns=None: list(infos) + ) return ar