From 8180036fbd6597f836b9a559b0dfb3d5b0851205 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 3 Sep 2026 22:20:29 +0200 Subject: [PATCH 1/2] diff: add --stats, showing a summary of the differences, fixes #796 borg diff so far only reported per-item changes. Add -s/--stats, printing an aggregated summary after the per-path output: Added items: 23 Removed items: 2 Changed items: 315 Added chunk volume: 53.70 MB Removed chunk volume: 51.10 MB Added/removed items only exist in one of the archives, changed items exist in both but differ. The chunk volumes sum up the size of the content chunks added/removed by all of these items. If the archives were created with different chunker params, borg compares the content byte by byte and cannot tell by how much it changed. Such items contribute no byte counts, so an additional "Items with unknown size changes" line reports how many there are. The text summary goes to the borg.output.stats logger (stderr), like the --stats output of other commands, so stdout stays clean for piping. With --json-lines, the summary is emitted as a final {"stats": {...}} line instead, which is easy to tell apart from the per-path lines. The change filtering that the text and JSON output paths did separately is now done once in reported_changes(), feeding both the output and the stats, so the summary counts exactly what was printed. Co-Authored-By: Claude Opus 5 --- src/borg/archiver/diff_cmd.py | 125 ++++++++++++++++--- src/borg/testsuite/archiver/diff_cmd_test.py | 89 +++++++++++++ 2 files changed, 196 insertions(+), 18 deletions(-) diff --git a/src/borg/archiver/diff_cmd.py b/src/borg/archiver/diff_cmd.py index 0fe4302540..a1fa032bf3 100644 --- a/src/borg/archiver/diff_cmd.py +++ b/src/borg/archiver/diff_cmd.py @@ -1,5 +1,6 @@ import textwrap import json +import logging import sys import os @@ -8,6 +9,7 @@ from ..constants import * # NOQA from ..helpers import BaseFormatter, DiffFormatter, archivename_validator, PathSpec, BorgJsonEncoder from ..helpers import IncludePatternNeverMatchedWarning, remove_surrogates +from ..helpers import format_file_size, log_multi from ..helpers.argparsing import ArgumentParser from ..helpers.sorting import sort_spec_validator, sorted_by_spec from ..item import ItemDiff @@ -35,6 +37,59 @@ diff_sort_spec = sort_spec_validator(DIFF_SORT_KEYS, name="diff_sort_spec") +class DiffStats: + """Aggregated statistics over all reported item diffs, for ``borg diff --stats``.""" + + def __init__(self): + self.added_items = 0 # items present in ARCHIVE2 only + self.removed_items = 0 # items present in ARCHIVE1 only + self.changed_items = 0 # items present in both archives, but not equal + self.added_chunk_volume = 0 # size of the content chunks added (by added and by changed items) + self.removed_chunk_volume = 0 # size of the content chunks removed (by removed and by changed items) + self.unknown_size_items = 0 # items whose content changed by an unknown amount + + def add(self, diff: ItemDiff, changes: dict) -> None: + """Account for one item diff, using the already filtered/reported changes.""" + content = changes.get("content") + if content is not None: + info = content.to_dict() + if "added" in info or "removed" in info: + self.added_chunk_volume += info.get("added", 0) + self.removed_chunk_volume += info.get("removed", 0) + else: + # a "modified" that was determined by comparing the content: no byte counts. + self.unknown_size_items += 1 + if diff._item1.get("deleted"): + self.added_items += 1 + elif diff._item2.get("deleted"): + self.removed_items += 1 + else: + self.changed_items += 1 + + def as_dict(self) -> dict: + return { + "added_items": self.added_items, + "removed_items": self.removed_items, + "changed_items": self.changed_items, + "added_chunk_volume": self.added_chunk_volume, + "removed_chunk_volume": self.removed_chunk_volume, + "unknown_size_items": self.unknown_size_items, + } + + def __str__(self) -> str: + lines = [ + f"Added items: {self.added_items}", + f"Removed items: {self.removed_items}", + f"Changed items: {self.changed_items}", + f"Added chunk volume: {format_file_size(self.added_chunk_volume)}", + f"Removed chunk volume: {format_file_size(self.removed_chunk_volume)}", + ] + if self.unknown_size_items: + # these items are not accounted for in the added/removed data above, so say so. + lines.append(f"Items with unknown size changes: {self.unknown_size_items}") + return "\n".join(lines) + + class DiffMixIn: @with_repository(compatibility=(Manifest.Operation.READ,)) def do_diff(self, args, repository, manifest): @@ -52,29 +107,25 @@ def actual_change(j): # All other change types are indeed changes. return True - def print_json_output(diff): + def reported_changes(diff): + """The changes of diff that are actually shown to the user.""" + return { + name: change + for name, change in diff.changes().items() + if actual_change(change) and (not args.content_only or (name not in DiffFormatter.METADATA)) + } + + def print_json_output(diff, changes): print( json.dumps( - { - "path": diff.path, - "changes": [ - change.to_dict() - for name, change in diff.changes().items() - if actual_change(change) and (not args.content_only or (name not in DiffFormatter.METADATA)) - ], - }, + {"path": diff.path, "changes": [change.to_dict() for change in changes.values()]}, sort_keys=True, cls=BorgJsonEncoder, ) ) - def print_text_output(diff, formatter): - actual_changes = { - name: change - for name, change in diff.changes().items() - if actual_change(change) and (not args.content_only or (name not in DiffFormatter.METADATA)) - } - diff._changes = actual_changes + def print_text_output(diff, changes, formatter): + diff._changes = changes res: str = formatter.format_item(diff) if res.strip(): sys.stdout.write(res) @@ -162,11 +213,23 @@ def key_for(field: str, d: "ItemDiff"): diffs = sorted_by_spec(diffs, args.sort_by, key_for) formatter = DiffFormatter(format, args.content_only) + stats = DiffStats() if args.stats else None for diff in diffs: + changes = reported_changes(diff) + if stats is not None and changes: + # items without any reported change do not show up in the output, so don't count them. + stats.add(diff, changes) + if args.json_lines: + print_json_output(diff, changes) + else: + print_text_output(diff, changes, formatter) + + if stats is not None: if args.json_lines: - print_json_output(diff) + # a final line of a different shape than the per-path lines, so it is easy to tell apart. + print(json.dumps({"stats": stats.as_dict()}, sort_keys=True, cls=BorgJsonEncoder)) else: - print_text_output(diff, formatter) + log_multi(str(stats), logger=logging.getLogger("borg.output.stats")) for pattern in matcher.get_unmatched_include_patterns(): self.print_warning_instance(IncludePatternNeverMatchedWarning(pattern)) @@ -252,6 +315,29 @@ def build_parser_diff(self, subparsers, common_parser, mid_common_parser): {"changes": [{"added": 4, "removed": 0, "type": "added"}], "path": "path/to/added-file"} {"changes": [{"added": 0, "removed": 5, "type": "removed"}], "path": "path/to/removed-file"} + Statistics + ++++++++++ + With ``--stats``, borg prints a summary of the differences after the per-path output:: + + Added items: 23 + Removed items: 2 + Changed items: 315 + Added chunk volume: 53.70 MB + Removed chunk volume: 51.10 MB + + "Added"/"Removed" items only exist in ARCHIVE2/ARCHIVE1, "changed" items exist in both + archives but differ. "Added chunk volume"/"Removed chunk volume" sum up the size of the + content chunks added/removed by all of these items. Items whose content borg could only + compare byte by byte (see "Performance considerations" below) contribute no byte counts; + if there are any, an additional "Items with unknown size changes" line reports how many. + + Together with ``--json-lines``, the summary is emitted as a final JSON line of the shape + ``{"stats": {...}}`` instead, so it is easy to tell apart from the per-path lines + (wrapped here for readability, borg prints it as a single line):: + + {"stats": {"added_chunk_volume": 53700000, "added_items": 23, "changed_items": 315, + "removed_chunk_volume": 51100000, "removed_items": 2, "unknown_size_items": 0}} + Sorting ++++++++ Use ``--sort-by FIELDS`` where FIELDS is a comma-separated list of fields. @@ -297,6 +383,9 @@ def build_parser_diff(self, subparsers, common_parser, mid_common_parser): help='specify format for differences between archives (default: "{change} {path}{NL}")', ) subparser.add_argument("--json-lines", action="store_true", help="Format output as JSON Lines.") + subparser.add_argument( + "-s", "--stats", dest="stats", action="store_true", help="print a summary of the differences at the end" + ) subparser.add_argument( "--sort-by", dest="sort_by", diff --git a/src/borg/testsuite/archiver/diff_cmd_test.py b/src/borg/testsuite/archiver/diff_cmd_test.py index e3b60ee026..6d764b438d 100644 --- a/src/borg/testsuite/archiver/diff_cmd_test.py +++ b/src/borg/testsuite/archiver/diff_cmd_test.py @@ -599,3 +599,92 @@ def test_hard_link_deletion_and_replacement(archivers, request): else: # But the b/hardlink file was not modified at all. assert_line_not_exists(lines, ".*input/b/hardlink") + + +def _setup_stats_archives(archiver): + """Create two archives differing in one added, one removed and one changed file.""" + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file_changed", contents=b"a" * 100) + create_regular_file(archiver.input_path, "file_removed", contents=b"b" * 5) + create_regular_file(archiver.input_path, "file_touched", contents=b"c" * 7) + cmd(archiver, "create", "test0", "input") + create_regular_file(archiver.input_path, "file_changed", contents=b"d" * 120) + os.unlink("input/file_removed") + create_regular_file(archiver.input_path, "file_added", contents=b"e" * 30) + os.chmod("input/file_touched", 0o700) + cmd(archiver, "create", "test1", "input") + + +def test_stats(archivers, request): + archiver = request.getfixturevalue(archivers) + _setup_stats_archives(archiver) + output = cmd(archiver, "diff", "--stats", "--content-only", "test0", "test1") + lines = output.splitlines() + # only the content changes are considered, so file_touched (mode only) is not counted. + assert "Added items: 1" in lines + assert "Removed items: 1" in lines + assert "Changed items: 1" in lines + # added: file_added (30) + the new content of file_changed (120) + assert_line_exists(lines, r"^Added chunk volume: 150 B$") + # removed: file_removed (5) + the old content of file_changed (100) + assert_line_exists(lines, r"^Removed chunk volume: 105 B$") + # all sizes are known, so this line is omitted + assert_line_not_exists(lines, r"^Items with unknown size changes:") + + +def test_stats_counts_metadata_only_changes(archivers, request): + archiver = request.getfixturevalue(archivers) + _setup_stats_archives(archiver) + # without --content-only, file_touched (changed mode) and the input directory + # (changed mtime/ctime) are reported as changed items, too. + output = cmd(archiver, "diff", "--stats", "test0", "test1") + lines = output.splitlines() + assert "Added items: 1" in lines + assert "Removed items: 1" in lines + assert "Changed items: 3" in lines + assert_line_exists(lines, r"^Added chunk volume: 150 B$") + assert_line_exists(lines, r"^Removed chunk volume: 105 B$") + + +def test_stats_json_lines(archivers, request): + archiver = request.getfixturevalue(archivers) + _setup_stats_archives(archiver) + output = cmd(archiver, "diff", "--stats", "--content-only", "--json-lines", "test0", "test1") + lines = [line for line in output.splitlines() if line.startswith("{")] + # the stats are the last line and have a shape of their own + assert all("stats" not in json.loads(line) for line in lines[:-1]) + assert json.loads(lines[-1]) == { + "stats": { + "added_items": 1, + "removed_items": 1, + "changed_items": 1, + "added_chunk_volume": 150, + "removed_chunk_volume": 105, + "unknown_size_items": 0, + } + } + + +def test_stats_unknown_sizes(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file_changed", contents=b"a" * 100) + cmd(archiver, "create", "test0", "input") + create_regular_file(archiver.input_path, "file_changed", contents=b"b" * 120) + # different chunker params: borg has to compare the content and can't tell sizes. + cmd(archiver, "create", "--chunker-params", "buzhash,10,23,16,4095", "test1", "input") + output = cmd(archiver, "diff", "--stats", "--content-only", "test0", "test1") + lines = output.splitlines() + assert "Changed items: 1" in lines + assert_line_exists(lines, r"^Added chunk volume: 0 B$") + assert_line_exists(lines, r"^Removed chunk volume: 0 B$") + assert "Items with unknown size changes: 1" in lines + + +def test_no_stats_by_default(archivers, request): + archiver = request.getfixturevalue(archivers) + _setup_stats_archives(archiver) + output = cmd(archiver, "diff", "--content-only", "test0", "test1") + assert_line_not_exists(output.splitlines(), r"^Added items:") + output = cmd(archiver, "diff", "--content-only", "--json-lines", "test0", "test1") + assert all("stats" not in json.loads(line) for line in output.splitlines() if line.startswith("{")) From f554346eb7b89980c8c17ded6f41e64b183229fb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 4 Sep 2026 02:11:08 +0200 Subject: [PATCH 2/2] tests: fix diff --stats metadata test on Windows test_stats_counts_metadata_only_changes made its metadata-only change with os.chmod(0o700). On Windows, chmod only toggles the read-only bit, so the stored mode did not change and file_touched was not reported as a changed item at all: the test expected 3 changed items, but got 2. Change the file's mtime instead, which is a metadata change borg reports on every platform. Co-Authored-By: Claude Opus 5 --- src/borg/testsuite/archiver/diff_cmd_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/borg/testsuite/archiver/diff_cmd_test.py b/src/borg/testsuite/archiver/diff_cmd_test.py index 6d764b438d..5937089069 100644 --- a/src/borg/testsuite/archiver/diff_cmd_test.py +++ b/src/borg/testsuite/archiver/diff_cmd_test.py @@ -611,7 +611,9 @@ def _setup_stats_archives(archiver): create_regular_file(archiver.input_path, "file_changed", contents=b"d" * 120) os.unlink("input/file_removed") create_regular_file(archiver.input_path, "file_added", contents=b"e" * 30) - os.chmod("input/file_touched", 0o700) + # touch the mtime only: a metadata change that works on every platform + # (os.chmod only toggles the read-only bit on Windows). + os.utime("input/file_touched", (1000000000, 1000000000)) cmd(archiver, "create", "test1", "input") @@ -620,7 +622,7 @@ def test_stats(archivers, request): _setup_stats_archives(archiver) output = cmd(archiver, "diff", "--stats", "--content-only", "test0", "test1") lines = output.splitlines() - # only the content changes are considered, so file_touched (mode only) is not counted. + # only the content changes are considered, so file_touched (mtime only) is not counted. assert "Added items: 1" in lines assert "Removed items: 1" in lines assert "Changed items: 1" in lines @@ -635,8 +637,9 @@ def test_stats(archivers, request): def test_stats_counts_metadata_only_changes(archivers, request): archiver = request.getfixturevalue(archivers) _setup_stats_archives(archiver) - # without --content-only, file_touched (changed mode) and the input directory - # (changed mtime/ctime) are reported as changed items, too. + # without --content-only, file_touched (changed mtime) and the input directory + # (changed mtime, as items were added to / removed from it) are reported as + # changed items, too. output = cmd(archiver, "diff", "--stats", "test0", "test1") lines = output.splitlines() assert "Added items: 1" in lines