From c9493edc94be10946f9fd783b8fbe2dbf2aacd88 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 1 Sep 2026 20:27:26 +0200 Subject: [PATCH] analyze: replace --by-name by --group-by --by-name decomposed the repository by archive name, which is a grouping with the keys hardwired. Archive series names are not unique in a repository shared by multiple machines or users, so different hosts backing up a "home" series ended up in one row: its exclusive size was the combined cost of unrelated data sets and could not be attributed to either host. It also no longer matched what prune does. prune groups by name,host, and the number analyze reports per row - "what would deleting this free" - is only actionable if the row is the unit that prune applies its retention rules to. --by-name is replaced by --group-by KEYS, taking the same comma-separated archive attributes (name, host, user, tags) and using the same grouping as prune, so "--group-by name" is the previous behaviour. As before, giving the option selects the decomposition mode, and it can not be combined with archive filters, because the shared and unreferenced rows need the repository-wide view. Grouping by more attributes moves chunks from the group rows into the shared row: if two hosts back up identical content, --group-by name reports it as exclusive to that name, while --group-by name,host correctly shows that deleting either host's archives alone would not free it. The JSON output changes accordingly, without keeping compatibility: the by_name key becomes by_group, its names list becomes groups, and each entry has a group object (attribute -> value) instead of a name. The list of attributes grouped by is in the new group_by key. --- docs/internals/frontends.rst | 19 ++- src/borg/archiver/analyze_cmd.py | 159 ++++++++++-------- .../testsuite/archiver/analyze_cmd_test.py | 103 +++++++++--- 3 files changed, 179 insertions(+), 102 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 3cfb1df4eb..ee408b3c6f 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -677,7 +677,7 @@ Archive Analysis :ref:`borg_analyze` ``--json`` emits the numbers of its text report as one object. All sizes are byte values; the compression factor the text report shows is ``stored_size / source_size``. -Without ``--by-name``, the *dedup_size* and *hotspots* keys are present. +Without ``--group-by``, the *dedup_size* and *hotspots* keys are present. *dedup_size* describes the considered set of archives: @@ -708,21 +708,24 @@ missing_chunks removed in that directory between consecutive archives), busiest directory first. It is ``null`` if fewer than two archives matched, as hot spots need at least two archives to compare. -With ``--by-name``, the *by_name* key is present instead, decomposing the whole repository: +With ``--group-by``, the *by_group* key is present instead, decomposing the whole repository: archives Number of non-deleted archives in the repository -names - List of objects with *name*, *archives* (number of archives with that name), *source_size* - and *stored_size*. The sizes are what is exclusive to that name: no archive of another name - references those chunks. Biggest *stored_size* first. +group_by + List of the archive attributes the archives were grouped by, as given to ``--group-by`` +groups + List of objects with *group*, *archives* (number of archives in that group), *source_size* + and *stored_size*. *group* is an object mapping each *group_by* attribute to this group's + value for it, e.g. ``{"name": "home", "host": "host1"}``. The sizes are what is exclusive to + that group: no archive of another group references those chunks. Biggest *stored_size* first. shared - Object with *source_size* and *stored_size*: the chunks referenced by two or more names + Object with *source_size* and *stored_size*: the chunks referenced by two or more groups unreferenced As above total Object with *archives*, *source_size* and *stored_size*. Each chunk is counted in exactly one - of *names*, *shared* and *unreferenced*, so the *names* and *shared* sizes add up to *total*. + of *groups*, *shared* and *unreferenced*, so the *groups* and *shared* sizes add up to *total*. total_chunks, missing_chunks As above diff --git a/src/borg/archiver/analyze_cmd.py b/src/borg/archiver/analyze_cmd.py index 639fb8a79b..51cf3cfdbd 100644 --- a/src/borg/archiver/analyze_cmd.py +++ b/src/borg/archiver/analyze_cmd.py @@ -1,14 +1,15 @@ from collections import defaultdict import os -from ._common import with_repository, define_archive_filters_group +from ._common import with_repository, define_archive_filters_group, Highlander from ..archive import Archive from ..cache import get_archive_references, list_archive_reference_caches from ..constants import * # NOQA from ..helpers import basic_json_data, bin_to_hex, Error, format_file_size, json_print from ..helpers import ProgressIndicatorPercent from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest +from ..helpers import GroupBySpec +from ..manifest import AI_GROUP_BY_KEYS, Manifest, archive_group_key, format_group_key from ..repository import Repository from ..logger import create_logger @@ -23,13 +24,14 @@ # ThomasWaldmann's proposal there). F_CONSIDERED = 2**3 F_REST = 2**4 -# By-name mode: bits 5..22 hold the owning archive name (as index + 1, so 0 means "referenced by no -# archive"), bit 23 marks a chunk referenced by more than one name. This decomposes the repository -# in a single pass: each chunk is either exclusive to one name, shared, or unreferenced. +# Decomposition mode (--group-by): bits 5..22 hold the owning archive group (as index + 1, so 0 +# means "referenced by no archive"), bit 23 marks a chunk referenced by more than one group. This +# decomposes the repository in a single pass: each chunk is either exclusive to one group, shared, +# or unreferenced. OWNER_SHIFT = 5 OWNER_MASK = 2**18 - 1 # bits 5..22 F_MULTI = 2**23 -MAX_NAMES = OWNER_MASK - 1 # owner values are name index + 1 +MAX_GROUPS = OWNER_MASK - 1 # owner values are group index + 1 class ArchiveAnalyzer: @@ -43,17 +45,22 @@ def __init__(self, args, repository, manifest): def analyze(self): logger.info("Starting archives analysis...") json_data = {} if self.args.json else None - if self.args.by_name: + if self.args.group_by is not None: + group_by = tuple(self.args.group_by.split(",")) if self.args.group_by else () + if not group_by: + raise Error( + "--group-by needs at least one archive attribute to decompose by, e.g. --group-by name,host." + ) # the decomposition is inherently repository-wide: "shared" and "unreferenced" can only # be determined by looking at every archive, so archive filters must not be applied. filters = ["match_archives", "first", "last", "older", "newer", "oldest", "newest"] if any(getattr(self.args, name, None) for name in filters): - raise Error("--by-name analyzes the whole repository and cannot be combined with archive filters.") - by_name = self.analyze_by_name() + raise Error("--group-by analyzes the whole repository and cannot be combined with archive filters.") + by_group = self.analyze_by_group(group_by) if json_data is None: - self.report_by_name(by_name) + self.report_by_group(by_group) else: - json_data["by_name"] = by_name + json_data["by_group"] = by_group else: considered_infos = self.manifest.archives.list_considering(self.args) if not considered_infos: @@ -118,36 +125,38 @@ def mark_references(self, archive_infos, update_flags): pi.finish() return missing - def analyze_by_name(self) -> dict: - """Decompose the whole repository by archive name. + def analyze_by_group(self, group_by) -> dict: + """Decompose the whole repository by groups of archives, see *group_by* (--group-by). - Archives sharing a name form a series, so a name usually groups all backups of one source; - for old-style archives that do not form a series, each name is just one archive. + Archives sharing a name form a series, so grouping by name usually groups all backups of one + source. In a repository shared by multiple machines or users the name alone is not specific + enough, as they may use the same series name for their own, unrelated data - grouping by + e.g. name,host separates those. - Every chunk falls into exactly one bucket: exclusive to a single name (no archive of another - name references it, so deleting all archives of that name would free it), shared by two or - more names, or referenced by no non-deleted archive at all (reclaimable by borg compact). + Every chunk falls into exactly one bucket: exclusive to a single group (no archive of another + group references it, so deleting all archives of that group would free it), shared by two or + more groups, or referenced by no non-deleted archive at all (reclaimable by borg compact). The buckets add up to the repository's deduplicated size. - This needs only a single pass over all archives: each chunk records the name that first - referenced it, and a reference from a different name sets the F_MULTI bit. + This needs only a single pass over all archives: each chunk records the group that first + referenced it, and a reference from a different group sets the F_MULTI bit. - Returns the decomposition as raw byte values, for report_by_name() or --json. + Returns the decomposition as raw byte values, for report_by_group() or --json. """ all_infos = self.manifest.archives.list() # non-deleted archives if not all_infos: raise Error("The repository does not contain any archives.") - archives_per_name: dict[str, int] = defaultdict(int) + archives_per_group: dict[tuple[str, ...], int] = defaultdict(int) for info in all_infos: - archives_per_name[info.name] += 1 - names = sorted(archives_per_name) - if len(names) > MAX_NAMES: - raise Error(f"Too many distinct archive names ({len(names)}) to decompose, limit is {MAX_NAMES}.") - owner_of = {name: i + 1 for i, name in enumerate(names)} # 0 means "unreferenced" + archives_per_group[archive_group_key(info, group_by)] += 1 + groups = sorted(archives_per_group) + if len(groups) > MAX_GROUPS: + raise Error(f"Too many distinct archive groups ({len(groups)}) to decompose, limit is {MAX_GROUPS}.") + owner_of = {group: i + 1 for i, group in enumerate(groups)} # 0 means "unreferenced" missing = 0 for info in all_infos: - owner = owner_of[info.name] + owner = owner_of[archive_group_key(info, group_by)] def update_flags(flags, owner=owner): previous = (flags >> OWNER_SHIFT) & OWNER_MASK @@ -159,7 +168,7 @@ def update_flags(flags, owner=owner): missing += self.mark_references([info], update_flags) - exclusive = {name: [0, 0] for name in names} # archive name -> [source size, stored size] + exclusive = {group: [0, 0] for group in groups} # archive group -> [source size, stored size] shared = [0, 0] unref_count = unref_stored = total_count = 0 for id, entry in self.repository.chunks.iteritems(): @@ -169,7 +178,7 @@ def update_flags(flags, owner=owner): unref_count += 1 unref_stored += entry.obj_size continue - bucket = shared if entry.flags & F_MULTI else exclusive[names[owner - 1]] + bucket = shared if entry.flags & F_MULTI else exclusive[groups[owner - 1]] bucket[0] += entry.size bucket[1] += entry.obj_size @@ -181,15 +190,16 @@ def update_flags(flags, owner=owner): return { "archives": len(all_infos), + "group_by": list(group_by), # biggest exclusive consumer first - that is what one would act on - "names": [ + "groups": [ { - "name": name, - "archives": archives_per_name[name], - "source_size": exclusive[name][0], - "stored_size": exclusive[name][1], + "group": dict(zip(group_by, group)), + "archives": archives_per_group[group], + "source_size": exclusive[group][0], + "stored_size": exclusive[group][1], } - for name in sorted(names, key=lambda n: exclusive[n][1], reverse=True) + for group in sorted(groups, key=lambda g: exclusive[g][1], reverse=True) ], "shared": {"source_size": shared[0], "stored_size": shared[1]}, "unreferenced": {"stored_size": unref_stored, "chunks": unref_count}, @@ -198,10 +208,11 @@ def update_flags(flags, owner=owner): "missing_chunks": missing, } - def report_by_name(self, data) -> None: - """Print the --by-name decomposition computed by analyze_by_name().""" - names = [entry["name"] for entry in data["names"]] - width = min(max([30] + [len(name) for name in names]), 60) + def report_by_group(self, data) -> None: + """Print the --group-by decomposition computed by analyze_by_group().""" + group_by = data["group_by"] + labels = [format_group_key(tuple(entry["group"].values()), group_by) for entry in data["groups"]] + width = min(max([30] + [len(label) for label in labels]), 60) def row(label, archives, source, stored, *, source_known=True): sizes = f"{self.fmt(source) if source_known else 'n/a':>14}{self.fmt(stored):>14}" @@ -209,14 +220,14 @@ def row(label, archives, source, stored, *, source_known=True): print(f"{label:<{width}}{archives:>10}{sizes}{ratio:>13}") print() - print("Repository decomposition by archive name") + print(f"Repository decomposition by {','.join(group_by)}") print("=" * (width + 51)) - print(f"{data['archives']} archive(s) with {len(names)} distinct name(s)") + print(f"{data['archives']} archive(s) in {len(labels)} group(s)") print() - print(f"{'name':<{width}}{'archives':>10}{'source':>14}{'stored':>14}{'compression':>13}") - for entry in data["names"]: - row(entry["name"], entry["archives"], entry["source_size"], entry["stored_size"]) - row("(shared by 2+ names)", "", data["shared"]["source_size"], data["shared"]["stored_size"]) + print(f"{'group':<{width}}{'archives':>10}{'source':>14}{'stored':>14}{'compression':>13}") + for label, entry in zip(labels, data["groups"]): + row(label, entry["archives"], entry["source_size"], entry["stored_size"]) + row("(shared by 2+ groups)", "", data["shared"]["source_size"], data["shared"]["stored_size"]) row("(unreferenced)", "", 0, data["unreferenced"]["stored_size"], source_known=False) print("-" * (width + 51)) row( @@ -231,9 +242,9 @@ def row(label, archives, source, stored, *, source_known=True): ) print() print("Each chunk is counted in exactly one row, so the rows add up to the total.") - print("A name row shows what is exclusive to it: no archive of another name references these") - print("chunks, so deleting all archives of that name would free them. Chunks used by several") - print("names are counted in the shared row, not against any single name.") + print("A group row shows what is exclusive to it: no archive of another group references these") + print("chunks, so deleting all archives of that group would free them. Chunks used by several") + print("groups are counted in the shared row, not against any single group.") print() print(f"{'source':<12} = uncompressed source data size (each chunk counted once)") print(f"{'stored':<12} = deduplicated size as stored in the repository (compressed)") @@ -444,7 +455,8 @@ def build_parser_analyze(self, subparsers, common_parser, mid_common_parser): analyze_epilog = process_epilog( """ - Analyze the archives matching the usual archive selection options (e.g. ``-a series_name``). + Analyze the archives matching the usual archive selection options (e.g. ``-a series_name``), + or decompose the whole repository into groups of archives, see ``--group-by``. **Deduplicated size of a set of archives** @@ -472,25 +484,36 @@ def build_parser_analyze(self, subparsers, common_parser, mid_common_parser): source sizes come from the per-archive references cache that ``borg compact`` maintains, so unchanged archives usually do not need to be opened. - **Decomposition by archive name (--by-name)** + **Decomposition by group of archives (--group-by)** - With ``--by-name``, the whole repository is decomposed by archive name instead. Archives - sharing a name form a series, so a name usually groups all backups of one source; for - old-style archives that do not form a series, each name is just one archive. + With ``--group-by``, the whole repository is decomposed into groups of archives instead. + The groups are formed by the given comma-separated archive attributes ``name``, ``host``, + ``user`` and ``tags`` - the same grouping ``borg prune --group-by`` uses, so the numbers + below relate to the same units that prune applies its retention rules to. + + Archives sharing a name form a series, so ``--group-by name`` usually groups all backups + of one source. In a repository shared by multiple machines or users that is not specific + enough, because they may use the same series name for their own, unrelated data - use + ``--group-by name,host`` there. Every chunk is counted in exactly one row, so the rows add up to the repository's deduplicated size: - - one row per archive name, showing what is *exclusive* to it: no archive of another name - references these chunks, so deleting all archives of that name would free them; - - one row for the chunks shared by two or more names; + - one row per group, showing what is *exclusive* to it: no archive of another group + references these chunks, so deleting all archives of that group would free them; + - one row for the chunks shared by two or more groups; - one row for the unreferenced chunks (see above). - This answers "which name costs how much, and what would I get back by dropping it" for - all names at once, in a single pass over the archives. As the shared and unreferenced - rows can only be determined by looking at every archive, ``--by-name`` always covers the + This answers "which group costs how much, and what would I get back by dropping it" for + all groups at once, in a single pass over the archives. As the shared and unreferenced + rows can only be determined by looking at every archive, ``--group-by`` always covers the whole repository and cannot be combined with archive filters. + Grouping by more attributes moves chunks from the group rows into the shared row: if two + hosts back up identical content, ``--group-by name`` reports it as exclusive to that + name, while ``--group-by name,host`` correctly shows that deleting either host's + archives alone would not free it. + **Hot spots** If at least two archives match, ``borg analyze`` additionally iterates over all matching @@ -508,8 +531,8 @@ def build_parser_analyze(self, subparsers, common_parser, mid_common_parser): With ``--json``, the same numbers are emitted as a single JSON object instead of the text report, with raw byte values rather than formatted sizes. The default mode fills the - *dedup_size* and *hotspots* keys, ``--by-name`` fills the *by_name* key. The compression - factor is not included: it is ``stored_size / source_size``. + *dedup_size* and *hotspots* keys, ``--group-by`` fills the *by_group* key. The + compression factor is not included: it is ``stored_size / source_size``. See :ref:`json_output` for the object's structure. """ @@ -517,10 +540,14 @@ def build_parser_analyze(self, subparsers, common_parser, mid_common_parser): subparser = ArgumentParser(parents=[common_parser], description=self.do_analyze.__doc__, epilog=analyze_epilog) subparsers.add_subcommand("analyze", subparser, help="analyze archives") subparser.add_argument( - "--by-name", - dest="by_name", - action="store_true", - help="decompose the whole repository by archive name (not combinable with archive filters)", + "--group-by", + metavar="KEYS", + dest="group_by", + type=GroupBySpec, + action=Highlander, + help="decompose the whole repository by groups of archives, grouped by the given " + "comma-separated archive attributes (not combinable with archive filters); " + "valid keys are: {}".format(", ".join(AI_GROUP_BY_KEYS)), ) subparser.add_argument("--json", action="store_true", help="format output as JSON") define_archive_filters_group(subparser) diff --git a/src/borg/testsuite/archiver/analyze_cmd_test.py b/src/borg/testsuite/archiver/analyze_cmd_test.py index df49bfc124..059db150b7 100644 --- a/src/borg/testsuite/archiver/analyze_cmd_test.py +++ b/src/borg/testsuite/archiver/analyze_cmd_test.py @@ -107,8 +107,8 @@ def test_analyze_whole_repository(archivers, request): assert "Exclusive size" not in output -def test_analyze_by_name(archivers, request): - """--by-name decomposes the repository into per-name exclusive, shared and unreferenced.""" +def test_analyze_group_by_name(archivers, request): + """--group-by decomposes the repository into per-group exclusive, shared and unreferenced.""" archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) @@ -125,18 +125,18 @@ def test_analyze_by_name(archivers, request): (input_path / "b_only").write_text("b" * 1000) cmd(archiver, "create", "beta", archiver.input_path) - output = cmd(archiver, "analyze", "--by-name") - assert "3 archive(s) with 2 distinct name(s)" in output - # exclusive to each name is its own 1000 B file; "shared" is used by archives of both names - assert re.search(r"^alpha\s+2\s+1\.00 kB", output, re.MULTILINE) - assert re.search(r"^beta\s+1\s+1\.00 kB", output, re.MULTILINE) - assert re.search(r"\(shared by 2\+ names\)\s+1\.00 kB", output) + output = cmd(archiver, "analyze", "--group-by", "name") + assert "3 archive(s) in 2 group(s)" in output + # exclusive to each group is its own 1000 B file; "shared" is used by archives of both groups + assert re.search(r"^name='alpha'\s+2\s+1\.00 kB", output, re.MULTILINE) + assert re.search(r"^name='beta'\s+1\s+1\.00 kB", output, re.MULTILINE) + assert re.search(r"\(shared by 2\+ groups\)\s+1\.00 kB", output) # the rows add up: 1000 (alpha) + 1000 (beta) + 1000 (shared) = 3000 B assert re.search(r"total \(deduplicated\)\s+3\s+3\.00 kB", output) -def test_analyze_by_name_rejects_filters(archivers, request): - """--by-name is repository-wide, so combining it with an archive filter is an error.""" +def test_analyze_group_by_rejects_filters(archivers, request): + """--group-by is repository-wide, so combining it with an archive filter is an error.""" archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) @@ -145,7 +145,52 @@ def test_analyze_by_name_rejects_filters(archivers, request): cmd(archiver, "create", "one", archiver.input_path) with pytest.raises(Error, match="cannot be combined with archive filters"): - cmd(archiver, "analyze", "--by-name", "-a", "sh:one") + cmd(archiver, "analyze", "--group-by", "name", "-a", "sh:one") + + +def test_analyze_group_by_needs_a_key(archivers, request): + """An empty --group-by has nothing to decompose by.""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + (input_path / "file1").write_text("x" * 1000) + cmd(archiver, "create", "one", archiver.input_path) + + with pytest.raises(Error, match="needs at least one archive attribute"): + cmd(archiver, "analyze", "--group-by", "") + + +def test_analyze_group_by_name_and_host(archivers, request, monkeypatch): + """Grouping by name,host separates the same series name of different hosts.""" + archiver = request.getfixturevalue(archivers) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + input_path = pathlib.Path(archiver.input_path) + (input_path / "shared").write_text("s" * 1000) + + # both hosts back up a "home" series, each with an own file besides the shared one + (input_path / "a_only").write_text("a" * 1000) + monkeypatch.setenv("BORG_HOSTNAME", "host1") + cmd(archiver, "create", "home", archiver.input_path) + + (input_path / "a_only").unlink() + (input_path / "b_only").write_text("b" * 1000) + monkeypatch.setenv("BORG_HOSTNAME", "host2") + cmd(archiver, "create", "home", archiver.input_path) + monkeypatch.delenv("BORG_HOSTNAME") + + # grouping by name only, both hosts are one group, so everything is exclusive to it: + output = cmd(archiver, "analyze", "--group-by", "name") + assert "2 archive(s) in 1 group(s)" in output + assert re.search(r"^name='home'\s+2\s+3\.00 kB", output, re.MULTILINE) + + # grouping by name,host separates them and moves the common file into the shared row: + output = cmd(archiver, "analyze", "--group-by", "name,host") + assert "2 archive(s) in 2 group(s)" in output + assert re.search(r"^name='home', host='host1'\s+1\s+1\.00 kB", output, re.MULTILINE) + assert re.search(r"^name='home', host='host2'\s+1\s+1\.00 kB", output, re.MULTILINE) + assert re.search(r"\(shared by 2\+ groups\)\s+1\.00 kB", output) def test_analyze_unreferenced_chunks(archivers, request): @@ -284,14 +329,14 @@ def test_analyze_json_hotspots(archivers, request): assert [hotspot["size"] for hotspot in hotspots] == sorted((hotspot["size"] for hotspot in hotspots), reverse=True) -def test_analyze_json_by_name(archivers, request): - """--by-name --json decomposes the repository into per-name exclusive, shared and unreferenced.""" +def test_analyze_json_group_by(archivers, request): + """--group-by --json decomposes the repository into per-group exclusive, shared and unreferenced.""" archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) input_path = pathlib.Path(archiver.input_path) - # same layout as test_analyze_by_name + # same layout as test_analyze_group_by_name (input_path / "shared").write_text("s" * 1000) (input_path / "a_only").write_text("a" * 1000) cmd(archiver, "create", "alpha", archiver.input_path) @@ -301,23 +346,25 @@ def test_analyze_json_by_name(archivers, request): (input_path / "b_only").write_text("b" * 1000) cmd(archiver, "create", "beta", archiver.input_path) - result = json.loads(cmd(archiver, "analyze", "--by-name", "--json")) + result = json.loads(cmd(archiver, "analyze", "--group-by", "name", "--json")) assert "dedup_size" not in result and "hotspots" not in result - by_name = result["by_name"] - - assert by_name["archives"] == 3 - assert {entry["name"]: entry["archives"] for entry in by_name["names"]} == {"alpha": 2, "beta": 1} - assert {entry["name"]: entry["source_size"] for entry in by_name["names"]} == {"alpha": 1000, "beta": 1000} - assert by_name["shared"]["source_size"] == 1000 - assert by_name["total"]["archives"] == 3 - assert by_name["total"]["source_size"] == 3000 # 1000 alpha + 1000 beta + 1000 shared - assert by_name["total"]["stored_size"] > 0 - assert by_name["missing_chunks"] == 0 + by_group = result["by_group"] + + assert by_group["archives"] == 3 + assert by_group["group_by"] == ["name"] + groups = by_group["groups"] + assert {entry["group"]["name"]: entry["archives"] for entry in groups} == {"alpha": 2, "beta": 1} + assert {entry["group"]["name"]: entry["source_size"] for entry in groups} == {"alpha": 1000, "beta": 1000} + assert by_group["shared"]["source_size"] == 1000 + assert by_group["total"]["archives"] == 3 + assert by_group["total"]["source_size"] == 3000 # 1000 alpha + 1000 beta + 1000 shared + assert by_group["total"]["stored_size"] > 0 + assert by_group["missing_chunks"] == 0 # every chunk is counted in exactly one row, so the rows add up to the total for size in ("source_size", "stored_size"): - assert sum(entry[size] for entry in by_name["names"]) + by_name["shared"][size] == by_name["total"][size] + assert sum(entry[size] for entry in groups) + by_group["shared"][size] == by_group["total"][size] # biggest exclusive consumer first - assert [entry["stored_size"] for entry in by_name["names"]] == sorted( - (entry["stored_size"] for entry in by_name["names"]), reverse=True + assert [entry["stored_size"] for entry in groups] == sorted( + (entry["stored_size"] for entry in groups), reverse=True )