From b4ae613c31af375b2f8294f6698583d6fe74e026 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Sep 2026 20:58:44 +0200 Subject: [PATCH] diff --stats: use the size_added / size_removed vocabulary, document the JSON stats line Rename the "Added chunk volume" / "Removed chunk volume" summary lines to "Added size" / "Removed size" and the JSON keys added_chunk_volume / removed_chunk_volume to size_added / size_removed. These totals are the sums over all reported items of what the per-path output calls "added" / "removed" and what --sort-by size_added / size_removed sorts by, so use that existing vocabulary instead of introducing a new one. #10327 was not released yet, so nothing depends on the old names. Document the --stats summary line of borg diff --json-lines in docs/internals/frontends.rst, which describes the JSON Lines output for frontends but did not mention it. Also add a granularity_sleep() between the two archives of the diff --stats tests: test_stats_counts_metadata_only_changes expects the input directory to be reported as changed (mtime/ctime), which fails on filesystems with whole-second timestamps (reproduced on HFS+: "Changed items: 2" instead of 3) when both archives are created within the same second. The other tests in that file already sleep there. Follow-up to #10327 (#796). Co-Authored-By: Claude Fable 5.1 --- docs/internals/frontends.rst | 31 ++++++++++++++++++++ src/borg/archiver/diff_cmd.py | 29 +++++++++--------- src/borg/testsuite/archiver/diff_cmd_test.py | 20 ++++++++----- 3 files changed, 58 insertions(+), 22 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index f511b5719b..b73a7cbaa5 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -667,6 +667,33 @@ item2: *changed owner*, *changed user*, *changed group*, *ctime* and *mtime* changes. Items that have only such changes are then not printed at all. +With ``--stats``, a final line of the shape ``{"stats": {...}}`` follows the per-path lines. It has +no **path** and no **changes** property, so it is easy to tell apart. The *stats* object has these +properties (all of them JSON numbers): + +added_items: + The number of items that only exist in ARCHIVE2. + +removed_items: + The number of items that only exist in ARCHIVE1. + +changed_items: + The number of items that exist in both archives, but differ. + +size_added: + The total amount of file content (in bytes) added by all of these items, i.e. the sum of the + **added** properties of their content changes (see above). + +size_removed: + The total amount of file content (in bytes) removed by all of these items, i.e. the sum of the + **removed** properties of their content changes (see above). + +unknown_size_items: + The number of items whose content was modified by an unknown amount (see **added** above). + They are counted in *changed_items*, but contribute nothing to *size_added* / *size_removed*. + +``--content-only`` narrows these counts the same way it narrows the per-path output. + Example of ``borg diff --json-lines --sort-by path ARCHIVE1 ARCHIVE2``:: @@ -680,6 +707,10 @@ Example of ``borg diff --json-lines --sort-by path ARCHIVE1 ARCHIVE2``:: {"changes": [{"added": 8, "removed": 4, "type": "modified"}, {"item1": "2026-08-28T09:00:12.816498270+02:00", "item2": "2026-08-28T09:00:14.334788977+02:00", "type": "ctime"}, {"item1": "2026-08-28T09:00:12.816498270+02:00", "item2": "2026-08-28T09:00:14.334788977+02:00", "type": "mtime"}], "path": "data/file5"} {"changes": [{"type": "changed link"}, {"item1": "2026-08-28T09:00:12.820243127+02:00", "item2": "2026-08-28T09:00:14.332850526+02:00", "type": "ctime"}, {"item1": "2026-08-28T09:00:12.820210210+02:00", "item2": "2026-08-28T09:00:14.332821942+02:00", "type": "mtime"}], "path": "data/link1"} +With ``--stats``, the same command additionally ends with:: + + {"stats": {"added_items": 2, "changed_items": 4, "removed_items": 3, "size_added": 16, "size_removed": 8, "unknown_size_items": 0}} + Archive Analysis ++++++++++++++++ diff --git a/src/borg/archiver/diff_cmd.py b/src/borg/archiver/diff_cmd.py index a1fa032bf3..82b3482817 100644 --- a/src/borg/archiver/diff_cmd.py +++ b/src/borg/archiver/diff_cmd.py @@ -44,8 +44,8 @@ def __init__(self): self.added_items = 0 # items present in ARCHIVE2 only self.removed_items = 0 # items present in ARCHIVE1 only self.changed_items = 0 # items present in both archives, but not equal - self.added_chunk_volume = 0 # size of the content chunks added (by added and by changed items) - self.removed_chunk_volume = 0 # size of the content chunks removed (by removed and by changed items) + self.size_added = 0 # total size of the content added (by added and by changed items) + self.size_removed = 0 # total size of the content removed (by removed and by changed items) self.unknown_size_items = 0 # items whose content changed by an unknown amount def add(self, diff: ItemDiff, changes: dict) -> None: @@ -54,8 +54,8 @@ def add(self, diff: ItemDiff, changes: dict) -> None: if content is not None: info = content.to_dict() if "added" in info or "removed" in info: - self.added_chunk_volume += info.get("added", 0) - self.removed_chunk_volume += info.get("removed", 0) + self.size_added += info.get("added", 0) + self.size_removed += info.get("removed", 0) else: # a "modified" that was determined by comparing the content: no byte counts. self.unknown_size_items += 1 @@ -71,8 +71,8 @@ def as_dict(self) -> dict: "added_items": self.added_items, "removed_items": self.removed_items, "changed_items": self.changed_items, - "added_chunk_volume": self.added_chunk_volume, - "removed_chunk_volume": self.removed_chunk_volume, + "size_added": self.size_added, + "size_removed": self.size_removed, "unknown_size_items": self.unknown_size_items, } @@ -81,8 +81,8 @@ def __str__(self) -> str: f"Added items: {self.added_items}", f"Removed items: {self.removed_items}", f"Changed items: {self.changed_items}", - f"Added chunk volume: {format_file_size(self.added_chunk_volume)}", - f"Removed chunk volume: {format_file_size(self.removed_chunk_volume)}", + f"Added size: {format_file_size(self.size_added)}", + f"Removed size: {format_file_size(self.size_removed)}", ] if self.unknown_size_items: # these items are not accounted for in the added/removed data above, so say so. @@ -322,12 +322,13 @@ def build_parser_diff(self, subparsers, common_parser, mid_common_parser): Added items: 23 Removed items: 2 Changed items: 315 - Added chunk volume: 53.70 MB - Removed chunk volume: 51.10 MB + Added size: 53.70 MB + Removed size: 51.10 MB "Added"/"Removed" items only exist in ARCHIVE2/ARCHIVE1, "changed" items exist in both - archives but differ. "Added chunk volume"/"Removed chunk volume" sum up the size of the - content chunks added/removed by all of these items. Items whose content borg could only + archives but differ. "Added size"/"Removed size" sum up the file content (in bytes) added/removed + by all of these items, i.e. the per-path byte counts that ``--sort-by size_added`` / + ``size_removed`` sort by. Items whose content borg could only compare byte by byte (see "Performance considerations" below) contribute no byte counts; if there are any, an additional "Items with unknown size changes" line reports how many. @@ -335,8 +336,8 @@ def build_parser_diff(self, subparsers, common_parser, mid_common_parser): ``{"stats": {...}}`` instead, so it is easy to tell apart from the per-path lines (wrapped here for readability, borg prints it as a single line):: - {"stats": {"added_chunk_volume": 53700000, "added_items": 23, "changed_items": 315, - "removed_chunk_volume": 51100000, "removed_items": 2, "unknown_size_items": 0}} + {"stats": {"added_items": 23, "changed_items": 315, "removed_items": 2, + "size_added": 53700000, "size_removed": 51100000, "unknown_size_items": 0}} Sorting ++++++++ diff --git a/src/borg/testsuite/archiver/diff_cmd_test.py b/src/borg/testsuite/archiver/diff_cmd_test.py index 5937089069..7f2b49f26d 100644 --- a/src/borg/testsuite/archiver/diff_cmd_test.py +++ b/src/borg/testsuite/archiver/diff_cmd_test.py @@ -608,6 +608,10 @@ def _setup_stats_archives(archiver): create_regular_file(archiver.input_path, "file_removed", contents=b"b" * 5) create_regular_file(archiver.input_path, "file_touched", contents=b"c" * 7) cmd(archiver, "create", "test0", "input") + # let the timestamps move on before the second archive: the metadata test expects the input directory + # to be reported as changed (mtime/ctime), which needs more than one timestamp tick on filesystems with + # whole-second timestamps (e.g. HFS+). + granularity_sleep() create_regular_file(archiver.input_path, "file_changed", contents=b"d" * 120) os.unlink("input/file_removed") create_regular_file(archiver.input_path, "file_added", contents=b"e" * 30) @@ -627,9 +631,9 @@ def test_stats(archivers, request): assert "Removed items: 1" in lines assert "Changed items: 1" in lines # added: file_added (30) + the new content of file_changed (120) - assert_line_exists(lines, r"^Added chunk volume: 150 B$") + assert_line_exists(lines, r"^Added size: 150 B$") # removed: file_removed (5) + the old content of file_changed (100) - assert_line_exists(lines, r"^Removed chunk volume: 105 B$") + assert_line_exists(lines, r"^Removed size: 105 B$") # all sizes are known, so this line is omitted assert_line_not_exists(lines, r"^Items with unknown size changes:") @@ -645,8 +649,8 @@ def test_stats_counts_metadata_only_changes(archivers, request): assert "Added items: 1" in lines assert "Removed items: 1" in lines assert "Changed items: 3" in lines - assert_line_exists(lines, r"^Added chunk volume: 150 B$") - assert_line_exists(lines, r"^Removed chunk volume: 105 B$") + assert_line_exists(lines, r"^Added size: 150 B$") + assert_line_exists(lines, r"^Removed size: 105 B$") def test_stats_json_lines(archivers, request): @@ -661,8 +665,8 @@ def test_stats_json_lines(archivers, request): "added_items": 1, "removed_items": 1, "changed_items": 1, - "added_chunk_volume": 150, - "removed_chunk_volume": 105, + "size_added": 150, + "size_removed": 105, "unknown_size_items": 0, } } @@ -679,8 +683,8 @@ def test_stats_unknown_sizes(archivers, request): output = cmd(archiver, "diff", "--stats", "--content-only", "test0", "test1") lines = output.splitlines() assert "Changed items: 1" in lines - assert_line_exists(lines, r"^Added chunk volume: 0 B$") - assert_line_exists(lines, r"^Removed chunk volume: 0 B$") + assert_line_exists(lines, r"^Added size: 0 B$") + assert_line_exists(lines, r"^Removed size: 0 B$") assert "Items with unknown size changes: 1" in lines