Skip to content

diff --stats follow-up: size_added/size_removed naming, JSON docs, test timing #10339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ThomasWaldmann merged 1 commit into from
Sep 8, 2026
+58 −22
Merged

diff --stats follow-up: size_added/size_removed naming, JSON docs, test timing #10339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions docs/internals/frontends.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,33 @@ item2:
*changed owner*, *changed user*, *changed group*, *ctime* and *mtime* changes. Items that have
only such changes are then not printed at all.

With ``--stats``, a final line of the shape ``{"stats": {...}}`` follows the per-path lines. It has
no **path** and no **changes** property, so it is easy to tell apart. The *stats* object has these
properties (all of them JSON numbers):

added_items:
The number of items that only exist in ARCHIVE2.

removed_items:
The number of items that only exist in ARCHIVE1.

changed_items:
The number of items that exist in both archives, but differ.

size_added:
The total amount of file content (in bytes) added by all of these items, i.e. the sum of the
**added** properties of their content changes (see above).

size_removed:
The total amount of file content (in bytes) removed by all of these items, i.e. the sum of the
**removed** properties of their content changes (see above).

unknown_size_items:
The number of items whose content was modified by an unknown amount (see **added** above).
They are counted in *changed_items*, but contribute nothing to *size_added* / *size_removed*.

``--content-only`` narrows these counts the same way it narrows the per-path output.


Example of ``borg diff --json-lines --sort-by path ARCHIVE1 ARCHIVE2``::

Expand All @@ -680,6 +707,10 @@ Example of ``borg diff --json-lines --sort-by path ARCHIVE1 ARCHIVE2``::
{"changes": [{"added": 8, "removed": 4, "type": "modified"}, {"item1": "2026-08-28T09:00:12.816498270+02:00", "item2": "2026-08-28T09:00:14.334788977+02:00", "type": "ctime"}, {"item1": "2026-08-28T09:00:12.816498270+02:00", "item2": "2026-08-28T09:00:14.334788977+02:00", "type": "mtime"}], "path": "data/file5"}
{"changes": [{"type": "changed link"}, {"item1": "2026-08-28T09:00:12.820243127+02:00", "item2": "2026-08-28T09:00:14.332850526+02:00", "type": "ctime"}, {"item1": "2026-08-28T09:00:12.820210210+02:00", "item2": "2026-08-28T09:00:14.332821942+02:00", "type": "mtime"}], "path": "data/link1"}

With ``--stats``, the same command additionally ends with::

{"stats": {"added_items": 2, "changed_items": 4, "removed_items": 3, "size_added": 16, "size_removed": 8, "unknown_size_items": 0}}


Archive Analysis
++++++++++++++++
Expand Down
29 changes: 15 additions & 14 deletions src/borg/archiver/diff_cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ def __init__(self):
self.added_items = 0 # items present in ARCHIVE2 only
self.removed_items = 0 # items present in ARCHIVE1 only
self.changed_items = 0 # items present in both archives, but not equal
self.added_chunk_volume = 0 # size of the content chunks added (by added and by changed items)
self.removed_chunk_volume = 0 # size of the content chunks removed (by removed and by changed items)
self.size_added = 0 # total size of the content added (by added and by changed items)
self.size_removed = 0 # total size of the content removed (by removed and by changed items)
self.unknown_size_items = 0 # items whose content changed by an unknown amount

def add(self, diff: ItemDiff, changes: dict) -> None:
Expand All @@ -54,8 +54,8 @@ def add(self, diff: ItemDiff, changes: dict) -> None:
if content is not None:
info = content.to_dict()
if "added" in info or "removed" in info:
self.added_chunk_volume += info.get("added", 0)
self.removed_chunk_volume += info.get("removed", 0)
self.size_added += info.get("added", 0)
self.size_removed += info.get("removed", 0)
else:
# a "modified" that was determined by comparing the content: no byte counts.
self.unknown_size_items += 1
Expand All @@ -71,8 +71,8 @@ def as_dict(self) -> dict:
"added_items": self.added_items,
"removed_items": self.removed_items,
"changed_items": self.changed_items,
"added_chunk_volume": self.added_chunk_volume,
"removed_chunk_volume": self.removed_chunk_volume,
"size_added": self.size_added,
"size_removed": self.size_removed,
"unknown_size_items": self.unknown_size_items,
}

Expand All @@ -81,8 +81,8 @@ def __str__(self) -> str:
f"Added items: {self.added_items}",
f"Removed items: {self.removed_items}",
f"Changed items: {self.changed_items}",
f"Added chunk volume: {format_file_size(self.added_chunk_volume)}",
f"Removed chunk volume: {format_file_size(self.removed_chunk_volume)}",
f"Added size: {format_file_size(self.size_added)}",
f"Removed size: {format_file_size(self.size_removed)}",
]
if self.unknown_size_items:
# these items are not accounted for in the added/removed data above, so say so.
Expand Down Expand Up @@ -322,21 +322,22 @@ def build_parser_diff(self, subparsers, common_parser, mid_common_parser):
Added items: 23
Removed items: 2
Changed items: 315
Added chunk volume: 53.70 MB
Removed chunk volume: 51.10 MB
Added size: 53.70 MB
Removed size: 51.10 MB

"Added"/"Removed" items only exist in ARCHIVE2/ARCHIVE1, "changed" items exist in both
archives but differ. "Added chunk volume"/"Removed chunk volume" sum up the size of the
content chunks added/removed by all of these items. Items whose content borg could only
archives but differ. "Added size"/"Removed size" sum up the file content (in bytes) added/removed
by all of these items, i.e. the per-path byte counts that ``--sort-by size_added`` /
``size_removed`` sort by. Items whose content borg could only
compare byte by byte (see "Performance considerations" below) contribute no byte counts;
if there are any, an additional "Items with unknown size changes" line reports how many.

Together with ``--json-lines``, the summary is emitted as a final JSON line of the shape
``{"stats": {...}}`` instead, so it is easy to tell apart from the per-path lines
(wrapped here for readability, borg prints it as a single line)::

{"stats": {"added_chunk_volume": 53700000, "added_items": 23, "changed_items": 315,
"removed_chunk_volume": 51100000, "removed_items": 2, "unknown_size_items": 0}}
{"stats": {"added_items": 23, "changed_items": 315, "removed_items": 2,
"size_added": 53700000, "size_removed": 51100000, "unknown_size_items": 0}}

Sorting
++++++++
Expand Down
20 changes: 12 additions & 8 deletions src/borg/testsuite/archiver/diff_cmd_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,10 @@ def _setup_stats_archives(archiver):
create_regular_file(archiver.input_path, "file_removed", contents=b"b" * 5)
create_regular_file(archiver.input_path, "file_touched", contents=b"c" * 7)
cmd(archiver, "create", "test0", "input")
# let the timestamps move on before the second archive: the metadata test expects the input directory
# to be reported as changed (mtime/ctime), which needs more than one timestamp tick on filesystems with
# whole-second timestamps (e.g. HFS+).
granularity_sleep()
create_regular_file(archiver.input_path, "file_changed", contents=b"d" * 120)
os.unlink("input/file_removed")
create_regular_file(archiver.input_path, "file_added", contents=b"e" * 30)
Expand All @@ -627,9 +631,9 @@ def test_stats(archivers, request):
assert "Removed items: 1" in lines
assert "Changed items: 1" in lines
# added: file_added (30) + the new content of file_changed (120)
assert_line_exists(lines, r"^Added chunk volume: 150 B$")
assert_line_exists(lines, r"^Added size: 150 B$")
# removed: file_removed (5) + the old content of file_changed (100)
assert_line_exists(lines, r"^Removed chunk volume: 105 B$")
assert_line_exists(lines, r"^Removed size: 105 B$")
# all sizes are known, so this line is omitted
assert_line_not_exists(lines, r"^Items with unknown size changes:")

Expand All @@ -645,8 +649,8 @@ def test_stats_counts_metadata_only_changes(archivers, request):
assert "Added items: 1" in lines
assert "Removed items: 1" in lines
assert "Changed items: 3" in lines
assert_line_exists(lines, r"^Added chunk volume: 150 B$")
assert_line_exists(lines, r"^Removed chunk volume: 105 B$")
assert_line_exists(lines, r"^Added size: 150 B$")
assert_line_exists(lines, r"^Removed size: 105 B$")


def test_stats_json_lines(archivers, request):
Expand All @@ -661,8 +665,8 @@ def test_stats_json_lines(archivers, request):
"added_items": 1,
"removed_items": 1,
"changed_items": 1,
"added_chunk_volume": 150,
"removed_chunk_volume": 105,
"size_added": 150,
"size_removed": 105,
"unknown_size_items": 0,
}
}
Expand All @@ -679,8 +683,8 @@ def test_stats_unknown_sizes(archivers, request):
output = cmd(archiver, "diff", "--stats", "--content-only", "test0", "test1")
lines = output.splitlines()
assert "Changed items: 1" in lines
assert_line_exists(lines, r"^Added chunk volume: 0 B$")
assert_line_exists(lines, r"^Removed chunk volume: 0 B$")
assert_line_exists(lines, r"^Added size: 0 B$")
assert_line_exists(lines, r"^Removed size: 0 B$")
assert "Items with unknown size changes: 1" in lines


Expand Down
Loading