diff --git a/docs/changes.rst b/docs/changes.rst index dce1d41e36..1b0d115595 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -163,6 +163,14 @@ Compatibility notes: Change Log 2.x ============== +Version 2.0.0b25 (not released yet) +----------------------------------- + +New features: + +- create/import-tar --json: report the deduplicated size of the new archive, #10335. + It is also included in the archive_progress JSON output. + Version 2.0.0b24 (2026-09-02) ----------------------------- diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index ee408b3c6f..f511b5719b 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -94,6 +94,9 @@ archive_progress original_size Original size of the data processed so far (before compression and deduplication) + deduplicated_size + Deduplicated size of the data processed so far (before compression): the size of the + chunks that were new to the repository nfiles Number of (regular) files processed so far hashing_time @@ -196,8 +199,9 @@ See Prompts_ for the types used by prompts. :ref:`borg_create` file listing with progress:: - {"original_size": 0, "nfiles": 0, "hashing_time": 0.0, "chunking_time": 0.0, "files_stats": {}, - "store_stats": {}, "path": "src", "time": 1787900398.684961, "type": "archive_progress", "finished": false} + {"original_size": 0, "deduplicated_size": 0, "nfiles": 0, "hashing_time": 0.0, "chunking_time": 0.0, + "files_stats": {}, "store_stats": {}, "path": "src", "time": 1787900398.684961, "type": "archive_progress", + "finished": false} {"type": "file_status", "status": "A", "path": "src/linux/baz/file2"} {"type": "file_status", "status": "A", "path": "src/linux/baz/file3"} {"type": "file_status", "status": "d", "path": "src/linux/baz"} @@ -399,6 +403,10 @@ stats original_size Size of the file contents and the metadata in this archive, before compression and deduplication + deduplicated_size + Size of the file contents and the metadata in this archive that were new to the repository + when the archive was created (this archive's deduplicated size, before compression). + Only given by *borg create* and *borg import-tar*, see below. nfiles Number of regular files in the archive hashing_time @@ -411,13 +419,14 @@ stats Object with the statistics of the storage backend (call counts, transferred volumes, times, cache hits/misses, ...) - *borg create* fills all of these in for the archive it has just created. *borg info* only reads - *original_size* and *nfiles* from the archive metadata; *hashing_time*, *chunking_time*, - *files_stats* and *store_stats* are 0 or empty there. + *borg create* and *borg import-tar* fill all of these in for the archive they have just created. + *borg info* only reads *original_size* and *nfiles* from the archive metadata; *hashing_time*, + *chunking_time*, *files_stats* and *store_stats* are 0 or empty there and *deduplicated_size* + is absent: computing the deduplicated size of an existing archive is expensive, so it is only + known while the archive is being created. - Compressed and deduplicated sizes are not given: computing them per archive is expensive. - Use :ref:`borg_analyze` for the deduplicated size of a set of archives and - ``borg compact --stats`` for the repository-wide numbers. + Compressed sizes are not given at all. Use :ref:`borg_analyze` for the deduplicated size of a + set of archives and ``borg compact --stats`` for the repository-wide numbers. :ref:`borg_info` further has: @@ -531,6 +540,7 @@ collected while running:: "start": "2026-08-28T08:59:56.761172+02:00", "stats": { "chunking_time": 7.81649723649025e-05, + "deduplicated_size": 250510, "files_stats": { "A": 3, "d": 3 diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 81940410a2..b407f602fe 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -326,9 +326,8 @@ terminals, the compact format is used, so that the path stays readable. If the o does not go to a terminal (e.g. into a logfile), the precise format is always used. When using ``--stats``, you will get some statistics about how much data was -added - the "This Archive" deduplicated size there is most interesting as that is -how much your repository will grow. Please note that the "All archives" stats refer to -the state after creation. +added - the deduplicated size there is most interesting as that is how much your +repository will grow. ``--json`` outputs the same statistics as JSON. When ``--stats`` is used together with ``--dry-run``, only the number of files and the original size are reported. They are computed from file system metadata, without reading diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index fec796cb84..019a14de2c 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -85,5 +85,6 @@ The original size shown here is the total size of the archive's source data (uncompressed, counting duplicate content per occurrence). Deduplicated sizes are not shown here (computing them per archive is expensive). -For the deduplicated size of a set of archives, use ``borg analyze``; for the -repository-wide deduplicated size, use ``borg compact --stats``. \ No newline at end of file +``borg create --stats`` (also with ``--json``) reports the deduplicated size of the +archive it has just created. For the deduplicated size of a set of archives, use +``borg analyze``; for the repository-wide deduplicated size, use ``borg compact --stats``. \ No newline at end of file diff --git a/src/borg/archive.py b/src/borg/archive.py index a3cb0586a2..3bf157fb8d 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -109,6 +109,8 @@ def format_value(key, value): class Statistics: def __init__(self, output_json=False): self.output_json = output_json + # usize: size of the data that was new to the repository (the deduplicated size). + # None means unknown, see Archive.calc_stats(). self.osize = self.usize = self.nfiles = 0 self.last_progress = float("-inf") # monotonic timestamp when progress was last shown, -inf: never self.files_stats = defaultdict(int) @@ -169,14 +171,17 @@ def __repr__(self): ) def as_dict(self): - return { - "original_size": FileSize(self.osize), + d = {"original_size": FileSize(self.osize)} + if self.usize is not None: # unknown for an existing archive, see Archive.calc_stats() + d["deduplicated_size"] = FileSize(self.usize) + d |= { "nfiles": self.nfiles, "hashing_time": self.hashing_time, "chunking_time": self.chunking_time, "files_stats": self.files_stats, "store_stats": self.store_stats, } + return d def as_raw_dict(self): return {"size": self.osize, "nfiles": self.nfiles} @@ -811,7 +816,9 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m def calc_stats(self, cache, want_unique=True): stats = Statistics() - stats.usize = 0 # this is expensive to compute + # The deduplicated size of an existing archive is unknown: computing it is expensive (see borg analyze), + # it is only known for the stats collected while creating an archive. + stats.usize = None stats.nfiles = self.metadata.nfiles stats.osize = self.metadata.size return stats diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index b18b4c24f7..4a42ff5f70 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -874,9 +874,8 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): does not go to a terminal (e.g. into a logfile), the precise format is always used. When using ``--stats``, you will get some statistics about how much data was - added - the "This Archive" deduplicated size there is most interesting as that is - how much your repository will grow. Please note that the "All archives" stats refer to - the state after creation. + added - the deduplicated size there is most interesting as that is how much your + repository will grow. ``--json`` outputs the same statistics as JSON. When ``--stats`` is used together with ``--dry-run``, only the number of files and the original size are reported. They are computed from file system metadata, without reading diff --git a/src/borg/archiver/info_cmd.py b/src/borg/archiver/info_cmd.py index 5e901b2975..4a44b8841c 100644 --- a/src/borg/archiver/info_cmd.py +++ b/src/borg/archiver/info_cmd.py @@ -72,8 +72,9 @@ def build_parser_info(self, subparsers, common_parser, mid_common_parser): (uncompressed, counting duplicate content per occurrence). Deduplicated sizes are not shown here (computing them per archive is expensive). - For the deduplicated size of a set of archives, use ``borg analyze``; for the - repository-wide deduplicated size, use ``borg compact --stats``. + ``borg create --stats`` (also with ``--json``) reports the deduplicated size of the + archive it has just created. For the deduplicated size of a set of archives, use + ``borg analyze``; for the repository-wide deduplicated size, use ``borg compact --stats``. """ ) subparser = ArgumentParser(parents=[common_parser], description=self.do_info.__doc__, epilog=info_epilog) diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py index b62c6e8a95..7eb2138d19 100644 --- a/src/borg/testsuite/archive_test.py +++ b/src/borg/testsuite/archive_test.py @@ -161,6 +161,7 @@ def test_stats_progress_json(stats): assert result["finished"] is False assert result["path"] == "foo" assert result["original_size"] == 20 + assert result["deduplicated_size"] == 20 assert result["nfiles"] == 1 out = StringIO() @@ -174,6 +175,17 @@ def test_stats_progress_json(stats): assert "nfiles" not in result +def test_stats_as_dict(stats): + # stats collected while creating an archive know the deduplicated size + result = stats.as_dict() + assert result["original_size"] == 20 + assert result["deduplicated_size"] == 20 + assert result["nfiles"] == 1 + # the deduplicated size of an existing archive is unknown, so it is not reported (see Archive.calc_stats) + stats.usize = None + assert "deduplicated_size" not in stats.as_dict() + + @pytest.mark.parametrize( "isoformat, expected", [ diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 23718713e5..af3118b822 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -977,6 +977,21 @@ def test_create_json(archivers, request): assert "stats" in archive +def test_create_json_deduplicated_size(archivers, request): + """create --json reports the deduplicated size of the new archive, see #10335.""" + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=os.urandom(1024 * 80)) + cmd(archiver, "repo-create", RK_ENCRYPTION) + stats = json.loads(cmd(archiver, "create", "--json", "test", "input"))["archive"]["stats"] + # fresh repository: all of the file content was new to the repository. + assert 1024 * 80 <= stats["deduplicated_size"] <= stats["original_size"] + # same, unchanged input again: the file content gets deduplicated against the first archive, + # only the new archive's metadata gets added to the repository. + stats = json.loads(cmd(archiver, "create", "--json", "test", "input"))["archive"]["stats"] + assert 1024 * 80 <= stats["original_size"] + assert 0 < stats["deduplicated_size"] < 1024 * 80 + + def test_hostname_and_username_override(archivers, request, monkeypatch): archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=1024 * 80) diff --git a/src/borg/testsuite/archiver/info_cmd_test.py b/src/borg/testsuite/archiver/info_cmd_test.py index 71b34f3085..42aeb8f80c 100644 --- a/src/borg/testsuite/archiver/info_cmd_test.py +++ b/src/borg/testsuite/archiver/info_cmd_test.py @@ -35,6 +35,8 @@ def test_info_json(archivers, request): assert len(archive["id"]) == 64 assert archive["tags"] == [] assert "stats" in archive + # unknown for an existing archive (expensive to compute), so it is not reported, see #10335 + assert "deduplicated_size" not in archive["stats"] checkts(archive["start"]) checkts(archive["end"]) diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py index 4e5d29bc4d..497fd96cf2 100644 --- a/src/borg/testsuite/archiver/tar_cmds_test.py +++ b/src/borg/testsuite/archiver/tar_cmds_test.py @@ -189,6 +189,21 @@ def test_import_tar_nfiles(archivers, request): assert info["archives"][0]["stats"]["nfiles"] == 3 +def test_import_tar_json(archivers, request): + """import-tar --json reports the stats of the new archive like create --json does, see #10335.""" + archiver = request.getfixturevalue(archivers) + data = os.urandom(1024 * 80) + with tarfile.open("input.tar", "w") as tar: + tarinfo = tarfile.TarInfo("dir/file1") + tarinfo.size = len(data) + tar.addfile(tarinfo, io.BytesIO(data)) + cmd(archiver, "repo-create", "--encryption=none-sha256") + stats = json.loads(cmd(archiver, "import-tar", "--json", "dst", "input.tar"))["archive"]["stats"] + assert stats["nfiles"] == 1 + # fresh repository: all of the file content was new to the repository. + assert len(data) <= stats["deduplicated_size"] <= stats["original_size"] + + def tar_item_digests(archiver, archive): """{path: item.digests} as STORED in the items of an archive, see create_cmd_test.item_digests""" archive_obj, repository = open_archive(archiver.repository_path, archive)