diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index 0fbdbbf15e..a2224d6942 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -39,6 +39,9 @@ of their content. The store hash is the unkeyed 256 bit BLAKE3 hash, see config/ config the repository config (see :ref:`repo_config`), a text object + defaults + the repository defaults (see :ref:`repo_defaults`), in the key's store object + envelope (see below). Only present if ``borg repo-create`` was given a default. space-reserve.N purely random binary data to reserve space, e.g. for disk-full emergencies. These objects are created and removed by ``borg repo-space``. @@ -110,8 +113,8 @@ locks/ .. _store_object_envelope: -The index fragments, the lock objects, ``checked-packs`` and the -``referenced-by-archive.*`` objects are stored in the **store object envelope**: the repository key's ``encrypt()``, +The index fragments, the lock objects, ``checked-packs``, the +``referenced-by-archive.*`` objects and ``config/defaults`` are stored in the **store object envelope**: the repository key's ``encrypt()``, exactly as for the metadata and data slots of the objects in a pack (see :ref:`security_encryption`), with an empty id and an AAD of ``b"borg-store-object\0"`` followed by the repository id, the tag ``b"n"`` and the @@ -129,7 +132,8 @@ packs; any other command that needs the chunks index aborts, except ``borg compa and ``borg repo-compress``, which rebuild it from the packs, as they rewrite the whole chunks index anyway (under an exclusive lock). A corrupted cache is ignored and rebuilt. A lock object that fails the authentication is treated as a foreign exclusive -lock, see :ref:`storelocking`. The ``chunkindex-invalid`` marker has no content and is +lock, see :ref:`storelocking`. Commands that use ``config/defaults`` abort if it fails +the authentication. The ``chunkindex-invalid`` marker has no content and is stored as is. @@ -308,6 +312,36 @@ a ``keys/`` object, see :ref:`key_files`), and the encryption mode and id hash are what ``borg repo-create`` was given (the key type byte of any repository object encodes them as well, see ``KeyType`` in ``constants.py``). +.. _repo_defaults: + +Repository defaults +~~~~~~~~~~~~~~~~~~~ + +The ``config/defaults`` store object holds default values for command options, +as a msgpacked dict mapping the option name to its value, both as strings:: + + {"compression": "zstd,3", "chunker_params": "fastcdc,19,23,21,2"} + +*compression* is the default compression spec (see ``borg help compression``), +set by ``borg repo-create --compression``. The commands with a ``--compression`` +option use it if no compression was given via the command line, the environment or +the default config file; without it, they use lz4. + +*chunker_params* are the default :ref:`chunker-params `, set by +``borg repo-create --chunker-params``. ``borg create`` and ``borg import-tar`` use +them if no chunker params were given (in the same ways as above), ``borg recreate`` +and ``borg transfer`` for ``--chunker-params default``; without them, the built-in +default chunker params are used. + +Each entry is optional, a missing entry means that the repository has no default +for it. + +Unlike the repository config, which borg must read before it knows the key, the +defaults are stored in the :ref:`store object envelope `, +so they are authenticated: an attacker with write access to the storage can not +change them (e.g. remove an ``obfuscate`` compression) without being noticed. A +repository without the object has no defaults. + .. _archive: Archives diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 686155c5d5..ad0f45a2e5 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -345,7 +345,14 @@ path Path to the local repository cache :ref:`borg_repo-info` additionally emits a *security_dir* key with the path of the local security -directory of the repository. +directory of the repository and a *defaults* key with an object containing: + +compression + The compression spec the commands use if no compression is given: the repository default (see + :ref:`borg_repo-create` ``--compression``), else ``lz4`` +chunker_params + The chunker params the commands use if no chunker params are given: the repository default (see + :ref:`borg_repo-create` ``--chunker-params``), else the built-in default .. highlight: json @@ -355,6 +362,10 @@ Example ``borg repo-info --json`` output:: "cache": { "path": "/home/user/.cache/borg/65d7898e2142485f44506fb11c0fcd6d7dfd0341716385246068584a62632a94" }, + "defaults": { + "chunker_params": "fastcdc,19,23,21,2", + "compression": "lz4" + }, "encryption": { "encryption": "aes256-ocb", "id_hash": "sha256" diff --git a/docs/quickstart.rst b/docs/quickstart.rst index d561a0474b..c1f7d4036e 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -352,6 +352,14 @@ specified algorithm:: You'll need to experiment a bit to find the best compression for your use case. Keep an eye on CPU load and throughput. +Instead of giving ``--compression`` to every command, you can set a default compression +for the repository when creating it:: + + $ borg repo-create --encryption aes256-ocb --compression zstd,3 + +Commands that compress data use this default when you do not give ``--compression``. +``borg repo-create --chunker-params`` sets the default chunker parameters in the same way. + .. _encrypted_repos: Repository encryption diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 8d84c6d2c5..d1c1994ab9 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -10,8 +10,9 @@ from ..helpers import CommandError, Error from ..helpers import SortBySpec, location_validator, Location, relative_time_marker_validator from ..helpers import FilesystemPathSpec +from ..helpers import ChunkerParams, CompressionSpec from ..helpers import Highlander, octal_int -from ..helpers.argparsing import SUPPRESS, PositiveInt +from ..helpers.argparsing import SUPPRESS, ArgumentTypeError, PositiveInt from ..helpers.nanorst import rst_to_terminal from ..manifest import Manifest, AI_HUMAN_SORT_KEYS from ..patterns import PatternMatcher @@ -80,6 +81,37 @@ def get_repository( return repository +def _repository_default(repository, name, parse, builtin): + """Return the repository default for option name (see Repository.save_defaults), parsed, else builtin.""" + value = repository.load_defaults().get(name) if isinstance(repository, Repository) else None + if value is None: + return builtin + try: + return parse(value) + except (ArgumentTypeError, ValueError) as err: + raise Repository.InvalidRepositoryConfig( + repository._location.canonical_path(), f"invalid default {name} {value!r}: {err}" + ) from None + + +def default_compression(repository): + """Return the CompressionSpec a command uses if --compression was not given. + + That is the repository default (set by "borg repo-create --compression"), else lz4. + An explicitly configured compression (command line, environment, default.yaml) always wins. + """ + return _repository_default(repository, "compression", CompressionSpec, CompressionSpec(BUILTIN_COMPRESSION)) + + +def default_chunker_params(repository): + """Return the chunker params for "--chunker-params default" (the default of create and import-tar). + + That is the repository default (set by "borg repo-create --chunker-params"), else CHUNKER_PARAMS. + Explicitly configured chunker params (command line, environment, default.yaml) always win. + """ + return _repository_default(repository, "chunker_params", ChunkerParams, CHUNKER_PARAMS) + + def with_repository( create=False, lock=True, @@ -152,7 +184,11 @@ def wrapper(self, args, **kwargs): manifest_ = Manifest.load(repository, other=False, ro_cls=ro_cls) kwargs["manifest"] = manifest_ if "compression" in args: + if args.compression is None: # not given, see default_compression() + args.compression = default_compression(repository) manifest_.repo_objs.compressor = args.compression.compressor + if "chunker_params" in args and args.chunker_params == DEFAULT_CHUNKER_PARAMS: + args.chunker_params = default_chunker_params(repository) if secure: assert_secure(repository, manifest_) if cache: diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 6285106f1e..472d373ef7 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -1333,10 +1333,11 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): metavar="PARAMS", dest="chunker_params", type=ChunkerParams, - default=CHUNKER_PARAMS, + default=DEFAULT_CHUNKER_PARAMS, # see default_chunker_params() action=Highlander, help="specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, " - "HASH_MASK_BITS, NC_LEVEL). default: %s,%d,%d,%d,%d" % CHUNKER_PARAMS, + "HASH_MASK_BITS, NC_LEVEL). default: the repository default (see borg repo-create), " + "else %s,%d,%d,%d,%d" % CHUNKER_PARAMS, ) archive_group.add_argument( "-C", @@ -1344,9 +1345,11 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=None, # None: not given, see default_compression() action=Highlander, - help="select compression algorithm, see the output of the " '"borg help compression" command for details.', + help="select compression algorithm, see the output of the " + '"borg help compression" command for details. ' + "Default: the repository default (see borg repo-create), else lz4.", ) archive_group.add_argument( "--tag", diff --git a/src/borg/archiver/debug_cmd.py b/src/borg/archiver/debug_cmd.py index 4379a584d2..18c3dd779b 100644 --- a/src/borg/archiver/debug_cmd.py +++ b/src/borg/archiver/debug_cmd.py @@ -462,9 +462,11 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=None, # None: not given, see default_compression() action=Highlander, - help="select compression algorithm, see the output of the " '"borg help compression" command for details.', + help="select compression algorithm, see the output of the " + '"borg help compression" command for details. ' + "Default: the repository default (see borg repo-create), else lz4.", ) subparser.add_argument( "object_path", diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py index 74584b6f8a..9d6d011538 100644 --- a/src/borg/archiver/help_cmd.py +++ b/src/borg/archiver/help_cmd.py @@ -487,7 +487,9 @@ class HelpMixIn: So if you use different compression specs for the backups, whichever stores a chunk first determines its compression. See also ``borg recreate``. - Compression is lz4 by default. If you want something else, you have to specify what you want. + If you do not specify a compression via ``--compression`` (or the environment or the + default config file), the repository's default compression is used, which can be set + with ``borg repo-create --compression``. Without a repository default, compression is lz4. Valid compression specifiers are: @@ -495,7 +497,7 @@ class HelpMixIn: Do not compress. lz4 - Use lz4 compression. Very high speed, very low compression. (default) + Use lz4 compression. Very high speed, very low compression. (built-in default) zstd[,L] Use zstd ("zstandard") compression, a modern wide-range algorithm. diff --git a/src/borg/archiver/recreate_cmd.py b/src/borg/archiver/recreate_cmd.py index 57d641b85a..7d67131282 100644 --- a/src/borg/archiver/recreate_cmd.py +++ b/src/borg/archiver/recreate_cmd.py @@ -159,13 +159,14 @@ def build_parser_recreate(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=None, # None: not given, see default_compression() action=Highlander, help="select compression algorithm, see the output of the " '"borg help compression" command for details. ' "Only applies to newly written data, e.g. when re-chunking with --chunker-params " "(and to the new archive metadata); data chunks reused from the existing archive " - "are not recompressed, use borg repo-compress for that.", + "are not recompressed, use borg repo-compress for that. " + "Default: the repository default (see borg repo-create), else lz4.", ) archive_group.add_argument( "--chunker-params", @@ -178,7 +179,8 @@ def build_parser_recreate(self, subparsers, common_parser, mid_common_parser): "buzhash,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,WINDOW_SIZE or " "buzhash64,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,WINDOW_SIZE,NC_LEVEL or " "fastcdc,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,NC_LEVEL or " - "`default` to use the chunker defaults. default: do not rechunk", + "`default` to use the repository default (see borg repo-create), else the built-in " + "chunker defaults. default: do not rechunk", ) subparser.add_argument( diff --git a/src/borg/archiver/repo_compress_cmd.py b/src/borg/archiver/repo_compress_cmd.py index 23b331dc2a..4acc75ab6a 100644 --- a/src/borg/archiver/repo_compress_cmd.py +++ b/src/borg/archiver/repo_compress_cmd.py @@ -223,7 +223,9 @@ def build_parser_repo_compress(self, subparsers, common_parser, mid_common_parse Repository (re-)compression (and/or re-obfuscation). Reads all repository objects and recompresses the ones that are not already using - the compression type/level and obfuscation level given via ``--compression``. + the compression type/level and obfuscation level given via ``--compression``. Without + ``--compression``, that is the repository's default compression (see ``borg repo-create``), + else lz4. The repository is processed one pack file at a time: a pack is read as a whole and, if it holds objects that need recompression, rewritten as a whole - objects already @@ -267,9 +269,10 @@ def build_parser_repo_compress(self, subparsers, common_parser, mid_common_parse metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=None, # None: not given, see default_compression() action=Highlander, - help='select compression algorithm, see the output of the "borg help compression" command for details.', + help='select compression algorithm, see the output of the "borg help compression" command for details. ' + "Default: the repository default (see borg repo-create), else lz4.", ) subparser.add_argument("-s", "--stats", dest="stats", action="store_true", help="print statistics") diff --git a/src/borg/archiver/repo_create_cmd.py b/src/borg/archiver/repo_create_cmd.py index 5ab8caf77c..58d083557d 100644 --- a/src/borg/archiver/repo_create_cmd.py +++ b/src/borg/archiver/repo_create_cmd.py @@ -3,7 +3,7 @@ from ..constants import * # NOQA from ..crypto.key import key_creator, encryption_argument_names, id_hash_argument_names from ..helpers import CancelledByUser -from ..helpers import location_validator, Location +from ..helpers import location_validator, Location, ChunkerParams, CompressionSpec from ..hashindex import ChunkIndex from ..helpers.argparsing import ArgumentParser from ..manifest import Manifest @@ -41,6 +41,13 @@ def do_repo_create(self, args, repository, *, other_repository=None, other_manif repository.acquire_lock() # writing the config is what makes the store a repository, see Repository.create(). repository.save_config(key) + defaults = {} + if args.compression is not None: + defaults["compression"] = str(args.compression) + if args.chunker_params not in (None, DEFAULT_CHUNKER_PARAMS): + defaults["chunker_params"] = ",".join(str(p) for p in args.chunker_params) + if defaults: + repository.save_defaults(defaults) # we know repo/packs/ still does not have any chunks stored in it, but for some stores, there # might be a lot of empty directories and listing them all might be rather slow, so we better # store an empty ChunkIndex now, so that the first repo operation does not have to build the @@ -191,6 +198,29 @@ def build_parser_repo_create(self, subparsers, common_parser, mid_common_parser) To normally work with ``authenticated-*`` repositories, you will need the passphrase, but there is an emergency workaround; see ``BORG_WORKAROUNDS=authenticated_no_key`` docs. + Repository defaults + +++++++++++++++++++ + + ``--compression`` sets the repository's default compression: the commands that compress + data (``borg create``, ``borg recreate``, ``borg import-tar``, ``borg transfer``, + ``borg repo-compress``) use it if no compression was given via ``--compression``, the + environment or the default config file (``default.yaml``). Without a repository default, + they use lz4. See ``borg help compression`` for the compression specs. + + ``--chunker-params`` sets the repository's default chunker parameters: ``borg create`` and + ``borg import-tar`` use them if no chunker parameters were given (in the same ways as above). + ``borg recreate`` and ``borg transfer`` only rechunk if ``--chunker-params`` is given, with + ``--chunker-params default``, they rechunk to the repository's default chunker parameters. + Without a repository default, the built-in default chunker parameters are used. + + This is useful if several clients back up into the same repository or if some commands are + run manually: they all compress and chunk the same way without having to give these options. + Using the same chunker parameters is important for deduplication. + ``borg repo-info`` shows the defaults. + + The defaults are stored in the repository and protected by the repository key, so nobody without + the key can change them (e.g. remove an ``obfuscate`` compression) without being noticed. + Creating a related repository +++++++++++++++++++++++++++++ @@ -263,6 +293,27 @@ def build_parser_repo_create(self, subparsers, common_parser, mid_common_parser) help="where to store the key: 'repokey' (in the repository, default) or 'keyfile' " "(in the local keys directory).", ) + subparser.add_argument( + "-C", + "--compression", + metavar="COMPRESSION", + dest="compression", + type=CompressionSpec, + default=None, + action=Highlander, + help="set the default compression of the repository, see the output of the " + '"borg help compression" command for details. Default: no repository default (lz4 is used).', + ) + subparser.add_argument( + "--chunker-params", + metavar="PARAMS", + dest="chunker_params", + type=ChunkerParams, + default=None, + action=Highlander, + help="set the default chunker parameters of the repository (same format as for borg create). " + "Default: no repository default (%s,%d,%d,%d,%d is used)." % CHUNKER_PARAMS, + ) subparser.add_argument( "--copy-crypt-key", dest="copy_crypt_key", diff --git a/src/borg/archiver/repo_info_cmd.py b/src/borg/archiver/repo_info_cmd.py index 07c1402633..9b7b997039 100644 --- a/src/borg/archiver/repo_info_cmd.py +++ b/src/borg/archiver/repo_info_cmd.py @@ -1,6 +1,6 @@ import textwrap -from ._common import with_repository +from ._common import with_repository, default_compression, default_chunker_params from ..constants import * # NOQA from ..helpers import bin_to_hex, json_print, basic_json_data from ..helpers.argparsing import ArgumentParser @@ -16,6 +16,10 @@ def do_repo_info(self, args, repository, manifest, cache): """Show repository information.""" key = manifest.key info = basic_json_data(manifest, cache=cache, extra={"security_dir": cache.security_manager.dir}) + info["defaults"] = { + "compression": str(default_compression(repository)), + "chunker_params": ",".join(str(p) for p in default_chunker_params(repository)), + } if args.json: json_print(info) @@ -40,6 +44,10 @@ def do_repo_info(self, args, repository, manifest, cache): if storage == KeyBlobStorage.KEYFILE: encryption += "\nKey file: %s" % key.find_key() info["encryption"] = encryption + defaults = dict(info["defaults"]) + for name in defaults: + if name not in repository.load_defaults(): + defaults[name] += " (built-in)" output = ( textwrap.dedent( @@ -49,6 +57,8 @@ def do_repo_info(self, args, repository, manifest, cache): Repository version: {version} {encryption} Security directory: {security_dir} + Default compression: {compression} + Default chunker params: {chunker_params} """ ) .strip() @@ -58,6 +68,8 @@ def do_repo_info(self, args, repository, manifest, cache): version=repository.version, encryption=info["encryption"], security_dir=info["security_dir"], + compression=defaults["compression"], + chunker_params=defaults["chunker_params"], ) ) diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index c9f138a522..daf73e86ab 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -849,14 +849,14 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): "--chunker-params", dest="chunker_params", type=ChunkerParams, - default=CHUNKER_PARAMS, + default=DEFAULT_CHUNKER_PARAMS, # see default_chunker_params() action=Highlander, metavar="PARAMS", help="specify the chunker parameters: " "buzhash,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,WINDOW_SIZE or " "buzhash64,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,WINDOW_SIZE,NC_LEVEL or " "fastcdc,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,NC_LEVEL. " - "default: %s,%d,%d,%d,%d" % CHUNKER_PARAMS, + "default: the repository default (see borg repo-create), else %s,%d,%d,%d,%d" % CHUNKER_PARAMS, ) archive_group.add_argument( "-C", @@ -864,9 +864,11 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=None, # None: not given, see default_compression() action=Highlander, - help="select compression algorithm, see the output of the " '"borg help compression" command for details.', + help="select compression algorithm, see the output of the " + '"borg help compression" command for details. ' + "Default: the repository default (see borg repo-create), else lz4.", ) archive_group.add_argument( "--digests", diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index 438cb23279..56750aab61 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -405,9 +405,11 @@ def build_parser_transfer(self, subparsers, common_parser, mid_common_parser): metavar="COMPRESSION", dest="compression", type=CompressionSpec, - default=CompressionSpec("lz4"), + default=None, # None: not given, see default_compression() action=Highlander, - help="select compression algorithm, see the output of the " '"borg help compression" command for details.', + help="select compression algorithm, see the output of the " + '"borg help compression" command for details. ' + "Default: the repository default (see borg repo-create), else lz4.", ) subparser.add_argument( "--recompress", @@ -436,7 +438,8 @@ def build_parser_transfer(self, subparsers, common_parser, mid_common_parser): "buzhash,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,WINDOW_SIZE or " "buzhash64,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,WINDOW_SIZE,NC_LEVEL or " "fastcdc,CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,NC_LEVEL or " - "`default` to use the chunker defaults. default: do not rechunk", + "`default` to use the repository default (see borg repo-create), else the built-in " + "chunker defaults. default: do not rechunk", ) define_archive_filters_group(subparser) diff --git a/src/borg/constants.py b/src/borg/constants.py index ca1c5afe15..0cc22219c6 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -187,6 +187,13 @@ TOEPLITZ_AES_PARAMS = (CH_TOEPLITZ_AES, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, NC_LEVEL) CHUNKER_PARAMS = FASTCDC_PARAMS # the default chunker for file content data +# the compression used if neither --compression nor the repository default (see repo-create) gives one +BUILTIN_COMPRESSION = "lz4" + +# what ChunkerParams returns for "--chunker-params default": the repository default (see repo-create) if it +# has one, else CHUNKER_PARAMS. with_repository replaces it by the real chunker params. +DEFAULT_CHUNKER_PARAMS = ("default",) + # chunker params for the items metadata stream, finer granularity ITEMS_CHUNKER_PARAMS = (CH_FASTCDC, 15, 19, 17, NC_LEVEL) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 62d5258324..3b5ce23b0d 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -318,8 +318,8 @@ def ChunkerParams(s): if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE: raise ArgumentTypeError("block_size and header_size must not exceed MAX_DATA_SIZE [%d]" % MAX_DATA_SIZE) return algo, block_size, header_size - if algo == "default" and count == 1: # default - return CHUNKER_PARAMS + if algo == "default" and count == 1: # the repository default or CHUNKER_PARAMS, see with_repository + return DEFAULT_CHUNKER_PARAMS if algo == CH_BUZHASH64: # buzhash64, chunk_min, chunk_max, chunk_mask, window_size, nc_level # use nc_level 0 to disable normalized chunking. diff --git a/src/borg/repository.py b/src/borg/repository.py index 64db5125e4..9f4f6702f8 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -26,6 +26,7 @@ from .helpers import replace_placeholders from .helpers import sig_int from .helpers import ProgressIndicatorPercent +from .helpers import msgpack from .helpers.lrucache import LRUCache from .storelocking import Lock from .logger import create_logger @@ -52,6 +53,8 @@ # from the metadata and data slots of pack objects, whose AAD starts with OBJ_MAGIC (b"BORG_OBJ"). The # repository id, a tag and the object name (or namespace) follow it, see Repository._store_obj_aad. STORE_OBJ_AAD = b"borg-store-object\0" +# the store object with the repository defaults, e.g. the default compression, see Repository.save_defaults. +DEFAULTS_NAME = "config/defaults" def repo_lister(repository, *, limit=None): @@ -984,6 +987,7 @@ def __init__( # key_loader(repository) returns the repository's key; acquire_lock() calls it if no key was set yet. # None: key_factory(repository). self._key_loader = key_loader + self._defaults = None # cache of load_defaults() # plaintext store hash -> fragment hash of the index/ fragments read in this session, so a # fragment with the same content is not stored again, see cache._store_chunkindex_fragment. self.chunkindex_fragment_hashes = {} @@ -2458,6 +2462,44 @@ def store_load_decrypt(self, name, *, hashed_name=False): except IntegrityError as err: raise IntegrityError(f"Store object {name}: authentication failed") from err + def save_defaults(self, defaults): + """Store the repository defaults (the config/defaults store object). + + defaults: a dict mapping option names to their default values as strings, e.g. + {"compression": "zstd,3"}. The commands use such a default if the option was not given (see + with_repository). Unlike config/config, which is read before the key is known, the object is + stored in the key's envelope (see store_encrypt_store), so nobody without the key can change the + defaults (e.g. remove an "obfuscate" compression) without being noticed. + """ + self.store_encrypt_store(DEFAULTS_NAME, msgpack.packb(defaults)) + self._defaults = dict(defaults) + + def load_defaults(self): + """Return the repository defaults stored by save_defaults(), or {} if there are none. + + The store object is only read once, later calls return the same defaults. + + Raises IntegrityError if the envelope authentication fails, InvalidRepositoryConfig if the + content is not a dict of strings. + """ + if self._defaults is not None: + return dict(self._defaults) + try: + data = self.store_load_decrypt(DEFAULTS_NAME) + except StoreObjectNotFound: + self._defaults = {} + return {} + try: + defaults = msgpack.unpackb(data) + except msgpack.UnpackException: + defaults = None + if not ( + isinstance(defaults, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in defaults.items()) + ): + raise self.InvalidRepositoryConfig(self._location.canonical_path(), f"{DEFAULTS_NAME} is malformed") + self._defaults = defaults + return dict(defaults) + def store_delete(self, name, *, deleted=False): self._lock_refresh() return self.store.delete(name, deleted=deleted) diff --git a/src/borg/testsuite/archiver/repo_create_cmd_test.py b/src/borg/testsuite/archiver/repo_create_cmd_test.py index 3a8b26ef63..07d5086d06 100644 --- a/src/borg/testsuite/archiver/repo_create_cmd_test.py +++ b/src/borg/testsuite/archiver/repo_create_cmd_test.py @@ -1,3 +1,4 @@ +import json import os from unittest.mock import patch @@ -5,10 +6,13 @@ from ...archiver import repo_create_cmd from ...cache import list_chunkindex_hashes, read_chunkindex_from_repo -from ...helpers.errors import Error, CancelledByUser +from ...compress import CNONE, LZ4, ZLIB, ZSTD +from ...helpers.errors import Error, CancelledByUser, IntegrityError from ...constants import * # NOQA from ...crypto.key import FlexiKey -from . import cmd, create_src_archive, generate_archiver_tests, open_repository +from ...manifest import Manifest +from ...repository import Repository, repo_lister +from . import cmd, create_regular_file, create_src_archive, generate_archiver_tests, open_repository from . import RK_ENCRYPTION, KF_ENCRYPTION, KF_LOCATION pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA @@ -194,3 +198,138 @@ def failing_write_chunkindex_to_repo(*args, **kwargs): cmd(archiver, "repo-create", KF_ENCRYPTION, KF_LOCATION) assert not os.path.exists(archiver.repository_location) assert not os.path.exists(keys_dir) or not os.listdir(keys_dir) + + +def stored_compression(archiver): + """Return the set of (ctype, clevel) of the compressed objects in the repository. + + Objects that do not get smaller by compression are stored uncompressed, they are left out. + """ + with open_repository(archiver) as repository: + manifest = Manifest.load(repository) + result = set() + for id, _ in repo_lister(repository, limit=LIST_SCAN_LIMIT): + meta = manifest.repo_objs.parse_meta(id, repository.get(id, read_data=False), ro_type=ROBJ_DONTCARE) + if meta["ctype"] != CNONE.ID: + result.add((meta["ctype"], meta["clevel"])) + return result + + +def test_repo_create_default_compression(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + cmd(archiver, "repo-create", RK_ENCRYPTION, "--compression=zstd,5") + assert "Default compression: zstd,5\n" in cmd(archiver, "repo-info") + assert json.loads(cmd(archiver, "repo-info", "--json"))["defaults"]["compression"] == "zstd,5" + # without --compression, create uses the repository default. + cmd(archiver, "create", "test", "input") + assert stored_compression(archiver) == {(ZSTD.ID, 5)} + # --compression wins over the repository default, and without it, the repository default is used again. + cmd(archiver, "repo-compress", "--compression=lz4") + assert stored_compression(archiver) == {(LZ4.ID, 255)} + cmd(archiver, "repo-compress") + assert stored_compression(archiver) == {(ZSTD.ID, 5)} + # a compression given via the environment wins over the repository default, too. + monkeypatch.setenv("BORG_REPO_COMPRESS__COMPRESSION", "zlib,3") + cmd(archiver, "repo-compress") + assert stored_compression(archiver) == {(ZLIB.ID, 3)} + + +def test_repo_create_without_default_compression(archivers, request): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + cmd(archiver, "repo-create", RK_ENCRYPTION) + with open_repository(archiver) as repository: + assert repository.load_defaults() == {} + output = cmd(archiver, "repo-info") + assert "Default compression: lz4 (built-in)\n" in output + assert "Default chunker params: %s,%d,%d,%d,%d (built-in)\n" % CHUNKER_PARAMS in output + assert json.loads(cmd(archiver, "repo-info", "--json"))["defaults"] == { + "compression": "lz4", + "chunker_params": "%s,%d,%d,%d,%d" % CHUNKER_PARAMS, + } + cmd(archiver, "create", "test", "input") + assert stored_compression(archiver) == {(LZ4.ID, 255)} + assert archive_chunker_params(archiver, "test") == list(CHUNKER_PARAMS) + + +def test_repo_create_rejects_invalid_compression(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION, "--compression=zstd,99", exit_code=2) + assert not os.path.exists(archiver.repository_path) + + +def test_default_compression_tampered(archivers, request): + # the repository defaults are authenticated: changing them without the key is detected. + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION, "--compression=obfuscate,110,zstd,3") + with open_repository(archiver) as repository: + envelope = bytearray(repository.store_load("config/defaults")) + envelope[-1] ^= 1 + repository.store_store("config/defaults", bytes(envelope)) + if archiver.FORK_DEFAULT: + cmd(archiver, "create", "test", "input", exit_code=IntegrityError("x").exit_code) + else: + with pytest.raises(IntegrityError): + cmd(archiver, "create", "test", "input") + + +def test_default_compression_invalid(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + with open_repository(archiver) as repository: + repository.save_defaults({"compression": "nosuchcompression"}) + if archiver.FORK_DEFAULT: + exit_code = Repository.InvalidRepositoryConfig("x", "y").exit_code + cmd(archiver, "create", "test", "input", exit_code=exit_code) + else: + with pytest.raises(Repository.InvalidRepositoryConfig): + cmd(archiver, "create", "test", "input") + + +def archive_chunker_params(archiver, name): + return json.loads(cmd(archiver, "info", "--json", name))["archives"][0]["chunker_params"] + + +def test_repo_create_default_chunker_params(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024 * 80) + cmd(archiver, "repo-create", RK_ENCRYPTION, "--chunker-params=fixed,4096") + assert "Default chunker params: fixed,4096,0\n" in cmd(archiver, "repo-info") + assert json.loads(cmd(archiver, "repo-info", "--json"))["defaults"]["chunker_params"] == "fixed,4096,0" + # without --chunker-params (or with "default"), create uses the repository default. + cmd(archiver, "create", "test1", "input") + assert archive_chunker_params(archiver, "test1") == ["fixed", 4096, 0] + cmd(archiver, "create", "--chunker-params=default", "test2", "input") + assert archive_chunker_params(archiver, "test2") == ["fixed", 4096, 0] + # given chunker params win over the repository default, also when given via the environment. + cmd(archiver, "create", "--chunker-params=fixed,8192", "test3", "input") + assert archive_chunker_params(archiver, "test3") == ["fixed", 8192, 0] + monkeypatch.setenv("BORG_CREATE__CHUNKER_PARAMS", "fixed,16384") + cmd(archiver, "create", "test4", "input") + assert archive_chunker_params(archiver, "test4") == ["fixed", 16384, 0] + # recreate only rechunks if asked to, "default" rechunks to the repository default. + cmd(archiver, "recreate", "-a", "test3") + assert archive_chunker_params(archiver, "test3") == ["fixed", 8192, 0] + cmd(archiver, "recreate", "-a", "test3", "--chunker-params=default") + assert archive_chunker_params(archiver, "test3") == ["fixed", 4096, 0] + + +def test_repo_create_chunker_params_default_is_no_repository_default(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION, "--chunker-params=default") + with open_repository(archiver) as repository: + assert repository.load_defaults() == {} + + +def test_default_chunker_params_invalid(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + with open_repository(archiver) as repository: + repository.save_defaults({"chunker_params": "fixed,1"}) + if archiver.FORK_DEFAULT: + exit_code = Repository.InvalidRepositoryConfig("x", "y").exit_code + cmd(archiver, "create", "test", "input", exit_code=exit_code) + else: + with pytest.raises(Repository.InvalidRepositoryConfig): + cmd(archiver, "create", "test", "input") diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 03e7576d57..192135963a 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -876,7 +876,7 @@ def test_eval_escapes(): @pytest.mark.parametrize( "chunker_params, expected_return", [ - ("default", ("fastcdc", 19, 23, 21, 2)), + ("default", DEFAULT_CHUNKER_PARAMS), # resolved to the repository default by with_repository ("19,23,21,4095", ("buzhash", 19, 23, 21, 4095)), ("buzhash,19,23,21,4095", ("buzhash", 19, 23, 21, 4095)), ("10,23,16,4095", ("buzhash", 10, 23, 16, 4095)),