From 13fdb3182fe351fc2329244143e5d1686767fb34 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Tue, 26 May 2026 22:49:25 +0200 Subject: [PATCH] Add opendal as backup source --- config/README.md | 17 ++++-- config/full.toml | 11 +++- config/opendal-source.toml | 16 +++++ src/commands/backup.rs | 61 ++++++++++++++++--- ..._tests__default_config_display_passes.snap | 2 + ..._config__tests__default_config_passes.snap | 1 + ..._tests__global_env_roundtrip_passes-2.snap | 2 + ..._tests__global_env_roundtrip_passes-3.snap | 1 + ...g__tests__global_env_roundtrip_passes.snap | 2 + .../show_config__show_config_passes.snap | 2 + 10 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 config/opendal-source.toml diff --git a/config/README.md b/config/README.md index 7f4785471..643ffde96 100644 --- a/config/README.md +++ b/config/README.md @@ -277,15 +277,22 @@ See [Global Metrics labels](#global-metrics-labels-globalmetrics-labels). **Note**: All of the backup options mentioned before can also be used as snapshot-specific option and then only apply to this snapshot. -| Attribute | Description | Default Value | Example Value | -| --------- | ---------------------------------------------------------------------- | ------------- | ---------------------------------------------------------------------- | -| name | Name to identify this snapshot (to be used with the --name CLI option) | "" | "myid" | -| sources | Array of source directories or file(s) to back up. | [] | ["/dir1", "/dir2"] | -| hooks | Hooks to run before and after backing up the defined sources. | Not set | { run-before = [], run-after = [], run-failed = [], run-finally = [] } | +| Attribute | Description | Default Value | Example Value | +| --------- | ------------------------------------------------------------------------------------------------------------------------ | ------------- | ---------------------------------------------------------------------- | +| name | Name to identify this snapshot (to be used with the --name CLI option) | "" | "myid" | +| sources | Array of source directories or file(s) to back up. Allows "opendal:" for a remote source if only a single source is used | [] | ["/dir1", "/dir2"], ["opendal:s3"] | +| hooks | Hooks to run before and after backing up the defined sources. | Not set | { run-before = [], run-after = [], run-failed = [], run-finally = [] } | Source-specific hooks are called additionally to global, repository and backup hooks when backing up the defined sources into a snapshot. +### Snapshot Sourcey Options `[backup.snapshots.options]` + +Additional source options - depending on the used source. This is currently only +relevant for an opendal source. These can be only set in the config file. + +see [Repository Options](#repository-options-repository) + ### Forget Options `[forget]` **Note**: At least one of the `keep-*` options must be given. Use diff --git a/config/full.toml b/config/full.toml index b1ab4445c..caa44141f 100644 --- a/config/full.toml +++ b/config/full.toml @@ -172,10 +172,19 @@ label-a = "xxx" # Backup options for specific sources - all above options are also available here and replace them for the given source [[backup.snapshots]] -sources = ["/path/to/source1"] +sources = [ + "opendal:s3", +] # Note: For opendal, currently only a single source is supported; see next [[backup.snapshots]] section for a local path example label = "label" # Default: not set # .. and so on. see [backup] +# Additional backup source options - depending on source; currently only relevant for a "opendal:" source. +# These can be only set in the config file. +[backup.snapshots.options] +connections = "20" # Default: Not set +# Note that an opendal source use several service-dependent options which may be specified here, see +# https://opendal.apache.org/docs/rust/opendal/services/index.html + # Source-specific hooks: The given commands when backing up the defined source [backup.snapshots.hooks] run-before = ["echo before"] # Default: [] diff --git a/config/opendal-source.toml b/config/opendal-source.toml new file mode 100644 index 000000000..2556c98dd --- /dev/null +++ b/config/opendal-source.toml @@ -0,0 +1,16 @@ +# rustic config file to backup a s3 storage to a local repository +[repository] +repository = "/path/to/repo" +password = "password" + +[[backup.snapshots]] +# Here only a single opendal source is allowed, the used service is given here +sources = ["opendal:s3"] +host = "s3" # we manually set the hostname, so we can easily distinguish the snapshot(s) created from here + +# Other options can be given here - note that opendal also support reading config from env files or AWS config dirs, see the opendal S3 docu +[backup.snapshots.options] +access_key_id = "xxx" # this can be omitted, when AWS config is used +secret_access_key = "xxx" # this can be omitted, when AWS config is used +bucket = "bucket_name" +root = "/path/to/repo" diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 7eebffd54..6bd24b4de 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -21,7 +21,8 @@ use clap::ValueHint; use comfy_table::Cell; use conflate::{Merge, MergeFrom}; use log::{debug, error, info, warn}; -use rustic_core::{Excludes, StringList}; +use rustic_backend::OpenDALBackend; +use rustic_core::{ChildStdoutSource, Excludes, LocalSource, StdinSource, StringList}; use serde::{Deserialize, Serialize}; use serde_with::serde_as; @@ -145,6 +146,11 @@ pub struct BackupCmd { #[merge(skip)] sources: Vec, + /// Other options for this source + #[clap(skip)] + #[merge(strategy = conflate::btreemap::append_or_ignore)] + options: BTreeMap, + /// Job name for the metrics. Default: rustic-backup #[clap(long, value_name = "JOB_NAME", env = "RUSTIC_METRICS_JOB")] #[merge(strategy=conflate::option::overwrite_none)] @@ -327,6 +333,51 @@ impl BackupCmd { hooks.with_env(&hooks_variables) } + fn backup_source( + source: &PathList, + options: BTreeMap, + backup_opts: BackupOptions, + snap: SnapshotFile, + repo: &IndexedIdsRepo, + ) -> Result { + let backup_stdin = PathList::from_string("-")?; + let source = source + .clone() + .sanitize() + .with_context(|| format!("error sanitizing source=s\"{:?}\"", source))? + .merge(); + + let snap = if source.len() == 1 + // TODO: This check should not be done on PathList, but in the sources list directly + && let Some(path) = source[0].to_string_lossy().strip_prefix("opendal:") + { + let source = OpenDALBackend::new(path, options)?.as_source()?; + repo.archive(&backup_opts, &source, snap, &[PathBuf::new()])? + } else if source == backup_stdin { + let path = PathBuf::from(&backup_opts.stdin_filename); + let backup_paths = vec![path.clone()]; + if let Some(command) = &backup_opts.stdin_command { + let src = ChildStdoutSource::new(command, path)?; + let res = repo.archive(&backup_opts, &src, snap, &backup_paths)?; + src.finish()?; + res + } else { + let src = StdinSource::new(path); + repo.archive(&backup_opts, &src, snap, &backup_paths)? + } + } else { + let backup_path = source.paths(); + let src = LocalSource::new( + backup_opts.ignore_save_opts, + &backup_opts.excludes, + &backup_opts.ignore_filter_opts, + &backup_path, + )?; + repo.archive(&backup_opts, &src, snap, &backup_path)? + }; + Ok(snap) + } + fn backup_snapshot(mut self, source: PathList, repo: &IndexedIdsRepo) -> Result<()> { let config = RUSTIC_APP.config(); let snapshot_opts = &config.backup.snapshots; @@ -369,13 +420,9 @@ impl BackupCmd { .no_scan(self.no_scan) .dry_run(config.global.dry_run); + let snap = self.snap_opts.to_snapshot()?; let snap = hooks.use_with(|| -> Result<_> { - let source = source - .clone() - .sanitize() - .with_context(|| format!("error sanitizing source=s\"{:?}\"", source))? - .merge(); - Ok(repo.backup(&backup_opts, &source, self.snap_opts.to_snapshot()?)?) + Self::backup_source(&source, self.options, backup_opts, snap, repo) })?; if config.global.progress_options.json_progress { diff --git a/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap b/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap index a110300ab..9c521210e 100644 --- a/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap @@ -78,6 +78,8 @@ run-after = [] run-failed = [] run-finally = [] +[backup.options] + [backup.metrics-labels] [copy] diff --git a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap index 5ce6b9bc6..6024135ea 100644 --- a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap @@ -171,6 +171,7 @@ RusticConfig { }, snapshots: [], sources: [], + options: {}, metrics_job: None, metrics_labels: {}, }, diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap index 2f44f627f..339016f3b 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap @@ -88,6 +88,8 @@ run-after = [] run-failed = [] run-finally = [] +[backup.options] + [backup.metrics-labels] [copy] diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap index a83536c47..0b5832086 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap @@ -182,6 +182,7 @@ RusticConfig { }, snapshots: [], sources: [], + options: {}, metrics_job: None, metrics_labels: {}, }, diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap index 230704c10..577ec10b6 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap @@ -88,6 +88,8 @@ run-after = [] run-failed = [] run-finally = [] +[backup.options] + [backup.metrics-labels] [copy] diff --git a/tests/snapshots/show_config__show_config_passes.snap b/tests/snapshots/show_config__show_config_passes.snap index 5281284e8..845668984 100644 --- a/tests/snapshots/show_config__show_config_passes.snap +++ b/tests/snapshots/show_config__show_config_passes.snap @@ -78,6 +78,8 @@ run-after = [] run-failed = [] run-finally = [] +[backup.options] + [backup.metrics-labels] [copy]