From 2115900545d207e3df7a4eda986794a14d7ac6a0 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sat, 28 Feb 2026 15:24:46 +0100 Subject: [PATCH 1/2] feat: make backup generic over a ReadSource; add openDAL ReadSource --- crates/backend/src/opendal.rs | 152 ++++++++++++++++++++++- crates/core/src/backend.rs | 5 + crates/core/src/commands/backup.rs | 125 +++++++++++-------- crates/core/src/lib.rs | 2 + crates/core/src/repofile/snapshotfile.rs | 11 +- crates/core/src/repository.rs | 35 +++++- 6 files changed, 270 insertions(+), 60 deletions(-) diff --git a/crates/backend/src/opendal.rs b/crates/backend/src/opendal.rs index e3dd11e99..bb8b14f1d 100644 --- a/crates/backend/src/opendal.rs +++ b/crates/backend/src/opendal.rs @@ -1,12 +1,18 @@ /// `OpenDAL` backend for rustic. -use std::{collections::BTreeMap, str::FromStr, sync::OnceLock}; +use std::{ + collections::BTreeMap, + ffi::OsStr, + str::FromStr, + sync::{Arc, OnceLock}, + vec::IntoIter, +}; use bytes::Bytes; use bytesize::ByteSize; -use log::{error, trace}; +use log::{error, trace, warn}; use opendal::{ - Metadata, - blocking::Operator, + Entry, Metadata, + blocking::{Operator, StdReader}, layers::{ConcurrentLimitLayer, LoggingLayer, RetryLayer, ThrottleLayer}, options::{ListOptions, ReadOptions}, }; @@ -15,7 +21,9 @@ use tokio::runtime::Runtime; use typed_path::UnixPathBuf; use rustic_core::{ - ALL_FILE_TYPES, ErrorKind, FileType, Id, ReadBackend, RusticError, RusticResult, WriteBackend, + ALL_FILE_TYPES, ErrorKind, FileType, Id, ReadBackend, ReadSource, ReadSourceEntry, + ReadSourceOpen, RusticError, RusticResult, WriteBackend, + repofile::{Node, NodeType}, }; mod constants { @@ -198,6 +206,43 @@ impl OpenDALBackend { } .to_string() } + + /// Turn this `OpenDALBackend into a ReadSource` + /// + /// # Errors + /// If listing fails + pub fn as_source(self) -> RusticResult { + let list_options = ListOptions { + recursive: true, + ..Default::default() + }; + // openDAL lister may entries in random order; hence we collect and sort them here. + // This also allows to handle listing errors directly + let mut entries: Vec<_> = self + .operator + .lister_options("", list_options) + .map_err(|err| { + RusticError::with_source(ErrorKind::Backend, "Error listong openDAL source.", err) + })? + .filter_map(|entry| { + // Ignore not needed root path + if let Ok(e) = &entry + && e.path() == "/" + { + return None; + } + entry + .inspect_err(|err| warn!("ignoring error on openDAL entry: {err}")) + .ok() + }) + .collect(); + entries.sort_unstable_by(|e1, e2| e1.path().cmp(e2.path())); + + Ok(OpenDALReadSource { + entries, + be: Arc::new(self), + }) + } } impl ReadBackend for OpenDALBackend { @@ -572,3 +617,100 @@ mod tests { Ok(()) } } + +#[derive(Debug)] +/// Describes an open file from the local backend. +pub struct OpenFile(Arc, String); + +impl ReadSourceOpen for OpenFile { + type Reader = StdReader; + + /// Open the file from the local backend. + /// + /// # Returns + /// + /// The read handle to the file from the local backend. + /// + /// # Errors + /// + /// * If the file could not be opened. + fn open(self) -> RusticResult { + let path = self.1; + + let reader = || self.0.operator.reader(&path)?.into_std_read(..); + + let reader = reader() + .map_err(|err| { + RusticError::with_source( + ErrorKind::InputOutput, + "Failed to open file at `{path}`. Please make sure the file exists and is accessible.", + err, + ) + .attach_context("path", path) + })?; + Ok(reader) + } +} + +// Walk doesn't implement Debug +#[allow(missing_debug_implementations)] +/// A Lister for a `OpenDALSource` +pub struct OpenDALLister(IntoIter, Arc); + +impl Iterator for OpenDALLister { + type Item = RusticResult>; + + fn next(&mut self) -> Option { + Ok(self.0.next().map(|e| { + let path = e.path(); + // strip "/" suffix from dirs + let path = path.strip_suffix('/').unwrap_or(path); + let name = OsStr::new(e.name()); + let metadata = e.metadata(); + let node_type = if metadata.is_dir() { + NodeType::Dir + } else { + NodeType::File + }; + let meta = rustic_core::repofile::Metadata { + mtime: metadata + .last_modified() + .map(opendal::raw::Timestamp::into_inner), + size: metadata.content_length(), + ..Default::default() + }; + let node = Node::new_node(name, node_type, meta); + let open = Some(OpenFile(self.1.clone(), path.to_string())); + ReadSourceEntry { + path: path.into(), + node, + open, + } + })) + .transpose() + } +} + +#[allow(missing_debug_implementations)] +/// A source to backup using openDAL for the access +pub struct OpenDALReadSource { + entries: Vec, + be: Arc, +} + +impl ReadSource for OpenDALReadSource { + type Open = OpenFile; + type Iter = OpenDALLister; + /// Returns the size of the source. + fn size(&self) -> RusticResult> { + let size = self + .entries + .iter() + .map(|e| e.metadata().content_length()) + .sum(); + Ok(Some(size)) + } + fn entries(&self) -> Self::Iter { + OpenDALLister(self.entries.clone().into_iter(), self.be.clone()) + } +} diff --git a/crates/core/src/backend.rs b/crates/core/src/backend.rs index 7720414a3..b59618ac3 100644 --- a/crates/core/src/backend.rs +++ b/crates/core/src/backend.rs @@ -431,6 +431,11 @@ impl ReadSourceEntry { ); Ok(Self { path, node, open }) } + + /// Turn this into a tree entry + pub fn as_tree_entry(self) -> (PathBuf, Node) { + (self.path, self.node) + } } /// Trait for backends that can read and open sources. diff --git a/crates/core/src/commands/backup.rs b/crates/core/src/commands/backup.rs index 663585c03..0237fa87e 100644 --- a/crates/core/src/commands/backup.rs +++ b/crates/core/src/commands/backup.rs @@ -10,7 +10,7 @@ use serde_derive::{Deserialize, Serialize}; use serde_with::{DisplayFromStr, serde_as}; use crate::{ - CommandInput, Excludes, + CommandInput, Excludes, ReadSource, archiver::{Archiver, parent::Parent}, backend::{ childstdout::ChildStdoutSource, @@ -90,19 +90,17 @@ impl ParentOptions { /// /// * `repo` - The repository to use /// * `snap` - The snapshot to use - /// * `backup_stdin` - Whether the backup is from stdin /// /// # Returns /// - /// The parent snapshot id and the parent object or `None` if no parent is used. + /// The parent snapshot ids and the parent object. pub(crate) fn get_parent( &self, repo: &Repository, snap: &SnapshotFile, - backup_stdin: bool, ) -> (Vec, Parent) { let group = SnapshotGroup::from_snapshot(snap, self.group_by.unwrap_or_default()); - let parent = if backup_stdin || self.force { + let parent = if self.force { Vec::new() } else if self.parents.is_empty() { // get suitable snapshot group from snapshot and opts.group_by. This is used to filter snapshots for the parent detection @@ -210,8 +208,8 @@ pub struct BackupOptions { /// /// * `repo` - The repository to use /// * `opts` - The backup options -/// * `source` - The source to backup -/// * `snap` - The snapshot to backup +/// * `src` - The source to backup +/// * `snap` - The snapshot with raw information /// /// # Errors /// @@ -224,22 +222,21 @@ pub struct BackupOptions { /// # Returns /// /// The snapshot pointing to the backup'ed data. -#[allow(clippy::too_many_lines)] -pub(crate) fn backup( +pub(crate) fn archive( repo: &Repository, opts: &BackupOptions, - source: &PathList, + src: &R, mut snap: SnapshotFile, -) -> RusticResult { + backup_paths: &[PathBuf], +) -> RusticResult +where + S: IndexedIds, + R: ReadSource + 'static, + ::Open: Send, + ::Iter: Send, +{ let index = repo.index(); - let backup_stdin = *source == PathList::from_string("-")?; - let backup_path = if backup_stdin { - vec![PathBuf::from(&opts.stdin_filename)] - } else { - source.paths() - }; - let as_path = opts .as_path .as_ref() @@ -257,10 +254,9 @@ pub(crate) fn backup( }) .transpose()?; - let paths = match &as_path { - Some(p) => std::slice::from_ref(p), - None => &backup_path, - }; + let paths = as_path + .as_ref() + .map_or(backup_paths, |p| std::slice::from_ref(p)); snap.paths.set_paths(paths).map_err(|err| { RusticError::with_source( @@ -270,14 +266,14 @@ pub(crate) fn backup( ) .attach_context( "paths", - backup_path + backup_paths .iter() .map(|p| p.display().to_string()) .join(","), ) })?; - let (parent_ids, parent) = opts.parent_opts.get_parent(repo, &snap, backup_stdin); + let (parent_ids, parent) = opts.parent_opts.get_parent(repo, &snap); if parent_ids.is_empty() { info!("using no parent"); } else { @@ -287,50 +283,73 @@ pub(crate) fn backup( } let be = DryRunBackend::new(repo.dbe().clone(), opts.dry_run); - info!("starting to backup {source} ..."); + info!("starting to backup {backup_paths:?} ..."); let archiver = Archiver::new(be, index, repo.config(), parent, snap)?; let p = repo.progress_bytes("backing up..."); - let snap = if backup_stdin { - let path = &backup_path[0]; + archiver.archive( + src, + &backup_paths[0], + as_path.as_ref(), + opts.parent_opts.skip_if_unchanged, + opts.no_scan, + &p, + ) +} + +/// Backup data, create a snapshot. +/// +/// # Type Parameters +/// +/// * `S` - The type of the indexed tree. +/// +/// # Arguments +/// +/// * `repo` - The repository to use +/// * `opts` - The backup options +/// * `source` - The source to backup +/// * `snap` - The snapshot with raw information +/// +/// # Errors +/// +/// * If sending the message to the raw packer fails. +/// * If converting the data length to u64 fails +/// * If sending the message to the raw packer fails. +/// * If the index file could not be serialized. +/// * If the time is not in the range of `Local::now()` +/// +/// # Returns +/// +/// The snapshot pointing to the backup'ed data. +pub(crate) fn backup( + repo: &Repository, + opts: &BackupOptions, + source: &PathList, + snap: SnapshotFile, +) -> RusticResult { + let backup_stdin = PathList::from_string("-")?; + + let snap = if *source == backup_stdin { + let path = PathBuf::from(&opts.stdin_filename); + let backup_paths = vec![path.clone()]; if let Some(command) = &opts.stdin_command { - let src = ChildStdoutSource::new(command, path.clone())?; - let res = archiver.archive( - &src, - path, - as_path.as_ref(), - opts.parent_opts.skip_if_unchanged, - opts.no_scan, - &p, - )?; + let src = ChildStdoutSource::new(command, path)?; + let res = archive(repo, opts, &src, snap, &backup_paths)?; src.finish()?; res } else { - let src = StdinSource::new(path.clone()); - archiver.archive( - &src, - path, - as_path.as_ref(), - opts.parent_opts.skip_if_unchanged, - opts.no_scan, - &p, - )? + let src = StdinSource::new(path); + archive(repo, opts, &src, snap, &backup_paths)? } } else { + let backup_path = source.paths(); let src = LocalSource::new( opts.ignore_save_opts, &opts.excludes, &opts.ignore_filter_opts, &backup_path, )?; - archiver.archive( - &src, - &backup_path[0], - as_path.as_ref(), - opts.parent_opts.skip_if_unchanged, - opts.no_scan, - &p, - )? + archive(repo, opts, &src, snap, &backup_path)? }; Ok(snap) diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 6eef68d88..d6c02edc1 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -132,6 +132,7 @@ pub use crate::{ backend::{ ALL_FILE_TYPES, FileType, ReadBackend, ReadSource, ReadSourceEntry, ReadSourceOpen, RepositoryBackends, WriteBackend, + childstdout::ChildStdoutSource, decrypt::{compression_level_range, max_compression_level}, ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions}, local_destination::LocalDestination, @@ -141,6 +142,7 @@ pub use crate::{ BlockdevOption, DevIdOption, NodeModification, TimeOption, XattrOption, }, }, + stdin::StdinSource, }, blob::{ BlobId, DataId, PackedId, diff --git a/crates/core/src/repofile/snapshotfile.rs b/crates/core/src/repofile/snapshotfile.rs index 3d90e27d7..ac4016daa 100644 --- a/crates/core/src/repofile/snapshotfile.rs +++ b/crates/core/src/repofile/snapshotfile.rs @@ -7,6 +7,7 @@ use std::{ cmp::Ordering, collections::{BTreeMap, BTreeSet}, fmt::{self, Display}, + ops::Index, path::{Path, PathBuf}, str::FromStr, }; @@ -1198,6 +1199,14 @@ impl> FromIterator for PathList { } } +impl Index for PathList { + type Output = PathBuf; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + impl PathList { /// Create a `PathList` from a String containing a single path /// Note: for multiple paths, use `PathList::from_iter`. @@ -1228,7 +1237,7 @@ impl PathList { /// Clone the internal `Vec`. #[must_use] - pub(crate) fn paths(&self) -> Vec { + pub fn paths(&self) -> Vec { self.0.clone() } diff --git a/crates/core/src/repository.rs b/crates/core/src/repository.rs index f4ad8a463..66a09ffe9 100644 --- a/crates/core/src/repository.rs +++ b/crates/core/src/repository.rs @@ -19,7 +19,7 @@ use log::info; use serde_with::{DisplayFromStr, serde_as}; use crate::{ - RepositoryBackends, RusticError, + ReadSource, RepositoryBackends, RusticError, backend::{ FileType, FindInBackend, ReadBackend, WriteBackend, cache::{Cache, CachedBackend}, @@ -1687,6 +1687,39 @@ impl Repository { ) -> RusticResult { commands::backup::backup(self, opts, source, snap) } + + /// Run a backup of `source` using a `ReadSource`. + /// + /// You have to give a preflled [`SnapshotFile`] which is modified and saved. + /// + /// # Arguments + /// + /// * `opts` - The options to use + /// * `src` - The source to backup + /// * `snap` - The snapshot to modify and save + /// + /// # Errors + /// + // TODO: Document errors + /// + /// # Returns + /// + /// The saved snapshot. + pub fn archive( + &self, + opts: &BackupOptions, + src: &R, + snap: SnapshotFile, + backup_paths: &[PathBuf], + ) -> RusticResult + where + S: IndexedIds, + R: ReadSource + 'static, + ::Open: Send, + ::Iter: Send, + { + commands::backup::archive(self, opts, src, snap, backup_paths) + } } impl Repository { From 5a8d6fba450870d9d17a8299f51d477126a46c4a Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Mon, 1 Jun 2026 15:48:01 +0200 Subject: [PATCH 2/2] fix clippy lints --- crates/core/src/backend/childstdout.rs | 9 +++++++++ crates/core/src/backend/stdin.rs | 1 + 2 files changed, 10 insertions(+) diff --git a/crates/core/src/backend/childstdout.rs b/crates/core/src/backend/childstdout.rs index ef38b5874..e2327747a 100644 --- a/crates/core/src/backend/childstdout.rs +++ b/crates/core/src/backend/childstdout.rs @@ -30,6 +30,9 @@ pub struct ChildStdoutSource { impl ChildStdoutSource { /// Creates a new `ChildSource`. + /// + /// # Errors + /// - if calling the command fails pub fn new(cmd: &CommandInput, path: PathBuf) -> RusticResult { let process = Command::new(cmd.command()) .args(cmd.args()) @@ -51,6 +54,12 @@ impl ChildStdoutSource { } /// Finishes the `ChildSource` + /// + /// # Errors + /// - if handling of the return code leads to an error + /// + /// # Panics + /// - if the lock for the process cannot be obtained (should not happen) pub fn finish(self) -> RusticResult<()> { let status = self.process.lock().unwrap().wait(); self.command diff --git a/crates/core/src/backend/stdin.rs b/crates/core/src/backend/stdin.rs index 3027bfe90..281ba8a77 100644 --- a/crates/core/src/backend/stdin.rs +++ b/crates/core/src/backend/stdin.rs @@ -18,6 +18,7 @@ pub struct StdinSource { impl StdinSource { /// Creates a new `StdinSource`. + #[must_use] pub const fn new(path: PathBuf) -> Self { Self { path } }