From a8b5c3aaf5801867ac227e67120e76f7bb2dfd57 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Tue, 11 Jul 2023 16:02:33 +0200 Subject: [PATCH 01/14] [Feature] Export snapshots without a running chain. --- src/cli/main.rs | 1 + src/cli/subcommands/archive_cmd.rs | 113 +++++++++++++++++++++++++++++ src/cli/subcommands/mod.rs | 6 ++ 3 files changed, 120 insertions(+) create mode 100644 src/cli/subcommands/archive_cmd.rs diff --git a/src/cli/main.rs b/src/cli/main.rs index 8ef98dcc8153..1b8c7a4da13d 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -63,6 +63,7 @@ where Subcommand::Info(cmd) => cmd.run(config, opts).await, Subcommand::DB(cmd) => cmd.run(&config).await, Subcommand::Snapshot(cmd) => cmd.run(config).await, + Subcommand::Archive(cmd) => cmd.run(config).await, Subcommand::Attach(cmd) => cmd.run(config), Subcommand::Shutdown(cmd) => cmd.run(config).await, } diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs new file mode 100644 index 000000000000..3b9ab0d48432 --- /dev/null +++ b/src/cli/subcommands/archive_cmd.rs @@ -0,0 +1,113 @@ +use crate::blocks::TipsetKeys; +use crate::car_backed_blockstore::CarBackedBlockstore; +use crate::chain::ChainStore; +use crate::cli_shared::snapshot; +use crate::cli_shared::snapshot::TrustedVendor; +use crate::genesis::read_genesis_header; +use crate::shim::clock::ChainEpoch; +use crate::Config; +use anyhow::{bail, Context}; +use chrono::Utc; +use clap::Subcommand; +use sha2::Sha256; +use std::path::PathBuf; +use std::sync::Arc; +use tempfile::TempDir; +use tokio_util::compat::TokioAsyncReadCompatExt; + +#[derive(Debug, Subcommand)] +pub enum ArchiveCommands { + /// Trim a snapshot of the chain and write it to `` + Export { + /// Snapshot input path. Currently supports only `.car` file format. + #[arg(index = 1)] + input_path: PathBuf, + /// Snapshot output filename or directory. Defaults to + /// `./forest_snapshot_{chain}_{year}-{month}-{day}_height_{epoch}.car.zst`. + #[arg(short, default_value = ".", verbatim_doc_comment)] + output_path: PathBuf, + /// Latest epoch that has to be exported for this snapshot, the upper bound. This value + /// cannot be greater than the latest epoch available in the input snapshot. + #[arg(short)] + epoch: ChainEpoch, + /// How far back we want to go. Think of it as `$epoch - $depth`, the lower bound of this + /// snapshot. This value cannot be less than `chain finality`, which is currently assumed + /// to be `900`. This parameter is optional due to the fact that we need to fetch the exact + /// default dynamically from a config. + // TODO: Investigate if we can have a dynamic default here somehow. + #[arg(short)] + depth: Option, + }, +} + +impl ArchiveCommands { + pub async fn run(&self, config: Config) -> anyhow::Result<()> { + match self { + ArchiveCommands::Export { + input_path, + output_path, + depth, + epoch, + } => { + let chain_finality = config.chain.policy.chain_finality; + let depth = depth.unwrap_or(chain_finality); + if depth < chain_finality { + bail!("depth has to be at least {}", chain_finality); + } + + let store = Arc::new( + CarBackedBlockstore::new(std::fs::File::open(input_path)?).context( + "couldn't read input CAR file - it's either compressed or corrupt", + )?, + ); + + let genesis = read_genesis_header( + config.client.genesis_file.as_ref(), + config.chain.genesis_bytes(), + &store, + ) + .await?; + + let chain_store = Arc::new(ChainStore::new( + store, + config.chain.clone(), + &genesis, + TempDir::new()?.path(), + )?); + + // TODO: This is totally unnecessary. It should be possible to do `tipset_by_height` without this step. + // One solution to this is making `ts` an `Option` in `tipset_by_height` method. + let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?; + + let ts = chain_store + .tipset_by_height(*epoch, ts, true) + .context("unable to get a tipset at given height")?; + + // The lower bound of this snapshot. + let recent_roots = epoch - depth; + + let output_path = match output_path.is_dir() { + true => output_path.join(snapshot::filename( + TrustedVendor::Forest, + config.chain.network.to_string(), + Utc::now().date_naive(), + *epoch, + )), + false => output_path.clone(), + }; + + let writer = tokio::fs::File::create(&output_path) + .await + .context(format!( + "unable to create the snapshot - is the output path '{}' correct?", + output_path.to_str().unwrap_or_default() + ))?; + + chain_store + .export::<_, Sha256>(&ts, recent_roots, writer.compat(), true, false) + .await?; + } + } + Ok(()) + } +} diff --git a/src/cli/subcommands/mod.rs b/src/cli/subcommands/mod.rs index 1733d6c0a9a9..280f581bd634 100644 --- a/src/cli/subcommands/mod.rs +++ b/src/cli/subcommands/mod.rs @@ -6,6 +6,7 @@ // check out the original commit history here: // https://github.com/ChainSafe/forest/commits/main/forest/src/cli/mod.rs +mod archive_cmd; mod attach_cmd; mod auth_cmd; mod chain_cmd; @@ -25,6 +26,7 @@ mod wallet_cmd; use std::io::{self, Write}; use crate::blocks::tipset_json::TipsetJson; +use crate::cli::subcommands::archive_cmd::ArchiveCommands; pub(crate) use crate::cli_shared::cli::Config; use crate::cli_shared::cli::{CliOpts, HELP_MESSAGE}; use crate::utils::version::FOREST_VERSION_STRING; @@ -108,6 +110,10 @@ pub enum Subcommand { #[command(subcommand)] DB(DBCommands), + /// Snapshot management that does not require a running node + #[command(subcommand)] + Archive(ArchiveCommands), + /// Attach to daemon via a JavaScript console Attach(AttachCommand), From d6d5bcf99679c267e486ba41f7ba7c1e6321747b Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Tue, 11 Jul 2023 16:23:44 +0200 Subject: [PATCH 02/14] refactor export into it's own method --- src/cli/subcommands/archive_cmd.rs | 106 ++++++++++++++++------------- 1 file changed, 58 insertions(+), 48 deletions(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 3b9ab0d48432..e3ac9f95ac63 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -46,8 +46,8 @@ impl ArchiveCommands { ArchiveCommands::Export { input_path, output_path, - depth, epoch, + depth, } => { let chain_finality = config.chain.policy.chain_finality; let depth = depth.unwrap_or(chain_finality); @@ -55,59 +55,69 @@ impl ArchiveCommands { bail!("depth has to be at least {}", chain_finality); } - let store = Arc::new( - CarBackedBlockstore::new(std::fs::File::open(input_path)?).context( - "couldn't read input CAR file - it's either compressed or corrupt", - )?, - ); + do_export(config, input_path, output_path, epoch, &depth).await + } + } + } +} - let genesis = read_genesis_header( - config.client.genesis_file.as_ref(), - config.chain.genesis_bytes(), - &store, - ) - .await?; +async fn do_export( + config: Config, + input_path: &PathBuf, + output_path: &PathBuf, + epoch: &ChainEpoch, + depth: &ChainEpoch, +) -> anyhow::Result<()> { + let store = Arc::new( + CarBackedBlockstore::new(std::fs::File::open(input_path)?) + .context("couldn't read input CAR file - it's either compressed or corrupt")?, + ); - let chain_store = Arc::new(ChainStore::new( - store, - config.chain.clone(), - &genesis, - TempDir::new()?.path(), - )?); + let genesis = read_genesis_header( + config.client.genesis_file.as_ref(), + config.chain.genesis_bytes(), + &store, + ) + .await?; - // TODO: This is totally unnecessary. It should be possible to do `tipset_by_height` without this step. - // One solution to this is making `ts` an `Option` in `tipset_by_height` method. - let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?; + let chain_store = Arc::new(ChainStore::new( + store, + config.chain.clone(), + &genesis, + TempDir::new()?.path(), + )?); - let ts = chain_store - .tipset_by_height(*epoch, ts, true) - .context("unable to get a tipset at given height")?; + // TODO: This is totally unnecessary. It should be possible to do `tipset_by_height` without this step. + // One solution to this is making `ts` an `Option` in `tipset_by_height` method. + let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?; - // The lower bound of this snapshot. - let recent_roots = epoch - depth; + let ts = chain_store + .tipset_by_height(*epoch, ts, true) + .context("unable to get a tipset at given height")?; - let output_path = match output_path.is_dir() { - true => output_path.join(snapshot::filename( - TrustedVendor::Forest, - config.chain.network.to_string(), - Utc::now().date_naive(), - *epoch, - )), - false => output_path.clone(), - }; + // The lower bound of this snapshot. + let recent_roots = epoch - depth; - let writer = tokio::fs::File::create(&output_path) - .await - .context(format!( - "unable to create the snapshot - is the output path '{}' correct?", - output_path.to_str().unwrap_or_default() - ))?; + let output_path = match output_path.is_dir() { + true => output_path.join(snapshot::filename( + TrustedVendor::Forest, + config.chain.network.to_string(), + Utc::now().date_naive(), + *epoch, + )), + false => output_path.clone(), + }; - chain_store - .export::<_, Sha256>(&ts, recent_roots, writer.compat(), true, false) - .await?; - } - } - Ok(()) - } + let writer = tokio::fs::File::create(&output_path) + .await + .context(format!( + "unable to create the snapshot - is the output path '{}' correct?", + output_path.to_str().unwrap_or_default() + ))?; + + chain_store + .export::<_, Sha256>(&ts, recent_roots, writer.compat(), true, false) + .await?; + + Ok(()) } From 389b50c53c917d8c24033250e4207a46ac8f44db Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 12:20:20 +0200 Subject: [PATCH 03/14] make sure that temp dir is not cleaned up during the export --- src/cli/subcommands/archive_cmd.rs | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index e3ac9f95ac63..81fc1791c2b2 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -14,6 +14,7 @@ use std::path::PathBuf; use std::sync::Arc; use tempfile::TempDir; use tokio_util::compat::TokioAsyncReadCompatExt; +use tracing::info; #[derive(Debug, Subcommand)] pub enum ArchiveCommands { @@ -32,8 +33,10 @@ pub enum ArchiveCommands { epoch: ChainEpoch, /// How far back we want to go. Think of it as `$epoch - $depth`, the lower bound of this /// snapshot. This value cannot be less than `chain finality`, which is currently assumed - /// to be `900`. This parameter is optional due to the fact that we need to fetch the exact - /// default dynamically from a config. + /// to be `900`. If this ever changes - the actual value is specified in the error message + /// that is thrown in case `depth` value is too low. + /// This parameter is optional due to the fact that we need to fetch the exact default + /// dynamically from a config. // TODO: Investigate if we can have a dynamic default here somehow. #[arg(short)] depth: Option, @@ -68,6 +71,11 @@ async fn do_export( epoch: &ChainEpoch, depth: &ChainEpoch, ) -> anyhow::Result<()> { + info!( + "indexing a car-backed store using snapshot: {}", + input_path.to_str().unwrap_or_default() + ); + let store = Arc::new( CarBackedBlockstore::new(std::fs::File::open(input_path)?) .context("couldn't read input CAR file - it's either compressed or corrupt")?, @@ -80,24 +88,25 @@ async fn do_export( ) .await?; + let tmp_chain_dir = TempDir::new()?; + let chain_store = Arc::new(ChainStore::new( store, config.chain.clone(), &genesis, - TempDir::new()?.path(), + tmp_chain_dir.path(), )?); // TODO: This is totally unnecessary. It should be possible to do `tipset_by_height` without this step. // One solution to this is making `ts` an `Option` in `tipset_by_height` method. let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?; + info!("looking up a tipset by epoch: {}", epoch); + let ts = chain_store .tipset_by_height(*epoch, ts, true) .context("unable to get a tipset at given height")?; - // The lower bound of this snapshot. - let recent_roots = epoch - depth; - let output_path = match output_path.is_dir() { true => output_path.join(snapshot::filename( TrustedVendor::Forest, @@ -111,12 +120,17 @@ async fn do_export( let writer = tokio::fs::File::create(&output_path) .await .context(format!( - "unable to create the snapshot - is the output path '{}' correct?", + "unable to create a snapshot - is the output path '{}' correct?", output_path.to_str().unwrap_or_default() ))?; + info!( + "exporting snapshot at location: {}", + output_path.to_str().unwrap_or_default() + ); + chain_store - .export::<_, Sha256>(&ts, recent_roots, writer.compat(), true, false) + .export::<_, Sha256>(&ts, *depth, writer.compat(), true, true) .await?; Ok(()) From 1351a02d64ead6d1b0b4c20a0adef6f056f431e9 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 12:42:29 +0200 Subject: [PATCH 04/14] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 785f18f6311e..5055fcbe4419 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ - [#3166](https://github.com/ChainSafe/forest/issues/3166): Add `forest-cli archive info` command for inspecting archives. +- [#3159](https://github.com/ChainSafe/forest/issues/3159): Add + `forest-cli archive snapshot --epoch=X` command for exporting archives. ### Changed From 4bcd7e083c990c84a27b9ea64607b7e1683f5362 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 12:56:27 +0200 Subject: [PATCH 05/14] Update src/cli/subcommands/archive_cmd.rs Co-authored-by: David Himmelstrup --- src/cli/subcommands/archive_cmd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 6bec5209bb0a..54cffd377a22 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -143,7 +143,7 @@ async fn do_export( // TODO: This is totally unnecessary. It should be possible to do `tipset_by_height` without this step. // One solution to this is making `ts` an `Option` in `tipset_by_height` method. - let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?; + let ts = Tipset::load(&store, &TipsetKeys::new(chain_store.db.roots()))?; info!("looking up a tipset by epoch: {}", epoch); From d893f72672f4dff224bc13d7d6be24ba66f39ef7 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 13:05:04 +0200 Subject: [PATCH 06/14] remove TODO, revert to using 'tipset_from_keys' --- src/cli/subcommands/archive_cmd.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 54cffd377a22..5d6de4e85903 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -141,9 +141,7 @@ async fn do_export( tmp_chain_dir.path(), )?); - // TODO: This is totally unnecessary. It should be possible to do `tipset_by_height` without this step. - // One solution to this is making `ts` an `Option` in `tipset_by_height` method. - let ts = Tipset::load(&store, &TipsetKeys::new(chain_store.db.roots()))?; + let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?; info!("looking up a tipset by epoch: {}", epoch); From b3736e536425184e885255a9a722c1713d48e775 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 13:46:14 +0200 Subject: [PATCH 07/14] fix spelling --- src/cli/subcommands/archive_cmd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 5d6de4e85903..6dbb6119042e 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -76,7 +76,7 @@ pub enum ArchiveCommands { /// to be `900`. If this ever changes - the actual value is specified in the error message /// that is thrown in case `depth` value is too low. /// This parameter is optional due to the fact that we need to fetch the exact default - /// dynamically from a config. + /// dynamically from configuration. // TODO: Investigate if we can have a dynamic default here somehow. #[arg(short)] depth: Option, From 718503f8f92da6398c6a61993f7adfb8de65689e Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 13:58:27 +0200 Subject: [PATCH 08/14] fix depth type --- src/chain/store/chain_store.rs | 4 ++++ src/cli/subcommands/archive_cmd.rs | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/chain/store/chain_store.rs b/src/chain/store/chain_store.rs index dad9e0ae065a..43a3a1c7eb67 100644 --- a/src/chain/store/chain_store.rs +++ b/src/chain/store/chain_store.rs @@ -56,6 +56,10 @@ const SINK_CAP: usize = 200; const DEFAULT_TIPSET_CACHE_SIZE: NonZeroUsize = nonzero!(8192usize); +/// Disambiguate the type to signify that we are expecting a delta and not an actual epoch/height +/// while maintaining the same type. +pub type ChainEpochDelta = ChainEpoch; + /// `Enum` for `pubsub` channel that defines message type variant and data /// contained in message type. #[derive(Clone, Debug)] diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 6dbb6119042e..188d437d521c 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -30,7 +30,7 @@ use crate::blocks::{Tipset, TipsetKeys}; use crate::car_backed_blockstore::CarBackedBlockstore; -use crate::chain::ChainStore; +use crate::chain::{ChainEpochDelta, ChainStore}; use crate::cli_shared::snapshot; use crate::cli_shared::snapshot::TrustedVendor; use crate::genesis::read_genesis_header; @@ -79,7 +79,7 @@ pub enum ArchiveCommands { /// dynamically from configuration. // TODO: Investigate if we can have a dynamic default here somehow. #[arg(short)] - depth: Option, + depth: Option, }, } @@ -113,7 +113,7 @@ async fn do_export( input_path: PathBuf, output_path: PathBuf, epoch: ChainEpoch, - depth: ChainEpoch, + depth: ChainEpochDelta, ) -> anyhow::Result<()> { info!( "indexing a car-backed store using snapshot: {}", From cfc004288feab83e44867a1a82c70b42c3b504b4 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 14:56:07 +0200 Subject: [PATCH 09/14] assert -> assert_eq --- src/cli/subcommands/archive_cmd.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 188d437d521c..909338ac8eed 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -310,8 +310,8 @@ mod tests { let info = ArchiveInfo::from_reader_with(std::io::Cursor::new(calibnet::DEFAULT_GENESIS), false) .unwrap(); - assert!(info.network == "calibnet"); - assert!(info.epoch == 0); + assert_eq!(info.network, "calibnet"); + assert_eq!(info.epoch, 0); } #[test] @@ -319,7 +319,7 @@ mod tests { let info = ArchiveInfo::from_reader_with(std::io::Cursor::new(mainnet::DEFAULT_GENESIS), false) .unwrap(); - assert!(info.network == "mainnet"); - assert!(info.epoch == 0); + assert_eq!(info.network, "mainnet"); + assert_eq!(info.epoch, 0); } } From bd2dd1d61973e7713ef163574e831c22c5535422 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 17:17:42 +0200 Subject: [PATCH 10/14] add a simple test --- src/cli/subcommands/archive_cmd.rs | 40 ++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 909338ac8eed..e314a99bdd7e 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -102,7 +102,14 @@ impl ArchiveCommands { bail!("depth has to be at least {}", chain_finality); } - do_export(config, input_path, output_path, epoch, depth).await + let reader = std::fs::File::open(&input_path)?; + + info!( + "indexing a car-backed store using snapshot: {}", + input_path.to_str().unwrap_or_default() + ); + + do_export(config, reader, output_path, epoch, depth).await } } } @@ -110,18 +117,13 @@ impl ArchiveCommands { async fn do_export( config: Config, - input_path: PathBuf, + reader: impl Read + Seek + Send + Sync, output_path: PathBuf, epoch: ChainEpoch, depth: ChainEpochDelta, ) -> anyhow::Result<()> { - info!( - "indexing a car-backed store using snapshot: {}", - input_path.to_str().unwrap_or_default() - ); - let store = Arc::new( - CarBackedBlockstore::new(std::fs::File::open(input_path)?) + CarBackedBlockstore::new(reader) .context("couldn't read input CAR file - it's either compressed or corrupt")?, ); @@ -304,6 +306,9 @@ impl ArchiveInfo { #[cfg(test)] mod tests { use super::*; + use async_compression::tokio::bufread::ZstdDecoder; + use fvm_ipld_car::CarReader; + use tokio::io::BufReader; #[test] fn archive_info_calibnet() { @@ -322,4 +327,23 @@ mod tests { assert_eq!(info.network, "mainnet"); assert_eq!(info.epoch, 0); } + + #[tokio::test] + async fn export() { + let output_path = TempDir::new().unwrap(); + do_export( + Config::default(), + std::io::Cursor::new(calibnet::DEFAULT_GENESIS), + output_path.path().into(), + 0, + 1, + ) + .await + .unwrap(); + let file = tokio::fs::File::open(output_path.path()).await.unwrap(); + let file = BufReader::new(file); + CarReader::new(ZstdDecoder::new(file).compat()) + .await + .unwrap(); + } } From a9894377228958ae5b83aea62db192ed18015257 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 17:20:52 +0200 Subject: [PATCH 11/14] fix CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5055fcbe4419..58d57906fa9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ - [#3166](https://github.com/ChainSafe/forest/issues/3166): Add `forest-cli archive info` command for inspecting archives. - [#3159](https://github.com/ChainSafe/forest/issues/3159): Add - `forest-cli archive snapshot --epoch=X` command for exporting archives. + `forest-cli archive export -=X` command for exporting archives. ### Changed From f0dd5fcfcb8d49adc1a2f2c2072b3e262e33148c Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 17:22:22 +0200 Subject: [PATCH 12/14] fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58d57906fa9d..2f4fb68376ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ - [#3166](https://github.com/ChainSafe/forest/issues/3166): Add `forest-cli archive info` command for inspecting archives. - [#3159](https://github.com/ChainSafe/forest/issues/3159): Add - `forest-cli archive export -=X` command for exporting archives. + `forest-cli archive export -e=X` command for exporting archives. ### Changed From 697358090b755fa1343e2e3f16bc0cfa16d3ef4f Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 17:41:23 +0200 Subject: [PATCH 13/14] more test fixes --- src/cli/subcommands/archive_cmd.rs | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index e314a99bdd7e..8cdd77931422 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -115,6 +115,20 @@ impl ArchiveCommands { } } +// This does nothing if the output path is a file. If it is a directory - it produces the following: +// `./forest_snapshot_{chain}_{year}-{month}-{day}_height_{epoch}.car.zst`. +fn build_output_path(chain: String, epoch: ChainEpoch, output_path: PathBuf) -> PathBuf { + match output_path.is_dir() { + true => output_path.join(snapshot::filename( + TrustedVendor::Forest, + chain, + Utc::now().date_naive(), + epoch, + )), + false => output_path.clone(), + } +} + async fn do_export( config: Config, reader: impl Read + Seek + Send + Sync, @@ -151,15 +165,7 @@ async fn do_export( .tipset_by_height(epoch, ts, true) .context("unable to get a tipset at given height")?; - let output_path = match output_path.is_dir() { - true => output_path.join(snapshot::filename( - TrustedVendor::Forest, - config.chain.network.to_string(), - Utc::now().date_naive(), - epoch, - )), - false => output_path.clone(), - }; + let output_path = build_output_path(config.chain.network.to_string(), epoch, output_path); let writer = tokio::fs::File::create(&output_path) .await @@ -330,9 +336,10 @@ mod tests { #[tokio::test] async fn export() { + let config = Config::default(); let output_path = TempDir::new().unwrap(); do_export( - Config::default(), + config.clone(), std::io::Cursor::new(calibnet::DEFAULT_GENESIS), output_path.path().into(), 0, @@ -340,7 +347,13 @@ mod tests { ) .await .unwrap(); - let file = tokio::fs::File::open(output_path.path()).await.unwrap(); + let file = tokio::fs::File::open(build_output_path( + config.chain.network.to_string(), + 0, + output_path.path().into(), + )) + .await + .unwrap(); let file = BufReader::new(file); CarReader::new(ZstdDecoder::new(file).compat()) .await From 1c9f543d3adfc81633c8762b2c7e6c9e7c4e4eb1 Mon Sep 17 00:00:00 2001 From: Roman Useinov Date: Wed, 12 Jul 2023 22:06:23 +0200 Subject: [PATCH 14/14] TODO -> issue --- src/cli/subcommands/archive_cmd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/subcommands/archive_cmd.rs b/src/cli/subcommands/archive_cmd.rs index 8cdd77931422..ff235796e84f 100644 --- a/src/cli/subcommands/archive_cmd.rs +++ b/src/cli/subcommands/archive_cmd.rs @@ -77,7 +77,7 @@ pub enum ArchiveCommands { /// that is thrown in case `depth` value is too low. /// This parameter is optional due to the fact that we need to fetch the exact default /// dynamically from configuration. - // TODO: Investigate if we can have a dynamic default here somehow. + // Potentially replace with dynamic default: https://github.com/ChainSafe/forest/issues/3182 #[arg(short)] depth: Option, },