[Feature] Export snapshots without a running chain.

CHANGELOG.md

@@ -31,6 +31,8 @@
 
 - [#3166](https://github.com/ChainSafe/forest/issues/3166): Add
   `forest-cli archive info` command for inspecting archives.
+- [#3159](https://github.com/ChainSafe/forest/issues/3159): Add
+  `forest-cli archive export -e=X` command for exporting archives.
 
 ### Changed
 

src/chain/store/chain_store.rs

@@ -56,6 +56,10 @@ const SINK_CAP: usize = 200;
 
 const DEFAULT_TIPSET_CACHE_SIZE: NonZeroUsize = nonzero!(8192usize);
 
+/// Disambiguate the type to signify that we are expecting a delta and not an actual epoch/height
+/// while maintaining the same type.
+pub type ChainEpochDelta = ChainEpoch;
+
 /// `Enum` for `pubsub` channel that defines message type variant and data
 /// contained in message type.
 #[derive(Clone, Debug)]

src/cli/subcommands/archive_cmd.rs

@@ -28,18 +28,28 @@
 //!     <https://github.com/ChainSafe/forest/blob/main/documentation/src/developer_documentation/filecoin_archive.md>
 //!     [`CarBackedBlockstore`]
 
-use super::Config;
 use crate::blocks::{Tipset, TipsetKeys};
 use crate::car_backed_blockstore::CarBackedBlockstore;
+use crate::chain::{ChainEpochDelta, ChainStore};
+use crate::cli_shared::snapshot;
+use crate::cli_shared::snapshot::TrustedVendor;
+use crate::genesis::read_genesis_header;
 use crate::networks::{calibnet, mainnet};
 use crate::shim::clock::ChainEpoch;
+use crate::Config;
 use anyhow::{bail, Context as _};
+use chrono::Utc;
 use clap::Subcommand;
 use fvm_ipld_blockstore::Blockstore;
 use indicatif::ProgressIterator;
 use itertools::Itertools;
+use sha2::Sha256;
 use std::io::{Read, Seek};
 use std::path::PathBuf;
+use std::sync::Arc;
+use tempfile::TempDir;
+use tokio_util::compat::TokioAsyncReadCompatExt;
+use tracing::info;
 
 #[derive(Debug, Subcommand)]
 pub enum ArchiveCommands {
@@ -48,19 +58,134 @@ pub enum ArchiveCommands {
         /// Path to an uncompressed archive (CAR)
         snapshot: PathBuf,
     },
+    /// Trim a snapshot of the chain and write it to `<output_path>`
+    Export {
+        /// Snapshot input path. Currently supports only `.car` file format.
+        #[arg(index = 1)]
+        input_path: PathBuf,
+        /// Snapshot output filename or directory. Defaults to
+        /// `./forest_snapshot_{chain}_{year}-{month}-{day}_height_{epoch}.car.zst`.
+        #[arg(short, default_value = ".", verbatim_doc_comment)]
+        output_path: PathBuf,
+        /// Latest epoch that has to be exported for this snapshot, the upper bound. This value
+        /// cannot be greater than the latest epoch available in the input snapshot.
+        #[arg(short)]
+        epoch: ChainEpoch,
+        /// How far back we want to go. Think of it as `$epoch - $depth`, the lower bound of this
+        /// snapshot. This value cannot be less than `chain finality`, which is currently assumed
+        /// to be `900`. If this ever changes - the actual value is specified in the error message
+        /// that is thrown in case `depth` value is too low.
+        /// This parameter is optional due to the fact that we need to fetch the exact default
+        /// dynamically from configuration.
+        // Potentially replace with dynamic default: https://github.com/ChainSafe/forest/issues/3182
+        #[arg(short)]
+        depth: Option<ChainEpochDelta>,
+    },
 }
 
 impl ArchiveCommands {
-    pub async fn run(self, _config: Config) -> anyhow::Result<()> {
+    pub async fn run(self, config: Config) -> anyhow::Result<()> {
         match self {
             Self::Info { snapshot } => {
                 println!("{}", ArchiveInfo::from_file(snapshot)?);
                 Ok(())
             }
+            Self::Export {
+                input_path,
+                output_path,
+                epoch,
+                depth,
+            } => {
+                let chain_finality = config.chain.policy.chain_finality;
+                let depth = depth.unwrap_or(chain_finality);
+                if depth < chain_finality {
+                    bail!("depth has to be at least {}", chain_finality);
+                }
+
+                let reader = std::fs::File::open(&input_path)?;
+
+                info!(
+                    "indexing a car-backed store using snapshot: {}",
+                    input_path.to_str().unwrap_or_default()
+                );
+
+                do_export(config, reader, output_path, epoch, depth).await
+            }
         }
     }
 }
 
+// This does nothing if the output path is a file. If it is a directory - it produces the following:
+// `./forest_snapshot_{chain}_{year}-{month}-{day}_height_{epoch}.car.zst`.
+fn build_output_path(chain: String, epoch: ChainEpoch, output_path: PathBuf) -> PathBuf {
+    match output_path.is_dir() {
+        true => output_path.join(snapshot::filename(
+            TrustedVendor::Forest,
+            chain,
+            Utc::now().date_naive(),
+            epoch,
+        )),
+        false => output_path.clone(),
+    }
+}
+
+async fn do_export(
+    config: Config,
+    reader: impl Read + Seek + Send + Sync,
+    output_path: PathBuf,
+    epoch: ChainEpoch,
+    depth: ChainEpochDelta,
+) -> anyhow::Result<()> {
+    let store = Arc::new(
+        CarBackedBlockstore::new(reader)
+            .context("couldn't read input CAR file - it's either compressed or corrupt")?,
+    );
+
+    let genesis = read_genesis_header(
+        config.client.genesis_file.as_ref(),
+        config.chain.genesis_bytes(),
+        &store,
+    )
+    .await?;
+
+    let tmp_chain_dir = TempDir::new()?;
+
+    let chain_store = Arc::new(ChainStore::new(
+        store,
+        config.chain.clone(),
+        &genesis,
+        tmp_chain_dir.path(),
+    )?);
+
+    let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?;
+
+    info!("looking up a tipset by epoch: {}", epoch);
+
+    let ts = chain_store
+        .tipset_by_height(epoch, ts, true)
+        .context("unable to get a tipset at given height")?;
+
+    let output_path = build_output_path(config.chain.network.to_string(), epoch, output_path);
+
+    let writer = tokio::fs::File::create(&output_path)
+        .await
+        .context(format!(
+            "unable to create a snapshot - is the output path '{}' correct?",
+            output_path.to_str().unwrap_or_default()
+        ))?;
+
+    info!(
+        "exporting snapshot at location: {}",
+        output_path.to_str().unwrap_or_default()
+    );
+
+    chain_store
+        .export::<_, Sha256>(&ts, depth, writer.compat(), true, true)
+        .await?;
+
+    Ok(())
+}
+
 #[derive(Debug)]
 struct ArchiveInfo {
     variant: String,
@@ -187,22 +312,51 @@ impl ArchiveInfo {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use async_compression::tokio::bufread::ZstdDecoder;
+    use fvm_ipld_car::CarReader;
+    use tokio::io::BufReader;
 
     #[test]
     fn archive_info_calibnet() {
         let info =
             ArchiveInfo::from_reader_with(std::io::Cursor::new(calibnet::DEFAULT_GENESIS), false)
                 .unwrap();
-        assert!(info.network == "calibnet");
-        assert!(info.epoch == 0);
+        assert_eq!(info.network, "calibnet");
+        assert_eq!(info.epoch, 0);
     }
 
     #[test]
     fn archive_info_mainnet() {
         let info =
             ArchiveInfo::from_reader_with(std::io::Cursor::new(mainnet::DEFAULT_GENESIS), false)
                 .unwrap();
-        assert!(info.network == "mainnet");
-        assert!(info.epoch == 0);
+        assert_eq!(info.network, "mainnet");
+        assert_eq!(info.epoch, 0);
+    }
+
+    #[tokio::test]
+    async fn export() {
+        let config = Config::default();
+        let output_path = TempDir::new().unwrap();
+        do_export(
+            config.clone(),
+            std::io::Cursor::new(calibnet::DEFAULT_GENESIS),
+            output_path.path().into(),
+            0,
+            1,
+        )
+        .await
+        .unwrap();
+        let file = tokio::fs::File::open(build_output_path(
+            config.chain.network.to_string(),
+            0,
+            output_path.path().into(),
+        ))
+        .await
+        .unwrap();
+        let file = BufReader::new(file);
+        CarReader::new(ZstdDecoder::new(file).compat())
+            .await
+            .unwrap();
     }
 }