From 484bd48b359928ade3ff66135adbcca139ba6f74 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Sat, 9 Jan 2021 18:29:09 +0000 Subject: [PATCH] Various snapshot-related code clean up (bp #14487) (#14513) * Create account paths once (cherry picked from commit fe0ba4a429287672612797a7d9c76956fcdc6f6a) * Replace incorrect symlink_dir usage with symlink_file (cherry picked from commit f2a7f561a07a1f9c56c24122b1537b3ab1b95163) * Reduce TempDir exposure (cherry picked from commit 9f70f7dc3e0ce4f960c9dceb3286c2c9e8d2256b) * Rename AccountsPackage::root to AccountsPackage::slot (cherry picked from commit 141e6706e67aa721e2ad22251dbb183799e76cc4) * Rename CompressionType to ArchiveFormat (cherry picked from commit 7be6770808ce53e591cec75a3366748b5661b906) Co-authored-by: Michael Vines --- core/src/accounts_hash_verifier.rs | 14 +-- core/src/rpc_service.rs | 6 +- core/src/snapshot_packager_service.rs | 10 +- core/src/test_validator.rs | 4 +- core/tests/snapshots.rs | 16 +-- download-utils/src/lib.rs | 8 +- ledger-tool/src/main.rs | 6 +- local-cluster/tests/local_cluster.rs | 10 +- runtime/src/accounts_background_service.rs | 2 +- runtime/src/bank_forks.rs | 12 +-- runtime/src/serde_snapshot.rs | 8 +- runtime/src/snapshot_package.rs | 14 +-- runtime/src/snapshot_utils.rs | 116 ++++++++++----------- validator/src/main.rs | 31 +++--- 14 files changed, 129 insertions(+), 128 deletions(-) diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 43601d8552..25d6b3a420 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -83,17 +83,17 @@ impl AccountsHashVerifier { snapshot_interval_slots: u64, ) { if fault_injection_rate_slots != 0 - && accounts_package.root % fault_injection_rate_slots == 0 + && accounts_package.slot % fault_injection_rate_slots == 0 { // For testing, publish an invalid hash to gossip. use rand::{thread_rng, Rng}; use solana_sdk::hash::extend_and_hash; - warn!("inserting fault at slot: {}", accounts_package.root); + warn!("inserting fault at slot: {}", accounts_package.slot); let rand = thread_rng().gen_range(0, 10); let hash = extend_and_hash(&accounts_package.hash, &[rand]); - hashes.push((accounts_package.root, hash)); + hashes.push((accounts_package.slot, hash)); } else { - hashes.push((accounts_package.root, accounts_package.hash)); + hashes.push((accounts_package.slot, accounts_package.hash)); } while hashes.len() > MAX_SNAPSHOT_HASHES { @@ -175,7 +175,7 @@ mod tests { use super::*; use crate::cluster_info::make_accounts_hashes_message; use crate::contact_info::ContactInfo; - use solana_runtime::bank_forks::CompressionType; + use solana_runtime::bank_forks::ArchiveFormat; use solana_runtime::snapshot_utils::SnapshotVersion; use solana_sdk::{ hash::hash, @@ -234,12 +234,12 @@ mod tests { let accounts_package = AccountsPackage { hash: hash(&[i as u8]), block_height: 100 + i as u64, - root: 100 + i as u64, + slot: 100 + i as u64, slot_deltas: vec![], snapshot_links, tar_output_file: PathBuf::from("."), storages: vec![], - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), }; diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index ca76c19068..ea55e6e54b 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -439,9 +439,7 @@ mod tests { genesis_utils::{create_genesis_config, GenesisConfigInfo}, get_tmp_ledger_path, }; - use solana_runtime::{ - bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion, - }; + use solana_runtime::{bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion}; use solana_sdk::{genesis_config::ClusterType, signature::Signer}; use std::net::{IpAddr, Ipv4Addr}; @@ -534,7 +532,7 @@ mod tests { snapshot_interval_slots: 0, snapshot_package_output_path: PathBuf::from("/"), snapshot_path: PathBuf::from("/"), - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), }), bank_forks, diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 1fc9038651..86a608b992 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -51,7 +51,7 @@ impl SnapshotPackagerService { { warn!("Failed to create snapshot archive: {}", err); } else { - hashes.push((snapshot_package.root, snapshot_package.hash)); + hashes.push((snapshot_package.slot, snapshot_package.hash)); while hashes.len() > MAX_SNAPSHOT_HASHES { hashes.remove(0); } @@ -81,7 +81,7 @@ mod tests { use solana_runtime::{ accounts_db::AccountStorageEntry, bank::BankSlotDelta, - bank_forks::CompressionType, + bank_forks::ArchiveFormat, snapshot_package::AccountsPackage, snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME}, }; @@ -163,7 +163,7 @@ mod tests { let output_tar_path = snapshot_utils::get_snapshot_archive_path( &snapshot_package_output_path, &(42, Hash::default()), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); let snapshot_package = AccountsPackage::new( 5, @@ -173,7 +173,7 @@ mod tests { vec![storage_entries], output_tar_path.clone(), Hash::default(), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, SnapshotVersion::default(), ); @@ -198,7 +198,7 @@ mod tests { output_tar_path, snapshots_dir, accounts_dir, - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, ); } } diff --git a/core/src/test_validator.rs b/core/src/test_validator.rs index 0070c862c4..80eaaf492d 100644 --- a/core/src/test_validator.rs +++ b/core/src/test_validator.rs @@ -8,7 +8,7 @@ use { solana_client::rpc_client::RpcClient, solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger}, solana_runtime::{ - bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion}, + bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion}, genesis_utils::create_genesis_config_with_leader_ex, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, }, @@ -354,7 +354,7 @@ impl TestValidator { snapshot_interval_slots: 100, snapshot_path: ledger_path.join("snapshot"), snapshot_package_output_path: ledger_path.to_path_buf(), - compression: CompressionType::NoCompression, + archive_format: ArchiveFormat::Tar, snapshot_version: SnapshotVersion::default(), }), enforce_ulimit_nofile: false, diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index e3a116ed23..a83b129eb1 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -45,7 +45,7 @@ mod tests { use solana_runtime::{ accounts_background_service::{ABSRequestSender, SnapshotRequestHandler}, bank::{Bank, BankSlotDelta}, - bank_forks::{BankForks, CompressionType, SnapshotConfig}, + bank_forks::{ArchiveFormat, BankForks, SnapshotConfig}, genesis_utils::{create_genesis_config, GenesisConfigInfo}, snapshot_utils, snapshot_utils::SnapshotVersion, @@ -106,7 +106,7 @@ mod tests { snapshot_interval_slots, snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_path: PathBuf::from(snapshot_dir.path()), - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version, }; bank_forks.set_snapshot_config(Some(snapshot_config.clone())); @@ -146,9 +146,9 @@ mod tests { snapshot_utils::get_snapshot_archive_path( snapshot_package_output_path, &(old_last_bank.slot(), old_last_bank.get_accounts_hash()), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, old_genesis_config, None, None, @@ -226,7 +226,7 @@ mod tests { last_bank.src.slot_deltas(&last_bank.src.roots()), &snapshot_config.snapshot_package_output_path, last_bank.get_snapshot_storages(), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, snapshot_version, ) .unwrap(); @@ -347,7 +347,7 @@ mod tests { &snapshot_path, &snapshot_package_output_path, snapshot_config.snapshot_version, - &snapshot_config.compression, + &snapshot_config.archive_format, ) .unwrap(); @@ -375,7 +375,7 @@ mod tests { saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( snapshot_package_output_path, &(slot, accounts_hash), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, )); } } @@ -432,7 +432,7 @@ mod tests { saved_accounts_dir .path() .join(accounts_dir.path().file_name().unwrap()), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, ); } diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index 1eeacfddcd..050afade87 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -1,7 +1,7 @@ use console::Emoji; use indicatif::{ProgressBar, ProgressStyle}; use log::*; -use solana_runtime::{bank_forks::CompressionType, snapshot_utils}; +use solana_runtime::{bank_forks::ArchiveFormat, snapshot_utils}; use solana_sdk::clock::Slot; use solana_sdk::hash::Hash; use std::fs::{self, File}; @@ -175,9 +175,9 @@ pub fn download_snapshot( snapshot_utils::purge_old_snapshot_archives(ledger_path); for compression in &[ - CompressionType::Zstd, - CompressionType::Gzip, - CompressionType::Bzip2, + ArchiveFormat::TarZstd, + ArchiveFormat::TarGzip, + ArchiveFormat::TarBzip2, ] { let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path( ledger_path, diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 69fad0c1b2..bce6715106 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -25,7 +25,7 @@ use solana_ledger::{ }; use solana_runtime::{ bank::{Bank, RewardCalculationEvent}, - bank_forks::{BankForks, CompressionType, SnapshotConfig}, + bank_forks::{ArchiveFormat, BankForks, SnapshotConfig}, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_utils, snapshot_utils::SnapshotVersion, @@ -667,7 +667,7 @@ fn load_bank_forks( snapshot_interval_slots: 0, // Value doesn't matter snapshot_package_output_path, snapshot_path, - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), }) }; @@ -1947,7 +1947,7 @@ fn main() { bank.src.slot_deltas(&bank.src.roots()), output_directory, storages, - CompressionType::Zstd, + ArchiveFormat::TarZstd, snapshot_version, ) }) diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 25cbfced2c..5b6772452d 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -32,7 +32,7 @@ use solana_local_cluster::{ local_cluster::{ClusterConfig, LocalCluster}, }; use solana_runtime::{ - bank_forks::{CompressionType, SnapshotConfig}, + bank_forks::{ArchiveFormat, SnapshotConfig}, snapshot_utils, }; use solana_sdk::{ @@ -1054,7 +1054,7 @@ fn test_snapshot_download() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &archive_snapshot_hash, - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); // Download the snapshot, then boot a validator from it. @@ -1124,7 +1124,7 @@ fn test_snapshot_restart_tower() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &archive_snapshot_hash, - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); fs::hard_link(archive_filename, &validator_archive_path).unwrap(); @@ -1189,7 +1189,7 @@ fn test_snapshots_blockstore_floor() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &(archive_slot, archive_hash), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); fs::hard_link(archive_filename, &validator_archive_path).unwrap(); let slot_floor = archive_slot; @@ -2446,7 +2446,7 @@ fn setup_snapshot_validator_config( snapshot_interval_slots, snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_path: PathBuf::from(snapshot_dir.path()), - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: snapshot_utils::SnapshotVersion::default(), }; diff --git a/runtime/src/accounts_background_service.rs b/runtime/src/accounts_background_service.rs index 4c559da268..384eba1b93 100644 --- a/runtime/src/accounts_background_service.rs +++ b/runtime/src/accounts_background_service.rs @@ -112,7 +112,7 @@ impl SnapshotRequestHandler { &self.snapshot_config.snapshot_path, &self.snapshot_config.snapshot_package_output_path, self.snapshot_config.snapshot_version, - &self.snapshot_config.compression, + &self.snapshot_config.archive_format, ); if r.is_err() { warn!( diff --git a/runtime/src/bank_forks.rs b/runtime/src/bank_forks.rs index dbd431c6dd..69327a024e 100644 --- a/runtime/src/bank_forks.rs +++ b/runtime/src/bank_forks.rs @@ -18,11 +18,11 @@ use std::{ pub use crate::snapshot_utils::SnapshotVersion; #[derive(Clone, Debug, Eq, PartialEq)] -pub enum CompressionType { - Bzip2, - Gzip, - Zstd, - NoCompression, +pub enum ArchiveFormat { + TarBzip2, + TarGzip, + TarZstd, + Tar, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -36,7 +36,7 @@ pub struct SnapshotConfig { // Where to place the snapshots for recent slots pub snapshot_path: PathBuf, - pub compression: CompressionType, + pub archive_format: ArchiveFormat, // Snapshot version to generate pub snapshot_version: SnapshotVersion, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 71687bf038..61482593e8 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -288,6 +288,12 @@ where map }; + // Ensure all account paths exist + for path in &accounts_db.paths { + std::fs::create_dir_all(path) + .unwrap_or_else(|err| panic!("Failed to create directory {}: {}", path.display(), err)); + } + let mut last_log_update = Instant::now(); let mut remaining_slots_to_process = storage.len(); @@ -307,8 +313,6 @@ where let path_index = thread_rng().gen_range(0, accounts_db.paths.len()); let local_dir = &accounts_db.paths[path_index]; - std::fs::create_dir_all(local_dir).expect("Create directory failed"); - // Move the corresponding AppendVec from the snapshot into the directory pointed // at by `local_dir` let append_vec_relative_path = diff --git a/runtime/src/snapshot_package.rs b/runtime/src/snapshot_package.rs index 31a8cda030..ef5523afbd 100644 --- a/runtime/src/snapshot_package.rs +++ b/runtime/src/snapshot_package.rs @@ -1,4 +1,4 @@ -use crate::bank_forks::CompressionType; +use crate::bank_forks::ArchiveFormat; use crate::snapshot_utils::SnapshotVersion; use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta}; use solana_sdk::clock::Slot; @@ -15,38 +15,38 @@ pub type AccountsPackageSendError = SendError; #[derive(Debug)] pub struct AccountsPackage { - pub root: Slot, + pub slot: Slot, pub block_height: Slot, pub slot_deltas: Vec, pub snapshot_links: TempDir, pub storages: SnapshotStorages, pub tar_output_file: PathBuf, pub hash: Hash, - pub compression: CompressionType, + pub archive_format: ArchiveFormat, pub snapshot_version: SnapshotVersion, } impl AccountsPackage { pub fn new( - root: Slot, + slot: Slot, block_height: u64, slot_deltas: Vec, snapshot_links: TempDir, storages: SnapshotStorages, tar_output_file: PathBuf, hash: Hash, - compression: CompressionType, + archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, ) -> Self { Self { - root, + slot, block_height, slot_deltas, snapshot_links, storages, tar_output_file, hash, - compression, + archive_format, snapshot_version, } } diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 8a0efb0ece..851f309a61 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -1,7 +1,7 @@ use crate::{ accounts_index::AccountIndex, bank::{Bank, BankSlotDelta, Builtins}, - bank_forks::CompressionType, + bank_forks::ArchiveFormat, hardened_unpack::{unpack_snapshot, UnpackError}, serde_snapshot::{ bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, @@ -27,7 +27,6 @@ use std::{ str::FromStr, }; use tar::Archive; -use tempfile::TempDir; use thiserror::Error; pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache"; @@ -159,7 +158,7 @@ pub fn package_snapshot, Q: AsRef>( status_cache_slot_deltas: Vec, snapshot_package_output_path: P, snapshot_storages: SnapshotStorages, - compression: CompressionType, + archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, ) -> Result { // Hard link all the snapshots we need for this package @@ -181,7 +180,7 @@ pub fn package_snapshot, Q: AsRef>( let snapshot_package_output_file = get_snapshot_archive_path( &snapshot_package_output_path, &(bank.slot(), bank.get_accounts_hash()), - &compression, + &archive_format, ); let package = AccountsPackage::new( @@ -192,19 +191,19 @@ pub fn package_snapshot, Q: AsRef>( snapshot_storages, snapshot_package_output_file, bank.get_accounts_hash(), - compression, + archive_format, snapshot_version, ); Ok(package) } -fn get_compression_ext(compression: &CompressionType) -> &'static str { - match compression { - CompressionType::Bzip2 => ".tar.bz2", - CompressionType::Gzip => ".tar.gz", - CompressionType::Zstd => ".tar.zst", - CompressionType::NoCompression => ".tar", +fn get_archive_ext(archive_format: &ArchiveFormat) -> &'static str { + match archive_format { + ArchiveFormat::TarBzip2 => ".tar.bz2", + ArchiveFormat::TarGzip => ".tar.gz", + ArchiveFormat::TarZstd => ".tar.zst", + ArchiveFormat::Tar => ".tar", } } @@ -230,13 +229,16 @@ pub fn remove_tmp_snapshot_archives(snapshot_path: &Path) { pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()> { info!( "Generating snapshot archive for slot {}", - snapshot_package.root + snapshot_package.slot ); serialize_status_cache( - snapshot_package.root, + snapshot_package.slot, &snapshot_package.slot_deltas, - &snapshot_package.snapshot_links, + &snapshot_package + .snapshot_links + .path() + .join(SNAPSHOT_STATUS_CACHE_FILE_NAME), )?; let mut timer = Measure::start("snapshot_package-package_snapshots"); @@ -274,10 +276,10 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() )); // `storage_path` - The file path where the AppendVec itself is located - // `output_path` - The directory where the AppendVec will be placed in the staging directory. + // `output_path` - The file path where the AppendVec will be placed in the staging directory. let storage_path = fs::canonicalize(storage_path).expect("Could not get absolute path for accounts"); - symlink::symlink_dir(storage_path, &output_path)?; + symlink::symlink_file(storage_path, &output_path)?; if !output_path.is_file() { return Err(SnapshotError::StoragePathSymlinkInvalid); } @@ -289,7 +291,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?; } - let file_ext = get_compression_ext(&snapshot_package.compression); + let file_ext = get_archive_ext(&snapshot_package.archive_format); // Tar the staging directory into the archive at `archive_path` // @@ -320,23 +322,23 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() Some(tar_output) => { let mut archive_file = fs::File::create(&archive_path)?; - match snapshot_package.compression { - CompressionType::Bzip2 => { + match snapshot_package.archive_format { + ArchiveFormat::TarBzip2 => { let mut encoder = bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best); io::copy(tar_output, &mut encoder)?; let _ = encoder.finish()?; } - CompressionType::Gzip => { + ArchiveFormat::TarGzip => { let mut encoder = flate2::write::GzEncoder::new(archive_file, flate2::Compression::default()); io::copy(tar_output, &mut encoder)?; let _ = encoder.finish()?; } - CompressionType::NoCompression => { + ArchiveFormat::Tar => { io::copy(tar_output, &mut archive_file)?; } - CompressionType::Zstd => { + ArchiveFormat::TarZstd => { let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?; io::copy(tar_output, &mut encoder)?; let _ = encoder.finish()?; @@ -361,13 +363,13 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() info!( "Successfully created {:?}. slot: {}, elapsed ms: {}, size={}", snapshot_package.tar_output_file, - snapshot_package.root, + snapshot_package.slot, timer.as_ms(), metadata.len() ); datapoint_info!( "snapshot-package", - ("slot", snapshot_package.root, i64), + ("slot", snapshot_package.slot, i64), ("duration_ms", timer.as_ms(), i64), ("size", metadata.len(), i64) ); @@ -546,14 +548,10 @@ pub fn add_snapshot>( fn serialize_status_cache( slot: Slot, slot_deltas: &[BankSlotDelta], - snapshot_links: &TempDir, + status_cache_path: &Path, ) -> Result<()> { - // the status cache is stored as snapshot_path/status_cache - let snapshot_status_cache_file_path = - snapshot_links.path().join(SNAPSHOT_STATUS_CACHE_FILE_NAME); - let mut status_cache_serialize = Measure::start("status_cache_serialize-ms"); - let consumed_size = serialize_snapshot_data_file(&snapshot_status_cache_file_path, |stream| { + let consumed_size = serialize_snapshot_data_file(status_cache_path, |stream| { serialize_into(stream, slot_deltas)?; Ok(()) })?; @@ -585,7 +583,7 @@ pub fn bank_from_archive>( frozen_account_pubkeys: &[Pubkey], snapshot_path: &PathBuf, snapshot_tar: P, - compression: CompressionType, + archive_format: ArchiveFormat, genesis_config: &GenesisConfig, debug_keys: Option>>, additional_builtins: Option<&Builtins>, @@ -595,7 +593,7 @@ pub fn bank_from_archive>( let unpack_dir = tempfile::Builder::new() .prefix(TMP_SNAPSHOT_DIR_PREFIX) .tempdir_in(snapshot_path)?; - untar_snapshot_in(&snapshot_tar, &unpack_dir, compression)?; + untar_snapshot_in(&snapshot_tar, &unpack_dir, archive_format)?; let mut measure = Measure::start("bank rebuild from snapshot"); let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR); @@ -629,27 +627,27 @@ pub fn bank_from_archive>( pub fn get_snapshot_archive_path>( snapshot_output_dir: P, snapshot_hash: &(Slot, Hash), - compression: &CompressionType, + archive_format: &ArchiveFormat, ) -> PathBuf { snapshot_output_dir.as_ref().join(format!( "snapshot-{}-{}{}", snapshot_hash.0, snapshot_hash.1, - get_compression_ext(compression), + get_archive_ext(archive_format), )) } -fn compression_type_from_str(compress: &str) -> Option { - match compress { - "tar.bz2" => Some(CompressionType::Bzip2), - "tar.gz" => Some(CompressionType::Gzip), - "tar.zst" => Some(CompressionType::Zstd), - "tar" => Some(CompressionType::NoCompression), +fn archive_format_from_str(archive_format: &str) -> Option { + match archive_format { + "tar.bz2" => Some(ArchiveFormat::TarBzip2), + "tar.gz" => Some(ArchiveFormat::TarGzip), + "tar.zst" => Some(ArchiveFormat::TarZstd), + "tar" => Some(ArchiveFormat::Tar), _ => None, } } -fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionType)> { +fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, ArchiveFormat)> { let snapshot_filename_regex = Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.(tar|tar\.bz2|tar\.zst|tar\.gz)$").unwrap(); @@ -658,12 +656,12 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy let hash_str = captures.get(2).unwrap().as_str(); let ext = captures.get(3).unwrap().as_str(); - if let (Ok(slot), Ok(hash), Some(compression)) = ( + if let (Ok(slot), Ok(hash), Some(archive_format)) = ( slot_str.parse::(), hash_str.parse::(), - compression_type_from_str(ext), + archive_format_from_str(ext), ) { - return Some((slot, hash, compression)); + return Some((slot, hash, archive_format)); } } None @@ -671,7 +669,7 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy pub fn get_snapshot_archives>( snapshot_output_dir: P, -) -> Vec<(PathBuf, (Slot, Hash, CompressionType))> { +) -> Vec<(PathBuf, (Slot, Hash, ArchiveFormat))> { match fs::read_dir(&snapshot_output_dir) { Err(err) => { info!("Unable to read snapshot directory: {}", err); @@ -702,7 +700,7 @@ pub fn get_snapshot_archives>( pub fn get_highest_snapshot_archive_path>( snapshot_output_dir: P, -) -> Option<(PathBuf, (Slot, Hash, CompressionType))> { +) -> Option<(PathBuf, (Slot, Hash, ArchiveFormat))> { let archives = get_snapshot_archives(snapshot_output_dir); archives.into_iter().next() } @@ -720,27 +718,27 @@ pub fn purge_old_snapshot_archives>(snapshot_output_dir: P) { pub fn untar_snapshot_in, Q: AsRef>( snapshot_tar: P, unpack_dir: Q, - compression: CompressionType, + archive_format: ArchiveFormat, ) -> Result<()> { let mut measure = Measure::start("snapshot untar"); let tar_name = File::open(&snapshot_tar)?; - match compression { - CompressionType::Bzip2 => { + match archive_format { + ArchiveFormat::TarBzip2 => { let tar = BzDecoder::new(BufReader::new(tar_name)); let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; } - CompressionType::Gzip => { + ArchiveFormat::TarGzip => { let tar = GzDecoder::new(BufReader::new(tar_name)); let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; } - CompressionType::Zstd => { + ArchiveFormat::TarZstd => { let tar = zstd::stream::read::Decoder::new(BufReader::new(tar_name))?; let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; } - CompressionType::NoCompression => { + ArchiveFormat::Tar => { let tar = BufReader::new(tar_name); let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; @@ -839,7 +837,7 @@ pub fn verify_snapshot_archive( snapshot_archive: P, snapshots_to_verify: Q, storages_to_verify: R, - compression: CompressionType, + archive_format: ArchiveFormat, ) where P: AsRef, Q: AsRef, @@ -847,7 +845,7 @@ pub fn verify_snapshot_archive( { let temp_dir = tempfile::TempDir::new().unwrap(); let unpack_dir = temp_dir.path(); - untar_snapshot_in(snapshot_archive, &unpack_dir, compression).unwrap(); + untar_snapshot_in(snapshot_archive, &unpack_dir, archive_format).unwrap(); // Check snapshots are the same let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR); @@ -878,7 +876,7 @@ pub fn snapshot_bank( snapshot_path: &Path, snapshot_package_output_path: &Path, snapshot_version: SnapshotVersion, - compression: &CompressionType, + archive_format: &ArchiveFormat, ) -> Result<()> { let storages: Vec<_> = root_bank.get_snapshot_storages(); let mut add_snapshot_time = Measure::start("add-snapshot-ms"); @@ -899,7 +897,7 @@ pub fn snapshot_bank( status_cache_slot_deltas, snapshot_package_output_path, storages, - compression.clone(), + archive_format.clone(), snapshot_version, )?; @@ -1024,15 +1022,15 @@ mod tests { fn test_snapshot_hash_of() { assert_eq!( snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())), - Some((42, Hash::default(), CompressionType::Bzip2)) + Some((42, Hash::default(), ArchiveFormat::TarBzip2)) ); assert_eq!( snapshot_hash_of(&format!("snapshot-43-{}.tar.zst", Hash::default())), - Some((43, Hash::default(), CompressionType::Zstd)) + Some((43, Hash::default(), ArchiveFormat::TarZstd)) ); assert_eq!( snapshot_hash_of(&format!("snapshot-42-{}.tar", Hash::default())), - Some((42, Hash::default(), CompressionType::NoCompression)) + Some((42, Hash::default(), ArchiveFormat::Tar)) ); assert!(snapshot_hash_of("invalid").is_none()); diff --git a/validator/src/main.rs b/validator/src/main.rs index 54ce79cde6..3dea288447 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -29,7 +29,7 @@ use solana_ledger::blockstore_db::BlockstoreRecoveryMode; use solana_perf::recycler::enable_recycler_warming; use solana_runtime::{ accounts_index::AccountIndex, - bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion}, + bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion}, hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_utils::get_highest_snapshot_archive_path, }; @@ -1359,13 +1359,14 @@ pub fn main() { other than increasing the account balance"), ) .arg( - Arg::with_name("snapshot_compression") - .long("snapshot-compression") - .possible_values(&["bz2", "gzip", "zstd", "none"]) + Arg::with_name("snapshot_archive_format") + .long("snapshot-archive-format") + .alias("snapshot-compression") // Legacy name used by Solana v1.5.x and older + .possible_values(&["bz2", "gzip", "zstd", "tar", "none"]) .default_value("zstd") - .value_name("COMPRESSION_TYPE") + .value_name("ARCHIVE_TYPE") .takes_value(true) - .help("Type of snapshot compression to use."), + .help("Snapshot archive format to use."), ) .arg( Arg::with_name("max_genesis_archive_unpacked_size") @@ -1670,14 +1671,14 @@ pub fn main() { exit(1); }); - let snapshot_compression = { - let compression_str = value_t_or_exit!(matches, "snapshot_compression", String); - match compression_str.as_str() { - "bz2" => CompressionType::Bzip2, - "gzip" => CompressionType::Gzip, - "zstd" => CompressionType::Zstd, - "none" => CompressionType::NoCompression, - _ => panic!("Compression type not recognized: {}", compression_str), + let archive_format = { + let archive_format_str = value_t_or_exit!(matches, "snapshot_archive_format", String); + match archive_format_str.as_str() { + "bz2" => ArchiveFormat::TarBzip2, + "gzip" => ArchiveFormat::TarGzip, + "zstd" => ArchiveFormat::TarZstd, + "tar" | "none" => ArchiveFormat::Tar, + _ => panic!("Archive format not recognized: {}", archive_format_str), } }; @@ -1698,7 +1699,7 @@ pub fn main() { }, snapshot_path, snapshot_package_output_path: ledger_path.clone(), - compression: snapshot_compression, + archive_format, snapshot_version, });