diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 4be8537672..25d6b3a420 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -175,7 +175,7 @@ mod tests { use super::*; use crate::cluster_info::make_accounts_hashes_message; use crate::contact_info::ContactInfo; - use solana_runtime::bank_forks::CompressionType; + use solana_runtime::bank_forks::ArchiveFormat; use solana_runtime::snapshot_utils::SnapshotVersion; use solana_sdk::{ hash::hash, @@ -239,7 +239,7 @@ mod tests { snapshot_links, tar_output_file: PathBuf::from("."), storages: vec![], - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), }; diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index ca76c19068..ea55e6e54b 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -439,9 +439,7 @@ mod tests { genesis_utils::{create_genesis_config, GenesisConfigInfo}, get_tmp_ledger_path, }; - use solana_runtime::{ - bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion, - }; + use solana_runtime::{bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion}; use solana_sdk::{genesis_config::ClusterType, signature::Signer}; use std::net::{IpAddr, Ipv4Addr}; @@ -534,7 +532,7 @@ mod tests { snapshot_interval_slots: 0, snapshot_package_output_path: PathBuf::from("/"), snapshot_path: PathBuf::from("/"), - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), }), bank_forks, diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 5432d7a517..86a608b992 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -81,7 +81,7 @@ mod tests { use solana_runtime::{ accounts_db::AccountStorageEntry, bank::BankSlotDelta, - bank_forks::CompressionType, + bank_forks::ArchiveFormat, snapshot_package::AccountsPackage, snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME}, }; @@ -163,7 +163,7 @@ mod tests { let output_tar_path = snapshot_utils::get_snapshot_archive_path( &snapshot_package_output_path, &(42, Hash::default()), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); let snapshot_package = AccountsPackage::new( 5, @@ -173,7 +173,7 @@ mod tests { vec![storage_entries], output_tar_path.clone(), Hash::default(), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, SnapshotVersion::default(), ); @@ -198,7 +198,7 @@ mod tests { output_tar_path, snapshots_dir, accounts_dir, - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, ); } } diff --git a/core/src/test_validator.rs b/core/src/test_validator.rs index 2ab8a6a05f..25c559128e 100644 --- a/core/src/test_validator.rs +++ b/core/src/test_validator.rs @@ -1,8 +1,45 @@ +<<<<<<< HEAD use crate::{ cluster_info::Node, contact_info::ContactInfo, gossip_service::discover_cluster, validator::{Validator, ValidatorConfig}, +======= +use { + crate::{ + cluster_info::Node, + gossip_service::discover_cluster, + rpc::JsonRpcConfig, + validator::{Validator, ValidatorConfig}, + }, + solana_client::rpc_client::RpcClient, + solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger}, + solana_runtime::{ + bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion}, + genesis_utils::create_genesis_config_with_leader_ex, + hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + }, + solana_sdk::{ + account::Account, + clock::DEFAULT_MS_PER_SLOT, + commitment_config::CommitmentConfig, + fee_calculator::{FeeCalculator, FeeRateGovernor}, + hash::Hash, + native_token::sol_to_lamports, + pubkey::Pubkey, + rent::Rent, + signature::{read_keypair_file, write_keypair_file, Keypair, Signer}, + }, + std::{ + collections::HashMap, + fs::remove_dir_all, + net::{IpAddr, Ipv4Addr, SocketAddr}, + path::PathBuf, + sync::Arc, + thread::sleep, + time::Duration, + }, +>>>>>>> 7be677080... Rename CompressionType to ArchiveFormat }; use solana_ledger::create_new_tmp_ledger; use solana_runtime::genesis_utils::{ @@ -106,8 +143,52 @@ impl TestValidator { let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config); +<<<<<<< HEAD let config = ValidatorConfig { rpc_addrs: Some((node.info.rpc, node.info.rpc_pubsub)), +======= + let validator_identity = + read_keypair_file(ledger_path.join("validator-keypair.json").to_str().unwrap())?; + let validator_vote_account = read_keypair_file( + ledger_path + .join("vote-account-keypair.json") + .to_str() + .unwrap(), + )?; + + let mut node = Node::new_localhost_with_pubkey(&validator_identity.pubkey()); + if let Some((rpc, rpc_pubsub)) = config.rpc_ports { + node.info.rpc = SocketAddr::new(node.info.gossip.ip(), rpc); + node.info.rpc_pubsub = SocketAddr::new(node.info.gossip.ip(), rpc_pubsub); + } + + let vote_account_address = validator_vote_account.pubkey(); + let rpc_url = format!("http://{}", node.info.rpc); + let rpc_pubsub_url = format!("ws://{}/", node.info.rpc_pubsub); + let tpu = node.info.tpu; + let gossip = node.info.gossip; + + let validator_config = ValidatorConfig { + rpc_addrs: Some(( + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()), + SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + node.info.rpc_pubsub.port(), + ), + )), + rpc_config: config.rpc_config.clone(), + accounts_hash_interval_slots: 100, + account_paths: vec![ledger_path.join("accounts")], + poh_verify: false, // Skip PoH verification of ledger on startup for speed + snapshot_config: Some(SnapshotConfig { + snapshot_interval_slots: 100, + snapshot_path: ledger_path.join("snapshot"), + snapshot_package_output_path: ledger_path.to_path_buf(), + archive_format: ArchiveFormat::Tar, + snapshot_version: SnapshotVersion::default(), + }), + enforce_ulimit_nofile: false, +>>>>>>> 7be677080... Rename CompressionType to ArchiveFormat ..ValidatorConfig::default() }; let vote_pubkey = voting_keypair.pubkey(); diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index ead2d834c7..7ca79051ab 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -45,7 +45,7 @@ mod tests { use solana_runtime::{ accounts_background_service::SnapshotRequestHandler, bank::{Bank, BankSlotDelta}, - bank_forks::{BankForks, CompressionType, SnapshotConfig}, + bank_forks::{ArchiveFormat, BankForks, SnapshotConfig}, genesis_utils::{create_genesis_config, GenesisConfigInfo}, snapshot_utils, snapshot_utils::SnapshotVersion, @@ -106,7 +106,7 @@ mod tests { snapshot_interval_slots, snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_path: PathBuf::from(snapshot_dir.path()), - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version, }; bank_forks.set_snapshot_config(Some(snapshot_config.clone())); @@ -146,9 +146,9 @@ mod tests { snapshot_utils::get_snapshot_archive_path( snapshot_package_output_path, &(old_last_bank.slot(), old_last_bank.get_accounts_hash()), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, old_genesis_config, None, None, @@ -226,7 +226,7 @@ mod tests { last_bank.src.slot_deltas(&last_bank.src.roots()), &snapshot_config.snapshot_package_output_path, last_bank.get_snapshot_storages(), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, snapshot_version, ) .unwrap(); @@ -347,7 +347,7 @@ mod tests { &snapshot_path, &snapshot_package_output_path, snapshot_config.snapshot_version, - &snapshot_config.compression, + &snapshot_config.archive_format, ) .unwrap(); @@ -375,7 +375,7 @@ mod tests { saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( snapshot_package_output_path, &(slot, accounts_hash), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, )); } } @@ -432,7 +432,7 @@ mod tests { saved_accounts_dir .path() .join(accounts_dir.path().file_name().unwrap()), - CompressionType::Bzip2, + ArchiveFormat::TarBzip2, ); } diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index 441c6be9a0..6ef976e201 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -1,7 +1,7 @@ use console::Emoji; use indicatif::{ProgressBar, ProgressStyle}; use log::*; -use solana_runtime::{bank_forks::CompressionType, snapshot_utils}; +use solana_runtime::{bank_forks::ArchiveFormat, snapshot_utils}; use solana_sdk::clock::Slot; use solana_sdk::hash::Hash; use std::fs::{self, File}; @@ -171,9 +171,9 @@ pub fn download_snapshot( snapshot_utils::purge_old_snapshot_archives(ledger_path); for compression in &[ - CompressionType::Zstd, - CompressionType::Gzip, - CompressionType::Bzip2, + ArchiveFormat::TarZstd, + ArchiveFormat::TarGzip, + ArchiveFormat::TarBzip2, ] { let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path( ledger_path, diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index e98d20607b..3dd1c717da 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -25,7 +25,7 @@ use solana_ledger::{ }; use solana_runtime::{ bank::{Bank, RewardCalculationEvent}, - bank_forks::{BankForks, CompressionType, SnapshotConfig}, + bank_forks::{ArchiveFormat, BankForks, SnapshotConfig}, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_utils, snapshot_utils::SnapshotVersion, @@ -678,7 +678,7 @@ fn load_bank_forks( snapshot_interval_slots: 0, // Value doesn't matter snapshot_package_output_path, snapshot_path, - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), }) }; @@ -1958,7 +1958,7 @@ fn main() { bank.src.slot_deltas(&bank.src.roots()), output_directory, storages, - CompressionType::Zstd, + ArchiveFormat::TarZstd, snapshot_version, ) }) diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index affe846644..9ba1f09fd3 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -32,7 +32,7 @@ use solana_local_cluster::{ local_cluster::{ClusterConfig, LocalCluster}, }; use solana_runtime::{ - bank_forks::{CompressionType, SnapshotConfig}, + bank_forks::{ArchiveFormat, SnapshotConfig}, snapshot_utils, }; use solana_sdk::{ @@ -1050,7 +1050,7 @@ fn test_snapshot_download() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &archive_snapshot_hash, - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); // Download the snapshot, then boot a validator from it. @@ -1120,7 +1120,7 @@ fn test_snapshot_restart_tower() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &archive_snapshot_hash, - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); fs::hard_link(archive_filename, &validator_archive_path).unwrap(); @@ -1185,7 +1185,7 @@ fn test_snapshots_blockstore_floor() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &(archive_slot, archive_hash), - &CompressionType::Bzip2, + &ArchiveFormat::TarBzip2, ); fs::hard_link(archive_filename, &validator_archive_path).unwrap(); let slot_floor = archive_slot; @@ -2440,7 +2440,7 @@ fn setup_snapshot_validator_config( snapshot_interval_slots, snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_path: PathBuf::from(snapshot_dir.path()), - compression: CompressionType::Bzip2, + archive_format: ArchiveFormat::TarBzip2, snapshot_version: snapshot_utils::SnapshotVersion::default(), }; diff --git a/runtime/src/accounts_background_service.rs b/runtime/src/accounts_background_service.rs index 619b403e4b..ab67d049b0 100644 --- a/runtime/src/accounts_background_service.rs +++ b/runtime/src/accounts_background_service.rs @@ -76,7 +76,7 @@ impl SnapshotRequestHandler { &self.snapshot_config.snapshot_path, &self.snapshot_config.snapshot_package_output_path, self.snapshot_config.snapshot_version, - &self.snapshot_config.compression, + &self.snapshot_config.archive_format, ); if r.is_err() { warn!( diff --git a/runtime/src/bank_forks.rs b/runtime/src/bank_forks.rs index 21b1e4082f..0a5881e0ab 100644 --- a/runtime/src/bank_forks.rs +++ b/runtime/src/bank_forks.rs @@ -18,11 +18,11 @@ use std::{ pub use crate::snapshot_utils::SnapshotVersion; #[derive(Clone, Debug, Eq, PartialEq)] -pub enum CompressionType { - Bzip2, - Gzip, - Zstd, - NoCompression, +pub enum ArchiveFormat { + TarBzip2, + TarGzip, + TarZstd, + Tar, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -36,7 +36,7 @@ pub struct SnapshotConfig { // Where to place the snapshots for recent slots pub snapshot_path: PathBuf, - pub compression: CompressionType, + pub archive_format: ArchiveFormat, // Snapshot version to generate pub snapshot_version: SnapshotVersion, diff --git a/runtime/src/snapshot_package.rs b/runtime/src/snapshot_package.rs index 917669c41d..ef5523afbd 100644 --- a/runtime/src/snapshot_package.rs +++ b/runtime/src/snapshot_package.rs @@ -1,4 +1,4 @@ -use crate::bank_forks::CompressionType; +use crate::bank_forks::ArchiveFormat; use crate::snapshot_utils::SnapshotVersion; use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta}; use solana_sdk::clock::Slot; @@ -22,7 +22,7 @@ pub struct AccountsPackage { pub storages: SnapshotStorages, pub tar_output_file: PathBuf, pub hash: Hash, - pub compression: CompressionType, + pub archive_format: ArchiveFormat, pub snapshot_version: SnapshotVersion, } @@ -35,7 +35,7 @@ impl AccountsPackage { storages: SnapshotStorages, tar_output_file: PathBuf, hash: Hash, - compression: CompressionType, + archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, ) -> Self { Self { @@ -46,7 +46,7 @@ impl AccountsPackage { storages, tar_output_file, hash, - compression, + archive_format, snapshot_version, } } diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 1cb74d473c..3785c98d19 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -1,7 +1,7 @@ use crate::{ accounts_index::AccountIndex, bank::{Bank, BankSlotDelta, Builtins}, - bank_forks::CompressionType, + bank_forks::ArchiveFormat, hardened_unpack::{unpack_snapshot, UnpackError}, serde_snapshot::{ bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, @@ -162,7 +162,7 @@ pub fn package_snapshot, Q: AsRef>( status_cache_slot_deltas: Vec, snapshot_package_output_path: P, snapshot_storages: SnapshotStorages, - compression: CompressionType, + archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, ) -> Result { // Hard link all the snapshots we need for this package @@ -184,7 +184,7 @@ pub fn package_snapshot, Q: AsRef>( let snapshot_package_output_file = get_snapshot_archive_path( &snapshot_package_output_path, &(bank.slot(), bank.get_accounts_hash()), - &compression, + &archive_format, ); let package = AccountsPackage::new( @@ -195,19 +195,19 @@ pub fn package_snapshot, Q: AsRef>( snapshot_storages, snapshot_package_output_file, bank.get_accounts_hash(), - compression, + archive_format, snapshot_version, ); Ok(package) } -fn get_compression_ext(compression: &CompressionType) -> &'static str { - match compression { - CompressionType::Bzip2 => ".tar.bz2", - CompressionType::Gzip => ".tar.gz", - CompressionType::Zstd => ".tar.zst", - CompressionType::NoCompression => ".tar", +fn get_archive_ext(archive_format: &ArchiveFormat) -> &'static str { + match archive_format { + ArchiveFormat::TarBzip2 => ".tar.bz2", + ArchiveFormat::TarGzip => ".tar.gz", + ArchiveFormat::TarZstd => ".tar.zst", + ArchiveFormat::Tar => ".tar", } } @@ -239,7 +239,10 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() serialize_status_cache( snapshot_package.slot, &snapshot_package.slot_deltas, - &snapshot_package.snapshot_links.path().join(SNAPSHOT_STATUS_CACHE_FILE_NAME), + &snapshot_package + .snapshot_links + .path() + .join(SNAPSHOT_STATUS_CACHE_FILE_NAME), )?; let mut timer = Measure::start("snapshot_package-package_snapshots"); @@ -292,7 +295,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?; } - let file_ext = get_compression_ext(&snapshot_package.compression); + let file_ext = get_archive_ext(&snapshot_package.archive_format); // Tar the staging directory into the archive at `archive_path` // @@ -323,23 +326,23 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() Some(tar_output) => { let mut archive_file = fs::File::create(&archive_path)?; - match snapshot_package.compression { - CompressionType::Bzip2 => { + match snapshot_package.archive_format { + ArchiveFormat::TarBzip2 => { let mut encoder = bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best); io::copy(tar_output, &mut encoder)?; let _ = encoder.finish()?; } - CompressionType::Gzip => { + ArchiveFormat::TarGzip => { let mut encoder = flate2::write::GzEncoder::new(archive_file, flate2::Compression::default()); io::copy(tar_output, &mut encoder)?; let _ = encoder.finish()?; } - CompressionType::NoCompression => { + ArchiveFormat::Tar => { io::copy(tar_output, &mut archive_file)?; } - CompressionType::Zstd => { + ArchiveFormat::TarZstd => { let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?; io::copy(tar_output, &mut encoder)?; let _ = encoder.finish()?; @@ -584,7 +587,7 @@ pub fn bank_from_archive>( frozen_account_pubkeys: &[Pubkey], snapshot_path: &PathBuf, snapshot_tar: P, - compression: CompressionType, + archive_format: ArchiveFormat, genesis_config: &GenesisConfig, debug_keys: Option>>, additional_builtins: Option<&Builtins>, @@ -594,7 +597,7 @@ pub fn bank_from_archive>( let unpack_dir = tempfile::Builder::new() .prefix(TMP_SNAPSHOT_DIR_PREFIX) .tempdir_in(snapshot_path)?; - untar_snapshot_in(&snapshot_tar, &unpack_dir, compression)?; + untar_snapshot_in(&snapshot_tar, &unpack_dir, archive_format)?; let mut measure = Measure::start("bank rebuild from snapshot"); let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR); @@ -642,27 +645,27 @@ pub fn bank_from_archive>( pub fn get_snapshot_archive_path>( snapshot_output_dir: P, snapshot_hash: &(Slot, Hash), - compression: &CompressionType, + archive_format: &ArchiveFormat, ) -> PathBuf { snapshot_output_dir.as_ref().join(format!( "snapshot-{}-{}{}", snapshot_hash.0, snapshot_hash.1, - get_compression_ext(compression), + get_archive_ext(archive_format), )) } -fn compression_type_from_str(compress: &str) -> Option { - match compress { - "tar.bz2" => Some(CompressionType::Bzip2), - "tar.gz" => Some(CompressionType::Gzip), - "tar.zst" => Some(CompressionType::Zstd), - "tar" => Some(CompressionType::NoCompression), +fn archive_format_from_str(archive_format: &str) -> Option { + match archive_format { + "tar.bz2" => Some(ArchiveFormat::TarBzip2), + "tar.gz" => Some(ArchiveFormat::TarGzip), + "tar.zst" => Some(ArchiveFormat::TarZstd), + "tar" => Some(ArchiveFormat::Tar), _ => None, } } -fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionType)> { +fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, ArchiveFormat)> { let snapshot_filename_regex = Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.(tar|tar\.bz2|tar\.zst|tar\.gz)$").unwrap(); @@ -671,12 +674,12 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy let hash_str = captures.get(2).unwrap().as_str(); let ext = captures.get(3).unwrap().as_str(); - if let (Ok(slot), Ok(hash), Some(compression)) = ( + if let (Ok(slot), Ok(hash), Some(archive_format)) = ( slot_str.parse::(), hash_str.parse::(), - compression_type_from_str(ext), + archive_format_from_str(ext), ) { - return Some((slot, hash, compression)); + return Some((slot, hash, archive_format)); } } None @@ -684,7 +687,7 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy pub fn get_snapshot_archives>( snapshot_output_dir: P, -) -> Vec<(PathBuf, (Slot, Hash, CompressionType))> { +) -> Vec<(PathBuf, (Slot, Hash, ArchiveFormat))> { match fs::read_dir(&snapshot_output_dir) { Err(err) => { info!("Unable to read snapshot directory: {}", err); @@ -715,7 +718,7 @@ pub fn get_snapshot_archives>( pub fn get_highest_snapshot_archive_path>( snapshot_output_dir: P, -) -> Option<(PathBuf, (Slot, Hash, CompressionType))> { +) -> Option<(PathBuf, (Slot, Hash, ArchiveFormat))> { let archives = get_snapshot_archives(snapshot_output_dir); archives.into_iter().next() } @@ -733,27 +736,27 @@ pub fn purge_old_snapshot_archives>(snapshot_output_dir: P) { pub fn untar_snapshot_in, Q: AsRef>( snapshot_tar: P, unpack_dir: Q, - compression: CompressionType, + archive_format: ArchiveFormat, ) -> Result<()> { let mut measure = Measure::start("snapshot untar"); let tar_name = File::open(&snapshot_tar)?; - match compression { - CompressionType::Bzip2 => { + match archive_format { + ArchiveFormat::TarBzip2 => { let tar = BzDecoder::new(BufReader::new(tar_name)); let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; } - CompressionType::Gzip => { + ArchiveFormat::TarGzip => { let tar = GzDecoder::new(BufReader::new(tar_name)); let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; } - CompressionType::Zstd => { + ArchiveFormat::TarZstd => { let tar = zstd::stream::read::Decoder::new(BufReader::new(tar_name))?; let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; } - CompressionType::NoCompression => { + ArchiveFormat::Tar => { let tar = BufReader::new(tar_name); let mut archive = Archive::new(tar); unpack_snapshot(&mut archive, unpack_dir)?; @@ -846,7 +849,7 @@ pub fn verify_snapshot_archive( snapshot_archive: P, snapshots_to_verify: Q, storages_to_verify: R, - compression: CompressionType, + archive_format: ArchiveFormat, ) where P: AsRef, Q: AsRef, @@ -854,7 +857,7 @@ pub fn verify_snapshot_archive( { let temp_dir = tempfile::TempDir::new().unwrap(); let unpack_dir = temp_dir.path(); - untar_snapshot_in(snapshot_archive, &unpack_dir, compression).unwrap(); + untar_snapshot_in(snapshot_archive, &unpack_dir, archive_format).unwrap(); // Check snapshots are the same let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR); @@ -885,7 +888,7 @@ pub fn snapshot_bank( snapshot_path: &Path, snapshot_package_output_path: &Path, snapshot_version: SnapshotVersion, - compression: &CompressionType, + archive_format: &ArchiveFormat, ) -> Result<()> { let storages: Vec<_> = root_bank.get_snapshot_storages(); let mut add_snapshot_time = Measure::start("add-snapshot-ms"); @@ -906,7 +909,7 @@ pub fn snapshot_bank( status_cache_slot_deltas, snapshot_package_output_path, storages, - compression.clone(), + archive_format.clone(), snapshot_version, )?; @@ -1031,15 +1034,15 @@ mod tests { fn test_snapshot_hash_of() { assert_eq!( snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())), - Some((42, Hash::default(), CompressionType::Bzip2)) + Some((42, Hash::default(), ArchiveFormat::TarBzip2)) ); assert_eq!( snapshot_hash_of(&format!("snapshot-43-{}.tar.zst", Hash::default())), - Some((43, Hash::default(), CompressionType::Zstd)) + Some((43, Hash::default(), ArchiveFormat::TarZstd)) ); assert_eq!( snapshot_hash_of(&format!("snapshot-42-{}.tar", Hash::default())), - Some((42, Hash::default(), CompressionType::NoCompression)) + Some((42, Hash::default(), ArchiveFormat::Tar)) ); assert!(snapshot_hash_of("invalid").is_none()); diff --git a/validator/src/main.rs b/validator/src/main.rs index 277b03975a..f2c6e2255f 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -29,7 +29,7 @@ use solana_ledger::blockstore_db::BlockstoreRecoveryMode; use solana_perf::recycler::enable_recycler_warming; use solana_runtime::{ accounts_index::AccountIndex, - bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion}, + bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion}, hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_utils::get_highest_snapshot_archive_path, }; @@ -1422,13 +1422,14 @@ pub fn main() { other than increasing the account balance"), ) .arg( - Arg::with_name("snapshot_compression") - .long("snapshot-compression") - .possible_values(&["bz2", "gzip", "zstd", "none"]) + Arg::with_name("snapshot_archive_format") + .long("snapshot-archive-format") + .alias("snapshot-compression") // Legacy name used by Solana v1.5.x and older + .possible_values(&["bz2", "gzip", "zstd", "tar", "none"]) .default_value("zstd") - .value_name("COMPRESSION_TYPE") + .value_name("ARCHIVE_TYPE") .takes_value(true) - .help("Type of snapshot compression to use."), + .help("Snapshot archive format to use."), ) .arg( Arg::with_name("max_genesis_archive_unpacked_size") @@ -1729,14 +1730,14 @@ pub fn main() { exit(1); }); - let snapshot_compression = { - let compression_str = value_t_or_exit!(matches, "snapshot_compression", String); - match compression_str.as_str() { - "bz2" => CompressionType::Bzip2, - "gzip" => CompressionType::Gzip, - "zstd" => CompressionType::Zstd, - "none" => CompressionType::NoCompression, - _ => panic!("Compression type not recognized: {}", compression_str), + let archive_format = { + let archive_format_str = value_t_or_exit!(matches, "snapshot_archive_format", String); + match archive_format_str.as_str() { + "bz2" => ArchiveFormat::TarBzip2, + "gzip" => ArchiveFormat::TarGzip, + "zstd" => ArchiveFormat::TarZstd, + "tar" | "none" => ArchiveFormat::Tar, + _ => panic!("Archive format not recognized: {}", archive_format_str), } }; @@ -1757,7 +1758,7 @@ pub fn main() { }, snapshot_path, snapshot_package_output_path: ledger_path.clone(), - compression: snapshot_compression, + archive_format, snapshot_version, });