diff --git a/ci/nits.sh b/ci/nits.sh index 2162d8fae0..93a71614fe 100755 --- a/ci/nits.sh +++ b/ci/nits.sh @@ -19,6 +19,7 @@ declare prints=( # Parts of the tree that are expected to be print free declare print_free_tree=( ':core/src/**.rs' + ':^core/src/validator.rs' ':faucet/src/**.rs' ':ledger/src/**.rs' ':metrics/src/**.rs' diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index c9f97fcc9a..1716406c54 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -158,7 +158,7 @@ mod tests { let output_tar_path = snapshot_utils::get_snapshot_archive_path( &snapshot_package_output_path, &(42, Hash::default()), - &ArchiveFormat::TarBzip2, + ArchiveFormat::TarBzip2, ); let snapshot_package = AccountsPackage::new( 5, diff --git a/core/src/test_validator.rs b/core/src/test_validator.rs index bf9a017158..b1dca2f5ed 100644 --- a/core/src/test_validator.rs +++ b/core/src/test_validator.rs @@ -14,7 +14,7 @@ use { }, solana_sdk::{ account::Account, - clock::DEFAULT_MS_PER_SLOT, + clock::{Slot, DEFAULT_MS_PER_SLOT}, commitment_config::CommitmentConfig, fee_calculator::{FeeCalculator, FeeRateGovernor}, hash::Hash, @@ -48,6 +48,7 @@ pub struct TestValidatorGenesis { rent: Rent, rpc_config: JsonRpcConfig, rpc_ports: Option<(u16, u16)>, // (JsonRpc, JsonRpcPubSub), None == random ports + warp_slot: Option, accounts: HashMap, programs: Vec, } @@ -78,6 +79,11 @@ impl TestValidatorGenesis { self } + pub fn warp_slot(&mut self, warp_slot: Slot) -> &mut Self { + self.warp_slot = Some(warp_slot); + self + } + /// Add an account to the test environment pub fn add_account(&mut self, address: Pubkey, account: Account) -> &mut Self { self.accounts.insert(address, account); @@ -99,9 +105,10 @@ impl TestValidatorGenesis { T: IntoIterator, { for address in addresses { - info!("Fetching {}...", address); + info!("Fetching {} over RPC...", address); let account = rpc_client.get_account(&address).unwrap_or_else(|err| { - panic!("Failed to fetch {}: {}", address, err); + error!("Failed to fetch {}: {}", address, err); + crate::validator::abort(); }); self.add_account(address, account); } @@ -280,7 +287,7 @@ impl TestValidator { ); } - let genesis_config = create_genesis_config_with_leader_ex( + let mut genesis_config = create_genesis_config_with_leader_ex( mint_lamports, &mint_address, &validator_identity.pubkey(), @@ -293,6 +300,7 @@ impl TestValidator { solana_sdk::genesis_config::ClusterType::Development, accounts.into_iter().collect(), ); + genesis_config.epoch_schedule = solana_sdk::epoch_schedule::EpochSchedule::without_warmup(); let ledger_path = match &config.ledger_path { None => create_new_tmp_ledger!(&genesis_config).0, @@ -385,6 +393,7 @@ impl TestValidator { snapshot_version: SnapshotVersion::default(), }), enforce_ulimit_nofile: false, + warp_slot: config.warp_slot, ..ValidatorConfig::default() }; diff --git a/core/src/validator.rs b/core/src/validator.rs index df92965295..bb64a9b719 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -120,6 +120,7 @@ pub struct ValidatorConfig { pub poh_pinned_cpu_core: usize, pub account_indexes: HashSet, pub accounts_db_caching_enabled: bool, + pub warp_slot: Option, } impl Default for ValidatorConfig { @@ -166,6 +167,7 @@ impl Default for ValidatorConfig { poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE, account_indexes: HashSet::new(), accounts_db_caching_enabled: false, + warp_slot: None, } } } @@ -223,9 +225,14 @@ pub struct Validator { } // in the distant future, get rid of ::new()/exit() and use Result properly... -fn abort() -> ! { +pub(crate) fn abort() -> ! { #[cfg(not(test))] - std::process::exit(1); + { + // standard error is usually redirected to a log file, cry for help on standard output as + // well + println!("Validator process aborted. The validator log may contain further details"); + std::process::exit(1); + } #[cfg(test)] panic!("process::exit(1) is intercepted for friendly test failure..."); @@ -848,6 +855,13 @@ fn post_process_restored_tower( } } + if let Some(warp_slot) = config.warp_slot { + // unconditionally relax tower requirement so that we can always restore tower + // from root bank after the warp + should_require_tower = false; + return Err(crate::consensus::TowerError::HardFork(warp_slot)); + } + tower }) .unwrap_or_else(|err| { @@ -991,6 +1005,50 @@ fn new_banks_from_ledger( abort() }); + if let Some(warp_slot) = config.warp_slot { + let snapshot_config = config.snapshot_config.as_ref().unwrap_or_else(|| { + error!("warp slot requires a snapshot config"); + abort(); + }); + + let working_bank = bank_forks.working_bank(); + + if warp_slot <= working_bank.slot() { + error!( + "warp slot ({}) cannot be less than the working bank slot ({})", + warp_slot, + working_bank.slot() + ); + abort(); + } + info!("warping to slot {}", warp_slot); + + bank_forks.insert(Bank::warp_from_parent( + &bank_forks.root_bank(), + &Pubkey::default(), + warp_slot, + )); + bank_forks.set_root( + warp_slot, + &solana_runtime::accounts_background_service::ABSRequestSender::default(), + Some(warp_slot), + ); + leader_schedule_cache.set_root(&bank_forks.root_bank()); + + let archive_file = solana_runtime::snapshot_utils::bank_to_snapshot_archive( + ledger_path, + &bank_forks.root_bank(), + None, + &snapshot_config.snapshot_package_output_path, + snapshot_config.archive_format, + ) + .unwrap_or_else(|err| { + error!("Unable to create snapshot: {}", err); + abort(); + }); + info!("created snapshot: {}", archive_file.display()); + } + let tower = post_process_restored_tower( restored_tower, &validator_identity, diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 1f36ecf27e..3e4743240a 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -155,7 +155,7 @@ mod tests { snapshot_utils::get_snapshot_archive_path( snapshot_package_output_path, &(old_last_bank.slot(), old_last_bank.get_accounts_hash()), - &ArchiveFormat::TarBzip2, + ArchiveFormat::TarBzip2, ), ArchiveFormat::TarBzip2, old_genesis_config, @@ -385,7 +385,7 @@ mod tests { saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( snapshot_package_output_path, &(slot, accounts_hash), - &ArchiveFormat::TarBzip2, + ArchiveFormat::TarBzip2, )); } } diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index 050afade87..82da7bfb24 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -182,7 +182,7 @@ pub fn download_snapshot( let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path( ledger_path, &desired_snapshot_hash, - compression, + *compression, ); if desired_snapshot_package.is_file() { diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 7ecefe08fc..c16d9eb3c0 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1918,6 +1918,7 @@ fn main() { &bank, Some(snapshot_version), output_directory, + ArchiveFormat::TarZstd, ) .unwrap_or_else(|err| { eprintln!("Unable to create snapshot: {}", err); diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 5b6772452d..789d49751a 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -1054,7 +1054,7 @@ fn test_snapshot_download() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &archive_snapshot_hash, - &ArchiveFormat::TarBzip2, + ArchiveFormat::TarBzip2, ); // Download the snapshot, then boot a validator from it. @@ -1124,7 +1124,7 @@ fn test_snapshot_restart_tower() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &archive_snapshot_hash, - &ArchiveFormat::TarBzip2, + ArchiveFormat::TarBzip2, ); fs::hard_link(archive_filename, &validator_archive_path).unwrap(); @@ -1189,7 +1189,7 @@ fn test_snapshots_blockstore_floor() { let validator_archive_path = snapshot_utils::get_snapshot_archive_path( &validator_snapshot_test_config.snapshot_output_path, &(archive_slot, archive_hash), - &ArchiveFormat::TarBzip2, + ArchiveFormat::TarBzip2, ); fs::hard_link(archive_filename, &validator_archive_path).unwrap(); let slot_floor = archive_slot; diff --git a/runtime/src/bank_forks.rs b/runtime/src/bank_forks.rs index 69327a024e..52ad799276 100644 --- a/runtime/src/bank_forks.rs +++ b/runtime/src/bank_forks.rs @@ -17,7 +17,7 @@ use std::{ pub use crate::snapshot_utils::SnapshotVersion; -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum ArchiveFormat { TarBzip2, TarGzip, diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 5feb9d75bc..2c2e7bb071 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -172,7 +172,7 @@ pub fn package_snapshot, Q: AsRef>( let snapshot_package_output_file = get_snapshot_archive_path( &snapshot_package_output_path, &(bank.slot(), bank.get_accounts_hash()), - &archive_format, + archive_format, ); let package = AccountsPackage::new( @@ -190,7 +190,7 @@ pub fn package_snapshot, Q: AsRef>( Ok(package) } -fn get_archive_ext(archive_format: &ArchiveFormat) -> &'static str { +fn get_archive_ext(archive_format: ArchiveFormat) -> &'static str { match archive_format { ArchiveFormat::TarBzip2 => ".tar.bz2", ArchiveFormat::TarGzip => ".tar.gz", @@ -291,7 +291,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?; } - let file_ext = get_archive_ext(&snapshot_package.archive_format); + let file_ext = get_archive_ext(snapshot_package.archive_format); // Tar the staging directory into the archive at `archive_path` // @@ -633,7 +633,7 @@ pub fn bank_from_archive>( pub fn get_snapshot_archive_path>( snapshot_output_dir: P, snapshot_hash: &(Slot, Hash), - archive_format: &ArchiveFormat, + archive_format: ArchiveFormat, ) -> PathBuf { snapshot_output_dir.as_ref().join(format!( "snapshot-{}-{}{}", @@ -906,7 +906,7 @@ pub fn snapshot_bank( status_cache_slot_deltas, snapshot_package_output_path, storages, - archive_format.clone(), + *archive_format, snapshot_version, )?; @@ -922,6 +922,7 @@ pub fn bank_to_snapshot_archive, Q: AsRef>( bank: &Bank, snapshot_version: Option, snapshot_package_output_path: Q, + archive_format: ArchiveFormat, ) -> Result { let snapshot_version = snapshot_version.unwrap_or_default(); @@ -943,7 +944,7 @@ pub fn bank_to_snapshot_archive, Q: AsRef>( bank.src.slot_deltas(&bank.src.roots()), snapshot_package_output_path, storages, - ArchiveFormat::TarZstd, + archive_format, snapshot_version, )?; diff --git a/sdk/program/src/epoch_schedule.rs b/sdk/program/src/epoch_schedule.rs index 87c5368498..6b0e9225c2 100644 --- a/sdk/program/src/epoch_schedule.rs +++ b/sdk/program/src/epoch_schedule.rs @@ -51,6 +51,13 @@ impl EpochSchedule { pub fn new(slots_per_epoch: u64) -> Self { Self::custom(slots_per_epoch, slots_per_epoch, true) } + pub fn without_warmup() -> Self { + Self::custom( + DEFAULT_SLOTS_PER_EPOCH, + DEFAULT_LEADER_SCHEDULE_SLOT_OFFSET, + false, + ) + } pub fn custom(slots_per_epoch: u64, leader_schedule_slot_offset: u64, warmup: bool) -> Self { assert!(slots_per_epoch >= MINIMUM_SLOTS_PER_EPOCH as u64); let (first_normal_epoch, first_normal_slot) = if warmup { diff --git a/validator/src/bin/solana-test-validator.rs b/validator/src/bin/solana-test-validator.rs index a6ce15a37d..e52549bb12 100644 --- a/validator/src/bin/solana-test-validator.rs +++ b/validator/src/bin/solana-test-validator.rs @@ -6,7 +6,8 @@ use { solana_clap_utils::{ input_parsers::{pubkey_of, pubkeys_of}, input_validators::{ - is_pubkey, is_pubkey_or_keypair, is_url_or_moniker, normalize_to_url_if_moniker, + is_pubkey, is_pubkey_or_keypair, is_slot, is_url_or_moniker, + normalize_to_url_if_moniker, }, }, solana_client::{client_error, rpc_client::RpcClient, rpc_request}, @@ -171,6 +172,22 @@ fn main() { If the ledger already exists then this parameter is silently ignored", ), ) + .arg( + Arg::with_name("warp_slot") + .required(false) + .long("warp-slot") + .short("w") + .takes_value(true) + .value_name("WARP_SLOT") + .validator(is_slot) + .min_values(0) + .max_values(1) + .help( + "Warp the ledger to WARP_SLOT after starting the validator. \ + If no slot is provided then the current slot of the cluster \ + referenced by the --url argument will be used", + ), + ) .get_matches(); let cli_config = if let Some(config_file) = matches.value_of("config_file") { @@ -179,7 +196,9 @@ fn main() { solana_cli_config::Config::default() }; - let json_rpc_url = value_t!(matches, "json_rpc_url", String).map(normalize_to_url_if_moniker); + let cluster_rpc_client = value_t!(matches, "json_rpc_url", String) + .map(normalize_to_url_if_moniker) + .map(RpcClient::new); let mint_address = pubkey_of(&matches, "mint_address").unwrap_or_else(|| { read_keypair_file(&cli_config.keypair_path) @@ -237,6 +256,25 @@ fn main() { .map(|v| v.into_iter().collect()) .unwrap_or_default(); + let warp_slot = if matches.is_present("warp_slot") { + Some(match matches.value_of("warp_slot") { + Some(_) => value_t_or_exit!(matches, "warp_slot", Slot), + None => { + cluster_rpc_client.as_ref().unwrap_or_else(|_| { + eprintln!("The --url argument must be provided if --warp-slot/-w is used without an explicit slot"); + exit(1); + + }).get_slot() + .unwrap_or_else(|err| { + eprintln!("Unable to get current cluster slot: {}", err); + exit(1); + }) + } + }) + } else { + None + }; + if !ledger_path.exists() { fs::create_dir(&ledger_path).unwrap_or_else(|err| { eprintln!( @@ -345,8 +383,16 @@ fn main() { .add_programs_with_path(&programs); if !clone_accounts.is_empty() { - let rpc_client = RpcClient::new(json_rpc_url.unwrap()); - genesis.clone_accounts(clone_accounts, &rpc_client); + genesis.clone_accounts( + clone_accounts, + cluster_rpc_client + .as_ref() + .expect("bug: --url argument missing?"), + ); + } + + if let Some(warp_slot) = warp_slot { + genesis.warp_slot(warp_slot); } genesis.start_with_mint_address(mint_address) }