gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add solana-test-validator --warp-slot argument

Browse Source
This commit is contained in:
Michael Vines
2021-01-21 18:34:51 -08:00
parent dd5a2ef05f
commit bf1943e489
12 changed files with 147 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ci/nits.sh
View File

@ -19,6 +19,7 @@ declare prints=(
# Parts of the tree that are expected to be print free
declare print_free_tree=(
':core/src/**.rs'
':^core/src/validator.rs'
':faucet/src/**.rs'
':ledger/src/**.rs'
':metrics/src/**.rs'

2
core/src/snapshot_packager_service.rs
View File

@ -158,7 +158,7 @@ mod tests {
let output_tar_path = snapshot_utils::get_snapshot_archive_path(
&snapshot_package_output_path,
&(42, Hash::default()),
&ArchiveFormat::TarBzip2,
ArchiveFormat::TarBzip2,
);
let snapshot_package = AccountsPackage::new(
5,

17
core/src/test_validator.rs
View File

@ -14,7 +14,7 @@ use {
},
solana_sdk::{
account::Account,
clock::DEFAULT_MS_PER_SLOT,
clock::{Slot, DEFAULT_MS_PER_SLOT},
commitment_config::CommitmentConfig,
fee_calculator::{FeeCalculator, FeeRateGovernor},
hash::Hash,
@ -48,6 +48,7 @@ pub struct TestValidatorGenesis {
rent: Rent,
rpc_config: JsonRpcConfig,
rpc_ports: Option<(u16, u16)>, // (JsonRpc, JsonRpcPubSub), None == random ports
warp_slot: Option<Slot>,
accounts: HashMap<Pubkey, Account>,
programs: Vec<ProgramInfo>,
}
@ -78,6 +79,11 @@ impl TestValidatorGenesis {
self
}
pub fn warp_slot(&mut self, warp_slot: Slot) -> &mut Self {
self.warp_slot = Some(warp_slot);
self
}
/// Add an account to the test environment
pub fn add_account(&mut self, address: Pubkey, account: Account) -> &mut Self {
self.accounts.insert(address, account);
@ -99,9 +105,10 @@ impl TestValidatorGenesis {
T: IntoIterator<Item = Pubkey>,
{
for address in addresses {
info!("Fetching {}...", address);
info!("Fetching {} over RPC...", address);
let account = rpc_client.get_account(&address).unwrap_or_else(|err| {
panic!("Failed to fetch {}: {}", address, err);
error!("Failed to fetch {}: {}", address, err);
crate::validator::abort();
});
self.add_account(address, account);
}
@ -280,7 +287,7 @@ impl TestValidator {
);
}
let genesis_config = create_genesis_config_with_leader_ex(
let mut genesis_config = create_genesis_config_with_leader_ex(
mint_lamports,
&mint_address,
&validator_identity.pubkey(),
@ -293,6 +300,7 @@ impl TestValidator {
solana_sdk::genesis_config::ClusterType::Development,
accounts.into_iter().collect(),
);
genesis_config.epoch_schedule = solana_sdk::epoch_schedule::EpochSchedule::without_warmup();
let ledger_path = match &config.ledger_path {
None => create_new_tmp_ledger!(&genesis_config).0,
@ -385,6 +393,7 @@ impl TestValidator {
snapshot_version: SnapshotVersion::default(),
}),
enforce_ulimit_nofile: false,
warp_slot: config.warp_slot,
..ValidatorConfig::default()
};

60
core/src/validator.rs
View File

@ -120,6 +120,7 @@ pub struct ValidatorConfig {
pub poh_pinned_cpu_core: usize,
pub account_indexes: HashSet<AccountIndex>,
pub accounts_db_caching_enabled: bool,
pub warp_slot: Option<Slot>,
}
impl Default for ValidatorConfig {
@ -166,6 +167,7 @@ impl Default for ValidatorConfig {
poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE,
account_indexes: HashSet::new(),
accounts_db_caching_enabled: false,
warp_slot: None,
}
}
}
@ -223,9 +225,14 @@ pub struct Validator {
}
// in the distant future, get rid of ::new()/exit() and use Result properly...
fn abort() -> ! {
pub(crate) fn abort() -> ! {
#[cfg(not(test))]
{
// standard error is usually redirected to a log file, cry for help on standard output as
// well
println!("Validator process aborted. The validator log may contain further details");
std::process::exit(1);
}
#[cfg(test)]
panic!("process::exit(1) is intercepted for friendly test failure...");
@ -848,6 +855,13 @@ fn post_process_restored_tower(
}
}
if let Some(warp_slot) = config.warp_slot {
// unconditionally relax tower requirement so that we can always restore tower
// from root bank after the warp
should_require_tower = false;
return Err(crate::consensus::TowerError::HardFork(warp_slot));
}
tower
})
.unwrap_or_else(|err| {
@ -991,6 +1005,50 @@ fn new_banks_from_ledger(
abort()
});
if let Some(warp_slot) = config.warp_slot {
let snapshot_config = config.snapshot_config.as_ref().unwrap_or_else(|| {
error!("warp slot requires a snapshot config");
abort();
});
let working_bank = bank_forks.working_bank();
if warp_slot <= working_bank.slot() {
error!(
"warp slot ({}) cannot be less than the working bank slot ({})",
warp_slot,
working_bank.slot()
);
abort();
}
info!("warping to slot {}", warp_slot);
bank_forks.insert(Bank::warp_from_parent(
&bank_forks.root_bank(),
&Pubkey::default(),
warp_slot,
));
bank_forks.set_root(
warp_slot,
&solana_runtime::accounts_background_service::ABSRequestSender::default(),
Some(warp_slot),
);
leader_schedule_cache.set_root(&bank_forks.root_bank());
let archive_file = solana_runtime::snapshot_utils::bank_to_snapshot_archive(
ledger_path,
&bank_forks.root_bank(),
None,
&snapshot_config.snapshot_package_output_path,
snapshot_config.archive_format,
)
.unwrap_or_else(|err| {
error!("Unable to create snapshot: {}", err);
abort();
});
info!("created snapshot: {}", archive_file.display());
}
let tower = post_process_restored_tower(
restored_tower,
&validator_identity,

4
core/tests/snapshots.rs
View File

@ -155,7 +155,7 @@ mod tests {
snapshot_utils::get_snapshot_archive_path(
snapshot_package_output_path,
&(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
&ArchiveFormat::TarBzip2,
ArchiveFormat::TarBzip2,
),
ArchiveFormat::TarBzip2,
old_genesis_config,
@ -385,7 +385,7 @@ mod tests {
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
snapshot_package_output_path,
&(slot, accounts_hash),
&ArchiveFormat::TarBzip2,
ArchiveFormat::TarBzip2,
));
}
}

2
download-utils/src/lib.rs
View File

@ -182,7 +182,7 @@ pub fn download_snapshot(
let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
ledger_path,
&desired_snapshot_hash,
compression,
*compression,
);
if desired_snapshot_package.is_file() {

1
ledger-tool/src/main.rs
View File

@ -1918,6 +1918,7 @@ fn main() {
&bank,
Some(snapshot_version),
output_directory,
ArchiveFormat::TarZstd,
)
.unwrap_or_else(|err| {
eprintln!("Unable to create snapshot: {}", err);

6
local-cluster/tests/local_cluster.rs
View File

@ -1054,7 +1054,7 @@ fn test_snapshot_download() {
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
&validator_snapshot_test_config.snapshot_output_path,
&archive_snapshot_hash,
&ArchiveFormat::TarBzip2,
ArchiveFormat::TarBzip2,
);
// Download the snapshot, then boot a validator from it.
@ -1124,7 +1124,7 @@ fn test_snapshot_restart_tower() {
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
&validator_snapshot_test_config.snapshot_output_path,
&archive_snapshot_hash,
&ArchiveFormat::TarBzip2,
ArchiveFormat::TarBzip2,
);
fs::hard_link(archive_filename, &validator_archive_path).unwrap();
@ -1189,7 +1189,7 @@ fn test_snapshots_blockstore_floor() {
let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
&validator_snapshot_test_config.snapshot_output_path,
&(archive_slot, archive_hash),
&ArchiveFormat::TarBzip2,
ArchiveFormat::TarBzip2,
);
fs::hard_link(archive_filename, &validator_archive_path).unwrap();
let slot_floor = archive_slot;

2
runtime/src/bank_forks.rs
View File

@ -17,7 +17,7 @@ use std::{
pub use crate::snapshot_utils::SnapshotVersion;
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum ArchiveFormat {
TarBzip2,
TarGzip,

13
runtime/src/snapshot_utils.rs
View File

@ -172,7 +172,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
let snapshot_package_output_file = get_snapshot_archive_path(
&snapshot_package_output_path,
&(bank.slot(), bank.get_accounts_hash()),
&archive_format,
archive_format,
);
let package = AccountsPackage::new(
@ -190,7 +190,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
Ok(package)
}
fn get_archive_ext(archive_format: &ArchiveFormat) -> &'static str {
fn get_archive_ext(archive_format: ArchiveFormat) -> &'static str {
match archive_format {
ArchiveFormat::TarBzip2 => ".tar.bz2",
ArchiveFormat::TarGzip => ".tar.gz",
@ -291,7 +291,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?;
}
let file_ext = get_archive_ext(&snapshot_package.archive_format);
let file_ext = get_archive_ext(snapshot_package.archive_format);
// Tar the staging directory into the archive at `archive_path`
//
@ -633,7 +633,7 @@ pub fn bank_from_archive<P: AsRef<Path>>(
pub fn get_snapshot_archive_path<P: AsRef<Path>>(
snapshot_output_dir: P,
snapshot_hash: &(Slot, Hash),
archive_format: &ArchiveFormat,
archive_format: ArchiveFormat,
) -> PathBuf {
snapshot_output_dir.as_ref().join(format!(
"snapshot-{}-{}{}",
@ -906,7 +906,7 @@ pub fn snapshot_bank(
status_cache_slot_deltas,
snapshot_package_output_path,
storages,
archive_format.clone(),
*archive_format,
snapshot_version,
)?;
@ -922,6 +922,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
bank: &Bank,
snapshot_version: Option<SnapshotVersion>,
snapshot_package_output_path: Q,
archive_format: ArchiveFormat,
) -> Result<PathBuf> {
let snapshot_version = snapshot_version.unwrap_or_default();
@ -943,7 +944,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
bank.src.slot_deltas(&bank.src.roots()),
snapshot_package_output_path,
storages,
ArchiveFormat::TarZstd,
archive_format,
snapshot_version,
)?;

7
sdk/program/src/epoch_schedule.rs
View File

@ -51,6 +51,13 @@ impl EpochSchedule {
pub fn new(slots_per_epoch: u64) -> Self {
Self::custom(slots_per_epoch, slots_per_epoch, true)
}
pub fn without_warmup() -> Self {
Self::custom(
DEFAULT_SLOTS_PER_EPOCH,
DEFAULT_LEADER_SCHEDULE_SLOT_OFFSET,
false,
)
}
pub fn custom(slots_per_epoch: u64, leader_schedule_slot_offset: u64, warmup: bool) -> Self {
assert!(slots_per_epoch >= MINIMUM_SLOTS_PER_EPOCH as u64);
let (first_normal_epoch, first_normal_slot) = if warmup {

54
validator/src/bin/solana-test-validator.rs
View File

@ -6,7 +6,8 @@ use {
solana_clap_utils::{
input_parsers::{pubkey_of, pubkeys_of},
input_validators::{
is_pubkey, is_pubkey_or_keypair, is_url_or_moniker, normalize_to_url_if_moniker,
is_pubkey, is_pubkey_or_keypair, is_slot, is_url_or_moniker,
normalize_to_url_if_moniker,
},
},
solana_client::{client_error, rpc_client::RpcClient, rpc_request},
@ -171,6 +172,22 @@ fn main() {
If the ledger already exists then this parameter is silently ignored",
),
)
.arg(
Arg::with_name("warp_slot")
.required(false)
.long("warp-slot")
.short("w")
.takes_value(true)
.value_name("WARP_SLOT")
.validator(is_slot)
.min_values(0)
.max_values(1)
.help(
"Warp the ledger to WARP_SLOT after starting the validator. \
If no slot is provided then the current slot of the cluster \
referenced by the --url argument will be used",
),
)
.get_matches();
let cli_config = if let Some(config_file) = matches.value_of("config_file") {
@ -179,7 +196,9 @@ fn main() {
solana_cli_config::Config::default()
};
let json_rpc_url = value_t!(matches, "json_rpc_url", String).map(normalize_to_url_if_moniker);
let cluster_rpc_client = value_t!(matches, "json_rpc_url", String)
.map(normalize_to_url_if_moniker)
.map(RpcClient::new);
let mint_address = pubkey_of(&matches, "mint_address").unwrap_or_else(|| {
read_keypair_file(&cli_config.keypair_path)
@ -237,6 +256,25 @@ fn main() {
.map(|v| v.into_iter().collect())
.unwrap_or_default();
let warp_slot = if matches.is_present("warp_slot") {
Some(match matches.value_of("warp_slot") {
Some(_) => value_t_or_exit!(matches, "warp_slot", Slot),
None => {
cluster_rpc_client.as_ref().unwrap_or_else(|_| {
eprintln!("The --url argument must be provided if --warp-slot/-w is used without an explicit slot");
exit(1);
}).get_slot()
.unwrap_or_else(|err| {
eprintln!("Unable to get current cluster slot: {}", err);
exit(1);
})
}
})
} else {
None
};
if !ledger_path.exists() {
fs::create_dir(&ledger_path).unwrap_or_else(|err| {
eprintln!(
@ -345,8 +383,16 @@ fn main() {
.add_programs_with_path(&programs);
if !clone_accounts.is_empty() {
let rpc_client = RpcClient::new(json_rpc_url.unwrap());
genesis.clone_accounts(clone_accounts, &rpc_client);
genesis.clone_accounts(
clone_accounts,
cluster_rpc_client
.as_ref()
.expect("bug: --url argument missing?"),
);
}
if let Some(warp_slot) = warp_slot {
genesis.warp_slot(warp_slot);
}
genesis.start_with_mint_address(mint_address)
}