Add solana-test-validator --warp-slot argument

ci/nits.sh

@@ -19,6 +19,7 @@ declare prints=(
 # Parts of the tree that are expected to be print free
 declare print_free_tree=(
   ':core/src/**.rs'
+  ':^core/src/validator.rs'
   ':faucet/src/**.rs'
   ':ledger/src/**.rs'
   ':metrics/src/**.rs'

core/src/snapshot_packager_service.rs

@@ -158,7 +158,7 @@ mod tests {
         let output_tar_path = snapshot_utils::get_snapshot_archive_path(
             &snapshot_package_output_path,
             &(42, Hash::default()),
-            &ArchiveFormat::TarBzip2,
+            ArchiveFormat::TarBzip2,
         );
         let snapshot_package = AccountsPackage::new(
             5,

core/src/test_validator.rs

@@ -14,7 +14,7 @@ use {
     },
     solana_sdk::{
         account::Account,
-        clock::DEFAULT_MS_PER_SLOT,
+        clock::{Slot, DEFAULT_MS_PER_SLOT},
         commitment_config::CommitmentConfig,
         fee_calculator::{FeeCalculator, FeeRateGovernor},
         hash::Hash,
@@ -48,6 +48,7 @@ pub struct TestValidatorGenesis {
     rent: Rent,
     rpc_config: JsonRpcConfig,
     rpc_ports: Option<(u16, u16)>, // (JsonRpc, JsonRpcPubSub), None == random ports
+    warp_slot: Option<Slot>,
     accounts: HashMap<Pubkey, Account>,
     programs: Vec<ProgramInfo>,
 }
@@ -78,6 +79,11 @@ impl TestValidatorGenesis {
         self
     }
 
+    pub fn warp_slot(&mut self, warp_slot: Slot) -> &mut Self {
+        self.warp_slot = Some(warp_slot);
+        self
+    }
+
     /// Add an account to the test environment
     pub fn add_account(&mut self, address: Pubkey, account: Account) -> &mut Self {
         self.accounts.insert(address, account);
@@ -99,9 +105,10 @@ impl TestValidatorGenesis {
         T: IntoIterator<Item = Pubkey>,
     {
         for address in addresses {
-            info!("Fetching {}...", address);
+            info!("Fetching {} over RPC...", address);
             let account = rpc_client.get_account(&address).unwrap_or_else(|err| {
-                panic!("Failed to fetch {}: {}", address, err);
+                error!("Failed to fetch {}: {}", address, err);
+                crate::validator::abort();
             });
             self.add_account(address, account);
         }
@@ -280,7 +287,7 @@ impl TestValidator {
             );
         }
 
-        let genesis_config = create_genesis_config_with_leader_ex(
+        let mut genesis_config = create_genesis_config_with_leader_ex(
             mint_lamports,
             &mint_address,
             &validator_identity.pubkey(),
@@ -293,6 +300,7 @@ impl TestValidator {
             solana_sdk::genesis_config::ClusterType::Development,
             accounts.into_iter().collect(),
         );
+        genesis_config.epoch_schedule = solana_sdk::epoch_schedule::EpochSchedule::without_warmup();
 
         let ledger_path = match &config.ledger_path {
             None => create_new_tmp_ledger!(&genesis_config).0,
@@ -385,6 +393,7 @@ impl TestValidator {
                 snapshot_version: SnapshotVersion::default(),
             }),
             enforce_ulimit_nofile: false,
+            warp_slot: config.warp_slot,
             ..ValidatorConfig::default()
         };
 

core/src/validator.rs

@@ -120,6 +120,7 @@ pub struct ValidatorConfig {
     pub poh_pinned_cpu_core: usize,
     pub account_indexes: HashSet<AccountIndex>,
     pub accounts_db_caching_enabled: bool,
+    pub warp_slot: Option<Slot>,
 }
 
 impl Default for ValidatorConfig {
@@ -166,6 +167,7 @@ impl Default for ValidatorConfig {
             poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE,
             account_indexes: HashSet::new(),
             accounts_db_caching_enabled: false,
+            warp_slot: None,
         }
     }
 }
@@ -223,9 +225,14 @@ pub struct Validator {
 }
 
 // in the distant future, get rid of ::new()/exit() and use Result properly...
-fn abort() -> ! {
+pub(crate) fn abort() -> ! {
     #[cfg(not(test))]
-    std::process::exit(1);
+    {
+        // standard error is usually redirected to a log file, cry for help on standard output as
+        // well
+        println!("Validator process aborted. The validator log may contain further details");
+        std::process::exit(1);
+    }
 
     #[cfg(test)]
     panic!("process::exit(1) is intercepted for friendly test failure...");
@@ -848,6 +855,13 @@ fn post_process_restored_tower(
                 }
             }
 
+            if let Some(warp_slot) = config.warp_slot {
+                // unconditionally relax tower requirement so that we can always restore tower
+                // from root bank after the warp
+                should_require_tower = false;
+                return Err(crate::consensus::TowerError::HardFork(warp_slot));
+            }
+
             tower
         })
         .unwrap_or_else(|err| {
@@ -991,6 +1005,50 @@ fn new_banks_from_ledger(
         abort()
     });
 
+    if let Some(warp_slot) = config.warp_slot {
+        let snapshot_config = config.snapshot_config.as_ref().unwrap_or_else(|| {
+            error!("warp slot requires a snapshot config");
+            abort();
+        });
+
+        let working_bank = bank_forks.working_bank();
+
+        if warp_slot <= working_bank.slot() {
+            error!(
+                "warp slot ({}) cannot be less than the working bank slot ({})",
+                warp_slot,
+                working_bank.slot()
+            );
+            abort();
+        }
+        info!("warping to slot {}", warp_slot);
+
+        bank_forks.insert(Bank::warp_from_parent(
+            &bank_forks.root_bank(),
+            &Pubkey::default(),
+            warp_slot,
+        ));
+        bank_forks.set_root(
+            warp_slot,
+            &solana_runtime::accounts_background_service::ABSRequestSender::default(),
+            Some(warp_slot),
+        );
+        leader_schedule_cache.set_root(&bank_forks.root_bank());
+
+        let archive_file = solana_runtime::snapshot_utils::bank_to_snapshot_archive(
+            ledger_path,
+            &bank_forks.root_bank(),
+            None,
+            &snapshot_config.snapshot_package_output_path,
+            snapshot_config.archive_format,
+        )
+        .unwrap_or_else(|err| {
+            error!("Unable to create snapshot: {}", err);
+            abort();
+        });
+        info!("created snapshot: {}", archive_file.display());
+    }
+
     let tower = post_process_restored_tower(
         restored_tower,
         &validator_identity,

core/tests/snapshots.rs

@@ -155,7 +155,7 @@ mod tests {
             snapshot_utils::get_snapshot_archive_path(
                 snapshot_package_output_path,
                 &(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
-                &ArchiveFormat::TarBzip2,
+                ArchiveFormat::TarBzip2,
             ),
             ArchiveFormat::TarBzip2,
             old_genesis_config,
@@ -385,7 +385,7 @@ mod tests {
                 saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
                     snapshot_package_output_path,
                     &(slot, accounts_hash),
-                    &ArchiveFormat::TarBzip2,
+                    ArchiveFormat::TarBzip2,
                 ));
             }
         }

download-utils/src/lib.rs

@@ -182,7 +182,7 @@ pub fn download_snapshot(
         let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
             ledger_path,
             &desired_snapshot_hash,
-            compression,
+            *compression,
         );
 
         if desired_snapshot_package.is_file() {

ledger-tool/src/main.rs

@@ -1918,6 +1918,7 @@ fn main() {
                         &bank,
                         Some(snapshot_version),
                         output_directory,
+                        ArchiveFormat::TarZstd,
                     )
                     .unwrap_or_else(|err| {
                         eprintln!("Unable to create snapshot: {}", err);

local-cluster/tests/local_cluster.rs

@@ -1054,7 +1054,7 @@ fn test_snapshot_download() {
     let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
         &validator_snapshot_test_config.snapshot_output_path,
         &archive_snapshot_hash,
-        &ArchiveFormat::TarBzip2,
+        ArchiveFormat::TarBzip2,
     );
 
     // Download the snapshot, then boot a validator from it.
@@ -1124,7 +1124,7 @@ fn test_snapshot_restart_tower() {
     let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
         &validator_snapshot_test_config.snapshot_output_path,
         &archive_snapshot_hash,
-        &ArchiveFormat::TarBzip2,
+        ArchiveFormat::TarBzip2,
     );
     fs::hard_link(archive_filename, &validator_archive_path).unwrap();
 
@@ -1189,7 +1189,7 @@ fn test_snapshots_blockstore_floor() {
     let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
         &validator_snapshot_test_config.snapshot_output_path,
         &(archive_slot, archive_hash),
-        &ArchiveFormat::TarBzip2,
+        ArchiveFormat::TarBzip2,
     );
     fs::hard_link(archive_filename, &validator_archive_path).unwrap();
     let slot_floor = archive_slot;

runtime/src/bank_forks.rs

@@ -17,7 +17,7 @@ use std::{
 
 pub use crate::snapshot_utils::SnapshotVersion;
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum ArchiveFormat {
     TarBzip2,
     TarGzip,

runtime/src/snapshot_utils.rs

@@ -172,7 +172,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
     let snapshot_package_output_file = get_snapshot_archive_path(
         &snapshot_package_output_path,
         &(bank.slot(), bank.get_accounts_hash()),
-        &archive_format,
+        archive_format,
     );
 
     let package = AccountsPackage::new(
@@ -190,7 +190,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
     Ok(package)
 }
 
-fn get_archive_ext(archive_format: &ArchiveFormat) -> &'static str {
+fn get_archive_ext(archive_format: ArchiveFormat) -> &'static str {
     match archive_format {
         ArchiveFormat::TarBzip2 => ".tar.bz2",
         ArchiveFormat::TarGzip => ".tar.gz",
@@ -291,7 +291,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
         f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?;
     }
 
-    let file_ext = get_archive_ext(&snapshot_package.archive_format);
+    let file_ext = get_archive_ext(snapshot_package.archive_format);
 
     // Tar the staging directory into the archive at `archive_path`
     //
@@ -633,7 +633,7 @@ pub fn bank_from_archive<P: AsRef<Path>>(
 pub fn get_snapshot_archive_path<P: AsRef<Path>>(
     snapshot_output_dir: P,
     snapshot_hash: &(Slot, Hash),
-    archive_format: &ArchiveFormat,
+    archive_format: ArchiveFormat,
 ) -> PathBuf {
     snapshot_output_dir.as_ref().join(format!(
         "snapshot-{}-{}{}",
@@ -906,7 +906,7 @@ pub fn snapshot_bank(
         status_cache_slot_deltas,
         snapshot_package_output_path,
         storages,
-        archive_format.clone(),
+        *archive_format,
         snapshot_version,
     )?;
 
@@ -922,6 +922,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
     bank: &Bank,
     snapshot_version: Option<SnapshotVersion>,
     snapshot_package_output_path: Q,
+    archive_format: ArchiveFormat,
 ) -> Result<PathBuf> {
     let snapshot_version = snapshot_version.unwrap_or_default();
 
@@ -943,7 +944,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
         bank.src.slot_deltas(&bank.src.roots()),
         snapshot_package_output_path,
         storages,
-        ArchiveFormat::TarZstd,
+        archive_format,
         snapshot_version,
     )?;
 

sdk/program/src/epoch_schedule.rs

@@ -51,6 +51,13 @@ impl EpochSchedule {
     pub fn new(slots_per_epoch: u64) -> Self {
         Self::custom(slots_per_epoch, slots_per_epoch, true)
     }
+    pub fn without_warmup() -> Self {
+        Self::custom(
+            DEFAULT_SLOTS_PER_EPOCH,
+            DEFAULT_LEADER_SCHEDULE_SLOT_OFFSET,
+            false,
+        )
+    }
     pub fn custom(slots_per_epoch: u64, leader_schedule_slot_offset: u64, warmup: bool) -> Self {
         assert!(slots_per_epoch >= MINIMUM_SLOTS_PER_EPOCH as u64);
         let (first_normal_epoch, first_normal_slot) = if warmup {

validator/src/bin/solana-test-validator.rs

@@ -6,7 +6,8 @@ use {
     solana_clap_utils::{
         input_parsers::{pubkey_of, pubkeys_of},
         input_validators::{
-            is_pubkey, is_pubkey_or_keypair, is_url_or_moniker, normalize_to_url_if_moniker,
+            is_pubkey, is_pubkey_or_keypair, is_slot, is_url_or_moniker,
+            normalize_to_url_if_moniker,
         },
     },
     solana_client::{client_error, rpc_client::RpcClient, rpc_request},
@@ -171,6 +172,22 @@ fn main() {
                      If the ledger already exists then this parameter is silently ignored",
                 ),
         )
+        .arg(
+            Arg::with_name("warp_slot")
+                .required(false)
+                .long("warp-slot")
+                .short("w")
+                .takes_value(true)
+                .value_name("WARP_SLOT")
+                .validator(is_slot)
+                .min_values(0)
+                .max_values(1)
+                .help(
+                    "Warp the ledger to WARP_SLOT after starting the validator. \
+                        If no slot is provided then the current slot of the cluster \
+                        referenced by the --url argument will be used",
+                ),
+        )
         .get_matches();
 
     let cli_config = if let Some(config_file) = matches.value_of("config_file") {
@@ -179,7 +196,9 @@ fn main() {
         solana_cli_config::Config::default()
     };
 
-    let json_rpc_url = value_t!(matches, "json_rpc_url", String).map(normalize_to_url_if_moniker);
+    let cluster_rpc_client = value_t!(matches, "json_rpc_url", String)
+        .map(normalize_to_url_if_moniker)
+        .map(RpcClient::new);
 
     let mint_address = pubkey_of(&matches, "mint_address").unwrap_or_else(|| {
         read_keypair_file(&cli_config.keypair_path)
@@ -237,6 +256,25 @@ fn main() {
         .map(|v| v.into_iter().collect())
         .unwrap_or_default();
 
+    let warp_slot = if matches.is_present("warp_slot") {
+        Some(match matches.value_of("warp_slot") {
+            Some(_) => value_t_or_exit!(matches, "warp_slot", Slot),
+            None => {
+                cluster_rpc_client.as_ref().unwrap_or_else(|_| {
+                        eprintln!("The --url argument must be provided if --warp-slot/-w is used without an explicit slot");
+                        exit(1);
+
+                }).get_slot()
+                    .unwrap_or_else(|err| {
+                        eprintln!("Unable to get current cluster slot: {}", err);
+                        exit(1);
+                    })
+            }
+        })
+    } else {
+        None
+    };
+
     if !ledger_path.exists() {
         fs::create_dir(&ledger_path).unwrap_or_else(|err| {
             eprintln!(
@@ -345,8 +383,16 @@ fn main() {
             .add_programs_with_path(&programs);
 
         if !clone_accounts.is_empty() {
-            let rpc_client = RpcClient::new(json_rpc_url.unwrap());
+            genesis.clone_accounts(
-            genesis.clone_accounts(clone_accounts, &rpc_client);
+                clone_accounts,
+                cluster_rpc_client
+                    .as_ref()
+                    .expect("bug: --url argument missing?"),
+            );
+        }
+
+        if let Some(warp_slot) = warp_slot {
+            genesis.warp_slot(warp_slot);
         }
         genesis.start_with_mint_address(mint_address)
     }