solana-validator set-identity now loads the tower file for the new identity

core/src/consensus.rs

@@ -119,7 +119,7 @@ pub struct Tower {
     last_vote_tx_blockhash: Hash,
     last_timestamp: BlockTimestamp,
     #[serde(skip)]
-    ledger_path: PathBuf,
+    pub(crate) ledger_path: PathBuf,
     #[serde(skip)]
     path: PathBuf,
     #[serde(skip)]
@@ -175,7 +175,7 @@ impl Tower {
         tower
     }
 
-    pub(crate) fn set_identity(&mut self, node_pubkey: Pubkey) {
+    fn set_identity(&mut self, node_pubkey: Pubkey) {
         let path = Self::get_filename(&self.ledger_path, &node_pubkey);
         let tmp_path = Self::get_tmp_filename(&path);
         self.node_pubkey = node_pubkey;

core/src/replay_stage.rs

@@ -640,7 +640,25 @@ impl ReplayStage {
                                 identity_keypair = cluster_info.keypair().clone();
                                 let my_old_pubkey = my_pubkey;
                                 my_pubkey = identity_keypair.pubkey();
-                                tower.set_identity(my_pubkey);
+
+                                // Load the new identity's tower
+                                tower = Tower::restore(&tower.ledger_path, &my_pubkey)
+                                    .and_then(|restored_tower| {
+                                        let root_bank = bank_forks.read().unwrap().root_bank();
+                                        let slot_history = root_bank.get_slot_history();
+                                        restored_tower.adjust_lockouts_after_replay(root_bank.slot(), &slot_history)
+                                    }).
+                                    unwrap_or_else(|err| {
+                                        // It's a fatal error if the tower is not present.  This is
+                                        // necessary to prevent the validator from violating
+                                        // lockouts for its new identity
+                                        error!("Failed to load tower for {}: {}", my_pubkey, err);
+                                        std::process::exit(1);
+                                    });
+
+                                // Ensure the validator can land votes with the new identity before
+                                // becoming leader
+                                has_new_vote_been_rooted = !wait_for_vote_to_start_leader;
                                 warn!("Identity changed from {} to {}", my_old_pubkey, my_pubkey);
                             }
 

validator/src/admin_rpc_service.rs

@@ -13,7 +13,7 @@ use {
     },
     std::{
         net::SocketAddr,
-        path::Path,
+        path::{Path, PathBuf},
         sync::{Arc, RwLock},
         thread::{self, Builder},
         time::{Duration, SystemTime},
@@ -28,6 +28,7 @@ pub struct AdminRpcRequestMetadata {
     pub validator_exit: Arc<RwLock<Exit>>,
     pub authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
     pub cluster_info: Arc<RwLock<Option<Arc<ClusterInfo>>>>,
+    pub tower_path: PathBuf,
 }
 impl Metadata for AdminRpcRequestMetadata {}
 
@@ -137,8 +138,22 @@ impl AdminRpc for AdminRpcImpl {
     fn set_identity(&self, meta: Self::Metadata, keypair_file: String) -> Result<()> {
         debug!("set_identity request received");
 
-        let identity_keypair = read_keypair_file(keypair_file)
+        let identity_keypair = read_keypair_file(&keypair_file).map_err(|err| {
-            .map_err(|err| jsonrpc_core::error::Error::invalid_params(format!("{}", err)))?;
+            jsonrpc_core::error::Error::invalid_params(format!(
+                "Failed to read identity keypair from {}: {}",
+                keypair_file, err
+            ))
+        })?;
+
+        // Ensure a Tower exists for the new identity and exit gracefully.
+        // ReplayStage will be less forgiving if it fails to load the new tower.
+        solana_core::consensus::Tower::restore(&meta.tower_path, &identity_keypair.pubkey())
+            .map_err(|err| {
+                jsonrpc_core::error::Error::invalid_params(format!(
+                    "Unable to load tower file for new identity: {}",
+                    err
+                ))
+            })?;
 
         if let Some(cluster_info) = meta.cluster_info.read().unwrap().as_ref() {
             solana_metrics::set_host_id(identity_keypair.pubkey().to_string());

validator/src/bin/solana-test-validator.rs

@@ -513,6 +513,7 @@ fn main() {
             validator_exit: genesis.validator_exit.clone(),
             authorized_voter_keypairs: genesis.authorized_voter_keypairs.clone(),
             cluster_info: admin_service_cluster_info.clone(),
+            tower_path: ledger_path.clone(),
         },
     );
     let dashboard = if output == Output::Dashboard {

validator/src/main.rs

@@ -2234,7 +2234,9 @@ pub fn main() {
 
     let mut validator_config = ValidatorConfig {
         require_tower: matches.is_present("require_tower"),
-        tower_path: value_t!(matches, "tower", PathBuf).ok(),
+        tower_path: value_t!(matches, "tower", PathBuf)
+            .ok()
+            .or_else(|| Some(ledger_path.clone())),
         dev_halt_at_slot: value_t!(matches, "dev_halt_at_slot", Slot).ok(),
         cuda: matches.is_present("cuda"),
         expected_genesis_hash: matches
@@ -2539,6 +2541,7 @@ pub fn main() {
             start_progress: start_progress.clone(),
             authorized_voter_keypairs: authorized_voter_keypairs.clone(),
             cluster_info: admin_service_cluster_info.clone(),
+            tower_path: validator_config.tower_path.clone().unwrap(),
         },
     );