Clean nonce

(cherry picked from commit 98e3e570d2)

Co-authored-by: Tyera Eulberg <tyera@solana.com>

.gitignore

@@ -1,7 +1,3 @@
-/docs/html/
-/docs/src/tests.ok
-/docs/src/cli/usage.md
-/docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/
 /solana-release.tar.bz2

Cargo.toml

@@ -28,6 +28,7 @@ members = [
     "local-cluster",
     "logger",
     "log-analyzer",
+    "merkle-root-bench",
     "merkle-tree",
     "stake-o-matic",
     "storage-bigtable",

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.5.0"
+version = "1.5.8"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,11 +18,11 @@ lazy_static = "1.4.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
+solana-config-program = { path = "../programs/config", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.8" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"
 

account-decoder/src/lib.rs

@@ -4,6 +4,7 @@ extern crate lazy_static;
 extern crate serde_derive;
 
 pub mod parse_account_data;
+pub mod parse_bpf_loader;
 pub mod parse_config;
 pub mod parse_nonce;
 pub mod parse_stake;

account-decoder/src/parse_account_data.rs

@@ -1,4 +1,5 @@
 use crate::{
+    parse_bpf_loader::parse_bpf_upgradeable_loader,
     parse_config::parse_config,
     parse_nonce::parse_nonce,
     parse_stake::parse_stake,
@@ -13,6 +14,7 @@ use std::collections::HashMap;
 use thiserror::Error;
 
 lazy_static! {
+    static ref BPF_UPGRADEABLE_LOADER_PROGRAM_ID: Pubkey = solana_sdk::bpf_loader_upgradeable::id();
     static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
     static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
@@ -21,6 +23,10 @@ lazy_static! {
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();
+        m.insert(
+            *BPF_UPGRADEABLE_LOADER_PROGRAM_ID,
+            ParsableAccount::BpfUpgradeableLoader,
+        );
         m.insert(*CONFIG_PROGRAM_ID, ParsableAccount::Config);
         m.insert(*SYSTEM_PROGRAM_ID, ParsableAccount::Nonce);
         m.insert(*TOKEN_PROGRAM_ID, ParsableAccount::SplToken);
@@ -60,6 +66,7 @@ pub struct ParsedAccount {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum ParsableAccount {
+    BpfUpgradeableLoader,
     Config,
     Nonce,
     SplToken,
@@ -84,6 +91,9 @@ pub fn parse_account_data(
         .ok_or(ParseAccountError::ProgramNotParsable)?;
     let additional_data = additional_data.unwrap_or_default();
     let parsed_json = match program_name {
+        ParsableAccount::BpfUpgradeableLoader => {
+            serde_json::to_value(parse_bpf_upgradeable_loader(data)?)?
+        }
         ParsableAccount::Config => serde_json::to_value(parse_config(data, pubkey)?)?,
         ParsableAccount::Nonce => serde_json::to_value(parse_nonce(data)?)?,
         ParsableAccount::SplToken => {
@@ -118,7 +128,7 @@ mod test {
 
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
         let parsed = parse_account_data(
             &account_pubkey,

account-decoder/src/parse_bpf_loader.rs

@@ -0,0 +1,181 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    UiAccountData, UiAccountEncoding,
+};
+use bincode::{deserialize, serialized_size};
+use solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey};
+
+pub fn parse_bpf_upgradeable_loader(
+    data: &[u8],
+) -> Result<BpfUpgradeableLoaderAccountType, ParseAccountError> {
+    let account_state: UpgradeableLoaderState = deserialize(data).map_err(|_| {
+        ParseAccountError::AccountNotParsable(ParsableAccount::BpfUpgradeableLoader)
+    })?;
+    let parsed_account = match account_state {
+        UpgradeableLoaderState::Uninitialized => BpfUpgradeableLoaderAccountType::Uninitialized,
+        UpgradeableLoaderState::Buffer { authority_address } => {
+            let offset = if authority_address.is_some() {
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+            } else {
+                // This case included for code completeness; in practice, a Buffer account will
+                // always have authority_address.is_some()
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+        UpgradeableLoaderState::Program {
+            programdata_address,
+        } => BpfUpgradeableLoaderAccountType::Program(UiProgram {
+            program_data: programdata_address.to_string(),
+        }),
+        UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address,
+        } => {
+            let offset = if upgrade_authority_address.is_some() {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+            } else {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: upgrade_authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+    };
+    Ok(parsed_account)
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum BpfUpgradeableLoaderAccountType {
+    Uninitialized,
+    Buffer(UiBuffer),
+    Program(UiProgram),
+    ProgramData(UiProgramData),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiBuffer {
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgram {
+    pub program_data: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgramData {
+    pub slot: u64,
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use bincode::serialize;
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_parse_bpf_upgradeable_loader_accounts() {
+        let bpf_loader_state = UpgradeableLoaderState::Uninitialized;
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Uninitialized
+        );
+
+        let program = vec![7u8; 64]; // Arbitrary program data
+
+        let authority = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        // This case included for code completeness; in practice, a Buffer account will always have
+        // authority_address.is_some()
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let programdata_address = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Program {
+            programdata_address,
+        };
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Program(UiProgram {
+                program_data: programdata_address.to_string(),
+            })
+        );
+
+        let authority = Pubkey::new_unique();
+        let slot = 42;
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+    }
+}

account-decoder/src/parse_vote.rs

@@ -128,7 +128,7 @@ mod test {
     fn test_parse_vote() {
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
         let expected_vote_state = UiVoteState {
             node_pubkey: Pubkey::default().to_string(),

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-logger = { path = "../logger", version = "1.5.8" }
+solana-runtime = { path = "../runtime", version = "1.5.8" }
+solana-measure = { path = "../measure", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -1,14 +1,14 @@
+#[macro_use]
+extern crate log;
 use clap::{crate_description, crate_name, value_t, App, Arg};
 use rayon::prelude::*;
 use solana_measure::measure::Measure;
 use solana_runtime::{
-    accounts::{create_test_accounts, update_accounts, Accounts},
+    accounts::{create_test_accounts, update_accounts_bench, Accounts},
     accounts_index::Ancestors,
 };
 use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
-use std::env;
-use std::fs;
-use std::path::PathBuf;
+use std::{collections::HashSet, env, fs, path::PathBuf};
 
 fn main() {
     solana_logger::setup();
@@ -53,10 +53,12 @@ fn main() {
 
     let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned()))
         .join("accounts-bench");
+    println!("cleaning file system: {:?}", path);
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
-    let accounts = Accounts::new(vec![path], &ClusterType::Testnet);
+    let accounts =
+        Accounts::new_with_config(vec![path], &ClusterType::Testnet, HashSet::new(), false);
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");
     let pubkeys: Vec<_> = (0..num_slots)
@@ -85,6 +87,8 @@ fn main() {
         ancestors.insert(i as u64, i - 1);
         accounts.add_root(i as u64);
     }
+    let mut elapsed = vec![0; iterations];
+    let mut elapsed_store = vec![0; iterations];
     for x in 0..iterations {
         if clean {
             let mut time = Measure::start("clean");
@@ -92,19 +96,45 @@ fn main() {
             time.stop();
             println!("{}", time);
             for slot in 0..num_slots {
-                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
+                update_accounts_bench(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
                 accounts.add_root((x * num_slots + slot) as u64);
             }
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let hash = accounts
+            let results = accounts
                 .accounts_db
-                .update_accounts_hash(0, &ancestors, true)
-                .0;
+                .update_accounts_hash(0, &ancestors, true);
             time.stop();
-            println!("hash: {} {}", hash, time);
+            let mut time_store = Measure::start("hash using store");
+            let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
+                true,
+                false,
+                solana_sdk::clock::Slot::default(),
+                &ancestors,
+                true,
+            );
+            time_store.stop();
+            if results != results_store {
+                error!("results different: \n{:?}\n{:?}", results, results_store);
+            }
+            println!(
+                "hash,{},{},{},{}%",
+                results.0,
+                time,
+                time_store,
+                (time_store.as_us() as f64 / time.as_us() as f64 * 100.0f64) as u32
+            );
             create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+            elapsed[x] = time.as_us();
+            elapsed_store[x] = time_store.as_us();
         }
     }
+
+    for x in elapsed {
+        info!("update_accounts_hash(us),{}", x);
+    }
+    for x in elapsed_store {
+        info!("calculate_accounts_hash_without_index(us),{}", x);
+    }
 }

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-streamer = { path = "../streamer", version = "1.5.0" }
-solana-perf = { path = "../perf", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-core = { path = "../core", version = "1.5.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-streamer = { path = "../streamer", version = "1.5.8" }
+solana-perf = { path = "../perf", version = "1.5.8" }
+solana-ledger = { path = "../ledger", version = "1.5.8" }
+solana-logger = { path = "../logger", version = "1.5.8" }
+solana-runtime = { path = "../runtime", version = "1.5.8" }
+solana-measure = { path = "../measure", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.5.0"
+version = "1.5.8"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,15 +12,15 @@ edition = "2018"
 bincode = "1.3.1"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
+solana-banks-interface = { path = "../banks-interface", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
 tarpc = { version = "0.23.0", features = ["full"] }
-tokio = { version = "0.3", features = ["full"] }
+tokio = { version = "0.3.5", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-banks-server = { path = "../banks-server", version = "1.5.0" }
+solana-runtime = { path = "../runtime", version = "1.5.8" }
+solana-banks-server = { path = "../banks-server", version = "1.5.8" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -122,7 +122,7 @@ impl BanksClient {
     pub fn get_fees(
         &mut self,
     ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
-        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::Root)
+        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::default())
     }
 
     /// Return the cluster rent
@@ -196,7 +196,7 @@ impl BanksClient {
     /// Return the most recent rooted slot height. All transactions at or below this height
     /// are said to be finalized. The cluster will not fork to a higher slot height.
     pub fn get_root_slot(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
-        self.get_slot_with_context(context::current(), CommitmentLevel::Root)
+        self.get_slot_with_context(context::current(), CommitmentLevel::default())
     }
 
     /// Return the account at the given address at the slot corresponding to the given
@@ -289,7 +289,10 @@ pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClie
 mod tests {
     use super::*;
     use solana_banks_server::banks_server::start_local_server;
-    use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
+    use solana_runtime::{
+        bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache,
+        genesis_utils::create_genesis_config,
+    };
     use solana_sdk::{message::Message, signature::Signer, system_instruction};
     use std::sync::{Arc, RwLock};
     use tarpc::transport;
@@ -308,9 +311,12 @@ mod tests {
         // `runtime.block_on()` just once, to run all the async code.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
         let bob_pubkey = solana_sdk::pubkey::new_rand();
         let mint_pubkey = genesis.mint_keypair.pubkey();
@@ -318,7 +324,7 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
             let mut banks_client = start_client(client_transport).await?;
 
             let recent_blockhash = banks_client.get_recent_blockhash().await?;
@@ -336,9 +342,12 @@ mod tests {
         // server-side functionality is available to the client.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
         let bob_pubkey = solana_sdk::pubkey::new_rand();
@@ -346,7 +355,7 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
             let mut banks_client = start_client(client_transport).await?;
             let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.5.0"
+version = "1.5.8"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,11 +11,11 @@ edition = "2018"
 [dependencies]
 mio = "0.7.6"
 serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
 tarpc = { version = "0.23.0", features = ["full"] }
 
 [dev-dependencies]
-tokio = { version = "0.3", features = ["full"] }
+tokio = { version = "0.3.5", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-interface/src/lib.rs

@@ -10,11 +10,19 @@ use solana_sdk::{
     transaction::{self, Transaction, TransactionError},
 };
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum TransactionConfirmationStatus {
+    Processed,
+    Confirmed,
+    Finalized,
+}
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct TransactionStatus {
     pub slot: Slot,
     pub confirmations: Option<usize>, // None = rooted
     pub err: Option<TransactionError>,
+    pub confirmation_status: Option<TransactionConfirmationStatus>,
 }
 
 #[tarpc::service]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.5.0"
+version = "1.5.8"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,10 +13,10 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
+solana-banks-interface = { path = "../banks-interface", version = "1.5.8" }
+solana-runtime = { path = "../runtime", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-metrics = { path = "../metrics", version = "1.5.8" }
 tarpc = { version = "0.23.0", features = ["full"] }
 tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -4,7 +4,9 @@ use futures::{
     future,
     prelude::stream::{self, StreamExt},
 };
-use solana_banks_interface::{Banks, BanksRequest, BanksResponse, TransactionStatus};
+use solana_banks_interface::{
+    Banks, BanksRequest, BanksResponse, TransactionConfirmationStatus, TransactionStatus,
+};
 use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache};
 use solana_sdk::{
     account::Account,
@@ -60,7 +62,7 @@ impl BanksServer {
         }
     }
 
-    fn run(bank: &Bank, transaction_receiver: Receiver<TransactionInfo>) {
+    fn run(bank_forks: Arc<RwLock<BankForks>>, transaction_receiver: Receiver<TransactionInfo>) {
         while let Ok(info) = transaction_receiver.recv() {
             let mut transaction_infos = vec![info];
             while let Ok(info) = transaction_receiver.try_recv() {
@@ -70,21 +72,28 @@ impl BanksServer {
                 .into_iter()
                 .map(|info| deserialize(&info.wire_transaction).unwrap())
                 .collect();
+            let bank = bank_forks.read().unwrap().working_bank();
             let _ = bank.process_transactions(&transactions);
         }
     }
 
     /// Useful for unit-testing
-    fn new_loopback(bank_forks: Arc<RwLock<BankForks>>) -> Self {
+    fn new_loopback(
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
         let slot = bank.slot();
-        let block_commitment_cache = Arc::new(RwLock::new(
-            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
-        ));
+        {
+            // ensure that the commitment cache and bank are synced
+            let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+            w_block_commitment_cache.set_all_slots(slot, slot);
+        }
+        let server_bank_forks = bank_forks.clone();
         Builder::new()
             .name("solana-bank-forks-client".to_string())
-            .spawn(move || Self::run(&bank, transaction_receiver))
+            .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
         Self::new(bank_forks, block_commitment_cache, transaction_sender)
     }
@@ -165,11 +174,17 @@ impl Banks for BanksServer {
         _: Context,
         signature: Signature,
     ) -> Option<TransactionStatus> {
-        let bank = self.bank(CommitmentLevel::Recent);
+        let bank = self.bank(CommitmentLevel::Processed);
         let (slot, status) = bank.get_signature_status_slot(&signature)?;
         let r_block_commitment_cache = self.block_commitment_cache.read().unwrap();
 
-        let confirmations = if r_block_commitment_cache.root() >= slot {
+        let optimistically_confirmed_bank = self.bank(CommitmentLevel::Confirmed);
+        let optimistically_confirmed =
+            optimistically_confirmed_bank.get_signature_status_slot(&signature);
+
+        let confirmations = if r_block_commitment_cache.root() >= slot
+            && r_block_commitment_cache.highest_confirmed_root() >= slot
+        {
             None
         } else {
             r_block_commitment_cache
@@ -180,6 +195,13 @@ impl Banks for BanksServer {
             slot,
             confirmations,
             err: status.err(),
+            confirmation_status: if confirmations.is_none() {
+                Some(TransactionConfirmationStatus::Finalized)
+            } else if optimistically_confirmed.is_some() {
+                Some(TransactionConfirmationStatus::Confirmed)
+            } else {
+                Some(TransactionConfirmationStatus::Processed)
+            },
         })
     }
 
@@ -225,9 +247,10 @@ impl Banks for BanksServer {
 }
 
 pub async fn start_local_server(
-    bank_forks: &Arc<RwLock<BankForks>>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(bank_forks.clone());
+    let banks_server = BanksServer::new_loopback(bank_forks, block_commitment_cache);
     let (client_transport, server_transport) = transport::channel::unbounded();
     let server = server::new(server::Config::default())
         .incoming(stream::once(future::ready(server_transport)))

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-core = { path = "../core", version = "1.5.0" }
-solana-genesis = { path = "../genesis", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-core = { path = "../core", version = "1.5.8" }
+solana-genesis = { path = "../genesis", version = "1.5.8" }
+solana-client = { path = "../client", version = "1.5.8" }
+solana-faucet = { path = "../faucet", version = "1.5.8" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.5.8" }
+solana-logger = { path = "../logger", version = "1.5.8" }
+solana-metrics = { path = "../metrics", version = "1.5.8" }
+solana-net-utils = { path = "../net-utils", version = "1.5.8" }
+solana-runtime = { path = "../runtime", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.5.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/src/bench.rs

@@ -390,7 +390,7 @@ fn swapper<T>(
             while client
                 .get_balance_with_commitment(
                     &trade_infos[trade_index].trade_account,
-                    CommitmentConfig::recent(),
+                    CommitmentConfig::processed(),
                 )
                 .unwrap_or(0)
                 == 0
@@ -445,7 +445,7 @@ fn swapper<T>(
             account_group = (account_group + 1) % account_groups as usize;
 
             let (blockhash, _fee_calculator, _last_valid_slot) = client
-                .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+                .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
                 .expect("Failed to get blockhash");
             let to_swap_txs: Vec<_> = to_swap
                 .par_iter()
@@ -571,7 +571,7 @@ fn trader<T>(
         account_group = (account_group + 1) % account_groups as usize;
 
         let (blockhash, _fee_calculator, _last_valid_slot) = client
-            .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+            .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
             .expect("Failed to get blockhash");
 
         trades.chunks(chunk_size).for_each(|chunk| {
@@ -658,7 +658,7 @@ where
 {
     for s in &tx.signatures {
         if let Ok(Some(r)) =
-            sync_client.get_signature_status_with_commitment(s, CommitmentConfig::recent())
+            sync_client.get_signature_status_with_commitment(s, CommitmentConfig::processed())
         {
             match r {
                 Ok(_) => {
@@ -681,7 +681,7 @@ fn verify_funding_transfer<T: SyncClient + ?Sized>(
     if verify_transaction(client, tx) {
         for a in &tx.message().account_keys[1..] {
             if client
-                .get_balance_with_commitment(a, CommitmentConfig::recent())
+                .get_balance_with_commitment(a, CommitmentConfig::processed())
                 .unwrap_or(0)
                 >= amount
             {
@@ -764,7 +764,7 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
                 );
 
                 let (blockhash, _fee_calculator, _last_valid_slot) = client
-                    .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+                    .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
                     .expect("blockhash");
                 to_fund_txs.par_iter_mut().for_each(|(k, tx)| {
                     tx.sign(&[*k], blockhash);
@@ -803,7 +803,7 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
         funded.append(&mut new_funded);
         funded.retain(|(k, b)| {
             client
-                .get_balance_with_commitment(&k.pubkey(), CommitmentConfig::recent())
+                .get_balance_with_commitment(&k.pubkey(), CommitmentConfig::processed())
                 .unwrap_or(0)
                 > lamports
                 && *b > lamports
@@ -857,7 +857,7 @@ pub fn create_token_accounts<T: Client>(
             let mut retries = 0;
             while !to_create_txs.is_empty() {
                 let (blockhash, _fee_calculator, _last_valid_slot) = client
-                    .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+                    .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
                     .expect("Failed to get blockhash");
                 to_create_txs
                     .par_iter_mut()
@@ -903,7 +903,7 @@ pub fn create_token_accounts<T: Client>(
         let mut new_notfunded: Vec<(&Arc<Keypair>, &Keypair)> = vec![];
         for f in &notfunded {
             if client
-                .get_balance_with_commitment(&f.1.pubkey(), CommitmentConfig::recent())
+                .get_balance_with_commitment(&f.1.pubkey(), CommitmentConfig::processed())
                 .unwrap_or(0)
                 == 0
             {
@@ -968,7 +968,7 @@ pub fn airdrop_lamports<T: Client>(
     id: &Keypair,
     amount: u64,
 ) {
-    let balance = client.get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent());
+    let balance = client.get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed());
     let balance = balance.unwrap_or(0);
     if balance >= amount {
         return;
@@ -986,7 +986,7 @@ pub fn airdrop_lamports<T: Client>(
     let mut tries = 0;
     loop {
         let (blockhash, _fee_calculator, _last_valid_slot) = client
-            .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+            .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
             .expect("Failed to get blockhash");
         match request_airdrop_transaction(&faucet_addr, &id.pubkey(), amount_to_drop, blockhash) {
             Ok(transaction) => {
@@ -995,14 +995,14 @@ pub fn airdrop_lamports<T: Client>(
                 for _ in 0..30 {
                     if let Ok(Some(_)) = client.get_signature_status_with_commitment(
                         &signature,
-                        CommitmentConfig::recent(),
+                        CommitmentConfig::processed(),
                     ) {
                         break;
                     }
                     sleep(Duration::from_millis(100));
                 }
                 if client
-                    .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent())
+                    .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed())
                     .unwrap_or(0)
                     >= amount
                 {

bench-exchange/tests/bench_exchange.rs

@@ -5,7 +5,7 @@ use solana_core::validator::ValidatorConfig;
 use solana_exchange_program::exchange_processor::process_instruction;
 use solana_exchange_program::id;
 use solana_exchange_program::solana_exchange_program;
-use solana_faucet::faucet::run_local_faucet;
+use solana_faucet::faucet::run_local_faucet_with_port;
 use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
 use solana_runtime::bank::Bank;
 use solana_runtime::bank_client::BankClient;
@@ -57,8 +57,11 @@ fn test_exchange_local_cluster() {
     );
 
     let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, Some(1_000_000_000_000));
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, Some(1_000_000_000_000), 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
 
     info!("Connecting to the cluster");
     let nodes =

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-streamer = { path = "../streamer", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-streamer = { path = "../streamer", version = "1.5.8" }
+solana-logger = { path = "../logger", version = "1.5.8" }
+solana-net-utils = { path = "../net-utils", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,23 +15,23 @@ log = "0.4.11"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-core = { path = "../core", version = "1.5.0" }
-solana-genesis = { path = "../genesis", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-core = { path = "../core", version = "1.5.8" }
+solana-genesis = { path = "../genesis", version = "1.5.8" }
+solana-client = { path = "../client", version = "1.5.8" }
+solana-faucet = { path = "../faucet", version = "1.5.8" }
+solana-logger = { path = "../logger", version = "1.5.8" }
+solana-metrics = { path = "../metrics", version = "1.5.8" }
+solana-measure = { path = "../measure", version = "1.5.8" }
+solana-net-utils = { path = "../net-utils", version = "1.5.8" }
+solana-runtime = { path = "../runtime", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.5.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -48,7 +48,7 @@ pub type SharedTransactions = Arc<RwLock<VecDeque<Vec<(Transaction, u64)>>>>;
 
 fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
     loop {
-        match client.get_recent_blockhash_with_commitment(CommitmentConfig::recent()) {
+        match client.get_recent_blockhash_with_commitment(CommitmentConfig::processed()) {
             Ok((blockhash, fee_calculator, _last_valid_slot)) => {
                 return (blockhash, fee_calculator)
             }
@@ -497,7 +497,7 @@ fn do_tx_transfers<T: Client>(
 
 fn verify_funding_transfer<T: Client>(client: &Arc<T>, tx: &Transaction, amount: u64) -> bool {
     for a in &tx.message().account_keys[1..] {
-        match client.get_balance_with_commitment(a, CommitmentConfig::recent()) {
+        match client.get_balance_with_commitment(a, CommitmentConfig::processed()) {
             Ok(balance) => return balance >= amount,
             Err(err) => error!("failed to get balance {:?}", err),
         }
@@ -762,7 +762,7 @@ pub fn airdrop_lamports<T: Client>(
         };
 
         let current_balance = client
-            .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent())
+            .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed())
             .unwrap_or_else(|e| {
                 info!("airdrop error {}", e);
                 starting_balance
@@ -967,7 +967,7 @@ mod tests {
         for kp in &keypairs {
             assert_eq!(
                 client
-                    .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::recent())
+                    .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::processed())
                     .unwrap(),
                 lamports
             );

bench-tps/tests/bench_tps.rs

@@ -4,7 +4,7 @@ use solana_bench_tps::cli::Config;
 use solana_client::thin_client::create_client;
 use solana_core::cluster_info::VALIDATOR_PORT_RANGE;
 use solana_core::validator::ValidatorConfig;
-use solana_faucet::faucet::run_local_faucet;
+use solana_faucet::faucet::run_local_faucet_with_port;
 use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
 use solana_sdk::signature::{Keypair, Signer};
 use std::sync::{mpsc::channel, Arc};
@@ -36,8 +36,11 @@ fn test_bench_tps_local_cluster(config: Config) {
     ));
 
     let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, None);
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, None, 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
 
     let lamports_per_account = 100;
 

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.48.0
+FROM solanalabs/rust:1.49.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.48.0
+FROM rust:1.49.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/localnet-sanity.sh

@@ -80,7 +80,7 @@ nodes=(
   "multinode-demo/validator.sh \
     --enable-rpc-exit \
     --no-restart \
-    --dynamic-port-range 8050-8100
+    --dynamic-port-range 8050-8100 \
     --init-complete-file init-complete-node1.log \
     --rpc-port 18899"
 )

ci/nits.sh

@@ -19,6 +19,7 @@ declare prints=(
 # Parts of the tree that are expected to be print free
 declare print_free_tree=(
   ':core/src/**.rs'
+  ':^core/src/validator.rs'
   ':faucet/src/**.rs'
   ':ledger/src/**.rs'
   ':metrics/src/**.rs'

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.48.0
+  stable_version=1.49.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-12-13
+  nightly_version=2021-01-23
 fi
 
 

ci/setup-new-buildkite-agent/setup-buildkite.sh

@@ -76,7 +76,7 @@ RestartForceExitStatus=SIGPIPE
 TimeoutStartSec=10
 TimeoutStopSec=0
 KillMode=process
-LimitNOFILE=500000
+LimitNOFILE=700000
 
 [Install]
 WantedBy=multi-user.target

ci/setup-new-buildkite-agent/setup-limits.sh

@@ -8,5 +8,5 @@ source "$HERE"/utils.sh
 ensure_env || exit 1
 
 # Allow more files to be opened by a user
-echo "* - nofile 500000" > /etc/security/limits.d/90-solana-nofiles.conf
+echo "* - nofile 700000" > /etc/security/limits.d/90-solana-nofiles.conf
 

ci/test-bench.sh

@@ -23,6 +23,10 @@ fi
 BENCH_FILE=bench_output.log
 BENCH_ARTIFACT=current_bench_results.log
 
+# solana-keygen required when building C programs
+_ "$cargo" build --manifest-path=keygen/Cargo.toml
+export PATH="$PWD/target/debug":$PATH
+
 # Clear the C dependency files, if dependeny moves these files are not regenerated
 test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete

ci/test-checks.sh

@@ -79,10 +79,14 @@ cargo_audit_ignores=(
   # Blocked on multiple crates updating `time` to >= 0.2.23
   --ignore RUSTSEC-2020-0071
 
-  # memmap crate is unmaintained
+  # difference is unmaintained
   #
-  # Blocked on us releasing new solana crates
-  --ignore RUSTSEC-2020-0077
+  # Blocked on predicates v1.0.6 removing its dependency on `difference`
+  --ignore RUSTSEC-2020-0095
+
+  # hyper is upgraded on master/v1.6 but not for v1.5
+  --ignore RUSTSEC-2021-0020
+
 )
 _ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"
 

ci/test-stable.sh

@@ -39,6 +39,10 @@ test-stable)
   _ "$cargo" stable test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
   ;;
 test-stable-perf)
+  # solana-keygen required when building C programs
+  _ "$cargo" build --manifest-path=keygen/Cargo.toml
+  export PATH="$PWD/target/debug":$PATH
+
   # BPF solana-sdk legacy compile test
   ./cargo-build-bpf --manifest-path sdk/Cargo.toml
 

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.5.0"
+version = "1.5.8"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/commitment.rs

@@ -1,22 +0,0 @@
-use crate::ArgConstant;
-use clap::Arg;
-
-pub const COMMITMENT_ARG: ArgConstant<'static> = ArgConstant {
-    name: "commitment",
-    long: "commitment",
-    help: "Return information at the selected commitment level",
-};
-
-pub fn commitment_arg<'a, 'b>() -> Arg<'a, 'b> {
-    commitment_arg_with_default("recent")
-}
-
-pub fn commitment_arg_with_default<'a, 'b>(default_value: &'static str) -> Arg<'a, 'b> {
-    Arg::with_name(COMMITMENT_ARG.name)
-        .long(COMMITMENT_ARG.long)
-        .takes_value(true)
-        .possible_values(&["recent", "single", "singleGossip", "root", "max"])
-        .default_value(default_value)
-        .value_name("COMMITMENT_LEVEL")
-        .help(COMMITMENT_ARG.help)
-}

clap-utils/src/input_parsers.rs

@@ -167,12 +167,12 @@ pub fn resolve_signer(
     name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Option<String>, Box<dyn std::error::Error>> {
-    Ok(resolve_signer_from_path(
+    resolve_signer_from_path(
         matches,
         matches.value_of(name).unwrap(),
         name,
         wallet_manager,
-    )?)
+    )
 }
 
 pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
@@ -184,14 +184,9 @@ pub fn cluster_type_of(matches: &ArgMatches<'_>, name: &str) -> Option<ClusterTy
 }
 
 pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentConfig> {
-    matches.value_of(name).map(|value| match value {
-        "max" => CommitmentConfig::max(),
-        "recent" => CommitmentConfig::recent(),
-        "root" => CommitmentConfig::root(),
-        "single" => CommitmentConfig::single(),
-        "singleGossip" => CommitmentConfig::single_gossip(),
-        _ => CommitmentConfig::default(),
-    })
+    matches
+        .value_of(name)
+        .map(|value| CommitmentConfig::from_str(value).unwrap_or_default())
 }
 
 #[cfg(test)]

clap-utils/src/input_validators.rs

@@ -1,7 +1,7 @@
 use crate::keypair::{parse_keypair_path, KeypairUrl, ASK_KEYWORD};
 use chrono::DateTime;
 use solana_sdk::{
-    clock::Slot,
+    clock::{Epoch, Slot},
     hash::Hash,
     pubkey::Pubkey,
     signature::{read_keypair_file, Signature},
@@ -148,6 +148,40 @@ where
     }
 }
 
+pub fn is_url_or_moniker<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    match url::Url::parse(&normalize_to_url_if_moniker(string.as_ref())) {
+        Ok(url) => {
+            if url.has_host() {
+                Ok(())
+            } else {
+                Err("no host provided".to_string())
+            }
+        }
+        Err(err) => Err(format!("{}", err)),
+    }
+}
+
+pub fn normalize_to_url_if_moniker<T: AsRef<str>>(url_or_moniker: T) -> String {
+    match url_or_moniker.as_ref() {
+        "m" | "mainnet-beta" => "https://api.mainnet-beta.solana.com",
+        "t" | "testnet" => "https://testnet.solana.com",
+        "d" | "devnet" => "https://devnet.solana.com",
+        "l" | "localhost" => "http://localhost:8899",
+        url => url,
+    }
+    .to_string()
+}
+
+pub fn is_epoch<T>(epoch: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<Epoch, _>(epoch)
+}
+
 pub fn is_slot<T>(slot: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,

clap-utils/src/keypair.rs

@@ -58,6 +58,15 @@ impl CliSignerInfo {
             Some(0)
         }
     }
+    pub fn index_of_or_none(&self, pubkey: Option<Pubkey>) -> Option<usize> {
+        if let Some(pubkey) = pubkey {
+            self.signers
+                .iter()
+                .position(|signer| signer.pubkey() == pubkey)
+        } else {
+            None
+        }
+    }
 }
 
 pub struct DefaultSigner {

clap-utils/src/lib.rs

@@ -23,7 +23,6 @@ impl std::fmt::Debug for DisplayError {
     }
 }
 
-pub mod commitment;
 pub mod fee_payer;
 pub mod input_parsers;
 pub mod input_validators;

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-config/src/config.rs

@@ -17,9 +17,10 @@ pub struct Config {
     pub json_rpc_url: String,
     pub websocket_url: String,
     pub keypair_path: String,
-
     #[serde(default)]
     pub address_labels: HashMap<String, String>,
+    #[serde(default)]
+    pub commitment: String,
 }
 
 impl Default for Config {
@@ -41,11 +42,14 @@ impl Default for Config {
             "System Program".to_string(),
         );
 
+        let commitment = "confirmed".to_string();
+
         Self {
             json_rpc_url,
             websocket_url,
             keypair_path,
             address_labels,
+            commitment,
         }
     }
 }

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,13 +17,13 @@ indicatif = "0.15.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-client = { path = "../client", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.8" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -1,37 +1,42 @@
-use crate::{
-    display::{build_balance_message, format_labeled_address, writeln_name_value},
-    QuietDisplay, VerboseDisplay,
-};
-use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
-use console::{style, Emoji};
-use inflector::cases::titlecase::to_title_case;
-use serde::{Deserialize, Serialize};
-use serde_json::{Map, Value};
-use solana_account_decoder::parse_token::UiTokenAccount;
-use solana_clap_utils::keypair::SignOnly;
-use solana_client::rpc_response::{
-    RpcAccountBalance, RpcKeyedAccount, RpcSupply, RpcVoteAccountInfo,
-};
-use solana_sdk::{
-    clock::{self, Epoch, Slot, UnixTimestamp},
-    epoch_info::EpochInfo,
-    hash::Hash,
-    native_token::lamports_to_sol,
-    pubkey::Pubkey,
-    signature::Signature,
-    stake_history::StakeHistoryEntry,
-    transaction::Transaction,
-};
-use solana_stake_program::stake_state::{Authorized, Lockup};
-use solana_vote_program::{
-    authorized_voters::AuthorizedVoters,
-    vote_state::{BlockTimestamp, Lockout},
-};
-use std::{
-    collections::{BTreeMap, HashMap},
-    fmt,
-    str::FromStr,
-    time::Duration,
+use {
+    crate::{
+        display::{
+            build_balance_message, build_balance_message_with_config, format_labeled_address,
+            unix_timestamp_to_string, writeln_name_value, BuildBalanceMessageConfig,
+        },
+        QuietDisplay, VerboseDisplay,
+    },
+    console::{style, Emoji},
+    inflector::cases::titlecase::to_title_case,
+    serde::{Deserialize, Serialize},
+    serde_json::{Map, Value},
+    solana_account_decoder::parse_token::UiTokenAccount,
+    solana_clap_utils::keypair::SignOnly,
+    solana_client::rpc_response::{
+        RpcAccountBalance, RpcInflationGovernor, RpcInflationRate, RpcKeyedAccount, RpcSupply,
+        RpcVoteAccountInfo,
+    },
+    solana_sdk::{
+        clock::{self, Epoch, Slot, UnixTimestamp},
+        epoch_info::EpochInfo,
+        hash::Hash,
+        native_token::lamports_to_sol,
+        pubkey::Pubkey,
+        signature::Signature,
+        stake_history::StakeHistoryEntry,
+        transaction::Transaction,
+    },
+    solana_stake_program::stake_state::{Authorized, Lockup},
+    solana_vote_program::{
+        authorized_voters::AuthorizedVoters,
+        vote_state::{BlockTimestamp, Lockout},
+    },
+    std::{
+        collections::{BTreeMap, HashMap},
+        fmt,
+        str::FromStr,
+        time::Duration,
+    },
 };
 
 static WARNING: Emoji = Emoji("⚠️", "!");
@@ -241,6 +246,9 @@ impl fmt::Display for CliEpochInfo {
         )?;
         writeln_name_value(f, "Slot:", &self.epoch_info.absolute_slot.to_string())?;
         writeln_name_value(f, "Epoch:", &self.epoch_info.epoch.to_string())?;
+        if let Some(transaction_count) = &self.epoch_info.transaction_count {
+            writeln_name_value(f, "Transaction Count:", &transaction_count.to_string())?;
+        }
         let start_slot = self.epoch_info.absolute_slot - self.epoch_info.slot_index;
         let end_slot = start_slot + self.epoch_info.slots_in_epoch;
         writeln_name_value(
@@ -355,6 +363,42 @@ impl fmt::Display for CliValidators {
                 },
             )
         }
+        writeln!(
+            f,
+            "{}",
+            style(format!(
+                "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
+                "Identity",
+                "Vote Account",
+                "Commission",
+                "Last Vote",
+                "Root Block",
+                "Credits",
+                "Version",
+                "Active Stake",
+            ))
+            .bold()
+        )?;
+        for validator in &self.current_validators {
+            write_vote_account(
+                f,
+                validator,
+                self.total_active_stake,
+                self.use_lamports_unit,
+                false,
+            )?;
+        }
+        for validator in &self.delinquent_validators {
+            write_vote_account(
+                f,
+                validator,
+                self.total_active_stake,
+                self.use_lamports_unit,
+                true,
+            )?;
+        }
+
+        writeln!(f)?;
         writeln_name_value(
             f,
             "Active Stake:",
@@ -406,41 +450,6 @@ impl fmt::Display for CliValidators {
             )?;
         }
 
-        writeln!(f)?;
-        writeln!(
-            f,
-            "{}",
-            style(format!(
-                "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
-                "Identity",
-                "Vote Account",
-                "Commission",
-                "Last Vote",
-                "Root Block",
-                "Credits",
-                "Version",
-                "Active Stake",
-            ))
-            .bold()
-        )?;
-        for validator in &self.current_validators {
-            write_vote_account(
-                f,
-                validator,
-                self.total_active_stake,
-                self.use_lamports_unit,
-                false,
-            )?;
-        }
-        for validator in &self.delinquent_validators {
-            write_vote_account(
-                f,
-                validator,
-                self.total_active_stake,
-                self.use_lamports_unit,
-                true,
-            )?;
-        }
         Ok(())
     }
 }
@@ -646,13 +655,13 @@ fn show_epoch_rewards(
         writeln!(f, "Epoch Rewards:")?;
         writeln!(
             f,
-            "  {:<8}  {:<11}  {:<15}  {:<15}  {:>14}  {:>14}",
+            "  {:<6}  {:<11}  {:<16}  {:<16}  {:>14}  {:>14}",
             "Epoch", "Reward Slot", "Amount", "New Balance", "Percent Change", "APR"
         )?;
         for reward in epoch_rewards {
             writeln!(
                 f,
-                "  {:<8}  {:<11}  ◎{:<14.9}  ◎{:<14.9}  {:>13.9}%  {}",
+                "  {:<6}  {:<11}  ◎{:<16.9}  ◎{:<14.9}  {:>13.2}%  {}",
                 reward.epoch,
                 reward.effective_slot,
                 lamports_to_sol(reward.amount),
@@ -660,7 +669,7 @@ fn show_epoch_rewards(
                 reward.percent_change,
                 reward
                     .apr
-                    .map(|apr| format!("{:>13.9}%", apr))
+                    .map(|apr| format!("{:>13.2}%", apr))
                     .unwrap_or_default(),
             )?;
         }
@@ -901,14 +910,19 @@ impl fmt::Display for CliStakeHistory {
             ))
             .bold()
         )?;
+        let config = BuildBalanceMessageConfig {
+            use_lamports_unit: self.use_lamports_unit,
+            show_unit: false,
+            trim_trailing_zeros: false,
+        };
         for entry in &self.entries {
             writeln!(
                 f,
                 "  {:>5}  {:>20}  {:>20}  {:>20} {}",
                 entry.epoch,
-                build_balance_message(entry.effective_stake, self.use_lamports_unit, false),
-                build_balance_message(entry.activating_stake, self.use_lamports_unit, false),
-                build_balance_message(entry.deactivating_stake, self.use_lamports_unit, false),
+                build_balance_message_with_config(entry.effective_stake, &config),
+                build_balance_message_with_config(entry.activating_stake, &config),
+                build_balance_message_with_config(entry.deactivating_stake, &config),
                 if self.use_lamports_unit {
                     "lamports"
                 } else {
@@ -1143,18 +1157,6 @@ pub struct CliBlockTime {
 impl QuietDisplay for CliBlockTime {}
 impl VerboseDisplay for CliBlockTime {}
 
-fn unix_timestamp_to_string(unix_timestamp: UnixTimestamp) -> String {
-    format!(
-        "{} (UnixTimestamp: {})",
-        match NaiveDateTime::from_timestamp_opt(unix_timestamp, 0) {
-            Some(ndt) =>
-                DateTime::<Utc>::from_utc(ndt, Utc).to_rfc3339_opts(SecondsFormat::Secs, true),
-            None => "unknown".to_string(),
-        },
-        unix_timestamp,
-    )
-}
-
 impl fmt::Display for CliBlockTime {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln_name_value(f, "Block:", &self.slot.to_string())?;
@@ -1162,6 +1164,104 @@ impl fmt::Display for CliBlockTime {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliLeaderSchedule {
+    pub epoch: Epoch,
+    pub leader_schedule_entries: Vec<CliLeaderScheduleEntry>,
+}
+
+impl QuietDisplay for CliLeaderSchedule {}
+impl VerboseDisplay for CliLeaderSchedule {}
+
+impl fmt::Display for CliLeaderSchedule {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        for entry in &self.leader_schedule_entries {
+            writeln!(f, "  {:<15} {:<44}", entry.slot, entry.leader)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliLeaderScheduleEntry {
+    pub slot: Slot,
+    pub leader: String,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliInflation {
+    pub governor: RpcInflationGovernor,
+    pub current_rate: RpcInflationRate,
+}
+
+impl QuietDisplay for CliInflation {}
+impl VerboseDisplay for CliInflation {}
+
+impl fmt::Display for CliInflation {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "{}", style("Inflation Governor:").bold())?;
+        if (self.governor.initial - self.governor.terminal).abs() < f64::EPSILON {
+            writeln!(
+                f,
+                "Fixed APR:               {:>5.2}%",
+                self.governor.terminal * 100.
+            )?;
+        } else {
+            writeln!(
+                f,
+                "Initial APR:             {:>5.2}%",
+                self.governor.initial * 100.
+            )?;
+            writeln!(
+                f,
+                "Terminal APR:            {:>5.2}%",
+                self.governor.terminal * 100.
+            )?;
+            writeln!(
+                f,
+                "Rate reduction per year: {:>5.2}%",
+                self.governor.taper * 100.
+            )?;
+        }
+        if self.governor.foundation_term > 0. {
+            writeln!(
+                f,
+                "Foundation percentage:   {:>5.2}%",
+                self.governor.foundation
+            )?;
+            writeln!(
+                f,
+                "Foundation term:         {:.1} years",
+                self.governor.foundation_term
+            )?;
+        }
+
+        writeln!(
+            f,
+            "\n{}",
+            style(format!("Inflation for Epoch {}:", self.current_rate.epoch)).bold()
+        )?;
+        writeln!(
+            f,
+            "Total APR:               {:>5.2}%",
+            self.current_rate.total * 100.
+        )?;
+        writeln!(
+            f,
+            "Staking APR:             {:>5.2}%",
+            self.current_rate.validator * 100.
+        )?;
+        writeln!(
+            f,
+            "Foundation APR:          {:>5.2}%",
+            self.current_rate.foundation * 100.
+        )
+    }
+}
+
 #[derive(Serialize, Deserialize, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct CliSignOnlyData {
@@ -1367,6 +1467,114 @@ impl fmt::Display for CliTokenAccount {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliProgramId {
+    pub program_id: String,
+}
+
+impl QuietDisplay for CliProgramId {}
+impl VerboseDisplay for CliProgramId {}
+
+impl fmt::Display for CliProgramId {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Program Id:", &self.program_id)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliProgramBuffer {
+    pub buffer: String,
+}
+
+impl QuietDisplay for CliProgramBuffer {}
+impl VerboseDisplay for CliProgramBuffer {}
+
+impl fmt::Display for CliProgramBuffer {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Buffer:", &self.buffer)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum CliProgramAccountType {
+    Buffer,
+    Program,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliProgramAuthority {
+    pub authority: String,
+    pub account_type: CliProgramAccountType,
+}
+
+impl QuietDisplay for CliProgramAuthority {}
+impl VerboseDisplay for CliProgramAuthority {}
+
+impl fmt::Display for CliProgramAuthority {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Account Type:", &format!("{:?}", self.account_type))?;
+        writeln_name_value(f, "Authority:", &self.authority)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliUpgradeableProgram {
+    pub program_id: String,
+    pub programdata_address: String,
+    pub authority: String,
+    pub last_deploy_slot: u64,
+    pub data_len: usize,
+}
+impl QuietDisplay for CliUpgradeableProgram {}
+impl VerboseDisplay for CliUpgradeableProgram {}
+impl fmt::Display for CliUpgradeableProgram {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Program Id:", &self.program_id)?;
+        writeln_name_value(f, "ProgramData Address:", &self.programdata_address)?;
+        writeln_name_value(f, "Authority:", &self.authority)?;
+        writeln_name_value(
+            f,
+            "Last Deployed In Slot:",
+            &self.last_deploy_slot.to_string(),
+        )?;
+        writeln_name_value(
+            f,
+            "Data Length:",
+            &format!("{:?} ({:#x?}) bytes", self.data_len, self.data_len),
+        )?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliUpgradeableBuffer {
+    pub address: String,
+    pub authority: String,
+    pub data_len: usize,
+}
+impl QuietDisplay for CliUpgradeableBuffer {}
+impl VerboseDisplay for CliUpgradeableBuffer {}
+impl fmt::Display for CliUpgradeableBuffer {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Buffer Address:", &self.address)?;
+        writeln_name_value(f, "Authority:", &self.authority)?;
+        writeln_name_value(
+            f,
+            "Data Length:",
+            &format!("{:?} ({:#x?}) bytes", self.data_len, self.data_len),
+        )?;
+        Ok(())
+    }
+}
+
 pub fn return_signers(
     tx: &Transaction,
     output_format: &OutputFormat,
@@ -1453,6 +1661,37 @@ pub fn parse_sign_only_reply_string(reply: &str) -> SignOnly {
     }
 }
 
+#[derive(Debug)]
+pub enum CliSignatureVerificationStatus {
+    None,
+    Pass,
+    Fail,
+}
+
+impl CliSignatureVerificationStatus {
+    pub fn verify_transaction(tx: &Transaction) -> Vec<Self> {
+        tx.verify_with_results()
+            .iter()
+            .zip(&tx.signatures)
+            .map(|(stat, sig)| match stat {
+                true => CliSignatureVerificationStatus::Pass,
+                false if sig == &Signature::default() => CliSignatureVerificationStatus::None,
+                false => CliSignatureVerificationStatus::Fail,
+            })
+            .collect()
+    }
+}
+
+impl fmt::Display for CliSignatureVerificationStatus {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::None => write!(f, "none"),
+            Self::Pass => write!(f, "pass"),
+            Self::Fail => write!(f, "fail"),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

cli-output/src/display.rs

@@ -1,28 +1,73 @@
-use console::style;
-use indicatif::{ProgressBar, ProgressStyle};
-use solana_sdk::{
-    hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
-    transaction::Transaction,
+use {
+    crate::cli_output::CliSignatureVerificationStatus,
+    chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc},
+    console::style,
+    indicatif::{ProgressBar, ProgressStyle},
+    solana_sdk::{
+        clock::UnixTimestamp, hash::Hash, native_token::lamports_to_sol,
+        program_utils::limited_deserialize, transaction::Transaction,
+    },
+    solana_transaction_status::UiTransactionStatusMeta,
+    std::{collections::HashMap, fmt, io},
 };
-use solana_transaction_status::UiTransactionStatusMeta;
-use std::{collections::HashMap, fmt, io};
 
-pub fn build_balance_message(lamports: u64, use_lamports_unit: bool, show_unit: bool) -> String {
-    if use_lamports_unit {
-        let ess = if lamports == 1 { "" } else { "s" };
-        let unit = if show_unit {
-            format!(" lamport{}", ess)
-        } else {
-            "".to_string()
-        };
-        format!("{:?}{}", lamports, unit)
+#[derive(Clone, Debug)]
+pub struct BuildBalanceMessageConfig {
+    pub use_lamports_unit: bool,
+    pub show_unit: bool,
+    pub trim_trailing_zeros: bool,
+}
+
+impl Default for BuildBalanceMessageConfig {
+    fn default() -> Self {
+        Self {
+            use_lamports_unit: false,
+            show_unit: true,
+            trim_trailing_zeros: true,
+        }
+    }
+}
+
+pub fn build_balance_message_with_config(
+    lamports: u64,
+    config: &BuildBalanceMessageConfig,
+) -> String {
+    let value = if config.use_lamports_unit {
+        lamports.to_string()
     } else {
         let sol = lamports_to_sol(lamports);
         let sol_str = format!("{:.9}", sol);
-        let pretty_sol = sol_str.trim_end_matches('0').trim_end_matches('.');
-        let unit = if show_unit { " SOL" } else { "" };
-        format!("{}{}", pretty_sol, unit)
-    }
+        if config.trim_trailing_zeros {
+            sol_str
+                .trim_end_matches('0')
+                .trim_end_matches('.')
+                .to_string()
+        } else {
+            sol_str
+        }
+    };
+    let unit = if config.show_unit {
+        if config.use_lamports_unit {
+            let ess = if lamports == 1 { "" } else { "s" };
+            format!(" lamport{}", ess)
+        } else {
+            " SOL".to_string()
+        }
+    } else {
+        "".to_string()
+    };
+    format!("{}{}", value, unit)
+}
+
+pub fn build_balance_message(lamports: u64, use_lamports_unit: bool, show_unit: bool) -> String {
+    build_balance_message_with_config(
+        lamports,
+        &BuildBalanceMessageConfig {
+            use_lamports_unit,
+            show_unit,
+            ..BuildBalanceMessageConfig::default()
+        },
+    )
 }
 
 // Pretty print a "name value"
@@ -35,7 +80,7 @@ pub fn println_name_value(name: &str, value: &str) {
     println!("{} {}", style(name).bold(), styled_value);
 }
 
-pub fn writeln_name_value(f: &mut fmt::Formatter, name: &str, value: &str) -> fmt::Result {
+pub fn writeln_name_value(f: &mut dyn fmt::Write, name: &str, value: &str) -> fmt::Result {
     let styled_value = if value.is_empty() {
         style("(not set)").italic()
     } else {
@@ -85,6 +130,7 @@ pub fn write_transaction<W: io::Write>(
     transaction: &Transaction,
     transaction_status: &Option<UiTransactionStatusMeta>,
     prefix: &str,
+    sigverify_status: Option<&[CliSignatureVerificationStatus]>,
 ) -> io::Result<()> {
     let message = &transaction.message;
     writeln!(
@@ -92,11 +138,24 @@ pub fn write_transaction<W: io::Write>(
         "{}Recent Blockhash: {:?}",
         prefix, message.recent_blockhash
     )?;
-    for (signature_index, signature) in transaction.signatures.iter().enumerate() {
+    let sigverify_statuses = if let Some(sigverify_status) = sigverify_status {
+        sigverify_status
+            .iter()
+            .map(|s| format!(" ({})", s))
+            .collect()
+    } else {
+        vec!["".to_string(); transaction.signatures.len()]
+    };
+    for (signature_index, (signature, sigverify_status)) in transaction
+        .signatures
+        .iter()
+        .zip(&sigverify_statuses)
+        .enumerate()
+    {
         writeln!(
             w,
-            "{}Signature {}: {:?}",
-            prefix, signature_index, signature
+            "{}Signature {}: {:?}{}",
+            prefix, signature_index, signature, sigverify_status,
         )?;
     }
     writeln!(w, "{}{:?}", prefix, message.header)?;
@@ -217,9 +276,18 @@ pub fn println_transaction(
     transaction: &Transaction,
     transaction_status: &Option<UiTransactionStatusMeta>,
     prefix: &str,
+    sigverify_status: Option<&[CliSignatureVerificationStatus]>,
 ) {
     let mut w = Vec::new();
-    if write_transaction(&mut w, transaction, transaction_status, prefix).is_ok() {
+    if write_transaction(
+        &mut w,
+        transaction,
+        transaction_status,
+        prefix,
+        sigverify_status,
+    )
+    .is_ok()
+    {
         if let Ok(s) = String::from_utf8(w) {
             print!("{}", s);
         }
@@ -235,6 +303,13 @@ pub fn new_spinner_progress_bar() -> ProgressBar {
     progress_bar
 }
 
+pub fn unix_timestamp_to_string(unix_timestamp: UnixTimestamp) -> String {
+    match NaiveDateTime::from_timestamp_opt(unix_timestamp, 0) {
+        Some(ndt) => DateTime::<Utc>::from_utc(ndt, Utc).to_rfc3339_opts(SecondsFormat::Secs, true),
+        None => format!("UnixTimestamp {}", unix_timestamp),
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,29 +27,29 @@ reqwest = { version = "0.10.8", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-cli-config = { path = "../cli-config", version = "1.5.0" }
-solana-cli-output = { path = "../cli-output", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-config-program = { path = "../programs/config", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana_rbpf = "=0.2.2"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.8" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-cli-config = { path = "../cli-config", version = "1.5.8" }
+solana-cli-output = { path = "../cli-output", version = "1.5.8" }
+solana-client = { path = "../client", version = "1.5.8" }
+solana-config-program = { path = "../programs/config", version = "1.5.8" }
+solana-faucet = { path = "../faucet", version = "1.5.8" }
+solana-logger = { path = "../logger", version = "1.5.8" }
+solana-net-utils = { path = "../net-utils", version = "1.5.8" }
+solana_rbpf = "=0.2.5"
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.8" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.8" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.5.0" }
+solana-core = { path = "../core", version = "1.5.8" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cluster_query.rs

@@ -6,8 +6,8 @@ use crate::{
 use chrono::{Local, TimeZone};
 use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
+use serde::{Deserialize, Serialize};
 use solana_clap_utils::{
-    commitment::{commitment_arg, commitment_arg_with_default},
     input_parsers::*,
     input_validators::*,
     keypair::DefaultSigner,
@@ -15,7 +15,8 @@ use solana_clap_utils::{
 };
 use solana_cli_output::{
     display::{
-        format_labeled_address, new_spinner_progress_bar, println_name_value, println_transaction,
+        build_balance_message, format_labeled_address, new_spinner_progress_bar,
+        println_name_value, println_transaction, unix_timestamp_to_string, writeln_name_value,
     },
     *,
 };
@@ -41,17 +42,21 @@ use solana_sdk::{
     message::Message,
     native_token::lamports_to_sol,
     pubkey::{self, Pubkey},
+    rent::Rent,
+    rpc_port::DEFAULT_RPC_PORT_STR,
     signature::Signature,
     system_instruction, system_program,
     sysvar::{
         self,
         stake_history::{self},
     },
+    timing,
     transaction::Transaction,
 };
 use solana_transaction_status::UiTransactionEncoding;
 use std::{
     collections::{BTreeMap, HashMap, VecDeque},
+    fmt,
     net::SocketAddr,
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -88,14 +93,14 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 .arg(
                     pubkey!(Arg::with_name("node_pubkey")
                         .index(1)
-                        .value_name("VALIDATOR_PUBKEY")
-                        .required(true),
+                        .value_name("OUR_VALIDATOR_PUBKEY")
+                        .required(false),
                         "Identity pubkey of the validator"),
                 )
                 .arg(
                     Arg::with_name("node_json_rpc_url")
                         .index(2)
-                        .value_name("URL")
+                        .value_name("OUR_URL")
                         .takes_value(true)
                         .validator(is_url)
                         .help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
@@ -106,17 +111,32 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .takes_value(false)
                         .help("Continue reporting progress even after the validator has caught up"),
                 )
-                .arg(commitment_arg()),
+                .arg(
+                    Arg::with_name("our_localhost")
+                        .long("our-localhost")
+                        .takes_value(false)
+                        .value_name("PORT")
+                        .default_value(&DEFAULT_RPC_PORT_STR)
+                        .validator(is_port)
+                        .help("Guess Identity pubkey and validator rpc node assuming local (possibly private) validator"),
+                )
+                .arg(
+                    Arg::with_name("log")
+                        .long("log")
+                        .takes_value(false)
+                        .help("Don't update the progress inplace; instead show updates with its own new lines"),
+                ),
         )
         .subcommand(
             SubCommand::with_name("cluster-date")
-                .about("Get current cluster date, computed from genesis creation time and network time")
+                .about("Get current cluster date, computed from genesis creation time and network time"),
         )
         .subcommand(
             SubCommand::with_name("cluster-version")
                 .about("Get the version of the cluster entrypoint"),
         )
-        .subcommand(SubCommand::with_name("fees").about("Display current cluster fees"))
+        .subcommand(SubCommand::with_name("fees").about("Display current cluster fees"),
+        )
         .subcommand(
             SubCommand::with_name("first-available-block")
                 .about("Get the first available block in the storage"),
@@ -132,12 +152,21 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .help("Slot number of the block to query")
             )
         )
-        .subcommand(SubCommand::with_name("leader-schedule").about("Display leader schedule"))
+        .subcommand(SubCommand::with_name("leader-schedule")
+            .about("Display leader schedule")
+            .arg(
+                Arg::with_name("epoch")
+                    .long("epoch")
+                    .takes_value(true)
+                    .value_name("EPOCH")
+                    .validator(is_epoch)
+                    .help("Epoch to show leader schedule for. (default: current)")
+            )
+        )
         .subcommand(
             SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
-            .alias("get-epoch-info")
-            .arg(commitment_arg()),
+            .alias("get-epoch-info"),
         )
         .subcommand(
             SubCommand::with_name("genesis-hash")
@@ -146,16 +175,13 @@ impl ClusterQuerySubCommands for App<'_, '_> {
         )
         .subcommand(
             SubCommand::with_name("slot").about("Get current slot")
-            .alias("get-slot")
-            .arg(commitment_arg()),
+            .alias("get-slot"),
         )
         .subcommand(
-            SubCommand::with_name("block-height").about("Get current block height")
-            .arg(commitment_arg()),
+            SubCommand::with_name("block-height").about("Get current block height"),
         )
         .subcommand(
-            SubCommand::with_name("epoch").about("Get current epoch")
-            .arg(commitment_arg()),
+            SubCommand::with_name("epoch").about("Get current epoch"),
         )
         .subcommand(
             SubCommand::with_name("largest-accounts").about("Get addresses of largest cluster accounts")
@@ -171,8 +197,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .takes_value(false)
                     .conflicts_with("circulating")
                     .help("Filter address list to only non-circulating accounts")
-            )
-            .arg(commitment_arg()),
+            ),
         )
         .subcommand(
             SubCommand::with_name("supply").about("Get information about the cluster supply of SOL")
@@ -181,18 +206,15 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .long("print-accounts")
                     .takes_value(false)
                     .help("Print list of non-circualting account addresses")
-            )
-            .arg(commitment_arg()),
+            ),
         )
         .subcommand(
             SubCommand::with_name("total-supply").about("Get total number of SOL")
-            .setting(AppSettings::Hidden)
-            .arg(commitment_arg()),
+            .setting(AppSettings::Hidden),
         )
         .subcommand(
             SubCommand::with_name("transaction-count").about("Get current transaction count")
-            .alias("get-transaction-count")
-            .arg(commitment_arg()),
+            .alias("get-transaction-count"),
         )
         .subcommand(
             SubCommand::with_name("ping")
@@ -239,8 +261,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .default_value("15")
                         .help("Wait up to timeout seconds for transaction confirmation"),
                 )
-                .arg(blockhash_arg())
-                .arg(commitment_arg()),
+                .arg(blockhash_arg()),
         )
         .subcommand(
             SubCommand::with_name("live-slots")
@@ -263,8 +284,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .takes_value(false)
                         .conflicts_with("address")
                         .help("Include vote transactions when monitoring all transactions")
-                )
-                .arg(commitment_arg_with_default("singleGossip")),
+                ),
         )
         .subcommand(
             SubCommand::with_name("block-production")
@@ -314,8 +334,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                )
-                .arg(commitment_arg()),
+                ),
         )
         .subcommand(
             SubCommand::with_name("transaction-history")
@@ -362,6 +381,23 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .index(1),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("rent")
+                .about("Calculate per-epoch and rent-exempt-minimum values for a given account data length.")
+                .arg(
+                    Arg::with_name("data_length")
+                        .index(1)
+                        .value_name("DATA_LENGTH")
+                        .required(true)
+                        .help("Length of data in the account to calculate rent for"),
+                )
+                .arg(
+                    Arg::with_name("lamports")
+                        .long("lamports")
+                        .takes_value(false)
+                        .help("Display rent in lamports instead of SOL"),
+                ),
+        )
     }
 }
 
@@ -369,14 +405,31 @@ pub fn parse_catchup(
     matches: &ArgMatches<'_>,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
+    let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?;
+    let mut our_localhost_port = value_t!(matches, "our_localhost", u16).ok();
+    // if there is no explicitly specified --our-localhost,
+    // disable the guess mode (= our_localhost_port)
+    if matches.occurrences_of("our_localhost") == 0 {
+        our_localhost_port = None
+    }
     let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
+    // requirement of node_pubkey is relaxed only if our_localhost_port
+    if our_localhost_port.is_none() && node_pubkey.is_none() {
+        return Err(CliError::BadParameter(
+            "OUR_VALIDATOR_PUBKEY (and possibly OUR_URL) must be specified \
+             unless --our-localhost is given"
+                .into(),
+        ));
+    }
     let follow = matches.is_present("follow");
+    let log = matches.is_present("log");
     Ok(CliCommandInfo {
         command: CliCommand::Catchup {
             node_pubkey,
             node_json_rpc_url,
             follow,
+            our_localhost_port,
+            log,
         },
         signers: vec![],
     })
@@ -556,38 +609,76 @@ pub fn parse_transaction_history(
 pub fn process_catchup(
     rpc_client: &RpcClient,
     config: &CliConfig,
-    node_pubkey: &Pubkey,
-    node_json_rpc_url: &Option<String>,
+    node_pubkey: Option<Pubkey>,
+    mut node_json_rpc_url: Option<String>,
     follow: bool,
+    our_localhost_port: Option<u16>,
+    log: bool,
 ) -> ProcessResult {
     let sleep_interval = 5;
 
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message("Connecting...");
 
-    let node_client = if let Some(node_json_rpc_url) = node_json_rpc_url {
-        RpcClient::new(node_json_rpc_url.to_string())
-    } else {
-        let rpc_addr = loop {
-            let cluster_nodes = rpc_client.get_cluster_nodes()?;
-            if let Some(contact_info) = cluster_nodes
-                .iter()
-                .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
-            {
-                if let Some(rpc_addr) = contact_info.rpc {
-                    break rpc_addr;
-                }
-                progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
-            } else {
-                progress_bar.set_message(&format!(
-                    "Contact information not found for {}",
-                    node_pubkey
-                ));
-            }
-            sleep(Duration::from_secs(sleep_interval as u64));
-        };
+    if let Some(our_localhost_port) = our_localhost_port {
+        let gussed_default = Some(format!("http://localhost:{}", our_localhost_port));
+        if node_json_rpc_url.is_some() && node_json_rpc_url != gussed_default {
+            // go to new line to leave this message on console
+            println!(
+                "Prefering explicitly given rpc ({}) as us, \
+                 although --our-localhost is given\n",
+                node_json_rpc_url.as_ref().unwrap()
+            );
+        } else {
+            node_json_rpc_url = gussed_default;
+        }
+    }
 
-        RpcClient::new_socket(rpc_addr)
+    let (node_client, node_pubkey) = if our_localhost_port.is_some() {
+        let client = RpcClient::new(node_json_rpc_url.unwrap());
+        let guessed_default = Some(client.get_identity()?);
+        (
+            client,
+            (if node_pubkey.is_some() && node_pubkey != guessed_default {
+                // go to new line to leave this message on console
+                println!(
+                    "Prefering explicitly given node pubkey ({}) as us, \
+                     although --our-localhost is given\n",
+                    node_pubkey.unwrap()
+                );
+                node_pubkey
+            } else {
+                guessed_default
+            })
+            .unwrap(),
+        )
+    } else if let Some(node_pubkey) = node_pubkey {
+        if let Some(node_json_rpc_url) = node_json_rpc_url {
+            (RpcClient::new(node_json_rpc_url), node_pubkey)
+        } else {
+            let rpc_addr = loop {
+                let cluster_nodes = rpc_client.get_cluster_nodes()?;
+                if let Some(contact_info) = cluster_nodes
+                    .iter()
+                    .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
+                {
+                    if let Some(rpc_addr) = contact_info.rpc {
+                        break rpc_addr;
+                    }
+                    progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
+                } else {
+                    progress_bar.set_message(&format!(
+                        "Contact information not found for {}",
+                        node_pubkey
+                    ));
+                }
+                sleep(Duration::from_secs(sleep_interval as u64));
+            };
+
+            (RpcClient::new_socket(rpc_addr), node_pubkey)
+        }
+    } else {
+        unreachable!()
     };
 
     let reported_node_pubkey = loop {
@@ -604,7 +695,7 @@ pub fn process_catchup(
         }
     };
 
-    if reported_node_pubkey != *node_pubkey {
+    if reported_node_pubkey != node_pubkey {
         return Err(format!(
             "The identity reported by node RPC URL does not match.  Expected: {:?}.  Reported: {:?}",
             node_pubkey, reported_node_pubkey
@@ -612,15 +703,41 @@ pub fn process_catchup(
         .into());
     }
 
-    if rpc_client.get_identity()? == *node_pubkey {
+    if rpc_client.get_identity()? == node_pubkey {
         return Err("Both RPC URLs reference the same node, unable to monitor for catchup.  Try a different --url".into());
     }
 
     let mut previous_rpc_slot = std::u64::MAX;
     let mut previous_slot_distance = 0;
+    let mut retry_count = 0;
+    let max_retry_count = 5;
+    let mut get_slot_while_retrying = |client: &RpcClient| {
+        loop {
+            match client.get_slot_with_commitment(config.commitment) {
+                Ok(r) => {
+                    retry_count = 0;
+                    return Ok(r);
+                }
+                Err(e) => {
+                    if retry_count >= max_retry_count {
+                        return Err(e);
+                    }
+                    retry_count += 1;
+                    if log {
+                        // go to new line to leave this message on console
+                        println!("Retrying({}/{}): {}\n", retry_count, max_retry_count, e);
+                    }
+                    sleep(Duration::from_secs(1));
+                }
+            };
+        }
+    };
+
     loop {
-        let rpc_slot = rpc_client.get_slot_with_commitment(config.commitment)?;
-        let node_slot = node_client.get_slot_with_commitment(config.commitment)?;
+        // humbly retry; the reference node (rpc_client) could be spotty,
+        // especially if pointing to api.meinnet-beta.solana.com at times
+        let rpc_slot = get_slot_while_retrying(rpc_client)?;
+        let node_slot = get_slot_while_retrying(&node_client)?;
         if !follow && node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
             progress_bar.finish_and_clear();
             return Ok(format!(
@@ -643,15 +760,21 @@ pub fn process_catchup(
         };
 
         progress_bar.set_message(&format!(
-            "{} slots behind (us:{} them:{}){}",
-            slot_distance,
+            "{} slot(s) {} (us:{} them:{}){}",
+            slot_distance.abs(),
+            if slot_distance >= 0 {
+                "behind"
+            } else {
+                "ahead"
+            },
             node_slot,
             rpc_slot,
             if slot_distance == 0 || previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
                 format!(
-                    ", {} at {:.1} slots/second{}",
+                    ", {} node is {} at {:.1} slots/second{}",
+                    if slot_distance >= 0 { "our" } else { "their" },
                     if slots_per_second < 0.0 {
                         "falling behind"
                     } else {
@@ -660,8 +783,11 @@ pub fn process_catchup(
                     slots_per_second,
                     time_remaining
                 )
-            }
+            },
         ));
+        if log {
+            println!();
+        }
 
         sleep(Duration::from_secs(sleep_interval as u64));
         previous_rpc_slot = rpc_slot;
@@ -670,8 +796,7 @@ pub fn process_catchup(
 }
 
 pub fn process_cluster_date(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let result = rpc_client
-        .get_account_with_commitment(&sysvar::clock::id(), CommitmentConfig::default())?;
+    let result = rpc_client.get_account_with_commitment(&sysvar::clock::id(), config.commitment)?;
     if let Some(clock_account) = result.value {
         let clock: Clock = from_account(&clock_account).ok_or_else(|| {
             CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
@@ -697,7 +822,7 @@ pub fn process_cluster_version(rpc_client: &RpcClient, config: &CliConfig) -> Pr
 }
 
 pub fn process_fees(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let result = rpc_client.get_recent_blockhash_with_commitment(CommitmentConfig::default())?;
+    let result = rpc_client.get_recent_blockhash_with_commitment(config.commitment)?;
     let (recent_blockhash, fee_calculator, last_valid_slot) = result.value;
     let fees = CliFees {
         slot: result.context.slot,
@@ -713,9 +838,27 @@ pub fn process_first_available_block(rpc_client: &RpcClient) -> ProcessResult {
     Ok(format!("{}", first_available_block))
 }
 
-pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
+pub fn parse_leader_schedule(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let epoch = value_of(matches, "epoch");
+    Ok(CliCommandInfo {
+        command: CliCommand::LeaderSchedule { epoch },
+        signers: vec![],
+    })
+}
+
+pub fn process_leader_schedule(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    epoch: Option<Epoch>,
+) -> ProcessResult {
     let epoch_info = rpc_client.get_epoch_info()?;
-    let first_slot_in_epoch = epoch_info.absolute_slot - epoch_info.slot_index;
+    let epoch = epoch.unwrap_or(epoch_info.epoch);
+    if epoch > epoch_info.epoch {
+        return Err(format!("Epoch {} is in the future", epoch).into());
+    }
+
+    let epoch_schedule = rpc_client.get_epoch_schedule()?;
+    let first_slot_in_epoch = epoch_schedule.get_first_slot_in_epoch(epoch);
 
     let leader_schedule = rpc_client.get_leader_schedule(Some(first_slot_in_epoch))?;
     if leader_schedule.is_none() {
@@ -737,15 +880,18 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
         }
     }
 
+    let mut leader_schedule_entries = vec![];
     for (slot_index, leader) in leader_per_slot_index.iter().enumerate() {
-        println!(
-            "  {:<15} {:<44}",
-            first_slot_in_epoch + slot_index as u64,
-            leader
-        );
+        leader_schedule_entries.push(CliLeaderScheduleEntry {
+            slot: first_slot_in_epoch + slot_index as u64,
+            leader: leader.to_string(),
+        });
     }
 
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&CliLeaderSchedule {
+        epoch,
+        leader_schedule_entries,
+    }))
 }
 
 pub fn process_get_block(
@@ -756,7 +902,7 @@ pub fn process_get_block(
     let slot = if let Some(slot) = slot {
         slot
     } else {
-        rpc_client.get_slot()?
+        rpc_client.get_slot_with_commitment(CommitmentConfig::finalized())?
     };
 
     let mut block =
@@ -800,8 +946,9 @@ pub fn process_get_block(
                     format!(
                         "◎{:<19.9}  {:>13.9}%",
                         lamports_to_sol(reward.post_balance),
-                        reward.lamports.abs() as f64
-                            / (reward.post_balance as f64 - reward.lamports as f64)
+                        (reward.lamports.abs() as f64
+                            / (reward.post_balance as f64 - reward.lamports as f64))
+                            * 100.0
                     )
                 }
             );
@@ -820,6 +967,7 @@ pub fn process_get_block(
             &transaction_with_meta.transaction.decode().unwrap(),
             &transaction_with_meta.meta,
             "  ",
+            None,
         );
     }
     Ok("".to_string())
@@ -833,22 +981,20 @@ pub fn process_get_block_time(
     let slot = if let Some(slot) = slot {
         slot
     } else {
-        rpc_client.get_slot()?
+        rpc_client.get_slot_with_commitment(CommitmentConfig::finalized())?
     };
     let timestamp = rpc_client.get_block_time(slot)?;
     let block_time = CliBlockTime { slot, timestamp };
     Ok(config.output_format.formatted_string(&block_time))
 }
 
-pub fn process_get_epoch(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let epoch_info = rpc_client.get_epoch_info_with_commitment(config.commitment)?;
+pub fn process_get_epoch(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
+    let epoch_info = rpc_client.get_epoch_info()?;
     Ok(epoch_info.epoch.to_string())
 }
 
 pub fn process_get_epoch_info(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let epoch_info: CliEpochInfo = rpc_client
-        .get_epoch_info_with_commitment(config.commitment)?
-        .into();
+    let epoch_info: CliEpochInfo = rpc_client.get_epoch_info()?.into();
     Ok(config.output_format.formatted_string(&epoch_info))
 }
 
@@ -857,15 +1003,13 @@ pub fn process_get_genesis_hash(rpc_client: &RpcClient) -> ProcessResult {
     Ok(genesis_hash.to_string())
 }
 
-pub fn process_get_slot(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let slot = rpc_client.get_slot_with_commitment(config.commitment)?;
+pub fn process_get_slot(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
+    let slot = rpc_client.get_slot()?;
     Ok(slot.to_string())
 }
 
-pub fn process_get_block_height(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let epoch_info: CliEpochInfo = rpc_client
-        .get_epoch_info_with_commitment(config.commitment)?
-        .into();
+pub fn process_get_block_height(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
+    let epoch_info: CliEpochInfo = rpc_client.get_epoch_info()?.into();
     Ok(epoch_info.epoch_info.block_height.to_string())
 }
 
@@ -886,7 +1030,7 @@ pub fn process_show_block_production(
     slot_limit: Option<u64>,
 ) -> ProcessResult {
     let epoch_schedule = rpc_client.get_epoch_schedule()?;
-    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::root())?;
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::finalized())?;
 
     let epoch = epoch.unwrap_or(epoch_info.epoch);
     if epoch > epoch_info.epoch {
@@ -945,7 +1089,7 @@ pub fn process_show_block_production(
 
     progress_bar.set_message(&format!("Fetching leader schedule for epoch {}...", epoch));
     let leader_schedule = rpc_client
-        .get_leader_schedule_with_commitment(Some(start_slot), CommitmentConfig::root())?;
+        .get_leader_schedule_with_commitment(Some(start_slot), CommitmentConfig::finalized())?;
     if leader_schedule.is_none() {
         return Err(format!("Unable to fetch leader schedule for slot {}", start_slot).into());
     }
@@ -1051,19 +1195,19 @@ pub fn process_supply(
     config: &CliConfig,
     print_accounts: bool,
 ) -> ProcessResult {
-    let supply_response = rpc_client.supply_with_commitment(config.commitment)?;
+    let supply_response = rpc_client.supply()?;
     let mut supply: CliSupply = supply_response.value.into();
     supply.print_accounts = print_accounts;
     Ok(config.output_format.formatted_string(&supply))
 }
 
-pub fn process_total_supply(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let total_supply = rpc_client.total_supply_with_commitment(config.commitment)?;
+pub fn process_total_supply(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
+    let total_supply = rpc_client.total_supply()?;
     Ok(format!("{} SOL", lamports_to_sol(total_supply)))
 }
 
-pub fn process_get_transaction_count(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let transaction_count = rpc_client.get_transaction_count_with_commitment(config.commitment)?;
+pub fn process_get_transaction_count(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
+    let transaction_count = rpc_client.get_transaction_count()?;
     Ok(transaction_count.to_string())
 }
 
@@ -1154,8 +1298,7 @@ pub fn process_ping(
             Ok(signature) => {
                 let transaction_sent = Instant::now();
                 loop {
-                    let signature_status = rpc_client
-                        .get_signature_status_with_commitment(&signature, config.commitment)?;
+                    let signature_status = rpc_client.get_signature_status(&signature)?;
                     let elapsed_time = Instant::now().duration_since(transaction_sent);
                     if let Some(transaction_status) = signature_status {
                         match transaction_status {
@@ -1495,7 +1638,7 @@ pub fn process_show_stakes(
         CliError::RpcRequestError("Failed to deserialize stake history".to_string())
     })?;
     // At v1.6, this check can be removed and simply passed as `true`
-    let stake_program_v2_enabled = is_stake_program_v2_enabled(rpc_client);
+    let stake_program_v2_enabled = is_stake_program_v2_enabled(rpc_client)?;
 
     let mut stake_accounts: Vec<CliKeyedStakeState> = vec![];
     for (stake_pubkey, stake_account) in all_stake_accounts {
@@ -1559,8 +1702,8 @@ pub fn process_show_validators(
     config: &CliConfig,
     use_lamports_unit: bool,
 ) -> ProcessResult {
-    let epoch_info = rpc_client.get_epoch_info_with_commitment(config.commitment)?;
-    let vote_accounts = rpc_client.get_vote_accounts_with_commitment(config.commitment)?;
+    let epoch_info = rpc_client.get_epoch_info()?;
+    let vote_accounts = rpc_client.get_vote_accounts()?;
 
     let mut node_version = HashMap::new();
     let unknown_version = "unknown".to_string();
@@ -1671,9 +1814,14 @@ pub fn process_transaction_history(
     for result in results {
         if config.verbose {
             println!(
-                "{} [slot={} status={}] {}",
+                "{} [slot={} {}status={}] {}",
                 result.signature,
                 result.slot,
+                match result.block_time {
+                    None => "".to_string(),
+                    Some(block_time) =>
+                        format!("timestamp={} ", unix_timestamp_to_string(block_time)),
+                },
                 match result.err {
                     None => "Confirmed".to_string(),
                     Some(err) => format!("Failed: {:?}", err),
@@ -1698,6 +1846,7 @@ pub fn process_transaction_history(
                                 .expect("Successful decode"),
                             &confirmed_transaction.transaction.meta,
                             "  ",
+                            None,
                         );
                     }
                     Err(err) => println!("  Unable to get confirmed transaction details: {}", err),
@@ -1709,6 +1858,62 @@ pub fn process_transaction_history(
     Ok(transactions_found)
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct CliRentCalculation {
+    pub lamports_per_byte_year: u64,
+    pub lamports_per_epoch: u64,
+    pub rent_exempt_minimum_lamports: u64,
+    #[serde(skip)]
+    pub use_lamports_unit: bool,
+}
+
+impl CliRentCalculation {
+    fn build_balance_message(&self, lamports: u64) -> String {
+        build_balance_message(lamports, self.use_lamports_unit, true)
+    }
+}
+
+impl fmt::Display for CliRentCalculation {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let per_byte_year = self.build_balance_message(self.lamports_per_byte_year);
+        let per_epoch = self.build_balance_message(self.lamports_per_epoch);
+        let exempt_minimum = self.build_balance_message(self.rent_exempt_minimum_lamports);
+        writeln_name_value(f, "Rent per byte-year:", &per_byte_year)?;
+        writeln_name_value(f, "Rent per epoch:", &per_epoch)?;
+        writeln_name_value(f, "Rent-exempt minimum:", &exempt_minimum)
+    }
+}
+
+impl QuietDisplay for CliRentCalculation {}
+impl VerboseDisplay for CliRentCalculation {}
+
+pub fn process_calculate_rent(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    data_length: usize,
+    use_lamports_unit: bool,
+) -> ProcessResult {
+    let epoch_schedule = rpc_client.get_epoch_schedule()?;
+    let rent_account = rpc_client.get_account(&sysvar::rent::id())?;
+    let rent: Rent = rent_account.deserialize_data()?;
+    let rent_exempt_minimum_lamports = rent.minimum_balance(data_length);
+    let seconds_per_tick = Duration::from_secs_f64(1.0 / clock::DEFAULT_TICKS_PER_SECOND as f64);
+    let slots_per_year =
+        timing::years_as_slots(1.0, &seconds_per_tick, clock::DEFAULT_TICKS_PER_SLOT);
+    let slots_per_epoch = epoch_schedule.slots_per_epoch as f64;
+    let years_per_epoch = slots_per_epoch / slots_per_year;
+    let (lamports_per_epoch, _) = rent.due(0, data_length, years_per_epoch);
+    let cli_rent_calculation = CliRentCalculation {
+        lamports_per_byte_year: rent.lamports_per_byte_year,
+        lamports_per_epoch,
+        rent_exempt_minimum_lamports,
+        use_lamports_unit,
+    };
+
+    Ok(config.output_format.formatted_string(&cli_rent_calculation))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1851,8 +2056,6 @@ mod tests {
             "-t",
             "3",
             "-D",
-            "--commitment",
-            "max",
             "--blockhash",
             "4CCNp28j6AhGq7PkjPDP4wbQWBS8LLbQin2xV5n8frKX",
         ]);

cli/src/feature.rs

@@ -19,10 +19,22 @@ use solana_sdk::{
 };
 use std::{collections::HashMap, fmt, sync::Arc};
 
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub enum ForceActivation {
+    No,
+    Almost,
+    Yes,
+}
+
 #[derive(Debug, PartialEq)]
 pub enum FeatureCliCommand {
-    Status { features: Vec<Pubkey> },
-    Activate { feature: Pubkey },
+    Status {
+        features: Vec<Pubkey>,
+    },
+    Activate {
+        feature: Pubkey,
+        force: ForceActivation,
+    },
 }
 
 #[derive(Serialize, Deserialize)]
@@ -58,8 +70,8 @@ impl fmt::Display for CliFeatures {
                 f,
                 "{}",
                 style(format!(
-                    "{:<44} {:<40} {}",
-                    "Feature", "Description", "Status"
+                    "{:<44} {:<28} {}",
+                    "Feature", "Status", "Description"
                 ))
                 .bold()
             )?;
@@ -67,15 +79,15 @@ impl fmt::Display for CliFeatures {
         for feature in &self.features {
             writeln!(
                 f,
-                "{:<44} {:<40} {}",
+                "{:<44} {:<28} {}",
                 feature.id,
-                feature.description,
                 match feature.status {
                     CliFeatureStatus::Inactive => style("inactive".to_string()).red(),
                     CliFeatureStatus::Pending => style("activation pending".to_string()).yellow(),
                     CliFeatureStatus::Active(activation_slot) =>
                         style(format!("active since slot {}", activation_slot)).green(),
-                }
+                },
+                feature.description,
             )?;
         }
         if self.inactive && !self.feature_activation_allowed {
@@ -126,6 +138,13 @@ impl FeatureSubCommands for App<'_, '_> {
                                 .index(1)
                                 .required(true)
                                 .help("The signer for the feature to activate"),
+                        )
+                        .arg(
+                            Arg::with_name("force")
+                                .long("yolo")
+                                .hidden(true)
+                                .multiple(true)
+                                .help("Override activation sanity checks. Don't use this flag"),
                         ),
                 ),
         )
@@ -152,13 +171,20 @@ pub fn parse_feature_subcommand(
         ("activate", Some(matches)) => {
             let (feature_signer, feature) = signer_of(matches, "feature", wallet_manager)?;
             let mut signers = vec![default_signer.signer_from_path(matches, wallet_manager)?];
+
+            let force = match matches.occurrences_of("force") {
+                2 => ForceActivation::Yes,
+                1 => ForceActivation::Almost,
+                _ => ForceActivation::No,
+            };
+
             signers.push(feature_signer.unwrap());
             let feature = feature.unwrap();
 
             known_feature(&feature)?;
 
             CliCommandInfo {
-                command: CliCommand::Feature(FeatureCliCommand::Activate { feature }),
+                command: CliCommand::Feature(FeatureCliCommand::Activate { feature, force }),
                 signers,
             }
         }
@@ -189,7 +215,9 @@ pub fn process_feature_subcommand(
 ) -> ProcessResult {
     match feature_subcommand {
         FeatureCliCommand::Status { features } => process_status(rpc_client, config, features),
-        FeatureCliCommand::Activate { feature } => process_activate(rpc_client, config, *feature),
+        FeatureCliCommand::Activate { feature, force } => {
+            process_activate(rpc_client, config, *feature, *force)
+        }
     }
 }
 
@@ -255,7 +283,7 @@ fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<boo
         }
         println!(
             "{}",
-            style(format!("Tool Feture Set: {}", my_feature_set)).bold()
+            style(format!("Tool Feature Set: {}", my_feature_set)).bold()
         );
         println!("{}", style("Cluster Feature Sets and Stakes:").bold());
         for (feature_set, percentage) in active_stake_by_feature_set.iter() {
@@ -329,12 +357,14 @@ fn process_activate(
     rpc_client: &RpcClient,
     config: &CliConfig,
     feature_id: Pubkey,
+    force: ForceActivation,
 ) -> ProcessResult {
     let account = rpc_client
         .get_multiple_accounts(&[feature_id])?
         .into_iter()
         .next()
         .unwrap();
+
     if let Some(account) = account {
         if feature::from_account(&account).is_some() {
             return Err(format!("{} has already been activated", feature_id).into());
@@ -342,7 +372,13 @@ fn process_activate(
     }
 
     if !feature_activation_allowed(rpc_client, false)? {
-        return Err("Feature activation is not allowed at this time".into());
+        match force {
+        ForceActivation::Almost =>
+            return Err("Add force argument once more to override the sanity check to force feature activation ".into()),
+        ForceActivation::Yes => println!("FEATURE ACTIVATION FORCED"),
+        ForceActivation::No =>
+            return Err("Feature activation is not allowed at this time".into()),
+        }
     }
 
     let rent = rpc_client.get_minimum_balance_for_rent_exemption(Feature::size_of())?;

cli/src/inflation.rs

@@ -1,7 +1,7 @@
 use crate::cli::{CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult};
 use clap::{App, ArgMatches, SubCommand};
-use console::style;
 use solana_clap_utils::keypair::*;
+use solana_cli_output::CliInflation;
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use std::sync::Arc;
@@ -34,56 +34,18 @@ pub fn parse_inflation_subcommand(
 
 pub fn process_inflation_subcommand(
     rpc_client: &RpcClient,
-    _config: &CliConfig,
+    config: &CliConfig,
     inflation_subcommand: &InflationCliCommand,
 ) -> ProcessResult {
     assert_eq!(*inflation_subcommand, InflationCliCommand::Show);
 
     let governor = rpc_client.get_inflation_governor()?;
-    let current_inflation_rate = rpc_client.get_inflation_rate()?;
+    let current_rate = rpc_client.get_inflation_rate()?;
 
-    println!("{}", style("Inflation Governor:").bold());
-    if (governor.initial - governor.terminal).abs() < f64::EPSILON {
-        println!(
-            "Fixed APR:               {:>5.2}%",
-            governor.terminal * 100.
-        );
-    } else {
-        println!("Initial APR:             {:>5.2}%", governor.initial * 100.);
-        println!(
-            "Terminal APR:            {:>5.2}%",
-            governor.terminal * 100.
-        );
-        println!("Rate reduction per year: {:>5.2}%", governor.taper * 100.);
-    }
-    if governor.foundation_term > 0. {
-        println!("Foundation percentage:   {:>5.2}%", governor.foundation);
-        println!(
-            "Foundation term:         {:.1} years",
-            governor.foundation_term
-        );
-    }
+    let inflation = CliInflation {
+        governor,
+        current_rate,
+    };
 
-    println!(
-        "\n{}",
-        style(format!(
-            "Inflation for Epoch {}:",
-            current_inflation_rate.epoch
-        ))
-        .bold()
-    );
-    println!(
-        "Total APR:               {:>5.2}%",
-        current_inflation_rate.total * 100.
-    );
-    println!(
-        "Staking APR:             {:>5.2}%",
-        current_inflation_rate.validator * 100.
-    );
-    println!(
-        "Foundation APR:          {:>5.2}%",
-        current_inflation_rate.foundation * 100.
-    );
-
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&inflation))
 }

cli/src/lib.rs

@@ -26,6 +26,7 @@ pub mod cluster_query;
 pub mod feature;
 pub mod inflation;
 pub mod nonce;
+pub mod program;
 pub mod send_tpu;
 pub mod spend_utils;
 pub mod stake;

cli/src/main.rs

@@ -3,11 +3,8 @@ use clap::{
     SubCommand,
 };
 use console::style;
-
 use solana_clap_utils::{
-    commitment::COMMITMENT_ARG,
-    input_parsers::commitment_of,
-    input_validators::is_url,
+    input_validators::{is_url, is_url_or_moniker, normalize_to_url_if_moniker},
     keypair::{CliSigners, DefaultSigner, SKIP_SEED_PHRASE_VALIDATION_ARG},
     DisplayError,
 };
@@ -63,12 +60,19 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     );
                     let (keypair_setting_type, keypair_path) =
                         CliConfig::compute_keypair_path_setting("", &config.keypair_path);
+                    let (commitment_setting_type, commitment) =
+                        CliConfig::compute_commitment_config("", &config.commitment);
 
                     if let Some(field) = subcommand_matches.value_of("specific_setting") {
                         let (field_name, value, setting_type) = match field {
                             "json_rpc_url" => ("RPC URL", json_rpc_url, url_setting_type),
                             "websocket_url" => ("WebSocket URL", websocket_url, ws_setting_type),
                             "keypair" => ("Key Path", keypair_path, keypair_setting_type),
+                            "commitment" => (
+                                "Commitment",
+                                commitment.commitment.to_string(),
+                                commitment_setting_type,
+                            ),
                             _ => unreachable!(),
                         };
                         println_name_value_or(&format!("{}:", field_name), &value, setting_type);
@@ -77,11 +81,16 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                         println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
                         println_name_value_or("WebSocket URL:", &websocket_url, ws_setting_type);
                         println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
+                        println_name_value_or(
+                            "Commitment:",
+                            &commitment.commitment.to_string(),
+                            commitment_setting_type,
+                        );
                     }
                 }
                 ("set", Some(subcommand_matches)) => {
                     if let Some(url) = subcommand_matches.value_of("json_rpc_url") {
-                        config.json_rpc_url = url.to_string();
+                        config.json_rpc_url = normalize_to_url_if_moniker(url);
                         // Revert to a computed `websocket_url` value when `json_rpc_url` is
                         // changed
                         config.websocket_url = "".to_string();
@@ -92,6 +101,9 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     if let Some(keypair) = subcommand_matches.value_of("keypair") {
                         config.keypair_path = keypair.to_string();
                     }
+                    if let Some(commitment) = subcommand_matches.value_of("commitment") {
+                        config.commitment = commitment.to_string();
+                    }
 
                     config.save(config_file)?;
 
@@ -105,11 +117,18 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     );
                     let (keypair_setting_type, keypair_path) =
                         CliConfig::compute_keypair_path_setting("", &config.keypair_path);
+                    let (commitment_setting_type, commitment) =
+                        CliConfig::compute_commitment_config("", &config.commitment);
 
                     println_name_value("Config File:", config_file);
                     println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
                     println_name_value_or("WebSocket URL:", &websocket_url, ws_setting_type);
                     println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
+                    println_name_value_or(
+                        "Commitment:",
+                        &commitment.commitment.to_string(),
+                        commitment_setting_type,
+                    );
                 }
                 ("import-address-labels", Some(subcommand_matches)) => {
                     let filename = value_t_or_exit!(subcommand_matches, "filename", PathBuf);
@@ -165,8 +184,18 @@ pub fn parse_args<'a>(
         path: default_signer_path.clone(),
     };
 
-    let CliCommandInfo { command, signers } =
-        parse_command(&matches, &default_signer, &mut wallet_manager)?;
+    let CliCommandInfo {
+        command,
+        mut signers,
+    } = parse_command(&matches, &default_signer, &mut wallet_manager)?;
+
+    if signers.is_empty() {
+        if let Ok(signer_info) =
+            default_signer.generate_unique_signers(vec![None], matches, &mut wallet_manager)
+        {
+            signers.extend(signer_info.signers);
+        }
+    }
 
     let verbose = matches.is_present("verbose");
     let output_format = matches
@@ -182,11 +211,10 @@ pub fn parse_args<'a>(
             OutputFormat::Display
         });
 
-    let commitment = matches
-        .subcommand_name()
-        .and_then(|name| matches.subcommand_matches(name))
-        .and_then(|sub_matches| commitment_of(sub_matches, COMMITMENT_ARG.long))
-        .unwrap_or_default();
+    let (_, commitment) = CliConfig::compute_commitment_config(
+        matches.value_of("commitment").unwrap_or(""),
+        &config.commitment,
+    );
 
     let address_labels = if matches.is_present("no_address_labels") {
         HashMap::new()
@@ -206,7 +234,10 @@ pub fn parse_args<'a>(
             verbose,
             output_format,
             commitment,
-            send_transaction_config: RpcSendTransactionConfig::default(),
+            send_transaction_config: RpcSendTransactionConfig {
+                preflight_commitment: Some(commitment.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
             address_labels,
         },
         signers,
@@ -214,7 +245,7 @@ pub fn parse_args<'a>(
 }
 
 fn main() -> Result<(), Box<dyn error::Error>> {
-    solana_logger::setup();
+    solana_logger::setup_with_default("off");
     let matches = app(
         crate_name!(),
         crate_description!(),
@@ -238,11 +269,14 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         Arg::with_name("json_rpc_url")
             .short("u")
             .long("url")
-            .value_name("URL")
+            .value_name("URL_OR_MONIKER")
             .takes_value(true)
             .global(true)
-            .validator(is_url)
-            .help("JSON RPC URL for the solana cluster"),
+            .validator(is_url_or_moniker)
+            .help(
+                "URL for Solana's JSON RPC or moniker (or their first letter): \
+                   [mainnet-beta, testnet, devnet, localhost]",
+            ),
     )
     .arg(
         Arg::with_name("websocket_url")
@@ -262,6 +296,25 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             .takes_value(true)
             .help("Filepath or URL to a keypair"),
     )
+    .arg(
+        Arg::with_name("commitment")
+            .long("commitment")
+            .takes_value(true)
+            .possible_values(&[
+                "processed",
+                "confirmed",
+                "finalized",
+                "recent", // Deprecated as of v1.5.5
+                "single", // Deprecated as of v1.5.5
+                "singleGossip", // Deprecated as of v1.5.5
+                "root", // Deprecated as of v1.5.5
+                "max", // Deprecated as of v1.5.5
+            ])
+            .value_name("COMMITMENT_LEVEL")
+            .hide_possible_values(true)
+            .global(true)
+            .help("Return information at the selected commitment level [possible values: processed, confirmed, finalized]"),
+    )
     .arg(
         Arg::with_name("verbose")
             .long("verbose")
@@ -313,7 +366,12 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                             .index(1)
                             .value_name("CONFIG_FIELD")
                             .takes_value(true)
-                            .possible_values(&["json_rpc_url", "websocket_url", "keypair"])
+                            .possible_values(&[
+                                "json_rpc_url",
+                                "websocket_url",
+                                "keypair",
+                                "commitment",
+                            ])
                             .help("Return a specific config setting"),
                     ),
             )
@@ -322,7 +380,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                     .about("Set a config setting")
                     .group(
                         ArgGroup::with_name("config_settings")
-                            .args(&["json_rpc_url", "websocket_url", "keypair"])
+                            .args(&["json_rpc_url", "websocket_url", "keypair", "commitment"])
                             .multiple(true)
                             .required(true),
                     ),

cli/src/nonce.rs

@@ -332,9 +332,7 @@ pub fn process_authorize_nonce_account(
     nonce_authority: SignerIndex,
     new_authority: &Pubkey,
 ) -> ProcessResult {
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let nonce_authority = config.signers[nonce_authority];
     let ix = authorize_nonce_account(nonce_account, &nonce_authority.pubkey(), new_authority);
@@ -349,11 +347,7 @@ pub fn process_authorize_nonce_account(
         &tx.message,
         config.commitment,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<NonceError>(result, &config)
 }
 
@@ -400,9 +394,7 @@ pub fn process_create_nonce_account(
         Message::new(&ixs, Some(&config.signers[0].pubkey()))
     };
 
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let (message, lamports) = resolve_spend_tx_and_check_account_balance(
         rpc_client,
@@ -414,9 +406,7 @@ pub fn process_create_nonce_account(
         config.commitment,
     )?;
 
-    if let Ok(nonce_account) =
-        get_account_with_commitment(rpc_client, &nonce_account_address, config.commitment)
-    {
+    if let Ok(nonce_account) = get_account(rpc_client, &nonce_account_address) {
         let err_msg = if state_from_account(&nonce_account).is_ok() {
             format!("Nonce account {} already exists", nonce_account_address)
         } else {
@@ -439,11 +429,7 @@ pub fn process_create_nonce_account(
 
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<SystemError>(result, &config)
 }
 
@@ -471,20 +457,17 @@ pub fn process_new_nonce(
         (&nonce_account, "nonce_account_pubkey".to_string()),
     )?;
 
-    let nonce_account_check =
-        rpc_client.get_account_with_commitment(&nonce_account, config.commitment);
-    if nonce_account_check.is_err() || nonce_account_check.unwrap().value.is_none() {
-        return Err(CliError::BadParameter(
-            "Unable to create new nonce, no nonce account found".to_string(),
-        )
+    if let Err(err) = rpc_client.get_account(&nonce_account) {
+        return Err(CliError::BadParameter(format!(
+            "Unable to advance nonce account {}. error: {}",
+            nonce_account, err
+        ))
         .into());
     }
 
     let nonce_authority = config.signers[nonce_authority];
     let ix = advance_nonce_account(&nonce_account, &nonce_authority.pubkey());
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
@@ -495,11 +478,7 @@ pub fn process_new_nonce(
         &tx.message,
         config.commitment,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<SystemError>(result, &config)
 }
 
@@ -541,9 +520,7 @@ pub fn process_withdraw_from_nonce_account(
     destination_account_pubkey: &Pubkey,
     lamports: u64,
 ) -> ProcessResult {
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let nonce_authority = config.signers[nonce_authority];
     let ix = withdraw_nonce_account(
@@ -562,11 +539,7 @@ pub fn process_withdraw_from_nonce_account(
         &tx.message,
         config.commitment,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<NonceError>(result, &config)
 }
 

cli/src/stake.rs

@@ -64,6 +64,12 @@ pub const WITHDRAW_AUTHORITY_ARG: ArgConstant<'static> = ArgConstant {
     help: "Authorized withdrawer [default: cli config keypair]",
 };
 
+pub const CUSTODIAN_ARG: ArgConstant<'static> = ArgConstant {
+    name: "custodian",
+    long: "custodian",
+    help: "Authority to override account lockup",
+};
+
 fn stake_authority_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name(STAKE_AUTHORITY_ARG.name)
         .long(STAKE_AUTHORITY_ARG.long)
@@ -82,6 +88,15 @@ fn withdraw_authority_arg<'a, 'b>() -> Arg<'a, 'b> {
         .help(WITHDRAW_AUTHORITY_ARG.help)
 }
 
+fn custodian_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(CUSTODIAN_ARG.name)
+        .long(CUSTODIAN_ARG.long)
+        .takes_value(true)
+        .value_name("KEYPAIR")
+        .validator(is_valid_signer)
+        .help(CUSTODIAN_ARG.help)
+}
+
 pub trait StakeSubCommands {
     fn stake_subcommands(self) -> Self;
 }
@@ -223,6 +238,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(custodian_arg())
         )
         .subcommand(
             SubCommand::with_name("deactivate-stake")
@@ -331,14 +347,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
-                .arg(
-                    Arg::with_name("custodian")
-                        .long("custodian")
-                        .takes_value(true)
-                        .value_name("KEYPAIR")
-                        .validator(is_valid_signer)
-                        .help("Authority to override account lockup")
-                )
+                .arg(custodian_arg())
         )
         .subcommand(
             SubCommand::with_name("stake-set-lockup")
@@ -403,7 +412,7 @@ impl StakeSubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL")
-                )
+                ),
         )
         .subcommand(
             SubCommand::with_name("stake-history")
@@ -561,11 +570,15 @@ pub fn parse_stake_authorize(
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
     let (fee_payer, fee_payer_pubkey) = signer_of(matches, FEE_PAYER_ARG.name, wallet_manager)?;
+    let (custodian, custodian_pubkey) = signer_of(matches, "custodian", wallet_manager)?;
 
     bulk_signers.push(fee_payer);
     if nonce_account.is_some() {
         bulk_signers.push(nonce_authority);
     }
+    if custodian.is_some() {
+        bulk_signers.push(custodian);
+    }
     let signer_info =
         default_signer.generate_unique_signers(bulk_signers, matches, wallet_manager)?;
 
@@ -591,6 +604,7 @@ pub fn parse_stake_authorize(
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
             fee_payer: signer_info.index_of(fee_payer_pubkey).unwrap(),
+            custodian: custodian_pubkey.and_then(|_| signer_info.index_of(custodian_pubkey)),
         },
         signers: signer_info.signers,
     })
@@ -959,11 +973,7 @@ pub fn process_create_stake_account(
         return_signers(&tx, &config.output_format)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &tx,
-            config.commitment,
-            config.send_transaction_config,
-        );
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
         log_instruction_custom_error::<SystemError>(result, &config)
     }
 }
@@ -974,6 +984,7 @@ pub fn process_stake_authorize(
     config: &CliConfig,
     stake_account_pubkey: &Pubkey,
     new_authorizations: &[(StakeAuthorize, Pubkey, SignerIndex)],
+    custodian: Option<SignerIndex>,
     sign_only: bool,
     blockhash_query: &BlockhashQuery,
     nonce_account: Option<Pubkey>,
@@ -981,6 +992,7 @@ pub fn process_stake_authorize(
     fee_payer: SignerIndex,
 ) -> ProcessResult {
     let mut ixs = Vec::new();
+    let custodian = custodian.map(|index| config.signers[index]);
     for (stake_authorize, authorized_pubkey, authority) in new_authorizations.iter() {
         check_unique_pubkeys(
             (stake_account_pubkey, "stake_account_pubkey".to_string()),
@@ -992,6 +1004,7 @@ pub fn process_stake_authorize(
             &authority.pubkey(),  // currently authorized
             authorized_pubkey,    // new stake signer
             *stake_authorize,     // stake or withdraw
+            custodian.map(|signer| signer.pubkey()).as_ref(),
         ));
     }
 
@@ -1033,11 +1046,7 @@ pub fn process_stake_authorize(
             &tx.message,
             config.commitment,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &tx,
-            config.commitment,
-            config.send_transaction_config,
-        );
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
         log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
@@ -1096,11 +1105,7 @@ pub fn process_deactivate_stake_account(
             &tx.message,
             config.commitment,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &tx,
-            config.commitment,
-            config.send_transaction_config,
-        );
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
         log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
@@ -1168,11 +1173,7 @@ pub fn process_withdraw_stake(
             &tx.message,
             config.commitment,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &tx,
-            config.commitment,
-            config.send_transaction_config,
-        );
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
         log_instruction_custom_error::<SystemError>(result, &config)
     }
 }
@@ -1311,11 +1312,7 @@ pub fn process_split_stake(
             &tx.message,
             config.commitment,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &tx,
-            config.commitment,
-            config.send_transaction_config,
-        );
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
         log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
@@ -1479,11 +1476,7 @@ pub fn process_stake_set_lockup(
             &tx.message,
             config.commitment,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &tx,
-            config.commitment,
-            config.send_transaction_config,
-        );
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
         log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
@@ -1662,13 +1655,13 @@ pub(crate) fn fetch_epoch_rewards(
                 .find(|reward| reward.pubkey == address.to_string())
             {
                 if reward.post_balance > reward.lamports.try_into().unwrap_or(0) {
-                    let percent_change = reward.lamports.abs() as f64
+                    let rate_change = reward.lamports.abs() as f64
                         / (reward.post_balance as f64 - reward.lamports as f64);
 
                     let apr = wallclock_epoch_duration.map(|wallclock_epoch_duration| {
                         let wallclock_epochs_per_year =
                             (SECONDS_PER_DAY * 356) as f64 / wallclock_epoch_duration;
-                        percent_change * wallclock_epochs_per_year
+                        rate_change * wallclock_epochs_per_year
                     });
 
                     all_epoch_rewards.push(CliEpochReward {
@@ -1676,8 +1669,8 @@ pub(crate) fn fetch_epoch_rewards(
                         effective_slot,
                         amount: reward.lamports.abs() as u64,
                         post_balance: reward.post_balance,
-                        percent_change,
-                        apr,
+                        percent_change: rate_change * 100.0,
+                        apr: apr.map(|r| r * 100.0),
                     });
                 }
             }
@@ -1725,7 +1718,7 @@ pub fn process_show_stake_account(
                 use_lamports_unit,
                 &stake_history,
                 &clock,
-                is_stake_program_v2_enabled(rpc_client), // At v1.6, this check can be removed and simply passed as `true`
+                is_stake_program_v2_enabled(rpc_client)?, // At v1.6, this check can be removed and simply passed as `true`
             );
 
             if state.stake_type == CliStakeType::Stake {
@@ -1792,23 +1785,15 @@ pub fn process_delegate_stake(
     if !sign_only {
         // Sanity check the vote account to ensure it is attached to a validator that has recently
         // voted at the tip of the ledger
-        let vote_account = rpc_client
-            .get_account_with_commitment(vote_account_pubkey, config.commitment)
-            .map_err(|_| {
+        let vote_account_data = rpc_client
+            .get_account(vote_account_pubkey)
+            .map_err(|err| {
                 CliError::RpcRequestError(format!(
-                    "Vote account not found: {}",
-                    vote_account_pubkey
+                    "Vote account not found: {}. error: {}",
+                    vote_account_pubkey, err,
                 ))
-            })?;
-        let vote_account_data = if let Some(account) = vote_account.value {
-            account.data
-        } else {
-            return Err(CliError::RpcRequestError(format!(
-                "Vote account not found: {}",
-                vote_account_pubkey
-            ))
-            .into());
-        };
+            })?
+            .data;
 
         let vote_state = VoteState::deserialize(&vote_account_data).map_err(|_| {
             CliError::RpcRequestError(
@@ -1888,22 +1873,18 @@ pub fn process_delegate_stake(
             &tx.message,
             config.commitment,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &tx,
-            config.commitment,
-            config.send_transaction_config,
-        );
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
         log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
 
-pub fn is_stake_program_v2_enabled(rpc_client: &RpcClient) -> bool {
-    rpc_client
-        .get_account(&feature_set::stake_program_v2::id())
-        .ok()
-        .and_then(|account| feature::from_account(&account))
+pub fn is_stake_program_v2_enabled(
+    rpc_client: &RpcClient,
+) -> Result<bool, Box<dyn std::error::Error>> {
+    let feature_account = rpc_client.get_account(&feature_set::stake_program_v2::id())?;
+    Ok(feature::from_account(&feature_account)
         .and_then(|feature| feature.activated_at)
-        .is_some()
+        .is_some())
 }
 
 #[cfg(test)]
@@ -1975,6 +1956,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
             },
@@ -2009,6 +1991,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2047,6 +2030,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2074,6 +2058,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
             },
@@ -2098,6 +2083,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2128,6 +2114,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2159,6 +2146,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
             },
@@ -2187,6 +2175,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2221,6 +2210,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2257,6 +2247,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2303,6 +2294,7 @@ mod tests {
                     nonce_account: Some(nonce_account),
                     nonce_authority: 2,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2335,6 +2327,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2372,6 +2365,7 @@ mod tests {
                     nonce_account: Some(nonce_account_pubkey),
                     nonce_authority: 1,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2405,6 +2399,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2442,6 +2437,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),

cli/src/test_utils.rs

@@ -5,7 +5,7 @@ use std::{thread::sleep, time::Duration};
 pub fn check_recent_balance(expected_balance: u64, client: &RpcClient, pubkey: &Pubkey) {
     (0..5).for_each(|tries| {
         let balance = client
-            .get_balance_with_commitment(pubkey, CommitmentConfig::recent())
+            .get_balance_with_commitment(pubkey, CommitmentConfig::processed())
             .unwrap()
             .value;
         if balance == expected_balance {
@@ -20,7 +20,7 @@ pub fn check_recent_balance(expected_balance: u64, client: &RpcClient, pubkey: &
 
 pub fn check_ready(rpc_client: &RpcClient) {
     while rpc_client
-        .get_slot_with_commitment(CommitmentConfig::recent())
+        .get_slot_with_commitment(CommitmentConfig::processed())
         .unwrap()
         < 5
     {

cli/src/validator_info.rs

@@ -20,7 +20,6 @@ use solana_config_program::{config_instruction, get_config_data, ConfigKeys, Con
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     account::Account,
-    commitment_config::CommitmentConfig,
     message::Message,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
@@ -288,9 +287,7 @@ pub fn process_set_validator_info(
     };
 
     // Check existence of validator-info account
-    let balance = rpc_client
-        .poll_get_balance_with_commitment(&info_pubkey, CommitmentConfig::default())
-        .unwrap_or(0);
+    let balance = rpc_client.get_balance(&info_pubkey).unwrap_or(0);
 
     let lamports =
         rpc_client.get_minimum_balance_for_rent_exemption(ValidatorInfo::max_space() as usize)?;

cli/src/vote.rs

@@ -8,7 +8,6 @@ use crate::{
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use solana_clap_utils::{
-    commitment::commitment_arg,
     input_parsers::*,
     input_validators::*,
     keypair::{DefaultSigner, SignerIndex},
@@ -208,8 +207,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                )
-                .arg(commitment_arg()),
+                ),
         )
         .subcommand(
             SubCommand::with_name("withdraw-from-vote-account")
@@ -494,9 +492,7 @@ pub fn process_create_vote_account(
         }
     }
 
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let (message, _) = resolve_spend_tx_and_check_account_balance(
         rpc_client,
@@ -509,11 +505,7 @@ pub fn process_create_vote_account(
     )?;
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<SystemError>(result, &config)
 }
 
@@ -536,9 +528,7 @@ pub fn process_vote_authorize(
         (&authorized.pubkey(), "authorized_account".to_string()),
         (new_authorized_pubkey, "new_authorized_pubkey".to_string()),
     )?;
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let ixs = vec![vote_instruction::authorize(
         vote_account_pubkey,   // vote account to update
         &authorized.pubkey(),  // current authorized
@@ -556,11 +546,7 @@ pub fn process_vote_authorize(
         &tx.message,
         config.commitment,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<VoteError>(result, &config)
 }
 
@@ -578,9 +564,7 @@ pub fn process_vote_update_validator(
         (vote_account_pubkey, "vote_account_pubkey".to_string()),
         (&new_identity_pubkey, "new_identity_account".to_string()),
     )?;
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let ixs = vec![vote_instruction::update_validator_identity(
         vote_account_pubkey,
         &authorized_withdrawer.pubkey(),
@@ -597,11 +581,7 @@ pub fn process_vote_update_validator(
         &tx.message,
         config.commitment,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<VoteError>(result, &config)
 }
 
@@ -613,9 +593,7 @@ pub fn process_vote_update_commission(
     withdraw_authority: SignerIndex,
 ) -> ProcessResult {
     let authorized_withdrawer = config.signers[withdraw_authority];
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let ixs = vec![vote_instruction::update_commission(
         vote_account_pubkey,
         &authorized_withdrawer.pubkey(),
@@ -632,11 +610,7 @@ pub fn process_vote_update_commission(
         &tx.message,
         config.commitment,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &tx,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
     log_instruction_custom_error::<VoteError>(result, &config)
 }
 
@@ -733,14 +707,10 @@ pub fn process_withdraw_from_vote_account(
     withdraw_amount: SpendAmount,
     destination_account_pubkey: &Pubkey,
 ) -> ProcessResult {
-    let (recent_blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let withdraw_authority = config.signers[withdraw_authority];
 
-    let current_balance = rpc_client
-        .get_balance_with_commitment(&vote_account_pubkey, config.commitment)?
-        .value;
+    let current_balance = rpc_client.get_balance(&vote_account_pubkey)?;
     let minimum_balance = rpc_client.get_minimum_balance_for_rent_exemption(VoteState::size_of())?;
 
     let lamports = match withdraw_amount {
@@ -773,11 +743,7 @@ pub fn process_withdraw_from_vote_account(
         &transaction.message,
         config.commitment,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-        &transaction,
-        config.commitment,
-        config.send_transaction_config,
-    );
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&transaction);
     log_instruction_custom_error::<VoteError>(result, &config)
 }
 

cli/tests/deploy.rs

@@ -1,422 +0,0 @@
-use serde_json::Value;
-use solana_cli::cli::{process_command, CliCommand, CliConfig};
-use solana_client::rpc_client::RpcClient;
-use solana_core::test_validator::TestValidator;
-use solana_faucet::faucet::run_local_faucet;
-use solana_sdk::{
-    bpf_loader,
-    bpf_loader_upgradeable::{self, UpgradeableLoaderState},
-    commitment_config::CommitmentConfig,
-    pubkey::Pubkey,
-    signature::{Keypair, Signer},
-};
-use std::{fs::File, io::Read, path::PathBuf, str::FromStr, sync::mpsc::channel};
-
-#[test]
-fn test_cli_deploy_program() {
-    solana_logger::setup();
-
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
-
-    let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
-
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
-
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
-    let mut program_data = Vec::new();
-    file.read_to_end(&mut program_data).unwrap();
-    let minimum_balance_for_rent_exemption = rpc_client
-        .get_minimum_balance_for_rent_exemption(program_data.len())
-        .unwrap();
-
-    let mut config = CliConfig::recent_for_tests();
-    let keypair = Keypair::new();
-    config.json_rpc_url = test_validator.rpc_url();
-    config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
-        pubkey: None,
-        lamports: 4 * minimum_balance_for_rent_exemption, // min balance for rent exemption for three programs + leftover for tx processing
-    };
-    config.signers = vec![&keypair];
-    process_command(&config).unwrap();
-
-    config.command = CliCommand::ProgramDeploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: None,
-        use_deprecated_loader: false,
-        use_upgradeable_loader: false,
-        allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: None,
-    };
-
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let program_id_str = json
-        .as_object()
-        .unwrap()
-        .get("programId")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    let program_id = Pubkey::from_str(&program_id_str).unwrap();
-    let account0 = rpc_client
-        .get_account_with_commitment(&program_id, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(account0.lamports, minimum_balance_for_rent_exemption);
-    assert_eq!(account0.owner, bpf_loader::id());
-    assert_eq!(account0.executable, true);
-
-    let mut file = File::open(pathbuf.to_str().unwrap().to_string()).unwrap();
-    let mut elf = Vec::new();
-    file.read_to_end(&mut elf).unwrap();
-
-    assert_eq!(account0.data, elf);
-
-    // Test custom address
-    let custom_address_keypair = Keypair::new();
-    config.signers = vec![&keypair, &custom_address_keypair];
-    config.command = CliCommand::ProgramDeploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: Some(1),
-        use_deprecated_loader: false,
-        use_upgradeable_loader: false,
-        allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: None,
-    };
-    process_command(&config).unwrap();
-    let account1 = rpc_client
-        .get_account_with_commitment(&custom_address_keypair.pubkey(), CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(account1.lamports, minimum_balance_for_rent_exemption);
-    assert_eq!(account1.owner, bpf_loader::id());
-    assert_eq!(account1.executable, true);
-    assert_eq!(account0.data, account1.data);
-
-    // Attempt to redeploy to the same address
-    process_command(&config).unwrap_err();
-
-    // Attempt to deploy to account with excess balance
-    let custom_address_keypair = Keypair::new();
-    config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
-        pubkey: None,
-        lamports: 2 * minimum_balance_for_rent_exemption, // Anything over minimum_balance_for_rent_exemption should trigger err
-    };
-    config.signers = vec![&custom_address_keypair];
-    process_command(&config).unwrap();
-
-    config.signers = vec![&keypair, &custom_address_keypair];
-    config.command = CliCommand::ProgramDeploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: Some(1),
-        use_deprecated_loader: false,
-        use_upgradeable_loader: false,
-        allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: None,
-    };
-    process_command(&config).unwrap_err();
-
-    // Use forcing parameter to deploy to account with excess balance
-    config.command = CliCommand::ProgramDeploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: Some(1),
-        use_deprecated_loader: false,
-        use_upgradeable_loader: false,
-        allow_excessive_balance: true,
-        upgrade_authority: None,
-        max_len: None,
-    };
-    process_command(&config).unwrap();
-    let account2 = rpc_client
-        .get_account_with_commitment(&custom_address_keypair.pubkey(), CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(account2.lamports, 2 * minimum_balance_for_rent_exemption);
-    assert_eq!(account2.owner, bpf_loader::id());
-    assert_eq!(account2.executable, true);
-    assert_eq!(account0.data, account2.data);
-}
-
-#[test]
-fn test_cli_deploy_upgradeable_program() {
-    solana_logger::setup();
-
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
-
-    let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
-
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
-
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
-    let mut program_data = Vec::new();
-    file.read_to_end(&mut program_data).unwrap();
-    let max_len = program_data.len();
-    println!(
-        "max_len {:?} {:?}",
-        max_len,
-        UpgradeableLoaderState::programdata_len(max_len)
-    );
-    let minimum_balance_for_programdata = rpc_client
-        .get_minimum_balance_for_rent_exemption(
-            UpgradeableLoaderState::programdata_len(max_len).unwrap(),
-        )
-        .unwrap();
-    let minimum_balance_for_program = rpc_client
-        .get_minimum_balance_for_rent_exemption(UpgradeableLoaderState::program_len().unwrap())
-        .unwrap();
-    let upgrade_authority = Keypair::new();
-
-    let mut config = CliConfig::recent_for_tests();
-    let keypair = Keypair::new();
-    config.json_rpc_url = test_validator.rpc_url();
-    config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
-        pubkey: None,
-        lamports: 100 * minimum_balance_for_programdata + minimum_balance_for_program,
-    };
-    config.signers = vec![&keypair];
-    process_command(&config).unwrap();
-
-    // Deploy and attempt to upgrade a non-upgradeable program
-    config.command = CliCommand::ProgramDeploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: None,
-        use_deprecated_loader: false,
-        use_upgradeable_loader: true,
-        allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: Some(max_len),
-    };
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let program_id_str = json
-        .as_object()
-        .unwrap()
-        .get("programId")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    let program_id = Pubkey::from_str(&program_id_str).unwrap();
-
-    config.signers = vec![&keypair, &upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
-    process_command(&config).unwrap_err();
-
-    // Deploy the upgradeable program
-    config.command = CliCommand::ProgramDeploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: None,
-        use_deprecated_loader: false,
-        use_upgradeable_loader: true,
-        allow_excessive_balance: false,
-        upgrade_authority: Some(upgrade_authority.pubkey()),
-        max_len: Some(max_len),
-    };
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let program_id_str = json
-        .as_object()
-        .unwrap()
-        .get("programId")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    let program_id = Pubkey::from_str(&program_id_str).unwrap();
-    let program_account = rpc_client
-        .get_account_with_commitment(&program_id, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(program_account.lamports, minimum_balance_for_program);
-    assert_eq!(program_account.owner, bpf_loader_upgradeable::id());
-    assert_eq!(program_account.executable, true);
-    let (programdata_pubkey, _) =
-        Pubkey::find_program_address(&[program_id.as_ref()], &bpf_loader_upgradeable::id());
-    let programdata_account = rpc_client
-        .get_account_with_commitment(&programdata_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(
-        programdata_account.lamports,
-        minimum_balance_for_programdata
-    );
-    assert_eq!(programdata_account.owner, bpf_loader_upgradeable::id());
-    assert_eq!(programdata_account.executable, false);
-    assert_eq!(
-        programdata_account.data[UpgradeableLoaderState::programdata_data_offset().unwrap()..],
-        program_data[..]
-    );
-
-    // Upgrade the program
-    config.signers = vec![&keypair, &upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let program_id_str = json
-        .as_object()
-        .unwrap()
-        .get("programId")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    let program_id = Pubkey::from_str(&program_id_str).unwrap();
-    let program_account = rpc_client
-        .get_account_with_commitment(&program_id, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(program_account.lamports, minimum_balance_for_program);
-    assert_eq!(program_account.owner, bpf_loader_upgradeable::id());
-    assert_eq!(program_account.executable, true);
-    let (programdata_pubkey, _) =
-        Pubkey::find_program_address(&[program_id.as_ref()], &bpf_loader_upgradeable::id());
-    let programdata_account = rpc_client
-        .get_account_with_commitment(&programdata_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(
-        programdata_account.lamports,
-        minimum_balance_for_programdata
-    );
-    assert_eq!(programdata_account.owner, bpf_loader_upgradeable::id());
-    assert_eq!(programdata_account.executable, false);
-    assert_eq!(
-        programdata_account.data[UpgradeableLoaderState::programdata_data_offset().unwrap()..],
-        program_data[..]
-    );
-
-    // Set a new authority
-    let new_upgrade_authority = Keypair::new();
-    config.signers = vec![&keypair, &upgrade_authority];
-    config.command = CliCommand::SetProgramUpgradeAuthority {
-        program: program_id,
-        upgrade_authority: 1,
-        new_upgrade_authority: Some(new_upgrade_authority.pubkey()),
-    };
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let new_upgrade_authority_str = json
-        .as_object()
-        .unwrap()
-        .get("UpgradeAuthority")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    assert_eq!(
-        Pubkey::from_str(&new_upgrade_authority_str).unwrap(),
-        new_upgrade_authority.pubkey()
-    );
-
-    // Upgrade with new authority
-    config.signers = vec![&keypair, &new_upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let program_id_str = json
-        .as_object()
-        .unwrap()
-        .get("programId")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    let program_id = Pubkey::from_str(&program_id_str).unwrap();
-    let program_account = rpc_client
-        .get_account_with_commitment(&program_id, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(program_account.lamports, minimum_balance_for_program);
-    assert_eq!(program_account.owner, bpf_loader_upgradeable::id());
-    assert_eq!(program_account.executable, true);
-    let (programdata_pubkey, _) =
-        Pubkey::find_program_address(&[program_id.as_ref()], &bpf_loader_upgradeable::id());
-    let programdata_account = rpc_client
-        .get_account_with_commitment(&programdata_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(
-        programdata_account.lamports,
-        minimum_balance_for_programdata
-    );
-    assert_eq!(programdata_account.owner, bpf_loader_upgradeable::id());
-    assert_eq!(programdata_account.executable, false);
-    assert_eq!(
-        programdata_account.data[UpgradeableLoaderState::programdata_data_offset().unwrap()..],
-        program_data[..]
-    );
-
-    // Set a no authority
-    config.signers = vec![&keypair, &new_upgrade_authority];
-    config.command = CliCommand::SetProgramUpgradeAuthority {
-        program: program_id,
-        upgrade_authority: 1,
-        new_upgrade_authority: None,
-    };
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let new_upgrade_authority_str = json
-        .as_object()
-        .unwrap()
-        .get("UpgradeAuthority")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    assert_eq!(new_upgrade_authority_str, "None");
-
-    // Upgrade with no authority
-    config.signers = vec![&keypair, &new_upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
-    process_command(&config).unwrap_err();
-}

cli/tests/nonce.rs

@@ -18,7 +18,6 @@ use solana_sdk::{
     signature::{keypair_from_seed, Keypair, Signer},
     system_program,
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_nonce() {
@@ -59,11 +58,10 @@ fn full_battery_tests(
     seed: Option<String>,
     use_nonce_authority: bool,
 ) {
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let json_rpc_url = test_validator.rpc_url();
 
     let mut config_payer = CliConfig::recent_for_tests();
@@ -218,9 +216,7 @@ fn test_create_account_with_seed() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
 
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let offline_nonce_authority_signer = keypair_from_seed(&[1u8; 32]).unwrap();
     let online_nonce_creator_signer = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -228,7 +224,8 @@ fn test_create_account_with_seed() {
     let config = CliConfig::recent_for_tests();
 
     // Setup accounts
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     request_and_confirm_airdrop(
         &rpc_client,
         &faucet_addr,
@@ -278,7 +275,7 @@ fn test_create_account_with_seed() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_address,
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()

cli/tests/request_airdrop.rs

@@ -6,16 +6,13 @@ use solana_sdk::{
     commitment_config::CommitmentConfig,
     signature::{Keypair, Signer},
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_cli_request_airdrop() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
 
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let mut bob_config = CliConfig::recent_for_tests();
     bob_config.json_rpc_url = test_validator.rpc_url();
@@ -31,11 +28,11 @@ fn test_cli_request_airdrop() {
     let sig_response = process_command(&bob_config);
     sig_response.unwrap();
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
     let balance = rpc_client
-        .get_balance_with_commitment(&bob_config.signers[0].pubkey(), CommitmentConfig::recent())
-        .unwrap()
-        .value;
+        .get_balance(&bob_config.signers[0].pubkey())
+        .unwrap();
     assert_eq!(balance, 50);
 }

cli/tests/stake.rs

@@ -22,17 +22,15 @@ use solana_stake_program::{
     stake_instruction::LockupArgs,
     stake_state::{Lockup, StakeAuthorize, StakeState},
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_stake_delegation_force() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let default_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
@@ -116,11 +114,10 @@ fn test_seed_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
     let validator_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let mut config_validator = CliConfig::recent_for_tests();
@@ -195,11 +192,10 @@ fn test_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let validator_keypair = Keypair::new();
 
     let mut config_validator = CliConfig::recent_for_tests();
@@ -270,11 +266,10 @@ fn test_offline_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
     let mut config_validator = CliConfig::recent_for_tests();
     config_validator.json_rpc_url = test_validator.rpc_url();
@@ -291,7 +286,7 @@ fn test_offline_stake_delegation_and_deactivation() {
     config_offline.command = CliCommand::ClusterVersion;
     let offline_keypair = Keypair::new();
     config_offline.signers = vec![&offline_keypair];
-    // Verfiy that we cannot reach the cluster
+    // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
     request_and_confirm_airdrop(
@@ -402,11 +397,10 @@ fn test_nonced_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
     let config_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let mut config = CliConfig::recent_for_tests();
@@ -460,7 +454,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -488,7 +482,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -516,11 +510,10 @@ fn test_stake_authorize() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let default_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
@@ -542,7 +535,7 @@ fn test_stake_authorize() {
     config_offline.json_rpc_url = String::default();
     let offline_authority_pubkey = config_offline.signers[0].pubkey();
     config_offline.command = CliCommand::ClusterVersion;
-    // Verfiy that we cannot reach the cluster
+    // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
     request_and_confirm_airdrop(
@@ -586,13 +579,10 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let current_authority = match stake_state {
         StakeState::Initialized(meta) => meta.authorized.staker,
@@ -617,13 +607,10 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let (current_staker, current_withdrawer) = match stake_state {
         StakeState::Initialized(meta) => (meta.authorized.staker, meta.authorized.withdrawer),
@@ -643,13 +630,10 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let current_authority = match stake_state {
         StakeState::Initialized(meta) => meta.authorized.staker,
@@ -660,10 +644,7 @@ fn test_stake_authorize() {
     // Offline assignment of new nonced stake authority
     let nonced_authority = Keypair::new();
     let nonced_authority_pubkey = nonced_authority.pubkey();
-    let (blockhash, _, _) = rpc_client
-        .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
-        .unwrap()
-        .value;
+    let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, nonced_authority_pubkey, 0)],
@@ -672,6 +653,7 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     config_offline.output_format = OutputFormat::JsonCompact;
     let sign_reply = process_command(&config_offline).unwrap();
@@ -687,13 +669,10 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let current_authority = match stake_state {
         StakeState::Initialized(meta) => meta.authorized.staker,
@@ -719,7 +698,7 @@ fn test_stake_authorize() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -737,6 +716,7 @@ fn test_stake_authorize() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     let sign_reply = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_reply);
@@ -756,13 +736,10 @@ fn test_stake_authorize() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let current_authority = match stake_state {
         StakeState::Initialized(meta) => meta.authorized.staker,
@@ -773,7 +750,7 @@ fn test_stake_authorize() {
     let new_nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -788,11 +765,10 @@ fn test_stake_authorize_with_fee_payer() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), SIG_FEE);
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let default_signer = Keypair::new();
     let default_pubkey = default_signer.pubkey();
 
@@ -861,6 +837,7 @@ fn test_stake_authorize_with_fee_payer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 1,
+        custodian: None,
     };
     process_command(&config).unwrap();
     // `config` balance has not changed, despite submitting the TX
@@ -870,10 +847,7 @@ fn test_stake_authorize_with_fee_payer() {
     check_recent_balance(100_000 - SIG_FEE - SIG_FEE, &rpc_client, &payer_pubkey);
 
     // Assign authority with offline fee payer
-    let (blockhash, _, _) = rpc_client
-        .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
-        .unwrap()
-        .value;
+    let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, payer_pubkey, 0)],
@@ -882,6 +856,7 @@ fn test_stake_authorize_with_fee_payer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     config_offline.output_format = OutputFormat::JsonCompact;
     let sign_reply = process_command(&config_offline).unwrap();
@@ -897,6 +872,7 @@ fn test_stake_authorize_with_fee_payer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
     // `config`'s balance again has not changed
@@ -912,11 +888,10 @@ fn test_stake_split() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let default_signer = Keypair::new();
     let offline_signer = Keypair::new();
 
@@ -993,7 +968,7 @@ fn test_stake_split() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -1055,11 +1030,10 @@ fn test_stake_set_lockup() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let default_signer = Keypair::new();
     let offline_signer = Keypair::new();
 
@@ -1142,11 +1116,7 @@ fn test_stake_set_lockup() {
         fee_payer: 0,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let current_lockup = match stake_state {
         StakeState::Initialized(meta) => meta.lockup,
@@ -1197,11 +1167,7 @@ fn test_stake_set_lockup() {
         fee_payer: 0,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let current_lockup = match stake_state {
         StakeState::Initialized(meta) => meta.lockup,
@@ -1252,7 +1218,7 @@ fn test_stake_set_lockup() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -1294,11 +1260,7 @@ fn test_stake_set_lockup() {
         fee_payer: 0,
     };
     process_command(&config).unwrap();
-    let stake_account = rpc_client
-        .get_account_with_commitment(&stake_account_pubkey, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
     let stake_state: StakeState = stake_account.state().unwrap();
     let current_lockup = match stake_state {
         StakeState::Initialized(meta) => meta.lockup,
@@ -1318,11 +1280,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let mut config = CliConfig::recent_for_tests();
     let default_signer = keypair_from_seed(&[1u8; 32]).unwrap();
     config.signers = vec![&default_signer];
@@ -1334,7 +1295,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     let offline_pubkey = config_offline.signers[0].pubkey();
     config_offline.json_rpc_url = String::default();
     config_offline.command = CliCommand::ClusterVersion;
-    // Verfiy that we cannot reach the cluster
+    // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
     request_and_confirm_airdrop(
@@ -1370,7 +1331,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -1425,7 +1386,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -1473,7 +1434,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()

cli/tests/transfer.rs

@@ -17,19 +17,16 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, NullSigner, Signer},
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_transfer() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
-
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
     let default_signer = Keypair::new();
     let default_offline_signer = Keypair::new();
@@ -92,10 +89,7 @@ fn test_transfer() {
     check_recent_balance(50, &rpc_client, &offline_pubkey);
 
     // Offline transfer
-    let (blockhash, _, _) = rpc_client
-        .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
-        .unwrap()
-        .value;
+    let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     offline.command = CliCommand::Transfer {
         amount: SpendAmount::Some(10),
         to: recipient_pubkey,
@@ -147,7 +141,7 @@ fn test_transfer() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -175,7 +169,7 @@ fn test_transfer() {
     let new_nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -196,7 +190,7 @@ fn test_transfer() {
     let nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
-        CommitmentConfig::recent(),
+        CommitmentConfig::processed(),
     )
     .and_then(|ref a| nonce_utils::data_from_account(a))
     .unwrap()
@@ -244,10 +238,7 @@ fn test_transfer_multisession_signing() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let to_pubkey = Pubkey::new(&[1u8; 32]);
     let offline_from_signer = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -256,7 +247,8 @@ fn test_transfer_multisession_signing() {
     let config = CliConfig::recent_for_tests();
 
     // Setup accounts
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     request_and_confirm_airdrop(
         &rpc_client,
         &faucet_addr,
@@ -279,10 +271,7 @@ fn test_transfer_multisession_signing() {
 
     check_ready(&rpc_client);
 
-    let (blockhash, _, _) = rpc_client
-        .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
-        .unwrap()
-        .value;
+    let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
 
     // Offline fee-payer signs first
     let mut fee_payer_config = CliConfig::recent_for_tests();
@@ -363,12 +352,10 @@ fn test_transfer_all() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
-
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
     let default_signer = Keypair::new();
 

cli/tests/vote.rs

@@ -15,17 +15,15 @@ use solana_sdk::{
     signature::{Keypair, Signer},
 };
 use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersions};
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_vote_authorize_and_withdraw() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
-    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let default_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
@@ -55,9 +53,7 @@ fn test_vote_authorize_and_withdraw() {
     };
     process_command(&config).unwrap();
     let vote_account = rpc_client
-        .get_account_with_commitment(&vote_account_keypair.pubkey(), CommitmentConfig::recent())
-        .unwrap()
-        .value
+        .get_account(&vote_account_keypair.pubkey())
         .unwrap();
     let vote_state: VoteStateVersions = vote_account.state().unwrap();
     let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;
@@ -95,9 +91,7 @@ fn test_vote_authorize_and_withdraw() {
     };
     process_command(&config).unwrap();
     let vote_account = rpc_client
-        .get_account_with_commitment(&vote_account_keypair.pubkey(), CommitmentConfig::recent())
-        .unwrap()
-        .value
+        .get_account(&vote_account_keypair.pubkey())
         .unwrap();
     let vote_state: VoteStateVersions = vote_account.state().unwrap();
     let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.5.0"
+version = "1.5.8"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -23,13 +23,13 @@ semver = "0.11.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-net-utils = { path = "../net-utils", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.8" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -38,7 +38,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "15.0.0"
 jsonrpc-http-server = "15.0.0"
-solana-logger = { path = "../logger", version = "1.5.0" }
+solana-logger = { path = "../logger", version = "1.5.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/client_error.rs

@@ -5,6 +5,8 @@ use solana_sdk::{
 use std::io;
 use thiserror::Error;
 
+pub use reqwest; // export `reqwest` for clients
+
 #[derive(Error, Debug)]
 pub enum ClientErrorKind {
     #[error(transparent)]
@@ -33,16 +35,16 @@ impl From<TransportError> for ClientErrorKind {
     }
 }
 
-impl Into<TransportError> for ClientErrorKind {
-    fn into(self) -> TransportError {
-        match self {
-            Self::Io(err) => TransportError::IoError(err),
-            Self::TransactionError(err) => TransportError::TransactionError(err),
-            Self::Reqwest(err) => TransportError::Custom(format!("{:?}", err)),
-            Self::RpcError(err) => TransportError::Custom(format!("{:?}", err)),
-            Self::SerdeJson(err) => TransportError::Custom(format!("{:?}", err)),
-            Self::SigningError(err) => TransportError::Custom(format!("{:?}", err)),
-            Self::Custom(err) => TransportError::Custom(format!("{:?}", err)),
+impl From<ClientErrorKind> for TransportError {
+    fn from(client_error_kind: ClientErrorKind) -> Self {
+        match client_error_kind {
+            ClientErrorKind::Io(err) => Self::IoError(err),
+            ClientErrorKind::TransactionError(err) => Self::TransactionError(err),
+            ClientErrorKind::Reqwest(err) => Self::Custom(format!("{:?}", err)),
+            ClientErrorKind::RpcError(err) => Self::Custom(format!("{:?}", err)),
+            ClientErrorKind::SerdeJson(err) => Self::Custom(format!("{:?}", err)),
+            ClientErrorKind::SigningError(err) => Self::Custom(format!("{:?}", err)),
+            ClientErrorKind::Custom(err) => Self::Custom(format!("{:?}", err)),
         }
     }
 }
@@ -98,9 +100,9 @@ impl From<TransportError> for ClientError {
     }
 }
 
-impl Into<TransportError> for ClientError {
-    fn into(self) -> TransportError {
-        self.kind.into()
+impl From<ClientError> for TransportError {
+    fn from(client_error: ClientError) -> Self {
+        client_error.kind.into()
     }
 }
 

client/src/http_sender.rs

@@ -85,6 +85,14 @@ impl RpcSender for HttpSender {
                                             }
                                         }
                                     },
+                                    rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY => {
+                                        match serde_json::from_value::<rpc_custom_error::NodeUnhealthyErrorData>(json["error"]["data"].clone()) {
+                                            Ok(rpc_custom_error::NodeUnhealthyErrorData {num_slots_behind}) => RpcResponseErrorData::NodeUnhealthy {num_slots_behind},
+                                            Err(_err) => {
+                                                RpcResponseErrorData::Empty
+                                            }
+                                        }
+                                    },
                                     _ => RpcResponseErrorData::Empty
                                 };
 

client/src/lib.rs

@@ -8,6 +8,7 @@ pub mod mock_sender;
 pub mod nonce_utils;
 pub mod perf_utils;
 pub mod pubsub_client;
+pub mod rpc_cache;
 pub mod rpc_client;
 pub mod rpc_config;
 pub mod rpc_custom_error;

client/src/mock_sender.rs

@@ -12,7 +12,7 @@ use solana_sdk::{
     signature::Signature,
     transaction::{self, Transaction, TransactionError},
 };
-use solana_transaction_status::TransactionStatus;
+use solana_transaction_status::{TransactionConfirmationStatus, TransactionStatus};
 use solana_version::Version;
 use std::{collections::HashMap, sync::RwLock};
 
@@ -48,6 +48,10 @@ impl RpcSender for MockSender {
             return Ok(Value::Null);
         }
         let val = match request {
+            RpcRequest::GetAccountInfo => serde_json::to_value(Response {
+                context: RpcResponseContext { slot: 1 },
+                value: Value::Null,
+            })?,
             RpcRequest::GetBalance => serde_json::to_value(Response {
                 context: RpcResponseContext { slot: 1 },
                 value: Value::Number(Number::from(50)),
@@ -65,6 +69,7 @@ impl RpcSender for MockSender {
                 slots_in_epoch: 32,
                 absolute_slot: 34,
                 block_height: 34,
+                transaction_count: Some(123),
             })?,
             RpcRequest::GetFeeCalculatorForBlockhash => {
                 let value = if self.url == "blockhash_expired" {
@@ -101,6 +106,7 @@ impl RpcSender for MockSender {
                         slot: 1,
                         confirmations: None,
                         err,
+                        confirmation_status: Some(TransactionConfirmationStatus::Finalized),
                     })
                 };
                 let statuses: Vec<Option<TransactionStatus>> = params.as_array().unwrap()[0]

client/src/perf_utils.rs

@@ -33,7 +33,7 @@ pub fn sample_txs<T>(
     let mut now = Instant::now();
     let start_time = now;
     let initial_txs = client
-        .get_transaction_count_with_commitment(CommitmentConfig::recent())
+        .get_transaction_count_with_commitment(CommitmentConfig::processed())
         .expect("transaction count");
     let mut last_txs = initial_txs;
 
@@ -42,7 +42,7 @@ pub fn sample_txs<T>(
         let elapsed = now.elapsed();
         now = Instant::now();
         let mut txs;
-        match client.get_transaction_count_with_commitment(CommitmentConfig::recent()) {
+        match client.get_transaction_count_with_commitment(CommitmentConfig::processed()) {
             Err(e) => {
                 // ThinClient with multiple options should pick a better one now.
                 info!("Couldn't get transaction count {:?}", e);

client/src/pubsub_client.rs

@@ -98,7 +98,7 @@ where
     }
 
     pub fn send_unsubscribe(&self) -> Result<(), PubsubClientError> {
-        let method = format!("{}Unubscribe", self.operation);
+        let method = format!("{}Unsubscribe", self.operation);
         self.socket
             .write()
             .unwrap()

client/src/rpc_cache.rs

@@ -0,0 +1,75 @@
+use crate::{rpc_config::RpcLargestAccountsFilter, rpc_response::RpcAccountBalance};
+use std::{
+    collections::HashMap,
+    time::{Duration, SystemTime},
+};
+
+#[derive(Debug, Clone)]
+pub struct LargestAccountsCache {
+    duration: u64,
+    cache: HashMap<Option<RpcLargestAccountsFilter>, LargestAccountsCacheValue>,
+}
+
+#[derive(Debug, Clone)]
+struct LargestAccountsCacheValue {
+    accounts: Vec<RpcAccountBalance>,
+    slot: u64,
+    cached_time: SystemTime,
+}
+
+impl LargestAccountsCache {
+    pub fn new(duration: u64) -> Self {
+        Self {
+            duration,
+            cache: HashMap::new(),
+        }
+    }
+
+    pub fn get_largest_accounts(
+        &self,
+        filter: &Option<RpcLargestAccountsFilter>,
+    ) -> Option<(u64, Vec<RpcAccountBalance>)> {
+        self.cache.get(&filter).and_then(|value| {
+            if let Ok(elapsed) = value.cached_time.elapsed() {
+                if elapsed < Duration::from_secs(self.duration) {
+                    return Some((value.slot, value.accounts.clone()));
+                }
+            }
+            None
+        })
+    }
+
+    pub fn set_largest_accounts(
+        &mut self,
+        filter: &Option<RpcLargestAccountsFilter>,
+        slot: u64,
+        accounts: &[RpcAccountBalance],
+    ) {
+        self.cache.insert(
+            filter.clone(),
+            LargestAccountsCacheValue {
+                accounts: accounts.to_owned(),
+                slot,
+                cached_time: SystemTime::now(),
+            },
+        );
+    }
+}
+
+#[cfg(test)]
+pub mod test {
+    use super::*;
+
+    #[test]
+    fn test_old_entries_expire() {
+        let mut cache = LargestAccountsCache::new(1);
+
+        let filter = Some(RpcLargestAccountsFilter::Circulating);
+
+        let accounts: Vec<RpcAccountBalance> = Vec::new();
+
+        cache.set_largest_accounts(&filter, 1000, &accounts);
+        std::thread::sleep(Duration::from_secs(1));
+        assert_eq!(cache.get_largest_accounts(&filter), None);
+    }
+}

client/src/rpc_client.rs

@@ -40,6 +40,7 @@ use solana_transaction_status::{
 };
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
+    cmp::min,
     net::SocketAddr,
     sync::RwLock,
     thread::sleep,
@@ -49,7 +50,7 @@ use std::{
 pub struct RpcClient {
     sender: Box<dyn RpcSender + Send + Sync + 'static>,
     commitment_config: CommitmentConfig,
-    default_cluster_transaction_encoding: RwLock<Option<UiTransactionEncoding>>,
+    node_version: RwLock<Option<semver::Version>>,
 }
 
 fn serialize_encode_transaction(
@@ -79,7 +80,7 @@ impl RpcClient {
     ) -> Self {
         Self {
             sender: Box::new(sender),
-            default_cluster_transaction_encoding: RwLock::new(None),
+            node_version: RwLock::new(None),
             commitment_config,
         }
     }
@@ -99,6 +100,17 @@ impl RpcClient {
         )
     }
 
+    pub fn new_with_timeout_and_commitment(
+        url: String,
+        timeout: Duration,
+        commitment_config: CommitmentConfig,
+    ) -> Self {
+        Self::new_sender(
+            HttpSender::new_with_timeout(url, timeout),
+            commitment_config,
+        )
+    }
+
     pub fn new_mock(url: String) -> Self {
         Self::new_sender(MockSender::new(url), CommitmentConfig::default())
     }
@@ -119,16 +131,54 @@ impl RpcClient {
         Self::new_with_timeout(url, timeout)
     }
 
-    pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
-        Ok(self
-            .confirm_transaction_with_commitment(signature, self.commitment_config)?
-            .value)
+    fn get_node_version(&self) -> Result<semver::Version, RpcError> {
+        let r_node_version = self.node_version.read().unwrap();
+        if let Some(version) = &*r_node_version {
+            Ok(version.clone())
+        } else {
+            drop(r_node_version);
+            let mut w_node_version = self.node_version.write().unwrap();
+            let node_version = self.get_version().map_err(|e| {
+                RpcError::RpcRequestError(format!("cluster version query failed: {}", e))
+            })?;
+            let node_version = semver::Version::parse(&node_version.solana_core).map_err(|e| {
+                RpcError::RpcRequestError(format!("failed to parse cluster version: {}", e))
+            })?;
+            *w_node_version = Some(node_version.clone());
+            Ok(node_version)
+        }
     }
 
     pub fn commitment(&self) -> CommitmentConfig {
         self.commitment_config
     }
 
+    fn use_deprecated_commitment(&self) -> Result<bool, RpcError> {
+        Ok(self.get_node_version()? < semver::Version::new(1, 5, 5))
+    }
+
+    fn maybe_map_commitment(
+        &self,
+        requested_commitment: CommitmentConfig,
+    ) -> Result<CommitmentConfig, RpcError> {
+        if matches!(
+            requested_commitment.commitment,
+            CommitmentLevel::Finalized | CommitmentLevel::Confirmed | CommitmentLevel::Processed
+        ) && self.use_deprecated_commitment()?
+        {
+            return Ok(CommitmentConfig::use_deprecated_commitment(
+                requested_commitment,
+            ));
+        }
+        Ok(requested_commitment)
+    }
+
+    pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
+        Ok(self
+            .confirm_transaction_with_commitment(signature, self.commitment_config)?
+            .value)
+    }
+
     pub fn confirm_transaction_with_commitment(
         &self,
         signature: &Signature,
@@ -150,34 +200,20 @@ impl RpcClient {
         self.send_transaction_with_config(
             transaction,
             RpcSendTransactionConfig {
-                preflight_commitment: Some(self.commitment_config.commitment),
+                preflight_commitment: Some(
+                    self.maybe_map_commitment(self.commitment_config)?
+                        .commitment,
+                ),
                 ..RpcSendTransactionConfig::default()
             },
         )
     }
 
     fn default_cluster_transaction_encoding(&self) -> Result<UiTransactionEncoding, RpcError> {
-        let default_cluster_transaction_encoding =
-            self.default_cluster_transaction_encoding.read().unwrap();
-        if let Some(encoding) = *default_cluster_transaction_encoding {
-            Ok(encoding)
+        if self.get_node_version()? < semver::Version::new(1, 3, 16) {
+            Ok(UiTransactionEncoding::Base58)
         } else {
-            drop(default_cluster_transaction_encoding);
-            let cluster_version = self.get_version().map_err(|e| {
-                RpcError::RpcRequestError(format!("cluster version query failed: {}", e))
-            })?;
-            let cluster_version =
-                semver::Version::parse(&cluster_version.solana_core).map_err(|e| {
-                    RpcError::RpcRequestError(format!("failed to parse cluster version: {}", e))
-                })?;
-            // Prefer base64 since 1.3.16
-            let encoding = if cluster_version < semver::Version::new(1, 3, 16) {
-                UiTransactionEncoding::Base58
-            } else {
-                UiTransactionEncoding::Base64
-            };
-            *self.default_cluster_transaction_encoding.write().unwrap() = Some(encoding);
-            Ok(encoding)
+            Ok(UiTransactionEncoding::Base64)
         }
     }
 
@@ -191,8 +227,13 @@ impl RpcClient {
         } else {
             self.default_cluster_transaction_encoding()?
         };
+        let preflight_commitment = CommitmentConfig {
+            commitment: config.preflight_commitment.unwrap_or_default(),
+        };
+        let preflight_commitment = self.maybe_map_commitment(preflight_commitment)?;
         let config = RpcSendTransactionConfig {
             encoding: Some(encoding),
+            preflight_commitment: Some(preflight_commitment.commitment),
             ..config
         };
         let serialized_encoded = serialize_encode_transaction(transaction, encoding)?;
@@ -218,6 +259,7 @@ impl RpcClient {
                         for (i, log) in logs.iter().enumerate() {
                             debug!("{:>3}: {}", i + 1, log);
                         }
+                        debug!("");
                     }
                 }
                 return Err(err);
@@ -265,8 +307,11 @@ impl RpcClient {
         } else {
             self.default_cluster_transaction_encoding()?
         };
+        let commitment = config.commitment.unwrap_or_default();
+        let commitment = self.maybe_map_commitment(commitment)?;
         let config = RpcSimulateTransactionConfig {
             encoding: Some(encoding),
+            commitment: Some(commitment),
             ..config
         };
         let serialized_encoded = serialize_encode_transaction(transaction, encoding)?;
@@ -276,6 +321,10 @@ impl RpcClient {
         )
     }
 
+    pub fn get_snapshot_slot(&self) -> ClientResult<Slot> {
+        self.send(RpcRequest::GetSnapshotSlot, Value::Null)
+    }
+
     pub fn get_signature_status(
         &self,
         signature: &Signature,
@@ -345,14 +394,24 @@ impl RpcClient {
         &self,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<Slot> {
-        self.send(RpcRequest::GetSlot, json!([commitment_config]))
+        self.send(
+            RpcRequest::GetSlot,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )
+    }
+
+    pub fn supply(&self) -> RpcResult<RpcSupply> {
+        self.supply_with_commitment(self.commitment_config)
     }
 
     pub fn supply_with_commitment(
         &self,
         commitment_config: CommitmentConfig,
     ) -> RpcResult<RpcSupply> {
-        self.send(RpcRequest::GetSupply, json!([commitment_config]))
+        self.send(
+            RpcRequest::GetSupply,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )
     }
 
     pub fn total_supply(&self) -> ClientResult<u64> {
@@ -363,13 +422,22 @@ impl RpcClient {
         &self,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<u64> {
-        self.send(RpcRequest::GetTotalSupply, json!([commitment_config]))
+        self.send(
+            RpcRequest::GetTotalSupply,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )
     }
 
     pub fn get_largest_accounts_with_config(
         &self,
         config: RpcLargestAccountsConfig,
     ) -> RpcResult<Vec<RpcAccountBalance>> {
+        let commitment = config.commitment.unwrap_or_default();
+        let commitment = self.maybe_map_commitment(commitment)?;
+        let config = RpcLargestAccountsConfig {
+            commitment: Some(commitment),
+            ..config
+        };
         self.send(RpcRequest::GetLargestAccounts, json!([config]))
     }
 
@@ -381,7 +449,10 @@ impl RpcClient {
         &self,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<RpcVoteAccountStatus> {
-        self.send(RpcRequest::GetVoteAccounts, json!([commitment_config]))
+        self.send(
+            RpcRequest::GetVoteAccounts,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )
     }
 
     pub fn wait_for_max_stake(
@@ -541,7 +612,10 @@ impl RpcClient {
         &self,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<EpochInfo> {
-        self.send(RpcRequest::GetEpochInfo, json!([commitment_config]))
+        self.send(
+            RpcRequest::GetEpochInfo,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )
     }
 
     pub fn get_leader_schedule(
@@ -558,7 +632,7 @@ impl RpcClient {
     ) -> ClientResult<Option<RpcLeaderSchedule>> {
         self.send(
             RpcRequest::GetLeaderSchedule,
-            json!([slot, commitment_config]),
+            json!([slot, self.maybe_map_commitment(commitment_config)?]),
         )
     }
 
@@ -599,7 +673,7 @@ impl RpcClient {
     ) -> ClientResult<Signature> {
         let signature = self.send_transaction(transaction)?;
         let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
-            self.get_recent_blockhash_with_commitment(CommitmentConfig::recent())?
+            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
                 .value
                 .0
         } else {
@@ -611,7 +685,7 @@ impl RpcClient {
                 if self
                     .get_fee_calculator_for_blockhash_with_commitment(
                         &recent_blockhash,
-                        CommitmentConfig::recent(),
+                        CommitmentConfig::processed(),
                     )?
                     .value
                     .is_none()
@@ -657,7 +731,7 @@ impl RpcClient {
     ) -> RpcResult<Option<Account>> {
         let config = RpcAccountInfoConfig {
             encoding: Some(UiAccountEncoding::Base64),
-            commitment: Some(commitment_config),
+            commitment: Some(self.maybe_map_commitment(commitment_config)?),
             data_slice: None,
         };
         let response = self.sender.send(
@@ -704,7 +778,7 @@ impl RpcClient {
     ) -> RpcResult<Vec<Option<Account>>> {
         let config = RpcAccountInfoConfig {
             encoding: Some(UiAccountEncoding::Base64),
-            commitment: Some(commitment_config),
+            commitment: Some(self.maybe_map_commitment(commitment_config)?),
             data_slice: None,
         };
         let pubkeys: Vec<_> = pubkeys.iter().map(|pubkey| pubkey.to_string()).collect();
@@ -758,7 +832,10 @@ impl RpcClient {
     ) -> RpcResult<u64> {
         self.send(
             RpcRequest::GetBalance,
-            json!([pubkey.to_string(), commitment_config]),
+            json!([
+                pubkey.to_string(),
+                self.maybe_map_commitment(commitment_config)?
+            ]),
         )
     }
 
@@ -769,6 +846,7 @@ impl RpcClient {
                 filters: None,
                 account_config: RpcAccountInfoConfig {
                     encoding: Some(UiAccountEncoding::Base64),
+                    commitment: Some(self.commitment_config),
                     ..RpcAccountInfoConfig::default()
                 },
             },
@@ -780,6 +858,16 @@ impl RpcClient {
         pubkey: &Pubkey,
         config: RpcProgramAccountsConfig,
     ) -> ClientResult<Vec<(Pubkey, Account)>> {
+        let commitment = config.account_config.commitment.unwrap_or_default();
+        let commitment = self.maybe_map_commitment(commitment)?;
+        let account_config = RpcAccountInfoConfig {
+            commitment: Some(commitment),
+            ..config.account_config
+        };
+        let config = RpcProgramAccountsConfig {
+            account_config,
+            ..config
+        };
         let accounts: Vec<RpcKeyedAccount> = self.send(
             RpcRequest::GetProgramAccounts,
             json!([pubkey.to_string(), config]),
@@ -796,7 +884,10 @@ impl RpcClient {
         &self,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<u64> {
-        self.send(RpcRequest::GetTransactionCount, json!([commitment_config]))
+        self.send(
+            RpcRequest::GetTransactionCount,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )
     }
 
     pub fn get_recent_blockhash(&self) -> ClientResult<(Hash, FeeCalculator)> {
@@ -818,9 +909,11 @@ impl RpcClient {
                     fee_calculator,
                     last_valid_slot,
                 },
-        }) =
-            self.send::<Response<RpcFees>>(RpcRequest::GetFees, json!([commitment_config]))
-        {
+        }) = self
+            .send::<Response<RpcFees>>(
+                RpcRequest::GetFees,
+                json!([self.maybe_map_commitment(commitment_config)?]),
+            ) {
             (context, blockhash, fee_calculator, last_valid_slot)
         } else if let Ok(Response {
             context,
@@ -831,7 +924,7 @@ impl RpcClient {
                 },
         }) = self.send::<Response<RpcBlockhashFeeCalculator>>(
             RpcRequest::GetRecentBlockhash,
-            json!([commitment_config]),
+            json!([self.maybe_map_commitment(commitment_config)?]),
         ) {
             (context, blockhash, fee_calculator, 0)
         } else {
@@ -869,7 +962,10 @@ impl RpcClient {
     ) -> RpcResult<Option<FeeCalculator>> {
         let Response { context, value } = self.send::<Response<Option<RpcFeeCalculator>>>(
             RpcRequest::GetFeeCalculatorForBlockhash,
-            json!([blockhash.to_string(), commitment_config]),
+            json!([
+                blockhash.to_string(),
+                self.maybe_map_commitment(commitment_config)?
+            ]),
         )?;
 
         Ok(Response {
@@ -932,6 +1028,11 @@ impl RpcClient {
         Ok(hash)
     }
 
+    pub fn get_health(&self) -> ClientResult<()> {
+        self.send::<String>(RpcRequest::GetHealth, Value::Null)
+            .map(|_| ())
+    }
+
     pub fn get_token_account(&self, pubkey: &Pubkey) -> ClientResult<Option<UiTokenAccount>> {
         Ok(self
             .get_token_account_with_commitment(pubkey, self.commitment_config)?
@@ -945,7 +1046,7 @@ impl RpcClient {
     ) -> RpcResult<Option<UiTokenAccount>> {
         let config = RpcAccountInfoConfig {
             encoding: Some(UiAccountEncoding::JsonParsed),
-            commitment: Some(commitment_config),
+            commitment: Some(self.maybe_map_commitment(commitment_config)?),
             data_slice: None,
         };
         let response = self.sender.send(
@@ -1006,7 +1107,10 @@ impl RpcClient {
     ) -> RpcResult<UiTokenAmount> {
         self.send(
             RpcRequest::GetTokenAccountBalance,
-            json!([pubkey.to_string(), commitment_config]),
+            json!([
+                pubkey.to_string(),
+                self.maybe_map_commitment(commitment_config)?
+            ]),
         )
     }
 
@@ -1039,7 +1143,7 @@ impl RpcClient {
 
         let config = RpcAccountInfoConfig {
             encoding: Some(UiAccountEncoding::JsonParsed),
-            commitment: Some(commitment_config),
+            commitment: Some(self.maybe_map_commitment(commitment_config)?),
             data_slice: None,
         };
 
@@ -1078,7 +1182,7 @@ impl RpcClient {
 
         let config = RpcAccountInfoConfig {
             encoding: Some(UiAccountEncoding::JsonParsed),
-            commitment: Some(commitment_config),
+            commitment: Some(self.maybe_map_commitment(commitment_config)?),
             data_slice: None,
         };
 
@@ -1101,7 +1205,10 @@ impl RpcClient {
     ) -> RpcResult<UiTokenAmount> {
         self.send(
             RpcRequest::GetTokenSupply,
-            json!([mint.to_string(), commitment_config]),
+            json!([
+                mint.to_string(),
+                self.maybe_map_commitment(commitment_config)?
+            ]),
         )
     }
 
@@ -1306,9 +1413,10 @@ impl RpcClient {
         commitment: CommitmentConfig,
         config: RpcSendTransactionConfig,
     ) -> ClientResult<Signature> {
-        let desired_confirmations = match commitment.commitment {
-            CommitmentLevel::Max | CommitmentLevel::Root => MAX_LOCKOUT_HISTORY + 1,
-            _ => 1,
+        let desired_confirmations = if commitment.is_finalized() {
+            MAX_LOCKOUT_HISTORY + 1
+        } else {
+            1
         };
         let mut confirmations = 0;
 
@@ -1319,7 +1427,7 @@ impl RpcClient {
             confirmations, desired_confirmations, transaction.signatures[0],
         ));
         let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
-            self.get_recent_blockhash_with_commitment(CommitmentConfig::recent())?
+            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
                 .value
                 .0
         } else {
@@ -1328,13 +1436,13 @@ impl RpcClient {
         let signature = self.send_transaction_with_config(transaction, config)?;
         let (signature, status) = loop {
             // Get recent commitment in order to count confirmations for successful transactions
-            let status =
-                self.get_signature_status_with_commitment(&signature, CommitmentConfig::recent())?;
+            let status = self
+                .get_signature_status_with_commitment(&signature, CommitmentConfig::processed())?;
             if status.is_none() {
                 if self
                     .get_fee_calculator_for_blockhash_with_commitment(
                         &recent_blockhash,
-                        CommitmentConfig::recent(),
+                        CommitmentConfig::processed(),
                     )?
                     .value
                     .is_none()
@@ -1364,29 +1472,20 @@ impl RpcClient {
         }
         let now = Instant::now();
         loop {
-            match commitment.commitment {
-                CommitmentLevel::Max | CommitmentLevel::Root =>
-                // Return when default (max) commitment is reached
-                // Failed transactions have already been eliminated, `is_some` check is sufficient
-                {
-                    if self.get_signature_status(&signature)?.is_some() {
-                        progress_bar.set_message("Transaction confirmed");
-                        progress_bar.finish_and_clear();
-                        return Ok(signature);
-                    }
-                }
-                _ => {
-                    // Return when one confirmation has been reached
-                    if confirmations >= desired_confirmations {
-                        progress_bar.set_message("Transaction reached commitment");
-                        progress_bar.finish_and_clear();
-                        return Ok(signature);
-                    }
-                }
+            // Return when specified commitment is reached
+            // Failed transactions have already been eliminated, `is_some` check is sufficient
+            if self
+                .get_signature_status_with_commitment(&signature, commitment)?
+                .is_some()
+            {
+                progress_bar.set_message("Transaction confirmed");
+                progress_bar.finish_and_clear();
+                return Ok(signature);
             }
+
             progress_bar.set_message(&format!(
                 "[{}/{}] Finalizing transaction {}",
-                confirmations + 1,
+                min(confirmations + 1, desired_confirmations),
                 desired_confirmations,
                 signature,
             ));

client/src/rpc_config.rs

@@ -31,7 +31,7 @@ pub struct RpcSimulateTransactionConfig {
     pub encoding: Option<UiTransactionEncoding>,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum RpcLargestAccountsFilter {
     Circulating,

client/src/rpc_custom_error.rs

@@ -8,8 +8,11 @@ pub const JSON_RPC_SERVER_ERROR_BLOCK_CLEANED_UP: i64 = -32001;
 pub const JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE: i64 = -32002;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE: i64 = -32003;
 pub const JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE: i64 = -32004;
-pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY: i64 = -32005;
+pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY: i64 = -32005;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_PRECOMPILE_VERIFICATION_FAILURE: i64 = -32006;
+pub const JSON_RPC_SERVER_ERROR_SLOT_SKIPPED: i64 = -32007;
+pub const JSON_RPC_SERVER_ERROR_NO_SNAPSHOT: i64 = -32008;
+pub const JSON_RPC_SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED: i64 = -32009;
 
 pub enum RpcCustomError {
     BlockCleanedUp {
@@ -24,8 +27,23 @@ pub enum RpcCustomError {
     BlockNotAvailable {
         slot: Slot,
     },
-    RpcNodeUnhealthy,
+    NodeUnhealthy {
+        num_slots_behind: Option<Slot>,
+    },
     TransactionPrecompileVerificationFailure(solana_sdk::transaction::TransactionError),
+    SlotSkipped {
+        slot: Slot,
+    },
+    NoSnapshot,
+    LongTermStorageSlotSkipped {
+        slot: Slot,
+    },
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct NodeUnhealthyErrorData {
+    pub num_slots_behind: Option<Slot>,
 }
 
 impl From<RpcCustomError> for Error {
@@ -61,10 +79,16 @@ impl From<RpcCustomError> for Error {
                 message: format!("Block not available for slot {}", slot),
                 data: None,
             },
-            RpcCustomError::RpcNodeUnhealthy => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY),
-                message: "RPC node is unhealthy".to_string(),
-                data: None,
+            RpcCustomError::NodeUnhealthy { num_slots_behind } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY),
+                message: if let Some(num_slots_behind) = num_slots_behind {
+                    format!("Node is behind by {} slots", num_slots_behind)
+                } else {
+                    "Node is unhealthy".to_string()
+                },
+                data: Some(serde_json::json!(NodeUnhealthyErrorData {
+                    num_slots_behind
+                })),
             },
             RpcCustomError::TransactionPrecompileVerificationFailure(e) => Self {
                 code: ErrorCode::ServerError(
@@ -73,6 +97,24 @@ impl From<RpcCustomError> for Error {
                 message: format!("Transaction precompile verification failure {:?}", e),
                 data: None,
             },
+            RpcCustomError::SlotSkipped { slot } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_SLOT_SKIPPED),
+                message: format!(
+                    "Slot {} was skipped, or missing due to ledger jump to recent snapshot",
+                    slot
+                ),
+                data: None,
+            },
+            RpcCustomError::NoSnapshot => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_NO_SNAPSHOT),
+                message: "No snapshot".to_string(),
+                data: None,
+            },
+            RpcCustomError::LongTermStorageSlotSkipped { slot } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED),
+                message: format!("Slot {} was skipped, or missing in long-term storage", slot),
+                data: None,
+            },
         }
     }
 }

client/src/rpc_request.rs

@@ -1,6 +1,6 @@
 use crate::rpc_response::RpcSimulateTransactionResult;
 use serde_json::{json, Value};
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::{clock::Slot, pubkey::Pubkey};
 use std::fmt;
 use thiserror::Error;
 
@@ -25,6 +25,7 @@ pub enum RpcRequest {
     GetFees,
     GetFirstAvailableBlock,
     GetGenesisHash,
+    GetHealth,
     GetIdentity,
     GetInflationGovernor,
     GetInflationRate,
@@ -34,6 +35,7 @@ pub enum RpcRequest {
     GetMultipleAccounts,
     GetProgramAccounts,
     GetRecentBlockhash,
+    GetSnapshotSlot,
     GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
@@ -80,6 +82,7 @@ impl fmt::Display for RpcRequest {
             RpcRequest::GetFees => "getFees",
             RpcRequest::GetFirstAvailableBlock => "getFirstAvailableBlock",
             RpcRequest::GetGenesisHash => "getGenesisHash",
+            RpcRequest::GetHealth => "getHealth",
             RpcRequest::GetIdentity => "getIdentity",
             RpcRequest::GetInflationGovernor => "getInflationGovernor",
             RpcRequest::GetInflationRate => "getInflationRate",
@@ -89,6 +92,7 @@ impl fmt::Display for RpcRequest {
             RpcRequest::GetMultipleAccounts => "getMultipleAccounts",
             RpcRequest::GetProgramAccounts => "getProgramAccounts",
             RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
+            RpcRequest::GetSnapshotSlot => "getSnapshotSlot",
             RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
@@ -143,6 +147,7 @@ impl RpcRequest {
 pub enum RpcResponseErrorData {
     Empty,
     SendTransactionPreflightFailure(RpcSimulateTransactionResult),
+    NodeUnhealthy { num_slots_behind: Option<Slot> },
 }
 
 impl fmt::Display for RpcResponseErrorData {
@@ -243,7 +248,7 @@ mod tests {
     #[test]
     fn test_build_request_json_config_options() {
         let commitment_config = CommitmentConfig {
-            commitment: CommitmentLevel::Max,
+            commitment: CommitmentLevel::Finalized,
         };
         let addr = json!("deadbeefXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNHhx");
 

client/src/rpc_response.rs

@@ -1,7 +1,7 @@
 use crate::client_error;
 use solana_account_decoder::{parse_token::UiTokenAmount, UiAccount};
 use solana_sdk::{
-    clock::{Epoch, Slot},
+    clock::{Epoch, Slot, UnixTimestamp},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
     inflation::Inflation,
     transaction::{Result, TransactionError},
@@ -286,6 +286,7 @@ pub struct RpcConfirmedTransactionStatusWithSignature {
     pub slot: Slot,
     pub err: Option<TransactionError>,
     pub memo: Option<String>,
+    pub block_time: Option<UnixTimestamp>,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -304,12 +305,14 @@ impl From<ConfirmedTransactionStatusWithSignature> for RpcConfirmedTransactionSt
             slot,
             err,
             memo,
+            block_time,
         } = value;
         Self {
             signature: signature.to_string(),
             slot,
             err,
             memo,
+            block_time,
         }
     }
 }

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.8"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -38,52 +38,56 @@ log = "0.4.11"
 lru = "0.6.1"
 miow = "0.2.2"
 net2 = "0.2.37"
-num_cpus = "1.13.0"
 num-traits = "0.2"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 raptorq = "1.4.2"
 rayon = "1.4.1"
 regex = "1.3.9"
+rustversion = "1.0.4"
 serde = "1.0.112"
+serde_bytes = "0.11"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-banks-server = { path = "../banks-server", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-perf = { path = "../perf", version = "1.5.0" }
-solana-program-test = { path = "../program-test", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "1.5.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.0" }
-solana-streamer = { path = "../streamer", version = "1.5.0" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.8" }
+solana-banks-server = { path = "../banks-server", version = "1.5.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.8" }
+solana-client = { path = "../client", version = "1.5.8" }
+solana-faucet = { path = "../faucet", version = "1.5.8" }
+solana-ledger = { path = "../ledger", version = "1.5.8" }
+solana-logger = { path = "../logger", version = "1.5.8" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.5.8" }
+solana-metrics = { path = "../metrics", version = "1.5.8" }
+solana-measure = { path = "../measure", version = "1.5.8" }
+solana-net-utils = { path = "../net-utils", version = "1.5.8" }
+solana-perf = { path = "../perf", version = "1.5.8" }
+solana-program-test = { path = "../program-test", version = "1.5.8" }
+solana-runtime = { path = "../runtime", version = "1.5.8" }
+solana-sdk = { path = "../sdk", version = "1.5.8" }
+solana-frozen-abi = { path = "../frozen-abi", version = "1.5.8" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.8" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.8" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.8" }
+solana-streamer = { path = "../streamer", version = "1.5.8" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.5.8" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.8" }
+solana-version = { path = "../version", version = "1.5.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.8" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "0.2", features = ["full"] }
 tokio_01 = { version = "0.1", package = "tokio" }
+tokio_01_bytes = { version = "0.4.7", package = "bytes" }
 tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
 tokio_io_01 = { version = "0.1", package = "tokio-io" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.0" }
+tokio_codec_01 = { version = "0.1", package = "tokio-codec" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.8" }
 trees = "0.2.1"
 
 [dev-dependencies]
 matches = "0.1.6"
+num_cpus = "1.13.0"
 reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"

core/benches/cluster_info.rs

@@ -35,9 +35,8 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
         cluster_info.insert_info(contact_info);
         stakes.insert(id, thread_rng().gen_range(1, NUM_PEERS) as u64);
     }
-    let stakes = Arc::new(stakes);
     let cluster_info = Arc::new(cluster_info);
-    let (peers, peers_and_stakes) = get_broadcast_peers(&cluster_info, Some(stakes));
+    let (peers, peers_and_stakes) = get_broadcast_peers(&cluster_info, Some(&stakes));
     let shreds = Arc::new(shreds);
     let last_datapoint = Arc::new(AtomicU64::new(0));
     bencher.iter(move || {

core/src/accounts_hash_verifier.rs

@@ -4,9 +4,14 @@
 // hash on gossip. Monitor gossip for messages from validators in the --trusted-validators
 // set and halt the node if a mismatch is detected.
 
-use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
-use solana_runtime::snapshot_package::{
-    AccountsPackage, AccountsPackageReceiver, AccountsPackageSender,
+use crate::{
+    cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES},
+    snapshot_packager_service::PendingSnapshotPackage,
+};
+use rayon::ThreadPool;
+use solana_runtime::{
+    accounts_db,
+    snapshot_package::{AccountsPackage, AccountsPackagePre, AccountsPackageReceiver},
 };
 use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
 use std::collections::{HashMap, HashSet};
@@ -27,7 +32,7 @@ pub struct AccountsHashVerifier {
 impl AccountsHashVerifier {
     pub fn new(
         accounts_package_receiver: AccountsPackageReceiver,
-        accounts_package_sender: Option<AccountsPackageSender>,
+        pending_snapshot_package: Option<PendingSnapshotPackage>,
         exit: &Arc<AtomicBool>,
         cluster_info: &Arc<ClusterInfo>,
         trusted_validators: Option<HashSet<Pubkey>>,
@@ -41,6 +46,7 @@ impl AccountsHashVerifier {
             .name("solana-accounts-hash".to_string())
             .spawn(move || {
                 let mut hashes = vec![];
+                let mut thread_pool_storage = None;
                 loop {
                     if exit.load(Ordering::Relaxed) {
                         break;
@@ -48,16 +54,24 @@ impl AccountsHashVerifier {
 
                     match accounts_package_receiver.recv_timeout(Duration::from_secs(1)) {
                         Ok(accounts_package) => {
-                            Self::process_accounts_package(
+                            if accounts_package.hash_for_testing.is_some()
+                                && thread_pool_storage.is_none()
+                            {
+                                thread_pool_storage =
+                                    Some(accounts_db::make_min_priority_thread_pool());
+                            }
+
+                            Self::process_accounts_package_pre(
                                 accounts_package,
                                 &cluster_info,
                                 &trusted_validators,
                                 halt_on_trusted_validators_accounts_hash_mismatch,
-                                &accounts_package_sender,
+                                &pending_snapshot_package,
                                 &mut hashes,
                                 &exit,
                                 fault_injection_rate_slots,
                                 snapshot_interval_slots,
+                                thread_pool_storage.as_ref(),
                             );
                         }
                         Err(RecvTimeoutError::Disconnected) => break,
@@ -71,29 +85,60 @@ impl AccountsHashVerifier {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
+    fn process_accounts_package_pre(
+        accounts_package: AccountsPackagePre,
+        cluster_info: &ClusterInfo,
+        trusted_validators: &Option<HashSet<Pubkey>>,
+        halt_on_trusted_validator_accounts_hash_mismatch: bool,
+        pending_snapshot_package: &Option<PendingSnapshotPackage>,
+        hashes: &mut Vec<(Slot, Hash)>,
+        exit: &Arc<AtomicBool>,
+        fault_injection_rate_slots: u64,
+        snapshot_interval_slots: u64,
+        thread_pool: Option<&ThreadPool>,
+    ) {
+        let accounts_package = solana_runtime::snapshot_utils::process_accounts_package_pre(
+            accounts_package,
+            thread_pool,
+        );
+        Self::process_accounts_package(
+            accounts_package,
+            cluster_info,
+            trusted_validators,
+            halt_on_trusted_validator_accounts_hash_mismatch,
+            pending_snapshot_package,
+            hashes,
+            exit,
+            fault_injection_rate_slots,
+            snapshot_interval_slots,
+        );
+    }
+
     fn process_accounts_package(
         accounts_package: AccountsPackage,
         cluster_info: &ClusterInfo,
         trusted_validators: &Option<HashSet<Pubkey>>,
         halt_on_trusted_validator_accounts_hash_mismatch: bool,
-        accounts_package_sender: &Option<AccountsPackageSender>,
+        pending_snapshot_package: &Option<PendingSnapshotPackage>,
         hashes: &mut Vec<(Slot, Hash)>,
         exit: &Arc<AtomicBool>,
         fault_injection_rate_slots: u64,
         snapshot_interval_slots: u64,
     ) {
+        let hash = accounts_package.hash;
         if fault_injection_rate_slots != 0
-            && accounts_package.root % fault_injection_rate_slots == 0
+            && accounts_package.slot % fault_injection_rate_slots == 0
         {
             // For testing, publish an invalid hash to gossip.
             use rand::{thread_rng, Rng};
             use solana_sdk::hash::extend_and_hash;
-            warn!("inserting fault at slot: {}", accounts_package.root);
+            warn!("inserting fault at slot: {}", accounts_package.slot);
             let rand = thread_rng().gen_range(0, 10);
-            let hash = extend_and_hash(&accounts_package.hash, &[rand]);
-            hashes.push((accounts_package.root, hash));
+            let hash = extend_and_hash(&hash, &[rand]);
+            hashes.push((accounts_package.slot, hash));
         } else {
-            hashes.push((accounts_package.root, accounts_package.hash));
+            hashes.push((accounts_package.slot, hash));
         }
 
         while hashes.len() > MAX_SNAPSHOT_HASHES {
@@ -111,8 +156,8 @@ impl AccountsHashVerifier {
         }
 
         if accounts_package.block_height % snapshot_interval_slots == 0 {
-            if let Some(sender) = accounts_package_sender.as_ref() {
-                if sender.send(accounts_package).is_err() {}
+            if let Some(pending_snapshot_package) = pending_snapshot_package.as_ref() {
+                *pending_snapshot_package.lock().unwrap() = Some(accounts_package);
             }
         }
 
@@ -175,7 +220,7 @@ mod tests {
     use super::*;
     use crate::cluster_info::make_accounts_hashes_message;
     use crate::contact_info::ContactInfo;
-    use solana_runtime::bank_forks::CompressionType;
+    use solana_runtime::bank_forks::ArchiveFormat;
     use solana_runtime::snapshot_utils::SnapshotVersion;
     use solana_sdk::{
         hash::hash,
@@ -234,12 +279,12 @@ mod tests {
             let accounts_package = AccountsPackage {
                 hash: hash(&[i as u8]),
                 block_height: 100 + i as u64,
-                root: 100 + i as u64,
+                slot: 100 + i as u64,
                 slot_deltas: vec![],
                 snapshot_links,
                 tar_output_file: PathBuf::from("."),
                 storages: vec![],
-                compression: CompressionType::Bzip2,
+                archive_format: ArchiveFormat::TarBzip2,
                 snapshot_version: SnapshotVersion::default(),
             };
 
@@ -254,6 +299,9 @@ mod tests {
                 0,
                 100,
             );
+            // sleep for 1ms to create a newer timestmap for gossip entry
+            // otherwise the timestamp won't be newer.
+            std::thread::sleep(Duration::from_millis(1));
         }
         cluster_info.flush_push_queue();
         let cluster_hashes = cluster_info

core/src/banking_stage.rs

@@ -4,7 +4,7 @@
 use crate::{
     cluster_info::ClusterInfo,
     poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
-    poh_service::PohService,
+    poh_service::{self, PohService},
 };
 use crossbeam_channel::{Receiver as CrossbeamReceiver, RecvTimeoutError};
 use itertools::Itertools;
@@ -23,7 +23,10 @@ use solana_perf::{
 };
 use solana_runtime::{
     accounts_db::ErrorCounters,
-    bank::{Bank, TransactionBalancesSet, TransactionCheckResult, TransactionExecutionResult},
+    bank::{
+        Bank, ExecuteTimings, TransactionBalancesSet, TransactionCheckResult,
+        TransactionExecutionResult,
+    },
     bank_utils,
     transaction_batch::TransactionBatch,
     vote_sender_types::ReplayVoteSender,
@@ -58,7 +61,7 @@ type PacketsAndOffsets = (Packets, Vec<usize>);
 pub type UnprocessedPackets = Vec<PacketsAndOffsets>;
 
 /// Transaction forwarding
-pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 1;
+pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 2;
 
 // Fixed thread size seems to be fastest on GCP setup
 pub const NUM_THREADS: u32 = 4;
@@ -544,6 +547,8 @@ impl BankingStage {
             vec![]
         };
 
+        let mut execute_timings = ExecuteTimings::default();
+
         let (
             mut loaded_accounts,
             results,
@@ -557,6 +562,7 @@ impl BankingStage {
             MAX_PROCESSING_AGE,
             transaction_status_sender.is_some(),
             transaction_status_sender.is_some(),
+            &mut execute_timings,
         );
         load_execute_time.stop();
 
@@ -583,10 +589,11 @@ impl BankingStage {
                 &results,
                 tx_count,
                 signature_count,
+                &mut execute_timings,
             );
 
             bank_utils::find_and_send_votes(txs, &tx_results, Some(gossip_vote_sender));
-            if let Some(sender) = transaction_status_sender {
+            if let Some(transaction_status_sender) = transaction_status_sender {
                 let post_balances = bank.collect_balances(batch);
                 let post_token_balances = collect_token_balances(&bank, &batch, &mut mint_decimals);
                 send_transaction_status_batch(
@@ -598,7 +605,7 @@ impl BankingStage {
                     TransactionTokenBalancesSet::new(pre_token_balances, post_token_balances),
                     inner_instructions,
                     transaction_logs,
-                    sender,
+                    transaction_status_sender,
                 );
             }
         }
@@ -1088,7 +1095,13 @@ pub fn create_test_recorder(
     poh_recorder.set_bank(&bank);
 
     let poh_recorder = Arc::new(Mutex::new(poh_recorder));
-    let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
+    let poh_service = PohService::new(
+        poh_recorder.clone(),
+        &poh_config,
+        &exit,
+        bank.ticks_per_slot(),
+        poh_service::DEFAULT_PINNED_CPU_CORE,
+    );
 
     (exit, poh_recorder, poh_service, entry_receiver)
 }
@@ -1108,7 +1121,7 @@ mod tests {
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
     };
-    use solana_perf::packet::to_packets;
+    use solana_perf::packet::to_packets_chunked;
     use solana_sdk::{
         instruction::InstructionError,
         signature::{Keypair, Signer},
@@ -1279,7 +1292,7 @@ mod tests {
             let tx_anf = system_transaction::transfer(&keypair, &to3, 1, start_hash);
 
             // send 'em over
-            let packets = to_packets(&[tx_no_ver, tx_anf, tx]);
+            let packets = to_packets_chunked(&[tx_no_ver, tx_anf, tx], 3);
 
             // glad they all fit
             assert_eq!(packets.len(), 1);
@@ -1355,7 +1368,7 @@ mod tests {
         let tx =
             system_transaction::transfer(&mint_keypair, &alice.pubkey(), 2, genesis_config.hash());
 
-        let packets = to_packets(&[tx]);
+        let packets = to_packets_chunked(&[tx], 1);
         let packets = packets
             .into_iter()
             .map(|packets| (packets, vec![1u8]))
@@ -1366,7 +1379,7 @@ mod tests {
         // Process a second batch that uses the same from account, so conflicts with above TX
         let tx =
             system_transaction::transfer(&mint_keypair, &alice.pubkey(), 1, genesis_config.hash());
-        let packets = to_packets(&[tx]);
+        let packets = to_packets_chunked(&[tx], 1);
         let packets = packets
             .into_iter()
             .map(|packets| (packets, vec![1u8]))
@@ -2061,7 +2074,10 @@ mod tests {
                 &transactions,
                 &poh_recorder,
                 0,
-                Some(transaction_status_sender),
+                Some(TransactionStatusSender {
+                    sender: transaction_status_sender,
+                    enable_cpi_and_log_storage: false,
+                }),
                 &gossip_vote_sender,
             );
 

core/src/bigtable_upload_service.rs

@@ -74,6 +74,7 @@ impl BigTableUploadService {
                 start_slot,
                 Some(end_slot),
                 true,
+                false,
                 exit.clone(),
             ));
 

core/src/broadcast_stage.rs

@@ -17,7 +17,7 @@ use crossbeam_channel::{
     Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
     Sender as CrossbeamSender,
 };
-use solana_ledger::{blockstore::Blockstore, shred::Shred, staking_utils};
+use solana_ledger::{blockstore::Blockstore, shred::Shred};
 use solana_measure::measure::Measure;
 use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
 use solana_runtime::bank::Bank;
@@ -306,7 +306,7 @@ impl BroadcastStage {
 
         for (_, bank) in retransmit_slots.iter() {
             let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-            let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+            let stakes = bank.epoch_staked_nodes(bank_epoch);
             let stakes = stakes.map(Arc::new);
             let data_shreds = Arc::new(
                 blockstore
@@ -347,7 +347,8 @@ fn update_peer_stats(
 ) {
     let now = timestamp();
     let last = last_datapoint_submit.load(Ordering::Relaxed);
-    if now - last > 1000
+    #[allow(deprecated)]
+    if now.saturating_sub(last) > 1000
         && last_datapoint_submit.compare_and_swap(last, now, Ordering::Relaxed) == last
     {
         datapoint_info!(
@@ -358,9 +359,9 @@ fn update_peer_stats(
     }
 }
 
-pub fn get_broadcast_peers<S: std::hash::BuildHasher>(
+pub fn get_broadcast_peers(
     cluster_info: &ClusterInfo,
-    stakes: Option<Arc<HashMap<Pubkey, u64, S>>>,
+    stakes: Option<&HashMap<Pubkey, u64>>,
 ) -> (Vec<ContactInfo>, Vec<(u64, usize)>) {
     use crate::cluster_info;
     let mut peers = cluster_info.tvu_peers();

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -102,7 +102,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         blockstore_sender.send((data_shreds.clone(), None))?;
         // 4) Start broadcast step
         let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-        let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+        let stakes = bank.epoch_staked_nodes(bank_epoch);
         let stakes = stakes.map(Arc::new);
         socket_sender.send(((stakes.clone(), data_shreds), None))?;
         if let Some((good_last_data_shred, bad_last_data_shred)) = last_shreds {
@@ -135,7 +135,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
     ) -> Result<()> {
         let ((stakes, shreds), _) = receiver.lock().unwrap().recv()?;
         // Broadcast data
-        let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
+        let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes.as_deref());
 
         broadcast_shreds(
             sock,

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -213,7 +213,7 @@ impl StandardBroadcastRun {
 
         let mut get_leader_schedule_time = Measure::start("broadcast_get_leader_schedule");
         let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-        let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+        let stakes = bank.epoch_staked_nodes(bank_epoch);
         let stakes = stakes.map(Arc::new);
 
         // Broadcast the last shred of the interrupted slot if necessary
@@ -319,7 +319,7 @@ impl StandardBroadcastRun {
         &mut self,
         sock: &UdpSocket,
         cluster_info: &ClusterInfo,
-        stakes: Option<Arc<HashMap<Pubkey, u64>>>,
+        stakes: Option<&HashMap<Pubkey, u64>>,
         shreds: Arc<Vec<Shred>>,
         broadcast_shred_batch_info: Option<BroadcastShredBatchInfo>,
     ) -> Result<()> {
@@ -329,6 +329,7 @@ impl StandardBroadcastRun {
         let mut get_peers_time = Measure::start("broadcast::get_peers");
         let now = timestamp();
         let last = self.last_peer_update.load(Ordering::Relaxed);
+        #[allow(deprecated)]
         if now - last > BROADCAST_PEER_UPDATE_INTERVAL_MS
             && self
                 .last_peer_update
@@ -431,7 +432,7 @@ impl BroadcastRun for StandardBroadcastRun {
         sock: &UdpSocket,
     ) -> Result<()> {
         let ((stakes, shreds), slot_start_ts) = receiver.lock().unwrap().recv()?;
-        self.broadcast(sock, cluster_info, stakes, shreds, slot_start_ts)
+        self.broadcast(sock, cluster_info, stakes.as_deref(), shreds, slot_start_ts)
     }
     fn record(
         &mut self,

core/src/cache_block_time_service.rs

@@ -2,9 +2,7 @@ use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
 use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
 use solana_runtime::bank::Bank;
-use solana_sdk::{feature_set, timing::slot_duration_from_slots_per_year};
 use std::{
-    collections::HashMap,
     sync::{
         atomic::{AtomicBool, Ordering},
         Arc,
@@ -60,24 +58,8 @@ impl CacheBlockTimeService {
     }
 
     fn cache_block_time(bank: Arc<Bank>, blockstore: &Arc<Blockstore>) {
-        if bank
-            .feature_set
-            .is_active(&feature_set::timestamp_correction::id())
-        {
-            if let Err(e) = blockstore.cache_block_time(bank.slot(), bank.clock().unix_timestamp) {
-                error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
-            }
-        } else {
-            let slot_duration = slot_duration_from_slots_per_year(bank.slots_per_year());
-            let epoch = bank.epoch_schedule().get_epoch(bank.slot());
-            let stakes = HashMap::new();
-            let stakes = bank.epoch_vote_accounts(epoch).unwrap_or(&stakes);
-
-            if let Err(e) =
-                blockstore.cache_block_time_from_slot_entries(bank.slot(), slot_duration, stakes)
-            {
-                error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
-            }
+        if let Err(e) = blockstore.cache_block_time(bank.slot(), bank.clock().unix_timestamp) {
+            error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
         }
     }
 

core/src/cluster_info_vote_listener.rs

@@ -4,7 +4,6 @@ use crate::{
     optimistic_confirmation_verifier::OptimisticConfirmationVerifier,
     optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender},
     poh_recorder::PohRecorder,
-    pubkey_references::LockedPubkeyReferences,
     result::{Error, Result},
     rpc_subscriptions::RpcSubscriptions,
     sigverify,
@@ -57,15 +56,15 @@ pub struct SlotVoteTracker {
     // Maps pubkeys that have voted for this slot
     // to whether or not we've seen the vote on gossip.
     // True if seen on gossip, false if only seen in replay.
-    voted: HashMap<Arc<Pubkey>, bool>,
+    voted: HashMap<Pubkey, bool>,
     optimistic_votes_tracker: HashMap<Hash, VoteStakeTracker>,
-    updates: Option<Vec<Arc<Pubkey>>>,
+    updates: Option<Vec<Pubkey>>,
     gossip_only_stake: u64,
 }
 
 impl SlotVoteTracker {
     #[allow(dead_code)]
-    pub fn get_updates(&mut self) -> Option<Vec<Arc<Pubkey>>> {
+    pub fn get_updates(&mut self) -> Option<Vec<Pubkey>> {
         self.updates.take()
     }
 
@@ -85,7 +84,6 @@ pub struct VoteTracker {
     epoch_authorized_voters: RwLock<HashMap<Epoch, Arc<EpochAuthorizedVoters>>>,
     leader_schedule_epoch: RwLock<Epoch>,
     current_epoch: RwLock<Epoch>,
-    keys: LockedPubkeyReferences,
     epoch_schedule: EpochSchedule,
 }
 
@@ -157,21 +155,19 @@ impl VoteTracker {
     }
 
     #[cfg(test)]
-    pub fn insert_vote(&self, slot: Slot, pubkey: Arc<Pubkey>) {
+    pub fn insert_vote(&self, slot: Slot, pubkey: Pubkey) {
         let mut w_slot_vote_trackers = self.slot_vote_trackers.write().unwrap();
 
         let slot_vote_tracker = w_slot_vote_trackers.entry(slot).or_default();
 
         let mut w_slot_vote_tracker = slot_vote_tracker.write().unwrap();
 
-        w_slot_vote_tracker.voted.insert(pubkey.clone(), true);
+        w_slot_vote_tracker.voted.insert(pubkey, true);
         if let Some(ref mut updates) = w_slot_vote_tracker.updates {
-            updates.push(pubkey.clone())
+            updates.push(pubkey)
         } else {
-            w_slot_vote_tracker.updates = Some(vec![pubkey.clone()]);
+            w_slot_vote_tracker.updates = Some(vec![pubkey]);
         }
-
-        self.keys.get_or_insert(&pubkey);
     }
 
     fn progress_leader_schedule_epoch(&self, root_bank: &Bank) {
@@ -221,7 +217,6 @@ impl VoteTracker {
                 .write()
                 .unwrap()
                 .retain(|epoch, _| *epoch >= root_epoch);
-            self.keys.purge();
             *self.current_epoch.write().unwrap() = root_epoch;
         }
     }
@@ -382,7 +377,7 @@ impl ClusterInfoVoteListener {
                 return Ok(());
             }
 
-            if let Err(e) = verified_vote_packets.get_and_process_vote_packets(
+            if let Err(e) = verified_vote_packets.receive_and_process_vote_packets(
                 &verified_vote_label_packets_receiver,
                 &mut update_version,
             ) {
@@ -403,6 +398,7 @@ impl ClusterInfoVoteListener {
                     let last_version = bank.last_vote_sync.load(Ordering::Relaxed);
                     let (new_version, msgs) = verified_vote_packets.get_latest_votes(last_version);
                     verified_packets_sender.send(msgs)?;
+                    #[allow(deprecated)]
                     bank.last_vote_sync.compare_and_swap(
                         last_version,
                         new_version,
@@ -546,7 +542,7 @@ impl ClusterInfoVoteListener {
         root_bank: &Bank,
         subscriptions: &RpcSubscriptions,
         verified_vote_sender: &VerifiedVoteSender,
-        diff: &mut HashMap<Slot, HashMap<Arc<Pubkey>, bool>>,
+        diff: &mut HashMap<Slot, HashMap<Pubkey, bool>>,
         new_optimistic_confirmed_slots: &mut Vec<(Slot, Hash)>,
         is_gossip_vote: bool,
         bank_notification_sender: &Option<BankNotificationSender>,
@@ -571,7 +567,6 @@ impl ClusterInfoVoteListener {
                 continue;
             }
             let epoch_stakes = epoch_stakes.unwrap();
-            let unduplicated_pubkey = vote_tracker.keys.get_or_insert(&vote_pubkey);
 
             // The last vote slot, which is the greatest slot in the stack
             // of votes in a vote transaction, qualifies for optimistic confirmation.
@@ -590,7 +585,7 @@ impl ClusterInfoVoteListener {
                     vote_tracker,
                     last_vote_slot,
                     last_vote_hash,
-                    unduplicated_pubkey.clone(),
+                    *vote_pubkey,
                     stake,
                     total_stake,
                 );
@@ -625,7 +620,7 @@ impl ClusterInfoVoteListener {
 
             diff.entry(slot)
                 .or_default()
-                .entry(unduplicated_pubkey)
+                .entry(*vote_pubkey)
                 .and_modify(|seen_in_gossip_previously| {
                     *seen_in_gossip_previously = *seen_in_gossip_previously || is_gossip_vote
                 })
@@ -680,7 +675,7 @@ impl ClusterInfoVoteListener {
         verified_vote_sender: &VerifiedVoteSender,
         bank_notification_sender: &Option<BankNotificationSender>,
     ) -> Vec<(Slot, Hash)> {
-        let mut diff: HashMap<Slot, HashMap<Arc<Pubkey>, bool>> = HashMap::new();
+        let mut diff: HashMap<Slot, HashMap<Pubkey, bool>> = HashMap::new();
         let mut new_optimistic_confirmed_slots = vec![];
 
         // Process votes from gossip and ReplayStage
@@ -747,9 +742,7 @@ impl ClusterInfoVoteListener {
                 // no other writers to `slot_vote_tracker` that
                 // `is_new || is_new_from_gossip`. In both cases we want to record
                 // `is_new_from_gossip` for the `pubkey` entry.
-                w_slot_tracker
-                    .voted
-                    .insert(pubkey.clone(), seen_in_gossip_above);
+                w_slot_tracker.voted.insert(pubkey, seen_in_gossip_above);
                 w_slot_tracker.updates.as_mut().unwrap().push(pubkey);
             }
 
@@ -764,7 +757,7 @@ impl ClusterInfoVoteListener {
         vote_tracker: &VoteTracker,
         slot: Slot,
         hash: Hash,
-        pubkey: Arc<Pubkey>,
+        pubkey: Pubkey,
         stake: u64,
         total_epoch_stake: u64,
     ) -> (bool, bool) {
@@ -824,7 +817,7 @@ mod tests {
         use bincode::serialized_size;
         info!("max vote size {}", serialized_size(&vote_tx).unwrap());
 
-        let msgs = packet::to_packets(&[vote_tx]); // panics if won't fit
+        let msgs = packet::to_packets_chunked(&[vote_tx], 1); // panics if won't fit
 
         assert_eq!(msgs.len(), 1);
     }
@@ -901,10 +894,10 @@ mod tests {
         let (vote_tracker, bank, _, _) = setup();
 
         // Check outdated slots are purged with new root
-        let new_voter = Arc::new(solana_sdk::pubkey::new_rand());
+        let new_voter = solana_sdk::pubkey::new_rand();
         // Make separate copy so the original doesn't count toward
         // the ref count, which would prevent cleanup
-        let new_voter_ = Arc::new(*new_voter);
+        let new_voter_ = new_voter;
         vote_tracker.insert_vote(bank.slot(), new_voter_);
         assert!(vote_tracker
             .slot_vote_trackers
@@ -921,7 +914,6 @@ mod tests {
 
         // Check `keys` and `epoch_authorized_voters` are purged when new
         // root bank moves to the next epoch
-        assert!(vote_tracker.keys.0.read().unwrap().contains(&new_voter));
         let current_epoch = bank.epoch();
         let new_epoch_bank = Bank::new_from_parent(
             &bank,
@@ -930,7 +922,6 @@ mod tests {
                 .get_first_slot_in_epoch(current_epoch + 1),
         );
         vote_tracker.progress_with_new_root_bank(&new_epoch_bank);
-        assert!(!vote_tracker.keys.0.read().unwrap().contains(&new_voter));
         assert_eq!(
             *vote_tracker.current_epoch.read().unwrap(),
             current_epoch + 1
@@ -1430,13 +1421,6 @@ mod tests {
 
     #[test]
     fn test_vote_tracker_references() {
-        // The number of references that get stored for a pubkey every time
-        // a vote is added to the tracking set via a transaction. One stored in the
-        // SlotVoteTracker.voted, one in SlotVoteTracker.updates, one in
-        // SlotVoteTracker.optimistic_votes_tracker
-        let ref_count_per_vote = 3;
-        let ref_count_per_new_key = 1;
-
         // Create some voters at genesis
         let validator_keypairs: Vec<_> =
             (0..2).map(|_| ValidatorVoteKeypairs::new_rand()).collect();
@@ -1491,22 +1475,6 @@ mod tests {
             &verified_vote_sender,
             &None,
         );
-        let ref_count = Arc::strong_count(
-            &vote_tracker
-                .keys
-                .0
-                .read()
-                .unwrap()
-                .get(&validator0_keypairs.vote_keypair.pubkey())
-                .unwrap(),
-        );
-
-        // This new pubkey submitted a vote for a slot, so ref count is
-        // `ref_count_per_vote + ref_count_per_new_key`.
-        // +ref_count_per_new_key for the new pubkey  in `vote_tracker.keys` and
-        // +ref_count_per_vote for the one new vote
-        let mut current_ref_count = ref_count_per_vote + ref_count_per_new_key;
-        assert_eq!(ref_count, current_ref_count);
 
         // Setup next epoch
         let old_epoch = bank.get_leader_schedule_epoch(bank.slot());
@@ -1561,35 +1529,6 @@ mod tests {
             &verified_vote_sender,
             &None,
         );
-
-        // Check new replay vote pubkey first
-        let ref_count = Arc::strong_count(
-            &vote_tracker
-                .keys
-                .0
-                .read()
-                .unwrap()
-                .get(&validator_keypairs[1].vote_keypair.pubkey())
-                .unwrap(),
-        );
-        // This new pubkey submitted a replay vote for a slot, so ref count is
-        // `ref_count_per_optimistic_vote + ref_count_per_new_key`.
-        // +ref_count_per_new_key for the new pubkey  in `vote_tracker.keys` and
-        // +ref_count_per_optimistic_vote for the one new vote
-        assert_eq!(ref_count, ref_count_per_vote + ref_count_per_new_key);
-
-        // Check the existing pubkey
-        let ref_count = Arc::strong_count(
-            &vote_tracker
-                .keys
-                .0
-                .read()
-                .unwrap()
-                .get(&validator0_keypairs.vote_keypair.pubkey())
-                .unwrap(),
-        );
-        current_ref_count += 2 * ref_count_per_vote;
-        assert_eq!(ref_count, current_ref_count);
     }
 
     fn setup() -> (

core/src/cluster_slots.rs

@@ -1,21 +1,23 @@
 use crate::{
     cluster_info::ClusterInfo, contact_info::ContactInfo, epoch_slots::EpochSlots,
-    pubkey_references::LockedPubkeyReferences, serve_repair::RepairType,
+    serve_repair::RepairType,
 };
 use solana_runtime::{bank_forks::BankForks, epoch_stakes::NodeIdToVoteAccounts};
 use solana_sdk::{clock::Slot, pubkey::Pubkey};
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{BTreeMap, HashMap, HashSet},
     sync::{Arc, RwLock},
 };
 
-pub type SlotPubkeys = HashMap<Arc<Pubkey>, u64>;
-pub type ClusterSlotsMap = RwLock<HashMap<Slot, Arc<RwLock<SlotPubkeys>>>>;
+// Limit the size of cluster-slots map in case
+// of receiving bogus epoch slots values.
+const CLUSTER_SLOTS_TRIM_SIZE: usize = 524_288; // 512K
+
+pub type SlotPubkeys = HashMap<Pubkey, u64>;
 
 #[derive(Default)]
 pub struct ClusterSlots {
-    cluster_slots: ClusterSlotsMap,
-    keys: LockedPubkeyReferences,
+    cluster_slots: RwLock<BTreeMap<Slot, Arc<RwLock<SlotPubkeys>>>>,
     since: RwLock<Option<u64>>,
     validator_stakes: RwLock<Arc<NodeIdToVoteAccounts>>,
     epoch: RwLock<Option<u64>>,
@@ -29,23 +31,29 @@ impl ClusterSlots {
     pub fn update(&self, root: Slot, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
         self.update_peers(cluster_info, bank_forks);
         let since = *self.since.read().unwrap();
-        let epoch_slots = cluster_info.get_epoch_slots_since(since);
-        self.update_internal(root, epoch_slots);
+        let (epoch_slots, since) = cluster_info.get_epoch_slots_since(since);
+        self.update_internal(root, epoch_slots, since);
     }
-    fn update_internal(&self, root: Slot, epoch_slots: (Vec<EpochSlots>, Option<u64>)) {
-        let (epoch_slots_list, since) = epoch_slots;
+    fn update_internal(&self, root: Slot, epoch_slots_list: Vec<EpochSlots>, since: Option<u64>) {
         for epoch_slots in epoch_slots_list {
             let slots = epoch_slots.to_slots(root);
             for slot in &slots {
                 if *slot <= root {
                     continue;
                 }
-                let unduplicated_pubkey = self.keys.get_or_insert(&epoch_slots.from);
-                self.insert_node_id(*slot, unduplicated_pubkey);
+                self.insert_node_id(*slot, epoch_slots.from);
+            }
+        }
+        {
+            let mut cluster_slots = self.cluster_slots.write().unwrap();
+            *cluster_slots = cluster_slots.split_off(&(root + 1));
+            // Trimming is done at 2x size so that amortized it has a constant
+            // cost. The slots furthest away from the root are discarded.
+            if cluster_slots.len() > 2 * CLUSTER_SLOTS_TRIM_SIZE {
+                let key = *cluster_slots.keys().nth(CLUSTER_SLOTS_TRIM_SIZE).unwrap();
+                cluster_slots.split_off(&key);
             }
         }
-        self.cluster_slots.write().unwrap().retain(|x, _| *x > root);
-        self.keys.purge();
         *self.since.write().unwrap() = since;
     }
 
@@ -54,13 +62,12 @@ impl ClusterSlots {
             .read()
             .unwrap()
             .iter()
-            .filter(|(_, keys)| keys.read().unwrap().get(id).is_some())
-            .map(|(slot, _)| slot)
-            .cloned()
+            .filter(|(_, keys)| keys.read().unwrap().contains_key(id))
+            .map(|(slot, _)| *slot)
             .collect()
     }
 
-    pub fn insert_node_id(&self, slot: Slot, node_id: Arc<Pubkey>) {
+    pub fn insert_node_id(&self, slot: Slot, node_id: Pubkey) {
         let balance = self
             .validator_stakes
             .read()
@@ -68,24 +75,18 @@ impl ClusterSlots {
             .get(&node_id)
             .map(|v| v.total_stake)
             .unwrap_or(0);
-        let mut slot_pubkeys = self.cluster_slots.read().unwrap().get(&slot).cloned();
-        if slot_pubkeys.is_none() {
-            let new_slot_pubkeys = Arc::new(RwLock::new(HashMap::default()));
-            self.cluster_slots
-                .write()
-                .unwrap()
-                .insert(slot, new_slot_pubkeys.clone());
-            slot_pubkeys = Some(new_slot_pubkeys);
-        }
-        slot_pubkeys
-            .unwrap()
+        let slot_pubkeys = self
+            .cluster_slots
             .write()
             .unwrap()
-            .insert(node_id, balance);
+            .entry(slot)
+            .or_default()
+            .clone();
+        slot_pubkeys.write().unwrap().insert(node_id, balance);
     }
 
     fn update_peers(&self, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
-        let root_bank = bank_forks.read().unwrap().root_bank().clone();
+        let root_bank = bank_forks.read().unwrap().root_bank();
         let root_epoch = root_bank.epoch();
         let my_epoch = *self.epoch.read().unwrap();
 
@@ -182,7 +183,7 @@ mod tests {
     #[test]
     fn test_update_noop() {
         let cs = ClusterSlots::default();
-        cs.update_internal(0, (vec![], None));
+        cs.update_internal(0, vec![], None);
         assert!(cs.cluster_slots.read().unwrap().is_empty());
         assert!(cs.since.read().unwrap().is_none());
     }
@@ -191,7 +192,7 @@ mod tests {
     fn test_update_empty() {
         let cs = ClusterSlots::default();
         let epoch_slot = EpochSlots::default();
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        cs.update_internal(0, vec![epoch_slot], Some(0));
         assert_eq!(*cs.since.read().unwrap(), Some(0));
         assert!(cs.lookup(0).is_none());
     }
@@ -202,7 +203,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[0], 0);
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        cs.update_internal(0, vec![epoch_slot], Some(0));
         assert_eq!(*cs.since.read().unwrap(), Some(0));
         assert!(cs.lookup(0).is_none());
     }
@@ -212,7 +213,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        cs.update_internal(0, vec![epoch_slot], Some(0));
         assert_eq!(*cs.since.read().unwrap(), Some(0));
         assert!(cs.lookup(0).is_none());
         assert!(cs.lookup(1).is_some());
@@ -241,8 +242,8 @@ mod tests {
         let mut map = HashMap::new();
         let k1 = solana_sdk::pubkey::new_rand();
         let k2 = solana_sdk::pubkey::new_rand();
-        map.insert(Arc::new(k1), std::u64::MAX / 2);
-        map.insert(Arc::new(k2), 0);
+        map.insert(k1, std::u64::MAX / 2);
+        map.insert(k2, 0);
         cs.cluster_slots
             .write()
             .unwrap()
@@ -263,14 +264,14 @@ mod tests {
         let mut map = HashMap::new();
         let k1 = solana_sdk::pubkey::new_rand();
         let k2 = solana_sdk::pubkey::new_rand();
-        map.insert(Arc::new(k2), 0);
+        map.insert(k2, 0);
         cs.cluster_slots
             .write()
             .unwrap()
             .insert(0, Arc::new(RwLock::new(map)));
         //make sure default weights are used as well
         let validator_stakes: HashMap<_, _> = vec![(
-            *Arc::new(k1),
+            k1,
             NodeVoteAccounts {
                 total_stake: std::u64::MAX / 2,
                 vote_accounts: vec![Pubkey::default()],
@@ -317,7 +318,7 @@ mod tests {
         // Mark the first validator as completed slot 9, should pick that validator,
         // even though it only has default stake, while the other validator has
         // max stake
-        cs.insert_node_id(slot, Arc::new(contact_infos[0].id));
+        cs.insert_node_id(slot, contact_infos[0].id);
         assert_eq!(
             cs.compute_weights_exclude_noncomplete(slot, &contact_infos),
             vec![(1, 0)]
@@ -343,7 +344,7 @@ mod tests {
         );
 
         *cs.validator_stakes.write().unwrap() = map;
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         assert!(cs.lookup(1).is_some());
         assert_eq!(
             cs.lookup(1)
@@ -360,7 +361,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         let self_id = solana_sdk::pubkey::new_rand();
         assert_eq!(
             cs.generate_repairs_for_missing_slots(&self_id, 0),
@@ -374,7 +375,7 @@ mod tests {
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
         let self_id = epoch_slot.from;
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         let slots: Vec<Slot> = cs.collect(&self_id).into_iter().collect();
         assert_eq!(slots, vec![1]);
     }
@@ -385,7 +386,7 @@ mod tests {
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
         let self_id = epoch_slot.from;
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         assert!(cs
             .generate_repairs_for_missing_slots(&self_id, 0)
             .is_empty());

core/src/commitment_service.rs

@@ -263,15 +263,9 @@ mod tests {
     #[test]
     fn test_get_highest_confirmed_root() {
         assert_eq!(get_highest_confirmed_root(vec![], 10), 0);
-        let mut rooted_stake = vec![];
-        rooted_stake.push((0, 5));
-        rooted_stake.push((1, 5));
+        let rooted_stake = vec![(0, 5), (1, 5)];
         assert_eq!(get_highest_confirmed_root(rooted_stake, 10), 0);
-        let mut rooted_stake = vec![];
-        rooted_stake.push((1, 5));
-        rooted_stake.push((0, 10));
-        rooted_stake.push((2, 5));
-        rooted_stake.push((1, 4));
+        let rooted_stake = vec![(1, 5), (0, 10), (2, 5), (1, 4)];
         assert_eq!(get_highest_confirmed_root(rooted_stake, 10), 1);
     }
 
@@ -434,26 +428,26 @@ mod tests {
         let mut vote_state1 = VoteState::from(&vote_account1).unwrap();
         vote_state1.process_slot_vote_unchecked(3);
         vote_state1.process_slot_vote_unchecked(5);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state1));
+        let versioned = VoteStateVersions::new_current(vote_state1);
         VoteState::to(&versioned, &mut vote_account1).unwrap();
         bank.store_account(&pk1, &vote_account1);
 
         let mut vote_state2 = VoteState::from(&vote_account2).unwrap();
         vote_state2.process_slot_vote_unchecked(9);
         vote_state2.process_slot_vote_unchecked(10);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state2));
+        let versioned = VoteStateVersions::new_current(vote_state2);
         VoteState::to(&versioned, &mut vote_account2).unwrap();
         bank.store_account(&pk2, &vote_account2);
 
         let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
         vote_state3.root_slot = Some(1);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
+        let versioned = VoteStateVersions::new_current(vote_state3);
         VoteState::to(&versioned, &mut vote_account3).unwrap();
         bank.store_account(&pk3, &vote_account3);
 
         let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
         vote_state4.root_slot = Some(2);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
+        let versioned = VoteStateVersions::new_current(vote_state4);
         VoteState::to(&versioned, &mut vote_account4).unwrap();
         bank.store_account(&pk4, &vote_account4);
 

core/src/consensus.rs

@@ -1,7 +1,4 @@
-use crate::{
-    progress_map::{LockoutIntervals, ProgressMap},
-    pubkey_references::PubkeyReferences,
-};
+use crate::progress_map::{LockoutIntervals, ProgressMap};
 use chrono::prelude::*;
 use solana_ledger::{ancestor_iterator::AncestorIterator, blockstore::Blockstore, blockstore_db};
 use solana_measure::measure::Measure;
@@ -26,7 +23,10 @@ use std::{
     collections::{HashMap, HashSet},
     fs::{self, File},
     io::BufReader,
-    ops::Bound::{Included, Unbounded},
+    ops::{
+        Bound::{Included, Unbounded},
+        Deref,
+    },
     path::{Path, PathBuf},
     sync::Arc,
 };
@@ -183,7 +183,7 @@ impl Tower {
         let root_bank = bank_forks.root_bank();
         let (_progress, heaviest_subtree_fork_choice) =
             crate::replay_stage::ReplayStage::initialize_progress_and_fork_choice(
-                root_bank,
+                root_bank.deref(),
                 bank_forks.frozen_banks().values().cloned().collect(),
                 &my_pubkey,
                 &vote_account,
@@ -211,11 +211,11 @@ impl Tower {
         bank_slot: Slot,
         vote_accounts: F,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
-        all_pubkeys: &mut PubkeyReferences,
     ) -> ComputedBankState
     where
         F: IntoIterator<Item = (Pubkey, (u64, ArcVoteAccount))>,
     {
+        let mut vote_slots = HashSet::new();
         let mut voted_stakes = HashMap::new();
         let mut total_stake = 0;
         let mut bank_weight = 0;
@@ -243,7 +243,6 @@ impl Tower {
                 Ok(vote_state) => vote_state.clone(),
             };
             for vote in &vote_state.votes {
-                let key = all_pubkeys.get_or_insert(&key);
                 lockout_intervals
                     .entry(vote.expiration_slot())
                     .or_insert_with(Vec::new)
@@ -278,7 +277,7 @@ impl Tower {
 
             for vote in &vote_state.votes {
                 bank_weight += vote.lockout() as u128 * voted_stake as u128;
-                Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
+                vote_slots.insert(vote.slot);
             }
 
             if start_root != vote_state.root_slot {
@@ -289,7 +288,7 @@ impl Tower {
                     };
                     trace!("ROOT: {}", vote.slot);
                     bank_weight += vote.lockout() as u128 * voted_stake as u128;
-                    Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
+                    vote_slots.insert(vote.slot);
                 }
             }
             if let Some(root) = vote_state.root_slot {
@@ -298,7 +297,7 @@ impl Tower {
                     slot: root,
                 };
                 bank_weight += vote.lockout() as u128 * voted_stake as u128;
-                Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
+                vote_slots.insert(vote.slot);
             }
 
             // The last vote in the vote stack is a simulated vote on bank_slot, which
@@ -326,6 +325,9 @@ impl Tower {
             total_stake += voted_stake;
         }
 
+        // TODO: populate_ancestor_voted_stakes only adds zeros. Comment why
+        // that is necessary (if so).
+        Self::populate_ancestor_voted_stakes(&mut voted_stakes, vote_slots, ancestors);
         ComputedBankState {
             voted_stakes,
             total_stake,
@@ -352,7 +354,7 @@ impl Tower {
         slot: Slot,
         hash: Hash,
         last_voted_slot_in_bank: Option<Slot>,
-    ) -> (Vote, usize) {
+    ) -> (Vote, Vec<Slot> /*VoteState.tower*/) {
         let mut local_vote_state = local_vote_state.clone();
         let vote = Vote::new(vec![slot], hash);
         local_vote_state.process_vote_unchecked(&vote);
@@ -372,7 +374,7 @@ impl Tower {
             slots,
             local_vote_state.votes
         );
-        (Vote::new(slots, hash), local_vote_state.votes.len() - 1)
+        (Vote::new(slots, hash), local_vote_state.tower())
     }
 
     fn last_voted_slot_in_bank(bank: &Bank, vote_account_pubkey: &Pubkey) -> Option<Slot> {
@@ -381,7 +383,11 @@ impl Tower {
         slot
     }
 
-    pub fn new_vote_from_bank(&self, bank: &Bank, vote_account_pubkey: &Pubkey) -> (Vote, usize) {
+    pub fn new_vote_from_bank(
+        &self,
+        bank: &Bank,
+        vote_account_pubkey: &Pubkey,
+    ) -> (Vote, Vec<Slot> /*VoteState.tower*/) {
         let voted_slot = Self::last_voted_slot_in_bank(bank, vote_account_pubkey);
         Self::new_vote(&self.lockouts, bank.slot(), bank.hash(), voted_slot)
     }
@@ -766,20 +772,19 @@ impl Tower {
     /// Update lockouts for all the ancestors
     pub(crate) fn populate_ancestor_voted_stakes(
         voted_stakes: &mut VotedStakes,
-        vote: &Lockout,
+        vote_slots: impl IntoIterator<Item = Slot>,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
     ) {
         // If there's no ancestors, that means this slot must be from before the current root,
         // in which case the lockouts won't be calculated in bank_weight anyways, so ignore
         // this slot
-        let vote_slot_ancestors = ancestors.get(&vote.slot);
-        if vote_slot_ancestors.is_none() {
-            return;
-        }
-        let mut slot_with_ancestors = vec![vote.slot];
-        slot_with_ancestors.extend(vote_slot_ancestors.unwrap());
-        for slot in slot_with_ancestors {
-            voted_stakes.entry(slot).or_default();
+        for vote_slot in vote_slots {
+            if let Some(slot_ancestors) = ancestors.get(&vote_slot) {
+                voted_stakes.entry(vote_slot).or_default();
+                for slot in slot_ancestors {
+                    voted_stakes.entry(*slot).or_default();
+                }
+            }
         }
     }
 
@@ -793,15 +798,11 @@ impl Tower {
     ) {
         // If there's no ancestors, that means this slot must be from
         // before the current root, so ignore this slot
-        let vote_slot_ancestors = ancestors.get(&voted_slot);
-        if vote_slot_ancestors.is_none() {
-            return;
-        }
-        let mut slot_with_ancestors = vec![voted_slot];
-        slot_with_ancestors.extend(vote_slot_ancestors.unwrap());
-        for slot in slot_with_ancestors {
-            let current = voted_stakes.entry(slot).or_default();
-            *current += voted_stake;
+        if let Some(vote_slot_ancestors) = ancestors.get(&voted_slot) {
+            *voted_stakes.entry(voted_slot).or_default() += voted_stake;
+            for slot in vote_slot_ancestors {
+                *voted_stakes.entry(*slot).or_default() += voted_stake;
+            }
         }
     }
 
@@ -1264,7 +1265,6 @@ pub mod test {
         collections::HashMap,
         fs::{remove_file, OpenOptions},
         io::{Read, Seek, SeekFrom, Write},
-        rc::Rc,
         sync::RwLock,
     };
     use tempfile::TempDir;
@@ -1374,7 +1374,6 @@ pub mod test {
                 &VoteTracker::default(),
                 &ClusterSlots::default(),
                 &self.bank_forks,
-                &mut PubkeyReferences::default(),
                 &mut self.heaviest_subtree_fork_choice,
             );
 
@@ -1387,7 +1386,7 @@ pub mod test {
                 .clone();
 
             // Try to vote on the given slot
-            let descendants = self.bank_forks.read().unwrap().descendants();
+            let descendants = self.bank_forks.read().unwrap().descendants().clone();
             let SelectVoteAndResetForkResult {
                 heaviest_fork_failures,
                 ..
@@ -1419,7 +1418,6 @@ pub mod test {
                 &self.bank_forks,
                 &mut self.progress,
                 &ABSRequestSender::default(),
-                &mut PubkeyReferences::default(),
                 None,
                 &mut self.heaviest_subtree_fork_choice,
             )
@@ -1461,7 +1459,7 @@ pub mod test {
                 .lockout_intervals
                 .entry(lockout_interval.1)
                 .or_default()
-                .push((lockout_interval.0, Rc::new(*vote_account_pubkey)));
+                .push((lockout_interval.0, *vote_account_pubkey));
         }
 
         fn can_progress_on_fork(
@@ -1584,7 +1582,7 @@ pub mod test {
                 vote_state.process_slot_vote_unchecked(*slot);
             }
             VoteState::serialize(
-                &VoteStateVersions::Current(Box::new(vote_state)),
+                &VoteStateVersions::new_current(vote_state),
                 &mut account.data,
             )
             .expect("serialize state");
@@ -1690,7 +1688,12 @@ pub mod test {
             fork_progress.fork_stats.computed = true;
         }
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let mut descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let mut descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
         let mut tower = Tower::new_with_key(&my_pubkey);
 
         // Last vote is 47
@@ -1821,7 +1824,12 @@ pub mod test {
         tower.lockouts.root_slot = Some(43);
         // Refresh ancestors and descendants for new root.
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
 
         assert_eq!(
             tower.check_switch_threshold(
@@ -1987,13 +1995,7 @@ pub mod test {
             bank_weight,
             pubkey_votes,
             ..
-        } = Tower::collect_vote_lockouts(
-            &Pubkey::default(),
-            1,
-            accounts.into_iter(),
-            &ancestors,
-            &mut PubkeyReferences::default(),
-        );
+        } = Tower::collect_vote_lockouts(&Pubkey::default(), 1, accounts.into_iter(), &ancestors);
         assert_eq!(voted_stakes[&0], 2);
         assert_eq!(total_stake, 2);
         let mut pubkey_votes = Arc::try_unwrap(pubkey_votes).unwrap();
@@ -2045,7 +2047,6 @@ pub mod test {
             MAX_LOCKOUT_HISTORY as u64,
             accounts.into_iter(),
             &ancestors,
-            &mut PubkeyReferences::default(),
         );
         for i in 0..MAX_LOCKOUT_HISTORY {
             assert_eq!(voted_stakes[&(i as u64)], 2);
@@ -2266,10 +2267,10 @@ pub mod test {
     #[test]
     fn test_new_vote() {
         let local = VoteState::default();
-        let vote = Tower::new_vote(&local, 0, Hash::default(), None);
+        let (vote, tower_slots) = Tower::new_vote(&local, 0, Hash::default(), None);
         assert_eq!(local.votes.len(), 0);
-        assert_eq!(vote.0.slots, vec![0]);
-        assert_eq!(vote.1, 0);
+        assert_eq!(vote.slots, vec![0]);
+        assert_eq!(tower_slots, vec![0]);
     }
 
     #[test]
@@ -2289,9 +2290,9 @@ pub mod test {
         };
         local.process_vote_unchecked(&vote);
         assert_eq!(local.votes.len(), 1);
-        let vote = Tower::new_vote(&local, 1, Hash::default(), Some(0));
-        assert_eq!(vote.0.slots, vec![1]);
-        assert_eq!(vote.1, 1);
+        let (vote, tower_slots) = Tower::new_vote(&local, 1, Hash::default(), Some(0));
+        assert_eq!(vote.slots, vec![1]);
+        assert_eq!(tower_slots, vec![0, 1]);
     }
 
     #[test]
@@ -2304,10 +2305,10 @@ pub mod test {
         };
         local.process_vote_unchecked(&vote);
         assert_eq!(local.votes.len(), 1);
-        let vote = Tower::new_vote(&local, 3, Hash::default(), Some(0));
-        //first vote expired, so index should be 0
-        assert_eq!(vote.0.slots, vec![3]);
-        assert_eq!(vote.1, 0);
+        let (vote, tower_slots) = Tower::new_vote(&local, 3, Hash::default(), Some(0));
+        assert_eq!(vote.slots, vec![3]);
+        // First vote expired, so should be evicted from tower.
+        assert_eq!(tower_slots, vec![3]);
     }
 
     #[test]
@@ -2352,7 +2353,6 @@ pub mod test {
             vote_to_evaluate,
             accounts.clone().into_iter(),
             &ancestors,
-            &mut PubkeyReferences::default(),
         );
         assert!(tower.check_vote_stake_threshold(vote_to_evaluate, &voted_stakes, total_stake,));
 
@@ -2369,7 +2369,6 @@ pub mod test {
             vote_to_evaluate,
             accounts.into_iter(),
             &ancestors,
-            &mut PubkeyReferences::default(),
         );
         assert!(!tower.check_vote_stake_threshold(vote_to_evaluate, &voted_stakes, total_stake,));
     }
@@ -2481,7 +2480,12 @@ pub mod test {
         }
 
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
         let mut tower = Tower::new_with_key(&my_pubkey);
 
         tower.record_vote(43, Hash::default());
@@ -2573,7 +2577,12 @@ pub mod test {
         let mut slot_history = SlotHistory::default();
         vote_simulator.set_root(replayed_root_slot);
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
         for slot in &[0, 1, 2, 43, replayed_root_slot] {
             slot_history.add(*slot);
         }

core/src/crds.rs

@@ -28,7 +28,7 @@ use crate::contact_info::ContactInfo;
 use crate::crds_shards::CrdsShards;
 use crate::crds_value::{CrdsData, CrdsValue, CrdsValueLabel, LowestSlot};
 use bincode::serialize;
-use indexmap::map::{rayon::ParValues, Entry, IndexMap, Iter, Values};
+use indexmap::map::{rayon::ParValues, Entry, IndexMap, Values};
 use indexmap::set::IndexSet;
 use rayon::{prelude::*, ThreadPool};
 use solana_sdk::hash::{hash, Hash};
@@ -40,6 +40,9 @@ use std::collections::{hash_map, HashMap};
 use std::ops::{Index, IndexMut};
 
 const CRDS_SHARDS_BITS: u32 = 8;
+// Limit number of crds values associated with each unique pubkey. This
+// excludes crds values which by label design are limited per each pubkey.
+const MAX_CRDS_VALUES_PER_PUBKEY: usize = 32;
 
 #[derive(Clone)]
 pub struct Crds {
@@ -47,8 +50,8 @@ pub struct Crds {
     table: IndexMap<CrdsValueLabel, VersionedCrdsValue>,
     pub num_inserts: usize, // Only used in tests.
     shards: CrdsShards,
-    // Indices of all crds values which are node ContactInfo.
-    nodes: IndexSet<usize>,
+    nodes: IndexSet<usize>, // Indices of nodes' ContactInfo.
+    votes: IndexSet<usize>, // Indices of Vote crds values.
     // Indices of all crds values associated with a node.
     records: HashMap<Pubkey, IndexSet<usize>>,
 }
@@ -109,6 +112,7 @@ impl Default for Crds {
             num_inserts: 0,
             shards: CrdsShards::new(CRDS_SHARDS_BITS),
             nodes: IndexSet::default(),
+            votes: IndexSet::default(),
             records: HashMap::default(),
         }
     }
@@ -141,9 +145,15 @@ impl Crds {
             Entry::Vacant(entry) => {
                 let entry_index = entry.index();
                 self.shards.insert(entry_index, &new_value);
-                if let CrdsData::ContactInfo(_) = new_value.value.data {
-                    self.nodes.insert(entry_index);
-                }
+                match new_value.value.data {
+                    CrdsData::ContactInfo(_) => {
+                        self.nodes.insert(entry_index);
+                    }
+                    CrdsData::Vote(_, _) => {
+                        self.votes.insert(entry_index);
+                    }
+                    _ => (),
+                };
                 self.records
                     .entry(new_value.value.pubkey())
                     .or_default()
@@ -215,6 +225,20 @@ impl Crds {
         })
     }
 
+    /// Returns all entries which are Vote.
+    pub(crate) fn get_votes(&self) -> impl Iterator<Item = &VersionedCrdsValue> {
+        self.votes.iter().map(move |i| self.table.index(*i))
+    }
+
+    /// Returns all records associated with a pubkey.
+    pub(crate) fn get_records(&self, pubkey: &Pubkey) -> impl Iterator<Item = &VersionedCrdsValue> {
+        self.records
+            .get(pubkey)
+            .into_iter()
+            .flat_map(|records| records.into_iter())
+            .map(move |i| self.table.index(*i))
+    }
+
     pub fn len(&self) -> usize {
         self.table.len()
     }
@@ -223,10 +247,6 @@ impl Crds {
         self.table.is_empty()
     }
 
-    pub fn iter(&self) -> Iter<'_, CrdsValueLabel, VersionedCrdsValue> {
-        self.table.iter()
-    }
-
     pub fn values(&self) -> Values<'_, CrdsValueLabel, VersionedCrdsValue> {
         self.table.values()
     }
@@ -267,21 +287,59 @@ impl Crds {
         now: u64,
         timeouts: &HashMap<Pubkey, u64>,
     ) -> Vec<CrdsValueLabel> {
+        #[rustversion::before(1.49.0)]
+        fn select_nth<T: Ord>(xs: &mut Vec<T>, _nth: usize) {
+            xs.sort_unstable();
+        }
+        #[rustversion::since(1.49.0)]
+        fn select_nth<T: Ord>(xs: &mut Vec<T>, nth: usize) {
+            xs.select_nth_unstable(nth);
+        }
         let default_timeout = *timeouts
             .get(&Pubkey::default())
             .expect("must have default timeout");
-        thread_pool.install(|| {
-            self.table
-                .par_iter()
-                .with_min_len(1024)
-                .filter_map(|(k, v)| {
-                    let timeout = timeouts.get(&k.pubkey()).unwrap_or(&default_timeout);
-                    if v.local_timestamp.saturating_add(*timeout) <= now {
-                        Some(k.clone())
-                    } else {
+        // Given an index of all crd values associated with a pubkey,
+        // returns crds labels of old values to be evicted.
+        let evict = |pubkey, index: &IndexSet<usize>| {
+            let timeout = *timeouts.get(pubkey).unwrap_or(&default_timeout);
+            let mut old_labels = Vec::new();
+            // Buffer of crds values to be evicted based on their wallclock.
+            let mut recent_unlimited_labels: Vec<(u64 /*wallclock*/, usize /*index*/)> = index
+                .into_iter()
+                .filter_map(|ix| {
+                    let (label, value) = self.table.get_index(*ix).unwrap();
+                    if value.local_timestamp.saturating_add(timeout) <= now {
+                        old_labels.push(label.clone());
                         None
+                    } else {
+                        match label.value_space() {
+                            Some(_) => None,
+                            None => Some((value.value.wallclock(), *ix)),
+                        }
                     }
                 })
+                .collect();
+            // Number of values to discard from the buffer:
+            let nth = recent_unlimited_labels
+                .len()
+                .saturating_sub(MAX_CRDS_VALUES_PER_PUBKEY);
+            // Partition on wallclock to discard the older ones.
+            if nth > 0 && nth < recent_unlimited_labels.len() {
+                select_nth(&mut recent_unlimited_labels, nth);
+            }
+            old_labels.extend(
+                recent_unlimited_labels
+                    .split_at(nth)
+                    .0
+                    .iter()
+                    .map(|(_ /*wallclock*/, ix)| self.table.get_index(*ix).unwrap().0.clone()),
+            );
+            old_labels
+        };
+        thread_pool.install(|| {
+            self.records
+                .par_iter()
+                .flat_map(|(pubkey, index)| evict(pubkey, index))
                 .collect()
         })
     }
@@ -289,8 +347,14 @@ impl Crds {
     pub fn remove(&mut self, key: &CrdsValueLabel) -> Option<VersionedCrdsValue> {
         let (index, _ /*label*/, value) = self.table.swap_remove_full(key)?;
         self.shards.remove(index, &value);
-        if let CrdsData::ContactInfo(_) = value.value.data {
-            self.nodes.swap_remove(&index);
+        match value.value.data {
+            CrdsData::ContactInfo(_) => {
+                self.nodes.swap_remove(&index);
+            }
+            CrdsData::Vote(_, _) => {
+                self.votes.swap_remove(&index);
+            }
+            _ => (),
         }
         // Remove the index from records associated with the value's pubkey.
         let pubkey = value.value.pubkey();
@@ -312,10 +376,17 @@ impl Crds {
             let value = self.table.index(index);
             self.shards.remove(size, value);
             self.shards.insert(index, value);
-            if let CrdsData::ContactInfo(_) = value.value.data {
-                self.nodes.swap_remove(&size);
-                self.nodes.insert(index);
-            }
+            match value.value.data {
+                CrdsData::ContactInfo(_) => {
+                    self.nodes.swap_remove(&size);
+                    self.nodes.insert(index);
+                }
+                CrdsData::Vote(_, _) => {
+                    self.votes.swap_remove(&size);
+                    self.votes.insert(index);
+                }
+                _ => (),
+            };
             let pubkey = value.value.pubkey();
             let records = self.records.get_mut(&pubkey).unwrap();
             records.swap_remove(&size);
@@ -328,10 +399,10 @@ impl Crds {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::contact_info::ContactInfo;
+    use crate::{contact_info::ContactInfo, crds_value::NodeInstance};
     use rand::{thread_rng, Rng};
     use rayon::ThreadPoolBuilder;
-    use std::iter::repeat_with;
+    use std::{collections::HashSet, iter::repeat_with};
 
     #[test]
     fn test_insert() {
@@ -445,6 +516,41 @@ mod test {
             vec![val.label()]
         );
     }
+
+    #[test]
+    fn test_find_old_records_unlimited() {
+        let thread_pool = ThreadPoolBuilder::new().build().unwrap();
+        let mut rng = thread_rng();
+        let now = 1_610_034_423_000;
+        let pubkey = Pubkey::new_unique();
+        let mut crds = Crds::default();
+        let mut timeouts = HashMap::new();
+        timeouts.insert(Pubkey::default(), 1);
+        timeouts.insert(pubkey, 180);
+        for _ in 0..1024 {
+            let wallclock = now - rng.gen_range(0, 240);
+            let val = NodeInstance::new(&mut rng, pubkey, wallclock);
+            let val = CrdsData::NodeInstance(val);
+            let val = CrdsValue::new_unsigned(val);
+            assert_eq!(crds.insert(val, now), Ok(None));
+        }
+        let now = now + 1;
+        let labels = crds.find_old_labels(&thread_pool, now, &timeouts);
+        assert_eq!(crds.table.len() - labels.len(), MAX_CRDS_VALUES_PER_PUBKEY);
+        let max_wallclock = labels
+            .iter()
+            .map(|label| crds.lookup(label).unwrap().wallclock())
+            .max()
+            .unwrap();
+        assert!(max_wallclock > now - 180);
+        let labels: HashSet<_> = labels.into_iter().collect();
+        for (label, value) in crds.table.iter() {
+            if !labels.contains(label) {
+                assert!(max_wallclock <= value.value.wallclock());
+            }
+        }
+    }
+
     #[test]
     fn test_remove_default() {
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
@@ -536,51 +642,67 @@ mod test {
     }
 
     #[test]
-    fn test_crds_nodes() {
-        fn check_crds_nodes(crds: &Crds) -> usize {
+    fn test_crds_value_indices() {
+        fn check_crds_value_indices(crds: &Crds) -> (usize, usize) {
             let num_nodes = crds
                 .table
                 .values()
                 .filter(|value| matches!(value.value.data, CrdsData::ContactInfo(_)))
                 .count();
+            let num_votes = crds
+                .table
+                .values()
+                .filter(|value| matches!(value.value.data, CrdsData::Vote(_, _)))
+                .count();
             assert_eq!(num_nodes, crds.get_nodes_contact_info().count());
-            num_nodes
+            assert_eq!(num_votes, crds.get_votes().count());
+            for vote in crds.get_votes() {
+                match vote.value.data {
+                    CrdsData::Vote(_, _) => (),
+                    _ => panic!("not a vote!"),
+                }
+            }
+            (num_nodes, num_votes)
         }
         let mut rng = thread_rng();
-        let keypairs: Vec<_> = std::iter::repeat_with(Keypair::new).take(256).collect();
+        let keypairs: Vec<_> = repeat_with(Keypair::new).take(128).collect();
         let mut crds = Crds::default();
         let mut num_inserts = 0;
         let mut num_overrides = 0;
-        for _ in 0..4096 {
+        for k in 0..4096 {
             let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
             let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
             match crds.insert_versioned(value) {
                 Ok(None) => {
                     num_inserts += 1;
-                    check_crds_nodes(&crds);
                 }
                 Ok(Some(_)) => {
                     num_inserts += 1;
                     num_overrides += 1;
-                    check_crds_nodes(&crds);
                 }
                 Err(_) => (),
             }
+            if k % 64 == 0 {
+                check_crds_value_indices(&crds);
+            }
         }
         assert_eq!(num_inserts, crds.num_inserts);
         assert!(num_inserts > 700);
         assert!(num_overrides > 500);
         assert!(crds.table.len() > 200);
         assert!(num_inserts > crds.table.len());
-        let num_nodes = check_crds_nodes(&crds);
+        let (num_nodes, num_votes) = check_crds_value_indices(&crds);
         assert!(num_nodes * 3 < crds.table.len());
-        assert!(num_nodes > 150);
+        assert!(num_nodes > 100, "num nodes: {}", num_nodes);
+        assert!(num_votes > 100, "num votes: {}", num_votes);
         // Remove values one by one and assert that nodes indices stay valid.
         while !crds.table.is_empty() {
             let index = rng.gen_range(0, crds.table.len());
             let key = crds.table.get_index(index).unwrap().0.clone();
             crds.remove(&key);
-            check_crds_nodes(&crds);
+            if crds.table.len() % 64 == 0 {
+                check_crds_value_indices(&crds);
+            }
         }
     }
 

core/src/crds_gossip.rs

@@ -8,10 +8,17 @@ use crate::{
     crds_gossip_error::CrdsGossipError,
     crds_gossip_pull::{CrdsFilter, CrdsGossipPull, ProcessPullStats},
     crds_gossip_push::{CrdsGossipPush, CRDS_GOSSIP_NUM_ACTIVE},
-    crds_value::{CrdsValue, CrdsValueLabel},
+    crds_value::{CrdsData, CrdsValue, CrdsValueLabel},
+    duplicate_shred::{self, DuplicateShredIndex, LeaderScheduleFn, MAX_DUPLICATE_SHREDS},
 };
 use rayon::ThreadPool;
-use solana_sdk::{hash::Hash, pubkey::Pubkey};
+use solana_ledger::shred::Shred;
+use solana_sdk::{
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::{Keypair, Signer},
+    timing::timestamp,
+};
 use std::collections::{HashMap, HashSet};
 
 ///The min size for bloom filters
@@ -105,6 +112,68 @@ impl CrdsGossip {
         (self.id, push_messages)
     }
 
+    pub(crate) fn push_duplicate_shred(
+        &mut self,
+        keypair: &Keypair,
+        shred: &Shred,
+        other_payload: &[u8],
+        leader_schedule: Option<impl LeaderScheduleFn>,
+        // Maximum serialized size of each DuplicateShred chunk payload.
+        max_payload_size: usize,
+    ) -> Result<(), duplicate_shred::Error> {
+        let pubkey = keypair.pubkey();
+        // Skip if there are already records of duplicate shreds for this slot.
+        let shred_slot = shred.slot();
+        if self
+            .crds
+            .get_records(&pubkey)
+            .any(|value| match &value.value.data {
+                CrdsData::DuplicateShred(_, value) => value.slot == shred_slot,
+                _ => false,
+            })
+        {
+            return Ok(());
+        }
+        let chunks = duplicate_shred::from_shred(
+            shred.clone(),
+            pubkey,
+            Vec::from(other_payload),
+            leader_schedule,
+            timestamp(),
+            max_payload_size,
+        )?;
+        // Find the index of oldest duplicate shred.
+        let mut num_dup_shreds = 0;
+        let offset = self
+            .crds
+            .get_records(&pubkey)
+            .filter_map(|value| match &value.value.data {
+                CrdsData::DuplicateShred(ix, value) => {
+                    num_dup_shreds += 1;
+                    Some((value.wallclock, *ix))
+                }
+                _ => None,
+            })
+            .min() // Override the oldest records.
+            .map(|(_ /*wallclock*/, ix)| ix)
+            .unwrap_or(0);
+        let offset = if num_dup_shreds < MAX_DUPLICATE_SHREDS {
+            num_dup_shreds
+        } else {
+            offset
+        };
+        let entries = chunks
+            .enumerate()
+            .map(|(k, chunk)| {
+                let index = (offset + k as DuplicateShredIndex) % MAX_DUPLICATE_SHREDS;
+                let data = CrdsData::DuplicateShred(index, chunk);
+                CrdsValue::new_signed(data, keypair)
+            })
+            .collect();
+        self.process_push_message(&pubkey, entries, timestamp());
+        Ok(())
+    }
+
     /// add the `from` to the peer's filter of nodes
     pub fn process_prune_msg(
         &self,
@@ -184,9 +253,11 @@ impl CrdsGossip {
     pub fn generate_pull_responses(
         &self,
         filters: &[(CrdsValue, CrdsFilter)],
+        output_size_limit: usize, // Limit number of crds values returned.
         now: u64,
     ) -> Vec<Vec<CrdsValue>> {
-        self.pull.generate_pull_responses(&self.crds, filters, now)
+        self.pull
+            .generate_pull_responses(&self.crds, filters, output_size_limit, now)
     }
 
     pub fn filter_pull_responses(

core/src/crds_gossip_pull.rs

@@ -14,6 +14,7 @@ use crate::crds::{Crds, VersionedCrdsValue};
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
+use itertools::Itertools;
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::Rng;
 use rayon::{prelude::*, ThreadPool};
@@ -144,10 +145,10 @@ impl CrdsFilterSet {
     }
 }
 
-impl Into<Vec<CrdsFilter>> for CrdsFilterSet {
-    fn into(self) -> Vec<CrdsFilter> {
-        let mask_bits = self.mask_bits;
-        self.filters
+impl From<CrdsFilterSet> for Vec<CrdsFilter> {
+    fn from(cfs: CrdsFilterSet) -> Self {
+        let mask_bits = cfs.mask_bits;
+        cfs.filters
             .into_iter()
             .enumerate()
             .map(|(seed, filter)| CrdsFilter {
@@ -304,9 +305,10 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         requests: &[(CrdsValue, CrdsFilter)],
+        output_size_limit: usize, // Limit number of crds values returned.
         now: u64,
     ) -> Vec<Vec<CrdsValue>> {
-        self.filter_crds_values(crds, requests, now)
+        self.filter_crds_values(crds, requests, output_size_limit, now)
     }
 
     // Checks if responses should be inserted and
@@ -474,6 +476,7 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         filters: &[(CrdsValue, CrdsFilter)],
+        mut output_size_limit: usize, // Limit number of crds values returned.
         now: u64,
     ) -> Vec<Vec<CrdsValue>> {
         let msg_timeout = CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
@@ -483,16 +486,20 @@ impl CrdsGossipPull {
         let past = now.saturating_sub(msg_timeout);
         let mut dropped_requests = 0;
         let mut total_skipped = 0;
-        let ret = filters
+        let ret: Vec<_> = filters
             .iter()
             .map(|(caller, filter)| {
+                if output_size_limit == 0 {
+                    return None;
+                }
                 let caller_wallclock = caller.wallclock();
                 if caller_wallclock >= future || caller_wallclock < past {
                     dropped_requests += 1;
-                    return vec![];
+                    return Some(vec![]);
                 }
                 let caller_wallclock = caller_wallclock.checked_add(jitter).unwrap_or(0);
-                crds.filter_bitmask(filter.mask, filter.mask_bits)
+                let out: Vec<_> = crds
+                    .filter_bitmask(filter.mask, filter.mask_bits)
                     .filter_map(|item| {
                         debug_assert!(filter.test_mask(&item.value_hash));
                         //skip values that are too new
@@ -505,12 +512,16 @@ impl CrdsGossipPull {
                             Some(item.value.clone())
                         }
                     })
-                    .collect()
+                    .take(output_size_limit)
+                    .collect();
+                output_size_limit -= out.len();
+                Some(out)
             })
+            .while_some()
             .collect();
         inc_new_counter_info!(
             "gossip_filter_crds_values-dropped_requests",
-            dropped_requests
+            dropped_requests + filters.len() - ret.len()
         );
         inc_new_counter_info!("gossip_filter_crds_values-dropped_values", total_skipped);
         ret
@@ -1029,7 +1040,12 @@ mod test {
         let dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
         let mut filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            0,
+        );
 
         assert_eq!(rsp[0].len(), 0);
 
@@ -1042,8 +1058,12 @@ mod test {
             .unwrap();
 
         //should skip new value since caller is to old
-        let rsp =
-            dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS,
+        );
         assert_eq!(rsp[0].len(), 0);
 
         assert_eq!(filters.len(), 1);
@@ -1054,8 +1074,12 @@ mod test {
             CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS + 1,
         )));
 
-        let rsp =
-            dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS,
+        );
         assert_eq!(rsp.len(), 2);
         assert_eq!(rsp[0].len(), 0);
         assert_eq!(rsp[1].len(), 1); // Orders are also preserved.
@@ -1092,7 +1116,12 @@ mod test {
         let mut dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
         let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            0,
+        );
         dest.process_pull_requests(
             &mut dest_crds,
             filters.into_iter().map(|(caller, _)| caller),
@@ -1170,7 +1199,12 @@ mod test {
             );
             let (_, filters, caller) = req.unwrap();
             let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-            let mut rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
+            let mut rsp = dest.generate_pull_responses(
+                &dest_crds,
+                &filters,
+                /*output_size_limit=*/ usize::MAX,
+                0,
+            );
             dest.process_pull_requests(
                 &mut dest_crds,
                 filters.into_iter().map(|(caller, _)| caller),
@@ -1394,7 +1428,7 @@ mod test {
 
         // construct something that's not a contact info
         let peer_vote =
-            CrdsValue::new_unsigned(CrdsData::Vote(0, Vote::new(&peer_pubkey, test_tx(), 0)));
+            CrdsValue::new_unsigned(CrdsData::Vote(0, Vote::new(peer_pubkey, test_tx(), 0)));
         // check that older CrdsValues (non-ContactInfos) infos pass even if are too old,
         // but a recent contact info (inserted above) exists
         assert_eq!(

core/src/crds_value.rs

@@ -1,9 +1,13 @@
-use crate::cluster_info::MAX_SNAPSHOT_HASHES;
-use crate::contact_info::ContactInfo;
-use crate::deprecated;
-use crate::epoch_slots::EpochSlots;
+use crate::{
+    cluster_info::MAX_SNAPSHOT_HASHES,
+    contact_info::ContactInfo,
+    deprecated,
+    duplicate_shred::{DuplicateShred, DuplicateShredIndex, MAX_DUPLICATE_SHREDS},
+    epoch_slots::EpochSlots,
+};
 use bincode::{serialize, serialized_size};
 use rand::{CryptoRng, Rng};
+use serde::de::{Deserialize, Deserializer};
 use solana_sdk::sanitize::{Sanitize, SanitizeError};
 use solana_sdk::timing::timestamp;
 use solana_sdk::{
@@ -13,9 +17,10 @@ use solana_sdk::{
     signature::{Keypair, Signable, Signature, Signer},
     transaction::Transaction,
 };
+use solana_vote_program::vote_transaction::parse_vote_transaction;
 use std::{
     borrow::{Borrow, Cow},
-    collections::{hash_map::Entry, BTreeSet, HashMap, HashSet},
+    collections::{hash_map::Entry, BTreeSet, HashMap},
     fmt,
 };
 
@@ -23,6 +28,8 @@ pub const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000;
 pub const MAX_SLOT: u64 = 1_000_000_000_000_000;
 
 pub type VoteIndex = u8;
+// TODO: Remove this in favor of vote_state::MAX_LOCKOUT_HISTORY once
+// the fleet is updated to the new ClusterInfo::push_vote code.
 pub const MAX_VOTES: VoteIndex = 32;
 
 pub type EpochSlotsIndex = u8;
@@ -80,6 +87,7 @@ pub enum CrdsData {
     LegacyVersion(LegacyVersion),
     Version(Version),
     NodeInstance(NodeInstance),
+    DuplicateShred(DuplicateShredIndex, DuplicateShred),
 }
 
 impl Sanitize for CrdsData {
@@ -109,6 +117,13 @@ impl Sanitize for CrdsData {
             CrdsData::LegacyVersion(version) => version.sanitize(),
             CrdsData::Version(version) => version.sanitize(),
             CrdsData::NodeInstance(node) => node.sanitize(),
+            CrdsData::DuplicateShred(ix, shred) => {
+                if *ix >= MAX_DUPLICATE_SHREDS {
+                    Err(SanitizeError::ValueOutOfBounds)
+                } else {
+                    shred.sanitize()
+                }
+            }
         }
     }
 }
@@ -122,7 +137,7 @@ pub(crate) fn new_rand_timestamp<R: Rng>(rng: &mut R) -> u64 {
 impl CrdsData {
     /// New random CrdsData for tests and benchmarks.
     fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> CrdsData {
-        let kind = rng.gen_range(0, 5);
+        let kind = rng.gen_range(0, 6);
         // TODO: Implement other kinds of CrdsData here.
         // TODO: Assign ranges to each arm proportional to their frequency in
         // the mainnet crds table.
@@ -131,7 +146,8 @@ impl CrdsData {
             1 => CrdsData::LowestSlot(rng.gen(), LowestSlot::new_rand(rng, pubkey)),
             2 => CrdsData::SnapshotHashes(SnapshotHash::new_rand(rng, pubkey)),
             3 => CrdsData::AccountsHashes(SnapshotHash::new_rand(rng, pubkey)),
-            _ => CrdsData::Version(Version::new_rand(rng, pubkey)),
+            4 => CrdsData::Version(Version::new_rand(rng, pubkey)),
+            _ => CrdsData::Vote(rng.gen_range(0, MAX_VOTES), Vote::new_rand(rng, pubkey)),
         }
     }
 }
@@ -145,9 +161,7 @@ pub struct SnapshotHash {
 
 impl Sanitize for SnapshotHash {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         for (slot, _) in &self.hashes {
             if *slot >= MAX_SLOT {
                 return Err(SanitizeError::ValueOutOfBounds);
@@ -220,9 +234,7 @@ impl LowestSlot {
 
 impl Sanitize for LowestSlot {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         if self.lowest >= MAX_SLOT {
             return Err(SanitizeError::ValueOutOfBounds);
         }
@@ -239,31 +251,77 @@ impl Sanitize for LowestSlot {
     }
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, AbiExample)]
+#[derive(Clone, Debug, PartialEq, AbiExample, Serialize)]
 pub struct Vote {
-    pub from: Pubkey,
-    pub transaction: Transaction,
-    pub wallclock: u64,
+    pub(crate) from: Pubkey,
+    transaction: Transaction,
+    pub(crate) wallclock: u64,
+    #[serde(skip_serializing)]
+    slot: Option<Slot>,
 }
 
 impl Sanitize for Vote {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()?;
         self.transaction.sanitize()
     }
 }
 
 impl Vote {
-    pub fn new(from: &Pubkey, transaction: Transaction, wallclock: u64) -> Self {
+    pub fn new(from: Pubkey, transaction: Transaction, wallclock: u64) -> Self {
+        let slot = parse_vote_transaction(&transaction)
+            .and_then(|(_, vote, _)| vote.slots.last().copied());
         Self {
-            from: *from,
+            from,
             transaction,
             wallclock,
+            slot,
         }
     }
+
+    /// New random Vote for tests and benchmarks.
+    fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
+        Self {
+            from: pubkey.unwrap_or_else(pubkey::new_rand),
+            transaction: Transaction::default(),
+            wallclock: new_rand_timestamp(rng),
+            slot: None,
+        }
+    }
+
+    pub(crate) fn transaction(&self) -> &Transaction {
+        &self.transaction
+    }
+
+    pub(crate) fn slot(&self) -> Option<Slot> {
+        self.slot
+    }
+}
+
+impl<'de> Deserialize<'de> for Vote {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        struct Vote {
+            from: Pubkey,
+            transaction: Transaction,
+            wallclock: u64,
+        }
+        let vote = Vote::deserialize(deserializer)?;
+        let vote = match vote.transaction.sanitize() {
+            Ok(_) => Self::new(vote.from, vote.transaction, vote.wallclock),
+            Err(_) => Self {
+                from: vote.from,
+                transaction: vote.transaction,
+                wallclock: vote.wallclock,
+                slot: None,
+            },
+        };
+        Ok(vote)
+    }
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, AbiExample)]
@@ -275,9 +333,7 @@ pub struct LegacyVersion {
 
 impl Sanitize for LegacyVersion {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()?;
         self.version.sanitize()
     }
@@ -292,9 +348,7 @@ pub struct Version {
 
 impl Sanitize for Version {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()?;
         self.version.sanitize()
     }
@@ -370,9 +424,7 @@ impl NodeInstance {
 
 impl Sanitize for NodeInstance {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()
     }
 }
@@ -390,6 +442,7 @@ pub enum CrdsValueLabel {
     LegacyVersion(Pubkey),
     Version(Pubkey),
     NodeInstance(Pubkey, u64 /*token*/),
+    DuplicateShred(DuplicateShredIndex, Pubkey),
 }
 
 impl fmt::Display for CrdsValueLabel {
@@ -403,7 +456,8 @@ impl fmt::Display for CrdsValueLabel {
             CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
             CrdsValueLabel::LegacyVersion(_) => write!(f, "LegacyVersion({})", self.pubkey()),
             CrdsValueLabel::Version(_) => write!(f, "Version({})", self.pubkey()),
-            CrdsValueLabel::NodeInstance(_, _) => write!(f, "NodeInstance({})", self.pubkey()),
+            CrdsValueLabel::NodeInstance(pk, token) => write!(f, "NodeInstance({}, {})", pk, token),
+            CrdsValueLabel::DuplicateShred(ix, pk) => write!(f, "DuplicateShred({}, {})", ix, pk),
         }
     }
 }
@@ -420,6 +474,24 @@ impl CrdsValueLabel {
             CrdsValueLabel::LegacyVersion(p) => *p,
             CrdsValueLabel::Version(p) => *p,
             CrdsValueLabel::NodeInstance(p, _ /*token*/) => *p,
+            CrdsValueLabel::DuplicateShred(_, p) => *p,
+        }
+    }
+
+    /// Returns number of possible distinct labels of the same type for
+    /// a fixed pubkey, and None if that is practically unlimited.
+    pub(crate) fn value_space(&self) -> Option<usize> {
+        match self {
+            CrdsValueLabel::ContactInfo(_) => Some(1),
+            CrdsValueLabel::Vote(_, _) => Some(MAX_VOTES as usize),
+            CrdsValueLabel::LowestSlot(_) => Some(1),
+            CrdsValueLabel::SnapshotHashes(_) => Some(1),
+            CrdsValueLabel::EpochSlots(_, _) => Some(MAX_EPOCH_SLOTS as usize),
+            CrdsValueLabel::AccountsHashes(_) => Some(1),
+            CrdsValueLabel::LegacyVersion(_) => Some(1),
+            CrdsValueLabel::Version(_) => Some(1),
+            CrdsValueLabel::NodeInstance(_, _) => None,
+            CrdsValueLabel::DuplicateShred(_, _) => Some(MAX_DUPLICATE_SHREDS as usize),
         }
     }
 }
@@ -467,6 +539,7 @@ impl CrdsValue {
             CrdsData::LegacyVersion(version) => version.wallclock,
             CrdsData::Version(version) => version.wallclock,
             CrdsData::NodeInstance(node) => node.wallclock,
+            CrdsData::DuplicateShred(_, shred) => shred.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
@@ -480,6 +553,7 @@ impl CrdsValue {
             CrdsData::LegacyVersion(version) => version.from,
             CrdsData::Version(version) => version.from,
             CrdsData::NodeInstance(node) => node.from,
+            CrdsData::DuplicateShred(_, shred) => shred.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
@@ -492,7 +566,8 @@ impl CrdsValue {
             CrdsData::EpochSlots(ix, _) => CrdsValueLabel::EpochSlots(*ix, self.pubkey()),
             CrdsData::LegacyVersion(_) => CrdsValueLabel::LegacyVersion(self.pubkey()),
             CrdsData::Version(_) => CrdsValueLabel::Version(self.pubkey()),
-            CrdsData::NodeInstance(node) => CrdsValueLabel::NodeInstance(self.pubkey(), node.token),
+            CrdsData::NodeInstance(node) => CrdsValueLabel::NodeInstance(node.from, node.token),
+            CrdsData::DuplicateShred(ix, shred) => CrdsValueLabel::DuplicateShred(*ix, shred.from),
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -501,20 +576,15 @@ impl CrdsValue {
             _ => None,
         }
     }
-    pub fn vote(&self) -> Option<&Vote> {
+
+    #[cfg(test)]
+    fn vote(&self) -> Option<&Vote> {
         match &self.data {
             CrdsData::Vote(_, vote) => Some(vote),
             _ => None,
         }
     }
 
-    pub fn vote_index(&self) -> Option<VoteIndex> {
-        match &self.data {
-            CrdsData::Vote(ix, _) => Some(*ix),
-            _ => None,
-        }
-    }
-
     pub fn lowest_slot(&self) -> Option<&LowestSlot> {
         match &self.data {
             CrdsData::LowestSlot(_, slots) => Some(slots),
@@ -562,33 +632,6 @@ impl CrdsValue {
         serialized_size(&self).expect("unable to serialize contact info")
     }
 
-    pub fn compute_vote_index(tower_index: usize, mut votes: Vec<&CrdsValue>) -> VoteIndex {
-        let mut available: HashSet<VoteIndex> = (0..MAX_VOTES).collect();
-        votes.iter().filter_map(|v| v.vote_index()).for_each(|ix| {
-            available.remove(&ix);
-        });
-
-        // free index
-        if !available.is_empty() {
-            return *available.iter().next().unwrap();
-        }
-
-        assert!(votes.len() == MAX_VOTES as usize);
-        votes.sort_by_key(|v| v.vote().expect("all values must be votes").wallclock);
-
-        // If Tower is full, oldest removed first
-        if tower_index + 1 == MAX_VOTES as usize {
-            return votes[0].vote_index().expect("all values must be votes");
-        }
-
-        // If Tower is not full, the early votes have expired
-        assert!(tower_index < MAX_VOTES as usize);
-
-        votes[tower_index]
-            .vote_index()
-            .expect("all values must be votes")
-    }
-
     /// Returns true if, regardless of prunes, this crds-value
     /// should be pushed to the receiving node.
     pub fn should_force_push(&self, peer: &Pubkey) -> bool {
@@ -623,14 +666,25 @@ where
     out.into_iter().map(|(_, (v, _))| v)
 }
 
+pub(crate) fn sanitize_wallclock(wallclock: u64) -> Result<(), SanitizeError> {
+    if wallclock >= MAX_WALLCLOCK {
+        Err(SanitizeError::ValueOutOfBounds)
+    } else {
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
     use crate::contact_info::ContactInfo;
-    use bincode::deserialize;
+    use bincode::{deserialize, Options};
+    use rand::SeedableRng;
+    use rand_chacha::ChaChaRng;
     use solana_perf::test_tx::test_tx;
     use solana_sdk::signature::{Keypair, Signer};
     use solana_sdk::timing::timestamp;
+    use solana_vote_program::{vote_instruction, vote_state};
     use std::cmp::Ordering;
     use std::iter::repeat_with;
 
@@ -643,7 +697,7 @@ mod test {
 
         let v = CrdsValue::new_unsigned(CrdsData::Vote(
             0,
-            Vote::new(&Pubkey::default(), test_tx(), 0),
+            Vote::new(Pubkey::default(), test_tx(), 0),
         ));
         assert_eq!(v.wallclock(), 0);
         let key = v.vote().unwrap().from;
@@ -695,7 +749,7 @@ mod test {
         verify_signatures(&mut v, &keypair, &wrong_keypair);
         v = CrdsValue::new_unsigned(CrdsData::Vote(
             0,
-            Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
+            Vote::new(keypair.pubkey(), test_tx(), timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
         v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
@@ -711,13 +765,45 @@ mod test {
         let vote = CrdsValue::new_signed(
             CrdsData::Vote(
                 MAX_VOTES,
-                Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
+                Vote::new(keypair.pubkey(), test_tx(), timestamp()),
             ),
             &keypair,
         );
         assert!(vote.sanitize().is_err());
     }
 
+    #[test]
+    fn test_vote_round_trip() {
+        let mut rng = rand::thread_rng();
+        let vote = vote_state::Vote::new(
+            vec![1, 3, 7], // slots
+            solana_sdk::hash::new_rand(&mut rng),
+        );
+        let ix = vote_instruction::vote(
+            &Pubkey::new_unique(), // vote_pubkey
+            &Pubkey::new_unique(), // authorized_voter_pubkey
+            vote,
+        );
+        let tx = Transaction::new_with_payer(
+            &[ix],                       // instructions
+            Some(&Pubkey::new_unique()), // payer
+        );
+        let vote = Vote::new(
+            Pubkey::new_unique(), // from
+            tx,
+            rng.gen(), // wallclock
+        );
+        assert_eq!(vote.slot, Some(7));
+        let bytes = bincode::serialize(&vote).unwrap();
+        let other = bincode::deserialize(&bytes[..]).unwrap();
+        assert_eq!(vote, other);
+        assert_eq!(other.slot, Some(7));
+        let bytes = bincode::options().serialize(&vote).unwrap();
+        let other = bincode::options().deserialize(&bytes[..]).unwrap();
+        assert_eq!(vote, other);
+        assert_eq!(other.slot, Some(7));
+    }
+
     #[test]
     fn test_max_epoch_slots_index() {
         let keypair = Keypair::new();
@@ -730,49 +816,6 @@ mod test {
         );
         assert_eq!(item.sanitize(), Err(SanitizeError::ValueOutOfBounds));
     }
-    #[test]
-    fn test_compute_vote_index_empty() {
-        for i in 0..MAX_VOTES {
-            let votes = vec![];
-            assert!(CrdsValue::compute_vote_index(i as usize, votes) < MAX_VOTES);
-        }
-    }
-
-    #[test]
-    fn test_compute_vote_index_one() {
-        let keypair = Keypair::new();
-        let vote = CrdsValue::new_unsigned(CrdsData::Vote(
-            0,
-            Vote::new(&keypair.pubkey(), test_tx(), 0),
-        ));
-        for i in 0..MAX_VOTES {
-            let votes = vec![&vote];
-            assert!(CrdsValue::compute_vote_index(i as usize, votes) > 0);
-            let votes = vec![&vote];
-            assert!(CrdsValue::compute_vote_index(i as usize, votes) < MAX_VOTES);
-        }
-    }
-
-    #[test]
-    fn test_compute_vote_index_full() {
-        let keypair = Keypair::new();
-        let votes: Vec<_> = (0..MAX_VOTES)
-            .map(|x| {
-                CrdsValue::new_unsigned(CrdsData::Vote(
-                    x,
-                    Vote::new(&keypair.pubkey(), test_tx(), x as u64),
-                ))
-            })
-            .collect();
-        let vote_refs = votes.iter().collect();
-        //pick the oldest vote when full
-        assert_eq!(CrdsValue::compute_vote_index(31, vote_refs), 0);
-        //pick the index
-        let vote_refs = votes.iter().collect();
-        assert_eq!(CrdsValue::compute_vote_index(0, vote_refs), 0);
-        let vote_refs = votes.iter().collect();
-        assert_eq!(CrdsValue::compute_vote_index(30, vote_refs), 30);
-    }
 
     fn serialize_deserialize_value(value: &mut CrdsValue, keypair: &Keypair) {
         let num_tries = 10;
@@ -806,13 +849,14 @@ mod test {
 
     #[test]
     fn test_filter_current() {
-        let mut rng = rand::thread_rng();
+        let seed = [48u8; 32];
+        let mut rng = ChaChaRng::from_seed(seed);
         let keys: Vec<_> = repeat_with(Keypair::new).take(16).collect();
         let values: Vec<_> = repeat_with(|| {
             let index = rng.gen_range(0, keys.len());
             CrdsValue::new_rand(&mut rng, Some(&keys[index]))
         })
-        .take(256)
+        .take(2048)
         .collect();
         let mut currents = HashMap::new();
         for value in filter_current(&values) {
@@ -826,17 +870,20 @@ mod test {
             match value.wallclock().cmp(&current_value.wallclock()) {
                 Ordering::Less => (),
                 Ordering::Equal => {
-                    assert_eq!(value, *current_value);
-                    count += 1;
+                    // There is a chance that two randomly generated
+                    // crds-values have the same label and wallclock.
+                    if value == *current_value {
+                        count += 1;
+                    }
                 }
                 Ordering::Greater => panic!("this should not happen!"),
             }
         }
         assert_eq!(count, currents.len());
         // Currently CrdsData::new_rand is only implemented for 5 different
-        // kinds and excludes Vote and EpochSlots, and so the unique labels
-        // cannot be more than 5 times number of keys.
-        assert!(currents.len() <= keys.len() * 5);
+        // kinds and excludes EpochSlots, and so the unique labels cannot be
+        // more than (5 + MAX_VOTES) times number of keys.
+        assert!(currents.len() <= keys.len() * (5 + MAX_VOTES as usize));
     }
 
     #[test]

core/src/data_budget.rs

@@ -52,27 +52,28 @@ impl DataBudget {
         }
     }
 
-    // Updates the budget if at least given milliseconds has passed since last
-    // update. Updater function maps current value of bytes to the new one.
-    pub fn update<F>(&self, duration_millis: u64, updater: F)
+    /// Updates the budget if at least given milliseconds has passed since last
+    /// update. Updater function maps current value of bytes to the new one.
+    /// Returns current data-budget after the update.
+    pub fn update<F>(&self, duration_millis: u64, updater: F) -> usize
     where
         F: Fn(usize) -> usize,
     {
-        if !self.can_update(duration_millis) {
-            return;
-        }
-        let mut bytes = self.bytes.load(Ordering::Acquire);
-        loop {
-            match self.bytes.compare_exchange_weak(
-                bytes,
-                updater(bytes),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-            ) {
-                Ok(_) => break,
-                Err(b) => bytes = b,
+        if self.can_update(duration_millis) {
+            let mut bytes = self.bytes.load(Ordering::Acquire);
+            loop {
+                match self.bytes.compare_exchange_weak(
+                    bytes,
+                    updater(bytes),
+                    Ordering::AcqRel,
+                    Ordering::Acquire,
+                ) {
+                    Ok(_) => break,
+                    Err(b) => bytes = b,
+                }
             }
         }
+        self.bytes.load(Ordering::Acquire)
     }
 
     // Non-atomic clone only for tests and simulations.
@@ -94,16 +95,16 @@ mod tests {
         let budget = DataBudget::default();
         assert!(!budget.take(1)); // budget = 0.
 
-        budget.update(1000, |bytes| bytes + 5); // budget updates to 5.
+        assert_eq!(budget.update(1000, |bytes| bytes + 5), 5); // budget updates to 5.
         assert!(budget.take(1));
         assert!(budget.take(2));
         assert!(!budget.take(3)); // budget = 2, out of budget.
 
-        budget.update(30, |_| 10); // no update, budget = 2.
+        assert_eq!(budget.update(30, |_| 10), 2); // no update, budget = 2.
         assert!(!budget.take(3)); // budget = 2, out of budget.
 
         std::thread::sleep(Duration::from_millis(50));
-        budget.update(30, |bytes| bytes * 2); // budget updates to 4.
+        assert_eq!(budget.update(30, |bytes| bytes * 2), 4); // budget updates to 4.
 
         assert!(budget.take(3));
         assert!(budget.take(1));

core/src/duplicate_shred.rs

@@ -0,0 +1,385 @@
+use crate::crds_value::sanitize_wallclock;
+use itertools::Itertools;
+use solana_ledger::{
+    blockstore_meta::DuplicateSlotProof,
+    shred::{Shred, ShredError, ShredType},
+};
+use solana_sdk::{
+    clock::Slot,
+    pubkey::Pubkey,
+    sanitize::{Sanitize, SanitizeError},
+};
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    convert::TryFrom,
+    num::TryFromIntError,
+};
+use thiserror::Error;
+
+const DUPLICATE_SHRED_HEADER_SIZE: usize = 63;
+
+pub(crate) type DuplicateShredIndex = u16;
+pub(crate) const MAX_DUPLICATE_SHREDS: DuplicateShredIndex = 512;
+
+/// Function returning leader at a given slot.
+pub trait LeaderScheduleFn: FnOnce(Slot) -> Option<Pubkey> {}
+impl<F> LeaderScheduleFn for F where F: FnOnce(Slot) -> Option<Pubkey> {}
+
+#[derive(Clone, Debug, PartialEq, AbiExample, Deserialize, Serialize)]
+pub struct DuplicateShred {
+    pub(crate) from: Pubkey,
+    pub(crate) wallclock: u64,
+    pub(crate) slot: Slot,
+    shred_index: u32,
+    shred_type: ShredType,
+    // Serialized DuplicateSlotProof split into chunks.
+    num_chunks: u8,
+    chunk_index: u8,
+    #[serde(with = "serde_bytes")]
+    chunk: Vec<u8>,
+}
+
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("data chunk mismatch")]
+    DataChunkMismatch,
+    #[error("invalid chunk index")]
+    InvalidChunkIndex,
+    #[error("invalid duplicate shreds")]
+    InvalidDuplicateShreds,
+    #[error("invalid duplicate slot proof")]
+    InvalidDuplicateSlotProof,
+    #[error("invalid signature")]
+    InvalidSignature,
+    #[error("invalid size limit")]
+    InvalidSizeLimit,
+    #[error("invalid shred")]
+    InvalidShred(#[from] ShredError),
+    #[error("number of chunks mismatch")]
+    NumChunksMismatch,
+    #[error("missing data chunk")]
+    MissingDataChunk,
+    #[error("(de)serialization error")]
+    SerializationError(#[from] bincode::Error),
+    #[error("shred index mismatch")]
+    ShredIndexMismatch,
+    #[error("shred type mismatch")]
+    ShredTypeMismatch,
+    #[error("slot mismatch")]
+    SlotMismatch,
+    #[error("type conversion error")]
+    TryFromIntError(#[from] TryFromIntError),
+    #[error("unknown slot leader")]
+    UnknownSlotLeader,
+}
+
+// Asserts that the two shreds can indicate duplicate proof for
+// the same triplet of (slot, shred-index, and shred-type_), and
+// that they have valid signatures from the slot leader.
+fn check_shreds(
+    leader_schedule: Option<impl LeaderScheduleFn>,
+    shred1: &Shred,
+    shred2: &Shred,
+) -> Result<(), Error> {
+    if shred1.slot() != shred2.slot() {
+        Err(Error::SlotMismatch)
+    } else if shred1.index() != shred2.index() {
+        Err(Error::ShredIndexMismatch)
+    } else if shred1.common_header.shred_type != shred2.common_header.shred_type {
+        Err(Error::ShredTypeMismatch)
+    } else if shred1.payload == shred2.payload {
+        Err(Error::InvalidDuplicateShreds)
+    } else {
+        if let Some(leader_schedule) = leader_schedule {
+            let slot_leader = leader_schedule(shred1.slot()).ok_or(Error::UnknownSlotLeader)?;
+            if !shred1.verify(&slot_leader) || !shred2.verify(&slot_leader) {
+                return Err(Error::InvalidSignature);
+            }
+        }
+        Ok(())
+    }
+}
+
+/// Splits a DuplicateSlotProof into DuplicateShred
+/// chunks with a size limit on each chunk.
+pub fn from_duplicate_slot_proof(
+    proof: &DuplicateSlotProof,
+    self_pubkey: Pubkey, // Pubkey of my node broadcasting crds value.
+    leader_schedule: Option<impl LeaderScheduleFn>,
+    wallclock: u64,
+    max_size: usize, // Maximum serialized size of each DuplicateShred.
+) -> Result<impl Iterator<Item = DuplicateShred>, Error> {
+    if proof.shred1 == proof.shred2 {
+        return Err(Error::InvalidDuplicateSlotProof);
+    }
+    let shred1 = Shred::new_from_serialized_shred(proof.shred1.clone())?;
+    let shred2 = Shred::new_from_serialized_shred(proof.shred2.clone())?;
+    check_shreds(leader_schedule, &shred1, &shred2)?;
+    let (slot, shred_index, shred_type) = (
+        shred1.slot(),
+        shred1.index(),
+        shred1.common_header.shred_type,
+    );
+    let data = bincode::serialize(proof)?;
+    let chunk_size = if DUPLICATE_SHRED_HEADER_SIZE < max_size {
+        max_size - DUPLICATE_SHRED_HEADER_SIZE
+    } else {
+        return Err(Error::InvalidSizeLimit);
+    };
+    let chunks: Vec<_> = data.chunks(chunk_size).map(Vec::from).collect();
+    let num_chunks = u8::try_from(chunks.len())?;
+    let chunks = chunks
+        .into_iter()
+        .enumerate()
+        .map(move |(i, chunk)| DuplicateShred {
+            from: self_pubkey,
+            wallclock,
+            slot,
+            shred_index,
+            shred_type,
+            num_chunks,
+            chunk_index: i as u8,
+            chunk,
+        });
+    Ok(chunks)
+}
+
+pub(crate) fn from_shred(
+    shred: Shred,
+    self_pubkey: Pubkey, // Pubkey of my node broadcasting crds value.
+    other_payload: Vec<u8>,
+    leader_schedule: Option<impl LeaderScheduleFn>,
+    wallclock: u64,
+    max_size: usize, // Maximum serialized size of each DuplicateShred.
+) -> Result<impl Iterator<Item = DuplicateShred>, Error> {
+    if shred.payload == other_payload {
+        return Err(Error::InvalidDuplicateShreds);
+    }
+    let other_shred = Shred::new_from_serialized_shred(other_payload.clone())?;
+    check_shreds(leader_schedule, &shred, &other_shred)?;
+    let (slot, shred_index, shred_type) =
+        (shred.slot(), shred.index(), shred.common_header.shred_type);
+    let proof = DuplicateSlotProof {
+        shred1: shred.payload,
+        shred2: other_payload,
+    };
+    let data = bincode::serialize(&proof)?;
+    let chunk_size = if DUPLICATE_SHRED_HEADER_SIZE < max_size {
+        max_size - DUPLICATE_SHRED_HEADER_SIZE
+    } else {
+        return Err(Error::InvalidSizeLimit);
+    };
+    let chunks: Vec<_> = data.chunks(chunk_size).map(Vec::from).collect();
+    let num_chunks = u8::try_from(chunks.len())?;
+    let chunks = chunks
+        .into_iter()
+        .enumerate()
+        .map(move |(i, chunk)| DuplicateShred {
+            from: self_pubkey,
+            wallclock,
+            slot,
+            shred_index,
+            shred_type,
+            num_chunks,
+            chunk_index: i as u8,
+            chunk,
+        });
+    Ok(chunks)
+}
+
+// Returns a predicate checking if a duplicate-shred chunk matches
+// (slot, shred_index, shred_type) and has valid chunk_index.
+fn check_chunk(
+    slot: Slot,
+    shred_index: u32,
+    shred_type: ShredType,
+    num_chunks: u8,
+) -> impl Fn(&DuplicateShred) -> Result<(), Error> {
+    move |dup| {
+        if dup.slot != slot {
+            Err(Error::SlotMismatch)
+        } else if dup.shred_index != shred_index {
+            Err(Error::ShredIndexMismatch)
+        } else if dup.shred_type != shred_type {
+            Err(Error::ShredTypeMismatch)
+        } else if dup.num_chunks != num_chunks {
+            Err(Error::NumChunksMismatch)
+        } else if dup.chunk_index >= num_chunks {
+            Err(Error::InvalidChunkIndex)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Reconstructs the duplicate shreds from chunks of DuplicateShred.
+pub fn into_shreds(
+    chunks: impl IntoIterator<Item = DuplicateShred>,
+    leader: impl LeaderScheduleFn,
+) -> Result<(Shred, Shred), Error> {
+    let mut chunks = chunks.into_iter();
+    let DuplicateShred {
+        slot,
+        shred_index,
+        shred_type,
+        num_chunks,
+        chunk_index,
+        chunk,
+        ..
+    } = match chunks.next() {
+        None => return Err(Error::InvalidDuplicateShreds),
+        Some(chunk) => chunk,
+    };
+    let slot_leader = leader(slot).ok_or(Error::UnknownSlotLeader)?;
+    let check_chunk = check_chunk(slot, shred_index, shred_type, num_chunks);
+    let mut data = HashMap::new();
+    data.insert(chunk_index, chunk);
+    for chunk in chunks {
+        check_chunk(&chunk)?;
+        match data.entry(chunk.chunk_index) {
+            Entry::Vacant(entry) => {
+                entry.insert(chunk.chunk);
+            }
+            Entry::Occupied(entry) => {
+                if *entry.get() != chunk.chunk {
+                    return Err(Error::DataChunkMismatch);
+                }
+            }
+        }
+    }
+    if data.len() != num_chunks as usize {
+        return Err(Error::MissingDataChunk);
+    }
+    let data = (0..num_chunks).map(|k| data.remove(&k).unwrap()).concat();
+    let proof: DuplicateSlotProof = bincode::deserialize(&data)?;
+    if proof.shred1 == proof.shred2 {
+        return Err(Error::InvalidDuplicateSlotProof);
+    }
+    let shred1 = Shred::new_from_serialized_shred(proof.shred1)?;
+    let shred2 = Shred::new_from_serialized_shred(proof.shred2)?;
+    if shred1.slot() != slot || shred2.slot() != slot {
+        Err(Error::SlotMismatch)
+    } else if shred1.index() != shred_index || shred2.index() != shred_index {
+        Err(Error::ShredIndexMismatch)
+    } else if shred1.common_header.shred_type != shred_type
+        || shred2.common_header.shred_type != shred_type
+    {
+        Err(Error::ShredTypeMismatch)
+    } else if shred1.payload == shred2.payload {
+        Err(Error::InvalidDuplicateShreds)
+    } else if !shred1.verify(&slot_leader) || !shred2.verify(&slot_leader) {
+        Err(Error::InvalidSignature)
+    } else {
+        Ok((shred1, shred2))
+    }
+}
+
+impl Sanitize for DuplicateShred {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        sanitize_wallclock(self.wallclock)?;
+        if self.chunk_index >= self.num_chunks {
+            return Err(SanitizeError::IndexOutOfBounds);
+        }
+        self.from.sanitize()
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+    use rand::Rng;
+    use solana_ledger::{entry::Entry, shred::Shredder};
+    use solana_sdk::{hash, signature::Keypair, signature::Signer, system_transaction};
+    use std::sync::Arc;
+
+    #[test]
+    fn test_duplicate_shred_header_size() {
+        let dup = DuplicateShred {
+            from: Pubkey::new_unique(),
+            wallclock: u64::MAX,
+            slot: Slot::MAX,
+            shred_index: u32::MAX,
+            shred_type: ShredType(u8::MAX),
+            num_chunks: u8::MAX,
+            chunk_index: u8::MAX,
+            chunk: Vec::default(),
+        };
+        assert_eq!(
+            bincode::serialize(&dup).unwrap().len(),
+            DUPLICATE_SHRED_HEADER_SIZE
+        );
+        assert_eq!(
+            bincode::serialized_size(&dup).unwrap(),
+            DUPLICATE_SHRED_HEADER_SIZE as u64
+        );
+    }
+
+    pub fn new_rand_shred<R: Rng>(
+        rng: &mut R,
+        next_shred_index: u32,
+        shredder: &Shredder,
+    ) -> Shred {
+        let entries: Vec<_> = std::iter::repeat_with(|| {
+            let tx = system_transaction::transfer(
+                &Keypair::new(),       // from
+                &Pubkey::new_unique(), // to
+                rng.gen(),             // lamports
+                hash::new_rand(rng),   // recent blockhash
+            );
+            Entry::new(
+                &hash::new_rand(rng), // prev_hash
+                1,                    // num_hashes,
+                vec![tx],             // transactions
+            )
+        })
+        .take(5)
+        .collect();
+        let (mut data_shreds, _coding_shreds, _last_shred_index) = shredder.entries_to_shreds(
+            &entries,
+            true, // is_last_in_slot
+            next_shred_index,
+        );
+        data_shreds.swap_remove(0)
+    }
+
+    #[test]
+    fn test_duplicate_shred_round_trip() {
+        let mut rng = rand::thread_rng();
+        let leader = Arc::new(Keypair::new());
+        let (slot, parent_slot, fec_rate, reference_tick, version) =
+            (53084024, 53084023, 0.0, 0, 0);
+        let shredder = Shredder::new(
+            slot,
+            parent_slot,
+            fec_rate,
+            leader.clone(),
+            reference_tick,
+            version,
+        )
+        .unwrap();
+        let next_shred_index = rng.gen();
+        let shred1 = new_rand_shred(&mut rng, next_shred_index, &shredder);
+        let shred2 = new_rand_shred(&mut rng, next_shred_index, &shredder);
+        let leader_schedule = |s| {
+            if s == slot {
+                Some(leader.pubkey())
+            } else {
+                None
+            }
+        };
+        let chunks: Vec<_> = from_shred(
+            shred1.clone(),
+            Pubkey::new_unique(), // self_pubkey
+            shred2.payload.clone(),
+            Some(leader_schedule),
+            rng.gen(), // wallclock
+            512,       // max_size
+        )
+        .unwrap()
+        .collect();
+        assert!(chunks.len() > 4);
+        let (shred3, shred4) = into_shreds(chunks, leader_schedule).unwrap();
+        assert_eq!(shred1, shred3);
+        assert_eq!(shred2, shred4);
+    }
+}

core/src/gossip_service.rs

@@ -33,6 +33,7 @@ impl GossipService {
         bank_forks: Option<Arc<RwLock<BankForks>>>,
         gossip_socket: UdpSocket,
         gossip_validators: Option<HashSet<Pubkey>>,
+        should_check_duplicate_instance: bool,
         exit: &Arc<AtomicBool>,
     ) -> Self {
         let (request_sender, request_receiver) = channel();
@@ -56,6 +57,7 @@ impl GossipService {
             bank_forks.clone(),
             request_receiver,
             response_sender.clone(),
+            should_check_duplicate_instance,
             exit,
         );
         let t_gossip = ClusterInfo::gossip(
@@ -108,8 +110,14 @@ pub fn discover(
     let keypair = keypair.unwrap_or_else(|| Arc::new(Keypair::new()));
 
     let exit = Arc::new(AtomicBool::new(false));
-    let (gossip_service, ip_echo, spy_ref) =
-        make_gossip_node(keypair, entrypoint, &exit, my_gossip_addr, my_shred_version);
+    let (gossip_service, ip_echo, spy_ref) = make_gossip_node(
+        keypair,
+        entrypoint,
+        &exit,
+        my_gossip_addr,
+        my_shred_version,
+        true, // should_check_duplicate_instance,
+    );
 
     let id = spy_ref.id();
     info!("Entrypoint: {:?}", entrypoint);
@@ -268,6 +276,7 @@ fn make_gossip_node(
     exit: &Arc<AtomicBool>,
     gossip_addr: Option<&SocketAddr>,
     shred_version: u16,
+    should_check_duplicate_instance: bool,
 ) -> (GossipService, Option<TcpListener>, Arc<ClusterInfo>) {
     let (node, gossip_socket, ip_echo) = if let Some(gossip_addr) = gossip_addr {
         ClusterInfo::gossip_node(&keypair.pubkey(), gossip_addr, shred_version)
@@ -279,7 +288,14 @@ fn make_gossip_node(
         cluster_info.set_entrypoint(ContactInfo::new_gossip_entry_point(entrypoint));
     }
     let cluster_info = Arc::new(cluster_info);
-    let gossip_service = GossipService::new(&cluster_info, None, gossip_socket, None, &exit);
+    let gossip_service = GossipService::new(
+        &cluster_info,
+        None,
+        gossip_socket,
+        None,
+        should_check_duplicate_instance,
+        &exit,
+    );
     (gossip_service, ip_echo, cluster_info)
 }
 
@@ -298,7 +314,14 @@ mod tests {
         let tn = Node::new_localhost();
         let cluster_info = ClusterInfo::new_with_invalid_keypair(tn.info.clone());
         let c = Arc::new(cluster_info);
-        let d = GossipService::new(&c, None, tn.sockets.gossip, None, &exit);
+        let d = GossipService::new(
+            &c,
+            None,
+            tn.sockets.gossip,
+            None,
+            true, // should_check_duplicate_instance
+            &exit,
+        );
         exit.store(true, Ordering::Relaxed);
         d.join().unwrap();
     }

core/src/ledger_cleanup_service.rs

@@ -1,15 +1,16 @@
 //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
 
+use rand::{thread_rng, Rng};
 use solana_ledger::blockstore::{Blockstore, PurgeType};
 use solana_ledger::blockstore_db::Result as BlockstoreResult;
 use solana_measure::measure::Measure;
 use solana_sdk::clock::{Slot, DEFAULT_TICKS_PER_SLOT, TICKS_PER_DAY};
 use std::string::ToString;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
 use std::sync::Arc;
 use std::thread;
-use std::thread::{Builder, JoinHandle};
+use std::thread::{sleep, Builder, JoinHandle};
 use std::time::Duration;
 
 // - To try and keep the RocksDB size under 400GB:
@@ -35,6 +36,7 @@ const DEFAULT_COMPACTION_SLOT_INTERVAL: u64 = TICKS_PER_DAY / DEFAULT_TICKS_PER_
 
 pub struct LedgerCleanupService {
     t_cleanup: JoinHandle<()>,
+    t_compact: JoinHandle<()>,
 }
 
 impl LedgerCleanupService {
@@ -43,6 +45,8 @@ impl LedgerCleanupService {
         blockstore: Arc<Blockstore>,
         max_ledger_shreds: u64,
         exit: &Arc<AtomicBool>,
+        compaction_interval: Option<u64>,
+        max_compaction_jitter: Option<u64>,
     ) -> Self {
         info!(
             "LedgerCleanupService active. Max Ledger Slots {}",
@@ -51,9 +55,16 @@ impl LedgerCleanupService {
         let exit = exit.clone();
         let mut last_purge_slot = 0;
         let mut last_compaction_slot = 0;
+        let mut compaction_jitter = 0;
+        let compaction_interval = compaction_interval.unwrap_or(DEFAULT_COMPACTION_SLOT_INTERVAL);
+        let last_compact_slot = Arc::new(AtomicU64::new(0));
+        let last_compact_slot2 = last_compact_slot.clone();
+
+        let exit_compact = exit.clone();
+        let blockstore_compact = blockstore.clone();
 
         let t_cleanup = Builder::new()
-            .name("solana-ledger-cleanup".to_string())
+            .name("sol-led-cleanup".to_string())
             .spawn(move || loop {
                 if exit.load(Ordering::Relaxed) {
                     break;
@@ -64,8 +75,7 @@ impl LedgerCleanupService {
                     max_ledger_shreds,
                     &mut last_purge_slot,
                     DEFAULT_PURGE_SLOT_INTERVAL,
-                    &mut last_compaction_slot,
-                    DEFAULT_COMPACTION_SLOT_INTERVAL,
+                    &last_compact_slot,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -74,7 +84,29 @@ impl LedgerCleanupService {
                 }
             })
             .unwrap();
-        Self { t_cleanup }
+
+        let t_compact = Builder::new()
+            .name("sol-led-compact".to_string())
+            .spawn(move || loop {
+                if exit_compact.load(Ordering::Relaxed) {
+                    break;
+                }
+                Self::compact_ledger(
+                    &blockstore_compact,
+                    &mut last_compaction_slot,
+                    compaction_interval,
+                    &last_compact_slot2,
+                    &mut compaction_jitter,
+                    max_compaction_jitter,
+                );
+                sleep(Duration::from_secs(1));
+            })
+            .unwrap();
+
+        Self {
+            t_cleanup,
+            t_compact,
+        }
     }
 
     fn find_slots_to_clean(
@@ -138,8 +170,7 @@ impl LedgerCleanupService {
         max_ledger_shreds: u64,
         last_purge_slot: &mut u64,
         purge_interval: u64,
-        last_compaction_slot: &mut u64,
-        compaction_interval: u64,
+        last_compact_slot: &Arc<AtomicU64>,
     ) -> Result<(), RecvTimeoutError> {
         let root = Self::receive_new_roots(new_root_receiver)?;
         if root - *last_purge_slot <= purge_interval {
@@ -148,8 +179,8 @@ impl LedgerCleanupService {
 
         let disk_utilization_pre = blockstore.storage_size();
         info!(
-            "purge: last_root={}, last_purge_slot={}, purge_interval={}, last_compaction_slot={}, disk_utilization={:?}",
-            root, last_purge_slot, purge_interval, last_compaction_slot, disk_utilization_pre
+            "purge: last_root={}, last_purge_slot={}, purge_interval={}, disk_utilization={:?}",
+            root, last_purge_slot, purge_interval, disk_utilization_pre
         );
 
         *last_purge_slot = root;
@@ -158,15 +189,10 @@ impl LedgerCleanupService {
             Self::find_slots_to_clean(&blockstore, root, max_ledger_shreds);
 
         if slots_to_clean {
-            let mut compact_first_slot = std::u64::MAX;
-            if lowest_cleanup_slot.saturating_sub(*last_compaction_slot) > compaction_interval {
-                compact_first_slot = *last_compaction_slot;
-                *last_compaction_slot = lowest_cleanup_slot;
-            }
-
             let purge_complete = Arc::new(AtomicBool::new(false));
             let blockstore = blockstore.clone();
             let purge_complete1 = purge_complete.clone();
+            let last_compact_slot1 = last_compact_slot.clone();
             let _t_purge = Builder::new()
                 .name("solana-ledger-purge".to_string())
                 .spawn(move || {
@@ -188,21 +214,7 @@ impl LedgerCleanupService {
                     purge_time.stop();
                     info!("{}", purge_time);
 
-                    if compact_first_slot < lowest_cleanup_slot {
-                        info!(
-                            "compacting data from slots {} to {}",
-                            compact_first_slot, lowest_cleanup_slot
-                        );
-                        if let Err(err) =
-                            blockstore.compact_storage(compact_first_slot, lowest_cleanup_slot)
-                        {
-                            // This error is not fatal and indicates an internal error?
-                            error!(
-                                "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
-                                err, compact_first_slot, lowest_cleanup_slot
-                            );
-                        }
-                    }
+                    last_compact_slot1.store(lowest_cleanup_slot, Ordering::Relaxed);
 
                     purge_complete1.store(true, Ordering::Relaxed);
                 })
@@ -223,6 +235,39 @@ impl LedgerCleanupService {
         Ok(())
     }
 
+    pub fn compact_ledger(
+        blockstore: &Arc<Blockstore>,
+        last_compaction_slot: &mut u64,
+        compaction_interval: u64,
+        highest_compact_slot: &Arc<AtomicU64>,
+        compaction_jitter: &mut u64,
+        max_jitter: Option<u64>,
+    ) {
+        let highest_compaction_slot = highest_compact_slot.load(Ordering::Relaxed);
+        if highest_compaction_slot.saturating_sub(*last_compaction_slot)
+            > (compaction_interval + *compaction_jitter)
+        {
+            info!(
+                "compacting data from slots {} to {}",
+                *last_compaction_slot, highest_compaction_slot,
+            );
+            if let Err(err) =
+                blockstore.compact_storage(*last_compaction_slot, highest_compaction_slot)
+            {
+                // This error is not fatal and indicates an internal error?
+                error!(
+                    "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
+                    err, last_compaction_slot, highest_compaction_slot,
+                );
+            }
+            *last_compaction_slot = highest_compaction_slot;
+            let jitter = max_jitter.unwrap_or(0);
+            if jitter > 0 {
+                *compaction_jitter = thread_rng().gen_range(0, jitter);
+            }
+        }
+    }
+
     fn report_disk_metrics(
         pre: BlockstoreResult<u64>,
         post: BlockstoreResult<u64>,
@@ -240,7 +285,8 @@ impl LedgerCleanupService {
     }
 
     pub fn join(self) -> thread::Result<()> {
-        self.t_cleanup.join()
+        self.t_cleanup.join()?;
+        self.t_compact.join()
     }
 }
 #[cfg(test)]
@@ -251,7 +297,7 @@ mod tests {
     use std::sync::mpsc::channel;
 
     #[test]
-    fn test_cleanup() {
+    fn test_cleanup1() {
         solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&blockstore_path).unwrap();
@@ -262,7 +308,7 @@ mod tests {
 
         //send a signal to kill all but 5 shreds, which will be in the newest slots
         let mut last_purge_slot = 0;
-        let mut last_compaction_slot = 0;
+        let highest_compaction_slot = Arc::new(AtomicU64::new(0));
         sender.send(50).unwrap();
         LedgerCleanupService::cleanup_ledger(
             &receiver,
@@ -270,10 +316,11 @@ mod tests {
             5,
             &mut last_purge_slot,
             10,
-            &mut last_compaction_slot,
-            10,
+            &highest_compaction_slot,
         )
         .unwrap();
+        assert_eq!(last_purge_slot, 50);
+        assert_eq!(highest_compaction_slot.load(Ordering::Relaxed), 44);
 
         //check that 0-40 don't exist
         blockstore
@@ -281,6 +328,18 @@ mod tests {
             .unwrap()
             .for_each(|(slot, _)| assert!(slot > 40));
 
+        let mut last_compaction_slot = 0;
+        let mut jitter = 0;
+        LedgerCleanupService::compact_ledger(
+            &blockstore,
+            &mut last_compaction_slot,
+            10,
+            &highest_compaction_slot,
+            &mut jitter,
+            None,
+        );
+        assert_eq!(jitter, 0);
+
         drop(blockstore);
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
@@ -303,7 +362,7 @@ mod tests {
         info!("{}", first_insert);
 
         let mut last_purge_slot = 0;
-        let mut last_compaction_slot = 0;
+        let last_compaction_slot = Arc::new(AtomicU64::new(0));
         let mut slot = initial_slots;
         let mut num_slots = 6;
         for _ in 0..5 {
@@ -327,8 +386,7 @@ mod tests {
                 initial_slots,
                 &mut last_purge_slot,
                 10,
-                &mut last_compaction_slot,
-                10,
+                &last_compaction_slot,
             )
             .unwrap();
             time.stop();