Add error reporting to system program (#15644) (#15650)

(cherry picked from commit a9c8dbfd0c)

Co-authored-by: Jack May <jack@solana.com>

.gitignore

@@ -1,7 +1,3 @@
-/docs/html/
-/docs/src/tests.ok
-/docs/src/cli/usage.md
-/docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/
 /solana-release.tar.bz2

Cargo.toml

@@ -28,6 +28,7 @@ members = [
     "local-cluster",
     "logger",
     "log-analyzer",
+    "merkle-root-bench",
     "merkle-tree",
     "stake-o-matic",
     "storage-bigtable",

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.5.5"
+version = "1.5.12"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,14 +15,14 @@ bs58 = "0.3.1"
 bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-serde = "1.0.112"
+serde = "1.0.118"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.5" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
+solana-config-program = { path = "../programs/config", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.12" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.12" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"
 

account-decoder/src/lib.rs

@@ -1,9 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
 #[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate serde_derive;
 
 pub mod parse_account_data;
+pub mod parse_bpf_loader;
 pub mod parse_config;
 pub mod parse_nonce;
 pub mod parse_stake;

account-decoder/src/parse_account_data.rs

@@ -1,4 +1,5 @@
 use crate::{
+    parse_bpf_loader::parse_bpf_upgradeable_loader,
     parse_config::parse_config,
     parse_nonce::parse_nonce,
     parse_stake::parse_stake,
@@ -13,6 +14,7 @@ use std::collections::HashMap;
 use thiserror::Error;
 
 lazy_static! {
+    static ref BPF_UPGRADEABLE_LOADER_PROGRAM_ID: Pubkey = solana_sdk::bpf_loader_upgradeable::id();
     static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
     static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
@@ -21,6 +23,10 @@ lazy_static! {
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();
+        m.insert(
+            *BPF_UPGRADEABLE_LOADER_PROGRAM_ID,
+            ParsableAccount::BpfUpgradeableLoader,
+        );
         m.insert(*CONFIG_PROGRAM_ID, ParsableAccount::Config);
         m.insert(*SYSTEM_PROGRAM_ID, ParsableAccount::Nonce);
         m.insert(*TOKEN_PROGRAM_ID, ParsableAccount::SplToken);
@@ -60,6 +66,7 @@ pub struct ParsedAccount {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum ParsableAccount {
+    BpfUpgradeableLoader,
     Config,
     Nonce,
     SplToken,
@@ -84,6 +91,9 @@ pub fn parse_account_data(
         .ok_or(ParseAccountError::ProgramNotParsable)?;
     let additional_data = additional_data.unwrap_or_default();
     let parsed_json = match program_name {
+        ParsableAccount::BpfUpgradeableLoader => {
+            serde_json::to_value(parse_bpf_upgradeable_loader(data)?)?
+        }
         ParsableAccount::Config => serde_json::to_value(parse_config(data, pubkey)?)?,
         ParsableAccount::Nonce => serde_json::to_value(parse_nonce(data)?)?,
         ParsableAccount::SplToken => {

account-decoder/src/parse_bpf_loader.rs

@@ -0,0 +1,181 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    UiAccountData, UiAccountEncoding,
+};
+use bincode::{deserialize, serialized_size};
+use solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey};
+
+pub fn parse_bpf_upgradeable_loader(
+    data: &[u8],
+) -> Result<BpfUpgradeableLoaderAccountType, ParseAccountError> {
+    let account_state: UpgradeableLoaderState = deserialize(data).map_err(|_| {
+        ParseAccountError::AccountNotParsable(ParsableAccount::BpfUpgradeableLoader)
+    })?;
+    let parsed_account = match account_state {
+        UpgradeableLoaderState::Uninitialized => BpfUpgradeableLoaderAccountType::Uninitialized,
+        UpgradeableLoaderState::Buffer { authority_address } => {
+            let offset = if authority_address.is_some() {
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+            } else {
+                // This case included for code completeness; in practice, a Buffer account will
+                // always have authority_address.is_some()
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+        UpgradeableLoaderState::Program {
+            programdata_address,
+        } => BpfUpgradeableLoaderAccountType::Program(UiProgram {
+            program_data: programdata_address.to_string(),
+        }),
+        UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address,
+        } => {
+            let offset = if upgrade_authority_address.is_some() {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+            } else {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: upgrade_authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+    };
+    Ok(parsed_account)
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum BpfUpgradeableLoaderAccountType {
+    Uninitialized,
+    Buffer(UiBuffer),
+    Program(UiProgram),
+    ProgramData(UiProgramData),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiBuffer {
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgram {
+    pub program_data: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgramData {
+    pub slot: u64,
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use bincode::serialize;
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_parse_bpf_upgradeable_loader_accounts() {
+        let bpf_loader_state = UpgradeableLoaderState::Uninitialized;
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Uninitialized
+        );
+
+        let program = vec![7u8; 64]; // Arbitrary program data
+
+        let authority = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        // This case included for code completeness; in practice, a Buffer account will always have
+        // authority_address.is_some()
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let programdata_address = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Program {
+            programdata_address,
+        };
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Program(UiProgram {
+                program_data: programdata_address.to_string(),
+            })
+        );
+
+        let authority = Pubkey::new_unique();
+        let slot = 42;
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+    }
+}

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.5.5" }
-solana-runtime = { path = "../runtime", version = "1.5.5" }
-solana-measure = { path = "../measure", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
+solana-logger = { path = "../logger", version = "1.5.12" }
+solana-runtime = { path = "../runtime", version = "1.5.12" }
+solana-measure = { path = "../measure", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -1,3 +1,6 @@
+#![allow(clippy::integer_arithmetic)]
+#[macro_use]
+extern crate log;
 use clap::{crate_description, crate_name, value_t, App, Arg};
 use rayon::prelude::*;
 use solana_measure::measure::Measure;
@@ -51,6 +54,7 @@ fn main() {
 
     let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned()))
         .join("accounts-bench");
+    println!("cleaning file system: {:?}", path);
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
@@ -84,6 +88,8 @@ fn main() {
         ancestors.insert(i as u64, i - 1);
         accounts.add_root(i as u64);
     }
+    let mut elapsed = vec![0; iterations];
+    let mut elapsed_store = vec![0; iterations];
     for x in 0..iterations {
         if clean {
             let mut time = Measure::start("clean");
@@ -97,13 +103,40 @@ fn main() {
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let hash = accounts
+            let results = accounts
                 .accounts_db
-                .update_accounts_hash(0, &ancestors, true)
-                .0;
+                .update_accounts_hash(0, &ancestors, true);
             time.stop();
-            println!("hash: {} {}", hash, time);
+            let mut time_store = Measure::start("hash using store");
+            let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
+                false,
+                false,
+                solana_sdk::clock::Slot::default(),
+                &ancestors,
+                true,
+                None,
+            );
+            time_store.stop();
+            if results != results_store {
+                error!("results different: \n{:?}\n{:?}", results, results_store);
+            }
+            println!(
+                "hash,{},{},{},{}%",
+                results.0,
+                time,
+                time_store,
+                (time_store.as_us() as f64 / time.as_us() as f64 * 100.0f64) as u32
+            );
             create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+            elapsed[x] = time.as_us();
+            elapsed_store[x] = time_store.as_us();
         }
     }
+
+    for x in elapsed {
+        info!("update_accounts_hash(us),{}", x);
+    }
+    for x in elapsed_store {
+        info!("calculate_accounts_hash_without_index(us),{}", x);
+    }
 }

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.5.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-streamer = { path = "../streamer", version = "1.5.5" }
-solana-perf = { path = "../perf", version = "1.5.5" }
-solana-ledger = { path = "../ledger", version = "1.5.5" }
-solana-logger = { path = "../logger", version = "1.5.5" }
-solana-runtime = { path = "../runtime", version = "1.5.5" }
-solana-measure = { path = "../measure", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
+solana-core = { path = "../core", version = "1.5.12" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-streamer = { path = "../streamer", version = "1.5.12" }
+solana-perf = { path = "../perf", version = "1.5.12" }
+solana-ledger = { path = "../ledger", version = "1.5.12" }
+solana-logger = { path = "../logger", version = "1.5.12" }
+solana-runtime = { path = "../runtime", version = "1.5.12" }
+solana-measure = { path = "../measure", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 use clap::{crate_description, crate_name, value_t, App, Arg};
 use crossbeam_channel::unbounded;
 use log::*;
@@ -18,7 +19,7 @@ use solana_ledger::{
 use solana_measure::measure::Measure;
 use solana_perf::packet::to_packets_chunked;
 use solana_runtime::{
-    accounts_background_service::ABSRequestSender, bank::Bank, bank_forks::BankForks,
+    accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
 };
 use solana_sdk::{
     hash::Hash,
@@ -325,7 +326,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &ABSRequestSender::default(), None);
+                    bank_forks.set_root(root, &AbsRequestSender::default(), None);
                     root += 1;
                 }
                 debug!(

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.5.5"
+version = "1.5.12"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,17 +10,20 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.3.1"
+borsh = "0.8.1"
+borsh-derive = "0.8.1"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
+solana-banks-interface = { path = "../banks-interface", version = "1.5.12" }
+solana-program = { path = "../sdk/program", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
 tarpc = { version = "0.23.0", features = ["full"] }
 tokio = { version = "0.3.5", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.5.5" }
-solana-banks-server = { path = "../banks-server", version = "1.5.5" }
+solana-runtime = { path = "../runtime", version = "1.5.12" }
+solana-banks-server = { path = "../banks-server", version = "1.5.12" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -5,19 +5,18 @@
 //! but they are undocumented, may change over time, and are generally more
 //! cumbersome to use.
 
+use borsh::BorshDeserialize;
 use futures::{future::join_all, Future, FutureExt};
 pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
 use solana_banks_interface::{BanksRequest, BanksResponse};
+use solana_program::{
+    clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
+    rent::Rent, sysvar,
+};
 use solana_sdk::{
     account::{from_account, Account},
-    clock::Slot,
     commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    pubkey::Pubkey,
-    rent::Rent,
     signature::Signature,
-    sysvar,
     transaction::{self, Transaction},
     transport,
 };
@@ -218,6 +217,33 @@ impl BanksClient {
         self.get_account_with_commitment(address, CommitmentLevel::default())
     }
 
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_packed_account_data<T: Pack>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account =
+                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Account not found"))?;
+            T::unpack_from_slice(&account.data)
+                .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to deserialize account"))
+        })
+    }
+
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_account_data_with_borsh<T: BorshDeserialize>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account =
+                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "account not found"))?;
+            T::try_from_slice(&account.data)
+        })
+    }
+
     /// Return the balance in lamports of an account at the given address at the slot
     /// corresponding to the given commitment level.
     pub fn get_balance_with_commitment(
@@ -289,7 +315,10 @@ pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClie
 mod tests {
     use super::*;
     use solana_banks_server::banks_server::start_local_server;
-    use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
+    use solana_runtime::{
+        bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache,
+        genesis_utils::create_genesis_config,
+    };
     use solana_sdk::{message::Message, signature::Signer, system_instruction};
     use std::sync::{Arc, RwLock};
     use tarpc::transport;
@@ -308,9 +337,12 @@ mod tests {
         // `runtime.block_on()` just once, to run all the async code.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
         let bob_pubkey = solana_sdk::pubkey::new_rand();
         let mint_pubkey = genesis.mint_keypair.pubkey();
@@ -318,7 +350,7 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
             let mut banks_client = start_client(client_transport).await?;
 
             let recent_blockhash = banks_client.get_recent_blockhash().await?;
@@ -336,9 +368,12 @@ mod tests {
         // server-side functionality is available to the client.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
         let bob_pubkey = solana_sdk::pubkey::new_rand();
@@ -346,7 +381,7 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
             let mut banks_client = start_client(client_transport).await?;
             let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.5.5"
+version = "1.5.12"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,8 +10,8 @@ edition = "2018"
 
 [dependencies]
 mio = "0.7.6"
-serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
+serde = { version = "1.0.118", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
 tarpc = { version = "0.23.0", features = ["full"] }
 
 [dev-dependencies]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.5.5"
+version = "1.5.12"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,10 +13,10 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.5.5" }
-solana-runtime = { path = "../runtime", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-metrics = { path = "../metrics", version = "1.5.5" }
+solana-banks-interface = { path = "../banks-interface", version = "1.5.12" }
+solana-runtime = { path = "../runtime", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-metrics = { path = "../metrics", version = "1.5.12" }
 tarpc = { version = "0.23.0", features = ["full"] }
 tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -62,7 +62,7 @@ impl BanksServer {
         }
     }
 
-    fn run(bank: &Bank, transaction_receiver: Receiver<TransactionInfo>) {
+    fn run(bank_forks: Arc<RwLock<BankForks>>, transaction_receiver: Receiver<TransactionInfo>) {
         while let Ok(info) = transaction_receiver.recv() {
             let mut transaction_infos = vec![info];
             while let Ok(info) = transaction_receiver.try_recv() {
@@ -72,21 +72,28 @@ impl BanksServer {
                 .into_iter()
                 .map(|info| deserialize(&info.wire_transaction).unwrap())
                 .collect();
+            let bank = bank_forks.read().unwrap().working_bank();
             let _ = bank.process_transactions(&transactions);
         }
     }
 
     /// Useful for unit-testing
-    fn new_loopback(bank_forks: Arc<RwLock<BankForks>>) -> Self {
+    fn new_loopback(
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
         let slot = bank.slot();
-        let block_commitment_cache = Arc::new(RwLock::new(
-            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
-        ));
+        {
+            // ensure that the commitment cache and bank are synced
+            let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+            w_block_commitment_cache.set_all_slots(slot, slot);
+        }
+        let server_bank_forks = bank_forks.clone();
         Builder::new()
             .name("solana-bank-forks-client".to_string())
-            .spawn(move || Self::run(&bank, transaction_receiver))
+            .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
         Self::new(bank_forks, block_commitment_cache, transaction_sender)
     }
@@ -240,9 +247,10 @@ impl Banks for BanksServer {
 }
 
 pub async fn start_local_server(
-    bank_forks: &Arc<RwLock<BankForks>>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(bank_forks.clone());
+    let banks_server = BanksServer::new_loopback(bank_forks, block_commitment_cache);
     let (client_transport, server_transport) = transport::channel::unbounded();
     let server = server::new(server::Config::default())
         .incoming(stream::once(future::ready(server_transport)))

banks-server/src/lib.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod banks_server;
 pub mod rpc_banks_service;
 pub mod send_transaction_service;

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-core = { path = "../core", version = "1.5.5" }
-solana-genesis = { path = "../genesis", version = "1.5.5" }
-solana-client = { path = "../client", version = "1.5.5" }
-solana-faucet = { path = "../faucet", version = "1.5.5" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.5.5" }
-solana-logger = { path = "../logger", version = "1.5.5" }
-solana-metrics = { path = "../metrics", version = "1.5.5" }
-solana-net-utils = { path = "../net-utils", version = "1.5.5" }
-solana-runtime = { path = "../runtime", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-core = { path = "../core", version = "1.5.12" }
+solana-genesis = { path = "../genesis", version = "1.5.12" }
+solana-client = { path = "../client", version = "1.5.12" }
+solana-faucet = { path = "../faucet", version = "1.5.12" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.5.12" }
+solana-logger = { path = "../logger", version = "1.5.12" }
+solana-metrics = { path = "../metrics", version = "1.5.12" }
+solana-net-utils = { path = "../net-utils", version = "1.5.12" }
+solana-runtime = { path = "../runtime", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.5.5" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/src/bench.rs

@@ -1,4 +1,5 @@
 #![allow(clippy::useless_attribute)]
+#![allow(clippy::integer_arithmetic)]
 
 use crate::order_book::*;
 use itertools::izip;

bench-exchange/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod bench;
 mod cli;
 pub mod order_book;

bench-exchange/tests/bench_exchange.rs

@@ -5,7 +5,7 @@ use solana_core::validator::ValidatorConfig;
 use solana_exchange_program::exchange_processor::process_instruction;
 use solana_exchange_program::id;
 use solana_exchange_program::solana_exchange_program;
-use solana_faucet::faucet::run_local_faucet;
+use solana_faucet::faucet::run_local_faucet_with_port;
 use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
 use solana_runtime::bank::Bank;
 use solana_runtime::bank_client::BankClient;
@@ -57,8 +57,11 @@ fn test_exchange_local_cluster() {
     );
 
     let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, Some(1_000_000_000_000));
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, Some(1_000_000_000_000), 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
 
     info!("Connecting to the cluster");
     let nodes =

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-streamer = { path = "../streamer", version = "1.5.5" }
-solana-logger = { path = "../logger", version = "1.5.5" }
-solana-net-utils = { path = "../net-utils", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-streamer = { path = "../streamer", version = "1.5.12" }
+solana-logger = { path = "../logger", version = "1.5.12" }
+solana-net-utils = { path = "../net-utils", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 use clap::{crate_description, crate_name, App, Arg};
 use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
 use solana_streamer::streamer::{receiver, PacketReceiver};
@@ -74,7 +75,7 @@ fn main() -> Result<()> {
 
     let mut read_channels = Vec::new();
     let mut read_threads = Vec::new();
-    let recycler = PacketsRecycler::default();
+    let recycler = PacketsRecycler::new_without_limit("bench-streamer-recycler-shrink-stats");
     for _ in 0..num_sockets {
         let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap();
         read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
@@ -90,6 +91,7 @@ fn main() -> Result<()> {
             s_reader,
             recycler.clone(),
             "bench-streamer-test",
+            1,
         ));
     }
 

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,23 +15,23 @@ log = "0.4.11"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-core = { path = "../core", version = "1.5.5" }
-solana-genesis = { path = "../genesis", version = "1.5.5" }
-solana-client = { path = "../client", version = "1.5.5" }
-solana-faucet = { path = "../faucet", version = "1.5.5" }
-solana-logger = { path = "../logger", version = "1.5.5" }
-solana-metrics = { path = "../metrics", version = "1.5.5" }
-solana-measure = { path = "../measure", version = "1.5.5" }
-solana-net-utils = { path = "../net-utils", version = "1.5.5" }
-solana-runtime = { path = "../runtime", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-core = { path = "../core", version = "1.5.12" }
+solana-genesis = { path = "../genesis", version = "1.5.12" }
+solana-client = { path = "../client", version = "1.5.12" }
+solana-faucet = { path = "../faucet", version = "1.5.12" }
+solana-logger = { path = "../logger", version = "1.5.12" }
+solana-metrics = { path = "../metrics", version = "1.5.12" }
+solana-measure = { path = "../measure", version = "1.5.12" }
+solana-net-utils = { path = "../net-utils", version = "1.5.12" }
+solana-runtime = { path = "../runtime", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.5.5" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/lib.rs

@@ -1,2 +1,3 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod bench;
 pub mod cli;

bench-tps/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 use log::*;
 use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs};
 use solana_bench_tps::cli;

bench-tps/tests/bench_tps.rs

@@ -1,10 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
 use serial_test_derive::serial;
 use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs};
 use solana_bench_tps::cli::Config;
 use solana_client::thin_client::create_client;
 use solana_core::cluster_info::VALIDATOR_PORT_RANGE;
 use solana_core::validator::ValidatorConfig;
-use solana_faucet::faucet::run_local_faucet;
+use solana_faucet::faucet::run_local_faucet_with_port;
 use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
 use solana_sdk::signature::{Keypair, Signer};
 use std::sync::{mpsc::channel, Arc};
@@ -36,8 +37,11 @@ fn test_bench_tps_local_cluster(config: Config) {
     ));
 
     let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, None);
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, None, 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
 
     let lamports_per_account = 100;
 

ci/do-audit.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e
+
+here="$(dirname "$0")"
+src_root="$(readlink -f "${here}/..")"
+
+cd "${src_root}"
+
+source ci/rust-version.sh stable
+
+cargo_audit_ignores=(
+  # failure is officially deprecated/unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `failure` dependency.
+  --ignore RUSTSEC-2020-0036
+
+  # `net2` crate has been deprecated; use `socket2` instead
+  #
+  # Blocked on https://github.com/paritytech/jsonrpc/issues/575
+  --ignore RUSTSEC-2020-0016
+
+  # stdweb is unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `stdweb` dependency.
+  --ignore RUSTSEC-2020-0056
+
+  # Potential segfault in the time crate
+  #
+  # Blocked on multiple crates updating `time` to >= 0.2.23
+  --ignore RUSTSEC-2020-0071
+
+  # difference is unmaintained
+  #
+  # Blocked on predicates v1.0.6 removing its dependency on `difference`
+  --ignore RUSTSEC-2020-0095
+
+  # hyper is upgraded on master/v1.6 but not for v1.5
+  --ignore RUSTSEC-2021-0020
+
+  # generic-array: arr! macro erases lifetimes
+  #
+  # ed25519-dalek and libsecp256k1 not upgraded for v1.5
+  --ignore RUSTSEC-2020-0146
+
+)
+scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"

ci/setup-new-buildkite-agent/setup-buildkite.sh

@@ -76,7 +76,7 @@ RestartForceExitStatus=SIGPIPE
 TimeoutStartSec=10
 TimeoutStopSec=0
 KillMode=process
-LimitNOFILE=500000
+LimitNOFILE=700000
 
 [Install]
 WantedBy=multi-user.target

ci/setup-new-buildkite-agent/setup-limits.sh

@@ -8,5 +8,5 @@ source "$HERE"/utils.sh
 ensure_env || exit 1
 
 # Allow more files to be opened by a user
-echo "* - nofile 500000" > /etc/security/limits.d/90-solana-nofiles.conf
+echo "* - nofile 700000" > /etc/security/limits.d/90-solana-nofiles.conf
 

ci/test-bench.sh

@@ -23,6 +23,10 @@ fi
 BENCH_FILE=bench_output.log
 BENCH_ARTIFACT=current_bench_results.log
 
+# solana-keygen required when building C programs
+_ "$cargo" build --manifest-path=keygen/Cargo.toml
+export PATH="$PWD/target/debug":$PATH
+
 # Clear the C dependency files, if dependeny moves these files are not regenerated
 test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete

ci/test-checks.sh

@@ -12,6 +12,16 @@ cargo="$(readlink -f "./cargo")"
 
 scripts/increment-cargo-version.sh check
 
+# Disallow uncommitted Cargo.lock changes
+(
+  _ scripts/cargo-for-all-lock-files.sh tree
+  set +e
+  if ! _ git diff --exit-code; then
+    echo -e "\nError: Uncommitted Cargo.lock changes" 1>&2
+    exit 1
+  fi
+)
+
 echo --- build environment
 (
   set -x
@@ -52,51 +62,24 @@ else
 fi
 
 _ ci/order-crates-for-publishing.py
-_ "$cargo" stable fmt --all -- --check
 
 # -Z... is needed because of clippy bug: https://github.com/rust-lang/rust-clippy/issues/4612
 # run nightly clippy for `sdk/` as there's a moderate amount of nightly-only code there
-_ "$cargo" nightly clippy -Zunstable-options --workspace --all-targets -- --deny=warnings
+_ "$cargo" nightly clippy -Zunstable-options --workspace --all-targets -- --deny=warnings --deny=clippy::integer_arithmetic
 
-cargo_audit_ignores=(
-  # failure is officially deprecated/unmaintained
-  #
-  # Blocked on multiple upstream crates removing their `failure` dependency.
-  --ignore RUSTSEC-2020-0036
+_ "$cargo" stable fmt --all -- --check
 
-  # `net2` crate has been deprecated; use `socket2` instead
-  #
-  # Blocked on https://github.com/paritytech/jsonrpc/issues/575
-  --ignore RUSTSEC-2020-0016
-
-  # stdweb is unmaintained
-  #
-  # Blocked on multiple upstream crates removing their `stdweb` dependency.
-  --ignore RUSTSEC-2020-0056
-
-  # Potential segfault in the time crate
-  #
-  # Blocked on multiple crates updating `time` to >= 0.2.23
-  --ignore RUSTSEC-2020-0071
-
-  # difference is unmaintained
-  #
-  # Blocked on predicates v1.0.6 removing its dependency on `difference`
-  --ignore RUSTSEC-2020-0095
-
-)
-_ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"
+_ ci/do-audit.sh
 
 {
   cd programs/bpf
-  _ "$cargo" stable audit
   for project in rust/*/ ; do
     echo "+++ do_bpf_checks $project"
     (
       cd "$project"
+      _ "$cargo" nightly clippy -- --deny=warnings --allow=clippy::missing_safety_doc
       _ "$cargo" stable fmt -- --check
       _ "$cargo" nightly test
-      _ "$cargo" nightly clippy -- --deny=warnings --allow=clippy::missing_safety_doc
     )
   done
 }

ci/test-stable.sh

@@ -39,6 +39,10 @@ test-stable)
   _ "$cargo" stable test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
   ;;
 test-stable-perf)
+  # solana-keygen required when building C programs
+  _ "$cargo" build --manifest-path=keygen/Cargo.toml
+  export PATH="$PWD/target/debug":$PATH
+
   # BPF solana-sdk legacy compile test
   ./cargo-build-bpf --manifest-path sdk/Cargo.toml
 

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.5.5"
+version = "1.5.12"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/input_validators.rs

@@ -3,7 +3,7 @@ use chrono::DateTime;
 use solana_sdk::{
     clock::{Epoch, Slot},
     hash::Hash,
-    pubkey::Pubkey,
+    pubkey::{Pubkey, MAX_SEED_LEN},
     signature::{read_keypair_file, Signature},
 };
 use std::fmt::Display;
@@ -291,6 +291,21 @@ where
         .map(|_| ())
 }
 
+pub fn is_derived_address_seed<T>(value: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    let value = value.as_ref();
+    if value.len() > MAX_SEED_LEN {
+        Err(format!(
+            "Address seed must not be longer than {} bytes",
+            MAX_SEED_LEN
+        ))
+    } else {
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,7 +11,7 @@ homepage = "https://solana.com/"
 [dependencies]
 dirs-next = "2.0.0"
 lazy_static = "1.4.0"
-serde = "1.0.112"
+serde = "1.0.118"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.13"
 url = "2.1.1"

cli-config/src/lib.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 #[macro_use]
 extern crate lazy_static;
 

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ console = "0.11.3"
 humantime = "2.0.1"
 Inflector = "0.11.4"
 indicatif = "0.15.0"
-serde = "1.0.112"
+serde = "1.0.118"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-client = { path = "../client", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.5" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.5" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.12" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-client = { path = "../client", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.12" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.12" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -1,38 +1,48 @@
-use crate::{
-    display::{build_balance_message, format_labeled_address, writeln_name_value},
-    QuietDisplay, VerboseDisplay,
-};
-use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
-use console::{style, Emoji};
-use inflector::cases::titlecase::to_title_case;
-use serde::{Deserialize, Serialize};
-use serde_json::{Map, Value};
-use solana_account_decoder::parse_token::UiTokenAccount;
-use solana_clap_utils::keypair::SignOnly;
-use solana_client::rpc_response::{
-    RpcAccountBalance, RpcInflationGovernor, RpcInflationRate, RpcKeyedAccount, RpcSupply,
-    RpcVoteAccountInfo,
-};
-use solana_sdk::{
-    clock::{self, Epoch, Slot, UnixTimestamp},
-    epoch_info::EpochInfo,
-    hash::Hash,
-    native_token::lamports_to_sol,
-    pubkey::Pubkey,
-    signature::Signature,
-    stake_history::StakeHistoryEntry,
-    transaction::Transaction,
-};
-use solana_stake_program::stake_state::{Authorized, Lockup};
-use solana_vote_program::{
-    authorized_voters::AuthorizedVoters,
-    vote_state::{BlockTimestamp, Lockout},
-};
-use std::{
-    collections::{BTreeMap, HashMap},
-    fmt,
-    str::FromStr,
-    time::Duration,
+use {
+    crate::{
+        display::{
+            build_balance_message, build_balance_message_with_config, format_labeled_address,
+            unix_timestamp_to_string, writeln_name_value, writeln_transaction,
+            BuildBalanceMessageConfig,
+        },
+        QuietDisplay, VerboseDisplay,
+    },
+    chrono::{Local, TimeZone},
+    console::{style, Emoji},
+    inflector::cases::titlecase::to_title_case,
+    serde::{Deserialize, Serialize},
+    serde_json::{Map, Value},
+    solana_account_decoder::parse_token::UiTokenAccount,
+    solana_clap_utils::keypair::SignOnly,
+    solana_client::rpc_response::{
+        RpcAccountBalance, RpcInflationGovernor, RpcInflationRate, RpcKeyedAccount, RpcSupply,
+        RpcVoteAccountInfo,
+    },
+    solana_sdk::{
+        clock::{self, Epoch, Slot, UnixTimestamp},
+        epoch_info::EpochInfo,
+        hash::Hash,
+        native_token::lamports_to_sol,
+        pubkey::Pubkey,
+        signature::Signature,
+        stake_history::StakeHistoryEntry,
+        transaction::{Transaction, TransactionError},
+    },
+    solana_stake_program::stake_state::{Authorized, Lockup},
+    solana_transaction_status::{
+        EncodedConfirmedBlock, EncodedTransaction, TransactionConfirmationStatus,
+        UiTransactionStatusMeta,
+    },
+    solana_vote_program::{
+        authorized_voters::AuthorizedVoters,
+        vote_state::{BlockTimestamp, Lockout},
+    },
+    std::{
+        collections::{BTreeMap, HashMap},
+        fmt,
+        str::FromStr,
+        time::Duration,
+    },
 };
 
 static WARNING: Emoji = Emoji("⚠️", "!");
@@ -359,6 +369,42 @@ impl fmt::Display for CliValidators {
                 },
             )
         }
+        writeln!(
+            f,
+            "{}",
+            style(format!(
+                "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
+                "Identity",
+                "Vote Account",
+                "Commission",
+                "Last Vote",
+                "Root Block",
+                "Credits",
+                "Version",
+                "Active Stake",
+            ))
+            .bold()
+        )?;
+        for validator in &self.current_validators {
+            write_vote_account(
+                f,
+                validator,
+                self.total_active_stake,
+                self.use_lamports_unit,
+                false,
+            )?;
+        }
+        for validator in &self.delinquent_validators {
+            write_vote_account(
+                f,
+                validator,
+                self.total_active_stake,
+                self.use_lamports_unit,
+                true,
+            )?;
+        }
+
+        writeln!(f)?;
         writeln_name_value(
             f,
             "Active Stake:",
@@ -410,41 +456,6 @@ impl fmt::Display for CliValidators {
             )?;
         }
 
-        writeln!(f)?;
-        writeln!(
-            f,
-            "{}",
-            style(format!(
-                "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
-                "Identity",
-                "Vote Account",
-                "Commission",
-                "Last Vote",
-                "Root Block",
-                "Credits",
-                "Version",
-                "Active Stake",
-            ))
-            .bold()
-        )?;
-        for validator in &self.current_validators {
-            write_vote_account(
-                f,
-                validator,
-                self.total_active_stake,
-                self.use_lamports_unit,
-                false,
-            )?;
-        }
-        for validator in &self.delinquent_validators {
-            write_vote_account(
-                f,
-                validator,
-                self.total_active_stake,
-                self.use_lamports_unit,
-                true,
-            )?;
-        }
         Ok(())
     }
 }
@@ -905,14 +916,19 @@ impl fmt::Display for CliStakeHistory {
             ))
             .bold()
         )?;
+        let config = BuildBalanceMessageConfig {
+            use_lamports_unit: self.use_lamports_unit,
+            show_unit: false,
+            trim_trailing_zeros: false,
+        };
         for entry in &self.entries {
             writeln!(
                 f,
                 "  {:>5}  {:>20}  {:>20}  {:>20} {}",
                 entry.epoch,
-                build_balance_message(entry.effective_stake, self.use_lamports_unit, false),
-                build_balance_message(entry.activating_stake, self.use_lamports_unit, false),
-                build_balance_message(entry.deactivating_stake, self.use_lamports_unit, false),
+                build_balance_message_with_config(entry.effective_stake, &config),
+                build_balance_message_with_config(entry.activating_stake, &config),
+                build_balance_message_with_config(entry.deactivating_stake, &config),
                 if self.use_lamports_unit {
                     "lamports"
                 } else {
@@ -1147,18 +1163,6 @@ pub struct CliBlockTime {
 impl QuietDisplay for CliBlockTime {}
 impl VerboseDisplay for CliBlockTime {}
 
-fn unix_timestamp_to_string(unix_timestamp: UnixTimestamp) -> String {
-    format!(
-        "{} (UnixTimestamp: {})",
-        match NaiveDateTime::from_timestamp_opt(unix_timestamp, 0) {
-            Some(ndt) =>
-                DateTime::<Utc>::from_utc(ndt, Utc).to_rfc3339_opts(SecondsFormat::Secs, true),
-            None => "unknown".to_string(),
-        },
-        unix_timestamp,
-    )
-}
-
 impl fmt::Display for CliBlockTime {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln_name_value(f, "Block:", &self.slot.to_string())?;
@@ -1402,17 +1406,17 @@ impl fmt::Display for CliSupply {
 
 #[derive(Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct CliFees {
+pub struct CliFeesInner {
     pub slot: Slot,
     pub blockhash: String,
     pub lamports_per_signature: u64,
-    pub last_valid_slot: Slot,
+    pub last_valid_slot: Option<Slot>,
 }
 
-impl QuietDisplay for CliFees {}
-impl VerboseDisplay for CliFees {}
+impl QuietDisplay for CliFeesInner {}
+impl VerboseDisplay for CliFeesInner {}
 
-impl fmt::Display for CliFees {
+impl fmt::Display for CliFeesInner {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln_name_value(f, "Blockhash:", &self.blockhash)?;
         writeln_name_value(
@@ -1420,8 +1424,51 @@ impl fmt::Display for CliFees {
             "Lamports per signature:",
             &self.lamports_per_signature.to_string(),
         )?;
-        writeln_name_value(f, "Last valid slot:", &self.last_valid_slot.to_string())?;
-        Ok(())
+        let last_valid_slot = self
+            .last_valid_slot
+            .map(|s| s.to_string())
+            .unwrap_or_default();
+        writeln_name_value(f, "Last valid slot:", &last_valid_slot)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliFees {
+    #[serde(flatten, skip_serializing_if = "Option::is_none")]
+    pub inner: Option<CliFeesInner>,
+}
+
+impl QuietDisplay for CliFees {}
+impl VerboseDisplay for CliFees {}
+
+impl fmt::Display for CliFees {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.inner.as_ref() {
+            Some(inner) => write!(f, "{}", inner),
+            None => write!(f, "Fees unavailable"),
+        }
+    }
+}
+
+impl CliFees {
+    pub fn some(
+        slot: Slot,
+        blockhash: Hash,
+        lamports_per_signature: u64,
+        last_valid_slot: Option<Slot>,
+    ) -> Self {
+        Self {
+            inner: Some(CliFeesInner {
+                slot,
+                blockhash: blockhash.to_string(),
+                lamports_per_signature,
+                last_valid_slot,
+            }),
+        }
+    }
+    pub fn none() -> Self {
+        Self { inner: None }
     }
 }
 
@@ -1469,6 +1516,114 @@ impl fmt::Display for CliTokenAccount {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliProgramId {
+    pub program_id: String,
+}
+
+impl QuietDisplay for CliProgramId {}
+impl VerboseDisplay for CliProgramId {}
+
+impl fmt::Display for CliProgramId {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Program Id:", &self.program_id)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliProgramBuffer {
+    pub buffer: String,
+}
+
+impl QuietDisplay for CliProgramBuffer {}
+impl VerboseDisplay for CliProgramBuffer {}
+
+impl fmt::Display for CliProgramBuffer {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Buffer:", &self.buffer)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum CliProgramAccountType {
+    Buffer,
+    Program,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliProgramAuthority {
+    pub authority: String,
+    pub account_type: CliProgramAccountType,
+}
+
+impl QuietDisplay for CliProgramAuthority {}
+impl VerboseDisplay for CliProgramAuthority {}
+
+impl fmt::Display for CliProgramAuthority {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Account Type:", &format!("{:?}", self.account_type))?;
+        writeln_name_value(f, "Authority:", &self.authority)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliUpgradeableProgram {
+    pub program_id: String,
+    pub programdata_address: String,
+    pub authority: String,
+    pub last_deploy_slot: u64,
+    pub data_len: usize,
+}
+impl QuietDisplay for CliUpgradeableProgram {}
+impl VerboseDisplay for CliUpgradeableProgram {}
+impl fmt::Display for CliUpgradeableProgram {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Program Id:", &self.program_id)?;
+        writeln_name_value(f, "ProgramData Address:", &self.programdata_address)?;
+        writeln_name_value(f, "Authority:", &self.authority)?;
+        writeln_name_value(
+            f,
+            "Last Deployed In Slot:",
+            &self.last_deploy_slot.to_string(),
+        )?;
+        writeln_name_value(
+            f,
+            "Data Length:",
+            &format!("{:?} ({:#x?}) bytes", self.data_len, self.data_len),
+        )?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliUpgradeableBuffer {
+    pub address: String,
+    pub authority: String,
+    pub data_len: usize,
+}
+impl QuietDisplay for CliUpgradeableBuffer {}
+impl VerboseDisplay for CliUpgradeableBuffer {}
+impl fmt::Display for CliUpgradeableBuffer {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Buffer Address:", &self.address)?;
+        writeln_name_value(f, "Authority:", &self.authority)?;
+        writeln_name_value(
+            f,
+            "Data Length:",
+            &format!("{:?} ({:#x?}) bytes", self.data_len, self.data_len),
+        )?;
+        Ok(())
+    }
+}
+
 pub fn return_signers(
     tx: &Transaction,
     output_format: &OutputFormat,
@@ -1555,6 +1710,224 @@ pub fn parse_sign_only_reply_string(reply: &str) -> SignOnly {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum CliSignatureVerificationStatus {
+    None,
+    Pass,
+    Fail,
+}
+
+impl CliSignatureVerificationStatus {
+    pub fn verify_transaction(tx: &Transaction) -> Vec<Self> {
+        tx.verify_with_results()
+            .iter()
+            .zip(&tx.signatures)
+            .map(|(stat, sig)| match stat {
+                true => CliSignatureVerificationStatus::Pass,
+                false if sig == &Signature::default() => CliSignatureVerificationStatus::None,
+                false => CliSignatureVerificationStatus::Fail,
+            })
+            .collect()
+    }
+}
+
+impl fmt::Display for CliSignatureVerificationStatus {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::None => write!(f, "none"),
+            Self::Pass => write!(f, "pass"),
+            Self::Fail => write!(f, "fail"),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliBlock {
+    #[serde(flatten)]
+    pub encoded_confirmed_block: EncodedConfirmedBlock,
+    #[serde(skip_serializing)]
+    pub slot: Slot,
+}
+
+impl QuietDisplay for CliBlock {}
+impl VerboseDisplay for CliBlock {}
+
+impl fmt::Display for CliBlock {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "Slot: {}", self.slot)?;
+        writeln!(
+            f,
+            "Parent Slot: {}",
+            self.encoded_confirmed_block.parent_slot
+        )?;
+        writeln!(f, "Blockhash: {}", self.encoded_confirmed_block.blockhash)?;
+        writeln!(
+            f,
+            "Previous Blockhash: {}",
+            self.encoded_confirmed_block.previous_blockhash
+        )?;
+        if let Some(block_time) = self.encoded_confirmed_block.block_time {
+            writeln!(f, "Block Time: {:?}", Local.timestamp(block_time, 0))?;
+        }
+        if !self.encoded_confirmed_block.rewards.is_empty() {
+            let mut rewards = self.encoded_confirmed_block.rewards.clone();
+            rewards.sort_by(|a, b| a.pubkey.cmp(&b.pubkey));
+            let mut total_rewards = 0;
+            writeln!(f, "Rewards:")?;
+            writeln!(
+                f,
+                "  {:<44}  {:^15}  {:<15}  {:<20}  {:>14}",
+                "Address", "Type", "Amount", "New Balance", "Percent Change"
+            )?;
+            for reward in rewards {
+                let sign = if reward.lamports < 0 { "-" } else { "" };
+
+                total_rewards += reward.lamports;
+                writeln!(
+                    f,
+                    "  {:<44}  {:^15}  {:>15}  {}",
+                    reward.pubkey,
+                    if let Some(reward_type) = reward.reward_type {
+                        format!("{}", reward_type)
+                    } else {
+                        "-".to_string()
+                    },
+                    format!(
+                        "{}◎{:<14.9}",
+                        sign,
+                        lamports_to_sol(reward.lamports.abs() as u64)
+                    ),
+                    if reward.post_balance == 0 {
+                        "          -                 -".to_string()
+                    } else {
+                        format!(
+                            "◎{:<19.9}  {:>13.9}%",
+                            lamports_to_sol(reward.post_balance),
+                            (reward.lamports.abs() as f64
+                                / (reward.post_balance as f64 - reward.lamports as f64))
+                                * 100.0
+                        )
+                    }
+                )?;
+            }
+
+            let sign = if total_rewards < 0 { "-" } else { "" };
+            writeln!(
+                f,
+                "Total Rewards: {}◎{:<12.9}",
+                sign,
+                lamports_to_sol(total_rewards.abs() as u64)
+            )?;
+        }
+        for (index, transaction_with_meta) in
+            self.encoded_confirmed_block.transactions.iter().enumerate()
+        {
+            writeln!(f, "Transaction {}:", index)?;
+            writeln_transaction(
+                f,
+                &transaction_with_meta.transaction.decode().unwrap(),
+                &transaction_with_meta.meta,
+                "  ",
+                None,
+                None,
+            )?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliTransaction {
+    pub transaction: EncodedTransaction,
+    pub meta: Option<UiTransactionStatusMeta>,
+    pub block_time: Option<UnixTimestamp>,
+    #[serde(skip_serializing)]
+    pub slot: Option<Slot>,
+    #[serde(skip_serializing)]
+    pub decoded_transaction: Transaction,
+    #[serde(skip_serializing)]
+    pub prefix: String,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub sigverify_status: Vec<CliSignatureVerificationStatus>,
+}
+
+impl QuietDisplay for CliTransaction {}
+impl VerboseDisplay for CliTransaction {}
+
+impl fmt::Display for CliTransaction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_transaction(
+            f,
+            &self.decoded_transaction,
+            &self.meta,
+            &self.prefix,
+            if !self.sigverify_status.is_empty() {
+                Some(&self.sigverify_status)
+            } else {
+                None
+            },
+            self.block_time,
+        )
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliTransactionConfirmation {
+    pub confirmation_status: Option<TransactionConfirmationStatus>,
+    #[serde(flatten, skip_serializing_if = "Option::is_none")]
+    pub transaction: Option<CliTransaction>,
+    #[serde(skip_serializing)]
+    pub get_transaction_error: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub err: Option<TransactionError>,
+}
+
+impl QuietDisplay for CliTransactionConfirmation {}
+impl VerboseDisplay for CliTransactionConfirmation {
+    fn write_str(&self, w: &mut dyn std::fmt::Write) -> std::fmt::Result {
+        if let Some(transaction) = &self.transaction {
+            writeln!(
+                w,
+                "\nTransaction executed in slot {}:",
+                transaction.slot.expect("slot should exist")
+            )?;
+            write!(w, "{}", transaction)?;
+        } else if let Some(confirmation_status) = &self.confirmation_status {
+            if confirmation_status != &TransactionConfirmationStatus::Finalized {
+                writeln!(w)?;
+                writeln!(
+                    w,
+                    "Unable to get finalized transaction details: not yet finalized"
+                )?;
+            } else if let Some(err) = &self.get_transaction_error {
+                writeln!(w)?;
+                writeln!(w, "Unable to get finalized transaction details: {}", err)?;
+            }
+        }
+        writeln!(w)?;
+        write!(w, "{}", self)
+    }
+}
+
+impl fmt::Display for CliTransactionConfirmation {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.confirmation_status {
+            None => write!(f, "Not found"),
+            Some(confirmation_status) => {
+                if let Some(err) = &self.err {
+                    write!(f, "Transaction failed: {}", err)
+                } else {
+                    write!(f, "{:?}", confirmation_status)
+                }
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

cli-output/src/display.rs

@@ -1,28 +1,73 @@
-use console::style;
-use indicatif::{ProgressBar, ProgressStyle};
-use solana_sdk::{
-    hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
-    transaction::Transaction,
+use {
+    crate::cli_output::CliSignatureVerificationStatus,
+    chrono::{DateTime, Local, NaiveDateTime, SecondsFormat, TimeZone, Utc},
+    console::style,
+    indicatif::{ProgressBar, ProgressStyle},
+    solana_sdk::{
+        clock::UnixTimestamp, hash::Hash, native_token::lamports_to_sol,
+        program_utils::limited_deserialize, transaction::Transaction,
+    },
+    solana_transaction_status::UiTransactionStatusMeta,
+    std::{collections::HashMap, fmt, io},
 };
-use solana_transaction_status::UiTransactionStatusMeta;
-use std::{collections::HashMap, fmt, io};
 
-pub fn build_balance_message(lamports: u64, use_lamports_unit: bool, show_unit: bool) -> String {
-    if use_lamports_unit {
-        let ess = if lamports == 1 { "" } else { "s" };
-        let unit = if show_unit {
-            format!(" lamport{}", ess)
-        } else {
-            "".to_string()
-        };
-        format!("{:?}{}", lamports, unit)
+#[derive(Clone, Debug)]
+pub struct BuildBalanceMessageConfig {
+    pub use_lamports_unit: bool,
+    pub show_unit: bool,
+    pub trim_trailing_zeros: bool,
+}
+
+impl Default for BuildBalanceMessageConfig {
+    fn default() -> Self {
+        Self {
+            use_lamports_unit: false,
+            show_unit: true,
+            trim_trailing_zeros: true,
+        }
+    }
+}
+
+pub fn build_balance_message_with_config(
+    lamports: u64,
+    config: &BuildBalanceMessageConfig,
+) -> String {
+    let value = if config.use_lamports_unit {
+        lamports.to_string()
     } else {
         let sol = lamports_to_sol(lamports);
         let sol_str = format!("{:.9}", sol);
-        let pretty_sol = sol_str.trim_end_matches('0').trim_end_matches('.');
-        let unit = if show_unit { " SOL" } else { "" };
-        format!("{}{}", pretty_sol, unit)
-    }
+        if config.trim_trailing_zeros {
+            sol_str
+                .trim_end_matches('0')
+                .trim_end_matches('.')
+                .to_string()
+        } else {
+            sol_str
+        }
+    };
+    let unit = if config.show_unit {
+        if config.use_lamports_unit {
+            let ess = if lamports == 1 { "" } else { "s" };
+            format!(" lamport{}", ess)
+        } else {
+            " SOL".to_string()
+        }
+    } else {
+        "".to_string()
+    };
+    format!("{}{}", value, unit)
+}
+
+pub fn build_balance_message(lamports: u64, use_lamports_unit: bool, show_unit: bool) -> String {
+    build_balance_message_with_config(
+        lamports,
+        &BuildBalanceMessageConfig {
+            use_lamports_unit,
+            show_unit,
+            ..BuildBalanceMessageConfig::default()
+        },
+    )
 }
 
 // Pretty print a "name value"
@@ -85,18 +130,41 @@ pub fn write_transaction<W: io::Write>(
     transaction: &Transaction,
     transaction_status: &Option<UiTransactionStatusMeta>,
     prefix: &str,
+    sigverify_status: Option<&[CliSignatureVerificationStatus]>,
+    block_time: Option<UnixTimestamp>,
 ) -> io::Result<()> {
     let message = &transaction.message;
+    if let Some(block_time) = block_time {
+        writeln!(
+            w,
+            "{}Block Time: {:?}",
+            prefix,
+            Local.timestamp(block_time, 0)
+        )?;
+    }
     writeln!(
         w,
         "{}Recent Blockhash: {:?}",
         prefix, message.recent_blockhash
     )?;
-    for (signature_index, signature) in transaction.signatures.iter().enumerate() {
+    let sigverify_statuses = if let Some(sigverify_status) = sigverify_status {
+        sigverify_status
+            .iter()
+            .map(|s| format!(" ({})", s))
+            .collect()
+    } else {
+        vec!["".to_string(); transaction.signatures.len()]
+    };
+    for (signature_index, (signature, sigverify_status)) in transaction
+        .signatures
+        .iter()
+        .zip(&sigverify_statuses)
+        .enumerate()
+    {
         writeln!(
             w,
-            "{}Signature {}: {:?}",
-            prefix, signature_index, signature
+            "{}Signature {}: {:?}{}",
+            prefix, signature_index, signature, sigverify_status,
         )?;
     }
     writeln!(w, "{}{:?}", prefix, message.header)?;
@@ -217,15 +285,52 @@ pub fn println_transaction(
     transaction: &Transaction,
     transaction_status: &Option<UiTransactionStatusMeta>,
     prefix: &str,
+    sigverify_status: Option<&[CliSignatureVerificationStatus]>,
+    block_time: Option<UnixTimestamp>,
 ) {
     let mut w = Vec::new();
-    if write_transaction(&mut w, transaction, transaction_status, prefix).is_ok() {
+    if write_transaction(
+        &mut w,
+        transaction,
+        transaction_status,
+        prefix,
+        sigverify_status,
+        block_time,
+    )
+    .is_ok()
+    {
         if let Ok(s) = String::from_utf8(w) {
             print!("{}", s);
         }
     }
 }
 
+pub fn writeln_transaction(
+    f: &mut dyn fmt::Write,
+    transaction: &Transaction,
+    transaction_status: &Option<UiTransactionStatusMeta>,
+    prefix: &str,
+    sigverify_status: Option<&[CliSignatureVerificationStatus]>,
+    block_time: Option<UnixTimestamp>,
+) -> fmt::Result {
+    let mut w = Vec::new();
+    if write_transaction(
+        &mut w,
+        transaction,
+        transaction_status,
+        prefix,
+        sigverify_status,
+        block_time,
+    )
+    .is_ok()
+    {
+        if let Ok(s) = String::from_utf8(w) {
+            write!(f, "{}", s)?;
+        }
+    }
+    Ok(())
+}
+
 /// Creates a new process bar for processing that will take an unknown amount of time
 pub fn new_spinner_progress_bar() -> ProgressBar {
     let progress_bar = ProgressBar::new(42);
@@ -235,6 +340,13 @@ pub fn new_spinner_progress_bar() -> ProgressBar {
     progress_bar
 }
 
+pub fn unix_timestamp_to_string(unix_timestamp: UnixTimestamp) -> String {
+    match NaiveDateTime::from_timestamp_opt(unix_timestamp, 0) {
+        Some(ndt) => DateTime::<Utc>::from_utc(ndt, Utc).to_rfc3339_opts(SecondsFormat::Secs, true),
+        None => format!("UnixTimestamp {}", unix_timestamp),
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

cli-output/src/lib.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 mod cli_output;
 pub mod display;
 pub use cli_output::*;

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.5"
+version = "1.5.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,32 +24,32 @@ humantime = "2.0.1"
 num-traits = "0.2"
 pretty-hex = "0.2.1"
 reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-serde = "1.0.112"
+serde = "1.0.118"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.5" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-cli-config = { path = "../cli-config", version = "1.5.5" }
-solana-cli-output = { path = "../cli-output", version = "1.5.5" }
-solana-client = { path = "../client", version = "1.5.5" }
-solana-config-program = { path = "../programs/config", version = "1.5.5" }
-solana-faucet = { path = "../faucet", version = "1.5.5" }
-solana-logger = { path = "../logger", version = "1.5.5" }
-solana-net-utils = { path = "../net-utils", version = "1.5.5" }
-solana_rbpf = "=0.2.4"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.5" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.5" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.12" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.12" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-cli-config = { path = "../cli-config", version = "1.5.12" }
+solana-cli-output = { path = "../cli-output", version = "1.5.12" }
+solana-client = { path = "../client", version = "1.5.12" }
+solana-config-program = { path = "../programs/config", version = "1.5.12" }
+solana-faucet = { path = "../faucet", version = "1.5.12" }
+solana-logger = { path = "../logger", version = "1.5.12" }
+solana-net-utils = { path = "../net-utils", version = "1.5.12" }
+solana_rbpf = "=0.2.5"
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.12" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.12" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.5.5" }
+solana-core = { path = "../core", version = "1.5.12" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/checks.rs

@@ -86,9 +86,13 @@ pub fn check_account_for_spend_multiple_fees_with_commitment(
             return Err(CliError::InsufficientFundsForSpendAndFee(
                 lamports_to_sol(balance),
                 lamports_to_sol(fee),
+                *account_pubkey,
             ));
         } else {
-            return Err(CliError::InsufficientFundsForFee(lamports_to_sol(fee)));
+            return Err(CliError::InsufficientFundsForFee(
+                lamports_to_sol(fee),
+                *account_pubkey,
+            ));
         }
     }
     Ok(())

cli/src/cli.rs

@@ -17,8 +17,9 @@ use solana_clap_utils::{
     offline::*,
 };
 use solana_cli_output::{
-    display::{build_balance_message, println_name_value, println_transaction},
-    return_signers, CliAccount, CliSignature, OutputFormat,
+    display::{build_balance_message, println_name_value},
+    return_signers, CliAccount, CliSignature, CliSignatureVerificationStatus, CliTransaction,
+    CliTransactionConfirmation, OutputFormat,
 };
 use solana_client::{
     blockhash_query::BlockhashQuery,
@@ -40,7 +41,7 @@ use solana_sdk::{
     hash::Hash,
     instruction::InstructionError,
     message::Message,
-    pubkey::{Pubkey, MAX_SEED_LEN},
+    pubkey::Pubkey,
     signature::{Signature, Signer, SignerError},
     system_instruction::{self, SystemError},
     system_program,
@@ -50,9 +51,7 @@ use solana_stake_program::{
     stake_instruction::LockupArgs,
     stake_state::{Lockup, StakeAuthorize},
 };
-use solana_transaction_status::{
-    EncodedTransaction, TransactionConfirmationStatus, UiTransactionEncoding,
-};
+use solana_transaction_status::{EncodedTransaction, UiTransactionEncoding};
 use solana_vote_program::vote_state::VoteAuthorize;
 use std::{
     collections::HashMap,
@@ -91,7 +90,9 @@ pub enum CliCommand {
     },
     Feature(FeatureCliCommand),
     Inflation(InflationCliCommand),
-    Fees,
+    Fees {
+        blockhash: Option<Hash>,
+    },
     FirstAvailableBlock,
     GetBlock {
         slot: Option<Slot>,
@@ -260,6 +261,7 @@ pub enum CliCommand {
         nonce_account: Option<Pubkey>,
         nonce_authority: SignerIndex,
         fee_payer: SignerIndex,
+        custodian: Option<SignerIndex>,
     },
     StakeSetLockup {
         stake_account_pubkey: Pubkey,
@@ -354,6 +356,8 @@ pub enum CliCommand {
         nonce_account: Option<Pubkey>,
         nonce_authority: SignerIndex,
         fee_payer: SignerIndex,
+        derived_address_seed: Option<String>,
+        derived_address_program_id: Option<Pubkey>,
     },
 }
 
@@ -365,25 +369,25 @@ pub struct CliCommandInfo {
 
 #[derive(Debug, Error)]
 pub enum CliError {
-    #[error("bad parameter: {0}")]
+    #[error("Bad parameter: {0}")]
     BadParameter(String),
     #[error(transparent)]
     ClientError(#[from] ClientError),
-    #[error("command not recognized: {0}")]
+    #[error("Command not recognized: {0}")]
     CommandNotRecognized(String),
-    #[error("insufficient funds for fee ({0} SOL)")]
-    InsufficientFundsForFee(f64),
-    #[error("insufficient funds for spend ({0} SOL)")]
-    InsufficientFundsForSpend(f64),
-    #[error("insufficient funds for spend ({0} SOL) and fee ({1} SOL)")]
-    InsufficientFundsForSpendAndFee(f64, f64),
+    #[error("Account {1} has insufficient funds for fee ({0} SOL)")]
+    InsufficientFundsForFee(f64, Pubkey),
+    #[error("Account {1} has insufficient funds for spend ({0} SOL)")]
+    InsufficientFundsForSpend(f64, Pubkey),
+    #[error("Account {2} has insufficient funds for spend ({0} SOL) + fee ({1} SOL)")]
+    InsufficientFundsForSpendAndFee(f64, f64, Pubkey),
     #[error(transparent)]
     InvalidNonce(nonce_utils::Error),
-    #[error("dynamic program error: {0}")]
+    #[error("Dynamic program error: {0}")]
     DynamicProgramError(String),
-    #[error("rpc request error: {0}")]
+    #[error("RPC request error: {0}")]
     RpcRequestError(String),
-    #[error("keypair file not found: {0}")]
+    #[error("Keypair file not found: {0}")]
     KeypairFileNotFound(String),
 }
 
@@ -470,11 +474,15 @@ impl CliConfig<'_> {
             (SettingType::Explicit, websocket_cfg_url.to_string()),
             (
                 SettingType::Computed,
-                solana_cli_config::Config::compute_websocket_url(json_rpc_cmd_url),
+                solana_cli_config::Config::compute_websocket_url(&normalize_to_url_if_moniker(
+                    json_rpc_cmd_url,
+                )),
             ),
             (
                 SettingType::Computed,
-                solana_cli_config::Config::compute_websocket_url(json_rpc_cfg_url),
+                solana_cli_config::Config::compute_websocket_url(&normalize_to_url_if_moniker(
+                    json_rpc_cfg_url,
+                )),
             ),
             (SettingType::SystemDefault, Self::default_websocket_url()),
         ])
@@ -587,10 +595,13 @@ pub fn parse_command(
         ("feature", Some(matches)) => {
             parse_feature_subcommand(matches, default_signer, wallet_manager)
         }
-        ("fees", Some(_matches)) => Ok(CliCommandInfo {
-            command: CliCommand::Fees,
-            signers: vec![],
-        }),
+        ("fees", Some(matches)) => {
+            let blockhash = value_of::<Hash>(matches, "blockhash");
+            Ok(CliCommandInfo {
+                command: CliCommand::Fees { blockhash },
+                signers: vec![],
+            })
+        }
         ("first-available-block", Some(_matches)) => Ok(CliCommandInfo {
             command: CliCommand::FirstAvailableBlock,
             signers: vec![],
@@ -853,6 +864,12 @@ pub fn parse_command(
             let signer_info =
                 default_signer.generate_unique_signers(bulk_signers, matches, wallet_manager)?;
 
+            let derived_address_seed = matches
+                .value_of("derived_address_seed")
+                .map(|s| s.to_string());
+            let derived_address_program_id =
+                resolve_derived_address_program_id(matches, "derived_address_program_id");
+
             Ok(CliCommandInfo {
                 command: CliCommand::Transfer {
                     amount,
@@ -864,6 +881,8 @@ pub fn parse_command(
                     nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
                     fee_payer: signer_info.index_of(fee_payer_pubkey).unwrap(),
                     from: signer_info.index_of(from_pubkey).unwrap(),
+                    derived_address_seed,
+                    derived_address_program_id,
                 },
                 signers: signer_info.signers,
             })
@@ -893,6 +912,15 @@ pub fn parse_command(
 
 pub type ProcessResult = Result<String, Box<dyn std::error::Error>>;
 
+fn resolve_derived_address_program_id(matches: &ArgMatches<'_>, arg_name: &str) -> Option<Pubkey> {
+    matches.value_of(arg_name).and_then(|v| match v {
+        "NONCE" => Some(system_program::id()),
+        "STAKE" => Some(solana_stake_program::id()),
+        "VOTE" => Some(solana_vote_program::id()),
+        _ => pubkey_of(matches, arg_name),
+    })
+}
+
 pub fn parse_create_address_with_seed(
     matches: &ArgMatches<'_>,
     default_signer: &DefaultSigner,
@@ -905,21 +933,10 @@ pub fn parse_create_address_with_seed(
         vec![default_signer.signer_from_path(matches, wallet_manager)?]
     };
 
-    let program_id = match matches.value_of("program_id").unwrap() {
-        "NONCE" => system_program::id(),
-        "STAKE" => solana_stake_program::id(),
-        "VOTE" => solana_vote_program::id(),
-        _ => pubkey_of(matches, "program_id").unwrap(),
-    };
+    let program_id = resolve_derived_address_program_id(matches, "program_id").unwrap();
 
     let seed = matches.value_of("seed").unwrap().to_string();
 
-    if seed.len() > MAX_SEED_LEN {
-        return Err(CliError::BadParameter(
-            "Address seed must not be longer than 32 bytes".to_string(),
-        ));
-    }
-
     Ok(CliCommandInfo {
         command: CliCommand::CreateAddressWithSeed {
             from_pubkey,
@@ -992,58 +1009,72 @@ fn process_confirm(
 ) -> ProcessResult {
     match rpc_client.get_signature_statuses_with_history(&[*signature]) {
         Ok(status) => {
-            if let Some(transaction_status) = &status.value[0] {
+            let cli_transaction = if let Some(transaction_status) = &status.value[0] {
+                let mut transaction = None;
+                let mut get_transaction_error = None;
                 if config.verbose {
                     match rpc_client
                         .get_confirmed_transaction(signature, UiTransactionEncoding::Base64)
                     {
                         Ok(confirmed_transaction) => {
-                            println!(
-                                "\nTransaction executed in slot {}:",
-                                confirmed_transaction.slot
-                            );
-                            println_transaction(
-                                &confirmed_transaction
-                                    .transaction
-                                    .transaction
-                                    .decode()
-                                    .expect("Successful decode"),
-                                &confirmed_transaction.transaction.meta,
-                                "  ",
+                            let decoded_transaction = confirmed_transaction
+                                .transaction
+                                .transaction
+                                .decode()
+                                .expect("Successful decode");
+                            let json_transaction = EncodedTransaction::encode(
+                                decoded_transaction.clone(),
+                                UiTransactionEncoding::Json,
                             );
+
+                            transaction = Some(CliTransaction {
+                                transaction: json_transaction,
+                                meta: confirmed_transaction.transaction.meta,
+                                block_time: confirmed_transaction.block_time,
+                                slot: Some(confirmed_transaction.slot),
+                                decoded_transaction,
+                                prefix: "  ".to_string(),
+                                sigverify_status: vec![],
+                            });
                         }
                         Err(err) => {
-                            if transaction_status.confirmation_status()
-                                != TransactionConfirmationStatus::Finalized
-                            {
-                                println!();
-                                println!("Unable to get finalized transaction details: not yet finalized")
-                            } else {
-                                println!();
-                                println!("Unable to get finalized transaction details: {}", err)
-                            }
+                            get_transaction_error = Some(format!("{:?}", err));
                         }
                     }
-                    println!();
                 }
-
-                if let Some(err) = &transaction_status.err {
-                    Ok(format!("Transaction failed: {}", err))
-                } else {
-                    Ok(format!("{:?}", transaction_status.confirmation_status()))
+                CliTransactionConfirmation {
+                    confirmation_status: Some(transaction_status.confirmation_status()),
+                    transaction,
+                    get_transaction_error,
+                    err: transaction_status.err.clone(),
                 }
             } else {
-                Ok("Not found".to_string())
-            }
+                CliTransactionConfirmation {
+                    confirmation_status: None,
+                    transaction: None,
+                    get_transaction_error: None,
+                    err: None,
+                }
+            };
+            Ok(config.output_format.formatted_string(&cli_transaction))
         }
         Err(err) => Err(CliError::RpcRequestError(format!("Unable to confirm: {}", err)).into()),
     }
 }
 
 #[allow(clippy::unnecessary_wraps)]
-fn process_decode_transaction(transaction: &Transaction) -> ProcessResult {
-    println_transaction(transaction, &None, "");
-    Ok("".to_string())
+fn process_decode_transaction(config: &CliConfig, transaction: &Transaction) -> ProcessResult {
+    let sigverify_status = CliSignatureVerificationStatus::verify_transaction(&transaction);
+    let decode_transaction = CliTransaction {
+        decoded_transaction: transaction.clone(),
+        transaction: EncodedTransaction::encode(transaction.clone(), UiTransactionEncoding::Json),
+        meta: None,
+        block_time: None,
+        slot: None,
+        prefix: "".to_string(),
+        sigverify_status,
+    };
+    Ok(config.output_format.formatted_string(&decode_transaction))
 }
 
 fn process_show_account(
@@ -1101,8 +1132,11 @@ fn process_transfer(
     nonce_account: Option<&Pubkey>,
     nonce_authority: SignerIndex,
     fee_payer: SignerIndex,
+    derived_address_seed: Option<String>,
+    derived_address_program_id: Option<&Pubkey>,
 ) -> ProcessResult {
     let from = config.signers[from];
+    let mut from_pubkey = from.pubkey();
 
     let (recent_blockhash, fee_calculator) =
         blockhash_query.get_blockhash_and_fee_calculator(rpc_client, config.commitment)?;
@@ -1110,8 +1144,28 @@ fn process_transfer(
     let nonce_authority = config.signers[nonce_authority];
     let fee_payer = config.signers[fee_payer];
 
+    let derived_parts = derived_address_seed.zip(derived_address_program_id);
+    let with_seed = if let Some((seed, program_id)) = derived_parts {
+        let base_pubkey = from_pubkey;
+        from_pubkey = Pubkey::create_with_seed(&base_pubkey, &seed, program_id)?;
+        Some((base_pubkey, seed, program_id, from_pubkey))
+    } else {
+        None
+    };
+
     let build_message = |lamports| {
-        let ixs = vec![system_instruction::transfer(&from.pubkey(), to, lamports)];
+        let ixs = if let Some((base_pubkey, seed, program_id, from_pubkey)) = with_seed.as_ref() {
+            vec![system_instruction::transfer_with_seed(
+                from_pubkey,
+                base_pubkey,
+                seed.clone(),
+                program_id,
+                to,
+                lamports,
+            )]
+        } else {
+            vec![system_instruction::transfer(&from_pubkey, to, lamports)]
+        };
 
         if let Some(nonce_account) = &nonce_account {
             Message::new_with_nonce(
@@ -1130,7 +1184,7 @@ fn process_transfer(
         sign_only,
         amount,
         &fee_calculator,
-        &from.pubkey(),
+        &from_pubkey,
         &fee_payer.pubkey(),
         build_message,
         config.commitment,
@@ -1214,7 +1268,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             seed,
             program_id,
         } => process_create_address_with_seed(config, from_pubkey.as_ref(), &seed, &program_id),
-        CliCommand::Fees => process_fees(&rpc_client, config),
+        CliCommand::Fees { ref blockhash } => process_fees(&rpc_client, config, blockhash.as_ref()),
         CliCommand::Feature(feature_subcommand) => {
             process_feature_subcommand(&rpc_client, config, feature_subcommand)
         }
@@ -1520,11 +1574,13 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             nonce_account,
             nonce_authority,
             fee_payer,
+            custodian,
         } => process_stake_authorize(
             &rpc_client,
             config,
             &stake_account_pubkey,
             new_authorizations,
+            *custodian,
             *sign_only,
             blockhash_query,
             *nonce_account,
@@ -1705,7 +1761,9 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         } => process_balance(&rpc_client, config, &pubkey, *use_lamports_unit),
         // Confirm the last client transaction by signature
         CliCommand::Confirm(signature) => process_confirm(&rpc_client, config, signature),
-        CliCommand::DecodeTransaction(transaction) => process_decode_transaction(transaction),
+        CliCommand::DecodeTransaction(transaction) => {
+            process_decode_transaction(config, transaction)
+        }
         CliCommand::ResolveSigner(path) => {
             if let Some(path) = path {
                 Ok(path.to_string())
@@ -1734,6 +1792,8 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             ref nonce_account,
             nonce_authority,
             fee_payer,
+            derived_address_seed,
+            ref derived_address_program_id,
         } => process_transfer(
             &rpc_client,
             config,
@@ -1746,6 +1806,8 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             nonce_account.as_ref(),
             *nonce_authority,
             *fee_payer,
+            derived_address_seed.clone(),
+            derived_address_program_id.as_ref(),
         ),
     }
 }
@@ -1929,7 +1991,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
         )
         .subcommand(
             SubCommand::with_name("decode-transaction")
-                .about("Decode a base-58 binary transaction")
+                .about("Decode a serialized transaction")
                 .arg(
                     Arg::with_name("transaction")
                         .index(1)
@@ -1958,6 +2020,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .value_name("SEED_STRING")
                         .takes_value(true)
                         .required(true)
+                        .validator(is_derived_address_seed)
                         .help("The seed.  Must not take more than 32 bytes to encode as utf-8"),
                 )
                 .arg(
@@ -2078,6 +2141,23 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .takes_value(false)
                         .help("Return signature immediately after submitting the transaction, instead of waiting for confirmations"),
                 )
+                .arg(
+                    Arg::with_name("derived_address_seed")
+                        .long("derived-address-seed")
+                        .takes_value(true)
+                        .value_name("SEED_STRING")
+                        .requires("derived_address_program_id")
+                        .validator(is_derived_address_seed)
+                        .hidden(true)
+                )
+                .arg(
+                    Arg::with_name("derived_address_program_id")
+                        .long("derived-address-program-id")
+                        .takes_value(true)
+                        .value_name("PROGRAM_ID")
+                        .requires("derived_address_seed")
+                        .hidden(true)
+                )
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg()),
@@ -2127,6 +2207,7 @@ mod tests {
         signature::{keypair_from_seed, read_keypair_file, write_keypair_file, Keypair, Presigner},
         transaction::TransactionError,
     };
+    use solana_transaction_status::TransactionConfirmationStatus;
     use std::path::PathBuf;
 
     fn make_tmp_path(name: &str) -> String {
@@ -2726,7 +2807,7 @@ mod tests {
         let program_id = json
             .as_object()
             .unwrap()
-            .get("ProgramId")
+            .get("programId")
             .unwrap()
             .as_str()
             .unwrap();
@@ -2778,6 +2859,8 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    derived_address_seed: None,
+                    derived_address_program_id: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2800,6 +2883,8 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    derived_address_seed: None,
+                    derived_address_program_id: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2826,6 +2911,8 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    derived_address_seed: None,
+                    derived_address_program_id: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2856,6 +2943,8 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    derived_address_seed: None,
+                    derived_address_program_id: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2894,6 +2983,8 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    derived_address_seed: None,
+                    derived_address_program_id: None,
                 },
                 signers: vec![Presigner::new(&from_pubkey, &from_sig).into()],
             }
@@ -2933,6 +3024,8 @@ mod tests {
                     nonce_account: Some(nonce_address),
                     nonce_authority: 1,
                     fee_payer: 0,
+                    derived_address_seed: None,
+                    derived_address_program_id: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2940,5 +3033,38 @@ mod tests {
                 ],
             }
         );
+
+        //Test Transfer Subcommand, with seed
+        let derived_address_seed = "seed".to_string();
+        let derived_address_program_id = "STAKE";
+        let test_transfer = test_commands.clone().get_matches_from(vec![
+            "test",
+            "transfer",
+            &to_string,
+            "42",
+            "--derived-address-seed",
+            &derived_address_seed,
+            "--derived-address-program-id",
+            derived_address_program_id,
+        ]);
+        assert_eq!(
+            parse_command(&test_transfer, &default_signer, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::Transfer {
+                    amount: SpendAmount::Some(42_000_000_000),
+                    to: to_pubkey,
+                    from: 0,
+                    sign_only: false,
+                    no_wait: false,
+                    blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+                    nonce_account: None,
+                    nonce_authority: 0,
+                    fee_payer: 0,
+                    derived_address_seed: Some(derived_address_seed),
+                    derived_address_program_id: Some(solana_stake_program::id()),
+                },
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
+            }
+        );
     }
 }

cli/src/cluster_query.rs

@@ -3,7 +3,6 @@ use crate::{
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
     stake::is_stake_program_v2_enabled,
 };
-use chrono::{Local, TimeZone};
 use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use serde::{Deserialize, Serialize};
@@ -16,7 +15,7 @@ use solana_clap_utils::{
 use solana_cli_output::{
     display::{
         build_balance_message, format_labeled_address, new_spinner_progress_bar,
-        println_name_value, println_transaction, writeln_name_value,
+        println_name_value, println_transaction, unix_timestamp_to_string, writeln_name_value,
     },
     *,
 };
@@ -135,7 +134,17 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("cluster-version")
                 .about("Get the version of the cluster entrypoint"),
         )
-        .subcommand(SubCommand::with_name("fees").about("Display current cluster fees"),
+        .subcommand(
+            SubCommand::with_name("fees")
+            .about("Display current cluster fees")
+            .arg(
+                Arg::with_name("blockhash")
+                    .long("blockhash")
+                    .takes_value(true)
+                    .value_name("BLOCKHASH")
+                    .validator(is_hash)
+                    .help("Query fees for BLOCKHASH instead of the the most recent blockhash")
+            ),
         )
         .subcommand(
             SubCommand::with_name("first-available-block")
@@ -821,14 +830,35 @@ pub fn process_cluster_version(rpc_client: &RpcClient, config: &CliConfig) -> Pr
     }
 }
 
-pub fn process_fees(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let result = rpc_client.get_recent_blockhash_with_commitment(config.commitment)?;
-    let (recent_blockhash, fee_calculator, last_valid_slot) = result.value;
-    let fees = CliFees {
-        slot: result.context.slot,
-        blockhash: recent_blockhash.to_string(),
-        lamports_per_signature: fee_calculator.lamports_per_signature,
-        last_valid_slot,
+pub fn process_fees(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    blockhash: Option<&Hash>,
+) -> ProcessResult {
+    let fees = if let Some(recent_blockhash) = blockhash {
+        let result = rpc_client.get_fee_calculator_for_blockhash_with_commitment(
+            recent_blockhash,
+            config.commitment,
+        )?;
+        if let Some(fee_calculator) = result.value {
+            CliFees::some(
+                result.context.slot,
+                *recent_blockhash,
+                fee_calculator.lamports_per_signature,
+                None,
+            )
+        } else {
+            CliFees::none()
+        }
+    } else {
+        let result = rpc_client.get_recent_blockhash_with_commitment(config.commitment)?;
+        let (recent_blockhash, fee_calculator, last_valid_slot) = result.value;
+        CliFees::some(
+            result.context.slot,
+            recent_blockhash,
+            fee_calculator.lamports_per_signature,
+            Some(last_valid_slot),
+        )
     };
     Ok(config.output_format.formatted_string(&fees))
 }
@@ -896,7 +926,7 @@ pub fn process_leader_schedule(
 
 pub fn process_get_block(
     rpc_client: &RpcClient,
-    _config: &CliConfig,
+    config: &CliConfig,
     slot: Option<Slot>,
 ) -> ProcessResult {
     let slot = if let Some(slot) = slot {
@@ -905,71 +935,13 @@ pub fn process_get_block(
         rpc_client.get_slot_with_commitment(CommitmentConfig::finalized())?
     };
 
-    let mut block =
+    let encoded_confirmed_block =
         rpc_client.get_confirmed_block_with_encoding(slot, UiTransactionEncoding::Base64)?;
-
-    println!("Slot: {}", slot);
-    println!("Parent Slot: {}", block.parent_slot);
-    println!("Blockhash: {}", block.blockhash);
-    println!("Previous Blockhash: {}", block.previous_blockhash);
-    if let Some(block_time) = block.block_time {
-        println!("Block Time: {:?}", Local.timestamp(block_time, 0));
-    }
-    if !block.rewards.is_empty() {
-        block.rewards.sort_by(|a, b| a.pubkey.cmp(&b.pubkey));
-        let mut total_rewards = 0;
-        println!("Rewards:",);
-        println!(
-            "  {:<44}  {:^15}  {:<15}  {:<20}  {:>14}",
-            "Address", "Type", "Amount", "New Balance", "Percent Change"
-        );
-        for reward in block.rewards {
-            let sign = if reward.lamports < 0 { "-" } else { "" };
-
-            total_rewards += reward.lamports;
-            println!(
-                "  {:<44}  {:^15}  {:>15}  {}",
-                reward.pubkey,
-                if let Some(reward_type) = reward.reward_type {
-                    format!("{}", reward_type)
-                } else {
-                    "-".to_string()
-                },
-                format!(
-                    "{}◎{:<14.9}",
-                    sign,
-                    lamports_to_sol(reward.lamports.abs() as u64)
-                ),
-                if reward.post_balance == 0 {
-                    "          -                 -".to_string()
-                } else {
-                    format!(
-                        "◎{:<19.9}  {:>13.9}%",
-                        lamports_to_sol(reward.post_balance),
-                        (reward.lamports.abs() as f64
-                            / (reward.post_balance as f64 - reward.lamports as f64))
-                            * 100.0
-                    )
-                }
-            );
-        }
-
-        let sign = if total_rewards < 0 { "-" } else { "" };
-        println!(
-            "Total Rewards: {}◎{:<12.9}",
-            sign,
-            lamports_to_sol(total_rewards.abs() as u64)
-        );
-    }
-    for (index, transaction_with_meta) in block.transactions.iter().enumerate() {
-        println!("Transaction {}:", index);
-        println_transaction(
-            &transaction_with_meta.transaction.decode().unwrap(),
-            &transaction_with_meta.meta,
-            "  ",
-        );
-    }
-    Ok("".to_string())
+    let cli_block = CliBlock {
+        encoded_confirmed_block,
+        slot,
+    };
+    Ok(config.output_format.formatted_string(&cli_block))
 }
 
 pub fn process_get_block_time(
@@ -1637,7 +1609,7 @@ pub fn process_show_stakes(
         CliError::RpcRequestError("Failed to deserialize stake history".to_string())
     })?;
     // At v1.6, this check can be removed and simply passed as `true`
-    let stake_program_v2_enabled = is_stake_program_v2_enabled(rpc_client);
+    let stake_program_v2_enabled = is_stake_program_v2_enabled(rpc_client)?;
 
     let mut stake_accounts: Vec<CliKeyedStakeState> = vec![];
     for (stake_pubkey, stake_account) in all_stake_accounts {
@@ -1813,9 +1785,14 @@ pub fn process_transaction_history(
     for result in results {
         if config.verbose {
             println!(
-                "{} [slot={} status={}] {}",
+                "{} [slot={} {}status={}] {}",
                 result.signature,
                 result.slot,
+                match result.block_time {
+                    None => "".to_string(),
+                    Some(block_time) =>
+                        format!("timestamp={} ", unix_timestamp_to_string(block_time)),
+                },
                 match result.err {
                     None => "Confirmed".to_string(),
                     Some(err) => format!("Failed: {:?}", err),
@@ -1840,6 +1817,8 @@ pub fn process_transaction_history(
                                 .expect("Successful decode"),
                             &confirmed_transaction.transaction.meta,
                             "  ",
+                            None,
+                            None,
                         );
                     }
                     Err(err) => println!("  Unable to get confirmed transaction details: {}", err),
@@ -1957,7 +1936,24 @@ mod tests {
         assert_eq!(
             parse_command(&test_fees, &default_signer, &mut None).unwrap(),
             CliCommandInfo {
-                command: CliCommand::Fees,
+                command: CliCommand::Fees { blockhash: None },
+                signers: vec![],
+            }
+        );
+
+        let blockhash = Hash::new_unique();
+        let test_fees = test_commands.clone().get_matches_from(vec![
+            "test",
+            "fees",
+            "--blockhash",
+            &blockhash.to_string(),
+        ]);
+        assert_eq!(
+            parse_command(&test_fees, &default_signer, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::Fees {
+                    blockhash: Some(blockhash)
+                },
                 signers: vec![],
             }
         );

cli/src/feature.rs

@@ -70,8 +70,8 @@ impl fmt::Display for CliFeatures {
                 f,
                 "{}",
                 style(format!(
-                    "{:<44} {:<40} {}",
-                    "Feature", "Description", "Status"
+                    "{:<44} | {:<27} | {}",
+                    "Feature", "Status", "Description"
                 ))
                 .bold()
             )?;
@@ -79,15 +79,15 @@ impl fmt::Display for CliFeatures {
         for feature in &self.features {
             writeln!(
                 f,
-                "{:<44} {:<40} {}",
+                "{:<44} | {:<27} | {}",
                 feature.id,
-                feature.description,
                 match feature.status {
                     CliFeatureStatus::Inactive => style("inactive".to_string()).red(),
                     CliFeatureStatus::Pending => style("activation pending".to_string()).yellow(),
                     CliFeatureStatus::Active(activation_slot) =>
                         style(format!("active since slot {}", activation_slot)).green(),
-                }
+                },
+                feature.description,
             )?;
         }
         if self.inactive && !self.feature_activation_allowed {
@@ -221,7 +221,7 @@ pub fn process_feature_subcommand(
     }
 }
 
-fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, u64>, ClientError> {
+fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, f64>, ClientError> {
     // Validator identity -> feature set
     let feature_set_map = rpc_client
         .get_cluster_nodes()?
@@ -239,7 +239,7 @@ fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, u6
         .sum();
 
     // Sum all active stake by feature set
-    let mut active_stake_by_feature_set = HashMap::new();
+    let mut active_stake_by_feature_set: HashMap<u32, u64> = HashMap::new();
     for vote_account in vote_accounts.current {
         if let Some(Some(feature_set)) = feature_set_map.get(&vote_account.node_pubkey) {
             *active_stake_by_feature_set.entry(*feature_set).or_default() +=
@@ -251,11 +251,15 @@ fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, u6
         }
     }
 
-    // Convert active stake to a percentage so the caller doesn't need `total_active_stake`
-    for (_, val) in active_stake_by_feature_set.iter_mut() {
-        *val = *val * 100 / total_active_stake;
-    }
-    Ok(active_stake_by_feature_set)
+    Ok(active_stake_by_feature_set
+        .into_iter()
+        .map(|(feature_set, active_stake)| {
+            (
+                feature_set,
+                active_stake as f64 * 100. / total_active_stake as f64,
+            )
+        })
+        .collect())
 }
 
 // Feature activation is only allowed when 95% of the active stake is on the current feature set
@@ -266,7 +270,7 @@ fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<boo
 
     let feature_activation_allowed = active_stake_by_feature_set
         .get(&my_feature_set)
-        .map(|percentage| *percentage >= 95)
+        .map(|percentage| *percentage >= 95.)
         .unwrap_or(false);
 
     if !feature_activation_allowed && !quiet {
@@ -283,15 +287,15 @@ fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<boo
         }
         println!(
             "{}",
-            style(format!("Tool Feture Set: {}", my_feature_set)).bold()
+            style(format!("Tool Feature Set: {}", my_feature_set)).bold()
         );
         println!("{}", style("Cluster Feature Sets and Stakes:").bold());
         for (feature_set, percentage) in active_stake_by_feature_set.iter() {
             if *feature_set == 0 {
-                println!("unknown - {}%", percentage);
+                println!("  unknown    - {:.2}%", percentage);
             } else {
                 println!(
-                    "{} - {}% {}",
+                    "  {:<10} - {:.2}% {}",
                     feature_set,
                     percentage,
                     if *feature_set == my_feature_set {

cli/src/lib.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 macro_rules! ACCOUNT_STRING {
     () => {
         r#", one of:

cli/src/main.rs

@@ -4,7 +4,7 @@ use clap::{
 };
 use console::style;
 use solana_clap_utils::{
-    input_validators::{is_url, is_url_or_moniker},
+    input_validators::{is_url, is_url_or_moniker, normalize_to_url_if_moniker},
     keypair::{CliSigners, DefaultSigner, SKIP_SEED_PHRASE_VALIDATION_ARG},
     DisplayError,
 };
@@ -90,7 +90,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                 }
                 ("set", Some(subcommand_matches)) => {
                     if let Some(url) = subcommand_matches.value_of("json_rpc_url") {
-                        config.json_rpc_url = url.to_string();
+                        config.json_rpc_url = normalize_to_url_if_moniker(url);
                         // Revert to a computed `websocket_url` value when `json_rpc_url` is
                         // changed
                         config.websocket_url = "".to_string();
@@ -245,6 +245,7 @@ pub fn parse_args<'a>(
 }
 
 fn main() -> Result<(), Box<dyn error::Error>> {
+    solana_logger::setup_with_default("off");
     let matches = app(
         crate_name!(),
         crate_description!(),

cli/src/nonce.rs

@@ -457,11 +457,11 @@ pub fn process_new_nonce(
         (&nonce_account, "nonce_account_pubkey".to_string()),
     )?;
 
-    let nonce_account_check = rpc_client.get_account(&nonce_account);
-    if nonce_account_check.is_err() {
-        return Err(CliError::BadParameter(
-            "Unable to create new nonce, no nonce account found".to_string(),
-        )
+    if let Err(err) = rpc_client.get_account(&nonce_account) {
+        return Err(CliError::BadParameter(format!(
+            "Unable to advance nonce account {}. error: {}",
+            nonce_account, err
+        ))
         .into());
     }
 

cli/src/send_tpu.rs

@@ -1,21 +1,38 @@
 use log::*;
 use solana_client::rpc_response::{RpcContactInfo, RpcLeaderSchedule};
+use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS;
 use std::net::{SocketAddr, UdpSocket};
 
-pub fn get_leader_tpu(
+pub fn get_leader_tpus(
     slot_index: u64,
+    num_leaders: u64,
     leader_schedule: Option<&RpcLeaderSchedule>,
     cluster_nodes: Option<&Vec<RpcContactInfo>>,
-) -> Option<SocketAddr> {
-    leader_schedule?
-        .iter()
-        .find(|(_pubkey, slots)| slots.iter().any(|slot| *slot as u64 == slot_index))
-        .and_then(|(pubkey, _)| {
-            cluster_nodes?
+) -> Vec<SocketAddr> {
+    let leaders: Vec<_> = (0..num_leaders)
+        .filter_map(|i| {
+            leader_schedule?
                 .iter()
-                .find(|contact_info| contact_info.pubkey == *pubkey)
-                .and_then(|contact_info| contact_info.tpu)
+                .find(|(_pubkey, slots)| {
+                    slots.iter().any(|slot| {
+                        *slot as u64 == (slot_index + (i * NUM_CONSECUTIVE_LEADER_SLOTS))
+                    })
+                })
+                .and_then(|(pubkey, _)| {
+                    cluster_nodes?
+                        .iter()
+                        .find(|contact_info| contact_info.pubkey == *pubkey)
+                        .and_then(|contact_info| contact_info.tpu)
+                })
         })
+        .collect();
+    let mut unique_leaders = vec![];
+    for leader in leaders.into_iter() {
+        if !unique_leaders.contains(&leader) {
+            unique_leaders.push(leader);
+        }
+    }
+    unique_leaders
 }
 
 pub fn send_transaction_tpu(

cli/src/spend_utils.rs

@@ -107,15 +107,22 @@ where
                 return Err(CliError::InsufficientFundsForSpendAndFee(
                     lamports_to_sol(spend),
                     lamports_to_sol(fee),
+                    *from_pubkey,
                 ));
             }
         } else {
             if from_balance < spend {
-                return Err(CliError::InsufficientFundsForSpend(lamports_to_sol(spend)));
+                return Err(CliError::InsufficientFundsForSpend(
+                    lamports_to_sol(spend),
+                    *from_pubkey,
+                ));
             }
             if !check_account_for_balance_with_commitment(rpc_client, fee_pubkey, fee, commitment)?
             {
-                return Err(CliError::InsufficientFundsForFee(lamports_to_sol(fee)));
+                return Err(CliError::InsufficientFundsForFee(
+                    lamports_to_sol(fee),
+                    *fee_pubkey,
+                ));
             }
         }
         Ok((message, spend))

cli/src/stake.rs

@@ -64,6 +64,12 @@ pub const WITHDRAW_AUTHORITY_ARG: ArgConstant<'static> = ArgConstant {
     help: "Authorized withdrawer [default: cli config keypair]",
 };
 
+pub const CUSTODIAN_ARG: ArgConstant<'static> = ArgConstant {
+    name: "custodian",
+    long: "custodian",
+    help: "Authority to override account lockup",
+};
+
 fn stake_authority_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name(STAKE_AUTHORITY_ARG.name)
         .long(STAKE_AUTHORITY_ARG.long)
@@ -82,6 +88,15 @@ fn withdraw_authority_arg<'a, 'b>() -> Arg<'a, 'b> {
         .help(WITHDRAW_AUTHORITY_ARG.help)
 }
 
+fn custodian_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(CUSTODIAN_ARG.name)
+        .long(CUSTODIAN_ARG.long)
+        .takes_value(true)
+        .value_name("KEYPAIR")
+        .validator(is_valid_signer)
+        .help(CUSTODIAN_ARG.help)
+}
+
 pub trait StakeSubCommands {
     fn stake_subcommands(self) -> Self;
 }
@@ -223,6 +238,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(custodian_arg())
         )
         .subcommand(
             SubCommand::with_name("deactivate-stake")
@@ -252,7 +268,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("split_stake_account")
                         .index(2)
-                        .value_name("ACCOUNT_KEYPAIR")
+                        .value_name("SPLIT_STAKE_ACCOUNT")
                         .takes_value(true)
                         .required(true)
                         .validator(is_valid_signer)
@@ -272,7 +288,7 @@ impl StakeSubCommands for App<'_, '_> {
                         .long("seed")
                         .value_name("STRING")
                         .takes_value(true)
-                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the SPLIT STAKE ACCOUNT pubkey")
+                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the SPLIT_STAKE_ACCOUNT pubkey")
                 )
                 .arg(stake_authority_arg())
                 .offline_args()
@@ -331,14 +347,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
-                .arg(
-                    Arg::with_name("custodian")
-                        .long("custodian")
-                        .takes_value(true)
-                        .value_name("KEYPAIR")
-                        .validator(is_valid_signer)
-                        .help("Authority to override account lockup")
-                )
+                .arg(custodian_arg())
         )
         .subcommand(
             SubCommand::with_name("stake-set-lockup")
@@ -561,11 +570,15 @@ pub fn parse_stake_authorize(
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
     let (fee_payer, fee_payer_pubkey) = signer_of(matches, FEE_PAYER_ARG.name, wallet_manager)?;
+    let (custodian, custodian_pubkey) = signer_of(matches, "custodian", wallet_manager)?;
 
     bulk_signers.push(fee_payer);
     if nonce_account.is_some() {
         bulk_signers.push(nonce_authority);
     }
+    if custodian.is_some() {
+        bulk_signers.push(custodian);
+    }
     let signer_info =
         default_signer.generate_unique_signers(bulk_signers, matches, wallet_manager)?;
 
@@ -591,6 +604,7 @@ pub fn parse_stake_authorize(
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
             fee_payer: signer_info.index_of(fee_payer_pubkey).unwrap(),
+            custodian: custodian_pubkey.and_then(|_| signer_info.index_of(custodian_pubkey)),
         },
         signers: signer_info.signers,
     })
@@ -970,6 +984,7 @@ pub fn process_stake_authorize(
     config: &CliConfig,
     stake_account_pubkey: &Pubkey,
     new_authorizations: &[(StakeAuthorize, Pubkey, SignerIndex)],
+    custodian: Option<SignerIndex>,
     sign_only: bool,
     blockhash_query: &BlockhashQuery,
     nonce_account: Option<Pubkey>,
@@ -977,6 +992,7 @@ pub fn process_stake_authorize(
     fee_payer: SignerIndex,
 ) -> ProcessResult {
     let mut ixs = Vec::new();
+    let custodian = custodian.map(|index| config.signers[index]);
     for (stake_authorize, authorized_pubkey, authority) in new_authorizations.iter() {
         check_unique_pubkeys(
             (stake_account_pubkey, "stake_account_pubkey".to_string()),
@@ -988,6 +1004,7 @@ pub fn process_stake_authorize(
             &authority.pubkey(),  // currently authorized
             authorized_pubkey,    // new stake signer
             *stake_authorize,     // stake or withdraw
+            custodian.map(|signer| signer.pubkey()).as_ref(),
         ));
     }
 
@@ -1701,7 +1718,7 @@ pub fn process_show_stake_account(
                 use_lamports_unit,
                 &stake_history,
                 &clock,
-                is_stake_program_v2_enabled(rpc_client), // At v1.6, this check can be removed and simply passed as `true`
+                is_stake_program_v2_enabled(rpc_client)?, // At v1.6, this check can be removed and simply passed as `true`
             );
 
             if state.stake_type == CliStakeType::Stake {
@@ -1770,10 +1787,10 @@ pub fn process_delegate_stake(
         // voted at the tip of the ledger
         let vote_account_data = rpc_client
             .get_account(vote_account_pubkey)
-            .map_err(|_| {
+            .map_err(|err| {
                 CliError::RpcRequestError(format!(
-                    "Vote account not found: {}",
-                    vote_account_pubkey
+                    "Vote account not found: {}. error: {}",
+                    vote_account_pubkey, err,
                 ))
             })?
             .data;
@@ -1861,13 +1878,13 @@ pub fn process_delegate_stake(
     }
 }
 
-pub fn is_stake_program_v2_enabled(rpc_client: &RpcClient) -> bool {
-    rpc_client
-        .get_account(&feature_set::stake_program_v2::id())
-        .ok()
-        .and_then(|account| feature::from_account(&account))
+pub fn is_stake_program_v2_enabled(
+    rpc_client: &RpcClient,
+) -> Result<bool, Box<dyn std::error::Error>> {
+    let feature_account = rpc_client.get_account(&feature_set::stake_program_v2::id())?;
+    Ok(feature::from_account(&feature_account)
         .and_then(|feature| feature.activated_at)
-        .is_some()
+        .is_some())
 }
 
 #[cfg(test)]
@@ -1939,6 +1956,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
             },
@@ -1973,6 +1991,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2011,6 +2030,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2038,6 +2058,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
             },
@@ -2062,6 +2083,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2092,6 +2114,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2123,6 +2146,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
             },
@@ -2151,6 +2175,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2185,6 +2210,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2221,6 +2247,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2267,6 +2294,7 @@ mod tests {
                     nonce_account: Some(nonce_account),
                     nonce_authority: 2,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2299,6 +2327,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -2336,6 +2365,7 @@ mod tests {
                     nonce_account: Some(nonce_account_pubkey),
                     nonce_authority: 1,
                     fee_payer: 0,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2369,6 +2399,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -2406,6 +2437,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     fee_payer: 1,
+                    custodian: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),

cli/tests/nonce.rs

@@ -18,7 +18,6 @@ use solana_sdk::{
     signature::{keypair_from_seed, Keypair, Signer},
     system_program,
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_nonce() {
@@ -59,9 +58,7 @@ fn full_battery_tests(
     seed: Option<String>,
     use_nonce_authority: bool,
 ) {
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -219,9 +216,7 @@ fn test_create_account_with_seed() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
 
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let offline_nonce_authority_signer = keypair_from_seed(&[1u8; 32]).unwrap();
     let online_nonce_creator_signer = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -303,6 +298,8 @@ fn test_create_account_with_seed() {
         nonce_account: Some(nonce_address),
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     authority_config.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&authority_config).unwrap();
@@ -327,6 +324,8 @@ fn test_create_account_with_seed() {
         nonce_account: Some(nonce_address),
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&submit_config).unwrap();
     check_recent_balance(241, &rpc_client, &nonce_address);

cli/tests/program.rs

@@ -3,6 +3,7 @@ use solana_cli::{
     cli::{process_command, CliCommand, CliConfig},
     program::ProgramCliCommand,
 };
+use solana_cli_output::OutputFormat;
 use solana_client::rpc_client::RpcClient;
 use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
@@ -14,7 +15,7 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::{Keypair, Signer},
 };
-use std::{fs::File, io::Read, path::PathBuf, str::FromStr, sync::mpsc::channel};
+use std::{env, fs::File, io::Read, path::PathBuf, str::FromStr};
 
 #[test]
 fn test_cli_program_deploy_non_upgradeable() {
@@ -28,10 +29,7 @@ fn test_cli_program_deploy_non_upgradeable() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -66,7 +64,7 @@ fn test_cli_program_deploy_non_upgradeable() {
     let program_id_str = json
         .as_object()
         .unwrap()
-        .get("ProgramId")
+        .get("programId")
         .unwrap()
         .as_str()
         .unwrap();
@@ -149,10 +147,7 @@ fn test_cli_program_deploy_no_authority() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -183,8 +178,8 @@ fn test_cli_program_deploy_no_authority() {
     config.signers = vec![&keypair];
     process_command(&config).unwrap();
 
-    // Deploy a program with no authority
-    config.signers = vec![&keypair];
+    // Deploy a program
+    config.signers = vec![&keypair, &upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
         program_location: Some(pathbuf.to_str().unwrap().to_string()),
         program_signer_index: None,
@@ -192,9 +187,8 @@ fn test_cli_program_deploy_no_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: None,
-        upgrade_authority_pubkey: None,
-        is_final: false,
+        upgrade_authority_signer_index: 1,
+        is_final: true,
         max_len: None,
     });
     let response = process_command(&config);
@@ -202,7 +196,7 @@ fn test_cli_program_deploy_no_authority() {
     let program_id_str = json
         .as_object()
         .unwrap()
-        .get("ProgramId")
+        .get("programId")
         .unwrap()
         .as_str()
         .unwrap();
@@ -217,8 +211,7 @@ fn test_cli_program_deploy_no_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: Some(1),
-        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
     });
@@ -237,10 +230,7 @@ fn test_cli_program_deploy_with_authority() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -281,17 +271,17 @@ fn test_cli_program_deploy_with_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: Some(1),
-        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
     });
+    config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let program_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("ProgramId")
+        .get("programId")
         .unwrap()
         .as_str()
         .unwrap();
@@ -328,8 +318,7 @@ fn test_cli_program_deploy_with_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: Some(1),
-        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
     });
@@ -338,7 +327,7 @@ fn test_cli_program_deploy_with_authority() {
     let program_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("ProgramId")
+        .get("programId")
         .unwrap()
         .as_str()
         .unwrap();
@@ -370,8 +359,7 @@ fn test_cli_program_deploy_with_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: Some(1),
-        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
     });
@@ -407,7 +395,7 @@ fn test_cli_program_deploy_with_authority() {
     let new_upgrade_authority_str = json
         .as_object()
         .unwrap()
-        .get("Authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
@@ -425,8 +413,7 @@ fn test_cli_program_deploy_with_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: Some(1),
-        upgrade_authority_pubkey: Some(new_upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
     });
@@ -451,7 +438,7 @@ fn test_cli_program_deploy_with_authority() {
 
     // Get upgrade authority
     config.signers = vec![&keypair];
-    config.command = CliCommand::Program(ProgramCliCommand::GetAuthority {
+    config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(program_pubkey),
     });
     let response = process_command(&config);
@@ -459,7 +446,7 @@ fn test_cli_program_deploy_with_authority() {
     let authority_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Program upgrade authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
@@ -480,11 +467,11 @@ fn test_cli_program_deploy_with_authority() {
     let new_upgrade_authority_str = json
         .as_object()
         .unwrap()
-        .get("Authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
-    assert_eq!(new_upgrade_authority_str, "None");
+    assert_eq!(new_upgrade_authority_str, "none");
 
     // Upgrade with no authority
     config.signers = vec![&keypair, &new_upgrade_authority];
@@ -495,8 +482,7 @@ fn test_cli_program_deploy_with_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: Some(1),
-        upgrade_authority_pubkey: Some(new_upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
     });
@@ -511,8 +497,7 @@ fn test_cli_program_deploy_with_authority() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         allow_excessive_balance: false,
-        upgrade_authority_signer_index: Some(1),
-        upgrade_authority_pubkey: Some(new_upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
     });
@@ -521,7 +506,7 @@ fn test_cli_program_deploy_with_authority() {
     let program_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("ProgramId")
+        .get("programId")
         .unwrap()
         .as_str()
         .unwrap();
@@ -541,7 +526,7 @@ fn test_cli_program_deploy_with_authority() {
 
     // Get buffer authority
     config.signers = vec![&keypair];
-    config.command = CliCommand::Program(ProgramCliCommand::GetAuthority {
+    config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(program_pubkey),
     });
     let response = process_command(&config);
@@ -549,11 +534,11 @@ fn test_cli_program_deploy_with_authority() {
     let authority_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Program upgrade authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
-    assert_eq!("None", authority_pubkey_str);
+    assert_eq!("none", authority_pubkey_str);
 }
 
 #[test]
@@ -568,10 +553,7 @@ fn test_cli_program_write_buffer() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -587,7 +569,7 @@ fn test_cli_program_write_buffer() {
         .unwrap();
     let minimum_balance_for_buffer_default = rpc_client
         .get_minimum_balance_for_rent_exemption(
-            UpgradeableLoaderState::programdata_len(max_len * 2).unwrap(),
+            UpgradeableLoaderState::programdata_len(max_len).unwrap(),
         )
         .unwrap();
 
@@ -610,15 +592,15 @@ fn test_cli_program_write_buffer() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         buffer_authority_signer_index: None,
-        is_final: false,
         max_len: None,
     });
+    config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let buffer_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Buffer")
+        .get("buffer")
         .unwrap()
         .as_str()
         .unwrap();
@@ -644,7 +626,6 @@ fn test_cli_program_write_buffer() {
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
-        is_final: false,
         max_len: Some(max_len),
     });
     let response = process_command(&config);
@@ -652,7 +633,7 @@ fn test_cli_program_write_buffer() {
     let buffer_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Buffer")
+        .get("buffer")
         .unwrap()
         .as_str()
         .unwrap();
@@ -675,7 +656,7 @@ fn test_cli_program_write_buffer() {
 
     // Get buffer authority
     config.signers = vec![&keypair];
-    config.command = CliCommand::Program(ProgramCliCommand::GetAuthority {
+    config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(buffer_keypair.pubkey()),
     });
     let response = process_command(&config);
@@ -683,7 +664,7 @@ fn test_cli_program_write_buffer() {
     let authority_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Buffer authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
@@ -701,7 +682,6 @@ fn test_cli_program_write_buffer() {
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
-        is_final: false,
         max_len: None,
     });
     let response = process_command(&config);
@@ -709,7 +689,7 @@ fn test_cli_program_write_buffer() {
     let buffer_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Buffer")
+        .get("buffer")
         .unwrap()
         .as_str()
         .unwrap();
@@ -739,7 +719,6 @@ fn test_cli_program_write_buffer() {
         buffer_signer_index: None,
         buffer_pubkey: None,
         buffer_authority_signer_index: Some(2),
-        is_final: false,
         max_len: None,
     });
     let response = process_command(&config);
@@ -747,7 +726,7 @@ fn test_cli_program_write_buffer() {
     let buffer_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Buffer")
+        .get("buffer")
         .unwrap()
         .as_str()
         .unwrap();
@@ -765,38 +744,9 @@ fn test_cli_program_write_buffer() {
         program_data[..]
     );
 
-    // Specify final
-    let buffer_keypair = Keypair::new();
-    let authority_keypair = Keypair::new();
-    config.signers = vec![&keypair, &buffer_keypair, &authority_keypair];
-    config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer_signer_index: None,
-        buffer_pubkey: None,
-        buffer_authority_signer_index: Some(2),
-        is_final: true,
-        max_len: None,
-    });
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let buffer_pubkey_str = json
-        .as_object()
-        .unwrap()
-        .get("Buffer")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    let buffer_pubkey = Pubkey::from_str(&buffer_pubkey_str).unwrap();
-    let buffer_account = rpc_client.get_account(&buffer_pubkey).unwrap();
-    if let UpgradeableLoaderState::Buffer { authority_address } = buffer_account.state().unwrap() {
-        assert_eq!(authority_address, None);
-    } else {
-        panic!("not a buffer account");
-    }
-
     // Get buffer authority
     config.signers = vec![&keypair];
-    config.command = CliCommand::Program(ProgramCliCommand::GetAuthority {
+    config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(buffer_pubkey),
     });
     let response = process_command(&config);
@@ -804,11 +754,14 @@ fn test_cli_program_write_buffer() {
     let authority_pubkey_str = json
         .as_object()
         .unwrap()
-        .get("Buffer authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
-    assert_eq!("None", authority_pubkey_str);
+    assert_eq!(
+        authority_keypair.pubkey(),
+        Pubkey::from_str(&authority_pubkey_str).unwrap()
+    );
 }
 
 #[test]
@@ -823,10 +776,7 @@ fn test_cli_program_set_buffer_authority() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -861,7 +811,6 @@ fn test_cli_program_set_buffer_authority() {
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
-        is_final: false,
         max_len: None,
     });
     process_command(&config).unwrap();
@@ -878,14 +827,14 @@ fn test_cli_program_set_buffer_authority() {
     config.command = CliCommand::Program(ProgramCliCommand::SetBufferAuthority {
         buffer_pubkey: buffer_keypair.pubkey(),
         buffer_authority_index: Some(0),
-        new_buffer_authority: Some(new_buffer_authority.pubkey()),
+        new_buffer_authority: new_buffer_authority.pubkey(),
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let new_buffer_authority_str = json
         .as_object()
         .unwrap()
-        .get("Authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
@@ -905,14 +854,14 @@ fn test_cli_program_set_buffer_authority() {
     config.command = CliCommand::Program(ProgramCliCommand::SetBufferAuthority {
         buffer_pubkey: buffer_keypair.pubkey(),
         buffer_authority_index: Some(1),
-        new_buffer_authority: Some(buffer_keypair.pubkey()),
+        new_buffer_authority: buffer_keypair.pubkey(),
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let buffer_authority_str = json
         .as_object()
         .unwrap()
-        .get("Authority")
+        .get("authority")
         .unwrap()
         .as_str()
         .unwrap();
@@ -926,28 +875,346 @@ fn test_cli_program_set_buffer_authority() {
     } else {
         panic!("not a buffer account");
     }
+}
 
-    // Set authority to None
-    config.signers = vec![&keypair, &buffer_keypair];
-    config.command = CliCommand::Program(ProgramCliCommand::SetBufferAuthority {
-        buffer_pubkey: buffer_keypair.pubkey(),
-        buffer_authority_index: Some(1),
-        new_buffer_authority: None,
-    });
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let buffer_authority_str = json
-        .as_object()
-        .unwrap()
-        .get("Authority")
-        .unwrap()
-        .as_str()
+#[test]
+fn test_cli_program_mismatch_buffer_authority() {
+    solana_logger::setup();
+
+    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    pathbuf.push("tests");
+    pathbuf.push("fixtures");
+    pathbuf.push("noop");
+    pathbuf.set_extension("so");
+
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut program_data = Vec::new();
+    file.read_to_end(&mut program_data).unwrap();
+    let max_len = program_data.len();
+    let minimum_balance_for_buffer = rpc_client
+        .get_minimum_balance_for_rent_exemption(
+            UpgradeableLoaderState::programdata_len(max_len).unwrap(),
+        )
         .unwrap();
-    assert_eq!(buffer_authority_str, "None");
+
+    let mut config = CliConfig::recent_for_tests();
+    let keypair = Keypair::new();
+    config.json_rpc_url = test_validator.rpc_url();
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Airdrop {
+        faucet_host: None,
+        faucet_port: faucet_addr.port(),
+        pubkey: None,
+        lamports: 100 * minimum_balance_for_buffer,
+    };
+    process_command(&config).unwrap();
+
+    // Write a buffer
+    let buffer_authority = Keypair::new();
+    let buffer_keypair = Keypair::new();
+    config.signers = vec![&keypair, &buffer_keypair, &buffer_authority];
+    config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
+        program_location: pathbuf.to_str().unwrap().to_string(),
+        buffer_signer_index: Some(1),
+        buffer_pubkey: Some(buffer_keypair.pubkey()),
+        buffer_authority_signer_index: Some(2),
+        max_len: None,
+    });
+    process_command(&config).unwrap();
     let buffer_account = rpc_client.get_account(&buffer_keypair.pubkey()).unwrap();
     if let UpgradeableLoaderState::Buffer { authority_address } = buffer_account.state().unwrap() {
-        assert_eq!(authority_address, None);
+        assert_eq!(authority_address, Some(buffer_authority.pubkey()));
     } else {
         panic!("not a buffer account");
     }
+
+    // Attempt to deploy with mismatched authority
+    let upgrade_authority = Keypair::new();
+    config.signers = vec![&keypair, &upgrade_authority];
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
+        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_signer_index: None,
+        program_pubkey: None,
+        buffer_signer_index: None,
+        buffer_pubkey: Some(buffer_keypair.pubkey()),
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: 1,
+        is_final: true,
+        max_len: None,
+    });
+    process_command(&config).unwrap_err();
+
+    // Attempt to deploy matched authority
+    config.signers = vec![&keypair, &buffer_authority];
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
+        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_signer_index: None,
+        program_pubkey: None,
+        buffer_signer_index: None,
+        buffer_pubkey: Some(buffer_keypair.pubkey()),
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: 1,
+        is_final: true,
+        max_len: None,
+    });
+    process_command(&config).unwrap();
+}
+
+#[test]
+fn test_cli_program_show() {
+    solana_logger::setup();
+
+    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    pathbuf.push("tests");
+    pathbuf.push("fixtures");
+    pathbuf.push("noop");
+    pathbuf.set_extension("so");
+
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut program_data = Vec::new();
+    file.read_to_end(&mut program_data).unwrap();
+    let max_len = program_data.len();
+    let minimum_balance_for_buffer = rpc_client
+        .get_minimum_balance_for_rent_exemption(
+            UpgradeableLoaderState::programdata_len(max_len).unwrap(),
+        )
+        .unwrap();
+
+    let mut config = CliConfig::recent_for_tests();
+    let keypair = Keypair::new();
+    config.json_rpc_url = test_validator.rpc_url();
+    config.output_format = OutputFormat::Json;
+
+    // Airdrop
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Airdrop {
+        faucet_host: None,
+        faucet_port: faucet_addr.port(),
+        pubkey: None,
+        lamports: 100 * minimum_balance_for_buffer,
+    };
+    process_command(&config).unwrap();
+
+    // Write a buffer
+    let buffer_keypair = Keypair::new();
+    let authority_keypair = Keypair::new();
+    config.signers = vec![&keypair, &buffer_keypair, &authority_keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
+        program_location: pathbuf.to_str().unwrap().to_string(),
+        buffer_signer_index: Some(1),
+        buffer_pubkey: Some(buffer_keypair.pubkey()),
+        buffer_authority_signer_index: Some(2),
+        max_len: None,
+    });
+    process_command(&config).unwrap();
+
+    // Verify show
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Show {
+        account_pubkey: Some(buffer_keypair.pubkey()),
+    });
+    let response = process_command(&config);
+    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
+    let address_str = json
+        .as_object()
+        .unwrap()
+        .get("address")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    assert_eq!(
+        buffer_keypair.pubkey(),
+        Pubkey::from_str(&address_str).unwrap()
+    );
+    let authority_str = json
+        .as_object()
+        .unwrap()
+        .get("authority")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    assert_eq!(
+        authority_keypair.pubkey(),
+        Pubkey::from_str(&authority_str).unwrap()
+    );
+    let data_len = json
+        .as_object()
+        .unwrap()
+        .get("dataLen")
+        .unwrap()
+        .as_u64()
+        .unwrap();
+    assert_eq!(max_len, data_len as usize);
+
+    // Deploy
+    let program_keypair = Keypair::new();
+    config.signers = vec![&keypair, &authority_keypair, &program_keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
+        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_signer_index: Some(2),
+        program_pubkey: Some(program_keypair.pubkey()),
+        buffer_signer_index: None,
+        buffer_pubkey: None,
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: 1,
+        is_final: false,
+        max_len: Some(max_len),
+    });
+    config.output_format = OutputFormat::JsonCompact;
+    let min_slot = rpc_client.get_slot().unwrap();
+    process_command(&config).unwrap();
+    let max_slot = rpc_client.get_slot().unwrap();
+
+    // Verify show
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Show {
+        account_pubkey: Some(program_keypair.pubkey()),
+    });
+    let response = process_command(&config);
+    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
+    let address_str = json
+        .as_object()
+        .unwrap()
+        .get("programId")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    assert_eq!(
+        program_keypair.pubkey(),
+        Pubkey::from_str(&address_str).unwrap()
+    );
+    let programdata_address_str = json
+        .as_object()
+        .unwrap()
+        .get("programdataAddress")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    let (programdata_pubkey, _) = Pubkey::find_program_address(
+        &[program_keypair.pubkey().as_ref()],
+        &bpf_loader_upgradeable::id(),
+    );
+    assert_eq!(
+        programdata_pubkey,
+        Pubkey::from_str(&programdata_address_str).unwrap()
+    );
+    let authority_str = json
+        .as_object()
+        .unwrap()
+        .get("authority")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    assert_eq!(
+        authority_keypair.pubkey(),
+        Pubkey::from_str(&authority_str).unwrap()
+    );
+    let deployed_slot = json
+        .as_object()
+        .unwrap()
+        .get("lastDeploySlot")
+        .unwrap()
+        .as_u64()
+        .unwrap();
+    assert!(deployed_slot >= min_slot);
+    assert!(deployed_slot <= max_slot);
+    let data_len = json
+        .as_object()
+        .unwrap()
+        .get("dataLen")
+        .unwrap()
+        .as_u64()
+        .unwrap();
+    assert_eq!(max_len, data_len as usize);
+}
+
+#[test]
+fn test_cli_program_dump() {
+    solana_logger::setup();
+
+    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    pathbuf.push("tests");
+    pathbuf.push("fixtures");
+    pathbuf.push("noop");
+    pathbuf.set_extension("so");
+
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut program_data = Vec::new();
+    file.read_to_end(&mut program_data).unwrap();
+    let max_len = program_data.len();
+    let minimum_balance_for_buffer = rpc_client
+        .get_minimum_balance_for_rent_exemption(
+            UpgradeableLoaderState::programdata_len(max_len).unwrap(),
+        )
+        .unwrap();
+
+    let mut config = CliConfig::recent_for_tests();
+    let keypair = Keypair::new();
+    config.json_rpc_url = test_validator.rpc_url();
+    config.output_format = OutputFormat::Json;
+
+    // Airdrop
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Airdrop {
+        faucet_host: None,
+        faucet_port: faucet_addr.port(),
+        pubkey: None,
+        lamports: 100 * minimum_balance_for_buffer,
+    };
+    process_command(&config).unwrap();
+
+    // Write a buffer
+    let buffer_keypair = Keypair::new();
+    let authority_keypair = Keypair::new();
+    config.signers = vec![&keypair, &buffer_keypair, &authority_keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
+        program_location: pathbuf.to_str().unwrap().to_string(),
+        buffer_signer_index: Some(1),
+        buffer_pubkey: Some(buffer_keypair.pubkey()),
+        buffer_authority_signer_index: Some(2),
+        max_len: None,
+    });
+    process_command(&config).unwrap();
+
+    // Verify dump
+    let mut out_file = {
+        let current_exe = env::current_exe().unwrap();
+        PathBuf::from(current_exe.parent().unwrap().parent().unwrap())
+    };
+    out_file.set_file_name("out.txt");
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Dump {
+        account_pubkey: Some(buffer_keypair.pubkey()),
+        output_location: out_file.clone().into_os_string().into_string().unwrap(),
+    });
+    process_command(&config).unwrap();
+
+    let mut file = File::open(out_file).unwrap();
+    let mut out_data = Vec::new();
+    file.read_to_end(&mut out_data).unwrap();
+    assert_eq!(program_data.len(), out_data.len());
+    for i in 0..program_data.len() {
+        assert_eq!(program_data[i], out_data[i]);
+    }
 }

cli/tests/request_airdrop.rs

@@ -6,16 +6,13 @@ use solana_sdk::{
     commitment_config::CommitmentConfig,
     signature::{Keypair, Signer},
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_cli_request_airdrop() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
 
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let mut bob_config = CliConfig::recent_for_tests();
     bob_config.json_rpc_url = test_validator.rpc_url();

cli/tests/stake.rs

@@ -22,15 +22,12 @@ use solana_stake_program::{
     stake_instruction::LockupArgs,
     stake_state::{Lockup, StakeAuthorize, StakeState},
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_stake_delegation_force() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -117,9 +114,7 @@ fn test_seed_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -197,9 +192,7 @@ fn test_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -273,9 +266,7 @@ fn test_offline_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -295,7 +286,7 @@ fn test_offline_stake_delegation_and_deactivation() {
     config_offline.command = CliCommand::ClusterVersion;
     let offline_keypair = Keypair::new();
     config_offline.signers = vec![&offline_keypair];
-    // Verfiy that we cannot reach the cluster
+    // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
     request_and_confirm_airdrop(
@@ -406,9 +397,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -521,9 +510,7 @@ fn test_stake_authorize() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -548,7 +535,7 @@ fn test_stake_authorize() {
     config_offline.json_rpc_url = String::default();
     let offline_authority_pubkey = config_offline.signers[0].pubkey();
     config_offline.command = CliCommand::ClusterVersion;
-    // Verfiy that we cannot reach the cluster
+    // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
     request_and_confirm_airdrop(
@@ -592,6 +579,7 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
     let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
@@ -619,6 +607,7 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
     let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
@@ -641,6 +630,7 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
     let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
@@ -663,6 +653,7 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     config_offline.output_format = OutputFormat::JsonCompact;
     let sign_reply = process_command(&config_offline).unwrap();
@@ -678,6 +669,7 @@ fn test_stake_authorize() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
     let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
@@ -724,6 +716,7 @@ fn test_stake_authorize() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     let sign_reply = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_reply);
@@ -743,6 +736,7 @@ fn test_stake_authorize() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
     let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
@@ -771,9 +765,7 @@ fn test_stake_authorize_with_fee_payer() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), SIG_FEE);
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -845,6 +837,7 @@ fn test_stake_authorize_with_fee_payer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 1,
+        custodian: None,
     };
     process_command(&config).unwrap();
     // `config` balance has not changed, despite submitting the TX
@@ -863,6 +856,7 @@ fn test_stake_authorize_with_fee_payer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     config_offline.output_format = OutputFormat::JsonCompact;
     let sign_reply = process_command(&config_offline).unwrap();
@@ -878,6 +872,7 @@ fn test_stake_authorize_with_fee_payer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        custodian: None,
     };
     process_command(&config).unwrap();
     // `config`'s balance again has not changed
@@ -893,9 +888,7 @@ fn test_stake_split() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1037,9 +1030,7 @@ fn test_stake_set_lockup() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1289,9 +1280,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
 
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1306,7 +1295,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     let offline_pubkey = config_offline.signers[0].pubkey();
     config_offline.json_rpc_url = String::default();
     config_offline.command = CliCommand::ClusterVersion;
-    // Verfiy that we cannot reach the cluster
+    // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
     request_and_confirm_airdrop(

cli/tests/transfer.rs

@@ -17,17 +17,13 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, NullSigner, Signer},
 };
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_transfer() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -60,6 +56,8 @@ fn test_transfer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&config).unwrap();
     check_recent_balance(49_989, &rpc_client, &sender_pubkey);
@@ -76,6 +74,8 @@ fn test_transfer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     assert!(process_command(&config).is_err());
     check_recent_balance(49_989, &rpc_client, &sender_pubkey);
@@ -104,6 +104,8 @@ fn test_transfer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     offline.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&offline).unwrap();
@@ -121,6 +123,8 @@ fn test_transfer() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&config).unwrap();
     check_recent_balance(39, &rpc_client, &offline_pubkey);
@@ -166,6 +170,8 @@ fn test_transfer() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&config).unwrap();
     check_recent_balance(49_976 - minimum_nonce_balance, &rpc_client, &sender_pubkey);
@@ -212,6 +218,8 @@ fn test_transfer() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     let sign_only_reply = process_command(&offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_only_reply);
@@ -231,6 +239,8 @@ fn test_transfer() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&config).unwrap();
     check_recent_balance(28, &rpc_client, &offline_pubkey);
@@ -242,10 +252,7 @@ fn test_transfer_multisession_signing() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let to_pubkey = Pubkey::new(&[1u8; 32]);
     let offline_from_signer = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -297,6 +304,8 @@ fn test_transfer_multisession_signing() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     fee_payer_config.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&fee_payer_config).unwrap();
@@ -323,6 +332,8 @@ fn test_transfer_multisession_signing() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     from_config.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&from_config).unwrap();
@@ -346,6 +357,8 @@ fn test_transfer_multisession_signing() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&config).unwrap();
 
@@ -359,10 +372,7 @@ fn test_transfer_all() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -394,8 +404,66 @@ fn test_transfer_all() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&config).unwrap();
     check_recent_balance(0, &rpc_client, &sender_pubkey);
     check_recent_balance(49_999, &rpc_client, &recipient_pubkey);
 }
+
+#[test]
+fn test_transfer_with_seed() {
+    solana_logger::setup();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+    let default_signer = Keypair::new();
+
+    let mut config = CliConfig::recent_for_tests();
+    config.json_rpc_url = test_validator.rpc_url();
+    config.signers = vec![&default_signer];
+
+    let sender_pubkey = config.signers[0].pubkey();
+    let recipient_pubkey = Pubkey::new(&[1u8; 32]);
+    let derived_address_seed = "seed".to_string();
+    let derived_address_program_id = solana_stake_program::id();
+    let derived_address = Pubkey::create_with_seed(
+        &sender_pubkey,
+        &derived_address_seed,
+        &derived_address_program_id,
+    )
+    .unwrap();
+
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 1, &config).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &derived_address, 50_000, &config)
+        .unwrap();
+    check_recent_balance(1, &rpc_client, &sender_pubkey);
+    check_recent_balance(50_000, &rpc_client, &derived_address);
+    check_recent_balance(0, &rpc_client, &recipient_pubkey);
+
+    check_ready(&rpc_client);
+
+    // Transfer with seed
+    config.command = CliCommand::Transfer {
+        amount: SpendAmount::Some(50_000),
+        to: recipient_pubkey,
+        from: 0,
+        sign_only: false,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+        derived_address_seed: Some(derived_address_seed),
+        derived_address_program_id: Some(derived_address_program_id),
+    };
+    process_command(&config).unwrap();
+    check_recent_balance(0, &rpc_client, &sender_pubkey);
+    check_recent_balance(50_000, &rpc_client, &recipient_pubkey);
+    check_recent_balance(0, &rpc_client, &derived_address);
+}

cli/tests/vote.rs

@@ -15,15 +15,12 @@ use solana_sdk::{
     signature::{Keypair, Signer},
 };
 use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersions};
-use std::sync::mpsc::channel;
 
 #[test]
 fn test_vote_authorize_and_withdraw() {
     let mint_keypair = Keypair::new();
     let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-    let (sender, receiver) = channel();
-    run_local_faucet(mint_keypair, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -79,6 +76,8 @@ fn test_vote_authorize_and_withdraw() {
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
     };
     process_command(&config).unwrap();
     let expected_balance = expected_balance + 1_000;

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.5.5"
+version = "1.5.12"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -20,16 +20,16 @@ net2 = "0.2.37"
 rayon = "1.4.0"
 reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 semver = "0.11.0"
-serde = "1.0.112"
+serde = "1.0.118"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-net-utils = { path = "../net-utils", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.5" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.12" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-net-utils = { path = "../net-utils", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.12" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -38,7 +38,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "15.0.0"
 jsonrpc-http-server = "15.0.0"
-solana-logger = { path = "../logger", version = "1.5.5" }
+solana-logger = { path = "../logger", version = "1.5.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/http_sender.rs

@@ -85,7 +85,7 @@ impl RpcSender for HttpSender {
                                             }
                                         }
                                     },
-                                    rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY => {
+                                    rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY => {
                                         match serde_json::from_value::<rpc_custom_error::NodeUnhealthyErrorData>(json["error"]["data"].clone()) {
                                             Ok(rpc_custom_error::NodeUnhealthyErrorData {num_slots_behind}) => RpcResponseErrorData::NodeUnhealthy {num_slots_behind},
                                             Err(_err) => {

client/src/lib.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 #[macro_use]
 extern crate serde_derive;
 
@@ -8,6 +9,7 @@ pub mod mock_sender;
 pub mod nonce_utils;
 pub mod perf_utils;
 pub mod pubsub_client;
+pub mod rpc_cache;
 pub mod rpc_client;
 pub mod rpc_config;
 pub mod rpc_custom_error;

client/src/pubsub_client.rs

@@ -98,7 +98,7 @@ where
     }
 
     pub fn send_unsubscribe(&self) -> Result<(), PubsubClientError> {
-        let method = format!("{}Unubscribe", self.operation);
+        let method = format!("{}Unsubscribe", self.operation);
         self.socket
             .write()
             .unwrap()

client/src/rpc_cache.rs

@@ -0,0 +1,75 @@
+use crate::{rpc_config::RpcLargestAccountsFilter, rpc_response::RpcAccountBalance};
+use std::{
+    collections::HashMap,
+    time::{Duration, SystemTime},
+};
+
+#[derive(Debug, Clone)]
+pub struct LargestAccountsCache {
+    duration: u64,
+    cache: HashMap<Option<RpcLargestAccountsFilter>, LargestAccountsCacheValue>,
+}
+
+#[derive(Debug, Clone)]
+struct LargestAccountsCacheValue {
+    accounts: Vec<RpcAccountBalance>,
+    slot: u64,
+    cached_time: SystemTime,
+}
+
+impl LargestAccountsCache {
+    pub fn new(duration: u64) -> Self {
+        Self {
+            duration,
+            cache: HashMap::new(),
+        }
+    }
+
+    pub fn get_largest_accounts(
+        &self,
+        filter: &Option<RpcLargestAccountsFilter>,
+    ) -> Option<(u64, Vec<RpcAccountBalance>)> {
+        self.cache.get(&filter).and_then(|value| {
+            if let Ok(elapsed) = value.cached_time.elapsed() {
+                if elapsed < Duration::from_secs(self.duration) {
+                    return Some((value.slot, value.accounts.clone()));
+                }
+            }
+            None
+        })
+    }
+
+    pub fn set_largest_accounts(
+        &mut self,
+        filter: &Option<RpcLargestAccountsFilter>,
+        slot: u64,
+        accounts: &[RpcAccountBalance],
+    ) {
+        self.cache.insert(
+            filter.clone(),
+            LargestAccountsCacheValue {
+                accounts: accounts.to_owned(),
+                slot,
+                cached_time: SystemTime::now(),
+            },
+        );
+    }
+}
+
+#[cfg(test)]
+pub mod test {
+    use super::*;
+
+    #[test]
+    fn test_old_entries_expire() {
+        let mut cache = LargestAccountsCache::new(1);
+
+        let filter = Some(RpcLargestAccountsFilter::Circulating);
+
+        let accounts: Vec<RpcAccountBalance> = Vec::new();
+
+        cache.set_largest_accounts(&filter, 1000, &accounts);
+        std::thread::sleep(Duration::from_secs(1));
+        assert_eq!(cache.get_largest_accounts(&filter), None);
+    }
+}

client/src/rpc_config.rs

@@ -31,7 +31,7 @@ pub struct RpcSimulateTransactionConfig {
     pub encoding: Option<UiTransactionEncoding>,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum RpcLargestAccountsFilter {
     Circulating,

client/src/rpc_custom_error.rs

@@ -8,7 +8,7 @@ pub const JSON_RPC_SERVER_ERROR_BLOCK_CLEANED_UP: i64 = -32001;
 pub const JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE: i64 = -32002;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE: i64 = -32003;
 pub const JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE: i64 = -32004;
-pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY: i64 = -32005;
+pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY: i64 = -32005;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_PRECOMPILE_VERIFICATION_FAILURE: i64 = -32006;
 pub const JSON_RPC_SERVER_ERROR_SLOT_SKIPPED: i64 = -32007;
 pub const JSON_RPC_SERVER_ERROR_NO_SNAPSHOT: i64 = -32008;
@@ -80,7 +80,7 @@ impl From<RpcCustomError> for Error {
                 data: None,
             },
             RpcCustomError::NodeUnhealthy { num_slots_behind } => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY),
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY),
                 message: if let Some(num_slots_behind) = num_slots_behind {
                     format!("Node is behind by {} slots", num_slots_behind)
                 } else {

client/src/rpc_filter.rs

@@ -16,10 +16,15 @@ impl RpcFilterType {
                 match encoding {
                     MemcmpEncoding::Binary => {
                         let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;
-                        bs58::decode(&bytes)
-                            .into_vec()
-                            .map(|_| ())
-                            .map_err(|e| e.into())
+
+                        if bytes.len() > 128 {
+                            Err(RpcFilterError::Base58DataTooLarge)
+                        } else {
+                            bs58::decode(&bytes)
+                                .into_vec()
+                                .map(|_| ())
+                                .map_err(|e| e.into())
+                        }
                     }
                 }
             }
@@ -27,10 +32,12 @@ impl RpcFilterType {
     }
 }
 
-#[derive(Error, Debug)]
+#[derive(Error, PartialEq, Debug)]
 pub enum RpcFilterError {
     #[error("bs58 decode error")]
     DecodeError(#[from] bs58::decode::Error),
+    #[error("encoded binary (base 58) data should be less than 129 bytes")]
+    Base58DataTooLarge,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -140,4 +147,36 @@ mod tests {
         }
         .bytes_match(&data));
     }
+
+    #[test]
+    fn test_verify_memcmp() {
+        let base58_bytes = "\
+            1111111111111111111111111111111111111111111111111111111111111111\
+            1111111111111111111111111111111111111111111111111111111111111111";
+        assert_eq!(base58_bytes.len(), 128);
+        assert_eq!(
+            RpcFilterType::Memcmp(Memcmp {
+                offset: 0,
+                bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
+                encoding: None,
+            })
+            .verify(),
+            Ok(())
+        );
+
+        let base58_bytes = "\
+            1111111111111111111111111111111111111111111111111111111111111111\
+            1111111111111111111111111111111111111111111111111111111111111111\
+            1";
+        assert_eq!(base58_bytes.len(), 129);
+        assert_eq!(
+            RpcFilterType::Memcmp(Memcmp {
+                offset: 0,
+                bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
+                encoding: None,
+            })
+            .verify(),
+            Err(RpcFilterError::Base58DataTooLarge)
+        );
+    }
 }

client/src/rpc_request.rs

@@ -127,6 +127,7 @@ pub const MAX_GET_CONFIRMED_BLOCKS_RANGE: u64 = 500_000;
 pub const MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS2_LIMIT: usize = 1_000;
 pub const MAX_MULTIPLE_ACCOUNTS: usize = 100;
 pub const NUM_LARGEST_ACCOUNTS: usize = 20;
+pub const MAX_GET_PROGRAM_ACCOUNT_FILTERS: usize = 4;
 
 // Validators that are this number of slots behind are considered delinquent
 pub const DELINQUENT_VALIDATOR_SLOT_DISTANCE: u64 = 128;

client/src/rpc_response.rs

@@ -101,6 +101,15 @@ pub struct SlotInfo {
     pub root: Slot,
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase", tag = "type")]
+pub enum SlotUpdate {
+    OptimisticConfirmation { slot: Slot, timestamp: u64 },
+    FirstShredReceived { slot: Slot, timestamp: u64 },
+    Frozen { slot: Slot, timestamp: u64 },
+    Root { slot: Slot, timestamp: u64 },
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase", untagged)]
 pub enum RpcSignatureResult {

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.5"
+version = "1.5.12"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -24,7 +24,7 @@ chrono = { version = "0.4.11", features = ["serde"] }
 core_affinity = "0.5.10"
 crossbeam-channel = "0.4"
 ed25519-dalek = "=1.0.0-pre.4"
-fs_extra = "1.1.0"
+fs_extra = "1.2.0"
 flate2 = "1.0"
 indexmap = { version = "1.5", features = ["rayon"] }
 itertools = "0.9.0"
@@ -44,36 +44,37 @@ rand_chacha = "0.2.2"
 raptorq = "1.4.2"
 rayon = "1.4.1"
 regex = "1.3.9"
+retain_mut = "0.1.2"
 rustversion = "1.0.4"
-serde = "1.0.112"
+serde = "1.0.118"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.5" }
-solana-banks-server = { path = "../banks-server", version = "1.5.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.5" }
-solana-client = { path = "../client", version = "1.5.5" }
-solana-faucet = { path = "../faucet", version = "1.5.5" }
-solana-ledger = { path = "../ledger", version = "1.5.5" }
-solana-logger = { path = "../logger", version = "1.5.5" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.5.5" }
-solana-metrics = { path = "../metrics", version = "1.5.5" }
-solana-measure = { path = "../measure", version = "1.5.5" }
-solana-net-utils = { path = "../net-utils", version = "1.5.5" }
-solana-perf = { path = "../perf", version = "1.5.5" }
-solana-program-test = { path = "../program-test", version = "1.5.5" }
-solana-runtime = { path = "../runtime", version = "1.5.5" }
-solana-sdk = { path = "../sdk", version = "1.5.5" }
-solana-frozen-abi = { path = "../frozen-abi", version = "1.5.5" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.5" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.5" }
-solana-streamer = { path = "../streamer", version = "1.5.5" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.5.5" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.5" }
-solana-version = { path = "../version", version = "1.5.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.5" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.12" }
+solana-banks-server = { path = "../banks-server", version = "1.5.12" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.12" }
+solana-client = { path = "../client", version = "1.5.12" }
+solana-faucet = { path = "../faucet", version = "1.5.12" }
+solana-ledger = { path = "../ledger", version = "1.5.12" }
+solana-logger = { path = "../logger", version = "1.5.12" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.5.12" }
+solana-metrics = { path = "../metrics", version = "1.5.12" }
+solana-measure = { path = "../measure", version = "1.5.12" }
+solana-net-utils = { path = "../net-utils", version = "1.5.12" }
+solana-perf = { path = "../perf", version = "1.5.12" }
+solana-program-test = { path = "../program-test", version = "1.5.12" }
+solana-runtime = { path = "../runtime", version = "1.5.12" }
+solana-sdk = { path = "../sdk", version = "1.5.12" }
+solana-frozen-abi = { path = "../frozen-abi", version = "1.5.12" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.12" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.12" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.12" }
+solana-streamer = { path = "../streamer", version = "1.5.12" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.5.12" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.12" }
+solana-version = { path = "../version", version = "1.5.12" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.12" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "0.2", features = ["full"] }
@@ -82,7 +83,7 @@ tokio_01_bytes = { version = "0.4.7", package = "bytes" }
 tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
 tokio_io_01 = { version = "0.1", package = "tokio-io" }
 tokio_codec_01 = { version = "0.1", package = "tokio-codec" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.5" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.12" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/benches/banking_stage.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 #![feature(test)]
 
 extern crate test;
@@ -28,6 +29,7 @@ use solana_sdk::system_instruction;
 use solana_sdk::system_transaction;
 use solana_sdk::timing::{duration_as_us, timestamp};
 use solana_sdk::transaction::Transaction;
+use std::collections::VecDeque;
 use std::sync::atomic::Ordering;
 use std::sync::mpsc::Receiver;
 use std::sync::Arc;
@@ -68,10 +70,10 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
         let len = 4096;
         let chunk_size = 1024;
         let batches = to_packets_chunked(&vec![tx; len], chunk_size);
-        let mut packets = vec![];
+        let mut packets = VecDeque::new();
         for batch in batches {
             let batch_len = batch.packets.len();
-            packets.push((batch, vec![0usize; batch_len]));
+            packets.push_back((batch, vec![0usize; batch_len]));
         }
         let (s, _r) = unbounded();
         // This tests the performance of buffering packets.
@@ -81,9 +83,10 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
                 &my_pubkey,
                 &poh_recorder,
                 &mut packets,
-                10_000,
                 None,
                 &s,
+                None::<Box<dyn Fn()>>,
+                None,
             );
         });
 

core/benches/blockstore.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 #![feature(test)]
 extern crate solana_ledger;
 extern crate test;

core/benches/retransmit_stage.rs

@@ -6,6 +6,7 @@ extern crate test;
 use log::*;
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
+use solana_core::max_slots::MaxSlots;
 use solana_core::retransmit_stage::retransmitter;
 use solana_ledger::entry::Entry;
 use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
@@ -92,6 +93,8 @@ fn bench_retransmitter(bencher: &mut Bencher) {
         &leader_schedule_cache,
         cluster_info,
         packet_receiver,
+        &Arc::new(MaxSlots::default()),
+        None,
     );
 
     let mut index = 0;

core/benches/shredder.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 #![feature(test)]
 
 extern crate test;

core/src/accounts_hash_verifier.rs

@@ -8,7 +8,11 @@ use crate::{
     cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES},
     snapshot_packager_service::PendingSnapshotPackage,
 };
-use solana_runtime::snapshot_package::{AccountsPackage, AccountsPackageReceiver};
+use rayon::ThreadPool;
+use solana_runtime::{
+    accounts_db,
+    snapshot_package::{AccountsPackage, AccountsPackagePre, AccountsPackageReceiver},
+};
 use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
 use std::collections::{HashMap, HashSet};
 use std::{
@@ -42,6 +46,7 @@ impl AccountsHashVerifier {
             .name("solana-accounts-hash".to_string())
             .spawn(move || {
                 let mut hashes = vec![];
+                let mut thread_pool_storage = None;
                 loop {
                     if exit.load(Ordering::Relaxed) {
                         break;
@@ -49,7 +54,14 @@ impl AccountsHashVerifier {
 
                     match accounts_package_receiver.recv_timeout(Duration::from_secs(1)) {
                         Ok(accounts_package) => {
-                            Self::process_accounts_package(
+                            if accounts_package.hash_for_testing.is_some()
+                                && thread_pool_storage.is_none()
+                            {
+                                thread_pool_storage =
+                                    Some(accounts_db::make_min_priority_thread_pool());
+                            }
+
+                            Self::process_accounts_package_pre(
                                 accounts_package,
                                 &cluster_info,
                                 &trusted_validators,
@@ -59,6 +71,7 @@ impl AccountsHashVerifier {
                                 &exit,
                                 fault_injection_rate_slots,
                                 snapshot_interval_slots,
+                                thread_pool_storage.as_ref(),
                             );
                         }
                         Err(RecvTimeoutError::Disconnected) => break,
@@ -72,6 +85,36 @@ impl AccountsHashVerifier {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
+    fn process_accounts_package_pre(
+        accounts_package: AccountsPackagePre,
+        cluster_info: &ClusterInfo,
+        trusted_validators: &Option<HashSet<Pubkey>>,
+        halt_on_trusted_validator_accounts_hash_mismatch: bool,
+        pending_snapshot_package: &Option<PendingSnapshotPackage>,
+        hashes: &mut Vec<(Slot, Hash)>,
+        exit: &Arc<AtomicBool>,
+        fault_injection_rate_slots: u64,
+        snapshot_interval_slots: u64,
+        thread_pool: Option<&ThreadPool>,
+    ) {
+        let accounts_package = solana_runtime::snapshot_utils::process_accounts_package_pre(
+            accounts_package,
+            thread_pool,
+        );
+        Self::process_accounts_package(
+            accounts_package,
+            cluster_info,
+            trusted_validators,
+            halt_on_trusted_validator_accounts_hash_mismatch,
+            pending_snapshot_package,
+            hashes,
+            exit,
+            fault_injection_rate_slots,
+            snapshot_interval_slots,
+        );
+    }
+
     fn process_accounts_package(
         accounts_package: AccountsPackage,
         cluster_info: &ClusterInfo,
@@ -83,6 +126,7 @@ impl AccountsHashVerifier {
         fault_injection_rate_slots: u64,
         snapshot_interval_slots: u64,
     ) {
+        let hash = accounts_package.hash;
         if fault_injection_rate_slots != 0
             && accounts_package.slot % fault_injection_rate_slots == 0
         {
@@ -91,10 +135,10 @@ impl AccountsHashVerifier {
             use solana_sdk::hash::extend_and_hash;
             warn!("inserting fault at slot: {}", accounts_package.slot);
             let rand = thread_rng().gen_range(0, 10);
-            let hash = extend_and_hash(&accounts_package.hash, &[rand]);
+            let hash = extend_and_hash(&hash, &[rand]);
             hashes.push((accounts_package.slot, hash));
         } else {
-            hashes.push((accounts_package.slot, accounts_package.hash));
+            hashes.push((accounts_package.slot, hash));
         }
 
         while hashes.len() > MAX_SNAPSHOT_HASHES {

core/src/banking_stage.rs

@@ -8,6 +8,7 @@ use crate::{
 };
 use crossbeam_channel::{Receiver as CrossbeamReceiver, RecvTimeoutError};
 use itertools::Itertools;
+use retain_mut::RetainMut;
 use solana_ledger::{
     blockstore::Blockstore,
     blockstore_processor::{send_transaction_status_batch, TransactionStatusSender},
@@ -46,10 +47,10 @@ use solana_transaction_status::token_balances::{
 };
 use std::{
     cmp,
-    collections::HashMap,
+    collections::{HashMap, VecDeque},
     env,
     net::UdpSocket,
-    sync::atomic::AtomicBool,
+    sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
     sync::mpsc::Receiver,
     sync::{Arc, Mutex},
     thread::{self, Builder, JoinHandle},
@@ -58,7 +59,7 @@ use std::{
 };
 
 type PacketsAndOffsets = (Packets, Vec<usize>);
-pub type UnprocessedPackets = Vec<PacketsAndOffsets>;
+pub type UnprocessedPackets = VecDeque<PacketsAndOffsets>;
 
 /// Transaction forwarding
 pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 2;
@@ -70,6 +71,80 @@ const TOTAL_BUFFERED_PACKETS: usize = 500_000;
 
 const MAX_NUM_TRANSACTIONS_PER_BATCH: usize = 128;
 
+#[derive(Debug, Default)]
+pub struct BankingStageStats {
+    last_report: AtomicU64,
+    id: u32,
+    process_packets_count: AtomicUsize,
+    new_tx_count: AtomicUsize,
+    dropped_batches_count: AtomicUsize,
+    newly_buffered_packets_count: AtomicUsize,
+    current_buffered_packets_count: AtomicUsize,
+    rebuffered_packets_count: AtomicUsize,
+    consumed_buffered_packets_count: AtomicUsize,
+}
+
+impl BankingStageStats {
+    pub fn new(id: u32) -> Self {
+        BankingStageStats {
+            id,
+            ..BankingStageStats::default()
+        }
+    }
+
+    fn report(&self, report_interval_ms: u64) {
+        let should_report = {
+            let last = self.last_report.load(Ordering::Relaxed);
+            let now = solana_sdk::timing::timestamp();
+            now.saturating_sub(last) > report_interval_ms
+                && self.last_report.compare_exchange(
+                    last,
+                    now,
+                    Ordering::Relaxed,
+                    Ordering::Relaxed,
+                ) == Ok(last)
+        };
+
+        if should_report {
+            datapoint_info!(
+                "banking_stage-loop-stats",
+                ("id", self.id as i64, i64),
+                (
+                    "process_packets_count",
+                    self.process_packets_count.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "new_tx_count",
+                    self.new_tx_count.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "dropped_batches_count",
+                    self.dropped_batches_count.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "newly_buffered_packets_count",
+                    self.newly_buffered_packets_count.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "current_buffered_packets_count",
+                    self.current_buffered_packets_count
+                        .swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "rebuffered_packets_count",
+                    self.rebuffered_packets_count.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+            );
+        }
+    }
+}
+
 /// Stores the stage's thread handle and output receiver.
 pub struct BankingStage {
     bank_thread_hdls: Vec<JoinHandle<()>>,
@@ -156,9 +231,10 @@ impl BankingStage {
         Self { bank_thread_hdls }
     }
 
-    fn filter_valid_packets_for_forwarding(all_packets: &[PacketsAndOffsets]) -> Vec<&Packet> {
+    fn filter_valid_packets_for_forwarding<'a>(
+        all_packets: impl Iterator<Item = &'a PacketsAndOffsets>,
+    ) -> Vec<&'a Packet> {
         all_packets
-            .iter()
             .flat_map(|(p, valid_indexes)| valid_indexes.iter().map(move |x| &p.packets[*x]))
             .collect()
     }
@@ -166,9 +242,9 @@ impl BankingStage {
     fn forward_buffered_packets(
         socket: &std::net::UdpSocket,
         tpu_forwards: &std::net::SocketAddr,
-        unprocessed_packets: &[PacketsAndOffsets],
+        unprocessed_packets: &VecDeque<PacketsAndOffsets>,
     ) -> std::io::Result<()> {
-        let packets = Self::filter_valid_packets_for_forwarding(unprocessed_packets);
+        let packets = Self::filter_valid_packets_for_forwarding(unprocessed_packets.iter());
         inc_new_counter_info!("banking_stage-forwarded_packets", packets.len());
         for p in packets {
             socket.send_to(&p.data[..p.meta.size], &tpu_forwards)?;
@@ -177,81 +253,89 @@ impl BankingStage {
         Ok(())
     }
 
+    // Returns whether the given `Packets` has any more remaining unprocessed
+    // transactions
+    fn update_buffered_packets_with_new_unprocessed(
+        original_unprocessed_indexes: &mut Vec<usize>,
+        new_unprocessed_indexes: Vec<usize>,
+    ) -> bool {
+        let has_more_unprocessed_transactions =
+            Self::packet_has_more_unprocessed_transactions(&new_unprocessed_indexes);
+        if has_more_unprocessed_transactions {
+            *original_unprocessed_indexes = new_unprocessed_indexes
+        };
+        has_more_unprocessed_transactions
+    }
+
     pub fn consume_buffered_packets(
         my_pubkey: &Pubkey,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
-        buffered_packets: &mut Vec<PacketsAndOffsets>,
-        batch_limit: usize,
+        buffered_packets: &mut UnprocessedPackets,
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: &ReplayVoteSender,
-    ) -> UnprocessedPackets {
-        let mut unprocessed_packets = vec![];
-        let mut rebuffered_packets = 0;
+        test_fn: Option<impl Fn()>,
+        banking_stage_stats: Option<&BankingStageStats>,
+    ) {
+        let mut rebuffered_packets_len = 0;
         let mut new_tx_count = 0;
         let buffered_len = buffered_packets.len();
-        let mut buffered_packets_iter = buffered_packets.drain(..);
-        let mut dropped_batches_count = 0;
-
         let mut proc_start = Measure::start("consume_buffered_process");
-        while let Some((msgs, unprocessed_indexes)) = buffered_packets_iter.next() {
-            let bank = poh_recorder.lock().unwrap().bank();
-            if bank.is_none() {
-                rebuffered_packets += unprocessed_indexes.len();
-                Self::push_unprocessed(
-                    &mut unprocessed_packets,
-                    msgs,
-                    unprocessed_indexes,
-                    &mut dropped_batches_count,
-                    batch_limit,
-                );
-                continue;
-            }
-            let bank = bank.unwrap();
-
-            let (processed, verified_txs_len, new_unprocessed_indexes) =
-                Self::process_received_packets(
+        let mut reached_end_of_slot = None;
+        buffered_packets.retain_mut(|(msgs, ref mut original_unprocessed_indexes)| {
+            if let Some((next_leader, bank)) = &reached_end_of_slot {
+                // We've hit the end of this slot, no need to perform more processing,
+                // just filter the remaining packets for the invalid (e.g. too old) ones
+                let new_unprocessed_indexes = Self::filter_unprocessed_packets(
                     &bank,
-                    &poh_recorder,
                     &msgs,
-                    unprocessed_indexes.to_owned(),
-                    transaction_status_sender.clone(),
-                    gossip_vote_sender,
+                    &original_unprocessed_indexes,
+                    my_pubkey,
+                    *next_leader,
                 );
-
-            new_tx_count += processed;
-
-            // Collect any unprocessed transactions in this batch for forwarding
-            rebuffered_packets += new_unprocessed_indexes.len();
-            Self::push_unprocessed(
-                &mut unprocessed_packets,
-                msgs,
-                new_unprocessed_indexes,
-                &mut dropped_batches_count,
-                batch_limit,
-            );
-
-            if processed < verified_txs_len {
-                let next_leader = poh_recorder.lock().unwrap().next_slot_leader();
-                // Walk thru rest of the transactions and filter out the invalid (e.g. too old) ones
-                #[allow(clippy::while_let_on_iterator)]
-                while let Some((msgs, unprocessed_indexes)) = buffered_packets_iter.next() {
-                    let unprocessed_indexes = Self::filter_unprocessed_packets(
-                        &bank,
-                        &msgs,
-                        &unprocessed_indexes,
-                        my_pubkey,
-                        next_leader,
-                    );
-                    Self::push_unprocessed(
-                        &mut unprocessed_packets,
-                        msgs,
-                        unprocessed_indexes,
-                        &mut dropped_batches_count,
-                        batch_limit,
-                    );
+                Self::update_buffered_packets_with_new_unprocessed(
+                    original_unprocessed_indexes,
+                    new_unprocessed_indexes,
+                )
+            } else {
+                let bank = poh_recorder.lock().unwrap().bank();
+                if let Some(bank) = bank {
+                    let (processed, verified_txs_len, new_unprocessed_indexes) =
+                        Self::process_received_packets(
+                            &bank,
+                            &poh_recorder,
+                            &msgs,
+                            original_unprocessed_indexes.to_owned(),
+                            transaction_status_sender.clone(),
+                            gossip_vote_sender,
+                        );
+                    if processed < verified_txs_len {
+                        reached_end_of_slot =
+                            Some((poh_recorder.lock().unwrap().next_slot_leader(), bank));
+                    }
+                    new_tx_count += processed;
+                    // Out of the buffered packets just retried, collect any still unprocessed
+                    // transactions in this batch for forwarding
+                    rebuffered_packets_len += new_unprocessed_indexes.len();
+                    let has_more_unprocessed_transactions =
+                        Self::update_buffered_packets_with_new_unprocessed(
+                            original_unprocessed_indexes,
+                            new_unprocessed_indexes,
+                        );
+                    if let Some(test_fn) = &test_fn {
+                        test_fn();
+                    }
+                    has_more_unprocessed_transactions
+                } else {
+                    rebuffered_packets_len += original_unprocessed_indexes.len();
+                    // `original_unprocessed_indexes` must have remaining packets to process
+                    // if not yet processed.
+                    assert!(Self::packet_has_more_unprocessed_transactions(
+                        &original_unprocessed_indexes
+                    ));
+                    true
                 }
             }
-        }
+        });
 
         proc_start.stop();
 
@@ -264,12 +348,14 @@ impl BankingStage {
             (new_tx_count as f32) / (proc_start.as_s())
         );
 
-        inc_new_counter_info!("banking_stage-rebuffered_packets", rebuffered_packets);
-        inc_new_counter_info!("banking_stage-consumed_buffered_packets", new_tx_count);
-        inc_new_counter_debug!("banking_stage-process_transactions", new_tx_count);
-        inc_new_counter_debug!("banking_stage-dropped_batches_count", dropped_batches_count);
-
-        unprocessed_packets
+        if let Some(stats) = banking_stage_stats {
+            stats
+                .rebuffered_packets_count
+                .fetch_add(rebuffered_packets_len, Ordering::Relaxed);
+            stats
+                .consumed_buffered_packets_count
+                .fetch_add(new_tx_count, Ordering::Relaxed);
+        }
     }
 
     fn consume_or_forward_packets(
@@ -306,11 +392,11 @@ impl BankingStage {
         socket: &std::net::UdpSocket,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         cluster_info: &ClusterInfo,
-        buffered_packets: &mut Vec<PacketsAndOffsets>,
+        buffered_packets: &mut UnprocessedPackets,
         enable_forwarding: bool,
-        batch_limit: usize,
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: &ReplayVoteSender,
+        banking_stage_stats: &BankingStageStats,
     ) -> BufferedPacketsDecision {
         let (leader_at_slot_offset, poh_has_bank, would_be_leader) = {
             let poh = poh_recorder.lock().unwrap();
@@ -332,15 +418,15 @@ impl BankingStage {
 
         match decision {
             BufferedPacketsDecision::Consume => {
-                let mut unprocessed = Self::consume_buffered_packets(
+                Self::consume_buffered_packets(
                     my_pubkey,
                     poh_recorder,
                     buffered_packets,
-                    batch_limit,
                     transaction_status_sender,
                     gossip_vote_sender,
+                    None::<Box<dyn Fn()>>,
+                    Some(banking_stage_stats),
                 );
-                buffered_packets.append(&mut unprocessed);
             }
             BufferedPacketsDecision::Forward => {
                 if enable_forwarding {
@@ -386,7 +472,8 @@ impl BankingStage {
         gossip_vote_sender: ReplayVoteSender,
     ) {
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-        let mut buffered_packets = vec![];
+        let mut buffered_packets = VecDeque::with_capacity(batch_limit);
+        let banking_stage_stats = BankingStageStats::new(id);
         loop {
             while !buffered_packets.is_empty() {
                 let decision = Self::process_buffered_packets(
@@ -396,9 +483,9 @@ impl BankingStage {
                     cluster_info,
                     &mut buffered_packets,
                     enable_forwarding,
-                    batch_limit,
                     transaction_status_sender.clone(),
                     &gossip_vote_sender,
+                    &banking_stage_stats,
                 );
                 if decision == BufferedPacketsDecision::Hold {
                     // If we are waiting on a new bank,
@@ -427,21 +514,14 @@ impl BankingStage {
                 batch_limit,
                 transaction_status_sender.clone(),
                 &gossip_vote_sender,
+                &mut buffered_packets,
+                &banking_stage_stats,
             ) {
-                Err(RecvTimeoutError::Timeout) => (),
+                Ok(()) | Err(RecvTimeoutError::Timeout) => (),
                 Err(RecvTimeoutError::Disconnected) => break,
-                Ok(mut unprocessed_packets) => {
-                    if unprocessed_packets.is_empty() {
-                        continue;
-                    }
-                    let num: usize = unprocessed_packets
-                        .iter()
-                        .map(|(_, unprocessed)| unprocessed.len())
-                        .sum();
-                    inc_new_counter_info!("banking_stage-buffered_packets", num);
-                    buffered_packets.append(&mut unprocessed_packets);
-                }
             }
+
+            banking_stage_stats.report(100);
         }
     }
 
@@ -490,6 +570,7 @@ impl BankingStage {
         debug!("num_to_commit: {} ", num_to_commit);
         // unlock all the accounts with errors which are filtered by the above `filter_map`
         if !processed_transactions.is_empty() {
+            inc_new_counter_info!("banking_stage-record_count", 1);
             inc_new_counter_info!("banking_stage-record_transactions", num_to_commit);
 
             let mut hash_time = Measure::start("record::hash");
@@ -502,10 +583,14 @@ impl BankingStage {
                 .lock()
                 .unwrap()
                 .record(bank_slot, hash, processed_transactions);
-
             match res {
                 Ok(()) => (),
                 Err(PohRecorderError::MaxHeightReached) => {
+                    inc_new_counter_info!("banking_stage-max_height_reached", 1);
+                    inc_new_counter_info!(
+                        "banking_stage-max_height_reached_num_to_commit",
+                        num_to_commit
+                    );
                     // If record errors, add all the committable transactions (the ones
                     // we just attempted to record) as retryable
                     return (
@@ -513,7 +598,7 @@ impl BankingStage {
                         processed_transactions_indexes,
                     );
                 }
-                Err(e) => panic!(format!("Poh recorder returned unexpected error: {:?}", e)),
+                Err(e) => panic!("Poh recorder returned unexpected error: {:?}", e),
             }
             poh_record.stop();
         }
@@ -571,6 +656,14 @@ impl BankingStage {
         let mut record_time = Measure::start("record_time");
         let (num_to_commit, retryable_record_txs) =
             Self::record_transactions(bank.slot(), txs, &results, poh);
+        inc_new_counter_info!(
+            "banking_stage-record_transactions_num_to_commit",
+            *num_to_commit.as_ref().unwrap_or(&0)
+        );
+        inc_new_counter_info!(
+            "banking_stage-record_transactions_retryable_record_txs",
+            retryable_record_txs.len()
+        );
         retryable_txs.extend(retryable_record_txs);
         if num_to_commit.is_err() {
             return (num_to_commit, retryable_txs);
@@ -593,7 +686,7 @@ impl BankingStage {
             );
 
             bank_utils::find_and_send_votes(txs, &tx_results, Some(gossip_vote_sender));
-            if let Some(sender) = transaction_status_sender {
+            if let Some(transaction_status_sender) = transaction_status_sender {
                 let post_balances = bank.collect_balances(batch);
                 let post_token_balances = collect_token_balances(&bank, &batch, &mut mint_decimals);
                 send_transaction_status_batch(
@@ -605,7 +698,7 @@ impl BankingStage {
                     TransactionTokenBalancesSet::new(pre_token_balances, post_token_balances),
                     inner_instructions,
                     transaction_logs,
-                    sender,
+                    transaction_status_sender,
                 );
             }
         }
@@ -943,7 +1036,9 @@ impl BankingStage {
         batch_limit: usize,
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: &ReplayVoteSender,
-    ) -> Result<UnprocessedPackets, RecvTimeoutError> {
+        buffered_packets: &mut UnprocessedPackets,
+        banking_stage_stats: &BankingStageStats,
+    ) -> Result<(), RecvTimeoutError> {
         let mut recv_time = Measure::start("process_packets_recv");
         let mms = verified_receiver.recv_timeout(recv_timeout)?;
         recv_time.stop();
@@ -962,17 +1057,18 @@ impl BankingStage {
         let mut new_tx_count = 0;
 
         let mut mms_iter = mms.into_iter();
-        let mut unprocessed_packets = vec![];
         let mut dropped_batches_count = 0;
+        let mut newly_buffered_packets_count = 0;
         while let Some(msgs) = mms_iter.next() {
             let packet_indexes = Self::generate_packet_indexes(&msgs.packets);
             let bank = poh.lock().unwrap().bank();
             if bank.is_none() {
                 Self::push_unprocessed(
-                    &mut unprocessed_packets,
+                    buffered_packets,
                     msgs,
                     packet_indexes,
                     &mut dropped_batches_count,
+                    &mut newly_buffered_packets_count,
                     batch_limit,
                 );
                 continue;
@@ -992,10 +1088,11 @@ impl BankingStage {
 
             // Collect any unprocessed transactions in this batch for forwarding
             Self::push_unprocessed(
-                &mut unprocessed_packets,
+                buffered_packets,
                 msgs,
                 unprocessed_indexes,
                 &mut dropped_batches_count,
+                &mut newly_buffered_packets_count,
                 batch_limit,
             );
 
@@ -1013,10 +1110,11 @@ impl BankingStage {
                         next_leader,
                     );
                     Self::push_unprocessed(
-                        &mut unprocessed_packets,
+                        buffered_packets,
                         msgs,
                         unprocessed_indexes,
                         &mut dropped_batches_count,
+                        &mut newly_buffered_packets_count,
                         batch_limit,
                     );
                 }
@@ -1036,13 +1134,23 @@ impl BankingStage {
             count,
             id,
         );
-        inc_new_counter_debug!("banking_stage-process_packets", count);
-        inc_new_counter_debug!("banking_stage-process_transactions", new_tx_count);
-        inc_new_counter_debug!("banking_stage-dropped_batches_count", dropped_batches_count);
-
+        banking_stage_stats
+            .process_packets_count
+            .fetch_add(count, Ordering::Relaxed);
+        banking_stage_stats
+            .new_tx_count
+            .fetch_add(new_tx_count, Ordering::Relaxed);
+        banking_stage_stats
+            .dropped_batches_count
+            .fetch_add(dropped_batches_count, Ordering::Relaxed);
+        banking_stage_stats
+            .newly_buffered_packets_count
+            .fetch_add(newly_buffered_packets_count, Ordering::Relaxed);
+        banking_stage_stats
+            .current_buffered_packets_count
+            .swap(buffered_packets.len(), Ordering::Relaxed);
         *recv_start = Instant::now();
-
-        Ok(unprocessed_packets)
+        Ok(())
     }
 
     fn push_unprocessed(
@@ -1050,17 +1158,23 @@ impl BankingStage {
         packets: Packets,
         packet_indexes: Vec<usize>,
         dropped_batches_count: &mut usize,
+        newly_buffered_packets_count: &mut usize,
         batch_limit: usize,
     ) {
-        if !packet_indexes.is_empty() {
+        if Self::packet_has_more_unprocessed_transactions(&packet_indexes) {
             if unprocessed_packets.len() >= batch_limit {
-                unprocessed_packets.remove(0);
                 *dropped_batches_count += 1;
+                unprocessed_packets.pop_front();
             }
-            unprocessed_packets.push((packets, packet_indexes));
+            *newly_buffered_packets_count += packet_indexes.len();
+            unprocessed_packets.push_back((packets, packet_indexes));
         }
     }
 
+    fn packet_has_more_unprocessed_transactions(packet_indexes: &[usize]) -> bool {
+        !packet_indexes.is_empty()
+    }
+
     pub fn join(self) -> thread::Result<()> {
         for bank_thread_hdl in self.bank_thread_hdls {
             bank_thread_hdl.join()?;
@@ -1121,7 +1235,7 @@ mod tests {
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
     };
-    use solana_perf::packet::to_packets;
+    use solana_perf::packet::to_packets_chunked;
     use solana_sdk::{
         instruction::InstructionError,
         signature::{Keypair, Signer},
@@ -1130,7 +1244,7 @@ mod tests {
         transaction::TransactionError,
     };
     use solana_transaction_status::TransactionWithStatusMeta;
-    use std::{sync::atomic::Ordering, thread::sleep};
+    use std::{net::SocketAddr, path::Path, sync::atomic::Ordering, thread::sleep};
 
     #[test]
     fn test_banking_stage_shutdown1() {
@@ -1292,7 +1406,7 @@ mod tests {
             let tx_anf = system_transaction::transfer(&keypair, &to3, 1, start_hash);
 
             // send 'em over
-            let packets = to_packets(&[tx_no_ver, tx_anf, tx]);
+            let packets = to_packets_chunked(&[tx_no_ver, tx_anf, tx], 3);
 
             // glad they all fit
             assert_eq!(packets.len(), 1);
@@ -1368,7 +1482,7 @@ mod tests {
         let tx =
             system_transaction::transfer(&mint_keypair, &alice.pubkey(), 2, genesis_config.hash());
 
-        let packets = to_packets(&[tx]);
+        let packets = to_packets_chunked(&[tx], 1);
         let packets = packets
             .into_iter()
             .map(|packets| (packets, vec![1u8]))
@@ -1379,7 +1493,7 @@ mod tests {
         // Process a second batch that uses the same from account, so conflicts with above TX
         let tx =
             system_transaction::transfer(&mint_keypair, &alice.pubkey(), 1, genesis_config.hash());
-        let packets = to_packets(&[tx]);
+        let packets = to_packets_chunked(&[tx], 1);
         let packets = packets
             .into_iter()
             .map(|packets| (packets, vec![1u8]))
@@ -1928,7 +2042,7 @@ mod tests {
             })
             .collect_vec();
 
-        let result = BankingStage::filter_valid_packets_for_forwarding(&all_packets);
+        let result = BankingStage::filter_valid_packets_for_forwarding(all_packets.iter());
 
         assert_eq!(result.len(), 256);
 
@@ -2074,13 +2188,16 @@ mod tests {
                 &transactions,
                 &poh_recorder,
                 0,
-                Some(transaction_status_sender),
+                Some(TransactionStatusSender {
+                    sender: transaction_status_sender,
+                    enable_cpi_and_log_storage: false,
+                }),
                 &gossip_vote_sender,
             );
 
             transaction_status_service.join().unwrap();
 
-            let confirmed_block = blockstore.get_confirmed_block(bank.slot()).unwrap();
+            let confirmed_block = blockstore.get_confirmed_block(bank.slot(), false).unwrap();
             assert_eq!(confirmed_block.transactions.len(), 3);
 
             for TransactionWithStatusMeta { transaction, meta } in
@@ -2105,4 +2222,250 @@ mod tests {
         }
         Blockstore::destroy(&ledger_path).unwrap();
     }
+
+    fn setup_conflicting_transactions(
+        ledger_path: &Path,
+    ) -> (
+        Vec<Transaction>,
+        Arc<Bank>,
+        Arc<Mutex<PohRecorder>>,
+        Receiver<WorkingBankEntry>,
+    ) {
+        Blockstore::destroy(&ledger_path).unwrap();
+        let genesis_config_info = create_genesis_config(10_000);
+        let GenesisConfigInfo {
+            genesis_config,
+            mint_keypair,
+            ..
+        } = &genesis_config_info;
+        let blockstore =
+            Blockstore::open(&ledger_path).expect("Expected to be able to open database ledger");
+        let bank = Arc::new(Bank::new(&genesis_config));
+        let (poh_recorder, entry_receiver) = PohRecorder::new(
+            bank.tick_height(),
+            bank.last_blockhash(),
+            bank.slot(),
+            Some((4, 4)),
+            bank.ticks_per_slot(),
+            &solana_sdk::pubkey::new_rand(),
+            &Arc::new(blockstore),
+            &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
+            &Arc::new(PohConfig::default()),
+        );
+        let poh_recorder = Arc::new(Mutex::new(poh_recorder));
+
+        // Set up unparallelizable conflicting transactions
+        let pubkey0 = solana_sdk::pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
+        let pubkey2 = solana_sdk::pubkey::new_rand();
+        let transactions = vec![
+            system_transaction::transfer(&mint_keypair, &pubkey0, 1, genesis_config.hash()),
+            system_transaction::transfer(&mint_keypair, &pubkey1, 1, genesis_config.hash()),
+            system_transaction::transfer(&mint_keypair, &pubkey2, 1, genesis_config.hash()),
+        ];
+        (transactions, bank, poh_recorder, entry_receiver)
+    }
+
+    #[test]
+    fn test_consume_buffered_packets() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let (transactions, bank, poh_recorder, _entry_receiver) =
+                setup_conflicting_transactions(&ledger_path);
+            let num_conflicting_transactions = transactions.len();
+            let mut packets_vec = to_packets_chunked(&transactions, num_conflicting_transactions);
+            assert_eq!(packets_vec.len(), 1);
+            assert_eq!(packets_vec[0].packets.len(), num_conflicting_transactions);
+            let all_packets = packets_vec.pop().unwrap();
+            let mut buffered_packets: UnprocessedPackets = vec![(
+                all_packets,
+                (0..num_conflicting_transactions).into_iter().collect(),
+            )]
+            .into_iter()
+            .collect();
+
+            let (gossip_vote_sender, _gossip_vote_receiver) = unbounded();
+
+            // When the working bank in poh_recorder is None, no packets should be processed
+            assert!(!poh_recorder.lock().unwrap().has_bank());
+            BankingStage::consume_buffered_packets(
+                &Pubkey::default(),
+                &poh_recorder,
+                &mut buffered_packets,
+                None,
+                &gossip_vote_sender,
+                None::<Box<dyn Fn()>>,
+                None,
+            );
+            assert_eq!(buffered_packets[0].1.len(), num_conflicting_transactions);
+            // When the poh recorder has a bank, should process all non conflicting buffered packets.
+            // Processes one packet per iteration of the loop
+            for num_expected_unprocessed in (0..num_conflicting_transactions).rev() {
+                poh_recorder.lock().unwrap().set_bank(&bank);
+                BankingStage::consume_buffered_packets(
+                    &Pubkey::default(),
+                    &poh_recorder,
+                    &mut buffered_packets,
+                    None,
+                    &gossip_vote_sender,
+                    None::<Box<dyn Fn()>>,
+                    None,
+                );
+                if num_expected_unprocessed == 0 {
+                    assert!(buffered_packets.is_empty())
+                } else {
+                    assert_eq!(buffered_packets[0].1.len(), num_expected_unprocessed);
+                }
+            }
+        }
+        Blockstore::destroy(&ledger_path).unwrap();
+    }
+
+    #[test]
+    fn test_consume_buffered_packets_interrupted() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let (transactions, bank, poh_recorder, _entry_receiver) =
+                setup_conflicting_transactions(&ledger_path);
+            let num_conflicting_transactions = transactions.len();
+            let packets_vec = to_packets_chunked(&transactions, 1);
+            assert_eq!(packets_vec.len(), num_conflicting_transactions);
+            for single_packets in &packets_vec {
+                assert_eq!(single_packets.packets.len(), 1);
+            }
+            let mut buffered_packets: UnprocessedPackets = packets_vec
+                .clone()
+                .into_iter()
+                .map(|single_packets| (single_packets, vec![0]))
+                .collect();
+
+            let (continue_sender, continue_receiver) = unbounded();
+            let (finished_packet_sender, finished_packet_receiver) = unbounded();
+
+            let test_fn = Some(move || {
+                finished_packet_sender.send(()).unwrap();
+                continue_receiver.recv().unwrap();
+            });
+            // When the poh recorder has a bank, it should process all non conflicting buffered packets.
+            // Because each conflicting transaction is in it's own `Packet` within `packets_vec`, then
+            // each iteration of this loop will process one element of `packets_vec`per iteration of the
+            // loop.
+            let interrupted_iteration = 1;
+            poh_recorder.lock().unwrap().set_bank(&bank);
+            let poh_recorder_ = poh_recorder.clone();
+            let (gossip_vote_sender, _gossip_vote_receiver) = unbounded();
+            // Start up thread to process the banks
+            let t_consume = Builder::new()
+                .name("consume-buffered-packets".to_string())
+                .spawn(move || {
+                    BankingStage::consume_buffered_packets(
+                        &Pubkey::default(),
+                        &poh_recorder_,
+                        &mut buffered_packets,
+                        None,
+                        &gossip_vote_sender,
+                        test_fn,
+                        None,
+                    );
+
+                    // Check everything is correct. All indexes after `interrupted_iteration`
+                    // should still be unprocessed
+                    assert_eq!(
+                        buffered_packets.len(),
+                        packets_vec[interrupted_iteration + 1..].iter().count()
+                    );
+                    for ((remaining_unprocessed_packet, _), original_packet) in buffered_packets
+                        .iter()
+                        .zip(&packets_vec[interrupted_iteration + 1..])
+                    {
+                        assert_eq!(
+                            remaining_unprocessed_packet.packets[0],
+                            original_packet.packets[0]
+                        );
+                    }
+                })
+                .unwrap();
+
+            for i in 0..=interrupted_iteration {
+                finished_packet_receiver.recv().unwrap();
+                if i == interrupted_iteration {
+                    poh_recorder
+                        .lock()
+                        .unwrap()
+                        .schedule_dummy_max_height_reached_failure();
+                }
+                continue_sender.send(()).unwrap();
+            }
+
+            t_consume.join().unwrap();
+        }
+        Blockstore::destroy(&ledger_path).unwrap();
+    }
+
+    #[test]
+    fn test_push_unprocessed_batch_limit() {
+        // Create `Packets` with 1 unprocessed element
+        let single_element_packets = Packets::new(vec![Packet::default()]);
+        let mut unprocessed_packets: UnprocessedPackets =
+            vec![(single_element_packets.clone(), vec![0])]
+                .into_iter()
+                .collect();
+        // Set the limit to 2
+        let batch_limit = 2;
+        // Create some new unprocessed packets
+        let new_packets = single_element_packets;
+        let packet_indexes = vec![];
+
+        let mut dropped_batches_count = 0;
+        let mut newly_buffered_packets_count = 0;
+        // Because the set of unprocessed `packet_indexes` is empty, the
+        // packets are not added to the unprocessed queue
+        BankingStage::push_unprocessed(
+            &mut unprocessed_packets,
+            new_packets.clone(),
+            packet_indexes,
+            &mut dropped_batches_count,
+            &mut newly_buffered_packets_count,
+            batch_limit,
+        );
+        assert_eq!(unprocessed_packets.len(), 1);
+        assert_eq!(dropped_batches_count, 0);
+        assert_eq!(newly_buffered_packets_count, 0);
+
+        // Because the set of unprocessed `packet_indexes` is non-empty, the
+        // packets are added to the unprocessed queue
+        let packet_indexes = vec![0];
+        BankingStage::push_unprocessed(
+            &mut unprocessed_packets,
+            new_packets,
+            packet_indexes.clone(),
+            &mut dropped_batches_count,
+            &mut newly_buffered_packets_count,
+            batch_limit,
+        );
+        assert_eq!(unprocessed_packets.len(), 2);
+        assert_eq!(dropped_batches_count, 0);
+        assert_eq!(newly_buffered_packets_count, 1);
+
+        // Because we've reached the batch limit, old unprocessed packets are
+        // dropped and the new one is appended to the end
+        let new_packets = Packets::new(vec![Packet::from_data(
+            &SocketAddr::from(([127, 0, 0, 1], 8001)),
+            42,
+        )
+        .unwrap()]);
+        assert_eq!(unprocessed_packets.len(), batch_limit);
+        BankingStage::push_unprocessed(
+            &mut unprocessed_packets,
+            new_packets.clone(),
+            packet_indexes,
+            &mut dropped_batches_count,
+            &mut newly_buffered_packets_count,
+            batch_limit,
+        );
+        assert_eq!(unprocessed_packets.len(), 2);
+        assert_eq!(unprocessed_packets[1].0.packets[0], new_packets.packets[0]);
+        assert_eq!(dropped_batches_count, 1);
+        assert_eq!(newly_buffered_packets_count, 2);
+    }
 }

core/src/bigtable_upload_service.rs

@@ -74,6 +74,7 @@ impl BigTableUploadService {
                 start_slot,
                 Some(end_slot),
                 true,
+                false,
                 exit.clone(),
             ));
 

core/src/broadcast_stage.rs

@@ -402,7 +402,7 @@ pub fn broadcast_shreds(
         match send_mmsg(s, &packets[sent..]) {
             Ok(n) => sent += n,
             Err(e) => {
-                return Err(Error::IO(e));
+                return Err(Error::Io(e));
             }
         }
     }

core/src/cache_block_time_service.rs

@@ -2,9 +2,7 @@ use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
 use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
 use solana_runtime::bank::Bank;
-use solana_sdk::{feature_set, timing::slot_duration_from_slots_per_year};
 use std::{
-    collections::HashMap,
     sync::{
         atomic::{AtomicBool, Ordering},
         Arc,
@@ -60,24 +58,8 @@ impl CacheBlockTimeService {
     }
 
     fn cache_block_time(bank: Arc<Bank>, blockstore: &Arc<Blockstore>) {
-        if bank
-            .feature_set
-            .is_active(&feature_set::timestamp_correction::id())
-        {
-            if let Err(e) = blockstore.cache_block_time(bank.slot(), bank.clock().unix_timestamp) {
-                error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
-            }
-        } else {
-            let slot_duration = slot_duration_from_slots_per_year(bank.slots_per_year());
-            let epoch = bank.epoch_schedule().get_epoch(bank.slot());
-            let stakes = HashMap::new();
-            let stakes = bank.epoch_vote_accounts(epoch).unwrap_or(&stakes);
-
-            if let Err(e) =
-                blockstore.cache_block_time_from_slot_entries(bank.slot(), slot_duration, stakes)
-            {
-                error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
-            }
+        if let Err(e) = blockstore.cache_block_time(bank.slot(), bank.clock().unix_timestamp) {
+            error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
         }
     }
 

core/src/cluster_info.rs

@@ -257,6 +257,17 @@ struct GossipStats {
     handle_batch_pull_requests_time: Counter,
     handle_batch_pull_responses_time: Counter,
     handle_batch_push_messages_time: Counter,
+    packets_received_count: Counter,
+    packets_received_prune_messages_count: Counter,
+    packets_received_pull_requests_count: Counter,
+    packets_received_pull_responses_count: Counter,
+    packets_received_push_messages_count: Counter,
+    packets_received_verified_count: Counter,
+    packets_sent_gossip_requests_count: Counter,
+    packets_sent_prune_messages_count: Counter,
+    packets_sent_pull_requests_count: Counter,
+    packets_sent_pull_responses_count: Counter,
+    packets_sent_push_messages_count: Counter,
     process_gossip_packets_time: Counter,
     process_pull_response: Counter,
     process_pull_response_count: Counter,
@@ -313,7 +324,7 @@ impl Default for ClusterInfo {
     }
 }
 
-#[derive(Debug, Default, Deserialize, Serialize, AbiExample)]
+#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)]
 pub struct PruneData {
     /// Pubkey of the node that sent this prune data
     pub pubkey: Pubkey,
@@ -405,7 +416,7 @@ pub fn make_accounts_hashes_message(
 type Ping = ping_pong::Ping<[u8; GOSSIP_PING_TOKEN_SIZE]>;
 
 // TODO These messages should go through the gpu pipeline for spam filtering
-#[frozen_abi(digest = "DdTxrwwnbe571Di4rLtrAQorFDE58vYnmzzbaeQ7sQMC")]
+#[frozen_abi(digest = "CH5BWuhAyvUiUQYgu2Lcwu7eoiW6bQitvtLS1yFsdmrE")]
 #[derive(Serialize, Deserialize, Debug, AbiEnumVisitor, AbiExample)]
 #[allow(clippy::large_enum_variant)]
 enum Protocol {
@@ -413,6 +424,8 @@ enum Protocol {
     PullRequest(CrdsFilter, CrdsValue),
     PullResponse(Pubkey, Vec<CrdsValue>),
     PushMessage(Pubkey, Vec<CrdsValue>),
+    // TODO: Remove the redundant outer pubkey here,
+    // and use the inner PruneData.pubkey instead.
     PruneMessage(Pubkey, PruneData),
     PingMessage(Ping),
     PongMessage(Pong),
@@ -496,7 +509,13 @@ impl Sanitize for Protocol {
             }
             Protocol::PullResponse(_, val) => val.sanitize(),
             Protocol::PushMessage(_, val) => val.sanitize(),
-            Protocol::PruneMessage(_, val) => val.sanitize(),
+            Protocol::PruneMessage(from, val) => {
+                if *from != val.pubkey {
+                    Err(SanitizeError::InvalidValue)
+                } else {
+                    val.sanitize()
+                }
+            }
             Protocol::PingMessage(ping) => ping.sanitize(),
             Protocol::PongMessage(pong) => pong.sanitize(),
         }
@@ -1488,7 +1507,7 @@ impl ClusterInfo {
                         1
                     );
                     error!("retransmit result {:?}", e);
-                    return Err(Error::IO(e));
+                    return Err(Error::Io(e));
                 }
             }
         }
@@ -1706,7 +1725,12 @@ impl ClusterInfo {
             vec![]
         };
         let mut pushes: Vec<_> = self.new_push_requests();
-
+        self.stats
+            .packets_sent_pull_requests_count
+            .add_relaxed(pulls.len() as u64);
+        self.stats
+            .packets_sent_push_messages_count
+            .add_relaxed(pushes.len() as u64);
         pulls.append(&mut pushes);
         pulls
     }
@@ -1729,6 +1753,9 @@ impl ClusterInfo {
         );
         if !reqs.is_empty() {
             let packets = to_packets_with_destination(recycler.clone(), &reqs);
+            self.stats
+                .packets_sent_gossip_requests_count
+                .add_relaxed(packets.packets.len() as u64);
             sender.send(packets)?;
         }
         Ok(())
@@ -1826,7 +1853,7 @@ impl ClusterInfo {
                 let mut last_contact_info_trace = timestamp();
                 let mut last_contact_info_save = timestamp();
                 let mut entrypoints_processed = false;
-                let recycler = PacketsRecycler::default();
+                let recycler = PacketsRecycler::new_without_limit("gossip-recycler-shrink-stats");
                 let crds_data = vec![
                     CrdsData::Version(Version::new(self.id())),
                     CrdsData::NodeInstance(self.instance.with_wallclock(timestamp())),
@@ -1995,6 +2022,9 @@ impl ClusterInfo {
                 .add_relaxed(requests.len() as u64);
             let response = self.handle_pull_requests(recycler, requests, stakes, feature_set);
             if !response.is_empty() {
+                self.stats
+                    .packets_sent_pull_responses_count
+                    .add_relaxed(response.packets.len() as u64);
                 let _ = response_sender.send(response);
             }
         }
@@ -2074,7 +2104,7 @@ impl ClusterInfo {
             .process_pull_requests(callers.cloned(), timestamp());
         let output_size_limit =
             self.update_data_budget(stakes.len()) / PULL_RESPONSE_MIN_SERIALIZED_SIZE;
-        let mut packets = Packets::new_with_recycler(recycler.clone(), 64, "handle_pull_requests");
+        let mut packets = Packets::new_with_recycler(recycler.clone(), 64).unwrap();
         let (caller_and_filters, addrs): (Vec<_>, Vec<_>) = {
             let mut rng = rand::thread_rng();
             let check_pull_request =
@@ -2359,8 +2389,7 @@ impl ClusterInfo {
         if packets.is_empty() {
             None
         } else {
-            let packets =
-                Packets::new_with_recycler_data(recycler, "handle_ping_messages", packets);
+            let packets = Packets::new_with_recycler_data(recycler, packets).unwrap();
             Some(packets)
         }
     }
@@ -2491,6 +2520,7 @@ impl ClusterInfo {
             return;
         }
         let mut packets = to_packets_with_destination(recycler.clone(), &prune_messages);
+        let num_prune_packets = packets.packets.len();
         self.stats
             .push_response_count
             .add_relaxed(packets.packets.len() as u64);
@@ -2506,6 +2536,12 @@ impl ClusterInfo {
                 trace!("Dropping Gossip push response, as destination is unknown");
             }
         }
+        self.stats
+            .packets_sent_prune_messages_count
+            .add_relaxed(num_prune_packets as u64);
+        self.stats
+            .packets_sent_push_messages_count
+            .add_relaxed((packets.packets.len() - num_prune_packets) as u64);
         let _ = response_sender.send(packets);
     }
 
@@ -2540,8 +2576,12 @@ impl ClusterInfo {
         stakes: HashMap<Pubkey, u64>,
         feature_set: Option<&FeatureSet>,
         epoch_time_ms: u64,
+        should_check_duplicate_instance: bool,
     ) -> Result<()> {
         let _st = ScopedTimer::from(&self.stats.process_gossip_packets_time);
+        self.stats
+            .packets_received_count
+            .add_relaxed(packets.len() as u64);
         let packets: Vec<_> = thread_pool.install(|| {
             packets
                 .into_par_iter()
@@ -2554,12 +2594,17 @@ impl ClusterInfo {
                 })
                 .collect()
         });
+        self.stats
+            .packets_received_verified_count
+            .add_relaxed(packets.len() as u64);
         // Check if there is a duplicate instance of
         // this node with more recent timestamp.
         let check_duplicate_instance = |values: &[CrdsValue]| {
-            for value in values {
-                if self.instance.check_duplicate(value) {
-                    return Err(Error::DuplicateNodeInstance);
+            if should_check_duplicate_instance {
+                for value in values {
+                    if self.instance.check_duplicate(value) {
+                        return Err(Error::DuplicateNodeInstance);
+                    }
                 }
             }
             Ok(())
@@ -2589,6 +2634,18 @@ impl ClusterInfo {
                 Protocol::PongMessage(pong) => pong_messages.push((from_addr, pong)),
             }
         }
+        self.stats
+            .packets_received_pull_requests_count
+            .add_relaxed(pull_requests.len() as u64);
+        self.stats
+            .packets_received_pull_responses_count
+            .add_relaxed(pull_responses.len() as u64);
+        self.stats
+            .packets_received_push_messages_count
+            .add_relaxed(push_messages.len() as u64);
+        self.stats
+            .packets_received_prune_messages_count
+            .add_relaxed(prune_messages.len() as u64);
         self.handle_batch_ping_messages(ping_messages, recycler, response_sender);
         self.handle_batch_prune_messages(prune_messages);
         self.handle_batch_push_messages(
@@ -2620,6 +2677,7 @@ impl ClusterInfo {
         response_sender: &PacketSender,
         thread_pool: &ThreadPool,
         last_print: &mut Instant,
+        should_check_duplicate_instance: bool,
     ) -> Result<()> {
         const RECV_TIMEOUT: Duration = Duration::from_secs(1);
         let packets: Vec<_> = requests_receiver.recv_timeout(RECV_TIMEOUT)?.packets.into();
@@ -2655,6 +2713,7 @@ impl ClusterInfo {
             stakes,
             feature_set.as_deref(),
             epoch_time_ms,
+            should_check_duplicate_instance,
         )?;
 
         self.print_reset_stats(last_print);
@@ -2889,6 +2948,61 @@ impl ClusterInfo {
                     self.stats.pull_requests_count.clear(),
                     i64
                 ),
+                (
+                    "packets_received_count",
+                    self.stats.packets_received_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_received_prune_messages_count",
+                    self.stats.packets_received_prune_messages_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_received_pull_requests_count",
+                    self.stats.packets_received_pull_requests_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_received_pull_responses_count",
+                    self.stats.packets_received_pull_responses_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_received_push_messages_count",
+                    self.stats.packets_received_push_messages_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_received_verified_count",
+                    self.stats.packets_received_verified_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_sent_gossip_requests_count",
+                    self.stats.packets_sent_gossip_requests_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_sent_prune_messages_count",
+                    self.stats.packets_sent_prune_messages_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_sent_pull_requests_count",
+                    self.stats.packets_sent_pull_requests_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_sent_pull_responses_count",
+                    self.stats.packets_sent_pull_responses_count.clear(),
+                    i64
+                ),
+                (
+                    "packets_sent_push_messages_count",
+                    self.stats.packets_sent_push_messages_count.clear(),
+                    i64
+                ),
             );
 
             *last_print = Instant::now();
@@ -2900,10 +3014,12 @@ impl ClusterInfo {
         bank_forks: Option<Arc<RwLock<BankForks>>>,
         requests_receiver: PacketReceiver,
         response_sender: PacketSender,
+        should_check_duplicate_instance: bool,
         exit: &Arc<AtomicBool>,
     ) -> JoinHandle<()> {
         let exit = exit.clone();
-        let recycler = PacketsRecycler::default();
+        let recycler =
+            PacketsRecycler::new_without_limit("cluster-info-listen-recycler-shrink-stats");
         Builder::new()
             .name("solana-listen".to_string())
             .spawn(move || {
@@ -2921,6 +3037,7 @@ impl ClusterInfo {
                         &response_sender,
                         &thread_pool,
                         &mut last_print,
+                        should_check_duplicate_instance,
                     ) {
                         match err {
                             Error::RecvTimeoutError(_) => {
@@ -3347,7 +3464,7 @@ mod tests {
             .iter()
             .map(|ping| Pong::new(ping, &this_node).unwrap())
             .collect();
-        let recycler = PacketsRecycler::default();
+        let recycler = PacketsRecycler::new_without_limit("");
         let packets = cluster_info
             .handle_ping_messages(
                 remote_nodes
@@ -4162,6 +4279,23 @@ mod tests {
         assert_eq!(msg.sanitize(), Err(SanitizeError::ValueOutOfBounds));
     }
 
+    #[test]
+    fn test_protocol_prune_message_sanitize() {
+        let keypair = Keypair::new();
+        let mut prune_data = PruneData {
+            pubkey: keypair.pubkey(),
+            prunes: vec![],
+            signature: Signature::default(),
+            destination: Pubkey::new_unique(),
+            wallclock: timestamp(),
+        };
+        prune_data.sign(&keypair);
+        let prune_message = Protocol::PruneMessage(keypair.pubkey(), prune_data.clone());
+        assert_eq!(prune_message.sanitize(), Ok(()));
+        let prune_message = Protocol::PruneMessage(Pubkey::new_unique(), prune_data);
+        assert_eq!(prune_message.sanitize(), Err(SanitizeError::InvalidValue));
+    }
+
     // computes the maximum size for pull request blooms
     fn max_bloom_size() -> usize {
         let filter_size = serialized_size(&CrdsFilter::default())

core/src/cluster_info_vote_listener.rs

@@ -4,7 +4,6 @@ use crate::{
     optimistic_confirmation_verifier::OptimisticConfirmationVerifier,
     optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender},
     poh_recorder::PohRecorder,
-    pubkey_references::LockedPubkeyReferences,
     result::{Error, Result},
     rpc_subscriptions::RpcSubscriptions,
     sigverify,
@@ -57,15 +56,15 @@ pub struct SlotVoteTracker {
     // Maps pubkeys that have voted for this slot
     // to whether or not we've seen the vote on gossip.
     // True if seen on gossip, false if only seen in replay.
-    voted: HashMap<Arc<Pubkey>, bool>,
+    voted: HashMap<Pubkey, bool>,
     optimistic_votes_tracker: HashMap<Hash, VoteStakeTracker>,
-    updates: Option<Vec<Arc<Pubkey>>>,
+    updates: Option<Vec<Pubkey>>,
     gossip_only_stake: u64,
 }
 
 impl SlotVoteTracker {
     #[allow(dead_code)]
-    pub fn get_updates(&mut self) -> Option<Vec<Arc<Pubkey>>> {
+    pub fn get_updates(&mut self) -> Option<Vec<Pubkey>> {
         self.updates.take()
     }
 
@@ -85,7 +84,6 @@ pub struct VoteTracker {
     epoch_authorized_voters: RwLock<HashMap<Epoch, Arc<EpochAuthorizedVoters>>>,
     leader_schedule_epoch: RwLock<Epoch>,
     current_epoch: RwLock<Epoch>,
-    keys: LockedPubkeyReferences,
     epoch_schedule: EpochSchedule,
 }
 
@@ -157,21 +155,19 @@ impl VoteTracker {
     }
 
     #[cfg(test)]
-    pub fn insert_vote(&self, slot: Slot, pubkey: Arc<Pubkey>) {
+    pub fn insert_vote(&self, slot: Slot, pubkey: Pubkey) {
         let mut w_slot_vote_trackers = self.slot_vote_trackers.write().unwrap();
 
         let slot_vote_tracker = w_slot_vote_trackers.entry(slot).or_default();
 
         let mut w_slot_vote_tracker = slot_vote_tracker.write().unwrap();
 
-        w_slot_vote_tracker.voted.insert(pubkey.clone(), true);
+        w_slot_vote_tracker.voted.insert(pubkey, true);
         if let Some(ref mut updates) = w_slot_vote_tracker.updates {
-            updates.push(pubkey.clone())
+            updates.push(pubkey)
         } else {
-            w_slot_vote_tracker.updates = Some(vec![pubkey.clone()]);
+            w_slot_vote_tracker.updates = Some(vec![pubkey]);
         }
-
-        self.keys.get_or_insert(&pubkey);
     }
 
     fn progress_leader_schedule_epoch(&self, root_bank: &Bank) {
@@ -221,7 +217,6 @@ impl VoteTracker {
                 .write()
                 .unwrap()
                 .retain(|epoch, _| *epoch >= root_epoch);
-            self.keys.purge();
             *self.current_epoch.write().unwrap() = root_epoch;
         }
     }
@@ -382,7 +377,7 @@ impl ClusterInfoVoteListener {
                 return Ok(());
             }
 
-            if let Err(e) = verified_vote_packets.get_and_process_vote_packets(
+            if let Err(e) = verified_vote_packets.receive_and_process_vote_packets(
                 &verified_vote_label_packets_receiver,
                 &mut update_version,
             ) {
@@ -402,7 +397,9 @@ impl ClusterInfoVoteListener {
                 if let Some(bank) = bank {
                     let last_version = bank.last_vote_sync.load(Ordering::Relaxed);
                     let (new_version, msgs) = verified_vote_packets.get_latest_votes(last_version);
-                    verified_packets_sender.send(msgs)?;
+                    inc_new_counter_info!("bank_send_loop_batch_size", msgs.packets.len());
+                    inc_new_counter_info!("bank_send_loop_num_batches", 1);
+                    verified_packets_sender.send(vec![msgs])?;
                     #[allow(deprecated)]
                     bank.last_vote_sync.compare_and_swap(
                         last_version,
@@ -547,7 +544,7 @@ impl ClusterInfoVoteListener {
         root_bank: &Bank,
         subscriptions: &RpcSubscriptions,
         verified_vote_sender: &VerifiedVoteSender,
-        diff: &mut HashMap<Slot, HashMap<Arc<Pubkey>, bool>>,
+        diff: &mut HashMap<Slot, HashMap<Pubkey, bool>>,
         new_optimistic_confirmed_slots: &mut Vec<(Slot, Hash)>,
         is_gossip_vote: bool,
         bank_notification_sender: &Option<BankNotificationSender>,
@@ -572,7 +569,6 @@ impl ClusterInfoVoteListener {
                 continue;
             }
             let epoch_stakes = epoch_stakes.unwrap();
-            let unduplicated_pubkey = vote_tracker.keys.get_or_insert(&vote_pubkey);
 
             // The last vote slot, which is the greatest slot in the stack
             // of votes in a vote transaction, qualifies for optimistic confirmation.
@@ -591,7 +587,7 @@ impl ClusterInfoVoteListener {
                     vote_tracker,
                     last_vote_slot,
                     last_vote_hash,
-                    unduplicated_pubkey.clone(),
+                    *vote_pubkey,
                     stake,
                     total_stake,
                 );
@@ -626,7 +622,7 @@ impl ClusterInfoVoteListener {
 
             diff.entry(slot)
                 .or_default()
-                .entry(unduplicated_pubkey)
+                .entry(*vote_pubkey)
                 .and_modify(|seen_in_gossip_previously| {
                     *seen_in_gossip_previously = *seen_in_gossip_previously || is_gossip_vote
                 })
@@ -681,7 +677,7 @@ impl ClusterInfoVoteListener {
         verified_vote_sender: &VerifiedVoteSender,
         bank_notification_sender: &Option<BankNotificationSender>,
     ) -> Vec<(Slot, Hash)> {
-        let mut diff: HashMap<Slot, HashMap<Arc<Pubkey>, bool>> = HashMap::new();
+        let mut diff: HashMap<Slot, HashMap<Pubkey, bool>> = HashMap::new();
         let mut new_optimistic_confirmed_slots = vec![];
 
         // Process votes from gossip and ReplayStage
@@ -748,9 +744,7 @@ impl ClusterInfoVoteListener {
                 // no other writers to `slot_vote_tracker` that
                 // `is_new || is_new_from_gossip`. In both cases we want to record
                 // `is_new_from_gossip` for the `pubkey` entry.
-                w_slot_tracker
-                    .voted
-                    .insert(pubkey.clone(), seen_in_gossip_above);
+                w_slot_tracker.voted.insert(pubkey, seen_in_gossip_above);
                 w_slot_tracker.updates.as_mut().unwrap().push(pubkey);
             }
 
@@ -765,7 +759,7 @@ impl ClusterInfoVoteListener {
         vote_tracker: &VoteTracker,
         slot: Slot,
         hash: Hash,
-        pubkey: Arc<Pubkey>,
+        pubkey: Pubkey,
         stake: u64,
         total_epoch_stake: u64,
     ) -> (bool, bool) {
@@ -825,7 +819,7 @@ mod tests {
         use bincode::serialized_size;
         info!("max vote size {}", serialized_size(&vote_tx).unwrap());
 
-        let msgs = packet::to_packets(&[vote_tx]); // panics if won't fit
+        let msgs = packet::to_packets_chunked(&[vote_tx], 1); // panics if won't fit
 
         assert_eq!(msgs.len(), 1);
     }
@@ -902,10 +896,10 @@ mod tests {
         let (vote_tracker, bank, _, _) = setup();
 
         // Check outdated slots are purged with new root
-        let new_voter = Arc::new(solana_sdk::pubkey::new_rand());
+        let new_voter = solana_sdk::pubkey::new_rand();
         // Make separate copy so the original doesn't count toward
         // the ref count, which would prevent cleanup
-        let new_voter_ = Arc::new(*new_voter);
+        let new_voter_ = new_voter;
         vote_tracker.insert_vote(bank.slot(), new_voter_);
         assert!(vote_tracker
             .slot_vote_trackers
@@ -922,7 +916,6 @@ mod tests {
 
         // Check `keys` and `epoch_authorized_voters` are purged when new
         // root bank moves to the next epoch
-        assert!(vote_tracker.keys.0.read().unwrap().contains(&new_voter));
         let current_epoch = bank.epoch();
         let new_epoch_bank = Bank::new_from_parent(
             &bank,
@@ -931,7 +924,6 @@ mod tests {
                 .get_first_slot_in_epoch(current_epoch + 1),
         );
         vote_tracker.progress_with_new_root_bank(&new_epoch_bank);
-        assert!(!vote_tracker.keys.0.read().unwrap().contains(&new_voter));
         assert_eq!(
             *vote_tracker.current_epoch.read().unwrap(),
             current_epoch + 1
@@ -1431,13 +1423,6 @@ mod tests {
 
     #[test]
     fn test_vote_tracker_references() {
-        // The number of references that get stored for a pubkey every time
-        // a vote is added to the tracking set via a transaction. One stored in the
-        // SlotVoteTracker.voted, one in SlotVoteTracker.updates, one in
-        // SlotVoteTracker.optimistic_votes_tracker
-        let ref_count_per_vote = 3;
-        let ref_count_per_new_key = 1;
-
         // Create some voters at genesis
         let validator_keypairs: Vec<_> =
             (0..2).map(|_| ValidatorVoteKeypairs::new_rand()).collect();
@@ -1492,22 +1477,6 @@ mod tests {
             &verified_vote_sender,
             &None,
         );
-        let ref_count = Arc::strong_count(
-            &vote_tracker
-                .keys
-                .0
-                .read()
-                .unwrap()
-                .get(&validator0_keypairs.vote_keypair.pubkey())
-                .unwrap(),
-        );
-
-        // This new pubkey submitted a vote for a slot, so ref count is
-        // `ref_count_per_vote + ref_count_per_new_key`.
-        // +ref_count_per_new_key for the new pubkey  in `vote_tracker.keys` and
-        // +ref_count_per_vote for the one new vote
-        let mut current_ref_count = ref_count_per_vote + ref_count_per_new_key;
-        assert_eq!(ref_count, current_ref_count);
 
         // Setup next epoch
         let old_epoch = bank.get_leader_schedule_epoch(bank.slot());
@@ -1562,35 +1531,6 @@ mod tests {
             &verified_vote_sender,
             &None,
         );
-
-        // Check new replay vote pubkey first
-        let ref_count = Arc::strong_count(
-            &vote_tracker
-                .keys
-                .0
-                .read()
-                .unwrap()
-                .get(&validator_keypairs[1].vote_keypair.pubkey())
-                .unwrap(),
-        );
-        // This new pubkey submitted a replay vote for a slot, so ref count is
-        // `ref_count_per_optimistic_vote + ref_count_per_new_key`.
-        // +ref_count_per_new_key for the new pubkey  in `vote_tracker.keys` and
-        // +ref_count_per_optimistic_vote for the one new vote
-        assert_eq!(ref_count, ref_count_per_vote + ref_count_per_new_key);
-
-        // Check the existing pubkey
-        let ref_count = Arc::strong_count(
-            &vote_tracker
-                .keys
-                .0
-                .read()
-                .unwrap()
-                .get(&validator0_keypairs.vote_keypair.pubkey())
-                .unwrap(),
-        );
-        current_ref_count += 2 * ref_count_per_vote;
-        assert_eq!(ref_count, current_ref_count);
     }
 
     fn setup() -> (

core/src/cluster_slots.rs

@@ -1,21 +1,23 @@
 use crate::{
     cluster_info::ClusterInfo, contact_info::ContactInfo, epoch_slots::EpochSlots,
-    pubkey_references::LockedPubkeyReferences, serve_repair::RepairType,
+    serve_repair::RepairType,
 };
 use solana_runtime::{bank_forks::BankForks, epoch_stakes::NodeIdToVoteAccounts};
 use solana_sdk::{clock::Slot, pubkey::Pubkey};
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{BTreeMap, HashMap, HashSet},
     sync::{Arc, RwLock},
 };
 
-pub type SlotPubkeys = HashMap<Arc<Pubkey>, u64>;
-pub type ClusterSlotsMap = RwLock<HashMap<Slot, Arc<RwLock<SlotPubkeys>>>>;
+// Limit the size of cluster-slots map in case
+// of receiving bogus epoch slots values.
+const CLUSTER_SLOTS_TRIM_SIZE: usize = 524_288; // 512K
+
+pub type SlotPubkeys = HashMap<Pubkey, u64>;
 
 #[derive(Default)]
 pub struct ClusterSlots {
-    cluster_slots: ClusterSlotsMap,
-    keys: LockedPubkeyReferences,
+    cluster_slots: RwLock<BTreeMap<Slot, Arc<RwLock<SlotPubkeys>>>>,
     since: RwLock<Option<u64>>,
     validator_stakes: RwLock<Arc<NodeIdToVoteAccounts>>,
     epoch: RwLock<Option<u64>>,
@@ -29,23 +31,29 @@ impl ClusterSlots {
     pub fn update(&self, root: Slot, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
         self.update_peers(cluster_info, bank_forks);
         let since = *self.since.read().unwrap();
-        let epoch_slots = cluster_info.get_epoch_slots_since(since);
-        self.update_internal(root, epoch_slots);
+        let (epoch_slots, since) = cluster_info.get_epoch_slots_since(since);
+        self.update_internal(root, epoch_slots, since);
     }
-    fn update_internal(&self, root: Slot, epoch_slots: (Vec<EpochSlots>, Option<u64>)) {
-        let (epoch_slots_list, since) = epoch_slots;
+    fn update_internal(&self, root: Slot, epoch_slots_list: Vec<EpochSlots>, since: Option<u64>) {
         for epoch_slots in epoch_slots_list {
             let slots = epoch_slots.to_slots(root);
             for slot in &slots {
                 if *slot <= root {
                     continue;
                 }
-                let unduplicated_pubkey = self.keys.get_or_insert(&epoch_slots.from);
-                self.insert_node_id(*slot, unduplicated_pubkey);
+                self.insert_node_id(*slot, epoch_slots.from);
+            }
+        }
+        {
+            let mut cluster_slots = self.cluster_slots.write().unwrap();
+            *cluster_slots = cluster_slots.split_off(&(root + 1));
+            // Trimming is done at 2x size so that amortized it has a constant
+            // cost. The slots furthest away from the root are discarded.
+            if cluster_slots.len() > 2 * CLUSTER_SLOTS_TRIM_SIZE {
+                let key = *cluster_slots.keys().nth(CLUSTER_SLOTS_TRIM_SIZE).unwrap();
+                cluster_slots.split_off(&key);
             }
         }
-        self.cluster_slots.write().unwrap().retain(|x, _| *x > root);
-        self.keys.purge();
         *self.since.write().unwrap() = since;
     }
 
@@ -54,13 +62,12 @@ impl ClusterSlots {
             .read()
             .unwrap()
             .iter()
-            .filter(|(_, keys)| keys.read().unwrap().get(id).is_some())
-            .map(|(slot, _)| slot)
-            .cloned()
+            .filter(|(_, keys)| keys.read().unwrap().contains_key(id))
+            .map(|(slot, _)| *slot)
             .collect()
     }
 
-    pub fn insert_node_id(&self, slot: Slot, node_id: Arc<Pubkey>) {
+    pub fn insert_node_id(&self, slot: Slot, node_id: Pubkey) {
         let balance = self
             .validator_stakes
             .read()
@@ -68,20 +75,14 @@ impl ClusterSlots {
             .get(&node_id)
             .map(|v| v.total_stake)
             .unwrap_or(0);
-        let mut slot_pubkeys = self.cluster_slots.read().unwrap().get(&slot).cloned();
-        if slot_pubkeys.is_none() {
-            let new_slot_pubkeys = Arc::new(RwLock::new(HashMap::default()));
-            self.cluster_slots
-                .write()
-                .unwrap()
-                .insert(slot, new_slot_pubkeys.clone());
-            slot_pubkeys = Some(new_slot_pubkeys);
-        }
-        slot_pubkeys
-            .unwrap()
+        let slot_pubkeys = self
+            .cluster_slots
             .write()
             .unwrap()
-            .insert(node_id, balance);
+            .entry(slot)
+            .or_default()
+            .clone();
+        slot_pubkeys.write().unwrap().insert(node_id, balance);
     }
 
     fn update_peers(&self, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
@@ -182,7 +183,7 @@ mod tests {
     #[test]
     fn test_update_noop() {
         let cs = ClusterSlots::default();
-        cs.update_internal(0, (vec![], None));
+        cs.update_internal(0, vec![], None);
         assert!(cs.cluster_slots.read().unwrap().is_empty());
         assert!(cs.since.read().unwrap().is_none());
     }
@@ -191,7 +192,7 @@ mod tests {
     fn test_update_empty() {
         let cs = ClusterSlots::default();
         let epoch_slot = EpochSlots::default();
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        cs.update_internal(0, vec![epoch_slot], Some(0));
         assert_eq!(*cs.since.read().unwrap(), Some(0));
         assert!(cs.lookup(0).is_none());
     }
@@ -202,7 +203,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[0], 0);
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        cs.update_internal(0, vec![epoch_slot], Some(0));
         assert_eq!(*cs.since.read().unwrap(), Some(0));
         assert!(cs.lookup(0).is_none());
     }
@@ -212,7 +213,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        cs.update_internal(0, vec![epoch_slot], Some(0));
         assert_eq!(*cs.since.read().unwrap(), Some(0));
         assert!(cs.lookup(0).is_none());
         assert!(cs.lookup(1).is_some());
@@ -241,8 +242,8 @@ mod tests {
         let mut map = HashMap::new();
         let k1 = solana_sdk::pubkey::new_rand();
         let k2 = solana_sdk::pubkey::new_rand();
-        map.insert(Arc::new(k1), std::u64::MAX / 2);
-        map.insert(Arc::new(k2), 0);
+        map.insert(k1, std::u64::MAX / 2);
+        map.insert(k2, 0);
         cs.cluster_slots
             .write()
             .unwrap()
@@ -263,14 +264,14 @@ mod tests {
         let mut map = HashMap::new();
         let k1 = solana_sdk::pubkey::new_rand();
         let k2 = solana_sdk::pubkey::new_rand();
-        map.insert(Arc::new(k2), 0);
+        map.insert(k2, 0);
         cs.cluster_slots
             .write()
             .unwrap()
             .insert(0, Arc::new(RwLock::new(map)));
         //make sure default weights are used as well
         let validator_stakes: HashMap<_, _> = vec![(
-            *Arc::new(k1),
+            k1,
             NodeVoteAccounts {
                 total_stake: std::u64::MAX / 2,
                 vote_accounts: vec![Pubkey::default()],
@@ -317,7 +318,7 @@ mod tests {
         // Mark the first validator as completed slot 9, should pick that validator,
         // even though it only has default stake, while the other validator has
         // max stake
-        cs.insert_node_id(slot, Arc::new(contact_infos[0].id));
+        cs.insert_node_id(slot, contact_infos[0].id);
         assert_eq!(
             cs.compute_weights_exclude_noncomplete(slot, &contact_infos),
             vec![(1, 0)]
@@ -343,7 +344,7 @@ mod tests {
         );
 
         *cs.validator_stakes.write().unwrap() = map;
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         assert!(cs.lookup(1).is_some());
         assert_eq!(
             cs.lookup(1)
@@ -360,7 +361,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         let self_id = solana_sdk::pubkey::new_rand();
         assert_eq!(
             cs.generate_repairs_for_missing_slots(&self_id, 0),
@@ -374,7 +375,7 @@ mod tests {
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
         let self_id = epoch_slot.from;
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         let slots: Vec<Slot> = cs.collect(&self_id).into_iter().collect();
         assert_eq!(slots, vec![1]);
     }
@@ -385,7 +386,7 @@ mod tests {
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
         let self_id = epoch_slot.from;
-        cs.update_internal(0, (vec![epoch_slot], None));
+        cs.update_internal(0, vec![epoch_slot], None);
         assert!(cs
             .generate_repairs_for_missing_slots(&self_id, 0)
             .is_empty());

core/src/commitment_service.rs

@@ -249,7 +249,7 @@ mod tests {
     use super::*;
     use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_runtime::{
-        accounts_background_service::ABSRequestSender,
+        accounts_background_service::AbsRequestSender,
         bank_forks::BankForks,
         genesis_utils::{create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs},
     };
@@ -534,7 +534,7 @@ mod tests {
             &working_bank,
         );
         for x in 0..root {
-            bank_forks.set_root(x, &ABSRequestSender::default(), None);
+            bank_forks.set_root(x, &AbsRequestSender::default(), None);
         }
 
         // Add an additional bank/vote that will root slot 2
@@ -573,7 +573,7 @@ mod tests {
             .highest_confirmed_root();
         bank_forks.set_root(
             root,
-            &ABSRequestSender::default(),
+            &AbsRequestSender::default(),
             Some(highest_confirmed_root),
         );
         let highest_confirmed_root_bank = bank_forks.get(highest_confirmed_root);
@@ -642,7 +642,7 @@ mod tests {
             .highest_confirmed_root();
         bank_forks.set_root(
             root,
-            &ABSRequestSender::default(),
+            &AbsRequestSender::default(),
             Some(highest_confirmed_root),
         );
         let highest_confirmed_root_bank = bank_forks.get(highest_confirmed_root);

core/src/completed_data_sets_service.rs

@@ -1,4 +1,4 @@
-use crate::rpc_subscriptions::RpcSubscriptions;
+use crate::{max_slots::MaxSlots, rpc_subscriptions::RpcSubscriptions};
 use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
 use solana_ledger::blockstore::{Blockstore, CompletedDataSetInfo};
 use solana_ledger::entry::Entry;
@@ -25,6 +25,7 @@ impl CompletedDataSetsService {
         blockstore: Arc<Blockstore>,
         rpc_subscriptions: Arc<RpcSubscriptions>,
         exit: &Arc<AtomicBool>,
+        max_slots: Arc<MaxSlots>,
     ) -> Self {
         let exit = exit.clone();
         let thread_hdl = Builder::new()
@@ -37,6 +38,7 @@ impl CompletedDataSetsService {
                     &completed_sets_receiver,
                     &blockstore,
                     &rpc_subscriptions,
+                    &max_slots,
                 ) {
                     break;
                 }
@@ -49,8 +51,10 @@ impl CompletedDataSetsService {
         completed_sets_receiver: &CompletedDataSetsReceiver,
         blockstore: &Blockstore,
         rpc_subscriptions: &RpcSubscriptions,
+        max_slots: &Arc<MaxSlots>,
     ) -> Result<(), RecvTimeoutError> {
         let completed_data_sets = completed_sets_receiver.recv_timeout(Duration::from_secs(1))?;
+        let mut max_slot = 0;
         for completed_set_info in std::iter::once(completed_data_sets)
             .chain(completed_sets_receiver.try_iter())
             .flatten()
@@ -60,6 +64,7 @@ impl CompletedDataSetsService {
                 start_index,
                 end_index,
             } = completed_set_info;
+            max_slot = max_slot.max(slot);
             match blockstore.get_entries_in_data_block(slot, start_index, end_index, None) {
                 Ok(entries) => {
                     let transactions = Self::get_transaction_signatures(entries);
@@ -70,6 +75,9 @@ impl CompletedDataSetsService {
                 Err(e) => warn!("completed-data-set-service deserialize error: {:?}", e),
             }
         }
+        max_slots
+            .shred_insert
+            .fetch_max(max_slot, Ordering::Relaxed);
 
         Ok(())
     }

core/src/consensus.rs

@@ -1,7 +1,4 @@
-use crate::{
-    progress_map::{LockoutIntervals, ProgressMap},
-    pubkey_references::PubkeyReferences,
-};
+use crate::progress_map::{LockoutIntervals, ProgressMap};
 use chrono::prelude::*;
 use solana_ledger::{ancestor_iterator::AncestorIterator, blockstore::Blockstore, blockstore_db};
 use solana_measure::measure::Measure;
@@ -214,7 +211,6 @@ impl Tower {
         bank_slot: Slot,
         vote_accounts: F,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
-        all_pubkeys: &mut PubkeyReferences,
     ) -> ComputedBankState
     where
         F: IntoIterator<Item = (Pubkey, (u64, ArcVoteAccount))>,
@@ -247,7 +243,6 @@ impl Tower {
                 Ok(vote_state) => vote_state.clone(),
             };
             for vote in &vote_state.votes {
-                let key = all_pubkeys.get_or_insert(&key);
                 lockout_intervals
                     .entry(vote.expiration_slot())
                     .or_insert_with(Vec::new)
@@ -850,10 +845,9 @@ impl Tower {
         assert!(
             self.last_vote == Vote::default() && self.lockouts.votes.is_empty()
                 || self.last_vote != Vote::default() && !self.lockouts.votes.is_empty(),
-            format!(
-                "last vote: {:?} lockouts.votes: {:?}",
-                self.last_vote, self.lockouts.votes
-            )
+            "last vote: {:?} lockouts.votes: {:?}",
+            self.last_vote,
+            self.lockouts.votes
         );
 
         if let Some(last_voted_slot) = self.last_voted_slot() {
@@ -1136,7 +1130,7 @@ impl Tower {
 #[derive(Error, Debug)]
 pub enum TowerError {
     #[error("IO Error: {0}")]
-    IOError(#[from] std::io::Error),
+    IoError(#[from] std::io::Error),
 
     #[error("Serialization Error: {0}")]
     SerializeError(#[from] bincode::Error),
@@ -1162,7 +1156,7 @@ pub enum TowerError {
 
 impl TowerError {
     pub fn is_file_missing(&self) -> bool {
-        if let TowerError::IOError(io_err) = &self {
+        if let TowerError::IoError(io_err) = &self {
             io_err.kind() == std::io::ErrorKind::NotFound
         } else {
             false
@@ -1251,7 +1245,7 @@ pub mod test {
     };
     use solana_ledger::{blockstore::make_slot_entries, get_tmp_ledger_path};
     use solana_runtime::{
-        accounts_background_service::ABSRequestSender,
+        accounts_background_service::AbsRequestSender,
         bank::Bank,
         bank_forks::BankForks,
         genesis_utils::{
@@ -1270,7 +1264,6 @@ pub mod test {
         collections::HashMap,
         fs::{remove_file, OpenOptions},
         io::{Read, Seek, SeekFrom, Write},
-        rc::Rc,
         sync::RwLock,
     };
     use tempfile::TempDir;
@@ -1380,7 +1373,6 @@ pub mod test {
                 &VoteTracker::default(),
                 &ClusterSlots::default(),
                 &self.bank_forks,
-                &mut PubkeyReferences::default(),
                 &mut self.heaviest_subtree_fork_choice,
             );
 
@@ -1393,7 +1385,7 @@ pub mod test {
                 .clone();
 
             // Try to vote on the given slot
-            let descendants = self.bank_forks.read().unwrap().descendants();
+            let descendants = self.bank_forks.read().unwrap().descendants().clone();
             let SelectVoteAndResetForkResult {
                 heaviest_fork_failures,
                 ..
@@ -1424,8 +1416,7 @@ pub mod test {
                 new_root,
                 &self.bank_forks,
                 &mut self.progress,
-                &ABSRequestSender::default(),
-                &mut PubkeyReferences::default(),
+                &AbsRequestSender::default(),
                 None,
                 &mut self.heaviest_subtree_fork_choice,
             )
@@ -1467,7 +1458,7 @@ pub mod test {
                 .lockout_intervals
                 .entry(lockout_interval.1)
                 .or_default()
-                .push((lockout_interval.0, Rc::new(*vote_account_pubkey)));
+                .push((lockout_interval.0, *vote_account_pubkey));
         }
 
         fn can_progress_on_fork(
@@ -1696,7 +1687,12 @@ pub mod test {
             fork_progress.fork_stats.computed = true;
         }
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let mut descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let mut descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
         let mut tower = Tower::new_with_key(&my_pubkey);
 
         // Last vote is 47
@@ -1827,7 +1823,12 @@ pub mod test {
         tower.lockouts.root_slot = Some(43);
         // Refresh ancestors and descendants for new root.
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
 
         assert_eq!(
             tower.check_switch_threshold(
@@ -1993,13 +1994,7 @@ pub mod test {
             bank_weight,
             pubkey_votes,
             ..
-        } = Tower::collect_vote_lockouts(
-            &Pubkey::default(),
-            1,
-            accounts.into_iter(),
-            &ancestors,
-            &mut PubkeyReferences::default(),
-        );
+        } = Tower::collect_vote_lockouts(&Pubkey::default(), 1, accounts.into_iter(), &ancestors);
         assert_eq!(voted_stakes[&0], 2);
         assert_eq!(total_stake, 2);
         let mut pubkey_votes = Arc::try_unwrap(pubkey_votes).unwrap();
@@ -2051,7 +2046,6 @@ pub mod test {
             MAX_LOCKOUT_HISTORY as u64,
             accounts.into_iter(),
             &ancestors,
-            &mut PubkeyReferences::default(),
         );
         for i in 0..MAX_LOCKOUT_HISTORY {
             assert_eq!(voted_stakes[&(i as u64)], 2);
@@ -2358,7 +2352,6 @@ pub mod test {
             vote_to_evaluate,
             accounts.clone().into_iter(),
             &ancestors,
-            &mut PubkeyReferences::default(),
         );
         assert!(tower.check_vote_stake_threshold(vote_to_evaluate, &voted_stakes, total_stake,));
 
@@ -2375,7 +2368,6 @@ pub mod test {
             vote_to_evaluate,
             accounts.into_iter(),
             &ancestors,
-            &mut PubkeyReferences::default(),
         );
         assert!(!tower.check_vote_stake_threshold(vote_to_evaluate, &voted_stakes, total_stake,));
     }
@@ -2487,7 +2479,12 @@ pub mod test {
         }
 
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
         let mut tower = Tower::new_with_key(&my_pubkey);
 
         tower.record_vote(43, Hash::default());
@@ -2579,7 +2576,12 @@ pub mod test {
         let mut slot_history = SlotHistory::default();
         vote_simulator.set_root(replayed_root_slot);
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
-        let descendants = vote_simulator.bank_forks.read().unwrap().descendants();
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
         for slot in &[0, 1, 2, 43, replayed_root_slot] {
             slot_history.add(*slot);
         }
@@ -2701,7 +2703,7 @@ pub mod test {
                 remove_file(path).unwrap();
             },
         );
-        assert_matches!(loaded, Err(TowerError::IOError(_)))
+        assert_matches!(loaded, Err(TowerError::IoError(_)))
     }
 
     #[test]

core/src/fetch_stage.rs

@@ -26,10 +26,19 @@ impl FetchStage {
         tpu_forwards_sockets: Vec<UdpSocket>,
         exit: &Arc<AtomicBool>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
+        coalesce_ms: u64,
     ) -> (Self, PacketReceiver) {
         let (sender, receiver) = channel();
         (
-            Self::new_with_sender(sockets, tpu_forwards_sockets, exit, &sender, &poh_recorder),
+            Self::new_with_sender(
+                sockets,
+                tpu_forwards_sockets,
+                exit,
+                &sender,
+                &poh_recorder,
+                None,
+                coalesce_ms,
+            ),
             receiver,
         )
     }
@@ -39,6 +48,8 @@ impl FetchStage {
         exit: &Arc<AtomicBool>,
         sender: &PacketSender,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
+        allocated_packet_limit: Option<u32>,
+        coalesce_ms: u64,
     ) -> Self {
         let tx_sockets = sockets.into_iter().map(Arc::new).collect();
         let tpu_forwards_sockets = tpu_forwards_sockets.into_iter().map(Arc::new).collect();
@@ -48,6 +59,8 @@ impl FetchStage {
             exit,
             &sender,
             &poh_recorder,
+            allocated_packet_limit,
+            coalesce_ms,
         )
     }
 
@@ -92,8 +105,11 @@ impl FetchStage {
         exit: &Arc<AtomicBool>,
         sender: &PacketSender,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
+        limit: Option<u32>,
+        coalesce_ms: u64,
     ) -> Self {
-        let recycler: PacketsRecycler = Recycler::warmed(1000, 1024);
+        let recycler: PacketsRecycler =
+            Recycler::warmed(1000, 1024, limit, "fetch_stage_recycler_shrink");
 
         let tpu_threads = sockets.into_iter().map(|socket| {
             streamer::receiver(
@@ -102,6 +118,7 @@ impl FetchStage {
                 sender.clone(),
                 recycler.clone(),
                 "fetch_stage",
+                coalesce_ms,
             )
         });
 
@@ -113,6 +130,7 @@ impl FetchStage {
                 forward_sender.clone(),
                 recycler.clone(),
                 "fetch_forward_stage",
+                coalesce_ms,
             )
         });
 

core/src/gossip_service.rs

@@ -33,6 +33,7 @@ impl GossipService {
         bank_forks: Option<Arc<RwLock<BankForks>>>,
         gossip_socket: UdpSocket,
         gossip_validators: Option<HashSet<Pubkey>>,
+        should_check_duplicate_instance: bool,
         exit: &Arc<AtomicBool>,
     ) -> Self {
         let (request_sender, request_receiver) = channel();
@@ -46,8 +47,9 @@ impl GossipService {
             gossip_socket.clone(),
             &exit,
             request_sender,
-            Recycler::default(),
+            Recycler::new_without_limit("gossip-receiver-recycler-shrink-stats"),
             "gossip_receiver",
+            1,
         );
         let (response_sender, response_receiver) = channel();
         let t_responder = streamer::responder("gossip", gossip_socket, response_receiver);
@@ -56,6 +58,7 @@ impl GossipService {
             bank_forks.clone(),
             request_receiver,
             response_sender.clone(),
+            should_check_duplicate_instance,
             exit,
         );
         let t_gossip = ClusterInfo::gossip(
@@ -108,8 +111,14 @@ pub fn discover(
     let keypair = keypair.unwrap_or_else(|| Arc::new(Keypair::new()));
 
     let exit = Arc::new(AtomicBool::new(false));
-    let (gossip_service, ip_echo, spy_ref) =
-        make_gossip_node(keypair, entrypoint, &exit, my_gossip_addr, my_shred_version);
+    let (gossip_service, ip_echo, spy_ref) = make_gossip_node(
+        keypair,
+        entrypoint,
+        &exit,
+        my_gossip_addr,
+        my_shred_version,
+        true, // should_check_duplicate_instance,
+    );
 
     let id = spy_ref.id();
     info!("Entrypoint: {:?}", entrypoint);
@@ -268,6 +277,7 @@ fn make_gossip_node(
     exit: &Arc<AtomicBool>,
     gossip_addr: Option<&SocketAddr>,
     shred_version: u16,
+    should_check_duplicate_instance: bool,
 ) -> (GossipService, Option<TcpListener>, Arc<ClusterInfo>) {
     let (node, gossip_socket, ip_echo) = if let Some(gossip_addr) = gossip_addr {
         ClusterInfo::gossip_node(&keypair.pubkey(), gossip_addr, shred_version)
@@ -279,7 +289,14 @@ fn make_gossip_node(
         cluster_info.set_entrypoint(ContactInfo::new_gossip_entry_point(entrypoint));
     }
     let cluster_info = Arc::new(cluster_info);
-    let gossip_service = GossipService::new(&cluster_info, None, gossip_socket, None, &exit);
+    let gossip_service = GossipService::new(
+        &cluster_info,
+        None,
+        gossip_socket,
+        None,
+        should_check_duplicate_instance,
+        &exit,
+    );
     (gossip_service, ip_echo, cluster_info)
 }
 
@@ -298,7 +315,14 @@ mod tests {
         let tn = Node::new_localhost();
         let cluster_info = ClusterInfo::new_with_invalid_keypair(tn.info.clone());
         let c = Arc::new(cluster_info);
-        let d = GossipService::new(&c, None, tn.sockets.gossip, None, &exit);
+        let d = GossipService::new(
+            &c,
+            None,
+            tn.sockets.gossip,
+            None,
+            true, // should_check_duplicate_instance
+            &exit,
+        );
         exit.store(true, Ordering::Relaxed);
         d.join().unwrap();
     }

core/src/ledger_cleanup_service.rs

@@ -1,15 +1,16 @@
 //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
 
+use rand::{thread_rng, Rng};
 use solana_ledger::blockstore::{Blockstore, PurgeType};
 use solana_ledger::blockstore_db::Result as BlockstoreResult;
 use solana_measure::measure::Measure;
 use solana_sdk::clock::{Slot, DEFAULT_TICKS_PER_SLOT, TICKS_PER_DAY};
 use std::string::ToString;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
 use std::sync::Arc;
 use std::thread;
-use std::thread::{Builder, JoinHandle};
+use std::thread::{sleep, Builder, JoinHandle};
 use std::time::Duration;
 
 // - To try and keep the RocksDB size under 400GB:
@@ -35,6 +36,7 @@ const DEFAULT_COMPACTION_SLOT_INTERVAL: u64 = TICKS_PER_DAY / DEFAULT_TICKS_PER_
 
 pub struct LedgerCleanupService {
     t_cleanup: JoinHandle<()>,
+    t_compact: JoinHandle<()>,
 }
 
 impl LedgerCleanupService {
@@ -43,6 +45,8 @@ impl LedgerCleanupService {
         blockstore: Arc<Blockstore>,
         max_ledger_shreds: u64,
         exit: &Arc<AtomicBool>,
+        compaction_interval: Option<u64>,
+        max_compaction_jitter: Option<u64>,
     ) -> Self {
         info!(
             "LedgerCleanupService active. Max Ledger Slots {}",
@@ -51,9 +55,16 @@ impl LedgerCleanupService {
         let exit = exit.clone();
         let mut last_purge_slot = 0;
         let mut last_compaction_slot = 0;
+        let mut compaction_jitter = 0;
+        let compaction_interval = compaction_interval.unwrap_or(DEFAULT_COMPACTION_SLOT_INTERVAL);
+        let last_compact_slot = Arc::new(AtomicU64::new(0));
+        let last_compact_slot2 = last_compact_slot.clone();
+
+        let exit_compact = exit.clone();
+        let blockstore_compact = blockstore.clone();
 
         let t_cleanup = Builder::new()
-            .name("solana-ledger-cleanup".to_string())
+            .name("sol-led-cleanup".to_string())
             .spawn(move || loop {
                 if exit.load(Ordering::Relaxed) {
                     break;
@@ -64,8 +75,7 @@ impl LedgerCleanupService {
                     max_ledger_shreds,
                     &mut last_purge_slot,
                     DEFAULT_PURGE_SLOT_INTERVAL,
-                    &mut last_compaction_slot,
-                    DEFAULT_COMPACTION_SLOT_INTERVAL,
+                    &last_compact_slot,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -74,7 +84,29 @@ impl LedgerCleanupService {
                 }
             })
             .unwrap();
-        Self { t_cleanup }
+
+        let t_compact = Builder::new()
+            .name("sol-led-compact".to_string())
+            .spawn(move || loop {
+                if exit_compact.load(Ordering::Relaxed) {
+                    break;
+                }
+                Self::compact_ledger(
+                    &blockstore_compact,
+                    &mut last_compaction_slot,
+                    compaction_interval,
+                    &last_compact_slot2,
+                    &mut compaction_jitter,
+                    max_compaction_jitter,
+                );
+                sleep(Duration::from_secs(1));
+            })
+            .unwrap();
+
+        Self {
+            t_cleanup,
+            t_compact,
+        }
     }
 
     fn find_slots_to_clean(
@@ -138,8 +170,7 @@ impl LedgerCleanupService {
         max_ledger_shreds: u64,
         last_purge_slot: &mut u64,
         purge_interval: u64,
-        last_compaction_slot: &mut u64,
-        compaction_interval: u64,
+        last_compact_slot: &Arc<AtomicU64>,
     ) -> Result<(), RecvTimeoutError> {
         let root = Self::receive_new_roots(new_root_receiver)?;
         if root - *last_purge_slot <= purge_interval {
@@ -148,8 +179,8 @@ impl LedgerCleanupService {
 
         let disk_utilization_pre = blockstore.storage_size();
         info!(
-            "purge: last_root={}, last_purge_slot={}, purge_interval={}, last_compaction_slot={}, disk_utilization={:?}",
-            root, last_purge_slot, purge_interval, last_compaction_slot, disk_utilization_pre
+            "purge: last_root={}, last_purge_slot={}, purge_interval={}, disk_utilization={:?}",
+            root, last_purge_slot, purge_interval, disk_utilization_pre
         );
 
         *last_purge_slot = root;
@@ -158,15 +189,10 @@ impl LedgerCleanupService {
             Self::find_slots_to_clean(&blockstore, root, max_ledger_shreds);
 
         if slots_to_clean {
-            let mut compact_first_slot = std::u64::MAX;
-            if lowest_cleanup_slot.saturating_sub(*last_compaction_slot) > compaction_interval {
-                compact_first_slot = *last_compaction_slot;
-                *last_compaction_slot = lowest_cleanup_slot;
-            }
-
             let purge_complete = Arc::new(AtomicBool::new(false));
             let blockstore = blockstore.clone();
             let purge_complete1 = purge_complete.clone();
+            let last_compact_slot1 = last_compact_slot.clone();
             let _t_purge = Builder::new()
                 .name("solana-ledger-purge".to_string())
                 .spawn(move || {
@@ -188,21 +214,7 @@ impl LedgerCleanupService {
                     purge_time.stop();
                     info!("{}", purge_time);
 
-                    if compact_first_slot < lowest_cleanup_slot {
-                        info!(
-                            "compacting data from slots {} to {}",
-                            compact_first_slot, lowest_cleanup_slot
-                        );
-                        if let Err(err) =
-                            blockstore.compact_storage(compact_first_slot, lowest_cleanup_slot)
-                        {
-                            // This error is not fatal and indicates an internal error?
-                            error!(
-                                "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
-                                err, compact_first_slot, lowest_cleanup_slot
-                            );
-                        }
-                    }
+                    last_compact_slot1.store(lowest_cleanup_slot, Ordering::Relaxed);
 
                     purge_complete1.store(true, Ordering::Relaxed);
                 })
@@ -223,6 +235,39 @@ impl LedgerCleanupService {
         Ok(())
     }
 
+    pub fn compact_ledger(
+        blockstore: &Arc<Blockstore>,
+        last_compaction_slot: &mut u64,
+        compaction_interval: u64,
+        highest_compact_slot: &Arc<AtomicU64>,
+        compaction_jitter: &mut u64,
+        max_jitter: Option<u64>,
+    ) {
+        let highest_compaction_slot = highest_compact_slot.load(Ordering::Relaxed);
+        if highest_compaction_slot.saturating_sub(*last_compaction_slot)
+            > (compaction_interval + *compaction_jitter)
+        {
+            info!(
+                "compacting data from slots {} to {}",
+                *last_compaction_slot, highest_compaction_slot,
+            );
+            if let Err(err) =
+                blockstore.compact_storage(*last_compaction_slot, highest_compaction_slot)
+            {
+                // This error is not fatal and indicates an internal error?
+                error!(
+                    "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
+                    err, last_compaction_slot, highest_compaction_slot,
+                );
+            }
+            *last_compaction_slot = highest_compaction_slot;
+            let jitter = max_jitter.unwrap_or(0);
+            if jitter > 0 {
+                *compaction_jitter = thread_rng().gen_range(0, jitter);
+            }
+        }
+    }
+
     fn report_disk_metrics(
         pre: BlockstoreResult<u64>,
         post: BlockstoreResult<u64>,
@@ -240,7 +285,8 @@ impl LedgerCleanupService {
     }
 
     pub fn join(self) -> thread::Result<()> {
-        self.t_cleanup.join()
+        self.t_cleanup.join()?;
+        self.t_compact.join()
     }
 }
 #[cfg(test)]
@@ -251,7 +297,7 @@ mod tests {
     use std::sync::mpsc::channel;
 
     #[test]
-    fn test_cleanup() {
+    fn test_cleanup1() {
         solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&blockstore_path).unwrap();
@@ -262,7 +308,7 @@ mod tests {
 
         //send a signal to kill all but 5 shreds, which will be in the newest slots
         let mut last_purge_slot = 0;
-        let mut last_compaction_slot = 0;
+        let highest_compaction_slot = Arc::new(AtomicU64::new(0));
         sender.send(50).unwrap();
         LedgerCleanupService::cleanup_ledger(
             &receiver,
@@ -270,10 +316,11 @@ mod tests {
             5,
             &mut last_purge_slot,
             10,
-            &mut last_compaction_slot,
-            10,
+            &highest_compaction_slot,
         )
         .unwrap();
+        assert_eq!(last_purge_slot, 50);
+        assert_eq!(highest_compaction_slot.load(Ordering::Relaxed), 44);
 
         //check that 0-40 don't exist
         blockstore
@@ -281,6 +328,18 @@ mod tests {
             .unwrap()
             .for_each(|(slot, _)| assert!(slot > 40));
 
+        let mut last_compaction_slot = 0;
+        let mut jitter = 0;
+        LedgerCleanupService::compact_ledger(
+            &blockstore,
+            &mut last_compaction_slot,
+            10,
+            &highest_compaction_slot,
+            &mut jitter,
+            None,
+        );
+        assert_eq!(jitter, 0);
+
         drop(blockstore);
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
@@ -303,7 +362,7 @@ mod tests {
         info!("{}", first_insert);
 
         let mut last_purge_slot = 0;
-        let mut last_compaction_slot = 0;
+        let last_compaction_slot = Arc::new(AtomicU64::new(0));
         let mut slot = initial_slots;
         let mut num_slots = 6;
         for _ in 0..5 {
@@ -327,8 +386,7 @@ mod tests {
                 initial_slots,
                 &mut last_purge_slot,
                 10,
-                &mut last_compaction_slot,
-                10,
+                &last_compaction_slot,
             )
             .unwrap();
             time.stop();

core/src/lib.rs

@@ -1,4 +1,5 @@
 #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))]
+#![allow(clippy::integer_arithmetic)]
 //! The `solana` library implements the Solana high-performance blockchain architecture.
 //! It includes a full Rust implementation of the architecture (see
 //! [Validator](server/struct.Validator.html)) as well as hooks to GPU implementations of its most
@@ -15,6 +16,7 @@ pub mod cluster_info_vote_listener;
 pub mod commitment_service;
 pub mod completed_data_sets_service;
 mod deprecated;
+pub mod max_slots;
 pub mod sample_performance_service;
 pub mod shred_fetch_stage;
 #[macro_use]
@@ -47,7 +49,6 @@ pub mod ping_pong;
 pub mod poh_recorder;
 pub mod poh_service;
 pub mod progress_map;
-pub mod pubkey_references;
 pub mod repair_response;
 pub mod repair_service;
 pub mod repair_weight;

core/src/max_slots.rs

@@ -0,0 +1,7 @@
+use std::sync::atomic::AtomicU64;
+
+#[derive(Default)]
+pub struct MaxSlots {
+    pub retransmit: AtomicU64,
+    pub shred_insert: AtomicU64,
+}

core/src/non_circulating_supply.rs

@@ -31,7 +31,7 @@ pub fn calculate_non_circulating_supply(bank: &Arc<Bank>) -> NonCirculatingSuppl
         bank.get_filtered_indexed_accounts(
             &IndexKey::ProgramId(solana_stake_program::id()),
             // The program-id account index checks for Account owner on inclusion. However, due to
-            // the current AccountsDB implementation, an account may remain in storage as a
+            // the current AccountsDb implementation, an account may remain in storage as a
             // zero-lamport Account::Default() after being wiped and reinitialized in later
             // updates. We include the redundant filter here to avoid returning these accounts.
             |account| account.owner == solana_stake_program::id(),

core/src/optimistically_confirmed_bank_tracker.rs

@@ -4,8 +4,9 @@
 
 use crate::rpc_subscriptions::RpcSubscriptions;
 use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
+use solana_client::rpc_response::SlotUpdate;
 use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::clock::Slot;
+use solana_sdk::{clock::Slot, timing::timestamp};
 use std::{
     collections::HashSet,
     sync::{
@@ -127,7 +128,15 @@ impl OptimisticallyConfirmedBankTracker {
                     drop(w_optimistically_confirmed_bank);
                 } else if slot > bank_forks.read().unwrap().root_bank().slot() {
                     pending_optimistically_confirmed_banks.insert(slot);
+                } else {
+                    inc_new_counter_info!("dropped-already-rooted-optimistic-bank-notification", 1);
                 }
+
+                // Send slot notification regardless of whether the bank is replayed
+                subscriptions.notify_slot_update(SlotUpdate::OptimisticConfirmation {
+                    slot,
+                    timestamp: timestamp(),
+                });
             }
             BankNotification::Frozen(bank) => {
                 let frozen_slot = bank.slot();
@@ -140,6 +149,10 @@ impl OptimisticallyConfirmedBankTracker {
                     }
                     drop(w_optimistically_confirmed_bank);
                 }
+                subscriptions.notify_slot_update(SlotUpdate::Frozen {
+                    slot: frozen_slot,
+                    timestamp: timestamp(),
+                });
             }
             BankNotification::Root(bank) => {
                 let root_slot = bank.slot();
@@ -168,7 +181,7 @@ mod tests {
     use super::*;
     use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_runtime::{
-        accounts_background_service::ABSRequestSender, commitment::BlockCommitmentCache,
+        accounts_background_service::AbsRequestSender, commitment::BlockCommitmentCache,
     };
     use solana_sdk::pubkey::Pubkey;
 
@@ -284,7 +297,7 @@ mod tests {
         bank_forks
             .write()
             .unwrap()
-            .set_root(7, &ABSRequestSender::default(), None);
+            .set_root(7, &AbsRequestSender::default(), None);
         OptimisticallyConfirmedBankTracker::process_notification(
             BankNotification::OptimisticallyConfirmed(6),
             &bank_forks,

core/src/poh_recorder.rs

@@ -501,6 +501,11 @@ impl PohRecorder {
             poh_config,
         )
     }
+
+    #[cfg(test)]
+    pub fn schedule_dummy_max_height_reached_failure(&mut self) {
+        self.reset(Hash::default(), 1, None);
+    }
 }
 
 #[cfg(test)]

core/src/poh_service.rs

@@ -248,11 +248,9 @@ mod tests {
                 if entry.is_tick() {
                     assert!(
                         entry.num_hashes <= poh_config.hashes_per_tick.unwrap(),
-                        format!(
-                            "{} <= {}",
-                            entry.num_hashes,
-                            poh_config.hashes_per_tick.unwrap()
-                        )
+                        "{} <= {}",
+                        entry.num_hashes,
+                        poh_config.hashes_per_tick.unwrap()
                     );
 
                     if entry.num_hashes == poh_config.hashes_per_tick.unwrap() {

core/src/progress_map.rs

@@ -1,7 +1,6 @@
 use crate::{
     cluster_info_vote_listener::SlotVoteTracker,
     cluster_slots::SlotPubkeys,
-    pubkey_references::PubkeyReferences,
     replay_stage::SUPERMINORITY_THRESHOLD,
     {consensus::Stake, consensus::VotedStakes},
 };
@@ -10,13 +9,12 @@ use solana_runtime::{bank::Bank, bank_forks::BankForks, vote_account::ArcVoteAcc
 use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
 use std::{
     collections::{BTreeMap, HashMap, HashSet},
-    rc::Rc,
     sync::{Arc, RwLock},
 };
 
 type VotedSlot = Slot;
 type ExpirationSlot = Slot;
-pub(crate) type LockoutIntervals = BTreeMap<ExpirationSlot, Vec<(VotedSlot, Rc<Pubkey>)>>;
+pub(crate) type LockoutIntervals = BTreeMap<ExpirationSlot, Vec<(VotedSlot, Pubkey)>>;
 
 #[derive(Default)]
 pub(crate) struct ReplaySlotStats(ConfirmationTiming);
@@ -128,9 +126,7 @@ impl ForkProgress {
                 (
                     true,
                     info.stake,
-                    vec![Rc::new(info.validator_vote_pubkey)]
-                        .into_iter()
-                        .collect(),
+                    vec![info.validator_vote_pubkey].into_iter().collect(),
                     {
                         if info.total_epoch_stake == 0 {
                             true
@@ -212,8 +208,8 @@ pub(crate) struct ForkStats {
 
 #[derive(Clone, Default)]
 pub(crate) struct PropagatedStats {
-    pub(crate) propagated_validators: HashSet<Rc<Pubkey>>,
-    pub(crate) propagated_node_ids: HashSet<Rc<Pubkey>>,
+    pub(crate) propagated_validators: HashSet<Pubkey>,
+    pub(crate) propagated_node_ids: HashSet<Pubkey>,
     pub(crate) propagated_validators_stake: u64,
     pub(crate) is_propagated: bool,
     pub(crate) is_leader_slot: bool,
@@ -224,25 +220,13 @@ pub(crate) struct PropagatedStats {
 }
 
 impl PropagatedStats {
-    pub fn add_vote_pubkey(
-        &mut self,
-        vote_pubkey: &Pubkey,
-        all_pubkeys: &mut PubkeyReferences,
-        stake: u64,
-    ) {
-        if !self.propagated_validators.contains(vote_pubkey) {
-            let cached_pubkey = all_pubkeys.get_or_insert(vote_pubkey);
-            self.propagated_validators.insert(cached_pubkey);
+    pub fn add_vote_pubkey(&mut self, vote_pubkey: Pubkey, stake: u64) {
+        if self.propagated_validators.insert(vote_pubkey) {
             self.propagated_validators_stake += stake;
         }
     }
 
-    pub fn add_node_pubkey(
-        &mut self,
-        node_pubkey: &Pubkey,
-        all_pubkeys: &mut PubkeyReferences,
-        bank: &Bank,
-    ) {
+    pub fn add_node_pubkey(&mut self, node_pubkey: &Pubkey, bank: &Bank) {
         if !self.propagated_node_ids.contains(node_pubkey) {
             let node_vote_accounts = bank
                 .epoch_vote_accounts_for_node_id(&node_pubkey)
@@ -251,7 +235,6 @@ impl PropagatedStats {
             if let Some(node_vote_accounts) = node_vote_accounts {
                 self.add_node_pubkey_internal(
                     node_pubkey,
-                    all_pubkeys,
                     node_vote_accounts,
                     bank.epoch_vote_accounts(bank.epoch())
                         .expect("Epoch stakes for bank's own epoch must exist"),
@@ -263,18 +246,16 @@ impl PropagatedStats {
     fn add_node_pubkey_internal(
         &mut self,
         node_pubkey: &Pubkey,
-        all_pubkeys: &mut PubkeyReferences,
         vote_account_pubkeys: &[Pubkey],
         epoch_vote_accounts: &HashMap<Pubkey, (u64, ArcVoteAccount)>,
     ) {
-        let cached_pubkey = all_pubkeys.get_or_insert(node_pubkey);
-        self.propagated_node_ids.insert(cached_pubkey);
+        self.propagated_node_ids.insert(*node_pubkey);
         for vote_account_pubkey in vote_account_pubkeys.iter() {
             let stake = epoch_vote_accounts
                 .get(vote_account_pubkey)
                 .map(|(stake, _)| *stake)
                 .unwrap_or(0);
-            self.add_vote_pubkey(vote_account_pubkey, all_pubkeys, stake);
+            self.add_vote_pubkey(*vote_account_pubkey, stake);
         }
     }
 }
@@ -403,34 +384,24 @@ mod test {
     #[test]
     fn test_add_vote_pubkey() {
         let mut stats = PropagatedStats::default();
-        let mut all_pubkeys = PubkeyReferences::default();
         let mut vote_pubkey = solana_sdk::pubkey::new_rand();
-        all_pubkeys.get_or_insert(&vote_pubkey);
 
         // Add a vote pubkey, the number of references in all_pubkeys
         // should be 2
-        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
+        stats.add_vote_pubkey(vote_pubkey, 1);
         assert!(stats.propagated_validators.contains(&vote_pubkey));
         assert_eq!(stats.propagated_validators_stake, 1);
-        assert_eq!(
-            Rc::strong_count(&all_pubkeys.get_or_insert(&vote_pubkey)),
-            3
-        );
 
         // Adding it again should change no state since the key already existed
-        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
+        stats.add_vote_pubkey(vote_pubkey, 1);
         assert!(stats.propagated_validators.contains(&vote_pubkey));
         assert_eq!(stats.propagated_validators_stake, 1);
 
         // Adding another pubkey should succeed
         vote_pubkey = solana_sdk::pubkey::new_rand();
-        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 2);
+        stats.add_vote_pubkey(vote_pubkey, 2);
         assert!(stats.propagated_validators.contains(&vote_pubkey));
         assert_eq!(stats.propagated_validators_stake, 3);
-        assert_eq!(
-            Rc::strong_count(&all_pubkeys.get_or_insert(&vote_pubkey)),
-            3
-        );
     }
 
     #[test]
@@ -447,35 +418,19 @@ mod test {
             .collect();
 
         let mut stats = PropagatedStats::default();
-        let mut all_pubkeys = PubkeyReferences::default();
         let mut node_pubkey = solana_sdk::pubkey::new_rand();
-        all_pubkeys.get_or_insert(&node_pubkey);
 
         // Add a vote pubkey, the number of references in all_pubkeys
         // should be 2
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
+        stats.add_node_pubkey_internal(&node_pubkey, &vote_account_pubkeys, &epoch_vote_accounts);
         assert!(stats.propagated_node_ids.contains(&node_pubkey));
         assert_eq!(
             stats.propagated_validators_stake,
             staked_vote_accounts as u64
         );
-        assert_eq!(
-            Rc::strong_count(&all_pubkeys.get_or_insert(&node_pubkey)),
-            3
-        );
 
         // Adding it again should not change any state
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
+        stats.add_node_pubkey_internal(&node_pubkey, &vote_account_pubkeys, &epoch_vote_accounts);
         assert!(stats.propagated_node_ids.contains(&node_pubkey));
         assert_eq!(
             stats.propagated_validators_stake,
@@ -485,21 +440,12 @@ mod test {
         // Adding another pubkey with same vote accounts should succeed, but stake
         // shouldn't increase
         node_pubkey = solana_sdk::pubkey::new_rand();
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
+        stats.add_node_pubkey_internal(&node_pubkey, &vote_account_pubkeys, &epoch_vote_accounts);
         assert!(stats.propagated_node_ids.contains(&node_pubkey));
         assert_eq!(
             stats.propagated_validators_stake,
             staked_vote_accounts as u64
         );
-        assert_eq!(
-            Rc::strong_count(&all_pubkeys.get_or_insert(&node_pubkey)),
-            3
-        );
 
         // Adding another pubkey with different vote accounts should succeed
         // and increase stake
@@ -512,21 +458,12 @@ mod test {
             .skip(num_vote_accounts - staked_vote_accounts)
             .map(|pubkey| (*pubkey, (1, ArcVoteAccount::default())))
             .collect();
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
+        stats.add_node_pubkey_internal(&node_pubkey, &vote_account_pubkeys, &epoch_vote_accounts);
         assert!(stats.propagated_node_ids.contains(&node_pubkey));
         assert_eq!(
             stats.propagated_validators_stake,
             2 * staked_vote_accounts as u64
         );
-        assert_eq!(
-            Rc::strong_count(&all_pubkeys.get_or_insert(&node_pubkey)),
-            3
-        );
     }
 
     #[test]

core/src/pubkey_references.rs

@@ -1,44 +0,0 @@
-use solana_sdk::pubkey::Pubkey;
-use std::{
-    collections::HashSet,
-    rc::Rc,
-    sync::{Arc, RwLock},
-};
-
-#[derive(Default)]
-pub struct PubkeyReferences(HashSet<Rc<Pubkey>>);
-
-impl PubkeyReferences {
-    pub fn get_or_insert(&mut self, pubkey: &Pubkey) -> Rc<Pubkey> {
-        let mut cached_pubkey: Option<Rc<Pubkey>> = self.0.get(pubkey).cloned();
-        if cached_pubkey.is_none() {
-            let new_pubkey = Rc::new(*pubkey);
-            self.0.insert(new_pubkey.clone());
-            cached_pubkey = Some(new_pubkey);
-        }
-        cached_pubkey.unwrap()
-    }
-
-    pub fn purge(&mut self) {
-        self.0.retain(|x| Rc::strong_count(x) > 1);
-    }
-}
-
-#[derive(Default)]
-pub struct LockedPubkeyReferences(pub RwLock<HashSet<Arc<Pubkey>>>);
-
-impl LockedPubkeyReferences {
-    pub fn get_or_insert(&self, pubkey: &Pubkey) -> Arc<Pubkey> {
-        let mut cached_pubkey = self.0.read().unwrap().get(pubkey).cloned();
-        if cached_pubkey.is_none() {
-            let new_pubkey = Arc::new(*pubkey);
-            self.0.write().unwrap().insert(new_pubkey.clone());
-            cached_pubkey = Some(new_pubkey);
-        }
-        cached_pubkey.unwrap()
-    }
-
-    pub fn purge(&self) {
-        self.0.write().unwrap().retain(|x| Arc::strong_count(x) > 1);
-    }
-}

core/src/repair_service.rs

@@ -1048,7 +1048,7 @@ mod test {
         // a valid target for repair
         let dead_slot = 9;
         let cluster_slots = ClusterSlots::default();
-        cluster_slots.insert_node_id(dead_slot, Arc::new(valid_repair_peer.id));
+        cluster_slots.insert_node_id(dead_slot, valid_repair_peer.id);
         cluster_info.insert_info(valid_repair_peer);
 
         // Not enough time has passed, should not update the
@@ -1178,7 +1178,7 @@ mod test {
         let cluster_slots = ClusterSlots::default();
         let duplicate_slot_repair_statuses = HashMap::new();
         let keypairs = ValidatorVoteKeypairs::new_rand();
-        let only_node_id = Arc::new(keypairs.node_keypair.pubkey());
+        let only_node_id = keypairs.node_keypair.pubkey();
         let GenesisConfigInfo { genesis_config, .. } =
             genesis_utils::create_genesis_config_with_vote_accounts(
                 1_000_000_000,

core/src/replay_stage.rs

@@ -13,7 +13,6 @@ use crate::{
     optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender},
     poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
     progress_map::{ForkProgress, ProgressMap, PropagatedStats},
-    pubkey_references::PubkeyReferences,
     repair_service::DuplicateSlotsResetReceiver,
     result::Result,
     rewards_recorder_service::RewardsRecorderSender,
@@ -29,7 +28,7 @@ use solana_ledger::{
 use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::inc_new_counter_info;
 use solana_runtime::{
-    accounts_background_service::ABSRequestSender, bank::Bank, bank_forks::BankForks,
+    accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
     commitment::BlockCommitmentCache, vote_sender_types::ReplayVoteSender,
 };
 use solana_sdk::{
@@ -44,7 +43,6 @@ use solana_sdk::{
 use solana_vote_program::{vote_instruction, vote_state::Vote};
 use std::{
     collections::{HashMap, HashSet},
-    ops::Deref,
     result,
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -100,7 +98,7 @@ pub struct ReplayStageConfig {
     pub subscriptions: Arc<RpcSubscriptions>,
     pub leader_schedule_cache: Arc<LeaderScheduleCache>,
     pub latest_root_senders: Vec<Sender<Slot>>,
-    pub accounts_background_request_sender: ABSRequestSender,
+    pub accounts_background_request_sender: AbsRequestSender,
     pub block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     pub transaction_status_sender: Option<TransactionStatusSender>,
     pub rewards_recorder_sender: Option<RewardsRecorderSender>,
@@ -111,6 +109,7 @@ pub struct ReplayStageConfig {
 #[derive(Default)]
 pub struct ReplayTiming {
     last_print: u64,
+    collect_frozen_banks_elapsed: u64,
     compute_bank_stats_elapsed: u64,
     select_vote_and_reset_forks_elapsed: u64,
     start_leader_elapsed: u64,
@@ -121,11 +120,15 @@ pub struct ReplayTiming {
     generate_new_bank_forks_elapsed: u64,
     replay_active_banks_elapsed: u64,
     reset_duplicate_slots_elapsed: u64,
+    wait_receive_elapsed: u64,
+    heaviest_fork_failures_elapsed: u64,
+    bank_count: u64,
 }
 impl ReplayTiming {
     #[allow(clippy::too_many_arguments)]
     fn update(
         &mut self,
+        collect_frozen_banks_elapsed: u64,
         compute_bank_stats_elapsed: u64,
         select_vote_and_reset_forks_elapsed: u64,
         start_leader_elapsed: u64,
@@ -136,7 +139,11 @@ impl ReplayTiming {
         generate_new_bank_forks_elapsed: u64,
         replay_active_banks_elapsed: u64,
         reset_duplicate_slots_elapsed: u64,
+        wait_receive_elapsed: u64,
+        heaviest_fork_failures_elapsed: u64,
+        bank_count: u64,
     ) {
+        self.collect_frozen_banks_elapsed += collect_frozen_banks_elapsed;
         self.compute_bank_stats_elapsed += compute_bank_stats_elapsed;
         self.select_vote_and_reset_forks_elapsed += select_vote_and_reset_forks_elapsed;
         self.start_leader_elapsed += start_leader_elapsed;
@@ -147,12 +154,20 @@ impl ReplayTiming {
         self.generate_new_bank_forks_elapsed += generate_new_bank_forks_elapsed;
         self.replay_active_banks_elapsed += replay_active_banks_elapsed;
         self.reset_duplicate_slots_elapsed += reset_duplicate_slots_elapsed;
+        self.wait_receive_elapsed += wait_receive_elapsed;
+        self.heaviest_fork_failures_elapsed += heaviest_fork_failures_elapsed;
+        self.bank_count += bank_count;
         let now = timestamp();
         let elapsed_ms = now - self.last_print;
         if elapsed_ms > 1000 {
             datapoint_info!(
                 "replay-loop-timing-stats",
                 ("total_elapsed_us", elapsed_ms * 1000, i64),
+                (
+                    "collect_frozen_banks_elapsed",
+                    self.collect_frozen_banks_elapsed as i64,
+                    i64
+                ),
                 (
                     "compute_bank_stats_elapsed",
                     self.compute_bank_stats_elapsed as i64,
@@ -195,6 +210,17 @@ impl ReplayTiming {
                     self.reset_duplicate_slots_elapsed as i64,
                     i64
                 ),
+                (
+                    "wait_receive_elapsed",
+                    self.wait_receive_elapsed as i64,
+                    i64
+                ),
+                (
+                    "heaviest_fork_failures_elapsed",
+                    self.heaviest_fork_failures_elapsed as i64,
+                    i64
+                ),
+                ("bank_count", self.bank_count as i64, i64),
             );
 
             *self = ReplayTiming::default();
@@ -252,7 +278,6 @@ impl ReplayStage {
         let t_replay = Builder::new()
             .name("solana-replay-stage".to_string())
             .spawn(move || {
-                let mut all_pubkeys = PubkeyReferences::default();
                 let verify_recyclers = VerifyRecyclers::default();
                 let _exit = Finalizer::new(exit.clone());
                 let (
@@ -286,7 +311,6 @@ impl ReplayStage {
                         &leader_schedule_cache,
                         &subscriptions,
                         &mut progress,
-                        &mut all_pubkeys,
                     );
                     generate_new_bank_forks_time.stop();
                     Self::report_memory(&allocated, "generate_new_bank_forks", start);
@@ -313,7 +337,7 @@ impl ReplayStage {
 
                     let mut reset_duplicate_slots_time = Measure::start("reset_duplicate_slots");
                     let mut ancestors = bank_forks.read().unwrap().ancestors();
-                    let mut descendants = bank_forks.read().unwrap().descendants();
+                    let mut descendants = bank_forks.read().unwrap().descendants().clone();
                     let forks_root = bank_forks.read().unwrap().root();
                     let start = allocated.get();
 
@@ -350,7 +374,6 @@ impl ReplayStage {
                         &vote_tracker,
                         &cluster_slots,
                         &bank_forks,
-                        &mut all_pubkeys,
                         &mut heaviest_subtree_fork_choice,
                     );
                     compute_bank_stats_time.stop();
@@ -401,6 +424,7 @@ impl ReplayStage {
                     );
                     select_vote_and_reset_forks_time.stop();
 
+                    let mut heaviest_fork_failures_time = Measure::start("heaviest_fork_failures_time");
                     if tower.is_recent(heaviest_bank.slot()) && !heaviest_fork_failures.is_empty() {
                         info!(
                             "Couldn't vote on heaviest fork: {:?}, heaviest_fork_failures: {:?}",
@@ -418,6 +442,7 @@ impl ReplayStage {
                             }
                         }
                     }
+                    heaviest_fork_failures_time.stop();
 
                     let start = allocated.get();
 
@@ -449,7 +474,6 @@ impl ReplayStage {
                             &lockouts_sender,
                             &accounts_background_request_sender,
                             &latest_root_senders,
-                            &mut all_pubkeys,
                             &subscriptions,
                             &block_commitment_cache,
                             &mut heaviest_subtree_fork_choice,
@@ -561,7 +585,22 @@ impl ReplayStage {
                     start_leader_time.stop();
                     Self::report_memory(&allocated, "start_leader", start);
 
+                    let mut wait_receive_time = Measure::start("wait_receive_time");
+                    if !did_complete_bank {
+                        // only wait for the signal if we did not just process a bank; maybe there are more slots available
+
+                        let timer = Duration::from_millis(100);
+                        let result = ledger_signal_receiver.recv_timeout(timer);
+                        match result {
+                            Err(RecvTimeoutError::Timeout) => (),
+                            Err(_) => break,
+                            Ok(_) => trace!("blockstore signal"),
+                        };
+                    }
+                    wait_receive_time.stop();
+
                     replay_timing.update(
+                        collect_frozen_banks_time.as_us(),
                         compute_bank_stats_time.as_us(),
                         select_vote_and_reset_forks_time.as_us(),
                         start_leader_time.as_us(),
@@ -572,19 +611,10 @@ impl ReplayStage {
                         generate_new_bank_forks_time.as_us(),
                         replay_active_banks_time.as_us(),
                         reset_duplicate_slots_time.as_us(),
+                        wait_receive_time.as_us(),
+                        heaviest_fork_failures_time.as_us(),
+                        if did_complete_bank {1} else {0},
                     );
-
-                    if did_complete_bank {
-                        //just processed a bank, skip the signal; maybe there's more slots available
-                        continue;
-                    }
-                    let timer = Duration::from_millis(100);
-                    let result = ledger_signal_receiver.recv_timeout(timer);
-                    match result {
-                        Err(RecvTimeoutError::Timeout) => continue,
-                        Err(_) => break,
-                        Ok(_) => trace!("blockstore signal"),
-                    };
                 }
                 Ok(())
             })
@@ -979,6 +1009,7 @@ impl ReplayStage {
             Some(replay_vote_sender),
             None,
             verify_recyclers,
+            false,
         );
         let tx_count_after = bank_progress.replay_progress.num_txs;
         let tx_count = tx_count_after - tx_count_before;
@@ -988,11 +1019,19 @@ impl ReplayStage {
             // that comes after the root, so we should not see any
             // errors related to the slot being purged
             let slot = bank.slot();
-            warn!("Fatal replay error in slot: {}, err: {:?}", slot, err);
-            let is_serious = matches!(
+
+            // Block producer can abandon the block if it detects a better one
+            // while producing. Somewhat common and expected in a
+            // network with variable network/machine configuration.
+            let is_serious = !matches!(
                 err,
-                BlockstoreProcessorError::InvalidBlock(BlockError::InvalidTickCount)
+                BlockstoreProcessorError::InvalidBlock(BlockError::TooFewTicks)
             );
+            if is_serious {
+                warn!("Fatal replay error in slot: {}, err: {:?}", slot, err);
+            } else {
+                info!("Slot had too few ticks: {}", slot);
+            }
             Self::mark_dead_slot(blockstore, bank_progress, slot, &err, is_serious);
             err
         })?;
@@ -1039,9 +1078,8 @@ impl ReplayStage {
         blockstore: &Arc<Blockstore>,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         lockouts_sender: &Sender<CommitmentAggregationData>,
-        accounts_background_request_sender: &ABSRequestSender,
+        accounts_background_request_sender: &AbsRequestSender,
         latest_root_senders: &[Sender<Slot>],
-        all_pubkeys: &mut PubkeyReferences,
         subscriptions: &Arc<RpcSubscriptions>,
         block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
         heaviest_subtree_fork_choice: &mut HeaviestSubtreeForkChoice,
@@ -1097,7 +1135,6 @@ impl ReplayStage {
                 &bank_forks,
                 progress,
                 accounts_background_request_sender,
-                all_pubkeys,
                 highest_confirmed_root,
                 heaviest_subtree_fork_choice,
             );
@@ -1393,7 +1430,6 @@ impl ReplayStage {
         vote_tracker: &VoteTracker,
         cluster_slots: &ClusterSlots,
         bank_forks: &RwLock<BankForks>,
-        all_pubkeys: &mut PubkeyReferences,
         heaviest_subtree_fork_choice: &mut dyn ForkChoice,
     ) -> Vec<Slot> {
         frozen_banks.sort_by_key(|bank| bank.slot());
@@ -1414,7 +1450,6 @@ impl ReplayStage {
                         bank_slot,
                         bank.vote_accounts().into_iter(),
                         &ancestors,
-                        all_pubkeys,
                     );
                     // Notify any listeners of the votes found in this newly computed
                     // bank
@@ -1459,7 +1494,6 @@ impl ReplayStage {
             Self::update_propagation_status(
                 progress,
                 bank_slot,
-                all_pubkeys,
                 bank_forks,
                 vote_tracker,
                 cluster_slots,
@@ -1481,7 +1515,6 @@ impl ReplayStage {
     fn update_propagation_status(
         progress: &mut ProgressMap,
         slot: Slot,
-        all_pubkeys: &mut PubkeyReferences,
         bank_forks: &RwLock<BankForks>,
         vote_tracker: &VoteTracker,
         cluster_slots: &ClusterSlots,
@@ -1535,7 +1568,6 @@ impl ReplayStage {
             cluster_slot_pubkeys,
             slot,
             bank_forks,
-            all_pubkeys,
         );
     }
 
@@ -1650,11 +1682,10 @@ impl ReplayStage {
 
     fn update_fork_propagated_threshold_from_votes(
         progress: &mut ProgressMap,
-        mut newly_voted_pubkeys: Vec<impl Deref<Target = Pubkey>>,
-        mut cluster_slot_pubkeys: Vec<impl Deref<Target = Pubkey>>,
+        mut newly_voted_pubkeys: Vec<Pubkey>,
+        mut cluster_slot_pubkeys: Vec<Pubkey>,
         fork_tip: Slot,
         bank_forks: &RwLock<BankForks>,
-        all_pubkeys: &mut PubkeyReferences,
     ) {
         let mut current_leader_slot = progress.get_latest_leader_slot(fork_tip);
         let mut did_newly_reach_threshold = false;
@@ -1700,7 +1731,6 @@ impl ReplayStage {
                 &mut cluster_slot_pubkeys,
                 &leader_bank,
                 leader_propagated_stats,
-                all_pubkeys,
                 did_newly_reach_threshold,
             ) || did_newly_reach_threshold;
 
@@ -1710,11 +1740,10 @@ impl ReplayStage {
     }
 
     fn update_slot_propagated_threshold_from_votes(
-        newly_voted_pubkeys: &mut Vec<impl Deref<Target = Pubkey>>,
-        cluster_slot_pubkeys: &mut Vec<impl Deref<Target = Pubkey>>,
+        newly_voted_pubkeys: &mut Vec<Pubkey>,
+        cluster_slot_pubkeys: &mut Vec<Pubkey>,
         leader_bank: &Bank,
         leader_propagated_stats: &mut PropagatedStats,
-        all_pubkeys: &mut PubkeyReferences,
         did_child_reach_threshold: bool,
     ) -> bool {
         // Track whether this slot newly confirm propagation
@@ -1748,10 +1777,9 @@ impl ReplayStage {
         newly_voted_pubkeys.retain(|vote_pubkey| {
             let exists = leader_propagated_stats
                 .propagated_validators
-                .contains(&**vote_pubkey);
+                .contains(vote_pubkey);
             leader_propagated_stats.add_vote_pubkey(
-                &*vote_pubkey,
-                all_pubkeys,
+                *vote_pubkey,
                 leader_bank.epoch_vote_account_stake(&vote_pubkey),
             );
             !exists
@@ -1760,8 +1788,8 @@ impl ReplayStage {
         cluster_slot_pubkeys.retain(|node_pubkey| {
             let exists = leader_propagated_stats
                 .propagated_node_ids
-                .contains(&**node_pubkey);
-            leader_propagated_stats.add_node_pubkey(&*node_pubkey, all_pubkeys, leader_bank);
+                .contains(node_pubkey);
+            leader_propagated_stats.add_node_pubkey(&*node_pubkey, leader_bank);
             !exists
         });
 
@@ -1815,22 +1843,16 @@ impl ReplayStage {
         new_root: Slot,
         bank_forks: &RwLock<BankForks>,
         progress: &mut ProgressMap,
-        accounts_background_request_sender: &ABSRequestSender,
-        all_pubkeys: &mut PubkeyReferences,
+        accounts_background_request_sender: &AbsRequestSender,
         highest_confirmed_root: Option<Slot>,
         heaviest_subtree_fork_choice: &mut HeaviestSubtreeForkChoice,
     ) {
-        let old_epoch = bank_forks.read().unwrap().root_bank().epoch();
         bank_forks.write().unwrap().set_root(
             new_root,
             accounts_background_request_sender,
             highest_confirmed_root,
         );
         let r_bank_forks = bank_forks.read().unwrap();
-        let new_epoch = bank_forks.read().unwrap().root_bank().epoch();
-        if old_epoch != new_epoch {
-            all_pubkeys.purge();
-        }
         progress.handle_new_root(&r_bank_forks);
         heaviest_subtree_fork_choice.set_root(new_root);
     }
@@ -1841,7 +1863,6 @@ impl ReplayStage {
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         subscriptions: &Arc<RpcSubscriptions>,
         progress: &mut ProgressMap,
-        all_pubkeys: &mut PubkeyReferences,
     ) {
         // Find the next slot that chains to the old slot
         let forks = bank_forks.read().unwrap();
@@ -1887,14 +1908,13 @@ impl ReplayStage {
                     &leader,
                     subscriptions,
                 );
-                let empty: Vec<&Pubkey> = vec![];
+                let empty: Vec<Pubkey> = vec![];
                 Self::update_fork_propagated_threshold_from_votes(
                     progress,
                     empty,
-                    vec![&leader],
+                    vec![leader],
                     parent_bank.slot(),
                     bank_forks,
-                    all_pubkeys,
                 );
                 new_banks.insert(child_slot, child_bank);
             }
@@ -2001,7 +2021,7 @@ pub(crate) mod tests {
         },
     };
     use solana_runtime::{
-        accounts_background_service::ABSRequestSender,
+        accounts_background_service::AbsRequestSender,
         commitment::BlockCommitment,
         genesis_utils::{self, GenesisConfigInfo, ValidatorVoteKeypairs},
     };
@@ -2023,7 +2043,6 @@ pub(crate) mod tests {
     use std::{
         fs::remove_dir_all,
         iter,
-        rc::Rc,
         sync::{Arc, RwLock},
     };
     use trees::tr;
@@ -2164,7 +2183,6 @@ pub(crate) mod tests {
             &leader_schedule_cache,
             &rpc_subscriptions,
             &mut progress,
-            &mut PubkeyReferences::default(),
         );
         assert!(bank_forks
             .read()
@@ -2187,7 +2205,6 @@ pub(crate) mod tests {
             &leader_schedule_cache,
             &rpc_subscriptions,
             &mut progress,
-            &mut PubkeyReferences::default(),
         );
         assert!(bank_forks
             .read()
@@ -2240,8 +2257,7 @@ pub(crate) mod tests {
             root,
             &bank_forks,
             &mut progress,
-            &ABSRequestSender::default(),
-            &mut PubkeyReferences::default(),
+            &AbsRequestSender::default(),
             None,
             &mut heaviest_subtree_fork_choice,
         );
@@ -2285,8 +2301,7 @@ pub(crate) mod tests {
             root,
             &bank_forks,
             &mut progress,
-            &ABSRequestSender::default(),
-            &mut PubkeyReferences::default(),
+            &AbsRequestSender::default(),
             Some(confirmed_root),
             &mut heaviest_subtree_fork_choice,
         );
@@ -2390,6 +2405,7 @@ pub(crate) mod tests {
 
     #[test]
     fn test_dead_fork_invalid_slot_tick_count() {
+        solana_logger::setup();
         // Too many ticks per slot
         let res = check_dead_fork(|_keypair, bank| {
             let blockhash = bank.last_blockhash();
@@ -2405,7 +2421,7 @@ pub(crate) mod tests {
         });
 
         if let Err(BlockstoreProcessorError::InvalidBlock(block_error)) = res {
-            assert_eq!(block_error, BlockError::InvalidTickCount);
+            assert_eq!(block_error, BlockError::TooManyTicks);
         } else {
             panic!();
         }
@@ -2425,7 +2441,7 @@ pub(crate) mod tests {
         });
 
         if let Err(BlockstoreProcessorError::InvalidBlock(block_error)) = res {
-            assert_eq!(block_error, BlockError::InvalidTickCount);
+            assert_eq!(block_error, BlockError::TooFewTicks);
         } else {
             panic!();
         }
@@ -2710,7 +2726,10 @@ pub(crate) mod tests {
             &bank,
             &entries,
             true,
-            Some(transaction_status_sender),
+            Some(TransactionStatusSender {
+                sender: transaction_status_sender,
+                enable_cpi_and_log_storage: false,
+            }),
             Some(&replay_vote_sender),
         );
 
@@ -2751,7 +2770,7 @@ pub(crate) mod tests {
                 blockstore.clone(),
             );
 
-            let confirmed_block = blockstore.get_confirmed_block(slot).unwrap();
+            let confirmed_block = blockstore.get_confirmed_block(slot, false).unwrap();
             assert_eq!(confirmed_block.transactions.len(), 3);
 
             for TransactionWithStatusMeta { transaction, meta } in
@@ -2821,7 +2840,6 @@ pub(crate) mod tests {
             &VoteTracker::default(),
             &ClusterSlots::default(),
             &bank_forks,
-            &mut PubkeyReferences::default(),
             &mut heaviest_subtree_fork_choice,
         );
 
@@ -2866,7 +2884,6 @@ pub(crate) mod tests {
             &VoteTracker::default(),
             &ClusterSlots::default(),
             &bank_forks,
-            &mut PubkeyReferences::default(),
             &mut heaviest_subtree_fork_choice,
         );
 
@@ -2901,7 +2918,6 @@ pub(crate) mod tests {
             &VoteTracker::default(),
             &ClusterSlots::default(),
             &bank_forks,
-            &mut PubkeyReferences::default(),
             &mut heaviest_subtree_fork_choice,
         );
         // No new stats should have been computed
@@ -2938,7 +2954,6 @@ pub(crate) mod tests {
             &VoteTracker::default(),
             &ClusterSlots::default(),
             &vote_simulator.bank_forks,
-            &mut PubkeyReferences::default(),
             &mut heaviest_subtree_fork_choice,
         );
 
@@ -3000,7 +3015,6 @@ pub(crate) mod tests {
             &VoteTracker::default(),
             &ClusterSlots::default(),
             &vote_simulator.bank_forks,
-            &mut PubkeyReferences::default(),
             &mut vote_simulator.heaviest_subtree_fork_choice,
         );
 
@@ -3126,29 +3140,19 @@ pub(crate) mod tests {
             ..PropagatedStats::default()
         };
 
-        let mut all_pubkeys = PubkeyReferences::default();
         let child_reached_threshold = false;
         for i in 0..std::cmp::max(new_vote_pubkeys.len(), new_node_pubkeys.len()) {
             propagated_stats.is_propagated = false;
             let len = std::cmp::min(i, new_vote_pubkeys.len());
-            let mut voted_pubkeys = new_vote_pubkeys[..len]
-                .iter()
-                .cloned()
-                .map(Arc::new)
-                .collect();
+            let mut voted_pubkeys = new_vote_pubkeys[..len].iter().copied().collect();
             let len = std::cmp::min(i, new_node_pubkeys.len());
-            let mut node_pubkeys = new_node_pubkeys[..len]
-                .iter()
-                .cloned()
-                .map(Arc::new)
-                .collect();
+            let mut node_pubkeys = new_node_pubkeys[..len].iter().copied().collect();
             let did_newly_reach_threshold =
                 ReplayStage::update_slot_propagated_threshold_from_votes(
                     &mut voted_pubkeys,
                     &mut node_pubkeys,
                     &root_bank,
                     &mut propagated_stats,
-                    &mut all_pubkeys,
                     child_reached_threshold,
                 );
 
@@ -3159,14 +3163,14 @@ pub(crate) mod tests {
                 if i == 0 || i >= new_vote_pubkeys.len() {
                     vec![]
                 } else {
-                    vec![Arc::new(new_vote_pubkeys[i - 1])]
+                    vec![new_vote_pubkeys[i - 1]]
                 }
             };
             let remaining_node_pubkeys = {
                 if i == 0 || i >= new_node_pubkeys.len() {
                     vec![]
                 } else {
-                    vec![Arc::new(new_node_pubkeys[i - 1])]
+                    vec![new_node_pubkeys[i - 1]]
                 }
             };
             assert_eq!(voted_pubkeys, remaining_vote_pubkeys);
@@ -3187,7 +3191,7 @@ pub(crate) mod tests {
 
     #[test]
     fn test_update_slot_propagated_threshold_from_votes2() {
-        let mut empty: Vec<&Pubkey> = vec![];
+        let mut empty: Vec<Pubkey> = vec![];
         let genesis_config = create_genesis_config(100_000_000).genesis_config;
         let root_bank = Bank::new(&genesis_config);
         let stake = 10_000;
@@ -3199,16 +3203,14 @@ pub(crate) mod tests {
             ..PropagatedStats::default()
         };
         propagated_stats.total_epoch_stake = stake * 10;
-        let mut all_pubkeys = PubkeyReferences::default();
         let child_reached_threshold = true;
-        let mut newly_voted_pubkeys: Vec<Arc<Pubkey>> = vec![];
+        let mut newly_voted_pubkeys: Vec<Pubkey> = vec![];
 
         assert!(ReplayStage::update_slot_propagated_threshold_from_votes(
             &mut newly_voted_pubkeys,
             &mut empty,
             &root_bank,
             &mut propagated_stats,
-            &mut all_pubkeys,
             child_reached_threshold,
         ));
 
@@ -3219,14 +3221,12 @@ pub(crate) mod tests {
             ..PropagatedStats::default()
         };
         propagated_stats.is_propagated = true;
-        all_pubkeys = PubkeyReferences::default();
         newly_voted_pubkeys = vec![];
         assert!(!ReplayStage::update_slot_propagated_threshold_from_votes(
             &mut newly_voted_pubkeys,
             &mut empty,
             &root_bank,
             &mut propagated_stats,
-            &mut all_pubkeys,
             child_reached_threshold,
         ));
 
@@ -3236,7 +3236,6 @@ pub(crate) mod tests {
             &mut empty,
             &root_bank,
             &mut propagated_stats,
-            &mut all_pubkeys,
             child_reached_threshold,
         ));
     }
@@ -3246,7 +3245,7 @@ pub(crate) mod tests {
         // Create genesis stakers
         let vote_keypairs = ValidatorVoteKeypairs::new_rand();
         let node_pubkey = vote_keypairs.node_keypair.pubkey();
-        let vote_pubkey = Arc::new(vote_keypairs.vote_keypair.pubkey());
+        let vote_pubkey = vote_keypairs.vote_keypair.pubkey();
         let keypairs: HashMap<_, _> = vec![(node_pubkey, vote_keypairs)].into_iter().collect();
         let stake = 10_000;
         let (mut bank_forks, mut progress_map, _) = initialize_state(&keypairs, stake);
@@ -3255,7 +3254,7 @@ pub(crate) mod tests {
         bank_forks.insert(Bank::new_from_parent(&bank0, &Pubkey::default(), 9));
         let bank9 = bank_forks.get(9).unwrap().clone();
         bank_forks.insert(Bank::new_from_parent(&bank9, &Pubkey::default(), 10));
-        bank_forks.set_root(9, &ABSRequestSender::default(), None);
+        bank_forks.set_root(9, &AbsRequestSender::default(), None);
         let total_epoch_stake = bank0.total_epoch_stake();
 
         // Insert new ForkProgress for slot 10 and its
@@ -3292,11 +3291,10 @@ pub(crate) mod tests {
         assert!(!progress_map.is_propagated(10));
 
         let vote_tracker = VoteTracker::new(&bank_forks.root_bank());
-        vote_tracker.insert_vote(10, vote_pubkey.clone());
+        vote_tracker.insert_vote(10, vote_pubkey);
         ReplayStage::update_propagation_status(
             &mut progress_map,
             10,
-            &mut PubkeyReferences::default(),
             &RwLock::new(bank_forks),
             &vote_tracker,
             &ClusterSlots::default(),
@@ -3321,7 +3319,7 @@ pub(crate) mod tests {
         // The voter should be recorded
         assert!(propagated_stats
             .propagated_validators
-            .contains(&*vote_pubkey));
+            .contains(&vote_pubkey));
 
         assert_eq!(propagated_stats.propagated_validators_stake, stake);
     }
@@ -3347,7 +3345,7 @@ pub(crate) mod tests {
             .get_propagated_stats_mut(0)
             .unwrap()
             .is_leader_slot = true;
-        bank_forks.set_root(0, &ABSRequestSender::default(), None);
+        bank_forks.set_root(0, &AbsRequestSender::default(), None);
         let total_epoch_stake = bank_forks.root_bank().total_epoch_stake();
 
         // Insert new ForkProgress representing a slot for all slots 1..=num_banks. Only
@@ -3380,7 +3378,7 @@ pub(crate) mod tests {
         let vote_tracker = VoteTracker::new(&bank_forks.root_bank());
         for vote_pubkey in &vote_pubkeys {
             // Insert a vote for the last bank for each voter
-            vote_tracker.insert_vote(10, Arc::new(*vote_pubkey));
+            vote_tracker.insert_vote(10, *vote_pubkey);
         }
 
         // The last bank should reach propagation threshold, and propagate it all
@@ -3388,7 +3386,6 @@ pub(crate) mod tests {
         ReplayStage::update_propagation_status(
             &mut progress_map,
             10,
-            &mut PubkeyReferences::default(),
             &RwLock::new(bank_forks),
             &vote_tracker,
             &ClusterSlots::default(),
@@ -3428,7 +3425,7 @@ pub(crate) mod tests {
             .get_propagated_stats_mut(0)
             .unwrap()
             .is_leader_slot = true;
-        bank_forks.set_root(0, &ABSRequestSender::default(), None);
+        bank_forks.set_root(0, &AbsRequestSender::default(), None);
 
         let total_epoch_stake = num_validators as u64 * stake_per_validator;
 
@@ -3458,11 +3455,8 @@ pub(crate) mod tests {
                     1
                 }
             };
-            fork_progress.propagated_stats.propagated_validators = vote_pubkeys[0..end_range]
-                .iter()
-                .cloned()
-                .map(Rc::new)
-                .collect();
+            fork_progress.propagated_stats.propagated_validators =
+                vote_pubkeys[0..end_range].iter().copied().collect();
             fork_progress.propagated_stats.propagated_validators_stake =
                 end_range as u64 * stake_per_validator;
             progress_map.insert(i, fork_progress);
@@ -3470,14 +3464,13 @@ pub(crate) mod tests {
 
         let vote_tracker = VoteTracker::new(&bank_forks.root_bank());
         // Insert a new vote
-        vote_tracker.insert_vote(10, Arc::new(vote_pubkeys[2]));
+        vote_tracker.insert_vote(10, vote_pubkeys[2]);
 
         // The last bank should reach propagation threshold, and propagate it all
         // the way back through earlier leader banks
         ReplayStage::update_propagation_status(
             &mut progress_map,
             10,
-            &mut PubkeyReferences::default(),
             &RwLock::new(bank_forks),
             &vote_tracker,
             &ClusterSlots::default(),
@@ -3686,7 +3679,7 @@ pub(crate) mod tests {
     #[test]
     fn test_purge_unconfirmed_duplicate_slot() {
         let (bank_forks, mut progress) = setup_forks();
-        let mut descendants = bank_forks.read().unwrap().descendants();
+        let mut descendants = bank_forks.read().unwrap().descendants().clone();
         let mut ancestors = bank_forks.read().unwrap().ancestors();
 
         // Purging slot 5 should purge only slots 5 and its descendant 6
@@ -3707,7 +3700,7 @@ pub(crate) mod tests {
         }
 
         // Purging slot 4 should purge only slot 4
-        let mut descendants = bank_forks.read().unwrap().descendants();
+        let mut descendants = bank_forks.read().unwrap().descendants().clone();
         let mut ancestors = bank_forks.read().unwrap().ancestors();
         ReplayStage::purge_unconfirmed_duplicate_slot(
             4,
@@ -3726,7 +3719,7 @@ pub(crate) mod tests {
         }
 
         // Purging slot 1 should purge both forks 2 and 3
-        let mut descendants = bank_forks.read().unwrap().descendants();
+        let mut descendants = bank_forks.read().unwrap().descendants().clone();
         let mut ancestors = bank_forks.read().unwrap().ancestors();
         ReplayStage::purge_unconfirmed_duplicate_slot(
             1,
@@ -3748,7 +3741,7 @@ pub(crate) mod tests {
         let (bank_forks, _) = setup_forks();
 
         // Purge branch rooted at slot 2
-        let mut descendants = bank_forks.read().unwrap().descendants();
+        let mut descendants = bank_forks.read().unwrap().descendants().clone();
         let mut ancestors = bank_forks.read().unwrap().ancestors();
         let slot_2_descendants = descendants.get(&2).unwrap().clone();
         ReplayStage::purge_ancestors_descendants(
@@ -3777,8 +3770,8 @@ pub(crate) mod tests {
         bank_forks
             .write()
             .unwrap()
-            .set_root(3, &ABSRequestSender::default(), None);
-        let mut descendants = bank_forks.read().unwrap().descendants();
+            .set_root(3, &AbsRequestSender::default(), None);
+        let mut descendants = bank_forks.read().unwrap().descendants().clone();
         let mut ancestors = bank_forks.read().unwrap().ancestors();
         let slot_3_descendants = descendants.get(&3).unwrap().clone();
         ReplayStage::purge_ancestors_descendants(
@@ -3834,7 +3827,7 @@ pub(crate) mod tests {
 
         // Add votes
         for vote_key in validator_voting_keys.values() {
-            vote_tracker.insert_vote(root_bank.slot(), Arc::new(*vote_key));
+            vote_tracker.insert_vote(root_bank.slot(), *vote_key);
         }
 
         assert!(!progress.is_propagated(root_bank.slot()));
@@ -3850,7 +3843,6 @@ pub(crate) mod tests {
             &vote_tracker,
             &ClusterSlots::default(),
             &bank_forks,
-            &mut PubkeyReferences::default(),
             &mut HeaviestSubtreeForkChoice::new_from_bank_forks(&bank_forks.read().unwrap()),
         );
 

core/src/result.rs

@@ -10,8 +10,8 @@ use std::any::Any;
 
 #[derive(Debug)]
 pub enum Error {
-    IO(std::io::Error),
-    JSON(serde_json::Error),
+    Io(std::io::Error),
+    Json(serde_json::Error),
     AddrParse(std::net::AddrParseError),
     JoinError(Box<dyn Any + Send + 'static>),
     RecvError(std::sync::mpsc::RecvError),
@@ -108,7 +108,7 @@ impl std::convert::From<Box<dyn Any + Send + 'static>> for Error {
 }
 impl std::convert::From<std::io::Error> for Error {
     fn from(e: std::io::Error) -> Error {
-        Error::IO(e)
+        Error::Io(e)
     }
 }
 impl std::convert::From<fs_extra::error::Error> for Error {
@@ -118,7 +118,7 @@ impl std::convert::From<fs_extra::error::Error> for Error {
 }
 impl std::convert::From<serde_json::Error> for Error {
     fn from(e: serde_json::Error) -> Error {
-        Error::JSON(e)
+        Error::Json(e)
     }
 }
 impl std::convert::From<std::net::AddrParseError> for Error {
@@ -199,7 +199,7 @@ mod tests {
         assert_matches!(send_error(), Err(Error::SendError));
         assert_matches!(join_error(), Err(Error::JoinError(_)));
         let ioe = io::Error::new(io::ErrorKind::NotFound, "hi");
-        assert_matches!(Error::from(ioe), Error::IO(_));
+        assert_matches!(Error::from(ioe), Error::Io(_));
     }
     #[test]
     fn fmt_test() {

core/src/retransmit_stage.rs

@@ -8,13 +8,16 @@ use crate::{
     cluster_slots_service::ClusterSlotsService,
     completed_data_sets_service::CompletedDataSetsSender,
     contact_info::ContactInfo,
+    max_slots::MaxSlots,
     repair_service::DuplicateSlotsResetSender,
     repair_service::RepairInfo,
     result::{Error, Result},
+    rpc_subscriptions::RpcSubscriptions,
     window_service::{should_retransmit_and_persist, WindowService},
 };
 use crossbeam_channel::Receiver;
 use lru::LruCache;
+use solana_client::rpc_response::SlotUpdate;
 use solana_ledger::shred::{get_shred_slot_index_type, ShredFetchStats};
 use solana_ledger::{
     blockstore::{Blockstore, CompletedSlotsReceiver},
@@ -31,7 +34,7 @@ use solana_streamer::streamer::PacketReceiver;
 use std::{
     cmp,
     collections::hash_set::HashSet,
-    collections::{BTreeMap, HashMap},
+    collections::{BTreeMap, BTreeSet, HashMap},
     net::UdpSocket,
     ops::{Deref, DerefMut},
     sync::atomic::{AtomicBool, AtomicU64, Ordering},
@@ -236,6 +239,43 @@ fn check_if_already_received(
     }
 }
 
+fn notify_first_shred_received(
+    shred_slot: Slot,
+    rpc_subscriptions: &RpcSubscriptions,
+    sent_received_slot_notification: &Mutex<BTreeSet<Slot>>,
+    root_bank: &Bank,
+) {
+    let notify_slot = {
+        let mut sent_received_slot_notification_locked =
+            sent_received_slot_notification.lock().unwrap();
+        if !sent_received_slot_notification_locked.contains(&shred_slot)
+            && shred_slot > root_bank.slot()
+        {
+            sent_received_slot_notification_locked.insert(shred_slot);
+            if sent_received_slot_notification_locked.len() > 100 {
+                let mut slots_before_root =
+                    sent_received_slot_notification_locked.split_off(&(root_bank.slot() + 1));
+                // `slots_before_root` now contains all slots <= root
+                std::mem::swap(
+                    &mut slots_before_root,
+                    &mut sent_received_slot_notification_locked,
+                );
+            }
+            Some(shred_slot)
+        } else {
+            None
+        }
+    };
+
+    if let Some(slot) = notify_slot {
+        info!("First time receiving a shred from slot: {}", slot);
+        rpc_subscriptions.notify_slot_update(SlotUpdate::FirstShredReceived {
+            slot,
+            timestamp: timestamp(),
+        });
+    }
+}
+
 // Returns true if turbine retransmit peers patch (#14565) is enabled.
 fn enable_turbine_retransmit_peers_patch(shred_slot: Slot, root_bank: &Bank) -> bool {
     let feature_slot = root_bank
@@ -264,6 +304,9 @@ fn retransmit(
     epoch_stakes_cache: &RwLock<EpochStakesCache>,
     last_peer_update: &AtomicU64,
     shreds_received: &Mutex<ShredFilterAndHasher>,
+    max_slots: &MaxSlots,
+    sent_received_slot_notification: &Mutex<BTreeSet<Slot>>,
+    rpc_subscriptions: &Option<Arc<RpcSubscriptions>>,
 ) -> Result<()> {
     let timer = Duration::new(1, 0);
     let r_lock = r.lock().unwrap();
@@ -320,6 +363,7 @@ fn retransmit(
     let mut compute_turbine_peers_total = 0;
     let mut packets_by_slot: HashMap<Slot, usize> = HashMap::new();
     let mut packets_by_source: HashMap<String, usize> = HashMap::new();
+    let mut max_slot = 0;
     for mut packets in packet_v {
         for packet in packets.packets.iter_mut() {
             // skip discarded packets and repair packets
@@ -337,6 +381,17 @@ fn retransmit(
                 Some(slot) => slot,
                 None => continue,
             };
+            max_slot = max_slot.max(shred_slot);
+
+            if let Some(rpc_subscriptions) = rpc_subscriptions {
+                notify_first_shred_received(
+                    shred_slot,
+                    rpc_subscriptions,
+                    sent_received_slot_notification,
+                    &root_bank,
+                );
+            }
+
             let mut compute_turbine_peers = Measure::start("turbine_start");
             let (my_index, mut shuffled_stakes_and_index) = ClusterInfo::shuffle_peers_and_index(
                 &my_id,
@@ -393,6 +448,7 @@ fn retransmit(
             retransmit_total += retransmit_time.as_us();
         }
     }
+    max_slots.retransmit.fetch_max(max_slot, Ordering::Relaxed);
     timer_start.stop();
     debug!(
         "retransmitted {} packets in {}ms retransmit_time: {}ms id: {}",
@@ -433,12 +489,15 @@ pub fn retransmitter(
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
     cluster_info: Arc<ClusterInfo>,
     r: Arc<Mutex<PacketReceiver>>,
+    max_slots: &Arc<MaxSlots>,
+    rpc_subscriptions: Option<Arc<RpcSubscriptions>>,
 ) -> Vec<JoinHandle<()>> {
     let stats = Arc::new(RetransmitStats::default());
     let shreds_received = Arc::new(Mutex::new((
         LruCache::new(DEFAULT_LRU_SIZE),
         PacketHasher::default(),
     )));
+    let sent_received_slot_notification = Arc::new(Mutex::new(BTreeSet::new()));
     (0..sockets.len())
         .map(|s| {
             let sockets = sockets.clone();
@@ -450,6 +509,9 @@ pub fn retransmitter(
             let epoch_stakes_cache = Arc::new(RwLock::new(EpochStakesCache::default()));
             let last_peer_update = Arc::new(AtomicU64::new(0));
             let shreds_received = shreds_received.clone();
+            let max_slots = max_slots.clone();
+            let sent_received_slot_notification = sent_received_slot_notification.clone();
+            let rpc_subscriptions = rpc_subscriptions.clone();
 
             Builder::new()
                 .name("solana-retransmitter".to_string())
@@ -467,6 +529,9 @@ pub fn retransmitter(
                             &epoch_stakes_cache,
                             &last_peer_update,
                             &shreds_received,
+                            &max_slots,
+                            &sent_received_slot_notification,
+                            &rpc_subscriptions,
                         ) {
                             match e {
                                 Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
@@ -511,6 +576,8 @@ impl RetransmitStage {
         verified_vote_receiver: VerifiedVoteReceiver,
         repair_validators: Option<HashSet<Pubkey>>,
         completed_data_sets_sender: CompletedDataSetsSender,
+        max_slots: &Arc<MaxSlots>,
+        rpc_subscriptions: Option<Arc<RpcSubscriptions>>,
     ) -> Self {
         let (retransmit_sender, retransmit_receiver) = channel();
 
@@ -521,6 +588,8 @@ impl RetransmitStage {
             leader_schedule_cache,
             cluster_info.clone(),
             retransmit_receiver,
+            max_slots,
+            rpc_subscriptions,
         );
 
         let leader_schedule_cache_clone = leader_schedule_cache.clone();
@@ -638,6 +707,8 @@ mod tests {
             &leader_schedule_cache,
             cluster_info,
             Arc::new(Mutex::new(retransmit_receiver)),
+            &Arc::new(MaxSlots::default()),
+            None,
         );
         let _thread_hdls = vec![t_retransmit];