Add StakeInstruction::Merge (#10503) (#10506)

automerge

.gitbook.yaml

@@ -3,3 +3,16 @@ root: ./docs/src
 structure:
     readme: introduction.md
     summary: SUMMARY.md
+
+redirects:
+    wallet: ./wallet-guide/README.md
+    wallet/app-wallets: ./wallet-guide/apps.md
+    wallet/app-wallets/trust-wallet: ./wallet-guide/trust-wallet.md
+    wallet/app-wallets/ledger-live:  ./wallet-guide/ledger-live.md
+    wallet/cli-wallets:  ./wallet-guide/cli.md
+    wallet/cli-wallets/paper-wallet:  ./paper-wallet/README.md
+    wallet/cli-wallets/paper-wallet/paper-wallet-usage: ./paper-wallet/paper-wallet-usage.md
+    wallet/cli-wallets/remote-wallet: ./hardware-wallets/README.md
+    wallet/cli-wallets/remote-wallet/ledger: ./hardware-wallets/ledger.md
+    wallet/cli-wallets/file-system-wallet: ./file-system-wallet/README.md
+    wallet/support: ./wallet-guide/support.md

Cargo.toml

@@ -58,6 +58,7 @@ members = [
     "transaction-status",
     "upload-perf",
     "net-utils",
+    "version",
     "vote-signer",
     "cli",
     "rayon-threadlimit",

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.3.0"
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-runtime = { path = "../runtime", version = "1.1.5" }
-solana-measure = { path = "../measure", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
-rand = "0.6.5"
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-runtime = { path = "../runtime", version = "1.1.17" }
+solana-measure = { path = "../measure", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
+rand = "0.7.0"
 clap = "2.33.0"
 crossbeam-channel = "0.4"
 

accounts-bench/src/main.rs

@@ -1,9 +1,11 @@
 use clap::{value_t, App, Arg};
 use rayon::prelude::*;
 use solana_measure::measure::Measure;
-use solana_runtime::accounts::{create_test_accounts, update_accounts, Accounts};
+use solana_runtime::{
+    accounts::{create_test_accounts, update_accounts, Accounts},
+    accounts_index::Ancestors,
+};
 use solana_sdk::pubkey::Pubkey;
-use std::collections::HashMap;
 use std::fs;
 use std::path::PathBuf;
 
@@ -76,7 +78,7 @@ fn main() {
         num_slots,
         create_time
     );
-    let mut ancestors: HashMap<u64, usize> = vec![(0, 0)].into_iter().collect();
+    let mut ancestors: Ancestors = vec![(0, 0)].into_iter().collect();
     for i in 1..num_slots {
         ancestors.insert(i as u64, i - 1);
         accounts.add_root(i as u64);

archiver-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-lib"
-version = "1.1.5"
+version = "1.1.17"
 description = "Solana Archiver Library"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,27 +11,27 @@ edition = "2018"
 [dependencies]
 bincode = "1.2.1"
 crossbeam-channel = "0.4"
-ed25519-dalek = "=1.0.0-pre.1"
+ed25519-dalek = "=1.0.0-pre.3"
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-client = { path = "../client", version = "1.1.5" }
-solana-storage-program = { path = "../programs/storage", version = "1.1.5" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-client = { path = "../client", version = "1.1.17" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.17" }
 thiserror = "1.0"
 serde = "1.0.105"
 serde_json = "1.0.48"
 serde_derive = "1.0.103"
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
-solana-chacha = { path = "../chacha", version = "1.1.5" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.1.5" }
-solana-ledger = { path = "../ledger", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-perf = { path = "../perf", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
-solana-core = { path = "../core", version = "1.1.5" }
-solana-streamer = { path = "../streamer", version = "1.1.5" }
-solana-archiver-utils = { path = "../archiver-utils", version = "1.1.5" }
-solana-metrics = { path = "../metrics", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
+solana-chacha = { path = "../chacha", version = "1.1.17" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.17" }
+solana-ledger = { path = "../ledger", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-perf = { path = "../perf", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
+solana-core = { path = "../core", version = "1.1.17" }
+solana-streamer = { path = "../streamer", version = "1.1.17" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.1.17" }
+solana-metrics = { path = "../metrics", version = "1.1.17" }
 
 [dev-dependencies]
 hex = "0.4.2"

archiver-lib/src/archiver.rs

@@ -1,7 +1,7 @@
 use crate::result::ArchiverError;
 use crossbeam_channel::unbounded;
-use rand::{thread_rng, Rng, SeedableRng};
-use rand_chacha::ChaChaRng;
+use rand::{thread_rng, Rng};
+use rand_chacha::{rand_core::SeedableRng, ChaChaRng};
 use solana_archiver_utils::sample_file;
 use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
 use solana_client::{
@@ -13,8 +13,7 @@ use solana_core::{
     cluster_slots::ClusterSlots,
     contact_info::ContactInfo,
     gossip_service::GossipService,
-    repair_service,
-    repair_service::{RepairService, RepairSlotRange, RepairStats, RepairStrategy},
+    repair_service::{self, RepairService, RepairSlotRange, RepairStats, RepairStrategy},
     serve_repair::ServeRepair,
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
@@ -53,7 +52,7 @@ use std::{
     result,
     sync::atomic::{AtomicBool, Ordering},
     sync::mpsc::{channel, Receiver, Sender},
-    sync::{Arc, RwLock},
+    sync::Arc,
     thread::{sleep, spawn, JoinHandle},
     time::Duration,
 };
@@ -185,9 +184,9 @@ impl Archiver {
 
         info!("Archiver: id: {}", keypair.pubkey());
         info!("Creating cluster info....");
-        let mut cluster_info = ClusterInfo::new(node.info.clone(), keypair.clone());
+        let cluster_info = ClusterInfo::new(node.info.clone(), keypair.clone());
         cluster_info.set_entrypoint(cluster_entrypoint.clone());
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
         let cluster_slots = Arc::new(ClusterSlots::default());
         // Note for now, this ledger will not contain any of the existing entries
         // in the ledger located at ledger_path, and will only append on newly received
@@ -200,7 +199,7 @@ impl Archiver {
 
         info!("Connecting to the cluster via {:?}", cluster_entrypoint);
         let (nodes, _) =
-            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
+            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 2) {
                 Ok(nodes_and_archivers) => nodes_and_archivers,
                 Err(e) => {
                     //shutdown services before exiting
@@ -308,7 +307,7 @@ impl Archiver {
     fn run(
         meta: &mut ArchiverMeta,
         blockstore: &Arc<Blockstore>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         archiver_keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
         exit: &Arc<AtomicBool>,
@@ -365,12 +364,12 @@ impl Archiver {
     }
 
     fn redeem_rewards(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         archiver_keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
         client_commitment: CommitmentConfig,
     ) {
-        let nodes = cluster_info.read().unwrap().tvu_peers();
+        let nodes = cluster_info.tvu_peers();
         let client = solana_core::gossip_service::get_client(&nodes);
 
         if let Ok(Some(account)) =
@@ -405,7 +404,7 @@ impl Archiver {
     #[allow(clippy::too_many_arguments)]
     fn setup(
         meta: &mut ArchiverMeta,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         blockstore: &Arc<Blockstore>,
         exit: &Arc<AtomicBool>,
         node_info: &ContactInfo,
@@ -491,7 +490,7 @@ impl Archiver {
         blockstore: &Arc<Blockstore>,
         exit: &Arc<AtomicBool>,
         node_info: &ContactInfo,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
     ) {
         info!(
             "window created, waiting for ledger download starting at slot {:?}",
@@ -519,11 +518,8 @@ impl Archiver {
         contact_info.tvu = "0.0.0.0:0".parse().unwrap();
         contact_info.wallclock = timestamp();
         // copy over the adopted shred_version from the entrypoint
-        contact_info.shred_version = cluster_info.read().unwrap().my_data().shred_version;
-        {
-            let mut cluster_info_w = cluster_info.write().unwrap();
-            cluster_info_w.insert_self(contact_info);
-        }
+        contact_info.shred_version = cluster_info.my_shred_version();
+        cluster_info.update_contact_info(|current| *current = contact_info);
     }
 
     fn encrypt_ledger(meta: &mut ArchiverMeta, blockstore: &Arc<Blockstore>) -> Result<()> {
@@ -626,12 +622,12 @@ impl Archiver {
 
     fn submit_mining_proof(
         meta: &ArchiverMeta,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         archiver_keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
     ) {
         // No point if we've got no storage account...
-        let nodes = cluster_info.read().unwrap().tvu_peers();
+        let nodes = cluster_info.tvu_peers();
         let client = solana_core::gossip_service::get_client(&nodes);
         let storage_balance = client
             .poll_get_balance_with_commitment(&storage_keypair.pubkey(), meta.client_commitment);
@@ -689,13 +685,10 @@ impl Archiver {
     }
 
     fn get_segment_config(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         client_commitment: CommitmentConfig,
     ) -> Result<u64> {
-        let rpc_peers = {
-            let cluster_info = cluster_info.read().unwrap();
-            cluster_info.all_rpc_peers()
-        };
+        let rpc_peers = cluster_info.all_rpc_peers();
         debug!("rpc peers: {:?}", rpc_peers);
         if !rpc_peers.is_empty() {
             let rpc_client = {
@@ -703,16 +696,10 @@ impl Archiver {
                 RpcClient::new_socket(rpc_peers[node_index].rpc)
             };
             Ok(rpc_client
-                .send(
-                    &RpcRequest::GetSlotsPerSegment,
+                .send::<u64>(
+                    RpcRequest::GetSlotsPerSegment,
                     serde_json::json!([client_commitment]),
-                    0,
                 )
-                .map_err(|err| {
-                    warn!("Error while making rpc request {:?}", err);
-                    ArchiverError::ClientError(err)
-                })?
-                .as_u64()
                 .unwrap())
         } else {
             Err(ArchiverError::NoRpcPeers)
@@ -721,7 +708,7 @@ impl Archiver {
 
     /// Waits until the first segment is ready, and returns the current segment
     fn poll_for_segment(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         slots_per_segment: u64,
         previous_blockhash: &Hash,
         exit: &Arc<AtomicBool>,
@@ -741,38 +728,24 @@ impl Archiver {
 
     /// Poll for a different blockhash and associated max_slot than `previous_blockhash`
     fn poll_for_blockhash_and_slot(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         slots_per_segment: u64,
         previous_blockhash: &Hash,
         exit: &Arc<AtomicBool>,
     ) -> Result<(Hash, u64)> {
         info!("waiting for the next turn...");
         loop {
-            let rpc_peers = {
-                let cluster_info = cluster_info.read().unwrap();
-                cluster_info.all_rpc_peers()
-            };
+            let rpc_peers = cluster_info.all_rpc_peers();
             debug!("rpc peers: {:?}", rpc_peers);
             if !rpc_peers.is_empty() {
                 let rpc_client = {
                     let node_index = thread_rng().gen_range(0, rpc_peers.len());
                     RpcClient::new_socket(rpc_peers[node_index].rpc)
                 };
-                let response = rpc_client
-                    .send(
-                        &RpcRequest::GetStorageTurn,
-                        serde_json::value::Value::Null,
-                        0,
-                    )
-                    .map_err(|err| {
-                        warn!("Error while making rpc request {:?}", err);
-                        ArchiverError::ClientError(err)
-                    })?;
                 let RpcStorageTurn {
                     blockhash: storage_blockhash,
                     slot: turn_slot,
-                } = serde_json::from_value::<RpcStorageTurn>(response)
-                    .map_err(ArchiverError::JsonError)?;
+                } = rpc_client.send(RpcRequest::GetStorageTurn, serde_json::value::Value::Null)?;
                 let turn_blockhash = storage_blockhash.parse().map_err(|err| {
                     io::Error::new(
                         io::ErrorKind::Other,
@@ -851,7 +824,7 @@ impl Archiver {
                     .into_iter()
                     .filter_map(|repair_request| {
                         serve_repair
-                            .map_repair_request(&repair_request, &mut repair_stats)
+                            .map_repair_request(&repair_request, &mut repair_stats, Some(0))
                             .map(|result| ((archiver_info.gossip, result), repair_request))
                             .ok()
                     })

archiver-lib/src/result.rs

@@ -1,4 +1,3 @@
-use serde_json;
 use solana_client::client_error;
 use solana_ledger::blockstore;
 use solana_sdk::transport;

archiver-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-utils"
-version = "1.1.5"
+version = "1.1.17"
 description = "Solana Archiver Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,13 +10,13 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-solana-chacha = { path = "../chacha", version = "1.1.5" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.1.5" }
-solana-ledger = { path = "../ledger", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-perf = { path = "../perf", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
+rand = "0.7.0"
+solana-chacha = { path = "../chacha", version = "1.1.17" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.17" }
+solana-ledger = { path = "../ledger", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-perf = { path = "../perf", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
 
 [dev-dependencies]
 hex = "0.4.2"

archiver/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-archiver"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,13 +10,13 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.33.0"
 console = "0.10.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
-solana-core = { path = "../core", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-metrics = { path = "../metrics", version = "1.1.5" }
-solana-archiver-lib = { path = "../archiver-lib", version = "1.1.5" }
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.17" }
+solana-core = { path = "../core", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-metrics = { path = "../metrics", version = "1.1.17" }
+solana-archiver-lib = { path = "../archiver-lib", version = "1.1.17" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
 
 
 [package.metadata.docs.rs]

banking-bench/Cargo.toml

@@ -2,24 +2,26 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
-log = "0.4.6"
-rayon = "1.3.0"
-solana-core = { path = "../core", version = "1.1.5" }
-solana-streamer = { path = "../streamer", version = "1.1.5" }
-solana-perf = { path = "../perf", version = "1.1.5" }
-solana-ledger = { path = "../ledger", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-runtime = { path = "../runtime", version = "1.1.5" }
-solana-measure = { path = "../measure", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
-rand = "0.6.5"
+clap = "2.33.1"
 crossbeam-channel = "0.4"
+log = "0.4.6"
+rand = "0.7.0"
+rayon = "1.3.0"
+solana-core = { path = "../core", version = "1.1.17" }
+solana-streamer = { path = "../streamer", version = "1.1.17" }
+solana-perf = { path = "../perf", version = "1.1.17" }
+solana-ledger = { path = "../ledger", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-runtime = { path = "../runtime", version = "1.1.17" }
+solana-measure = { path = "../measure", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
+solana-version = { path = "../version", version = "1.1.17" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,3 +1,4 @@
+use clap::{crate_description, crate_name, value_t, App, Arg};
 use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
@@ -28,7 +29,7 @@ use solana_sdk::{
     transaction::Transaction,
 };
 use std::{
-    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
+    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex},
     thread::sleep,
     time::{Duration, Instant},
 };
@@ -64,15 +65,22 @@ fn check_txs(
     no_bank
 }
 
-fn make_accounts_txs(txes: usize, mint_keypair: &Keypair, hash: Hash) -> Vec<Transaction> {
+fn make_accounts_txs(
+    total_num_transactions: usize,
+    hash: Hash,
+    same_payer: bool,
+) -> Vec<Transaction> {
     let to_pubkey = Pubkey::new_rand();
-    let dummy = system_transaction::transfer(mint_keypair, &to_pubkey, 1, hash);
-    (0..txes)
+    let payer_key = Keypair::new();
+    let dummy = system_transaction::transfer(&payer_key, &to_pubkey, 1, hash);
+    (0..total_num_transactions)
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
             let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
-            new.message.account_keys[0] = Pubkey::new_rand();
+            if !same_payer {
+                new.message.account_keys[0] = Pubkey::new_rand();
+            }
             new.message.account_keys[1] = Pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
@@ -96,13 +104,61 @@ fn bytes_as_usize(bytes: &[u8]) -> usize {
     bytes[0] as usize | (bytes[1] as usize) << 8
 }
 
+#[allow(clippy::cognitive_complexity)]
 fn main() {
     solana_logger::setup();
-    let num_threads = BankingStage::num_threads() as usize;
+
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(solana_version::version!())
+        .arg(
+            Arg::with_name("num_chunks")
+                .long("num-chunks")
+                .takes_value(true)
+                .value_name("SIZE")
+                .help("Number of transaction chunks."),
+        )
+        .arg(
+            Arg::with_name("packets_per_chunk")
+                .long("packets-per-chunk")
+                .takes_value(true)
+                .value_name("SIZE")
+                .help("Packets per chunk"),
+        )
+        .arg(
+            Arg::with_name("skip_sanity")
+                .long("skip-sanity")
+                .takes_value(false)
+                .help("Skip transaction sanity execution"),
+        )
+        .arg(
+            Arg::with_name("same_payer")
+                .long("same-payer")
+                .takes_value(false)
+                .help("Use the same payer for transfers"),
+        )
+        .arg(
+            Arg::with_name("iterations")
+                .long("iterations")
+                .takes_value(true)
+                .help("Number of iterations"),
+        )
+        .arg(
+            Arg::with_name("num_threads")
+                .long("num-threads")
+                .takes_value(true)
+                .help("Number of iterations"),
+        )
+        .get_matches();
+
+    let num_threads =
+        value_t!(matches, "num_threads", usize).unwrap_or(BankingStage::num_threads() as usize);
     //   a multiple of packet chunk duplicates to avoid races
-    const CHUNKS: usize = 8 * 2;
-    const PACKETS_PER_BATCH: usize = 192;
-    let txes = PACKETS_PER_BATCH * num_threads * CHUNKS;
+    let num_chunks = value_t!(matches, "num_chunks", usize).unwrap_or(16);
+    let packets_per_chunk = value_t!(matches, "packets_per_chunk", usize).unwrap_or(192);
+    let iterations = value_t!(matches, "iterations", usize).unwrap_or(1000);
+
+    let total_num_transactions = num_chunks * num_threads * packets_per_chunk;
     let mint_total = 1_000_000_000_000;
     let GenesisConfigInfo {
         genesis_config,
@@ -116,34 +172,44 @@ fn main() {
     let mut bank_forks = BankForks::new(0, bank0);
     let mut bank = bank_forks.working_bank();
 
-    info!("threads: {} txs: {}", num_threads, txes);
+    info!("threads: {} txs: {}", num_threads, total_num_transactions);
 
-    let mut transactions = make_accounts_txs(txes, &mint_keypair, genesis_config.hash());
+    let same_payer = matches.is_present("same_payer");
+    let mut transactions =
+        make_accounts_txs(total_num_transactions, genesis_config.hash(), same_payer);
 
     // fund all the accounts
     transactions.iter().for_each(|tx| {
-        let fund = system_transaction::transfer(
+        let mut fund = system_transaction::transfer(
             &mint_keypair,
             &tx.message.account_keys[0],
-            mint_total / txes as u64,
+            mint_total / total_num_transactions as u64,
             genesis_config.hash(),
         );
+        // Ignore any pesky duplicate signature errors in the case we are using single-payer
+        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+        fund.signatures = vec![Signature::new(&sig[0..64])];
         let x = bank.process_transaction(&fund);
         x.unwrap();
     });
-    //sanity check, make sure all the transactions can execute sequentially
-    transactions.iter().for_each(|tx| {
-        let res = bank.process_transaction(&tx);
-        assert!(res.is_ok(), "sanity test transactions");
-    });
-    bank.clear_signatures();
-    //sanity check, make sure all the transactions can execute in parallel
-    let res = bank.process_transactions(&transactions);
-    for r in res {
-        assert!(r.is_ok(), "sanity parallel execution");
+
+    let skip_sanity = matches.is_present("skip_sanity");
+    if !skip_sanity {
+        //sanity check, make sure all the transactions can execute sequentially
+        transactions.iter().for_each(|tx| {
+            let res = bank.process_transaction(&tx);
+            assert!(res.is_ok(), "sanity test transactions error: {:?}", res);
+        });
+        bank.clear_signatures();
+        //sanity check, make sure all the transactions can execute in parallel
+        let res = bank.process_transactions(&transactions);
+        for r in res {
+            assert!(r.is_ok(), "sanity parallel execution error: {:?}", r);
+        }
+        bank.clear_signatures();
     }
-    bank.clear_signatures();
-    let mut verified: Vec<_> = to_packets_chunked(&transactions.clone(), PACKETS_PER_BATCH);
+
+    let mut verified: Vec<_> = to_packets_chunked(&transactions.clone(), packets_per_chunk);
     let ledger_path = get_tmp_ledger_path!();
     {
         let blockstore = Arc::new(
@@ -152,7 +218,7 @@ fn main() {
         let (exit, poh_recorder, poh_service, signal_receiver) =
             create_test_recorder(&bank, &blockstore, None);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
         let banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,
@@ -162,7 +228,7 @@ fn main() {
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
-        let chunk_len = verified.len() / CHUNKS;
+        let chunk_len = verified.len() / num_chunks;
         let mut start = 0;
 
         // This is so that the signal_receiver does not go out of scope after the closure.
@@ -171,17 +237,17 @@ fn main() {
         let signal_receiver = Arc::new(signal_receiver);
         let mut total_us = 0;
         let mut tx_total_us = 0;
+        let base_tx_count = bank.transaction_count();
         let mut txs_processed = 0;
         let mut root = 1;
         let collector = Pubkey::new_rand();
-        const ITERS: usize = 1_000;
         let config = Config {
-            packets_per_batch: PACKETS_PER_BATCH,
+            packets_per_batch: packets_per_chunk,
             chunk_len,
             num_threads,
         };
         let mut total_sent = 0;
-        for _ in 0..ITERS {
+        for _ in 0..iterations {
             let now = Instant::now();
             let mut sent = 0;
 
@@ -222,7 +288,11 @@ fn main() {
                     sleep(Duration::from_millis(5));
                 }
             }
-            if check_txs(&signal_receiver, txes / CHUNKS, &poh_recorder) {
+            if check_txs(
+                &signal_receiver,
+                total_num_transactions / num_chunks,
+                &poh_recorder,
+            ) {
                 debug!(
                     "resetting bank {} tx count: {} txs_proc: {}",
                     bank.slot(),
@@ -253,7 +323,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None);
+                    bank_forks.set_root(root, &None, None);
                     root += 1;
                 }
                 debug!(
@@ -274,7 +344,7 @@ fn main() {
             debug!(
                 "time: {} us checked: {} sent: {}",
                 duration_as_us(&now.elapsed()),
-                txes / CHUNKS,
+                total_num_transactions / num_chunks,
                 sent,
             );
             total_sent += sent;
@@ -285,20 +355,26 @@ fn main() {
                     let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
                     tx.signatures[0] = Signature::new(&sig[0..64]);
                 }
-                verified = to_packets_chunked(&transactions.clone(), PACKETS_PER_BATCH);
+                verified = to_packets_chunked(&transactions.clone(), packets_per_chunk);
             }
 
             start += chunk_len;
             start %= verified.len();
         }
+        let txs_processed = bank_forks.working_bank().transaction_count();
+        debug!("processed: {} base: {}", txs_processed, base_tx_count);
         eprintln!(
-            "{{'name': 'banking_bench_total', 'median': '{}'}}",
+            "{{'name': 'banking_bench_total', 'median': '{:.2}'}}",
             (1000.0 * 1000.0 * total_sent as f64) / (total_us as f64),
         );
         eprintln!(
-            "{{'name': 'banking_bench_tx_total', 'median': '{}'}}",
+            "{{'name': 'banking_bench_tx_total', 'median': '{:.2}'}}",
             (1000.0 * 1000.0 * total_sent as f64) / (tx_total_us as f64),
         );
+        eprintln!(
+            "{{'name': 'banking_bench_success_tx_total', 'median': '{:.2}'}}",
+            (1000.0 * 1000.0 * (txs_processed - base_tx_count) as f64) / (total_us as f64),
+        );
 
         drop(verified_sender);
         drop(vote_sender);

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,24 +14,24 @@ itertools = "0.9.0"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
-rand = "0.6.5"
+rand = "0.7.0"
 rayon = "1.3.0"
 serde_json = "1.0.48"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
-solana-core = { path = "../core", version = "1.1.5" }
-solana-genesis = { path = "../genesis", version = "1.1.5" }
-solana-client = { path = "../client", version = "1.1.5" }
-solana-faucet = { path = "../faucet", version = "1.1.5" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-metrics = { path = "../metrics", version = "1.1.5" }
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
-solana-runtime = { path = "../runtime", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.17" }
+solana-core = { path = "../core", version = "1.1.17" }
+solana-genesis = { path = "../genesis", version = "1.1.17" }
+solana-client = { path = "../client", version = "1.1.17" }
+solana-faucet = { path = "../faucet", version = "1.1.17" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-metrics = { path = "../metrics", version = "1.1.17" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
+solana-runtime = { path = "../runtime", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.1.5" }
+solana-local-cluster = { path = "../local-cluster", version = "1.1.17" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/tests/bench_exchange.rs

@@ -86,7 +86,7 @@ fn test_exchange_bank_client() {
     solana_logger::setup();
     let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
     let mut bank = Bank::new(&genesis_config);
-    bank.add_instruction_processor(id(), process_instruction);
+    bank.add_static_program("exchange_program", id(), process_instruction);
     let clients = vec![BankClient::new(bank)];
 
     let mut config = Config::default();

bench-streamer/Cargo.toml

@@ -2,17 +2,17 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
-solana-streamer = { path = "../streamer", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.17" }
+solana-streamer = { path = "../streamer", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,27 +14,27 @@ log = "0.4.8"
 rayon = "1.3.0"
 serde_json = "1.0.48"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
-solana-core = { path = "../core", version = "1.1.5" }
-solana-genesis = { path = "../genesis", version = "1.1.5" }
-solana-client = { path = "../client", version = "1.1.5" }
-solana-faucet = { path = "../faucet", version = "1.1.5" }
-solana-librapay = { path = "../programs/librapay", version = "1.1.5", optional = true }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-metrics = { path = "../metrics", version = "1.1.5" }
-solana-measure = { path = "../measure", version = "1.1.5" }
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
-solana-runtime = { path = "../runtime", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.1.5", optional = true }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.17" }
+solana-core = { path = "../core", version = "1.1.17" }
+solana-genesis = { path = "../genesis", version = "1.1.17" }
+solana-client = { path = "../client", version = "1.1.17" }
+solana-faucet = { path = "../faucet", version = "1.1.17" }
+#solana-librapay = { path = "../programs/librapay", version = "1.1.8", optional = true }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-metrics = { path = "../metrics", version = "1.1.17" }
+solana-measure = { path = "../measure", version = "1.1.17" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
+solana-runtime = { path = "../runtime", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
+#solana-move-loader-program = { path = "../programs/move_loader", version = "1.1.8", optional = true }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.1.5" }
+solana-local-cluster = { path = "../local-cluster", version = "1.1.17" }
 
-[features]
-move = ["solana-librapay", "solana-move-loader-program"]
+#[features]
+#move = ["solana-librapay", "solana-move-loader-program"]
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

chacha-cuda/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-cuda"
-version = "1.1.5"
+version = "1.1.17"
 description = "Solana Chacha Cuda APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,12 +10,12 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-archiver-utils = { path = "../archiver-utils", version = "1.1.5" }
-solana-chacha = { path = "../chacha", version = "1.1.5" }
-solana-ledger = { path = "../ledger", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-perf = { path = "../perf", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.1.17" }
+solana-chacha = { path = "../chacha", version = "1.1.17" }
+solana-ledger = { path = "../ledger", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-perf = { path = "../perf", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"

chacha-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-sys"
-version = "1.1.5"
+version = "1.1.17"
 description = "Solana chacha-sys"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

chacha/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha"
-version = "1.1.5"
+version = "1.1.17"
 description = "Solana Chacha APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,13 +10,13 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-chacha-sys = { path = "../chacha-sys", version = "1.1.5" }
-solana-ledger = { path = "../ledger", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-perf = { path = "../perf", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.17" }
+solana-ledger = { path = "../ledger", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-perf = { path = "../perf", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"

ci/buildkite-secondary.yml

@@ -2,6 +2,16 @@
 # Build steps that run after the primary pipeline on pushes and tags.
 # Pull requests to not run these steps.
 steps:
+  - command: "ci/publish-tarball.sh"
+    timeout_in_minutes: 60
+    name: "publish tarball"
+  - command: "ci/publish-docs.sh"
+    timeout_in_minutes: 15
+    name: "publish docs"
+  - command: "ci/publish-bpf-sdk.sh"
+    timeout_in_minutes: 5
+    name: "publish bpf sdk"
+  - wait
   - command: "sdk/docker-solana/build.sh"
     timeout_in_minutes: 60
     name: "publish docker"
@@ -9,15 +19,6 @@ steps:
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"
-  - command: "ci/publish-bpf-sdk.sh"
-    timeout_in_minutes: 5
-    name: "publish bpf sdk"
-  - command: "ci/publish-tarball.sh"
-    timeout_in_minutes: 60
-    name: "publish tarball"
-  - command: "ci/publish-docs.sh"
-    timeout_in_minutes: 15
-    name: "publish docs"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
-    name: "move"
-    timeout_in_minutes: 20
+    #  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
+    #    name: "move"
+    #    timeout_in_minutes: 20

ci/docker-run.sh

@@ -49,7 +49,7 @@ else
   # ~/.cargo
   ARGS+=(--volume "$PWD:/home")
 fi
-ARGS+=(--env "CARGO_HOME=/home/.cargo")
+ARGS+=(--env "HOME=/home" --env "CARGO_HOME=/home/.cargo")
 
 # kcov tries to set the personality of the binary which docker
 # doesn't allow by default.

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.42.0
+FROM solanalabs/rust:1.43.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.42.0
+FROM rust:1.43.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/publish-tarball.sh

@@ -71,7 +71,7 @@ echo --- Creating release tarball
   export CHANNEL
 
   source ci/rust-version.sh stable
-  scripts/cargo-install-all.sh +"$rust_stable" --use-move solana-release
+  scripts/cargo-install-all.sh +"$rust_stable" solana-release
 
   tar cvf solana-release-$TARGET.tar solana-release
   bzip2 solana-release-$TARGET.tar

ci/run-sanity.sh

@@ -2,8 +2,10 @@
 set -e
 
 cd "$(dirname "$0")/.."
+# shellcheck source=multinode-demo/common.sh
+source multinode-demo/common.sh
 
-rm -f config/run/init-completed
+rm -rf config/run/init-completed config/ledger config/snapshot-ledger
 
 timeout 15 ./run.sh &
 pid=$!
@@ -17,6 +19,13 @@ while [[ ! -f config/run/init-completed ]]; do
   fi
 done
 
+while [[ $($solana_cli --url http://localhost:8899 slot --commitment recent) -eq 0 ]]; do
+  sleep 1
+done
 curl -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1, "method":"validatorExit"}' http://localhost:8899
 
 wait $pid
+
+$solana_ledger_tool create-snapshot --ledger config/ledger 1 config/snapshot-ledger
+cp config/ledger/genesis.tar.bz2 config/snapshot-ledger
+$solana_ledger_tool verify --ledger config/snapshot-ledger

ci/rust-version.sh

@@ -1,28 +1,30 @@
 #
 # This file maintains the rust versions for use by CI.
 #
-# Build with stable rust, updating the stable toolchain if necessary:
-#   $ source ci/rust-version.sh stable
-#   $ cargo +"$rust_stable" build
-#
-# Build with nightly rust, updating the nightly toolchain if necessary:
-#   $ source ci/rust-version.sh nightly
-#   $ cargo +"$rust_nightly" build
-#
 # Obtain the environment variables without any automatic toolchain updating:
 #   $ source ci/rust-version.sh
 #
+# Obtain the environment variables updating both stable and nightly, only stable, or
+# only nightly:
+#   $ source ci/rust-version.sh all
+#   $ source ci/rust-version.sh stable
+#   $ source ci/rust-version.sh nightly
+
+# Then to build with either stable or nightly:
+#   $ cargo +"$rust_stable" build
+#   $ cargo +"$rust_nightly" build
+#
 
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.42.0
+  stable_version=1.43.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-03-12
+  nightly_version=2020-04-23
 fi
 
 
@@ -51,6 +53,10 @@ export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version"
   nightly)
      rustup_install "$rust_nightly"
     ;;
+  all)
+     rustup_install "$rust_stable"
+     rustup_install "$rust_nightly"
+    ;;
   *)
     echo "Note: ignoring unknown argument: $1"
     ;;

ci/test-bench.sh

@@ -25,7 +25,7 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-source ci/rust-version.sh nightly
+source ci/rust-version.sh all
 
 set -o pipefail
 export RUST_BACKTRACE=1

ci/test-stable.sh

@@ -39,15 +39,15 @@ test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 rm -rf target/xargo # Issue #3105
 
 # Limit compiler jobs to reduce memory usage
-# on machines with 1gb/thread of memory
+# on machines with 2gb/thread of memory
 NPROC=$(nproc)
-NPROC=$((NPROC>16 ? 16 : NPROC))
+NPROC=$((NPROC>14 ? 14 : NPROC))
 
 echo "Executing $testName"
 case $testName in
 test-stable)
   _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
-  _ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
+  #_ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
   ;;
 test-stable-perf)
   ci/affects-files.sh \

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.1.5"
+version = "1.1.17"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/commitment.rs

@@ -0,0 +1,22 @@
+use crate::ArgConstant;
+use clap::Arg;
+
+pub const COMMITMENT_ARG: ArgConstant<'static> = ArgConstant {
+    name: "commitment",
+    long: "commitment",
+    help: "Return information at the selected commitment level",
+};
+
+pub fn commitment_arg<'a, 'b>() -> Arg<'a, 'b> {
+    commitment_arg_with_default("recent")
+}
+
+pub fn commitment_arg_with_default<'a, 'b>(default_value: &'static str) -> Arg<'a, 'b> {
+    Arg::with_name(COMMITMENT_ARG.name)
+        .long(COMMITMENT_ARG.long)
+        .takes_value(true)
+        .possible_values(&["recent", "root", "max"])
+        .default_value(default_value)
+        .value_name("COMMITMENT_LEVEL")
+        .help(COMMITMENT_ARG.help)
+}

clap-utils/src/input_parsers.rs

@@ -7,6 +7,7 @@ use clap::ArgMatches;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     clock::UnixTimestamp,
+    commitment_config::CommitmentConfig,
     native_token::sol_to_lamports,
     pubkey::Pubkey,
     signature::{read_keypair_file, Keypair, Signature, Signer},
@@ -177,6 +178,15 @@ pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
     value_of(matches, name).map(sol_to_lamports)
 }
 
+pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentConfig> {
+    matches.value_of(name).map(|value| match value {
+        "max" => CommitmentConfig::max(),
+        "recent" => CommitmentConfig::recent(),
+        "root" => CommitmentConfig::root(),
+        _ => CommitmentConfig::default(),
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

clap-utils/src/lib.rs

@@ -42,6 +42,7 @@ impl std::fmt::Debug for DisplayError {
     }
 }
 
+pub mod commitment;
 pub mod input_parsers;
 pub mod input_validators;
 pub mod keypair;

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.1.5"
+version = "1.1.17"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,28 +27,28 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.105"
 serde_derive = "1.0.103"
 serde_json = "1.0.48"
-solana-budget-program = { path = "../programs/budget", version = "1.1.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
-solana-cli-config = { path = "../cli-config", version = "1.1.5" }
-solana-client = { path = "../client", version = "1.1.5" }
-solana-config-program = { path = "../programs/config", version = "1.1.5" }
-solana-faucet = { path = "../faucet", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.1.5" }
-solana-runtime = { path = "../runtime", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.1.5" }
-solana-storage-program = { path = "../programs/storage", version = "1.1.5" }
-solana-transaction-status = { path = "../transaction-status", version = "1.1.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.1.5" }
-solana-vote-signer = { path = "../vote-signer", version = "1.1.5" }
+solana-budget-program = { path = "../programs/budget", version = "1.1.17" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.17" }
+solana-cli-config = { path = "../cli-config", version = "1.1.17" }
+solana-client = { path = "../client", version = "1.1.17" }
+solana-config-program = { path = "../programs/config", version = "1.1.17" }
+solana-faucet = { path = "../faucet", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.1.17" }
+solana-runtime = { path = "../runtime", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
+solana-stake-program = { path = "../programs/stake", version = "1.1.17" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.17" }
+solana-transaction-status = { path = "../transaction-status", version = "1.1.17" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.17" }
+solana-vote-signer = { path = "../vote-signer", version = "1.1.17" }
 thiserror = "1.0.13"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.1.5" }
-solana-budget-program = { path = "../programs/budget", version = "1.1.5" }
+solana-core = { path = "../core", version = "1.1.17" }
+solana-budget-program = { path = "../programs/budget", version = "1.1.17" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -1,7 +1,7 @@
 use crate::{
-    cli_output::{CliAccount, OutputFormat},
+    cli_output::{CliAccount, CliSignOnlyData, CliSignature, OutputFormat},
     cluster_query::*,
-    display::{println_name_value, println_signers},
+    display::{new_spinner_progress_bar, println_name_value, println_transaction},
     nonce::{self, *},
     offline::{blockhash_query::BlockhashQuery, *},
     stake::*,
@@ -16,12 +16,17 @@ use num_traits::FromPrimitive;
 use serde_json::{self, json, Value};
 use solana_budget_program::budget_instruction::{self, BudgetError};
 use solana_clap_utils::{
-    input_parsers::*, input_validators::*, keypair::signer_from_path, offline::SIGN_ONLY_ARG,
+    commitment::{commitment_arg_with_default, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+    keypair::signer_from_path,
+    offline::SIGN_ONLY_ARG,
     ArgConstant,
 };
 use solana_client::{
     client_error::{ClientErrorKind, Result as ClientResult},
     rpc_client::RpcClient,
+    rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig},
     rpc_response::{RpcAccount, RpcKeyedAccount},
 };
 #[cfg(not(test))]
@@ -31,7 +36,7 @@ use solana_faucet::faucet_mock::request_airdrop_transaction;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     bpf_loader,
-    clock::{Epoch, Slot},
+    clock::{Epoch, Slot, DEFAULT_TICKS_PER_SECOND},
     commitment_config::CommitmentConfig,
     fee_calculator::FeeCalculator,
     hash::Hash,
@@ -42,6 +47,7 @@ use solana_sdk::{
     program_utils::DecodeError,
     pubkey::{Pubkey, MAX_SEED_LEN},
     signature::{Keypair, Signature, Signer, SignerError},
+    signers::Signers,
     system_instruction::{self, SystemError},
     system_program,
     transaction::{Transaction, TransactionError},
@@ -51,9 +57,11 @@ use solana_stake_program::{
     stake_state::{Lockup, StakeAuthorize},
 };
 use solana_storage_program::storage_instruction::StorageAccountType;
+use solana_transaction_status::{EncodedTransaction, TransactionEncoding};
 use solana_vote_program::vote_state::VoteAuthorize;
 use std::{
     error,
+    fmt::Write as FmtWrite,
     fs::File,
     io::{Read, Write},
     net::{IpAddr, SocketAddr},
@@ -178,6 +186,7 @@ pub enum CliCommand {
         commitment_config: CommitmentConfig,
         follow: bool,
     },
+    ClusterDate,
     ClusterVersion,
     CreateAddressWithSeed {
         from_pubkey: Option<Pubkey>,
@@ -186,7 +195,7 @@ pub enum CliCommand {
     },
     Fees,
     GetBlockTime {
-        slot: Slot,
+        slot: Option<Slot>,
     },
     GetEpochInfo {
         commitment_config: CommitmentConfig,
@@ -198,6 +207,14 @@ pub enum CliCommand {
     GetSlot {
         commitment_config: CommitmentConfig,
     },
+    LargestAccounts {
+        commitment_config: CommitmentConfig,
+        filter: Option<RpcLargestAccountsFilter>,
+    },
+    Supply {
+        commitment_config: CommitmentConfig,
+        print_accounts: bool,
+    },
     TotalSupply {
         commitment_config: CommitmentConfig,
     },
@@ -226,6 +243,11 @@ pub enum CliCommand {
         use_lamports_unit: bool,
         commitment_config: CommitmentConfig,
     },
+    TransactionHistory {
+        address: Pubkey,
+        end_slot: Option<Slot>,  // None == latest slot
+        slot_limit: Option<u64>, // None == search full history
+    },
     // Nonce commands
     AuthorizeNonceAccount {
         nonce_account: Pubkey,
@@ -333,6 +355,7 @@ pub enum CliCommand {
         destination_account_pubkey: Pubkey,
         lamports: u64,
         withdraw_authority: SignerIndex,
+        custodian: Option<SignerIndex>,
         sign_only: bool,
         blockhash_query: BlockhashQuery,
         nonce_account: Option<Pubkey>,
@@ -396,9 +419,11 @@ pub enum CliCommand {
     Balance {
         pubkey: Option<Pubkey>,
         use_lamports_unit: bool,
+        commitment_config: CommitmentConfig,
     },
     Cancel(Pubkey),
     Confirm(Signature),
+    DecodeTransaction(Transaction),
     Pay(PayCommand),
     ResolveSigner(Option<String>),
     ShowAccount {
@@ -546,7 +571,7 @@ impl CliConfig<'_> {
         if !self.signers.is_empty() {
             self.signers[0].try_pubkey()
         } else {
-            Err(SignerError::CustomError(
+            Err(SignerError::Custom(
                 "Default keypair must be set if pubkey arg not provided".to_string(),
             ))
         }
@@ -559,6 +584,7 @@ impl Default for CliConfig<'_> {
             command: CliCommand::Balance {
                 pubkey: Some(Pubkey::default()),
                 use_lamports_unit: false,
+                commitment_config: CommitmentConfig::default(),
             },
             json_rpc_url: Self::default_json_rpc_url(),
             websocket_url: Self::default_websocket_url(),
@@ -579,6 +605,10 @@ pub fn parse_command(
     let response = match matches.subcommand() {
         // Cluster Query Commands
         ("catchup", Some(matches)) => parse_catchup(matches, wallet_manager),
+        ("cluster-date", Some(_matches)) => Ok(CliCommandInfo {
+            command: CliCommand::ClusterDate,
+            signers: vec![],
+        }),
         ("cluster-version", Some(_matches)) => Ok(CliCommandInfo {
             command: CliCommand::ClusterVersion,
             signers: vec![],
@@ -598,6 +628,8 @@ pub fn parse_command(
         }),
         ("epoch", Some(matches)) => parse_get_epoch(matches),
         ("slot", Some(matches)) => parse_get_slot(matches),
+        ("largest-accounts", Some(matches)) => parse_largest_accounts(matches),
+        ("supply", Some(matches)) => parse_supply(matches),
         ("total-supply", Some(matches)) => parse_total_supply(matches),
         ("transaction-count", Some(matches)) => parse_get_transaction_count(matches),
         ("leader-schedule", Some(_matches)) => Ok(CliCommandInfo {
@@ -616,6 +648,9 @@ pub fn parse_command(
         }),
         ("stakes", Some(matches)) => parse_show_stakes(matches, wallet_manager),
         ("validators", Some(matches)) => parse_show_validators(matches),
+        ("transaction-history", Some(matches)) => {
+            parse_transaction_history(matches, wallet_manager)
+        }
         // Nonce Commands
         ("authorize-nonce-account", Some(matches)) => {
             parse_authorize_nonce_account(matches, default_signer_path, wallet_manager)
@@ -765,6 +800,7 @@ pub fn parse_command(
         }
         ("balance", Some(matches)) => {
             let pubkey = pubkey_of_signer(matches, "pubkey", wallet_manager)?;
+            let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
             let signers = if pubkey.is_some() {
                 vec![]
             } else {
@@ -779,6 +815,7 @@ pub fn parse_command(
                 command: CliCommand::Balance {
                     pubkey,
                     use_lamports_unit: matches.is_present("lamports"),
+                    commitment_config,
                 },
                 signers,
             })
@@ -798,11 +835,23 @@ pub fn parse_command(
                 command: CliCommand::Confirm(signature),
                 signers: vec![],
             }),
-            _ => {
-                eprintln!("{}", matches.usage());
-                Err(CliError::BadParameter("Invalid signature".to_string()))
-            }
+            _ => Err(CliError::BadParameter("Invalid signature".to_string())),
         },
+        ("decode-transaction", Some(matches)) => {
+            let encoded_transaction = EncodedTransaction::Binary(
+                matches.value_of("base58_transaction").unwrap().to_string(),
+            );
+            if let Some(transaction) = encoded_transaction.decode() {
+                Ok(CliCommandInfo {
+                    command: CliCommand::DecodeTransaction(transaction),
+                    signers: vec![],
+                })
+            } else {
+                Err(CliError::BadParameter(
+                    "Unable to decode transaction".to_string(),
+                ))
+            }
+        }
         ("pay", Some(matches)) => {
             let lamports = lamports_of_sol(matches, "amount").unwrap();
             let to = pubkey_of_signer(matches, "to", wallet_manager)?.unwrap();
@@ -1021,7 +1070,7 @@ pub fn get_blockhash_and_fee_calculator(
     })
 }
 
-pub fn return_signers(tx: &Transaction) -> ProcessResult {
+pub fn return_signers(tx: &Transaction, config: &CliConfig) -> ProcessResult {
     let verify_results = tx.verify_with_results();
     let mut signers = Vec::new();
     let mut absent = Vec::new();
@@ -1040,15 +1089,14 @@ pub fn return_signers(tx: &Transaction) -> ProcessResult {
             }
         });
 
-    println_signers(&tx.message.recent_blockhash, &signers, &absent, &bad_sig);
+    let cli_command = CliSignOnlyData {
+        blockhash: tx.message.recent_blockhash.to_string(),
+        signers,
+        absent,
+        bad_sig,
+    };
 
-    Ok(json!({
-        "blockhash": tx.message.recent_blockhash.to_string(),
-        "signers": &signers,
-        "absent": &absent,
-        "badSig": &bad_sig,
-    })
-    .to_string())
+    Ok(config.output_format.formatted_string(&cli_command))
 }
 
 pub fn parse_create_address_with_seed(
@@ -1136,7 +1184,7 @@ fn process_airdrop(
         }
     };
 
-    request_and_confirm_airdrop(&rpc_client, faucet_addr, &pubkey, lamports)?;
+    request_and_confirm_airdrop(&rpc_client, faucet_addr, &pubkey, lamports, &config)?;
 
     let current_balance = rpc_client
         .retry_get_balance(&pubkey, 5)?
@@ -1150,19 +1198,17 @@ fn process_balance(
     config: &CliConfig,
     pubkey: &Option<Pubkey>,
     use_lamports_unit: bool,
+    commitment_config: CommitmentConfig,
 ) -> ProcessResult {
     let pubkey = if let Some(pubkey) = pubkey {
         *pubkey
     } else {
         config.pubkey()?
     };
-    let balance = rpc_client.retry_get_balance(&pubkey, 5)?;
-    match balance {
-        Some(lamports) => Ok(build_balance_message(lamports, use_lamports_unit, true)),
-        None => Err(
-            CliError::RpcRequestError("Received result of an unexpected type".to_string()).into(),
-        ),
-    }
+    let balance = rpc_client
+        .get_balance_with_commitment(&pubkey, commitment_config)?
+        .value;
+    Ok(build_balance_message(balance, use_lamports_unit, true))
 }
 
 fn process_confirm(
@@ -1178,16 +1224,15 @@ fn process_confirm(
         Ok(status) => {
             if let Some(transaction_status) = status {
                 if config.verbose {
-                    match rpc_client.get_confirmed_transaction(
-                        signature,
-                        solana_transaction_status::TransactionEncoding::Binary,
-                    ) {
+                    match rpc_client
+                        .get_confirmed_transaction(signature, TransactionEncoding::Binary)
+                    {
                         Ok(confirmed_transaction) => {
                             println!(
                                 "\nTransaction executed in slot {}:",
                                 confirmed_transaction.slot
                             );
-                            crate::display::println_transaction(
+                            println_transaction(
                                 &confirmed_transaction
                                     .transaction
                                     .transaction
@@ -1216,6 +1261,11 @@ fn process_confirm(
     }
 }
 
+fn process_decode_transaction(transaction: &Transaction) -> ProcessResult {
+    println_transaction(transaction, &None, "");
+    Ok("".to_string())
+}
+
 fn process_show_account(
     rpc_client: &RpcClient,
     config: &CliConfig,
@@ -1233,21 +1283,118 @@ fn process_show_account(
         use_lamports_unit,
     };
 
-    config.output_format.formatted_print(&cli_account);
+    let mut account_string = config.output_format.formatted_string(&cli_account);
 
     if config.output_format == OutputFormat::Display {
         if let Some(output_file) = output_file {
             let mut f = File::create(output_file)?;
             f.write_all(&data)?;
-            println!();
-            println!("Wrote account data to {}", output_file);
+            writeln!(&mut account_string)?;
+            writeln!(&mut account_string, "Wrote account data to {}", output_file)?;
         } else if !data.is_empty() {
             use pretty_hex::*;
-            println!("{:?}", data.hex_dump());
+            writeln!(&mut account_string, "{:?}", data.hex_dump())?;
         }
     }
 
-    Ok("".to_string())
+    Ok(account_string)
+}
+
+fn send_and_confirm_transactions_with_spinner<T: Signers>(
+    rpc_client: &RpcClient,
+    mut transactions: Vec<Transaction>,
+    signer_keys: &T,
+) -> Result<(), Box<dyn error::Error>> {
+    let progress_bar = new_spinner_progress_bar();
+    let mut send_retries = 5;
+    loop {
+        let mut status_retries = 15;
+
+        // Send all transactions
+        let mut transactions_signatures = vec![];
+        let num_transactions = transactions.len();
+        for transaction in transactions {
+            if cfg!(not(test)) {
+                // Delay ~1 tick between write transactions in an attempt to reduce AccountInUse errors
+                // when all the write transactions modify the same program account (eg, deploying a
+                // new program)
+                sleep(Duration::from_millis(1000 / DEFAULT_TICKS_PER_SECOND));
+            }
+
+            let signature = rpc_client
+                .send_transaction_with_config(
+                    &transaction,
+                    RpcSendTransactionConfig {
+                        skip_preflight: true,
+                    },
+                )
+                .ok();
+            transactions_signatures.push((transaction, signature));
+
+            progress_bar.set_message(&format!(
+                "[{}/{}] Transactions sent",
+                transactions_signatures.len(),
+                num_transactions
+            ));
+        }
+
+        // Collect statuses for all the transactions, drop those that are confirmed
+        while status_retries > 0 {
+            status_retries -= 1;
+
+            progress_bar.set_message(&format!(
+                "[{}/{}] Transactions confirmed",
+                num_transactions - transactions_signatures.len(),
+                num_transactions
+            ));
+
+            if cfg!(not(test)) {
+                // Retry twice a second
+                sleep(Duration::from_millis(500));
+            }
+
+            transactions_signatures = transactions_signatures
+                .into_iter()
+                .filter(|(_transaction, signature)| {
+                    if let Some(signature) = signature {
+                        if let Ok(status) = rpc_client.get_signature_status(&signature) {
+                            if rpc_client
+                                .get_num_blocks_since_signature_confirmation(&signature)
+                                .unwrap_or(0)
+                                > 1
+                            {
+                                return false;
+                            } else {
+                                return match status {
+                                    None => true,
+                                    Some(result) => result.is_err(),
+                                };
+                            }
+                        }
+                    }
+                    true
+                })
+                .collect();
+
+            if transactions_signatures.is_empty() {
+                return Ok(());
+            }
+        }
+
+        if send_retries == 0 {
+            return Err("Transactions failed".into());
+        }
+        send_retries -= 1;
+
+        // Re-sign any failed transactions with a new blockhash and retry
+        let (blockhash, _fee_calculator) = rpc_client
+            .get_new_blockhash(&transactions_signatures[0].0.message().recent_blockhash)?;
+        transactions = vec![];
+        for (mut transaction, _) in transactions_signatures.into_iter() {
+            transaction.try_sign(signer_keys, blockhash)?;
+            transactions.push(transaction);
+        }
+    }
 }
 
 fn process_deploy(
@@ -1311,19 +1458,26 @@ fn process_deploy(
     )?;
 
     trace!("Creating program account");
-    let result =
-        rpc_client.send_and_confirm_transaction_with_spinner(&mut create_account_tx, &signers);
-    log_instruction_custom_error::<SystemError>(result)
-        .map_err(|_| CliError::DynamicProgramError("Program allocate space failed".to_string()))?;
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&create_account_tx);
+    log_instruction_custom_error::<SystemError>(result, &config).map_err(|_| {
+        CliError::DynamicProgramError("Program account allocation failed".to_string())
+    })?;
 
     trace!("Writing program data");
-    rpc_client.send_and_confirm_transactions(write_transactions, &signers)?;
+    send_and_confirm_transactions_with_spinner(&rpc_client, write_transactions, &signers).map_err(
+        |_| CliError::DynamicProgramError("Data writes to program account failed".to_string()),
+    )?;
 
     trace!("Finalizing program account");
     rpc_client
-        .send_and_confirm_transaction_with_spinner(&mut finalize_tx, &signers)
-        .map_err(|_| {
-            CliError::DynamicProgramError("Program finalize transaction failed".to_string())
+        .send_and_confirm_transaction_with_spinner_and_config(
+            &finalize_tx,
+            RpcSendTransactionConfig {
+                skip_preflight: true,
+            },
+        )
+        .map_err(|e| {
+            CliError::DynamicProgramError(format!("Program finalize transaction failed: {}", e))
         })?;
 
     Ok(json!({
@@ -1373,7 +1527,7 @@ fn process_pay(
 
         if sign_only {
             tx.try_partial_sign(&config.signers, blockhash)?;
-            return_signers(&tx)
+            return_signers(&tx, &config)
         } else {
             tx.try_sign(&config.signers, blockhash)?;
             if let Some(nonce_account) = &nonce_account {
@@ -1386,9 +1540,8 @@ fn process_pay(
                 &fee_calculator,
                 &tx.message,
             )?;
-            let result =
-                rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-            log_instruction_custom_error::<SystemError>(result)
+            let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+            log_instruction_custom_error::<SystemError>(result, &config)
         }
     } else if *witnesses == None {
         let dt = timestamp.unwrap();
@@ -1413,7 +1566,7 @@ fn process_pay(
         let mut tx = Transaction::new_unsigned(message);
         if sign_only {
             tx.try_partial_sign(&[config.signers[0], &contract_state], blockhash)?;
-            return_signers(&tx)
+            return_signers(&tx, &config)
         } else {
             tx.try_sign(&[config.signers[0], &contract_state], blockhash)?;
             check_account_for_fee(
@@ -1422,14 +1575,10 @@ fn process_pay(
                 &fee_calculator,
                 &tx.message,
             )?;
-            let result = rpc_client.send_and_confirm_transaction_with_spinner(
-                &mut tx,
-                &[config.signers[0], &contract_state],
-            );
-            let signature_str = log_instruction_custom_error::<BudgetError>(result)?;
-
+            let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+            let signature = log_instruction_custom_error::<BudgetError>(result, &config)?;
             Ok(json!({
-                "signature": signature_str,
+                "signature": signature,
                 "processId": format!("{}", contract_state.pubkey()),
             })
             .to_string())
@@ -1459,23 +1608,19 @@ fn process_pay(
         let mut tx = Transaction::new_unsigned(message);
         if sign_only {
             tx.try_partial_sign(&[config.signers[0], &contract_state], blockhash)?;
-            return_signers(&tx)
+            return_signers(&tx, &config)
         } else {
             tx.try_sign(&[config.signers[0], &contract_state], blockhash)?;
-            let result = rpc_client.send_and_confirm_transaction_with_spinner(
-                &mut tx,
-                &[config.signers[0], &contract_state],
-            );
+            let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
             check_account_for_fee(
                 rpc_client,
                 &config.signers[0].pubkey(),
                 &fee_calculator,
                 &tx.message,
             )?;
-            let signature_str = log_instruction_custom_error::<BudgetError>(result)?;
-
+            let signature = log_instruction_custom_error::<BudgetError>(result, &config)?;
             Ok(json!({
-                "signature": signature_str,
+                "signature": signature,
                 "processId": format!("{}", contract_state.pubkey()),
             })
             .to_string())
@@ -1501,9 +1646,8 @@ fn process_cancel(rpc_client: &RpcClient, config: &CliConfig, pubkey: &Pubkey) -
         &fee_calculator,
         &tx.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0]]);
-    log_instruction_custom_error::<BudgetError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<BudgetError>(result, &config)
 }
 
 fn process_time_elapsed(
@@ -1525,9 +1669,8 @@ fn process_time_elapsed(
         &fee_calculator,
         &tx.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0]]);
-    log_instruction_custom_error::<BudgetError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<BudgetError>(result, &config)
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -1546,11 +1689,6 @@ fn process_transfer(
 ) -> ProcessResult {
     let from = config.signers[from];
 
-    check_unique_pubkeys(
-        (&from.pubkey(), "cli keypair".to_string()),
-        (to, "to".to_string()),
-    )?;
-
     let (recent_blockhash, fee_calculator) =
         blockhash_query.get_blockhash_and_fee_calculator(rpc_client)?;
     let ixs = vec![system_instruction::transfer(&from.pubkey(), to, lamports)];
@@ -1572,7 +1710,7 @@ fn process_transfer(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -1588,9 +1726,9 @@ fn process_transfer(
         let result = if no_wait {
             rpc_client.send_transaction(&tx)
         } else {
-            rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers)
+            rpc_client.send_and_confirm_transaction_with_spinner(&tx)
         };
-        log_instruction_custom_error::<SystemError>(result)
+        log_instruction_custom_error::<SystemError>(result, &config)
     }
 }
 
@@ -1612,9 +1750,8 @@ fn process_witness(
         &fee_calculator,
         &tx.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0]]);
-    log_instruction_custom_error::<BudgetError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<BudgetError>(result, &config)
 }
 
 pub fn process_command(config: &CliConfig) -> ProcessResult {
@@ -1653,6 +1790,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *commitment_config,
             *follow,
         ),
+        CliCommand::ClusterDate => process_cluster_date(&rpc_client, config),
         CliCommand::ClusterVersion => process_cluster_version(&rpc_client),
         CliCommand::CreateAddressWithSeed {
             from_pubkey,
@@ -1660,7 +1798,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             program_id,
         } => process_create_address_with_seed(config, from_pubkey.as_ref(), &seed, &program_id),
         CliCommand::Fees => process_fees(&rpc_client),
-        CliCommand::GetBlockTime { slot } => process_get_block_time(&rpc_client, *slot),
+        CliCommand::GetBlockTime { slot } => process_get_block_time(&rpc_client, config, *slot),
         CliCommand::GetGenesisHash => process_get_genesis_hash(&rpc_client),
         CliCommand::GetEpochInfo { commitment_config } => {
             process_get_epoch_info(&rpc_client, config, *commitment_config)
@@ -1671,6 +1809,14 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::GetSlot { commitment_config } => {
             process_get_slot(&rpc_client, *commitment_config)
         }
+        CliCommand::LargestAccounts {
+            commitment_config,
+            filter,
+        } => process_largest_accounts(&rpc_client, config, *commitment_config, filter.clone()),
+        CliCommand::Supply {
+            commitment_config,
+            print_accounts,
+        } => process_supply(&rpc_client, config, *commitment_config, *print_accounts),
         CliCommand::TotalSupply { commitment_config } => {
             process_total_supply(&rpc_client, *commitment_config)
         }
@@ -1711,6 +1857,11 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             use_lamports_unit,
             commitment_config,
         } => process_show_validators(&rpc_client, config, *use_lamports_unit, *commitment_config),
+        CliCommand::TransactionHistory {
+            address,
+            end_slot,
+            slot_limit,
+        } => process_transaction_history(&rpc_client, address, *end_slot, *slot_limit),
 
         // Nonce Commands
 
@@ -1937,6 +2088,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             destination_account_pubkey,
             lamports,
             withdraw_authority,
+            custodian,
             sign_only,
             blockhash_query,
             ref nonce_account,
@@ -1949,6 +2101,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             &destination_account_pubkey,
             *lamports,
             *withdraw_authority,
+            *custodian,
             *sign_only,
             blockhash_query,
             nonce_account.as_ref(),
@@ -2094,11 +2247,19 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::Balance {
             pubkey,
             use_lamports_unit,
-        } => process_balance(&rpc_client, config, &pubkey, *use_lamports_unit),
+            commitment_config,
+        } => process_balance(
+            &rpc_client,
+            config,
+            &pubkey,
+            *use_lamports_unit,
+            *commitment_config,
+        ),
         // Cancel a contract by contract Pubkey
         CliCommand::Cancel(pubkey) => process_cancel(&rpc_client, config, &pubkey),
         // Confirm the last client transaction by signature
         CliCommand::Confirm(signature) => process_confirm(&rpc_client, config, signature),
+        CliCommand::DecodeTransaction(transaction) => process_decode_transaction(transaction),
         // If client has positive balance, pay lamports to another address
         CliCommand::Pay(PayCommand {
             lamports,
@@ -2221,6 +2382,7 @@ pub fn request_and_confirm_airdrop(
     faucet_addr: &SocketAddr,
     to_pubkey: &Pubkey,
     lamports: u64,
+    config: &CliConfig,
 ) -> ProcessResult {
     let (blockhash, _fee_calculator) = rpc_client.get_recent_blockhash()?;
     let keypair = {
@@ -2234,12 +2396,15 @@ pub fn request_and_confirm_airdrop(
             sleep(Duration::from_secs(1));
         }
     }?;
-    let mut tx = keypair.airdrop_transaction();
-    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &[&keypair]);
-    log_instruction_custom_error::<SystemError>(result)
+    let tx = keypair.airdrop_transaction();
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<SystemError>(result, &config)
 }
 
-pub fn log_instruction_custom_error<E>(result: ClientResult<Signature>) -> ProcessResult
+pub fn log_instruction_custom_error<E>(
+    result: ClientResult<Signature>,
+    config: &CliConfig,
+) -> ProcessResult
 where
     E: 'static + std::error::Error + DecodeError<E> + FromPrimitive,
 {
@@ -2247,7 +2412,7 @@ where
         Err(err) => {
             if let ClientErrorKind::TransactionError(TransactionError::InstructionError(
                 _,
-                InstructionError::CustomError(code),
+                InstructionError::Custom(code),
             )) = err.kind()
             {
                 if let Some(specific_error) = E::decode_custom_error_to_enum(*code) {
@@ -2256,7 +2421,12 @@ where
             }
             Err(err.into())
         }
-        Ok(sig) => Ok(sig.to_string()),
+        Ok(sig) => {
+            let signature = CliSignature {
+                signature: sig.clone().to_string(),
+            };
+            Ok(config.output_format.formatted_string(&signature))
+        }
     }
 }
 
@@ -2353,7 +2523,8 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg_with_default("max")),
         )
         .subcommand(
             SubCommand::with_name("cancel")
@@ -2380,6 +2551,18 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .help("The transaction signature to confirm"),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("decode-transaction")
+                .about("Decode a base-58 binary transaction")
+                .arg(
+                    Arg::with_name("base58_transaction")
+                        .index(1)
+                        .value_name("BASE58_TRANSACTION")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The transaction to decode"),
+                ),
+        )
         .subcommand(
             SubCommand::with_name("create-address-with-seed")
                 .about("Generate a derived account address with a seed")
@@ -2752,7 +2935,8 @@ mod tests {
             CliCommandInfo {
                 command: CliCommand::Balance {
                     pubkey: Some(keypair.pubkey()),
-                    use_lamports_unit: false
+                    use_lamports_unit: false,
+                    commitment_config: CommitmentConfig::default(),
                 },
                 signers: vec![],
             }
@@ -2768,7 +2952,8 @@ mod tests {
             CliCommandInfo {
                 command: CliCommand::Balance {
                     pubkey: Some(keypair.pubkey()),
-                    use_lamports_unit: true
+                    use_lamports_unit: true,
+                    commitment_config: CommitmentConfig::default(),
                 },
                 signers: vec![],
             }
@@ -2782,7 +2967,8 @@ mod tests {
             CliCommandInfo {
                 command: CliCommand::Balance {
                     pubkey: None,
-                    use_lamports_unit: true
+                    use_lamports_unit: true,
+                    commitment_config: CommitmentConfig::default(),
                 },
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -3252,12 +3438,14 @@ mod tests {
         config.command = CliCommand::Balance {
             pubkey: None,
             use_lamports_unit: true,
+            commitment_config: CommitmentConfig::default(),
         };
         assert_eq!(process_command(&config).unwrap(), "50 lamports");
 
         config.command = CliCommand::Balance {
             pubkey: None,
             use_lamports_unit: false,
+            commitment_config: CommitmentConfig::default(),
         };
         assert_eq!(process_command(&config).unwrap(), "0.00000005 SOL");
 
@@ -3334,6 +3522,7 @@ mod tests {
             destination_account_pubkey: to_pubkey,
             lamports: 100,
             withdraw_authority: 0,
+            custodian: None,
             sign_only: false,
             blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
             nonce_account: None,
@@ -3490,6 +3679,7 @@ mod tests {
         config.command = CliCommand::Balance {
             pubkey: None,
             use_lamports_unit: false,
+            commitment_config: CommitmentConfig::default(),
         };
         assert!(process_command(&config).is_err());
 
@@ -3785,6 +3975,8 @@ mod tests {
             }
         }
 
+        let mut config = CliConfig::default();
+        config.output_format = OutputFormat::JsonCompact;
         let present: Box<dyn Signer> = Box::new(keypair_from_seed(&[2u8; 32]).unwrap());
         let absent: Box<dyn Signer> = Box::new(NullSigner::new(&Pubkey::new(&[3u8; 32])));
         let bad: Box<dyn Signer> = Box::new(BadSigner::new(Pubkey::new(&[4u8; 32])));
@@ -3803,7 +3995,7 @@ mod tests {
         let signers = vec![present.as_ref(), absent.as_ref(), bad.as_ref()];
         let blockhash = Hash::new(&[7u8; 32]);
         tx.try_partial_sign(&signers, blockhash).unwrap();
-        let res = return_signers(&tx).unwrap();
+        let res = return_signers(&tx, &config).unwrap();
         let sign_only = parse_sign_only_reply_string(&res);
         assert_eq!(sign_only.blockhash, blockhash);
         assert_eq!(sign_only.present_signers[0].0, present.pubkey());

cli/src/cli_output.rs

@@ -4,9 +4,12 @@ use console::{style, Emoji};
 use inflector::cases::titlecase::to_title_case;
 use serde::Serialize;
 use serde_json::{Map, Value};
-use solana_client::rpc_response::{RpcEpochInfo, RpcKeyedAccount, RpcVoteAccountInfo};
+use solana_client::rpc_response::{
+    RpcAccountBalance, RpcEpochInfo, RpcKeyedAccount, RpcSupply, RpcVoteAccountInfo,
+};
 use solana_sdk::{
     clock::{self, Epoch, Slot, UnixTimestamp},
+    native_token::lamports_to_sol,
     stake_history::StakeHistoryEntry,
 };
 use solana_stake_program::stake_state::{Authorized, Lockup};
@@ -26,20 +29,14 @@ pub enum OutputFormat {
 }
 
 impl OutputFormat {
-    pub fn formatted_print<T>(&self, item: &T)
+    pub fn formatted_string<T>(&self, item: &T) -> String
     where
         T: Serialize + fmt::Display,
     {
         match self {
-            OutputFormat::Display => {
-                println!("{}", item);
-            }
-            OutputFormat::Json => {
-                println!("{}", serde_json::to_string_pretty(item).unwrap());
-            }
-            OutputFormat::JsonCompact => {
-                println!("{}", serde_json::to_value(item).unwrap());
-            }
+            OutputFormat::Display => format!("{}", item),
+            OutputFormat::Json => serde_json::to_string_pretty(item).unwrap(),
+            OutputFormat::JsonCompact => serde_json::to_value(item).unwrap().to_string(),
         }
     }
 }
@@ -839,3 +836,147 @@ impl From<&Lockout> for CliLockout {
         }
     }
 }
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliBlockTime {
+    pub slot: Slot,
+    pub timestamp: UnixTimestamp,
+}
+
+impl fmt::Display for CliBlockTime {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Block:", &self.slot.to_string())?;
+        writeln_name_value(
+            f,
+            "Date:",
+            &format!(
+                "{} (UnixTimestamp: {})",
+                DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(self.timestamp, 0), Utc)
+                    .to_rfc3339_opts(SecondsFormat::Secs, true),
+                self.timestamp
+            ),
+        )
+    }
+}
+
+#[derive(Serialize, Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct CliSignOnlyData {
+    pub blockhash: String,
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    pub signers: Vec<String>,
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    pub absent: Vec<String>,
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    pub bad_sig: Vec<String>,
+}
+
+impl fmt::Display for CliSignOnlyData {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Blockhash:", &self.blockhash)?;
+        if !self.signers.is_empty() {
+            writeln!(f, "{}", style("Signers (Pubkey=Signature):").bold())?;
+            for signer in self.signers.iter() {
+                writeln!(f, " {}", signer)?;
+            }
+        }
+        if !self.absent.is_empty() {
+            writeln!(f, "{}", style("Absent Signers (Pubkey):").bold())?;
+            for pubkey in self.absent.iter() {
+                writeln!(f, " {}", pubkey)?;
+            }
+        }
+        if !self.bad_sig.is_empty() {
+            writeln!(f, "{}", style("Bad Signatures (Pubkey):").bold())?;
+            for pubkey in self.bad_sig.iter() {
+                writeln!(f, " {}", pubkey)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CliSignature {
+    pub signature: String,
+}
+
+impl fmt::Display for CliSignature {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Signature:", &self.signature)?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CliAccountBalances {
+    pub accounts: Vec<RpcAccountBalance>,
+}
+
+impl fmt::Display for CliAccountBalances {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(
+            f,
+            "{}",
+            style(format!("{:<44}  {}", "Address", "Balance",)).bold()
+        )?;
+        for account in &self.accounts {
+            writeln!(
+                f,
+                "{:<44}  {}",
+                account.address,
+                &format!("{} SOL", lamports_to_sol(account.lamports))
+            )?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CliSupply {
+    pub total: u64,
+    pub circulating: u64,
+    pub non_circulating: u64,
+    pub non_circulating_accounts: Vec<String>,
+    #[serde(skip_serializing)]
+    pub print_accounts: bool,
+}
+
+impl From<RpcSupply> for CliSupply {
+    fn from(rpc_supply: RpcSupply) -> Self {
+        Self {
+            total: rpc_supply.total,
+            circulating: rpc_supply.circulating,
+            non_circulating: rpc_supply.non_circulating,
+            non_circulating_accounts: rpc_supply.non_circulating_accounts,
+            print_accounts: false,
+        }
+    }
+}
+
+impl fmt::Display for CliSupply {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Total:", &format!("{} SOL", lamports_to_sol(self.total)))?;
+        writeln_name_value(
+            f,
+            "Circulating:",
+            &format!("{} SOL", lamports_to_sol(self.circulating)),
+        )?;
+        writeln_name_value(
+            f,
+            "Non-Circulating:",
+            &format!("{} SOL", lamports_to_sol(self.non_circulating)),
+        )?;
+        if self.print_accounts {
+            writeln!(f)?;
+            writeln_name_value(f, "Non-Circulating Accounts:", " ")?;
+            for account in &self.non_circulating_accounts {
+                writeln!(f, "  {}", account)?;
+            }
+        }
+        Ok(())
+    }
+}

cli/src/cluster_query.rs

@@ -1,24 +1,26 @@
 use crate::{
     cli::{check_account_for_fee, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult},
-    cli_output::{
-        CliBlockProduction, CliBlockProductionEntry, CliEpochInfo, CliKeyedStakeState,
-        CliSlotStatus, CliStakeVec, CliValidator, CliValidators,
-    },
-    display::println_name_value,
+    cli_output::*,
+    display::{new_spinner_progress_bar, println_name_value},
 };
-use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
-use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
+use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
-use indicatif::{ProgressBar, ProgressStyle};
-use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+    keypair::signer_from_path,
+};
 use solana_client::{
     pubsub_client::{PubsubClient, SlotInfoMessage},
     rpc_client::RpcClient,
+    rpc_config::{RpcLargestAccountsConfig, RpcLargestAccountsFilter},
+    rpc_request::MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE,
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     account_utils::StateMut,
-    clock::{self, Slot},
+    clock::{self, Clock, Slot},
     commitment_config::CommitmentConfig,
     epoch_schedule::Epoch,
     hash::Hash,
@@ -27,6 +29,7 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     system_instruction,
+    sysvar::{self, Sysvar},
     transaction::Transaction,
 };
 use std::{
@@ -69,20 +72,17 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .validator(is_url)
                         .help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
                 )
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("follow")
                         .long("follow")
                         .takes_value(false)
                         .help("Continue reporting progress even after the validator has caught up"),
-                ),
+                )
+                .arg(commitment_arg()),
+        )
+        .subcommand(
+            SubCommand::with_name("cluster-date")
+                .about("Get current cluster date, computed from genesis creation time and network time")
         )
         .subcommand(
             SubCommand::with_name("cluster-version")
@@ -97,7 +97,6 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .index(1)
                     .takes_value(true)
                     .value_name("SLOT")
-                    .required(true)
                     .help("Slot number of the block to query")
             )
         )
@@ -106,14 +105,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
             .alias("get-epoch-info")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return information at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("genesis-hash")
@@ -123,48 +115,48 @@ impl ClusterQuerySubCommands for App<'_, '_> {
         .subcommand(
             SubCommand::with_name("slot").about("Get current slot")
             .alias("get-slot")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return slot at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("epoch").about("Get current epoch")
+            .arg(commitment_arg()),
+        )
+        .subcommand(
+            SubCommand::with_name("largest-accounts").about("Get addresses of largest cluster accounts")
             .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
+                Arg::with_name("circulating")
+                    .long("circulating")
                     .takes_value(false)
-                    .help(
-                        "Return epoch at maximum-lockout commitment level",
-                    ),
-            ),
+                    .help("Filter address list to only circulating accounts")
+            )
+            .arg(
+                Arg::with_name("non_circulating")
+                    .long("non-circulating")
+                    .takes_value(false)
+                    .conflicts_with("circulating")
+                    .help("Filter address list to only non-circulating accounts")
+            )
+            .arg(commitment_arg()),
+        )
+        .subcommand(
+            SubCommand::with_name("supply").about("Get information about the cluster supply of SOL")
+            .arg(
+                Arg::with_name("print_accounts")
+                    .long("print-accounts")
+                    .takes_value(false)
+                    .help("Print list of non-circualting account addresses")
+            )
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("total-supply").about("Get total number of SOL")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return count at maximum-lockout commitment level",
-                    ),
-            ),
+            .setting(AppSettings::Hidden)
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("transaction-count").about("Get current transaction count")
             .alias("get-transaction-count")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return count at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("ping")
@@ -204,14 +196,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .default_value("15")
                         .help("Wait up to timeout seconds for transaction confirmation"),
                 )
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Wait until the transaction is confirmed at maximum-lockout commitment level",
-                        ),
-                ),
+                .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("live-slots")
@@ -262,19 +247,46 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("validators")
                 .about("Show summary information about the current validators")
                 .alias("show-validators")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("lamports")
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
+                )
+                .arg(commitment_arg()),
+        )
+        .subcommand(
+            SubCommand::with_name("transaction-history")
+                .about("Show historical transactions affecting the given address, \
+                       ordered based on the slot in which they were confirmed in \
+                       from lowest to highest slot")
+                .arg(
+                    Arg::with_name("address")
+                        .index(1)
+                        .value_name("ADDRESS")
+                        .required(true)
+                        .validator(is_valid_pubkey)
+                        .help("Account address"),
+                )
+                .arg(
+                    Arg::with_name("end_slot")
+                        .takes_value(false)
+                        .value_name("SLOT")
+                        .index(2)
+                        .validator(is_slot)
+                        .help(
+                            "Slot to start from [default: latest slot at maximum commitment]"
+                        ),
+                )
+                .arg(
+                    Arg::with_name("limit")
+                        .long("limit")
+                        .takes_value(true)
+                        .value_name("NUMBER OF SLOTS")
+                        .validator(is_slot)
+                        .help(
+                            "Limit the search to this many slots"
+                        ),
                 ),
         )
     }
@@ -286,11 +298,7 @@ pub fn parse_catchup(
 ) -> Result<CliCommandInfo, CliError> {
     let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
     let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     let follow = matches.is_present("follow");
     Ok(CliCommandInfo {
         command: CliCommand::Catchup {
@@ -316,11 +324,7 @@ pub fn parse_cluster_ping(
         None
     };
     let timeout = Duration::from_secs(value_t_or_exit!(matches, "timeout", u64));
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::Ping {
             lamports,
@@ -339,7 +343,7 @@ pub fn parse_cluster_ping(
 }
 
 pub fn parse_get_block_time(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let slot = value_t_or_exit!(matches, "slot", u64);
+    let slot = value_of(matches, "slot");
     Ok(CliCommandInfo {
         command: CliCommand::GetBlockTime { slot },
         signers: vec![],
@@ -347,11 +351,7 @@ pub fn parse_get_block_time(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetEpochInfo { commitment_config },
         signers: vec![],
@@ -359,11 +359,7 @@ pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetSlot { commitment_config },
         signers: vec![],
@@ -371,23 +367,45 @@ pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliErr
 }
 
 pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetEpoch { commitment_config },
         signers: vec![],
     })
 }
 
-pub fn parse_total_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
+pub fn parse_largest_accounts(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
+    let filter = if matches.is_present("circulating") {
+        Some(RpcLargestAccountsFilter::Circulating)
+    } else if matches.is_present("non_circulating") {
+        Some(RpcLargestAccountsFilter::NonCirculating)
     } else {
-        CommitmentConfig::recent()
+        None
     };
+    Ok(CliCommandInfo {
+        command: CliCommand::LargestAccounts {
+            commitment_config,
+            filter,
+        },
+        signers: vec![],
+    })
+}
+
+pub fn parse_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
+    let print_accounts = matches.is_present("print_accounts");
+    Ok(CliCommandInfo {
+        command: CliCommand::Supply {
+            commitment_config,
+            print_accounts,
+        },
+        signers: vec![],
+    })
+}
+
+pub fn parse_total_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::TotalSupply { commitment_config },
         signers: vec![],
@@ -395,11 +413,7 @@ pub fn parse_total_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, Cl
 }
 
 pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetTransactionCount { commitment_config },
         signers: vec![],
@@ -425,11 +439,7 @@ pub fn parse_show_stakes(
 
 pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
 
     Ok(CliCommandInfo {
         command: CliCommand::ShowValidators {
@@ -440,13 +450,22 @@ pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
     })
 }
 
-/// Creates a new process bar for processing that will take an unknown amount of time
-fn new_spinner_progress_bar() -> ProgressBar {
-    let progress_bar = ProgressBar::new(42);
-    progress_bar
-        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
-    progress_bar.enable_steady_tick(100);
-    progress_bar
+pub fn parse_transaction_history(
+    matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let address = pubkey_of_signer(matches, "address", wallet_manager)?.unwrap();
+    let end_slot = value_t!(matches, "end_slot", Slot).ok();
+    let slot_limit = value_t!(matches, "limit", u64).ok();
+
+    Ok(CliCommandInfo {
+        command: CliCommand::TransactionHistory {
+            address,
+            end_slot,
+            slot_limit,
+        },
+        signers: vec![],
+    })
 }
 
 pub fn process_catchup(
@@ -513,25 +532,35 @@ pub fn process_catchup(
         }
 
         let slot_distance = rpc_slot as i64 - node_slot as i64;
+        let slots_per_second =
+            (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
+        let time_remaining = (slot_distance as f64 / slots_per_second).round();
+        let time_remaining = if !time_remaining.is_normal() || time_remaining <= 0.0 {
+            "".to_string()
+        } else {
+            format!(
+                ". Time remaining: {}",
+                humantime::format_duration(Duration::from_secs_f64(time_remaining))
+            )
+        };
+
         progress_bar.set_message(&format!(
-            "Validator is {} slots away (us:{} them:{}){}",
+            "{} slots behind (us:{} them:{}){}",
             slot_distance,
             node_slot,
             rpc_slot,
             if slot_distance == 0 || previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
-                let slots_per_second =
-                    (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
-
                 format!(
-                    " and {} at {:.1} slots/second",
+                    ", {} at {:.1} slots/second{}",
                     if slots_per_second < 0.0 {
                         "falling behind"
                     } else {
                         "gaining"
                     },
                     slots_per_second,
+                    time_remaining
                 )
             }
         ));
@@ -542,6 +571,23 @@ pub fn process_catchup(
     }
 }
 
+pub fn process_cluster_date(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
+    let result = rpc_client
+        .get_account_with_commitment(&sysvar::clock::id(), CommitmentConfig::default())?;
+    if let Some(clock_account) = result.value {
+        let clock: Clock = Sysvar::from_account(&clock_account).ok_or_else(|| {
+            CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
+        })?;
+        let block_time = CliBlockTime {
+            slot: result.context.slot,
+            timestamp: clock.unix_timestamp,
+        };
+        Ok(config.output_format.formatted_string(&block_time))
+    } else {
+        Err(format!("AccountNotFound: pubkey={}", sysvar::clock::id()).into())
+    }
+}
+
 pub fn process_cluster_version(rpc_client: &RpcClient) -> ProcessResult {
     let remote_version = rpc_client.get_version()?;
     Ok(remote_version.solana_core)
@@ -591,15 +637,19 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
     Ok("".to_string())
 }
 
-pub fn process_get_block_time(rpc_client: &RpcClient, slot: Slot) -> ProcessResult {
+pub fn process_get_block_time(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    slot: Option<Slot>,
+) -> ProcessResult {
+    let slot = if let Some(slot) = slot {
+        slot
+    } else {
+        rpc_client.get_slot()?
+    };
     let timestamp = rpc_client.get_block_time(slot)?;
-    let result = format!(
-        "{} (UnixTimestamp: {})",
-        DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(timestamp, 0), Utc)
-            .to_rfc3339_opts(SecondsFormat::Secs, true),
-        timestamp
-    );
-    Ok(result)
+    let block_time = CliBlockTime { slot, timestamp };
+    Ok(config.output_format.formatted_string(&block_time))
 }
 
 pub fn process_get_epoch_info(
@@ -610,8 +660,7 @@ pub fn process_get_epoch_info(
     let epoch_info: CliEpochInfo = rpc_client
         .get_epoch_info_with_commitment(commitment_config.clone())?
         .into();
-    config.output_format.formatted_print(&epoch_info);
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&epoch_info))
 }
 
 pub fn process_get_genesis_hash(rpc_client: &RpcClient) -> ProcessResult {
@@ -652,7 +701,7 @@ pub fn process_show_block_production(
     slot_limit: Option<u64>,
 ) -> ProcessResult {
     let epoch_schedule = rpc_client.get_epoch_schedule()?;
-    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::max())?;
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::root())?;
 
     let epoch = epoch.unwrap_or(epoch_info.epoch);
     if epoch > epoch_info.epoch {
@@ -711,7 +760,7 @@ pub fn process_show_block_production(
 
     progress_bar.set_message(&format!("Fetching leader schedule for epoch {}...", epoch));
     let leader_schedule = rpc_client
-        .get_leader_schedule_with_commitment(Some(start_slot), CommitmentConfig::max())?;
+        .get_leader_schedule_with_commitment(Some(start_slot), CommitmentConfig::root())?;
     if leader_schedule.is_none() {
         return Err(format!("Unable to fetch leader schedule for slot {}", start_slot).into());
     }
@@ -793,8 +842,35 @@ pub fn process_show_block_production(
         individual_slot_status,
         verbose: config.verbose,
     };
-    config.output_format.formatted_print(&block_production);
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&block_production))
+}
+
+pub fn process_largest_accounts(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    commitment_config: CommitmentConfig,
+    filter: Option<RpcLargestAccountsFilter>,
+) -> ProcessResult {
+    let accounts = rpc_client
+        .get_largest_accounts_with_config(RpcLargestAccountsConfig {
+            commitment: Some(commitment_config),
+            filter,
+        })?
+        .value;
+    let largest_accounts = CliAccountBalances { accounts };
+    Ok(config.output_format.formatted_string(&largest_accounts))
+}
+
+pub fn process_supply(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    commitment_config: CommitmentConfig,
+    print_accounts: bool,
+) -> ProcessResult {
+    let supply_response = rpc_client.supply_with_commitment(commitment_config.clone())?;
+    let mut supply: CliSupply = supply_response.value.into();
+    supply.print_accounts = print_accounts;
+    Ok(config.output_format.formatted_string(&supply))
 }
 
 pub fn process_total_supply(
@@ -1123,10 +1199,9 @@ pub fn process_show_stakes(
             }
         }
     }
-    config
+    Ok(config
         .output_format
-        .formatted_print(&CliStakeVec::new(stake_accounts));
-    Ok("".to_string())
+        .formatted_string(&CliStakeVec::new(stake_accounts)))
 }
 
 pub fn process_show_validators(
@@ -1170,8 +1245,46 @@ pub fn process_show_validators(
         delinquent_validators,
         use_lamports_unit,
     };
-    config.output_format.formatted_print(&cli_validators);
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&cli_validators))
+}
+
+pub fn process_transaction_history(
+    rpc_client: &RpcClient,
+    address: &Pubkey,
+    end_slot: Option<Slot>, // None == use latest slot
+    slot_limit: Option<u64>,
+) -> ProcessResult {
+    let end_slot = {
+        if let Some(end_slot) = end_slot {
+            end_slot
+        } else {
+            rpc_client.get_slot_with_commitment(CommitmentConfig::max())?
+        }
+    };
+    let mut start_slot = match slot_limit {
+        Some(slot_limit) => end_slot.saturating_sub(slot_limit),
+        None => rpc_client.minimum_ledger_slot()?,
+    };
+
+    println!(
+        "Transactions affecting {} within slots [{},{}]",
+        address, start_slot, end_slot
+    );
+
+    let mut transaction_count = 0;
+    while start_slot < end_slot {
+        let signatures = rpc_client.get_confirmed_signatures_for_address(
+            address,
+            start_slot,
+            (start_slot + MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE).min(end_slot),
+        )?;
+        for signature in &signatures {
+            println!("{}", signature);
+        }
+        transaction_count += signatures.len();
+        start_slot += MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE;
+    }
+    Ok(format!("{} transactions found", transaction_count))
 }
 
 #[cfg(test)]
@@ -1193,6 +1306,17 @@ mod tests {
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
 
+        let test_cluster_version = test_commands
+            .clone()
+            .get_matches_from(vec!["test", "cluster-date"]);
+        assert_eq!(
+            parse_command(&test_cluster_version, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::ClusterDate,
+                signers: vec![],
+            }
+        );
+
         let test_cluster_version = test_commands
             .clone()
             .get_matches_from(vec!["test", "cluster-version"]);
@@ -1221,7 +1345,7 @@ mod tests {
         assert_eq!(
             parse_command(&test_get_block_time, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
-                command: CliCommand::GetBlockTime { slot },
+                command: CliCommand::GetBlockTime { slot: Some(slot) },
                 signers: vec![],
             }
         );
@@ -1309,7 +1433,8 @@ mod tests {
             "2",
             "-t",
             "3",
-            "--confirmed",
+            "--commitment",
+            "max",
         ]);
         assert_eq!(
             parse_command(&test_ping, &default_keypair_file, &mut None).unwrap(),
@@ -1319,7 +1444,7 @@ mod tests {
                     interval: Duration::from_secs(1),
                     count: Some(2),
                     timeout: Duration::from_secs(3),
-                    commitment_config: CommitmentConfig::default(),
+                    commitment_config: CommitmentConfig::max(),
                 },
                 signers: vec![default_keypair.into()],
             }

cli/src/display.rs

@@ -1,11 +1,12 @@
 use crate::cli::SettingType;
 use console::style;
+use indicatif::{ProgressBar, ProgressStyle};
 use solana_sdk::{
     hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
     transaction::Transaction,
 };
 use solana_transaction_status::RpcTransactionStatusMeta;
-use std::fmt;
+use std::{fmt, io};
 
 // Pretty print a "name value"
 pub fn println_name_value(name: &str, value: &str) {
@@ -64,33 +65,44 @@ pub fn println_signers(
     println!();
 }
 
-pub fn println_transaction(
+pub fn write_transaction<W: io::Write>(
+    w: &mut W,
     transaction: &Transaction,
     transaction_status: &Option<RpcTransactionStatusMeta>,
     prefix: &str,
-) {
+) -> io::Result<()> {
     let message = &transaction.message;
-    println!("{}Recent Blockhash: {:?}", prefix, message.recent_blockhash);
+    writeln!(
+        w,
+        "{}Recent Blockhash: {:?}",
+        prefix, message.recent_blockhash
+    )?;
     for (signature_index, signature) in transaction.signatures.iter().enumerate() {
-        println!("{}Signature {}: {:?}", prefix, signature_index, signature);
+        writeln!(
+            w,
+            "{}Signature {}: {:?}",
+            prefix, signature_index, signature
+        )?;
     }
-    println!("{}{:?}", prefix, message.header);
+    writeln!(w, "{}{:?}", prefix, message.header)?;
     for (account_index, account) in message.account_keys.iter().enumerate() {
-        println!("{}Account {}: {:?}", prefix, account_index, account);
+        writeln!(w, "{}Account {}: {:?}", prefix, account_index, account)?;
     }
     for (instruction_index, instruction) in message.instructions.iter().enumerate() {
         let program_pubkey = message.account_keys[instruction.program_id_index as usize];
-        println!("{}Instruction {}", prefix, instruction_index);
-        println!(
+        writeln!(w, "{}Instruction {}", prefix, instruction_index)?;
+        writeln!(
+            w,
             "{}  Program: {} ({})",
             prefix, program_pubkey, instruction.program_id_index
-        );
+        )?;
         for (account_index, account) in instruction.accounts.iter().enumerate() {
             let account_pubkey = message.account_keys[*account as usize];
-            println!(
+            writeln!(
+                w,
                 "{}  Account {}: {} ({})",
                 prefix, account_index, account_pubkey, account
-            );
+            )?;
         }
 
         let mut raw = true;
@@ -99,7 +111,7 @@ pub fn println_transaction(
                 solana_vote_program::vote_instruction::VoteInstruction,
             >(&instruction.data)
             {
-                println!("{}  {:?}", prefix, vote_instruction);
+                writeln!(w, "{}  {:?}", prefix, vote_instruction)?;
                 raw = false;
             }
         } else if program_pubkey == solana_stake_program::id() {
@@ -107,7 +119,7 @@ pub fn println_transaction(
                 solana_stake_program::stake_instruction::StakeInstruction,
             >(&instruction.data)
             {
-                println!("{}  {:?}", prefix, stake_instruction);
+                writeln!(w, "{}  {:?}", prefix, stake_instruction)?;
                 raw = false;
             }
         } else if program_pubkey == solana_sdk::system_program::id() {
@@ -115,26 +127,32 @@ pub fn println_transaction(
                 solana_sdk::system_instruction::SystemInstruction,
             >(&instruction.data)
             {
-                println!("{}  {:?}", prefix, system_instruction);
+                writeln!(w, "{}  {:?}", prefix, system_instruction)?;
                 raw = false;
             }
         }
 
         if raw {
-            println!("{}  Data: {:?}", prefix, instruction.data);
+            writeln!(w, "{}  Data: {:?}", prefix, instruction.data)?;
         }
     }
 
     if let Some(transaction_status) = transaction_status {
-        println!(
+        writeln!(
+            w,
             "{}Status: {}",
             prefix,
             match &transaction_status.status {
                 Ok(_) => "Ok".into(),
                 Err(err) => err.to_string(),
             }
-        );
-        println!("{}  Fee: {}", prefix, transaction_status.fee);
+        )?;
+        writeln!(
+            w,
+            "{}  Fee: {} SOL",
+            prefix,
+            lamports_to_sol(transaction_status.fee)
+        )?;
         assert_eq!(
             transaction_status.pre_balances.len(),
             transaction_status.post_balances.len()
@@ -146,23 +164,49 @@ pub fn println_transaction(
             .enumerate()
         {
             if pre == post {
-                println!(
+                writeln!(
+                    w,
                     "{}  Account {} balance: {} SOL",
                     prefix,
                     i,
                     lamports_to_sol(*pre)
-                );
+                )?;
             } else {
-                println!(
+                writeln!(
+                    w,
                     "{}  Account {} balance: {} SOL -> {} SOL",
                     prefix,
                     i,
                     lamports_to_sol(*pre),
                     lamports_to_sol(*post)
-                );
+                )?;
             }
         }
     } else {
-        println!("{}Status: Unavailable", prefix);
+        writeln!(w, "{}Status: Unavailable", prefix)?;
+    }
+
+    Ok(())
+}
+
+pub fn println_transaction(
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) {
+    let mut w = Vec::new();
+    if write_transaction(&mut w, transaction, transaction_status, prefix).is_ok() {
+        if let Ok(s) = String::from_utf8(w) {
+            print!("{}", s);
+        }
     }
 }
+
+/// Creates a new process bar for processing that will take an unknown amount of time
+pub fn new_spinner_progress_bar() -> ProgressBar {
+    let progress_bar = ProgressBar::new(42);
+    progress_bar
+        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
+    progress_bar.enable_steady_tick(100);
+    progress_bar
+}

cli/src/main.rs

@@ -2,8 +2,7 @@ use clap::{crate_description, crate_name, AppSettings, Arg, ArgGroup, ArgMatches
 use console::style;
 
 use solana_clap_utils::{
-    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, offline::SIGN_ONLY_ARG,
-    DisplayError,
+    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, DisplayError,
 };
 use solana_cli::{
     cli::{app, parse_command, process_command, CliCommandInfo, CliConfig, CliSigners},
@@ -262,13 +261,7 @@ fn do_main(matches: &ArgMatches<'_>) -> Result<(), Box<dyn error::Error>> {
         let (mut config, signers) = parse_args(&matches, &mut wallet_manager)?;
         config.signers = signers.iter().map(|s| s.as_ref()).collect();
         let result = process_command(&config)?;
-        let (_, submatches) = matches.subcommand();
-        let sign_only = submatches
-            .map(|m| m.is_present(SIGN_ONLY_ARG.name))
-            .unwrap_or(false);
-        if !sign_only {
-            println!("{}", result);
-        }
+        println!("{}", result);
     };
     Ok(())
 }

cli/src/nonce.rs

@@ -462,8 +462,8 @@ pub fn process_authorize_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-    log_instruction_custom_error::<NonceError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<NonceError>(result, &config)
 }
 
 pub fn process_create_nonce_account(
@@ -539,8 +539,8 @@ pub fn process_create_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-    log_instruction_custom_error::<SystemError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<SystemError>(result, &config)
 }
 
 pub fn process_get_nonce(rpc_client: &RpcClient, nonce_account_pubkey: &Pubkey) -> ProcessResult {
@@ -580,9 +580,8 @@ pub fn process_new_nonce(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client
-        .send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0], nonce_authority]);
-    log_instruction_custom_error::<SystemError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<SystemError>(result, &config)
 }
 
 pub fn process_show_nonce_account(
@@ -606,8 +605,7 @@ pub fn process_show_nonce_account(
             nonce_account.authority = Some(data.authority.to_string());
         }
 
-        config.output_format.formatted_print(&nonce_account);
-        Ok("".to_string())
+        Ok(config.output_format.formatted_string(&nonce_account))
     };
     match state_from_account(&nonce_account)? {
         State::Uninitialized => print_account(None),
@@ -641,8 +639,8 @@ pub fn process_withdraw_from_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-    log_instruction_custom_error::<NonceError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<NonceError>(result, &config)
 }
 
 #[cfg(test)]

cli/src/offline/mod.rs

@@ -79,32 +79,47 @@ pub fn parse_sign_only_reply_string(reply: &str) -> SignOnly {
     let object: Value = serde_json::from_str(&reply).unwrap();
     let blockhash_str = object.get("blockhash").unwrap().as_str().unwrap();
     let blockhash = blockhash_str.parse::<Hash>().unwrap();
-    let signer_strings = object.get("signers").unwrap().as_array().unwrap();
-    let present_signers = signer_strings
-        .iter()
-        .map(|signer_string| {
-            let mut signer = signer_string.as_str().unwrap().split('=');
-            let key = Pubkey::from_str(signer.next().unwrap()).unwrap();
-            let sig = Signature::from_str(signer.next().unwrap()).unwrap();
-            (key, sig)
-        })
-        .collect();
-    let signer_strings = object.get("absent").unwrap().as_array().unwrap();
-    let absent_signers = signer_strings
-        .iter()
-        .map(|val| {
-            let s = val.as_str().unwrap();
-            Pubkey::from_str(s).unwrap()
-        })
-        .collect();
-    let signer_strings = object.get("badSig").unwrap().as_array().unwrap();
-    let bad_signers = signer_strings
-        .iter()
-        .map(|val| {
-            let s = val.as_str().unwrap();
-            Pubkey::from_str(s).unwrap()
-        })
-        .collect();
+    let mut present_signers: Vec<(Pubkey, Signature)> = Vec::new();
+    let signer_strings = object.get("signers");
+    if let Some(sig_strings) = signer_strings {
+        present_signers = sig_strings
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|signer_string| {
+                let mut signer = signer_string.as_str().unwrap().split('=');
+                let key = Pubkey::from_str(signer.next().unwrap()).unwrap();
+                let sig = Signature::from_str(signer.next().unwrap()).unwrap();
+                (key, sig)
+            })
+            .collect();
+    }
+    let mut absent_signers: Vec<Pubkey> = Vec::new();
+    let signer_strings = object.get("absent");
+    if let Some(sig_strings) = signer_strings {
+        absent_signers = sig_strings
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|val| {
+                let s = val.as_str().unwrap();
+                Pubkey::from_str(s).unwrap()
+            })
+            .collect();
+    }
+    let mut bad_signers: Vec<Pubkey> = Vec::new();
+    let signer_strings = object.get("badSig");
+    if let Some(sig_strings) = signer_strings {
+        bad_signers = sig_strings
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|val| {
+                let s = val.as_str().unwrap();
+                Pubkey::from_str(s).unwrap()
+            })
+            .collect();
+    }
     SignOnly {
         blockhash,
         present_signers,

cli/src/stake.rs

@@ -313,6 +313,14 @@ impl StakeSubCommands for App<'_, '_> {
                 .arg(nonce_arg())
                 .arg(nonce_authority_arg())
                 .arg(fee_payer_arg())
+                .arg(
+                    Arg::with_name("custodian")
+                        .long("custodian")
+                        .takes_value(true)
+                        .value_name("KEYPAIR")
+                        .validator(is_valid_signer)
+                        .help("Authority to override account lockup")
+                )
         )
         .subcommand(
             SubCommand::with_name("stake-set-lockup")
@@ -676,11 +684,15 @@ pub fn parse_stake_withdraw_stake(
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
     let (fee_payer, fee_payer_pubkey) = signer_of(matches, FEE_PAYER_ARG.name, wallet_manager)?;
+    let (custodian, custodian_pubkey) = signer_of(matches, "custodian", wallet_manager)?;
 
     let mut bulk_signers = vec![withdraw_authority, fee_payer];
     if nonce_account.is_some() {
         bulk_signers.push(nonce_authority);
     }
+    if custodian.is_some() {
+        bulk_signers.push(custodian);
+    }
     let signer_info =
         generate_unique_signers(bulk_signers, matches, default_signer_path, wallet_manager)?;
 
@@ -695,6 +707,7 @@ pub fn parse_stake_withdraw_stake(
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
             fee_payer: signer_info.index_of(fee_payer_pubkey).unwrap(),
+            custodian: custodian_pubkey.and_then(|_| signer_info.index_of(custodian_pubkey)),
         },
         signers: signer_info.signers,
     })
@@ -868,7 +881,7 @@ pub fn process_create_stake_account(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -881,8 +894,8 @@ pub fn process_create_stake_account(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-        log_instruction_custom_error::<SystemError>(result)
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<SystemError>(result, &config)
     }
 }
 
@@ -933,7 +946,7 @@ pub fn process_stake_authorize(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -946,8 +959,8 @@ pub fn process_stake_authorize(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-        log_instruction_custom_error::<StakeError>(result)
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
 
@@ -987,7 +1000,7 @@ pub fn process_deactivate_stake_account(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -1000,8 +1013,8 @@ pub fn process_deactivate_stake_account(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-        log_instruction_custom_error::<StakeError>(result)
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
 
@@ -1013,6 +1026,7 @@ pub fn process_withdraw_stake(
     destination_account_pubkey: &Pubkey,
     lamports: u64,
     withdraw_authority: SignerIndex,
+    custodian: Option<SignerIndex>,
     sign_only: bool,
     blockhash_query: &BlockhashQuery,
     nonce_account: Option<&Pubkey>,
@@ -1022,12 +1036,14 @@ pub fn process_withdraw_stake(
     let (recent_blockhash, fee_calculator) =
         blockhash_query.get_blockhash_and_fee_calculator(rpc_client)?;
     let withdraw_authority = config.signers[withdraw_authority];
+    let custodian = custodian.map(|index| config.signers[index]);
 
     let ixs = vec![stake_instruction::withdraw(
         stake_account_pubkey,
         &withdraw_authority.pubkey(),
         destination_account_pubkey,
         lamports,
+        custodian.map(|signer| signer.pubkey()).as_ref(),
     )];
 
     let fee_payer = config.signers[fee_payer];
@@ -1047,7 +1063,7 @@ pub fn process_withdraw_stake(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -1060,8 +1076,8 @@ pub fn process_withdraw_stake(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-        log_instruction_custom_error::<SystemError>(result)
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<SystemError>(result, &config)
     }
 }
 
@@ -1181,7 +1197,7 @@ pub fn process_split_stake(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -1194,8 +1210,8 @@ pub fn process_split_stake(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-        log_instruction_custom_error::<StakeError>(result)
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
 
@@ -1238,7 +1254,7 @@ pub fn process_stake_set_lockup(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -1251,8 +1267,8 @@ pub fn process_stake_set_lockup(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-        log_instruction_custom_error::<StakeError>(result)
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
 
@@ -1325,8 +1341,7 @@ pub fn process_show_stake_account(
     match stake_account.state() {
         Ok(stake_state) => {
             let state = build_stake_state(stake_account.lamports, &stake_state, use_lamports_unit);
-            config.output_format.formatted_print(&state);
-            Ok("".to_string())
+            Ok(config.output_format.formatted_string(&state))
         }
         Err(err) => Err(CliError::RpcRequestError(format!(
             "Account data could not be deserialized to stake state: {}",
@@ -1354,8 +1369,7 @@ pub fn process_show_stake_history(
         entries,
         use_lamports_unit,
     };
-    config.output_format.formatted_print(&stake_history_output);
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&stake_history_output))
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -1448,7 +1462,7 @@ pub fn process_delegate_stake(
 
     if sign_only {
         tx.try_partial_sign(&config.signers, recent_blockhash)?;
-        return_signers(&tx)
+        return_signers(&tx, &config)
     } else {
         tx.try_sign(&config.signers, recent_blockhash)?;
         if let Some(nonce_account) = &nonce_account {
@@ -1461,8 +1475,8 @@ pub fn process_delegate_stake(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-        log_instruction_custom_error::<StakeError>(result)
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<StakeError>(result, &config)
     }
 }
 
@@ -1496,6 +1510,9 @@ mod tests {
         let (stake_authority_keypair_file, mut tmp_file) = make_tmp_file();
         let stake_authority_keypair = Keypair::new();
         write_keypair(&stake_authority_keypair, tmp_file.as_file_mut()).unwrap();
+        let (custodian_keypair_file, mut tmp_file) = make_tmp_file();
+        let custodian_keypair = Keypair::new();
+        write_keypair(&custodian_keypair, tmp_file.as_file_mut()).unwrap();
 
         // stake-authorize subcommand
         let stake_account_string = stake_account_pubkey.to_string();
@@ -2434,6 +2451,7 @@ mod tests {
                     destination_account_pubkey: stake_account_pubkey,
                     lamports: 42_000_000_000,
                     withdraw_authority: 0,
+                    custodian: None,
                     sign_only: false,
                     blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
                     nonce_account: None,
@@ -2463,6 +2481,7 @@ mod tests {
                     destination_account_pubkey: stake_account_pubkey,
                     lamports: 42_000_000_000,
                     withdraw_authority: 1,
+                    custodian: None,
                     sign_only: false,
                     blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
                     nonce_account: None,
@@ -2478,6 +2497,39 @@ mod tests {
             }
         );
 
+        // Test WithdrawStake Subcommand w/ custodian
+        let test_withdraw_stake = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-stake",
+            &stake_account_string,
+            &stake_account_string,
+            "42",
+            "--custodian",
+            &custodian_keypair_file,
+        ]);
+
+        assert_eq!(
+            parse_command(&test_withdraw_stake, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::WithdrawStake {
+                    stake_account_pubkey,
+                    destination_account_pubkey: stake_account_pubkey,
+                    lamports: 42_000_000_000,
+                    withdraw_authority: 0,
+                    custodian: Some(1),
+                    sign_only: false,
+                    blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+                    nonce_account: None,
+                    nonce_authority: 0,
+                    fee_payer: 0,
+                },
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    read_keypair_file(&custodian_keypair_file).unwrap().into()
+                ],
+            }
+        );
+
         // Test WithdrawStake Subcommand w/ authority and offline nonce
         let test_withdraw_stake = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2507,6 +2559,7 @@ mod tests {
                     destination_account_pubkey: stake_account_pubkey,
                     lamports: 42_000_000_000,
                     withdraw_authority: 0,
+                    custodian: None,
                     sign_only: false,
                     blockhash_query: BlockhashQuery::FeeCalculator(
                         blockhash_query::Source::NonceAccount(nonce_account),

cli/src/storage.rs

@@ -242,8 +242,8 @@ pub fn process_create_storage_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-    log_instruction_custom_error::<SystemError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<SystemError>(result, &config)
 }
 
 pub fn process_claim_storage_reward(
@@ -266,7 +266,7 @@ pub fn process_claim_storage_reward(
         &fee_calculator,
         &tx.message,
     )?;
-    let signature = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
+    let signature = rpc_client.send_and_confirm_transaction_with_spinner(&tx)?;
     Ok(signature.to_string())
 }
 

cli/src/validator_info.rs

@@ -367,7 +367,7 @@ pub fn process_set_validator_info(
         &fee_calculator,
         &tx.message,
     )?;
-    let signature_str = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
+    let signature_str = rpc_client.send_and_confirm_transaction_with_spinner(&tx)?;
 
     println!("Success! Validator info published at: {:?}", info_pubkey);
     println!("{}", signature_str);
@@ -410,10 +410,9 @@ pub fn process_get_validator_info(
             info: validator_info,
         });
     }
-    config
+    Ok(config
         .output_format
-        .formatted_print(&CliValidatorInfoVec::new(validator_info_list));
-    Ok("".to_string())
+        .formatted_string(&CliValidatorInfoVec::new(validator_info_list)))
 }
 
 #[cfg(test)]

cli/src/vote.rs

@@ -7,7 +7,11 @@ use crate::{
     cli_output::{CliEpochVotingHistory, CliLockout, CliVoteAccount},
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
-use solana_clap_utils::{input_parsers::*, input_validators::*};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+};
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
@@ -174,14 +178,6 @@ impl VoteSubCommands for App<'_, '_> {
             SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
                 .alias("show-vote-account")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
@@ -196,7 +192,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("withdraw-from-vote-account")
@@ -338,11 +335,7 @@ pub fn parse_vote_get_account_command(
     let vote_account_pubkey =
         pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ShowVoteAccount {
             pubkey: vote_account_pubkey,
@@ -464,8 +457,8 @@ pub fn process_create_vote_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-    log_instruction_custom_error::<SystemError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<SystemError>(result, &config)
 }
 
 pub fn process_vote_authorize(
@@ -504,9 +497,8 @@ pub fn process_vote_authorize(
         &fee_calculator,
         &tx.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0]]);
-    log_instruction_custom_error::<VoteError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<VoteError>(result, &config)
 }
 
 pub fn process_vote_update_validator(
@@ -538,8 +530,8 @@ pub fn process_vote_update_validator(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
-    log_instruction_custom_error::<VoteError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<VoteError>(result, &config)
 }
 
 fn get_vote_account(
@@ -613,8 +605,7 @@ pub fn process_show_vote_account(
         use_lamports_unit,
     };
 
-    config.output_format.formatted_print(&vote_account_data);
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&vote_account_data))
 }
 
 pub fn process_withdraw_from_vote_account(
@@ -644,9 +635,8 @@ pub fn process_withdraw_from_vote_account(
         &fee_calculator,
         &transaction.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction_with_spinner(&mut transaction, &config.signers);
-    log_instruction_custom_error::<VoteError>(result)
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&transaction);
+    log_instruction_custom_error::<VoteError>(result, &config)
 }
 
 #[cfg(test)]

cli/tests/nonce.rs

@@ -1,5 +1,6 @@
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
+    cli_output::OutputFormat,
     nonce,
     offline::{
         blockhash_query::{self, BlockhashQuery},
@@ -119,6 +120,7 @@ fn full_battery_tests(
         &faucet_addr,
         &config_payer.signers[0].pubkey(),
         2000,
+        &config_payer,
     )
     .unwrap();
     check_balance(2000, &rpc_client, &config_payer.signers[0].pubkey());
@@ -275,6 +277,7 @@ fn test_create_account_with_seed() {
     let offline_nonce_authority_signer = keypair_from_seed(&[1u8; 32]).unwrap();
     let online_nonce_creator_signer = keypair_from_seed(&[2u8; 32]).unwrap();
     let to_address = Pubkey::new(&[3u8; 32]);
+    let config = CliConfig::default();
 
     // Setup accounts
     let rpc_client = RpcClient::new_socket(leader_data.rpc);
@@ -283,6 +286,7 @@ fn test_create_account_with_seed() {
         &faucet_addr,
         &offline_nonce_authority_signer.pubkey(),
         42,
+        &config,
     )
     .unwrap();
     request_and_confirm_airdrop(
@@ -290,6 +294,7 @@ fn test_create_account_with_seed() {
         &faucet_addr,
         &online_nonce_creator_signer.pubkey(),
         4242,
+        &config,
     )
     .unwrap();
     check_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
@@ -344,6 +349,7 @@ fn test_create_account_with_seed() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    authority_config.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&authority_config).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_only_reply);
     let authority_presigner = sign_only.presigner_of(&authority_pubkey).unwrap();

cli/tests/pay.rs

@@ -2,6 +2,7 @@ use chrono::prelude::*;
 use serde_json::Value;
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig, PayCommand},
+    cli_output::OutputFormat,
     nonce,
     offline::{
         blockhash_query::{self, BlockhashQuery},
@@ -69,6 +70,7 @@ fn test_cli_timestamp_tx() {
         &faucet_addr,
         &config_payer.signers[0].pubkey(),
         50,
+        &config_witness,
     )
     .unwrap();
     check_balance(50, &rpc_client, &config_payer.signers[0].pubkey());
@@ -78,6 +80,7 @@ fn test_cli_timestamp_tx() {
         &faucet_addr,
         &config_witness.signers[0].pubkey(),
         1,
+        &config_witness,
     )
     .unwrap();
 
@@ -154,6 +157,7 @@ fn test_cli_witness_tx() {
         &faucet_addr,
         &config_payer.signers[0].pubkey(),
         50,
+        &config_witness,
     )
     .unwrap();
     request_and_confirm_airdrop(
@@ -161,6 +165,7 @@ fn test_cli_witness_tx() {
         &faucet_addr,
         &config_witness.signers[0].pubkey(),
         1,
+        &config_witness,
     )
     .unwrap();
 
@@ -234,6 +239,7 @@ fn test_cli_cancel_tx() {
         &faucet_addr,
         &config_payer.signers[0].pubkey(),
         50,
+        &config_witness,
     )
     .unwrap();
 
@@ -307,6 +313,7 @@ fn test_offline_pay_tx() {
         &faucet_addr,
         &config_offline.signers[0].pubkey(),
         50,
+        &config_offline,
     )
     .unwrap();
 
@@ -315,6 +322,7 @@ fn test_offline_pay_tx() {
         &faucet_addr,
         &config_online.signers[0].pubkey(),
         50,
+        &config_offline,
     )
     .unwrap();
     check_balance(50, &rpc_client, &config_offline.signers[0].pubkey());
@@ -328,6 +336,7 @@ fn test_offline_pay_tx() {
         sign_only: true,
         ..PayCommand::default()
     });
+    config_offline.output_format = OutputFormat::JsonCompact;
     let sig_response = process_command(&config_offline).unwrap();
 
     check_balance(50, &rpc_client, &config_offline.signers[0].pubkey());
@@ -388,6 +397,7 @@ fn test_nonced_pay_tx() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         50 + minimum_nonce_balance,
+        &config,
     )
     .unwrap();
     check_balance(

cli/tests/stake.rs

@@ -1,5 +1,6 @@
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
+    cli_output::OutputFormat,
     nonce,
     offline::{
         blockhash_query::{self, BlockhashQuery},
@@ -59,6 +60,7 @@ fn test_stake_delegation_force() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         100_000,
+        &config,
     )
     .unwrap();
 
@@ -155,6 +157,7 @@ fn test_seed_stake_delegation_and_deactivation() {
         &faucet_addr,
         &config_validator.signers[0].pubkey(),
         100_000,
+        &config_validator,
     )
     .unwrap();
     check_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
@@ -245,6 +248,7 @@ fn test_stake_delegation_and_deactivation() {
         &faucet_addr,
         &config_validator.signers[0].pubkey(),
         100_000,
+        &config_validator,
     )
     .unwrap();
     check_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
@@ -341,6 +345,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         &faucet_addr,
         &config_validator.signers[0].pubkey(),
         100_000,
+        &config_offline,
     )
     .unwrap();
     check_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
@@ -350,6 +355,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         &faucet_addr,
         &config_offline.signers[0].pubkey(),
         100_000,
+        &config_validator,
     )
     .unwrap();
     check_balance(100_000, &rpc_client, &config_offline.signers[0].pubkey());
@@ -385,6 +391,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    config_offline.output_format = OutputFormat::JsonCompact;
     let sig_response = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sig_response);
     assert!(sign_only.has_all_signers());
@@ -470,6 +477,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         100_000,
+        &config,
     )
     .unwrap();
 
@@ -579,6 +587,7 @@ fn test_stake_authorize() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         100_000,
+        &config,
     )
     .unwrap();
 
@@ -596,6 +605,7 @@ fn test_stake_authorize() {
         &faucet_addr,
         &config_offline.signers[0].pubkey(),
         100_000,
+        &config,
     )
     .unwrap();
 
@@ -703,6 +713,7 @@ fn test_stake_authorize() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    config_offline.output_format = OutputFormat::JsonCompact;
     let sign_reply = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_reply);
     assert!(sign_only.has_all_signers());
@@ -841,13 +852,16 @@ fn test_stake_authorize_with_fee_payer() {
     config_offline.command = CliCommand::ClusterVersion;
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &default_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &default_pubkey, 100_000, &config)
+        .unwrap();
     check_balance(100_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &payer_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &payer_pubkey, 100_000, &config)
+        .unwrap();
     check_balance(100_000, &rpc_client, &payer_pubkey);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
+        .unwrap();
     check_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
@@ -901,6 +915,7 @@ fn test_stake_authorize_with_fee_payer() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    config_offline.output_format = OutputFormat::JsonCompact;
     let sign_reply = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_reply);
     assert!(sign_only.has_all_signers());
@@ -966,11 +981,13 @@ fn test_stake_split() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         500_000,
+        &config,
     )
     .unwrap();
     check_balance(500_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
+        .unwrap();
     check_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
@@ -1038,6 +1055,7 @@ fn test_stake_split() {
         lamports: 2 * minimum_stake_balance,
         fee_payer: 0,
     };
+    config_offline.output_format = OutputFormat::JsonCompact;
     let sig_response = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sig_response);
     assert!(sign_only.has_all_signers());
@@ -1114,11 +1132,13 @@ fn test_stake_set_lockup() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         500_000,
+        &config,
     )
     .unwrap();
     check_balance(500_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
+        .unwrap();
     check_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
@@ -1292,6 +1312,7 @@ fn test_stake_set_lockup() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    config_offline.output_format = OutputFormat::JsonCompact;
     let sig_response = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sig_response);
     assert!(sign_only.has_all_signers());
@@ -1364,11 +1385,13 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         200_000,
+        &config,
     )
     .unwrap();
     check_balance(200_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
+        .unwrap();
     check_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create nonce account
@@ -1410,6 +1433,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         fee_payer: 0,
         from: 0,
     };
+    config_offline.output_format = OutputFormat::JsonCompact;
     let sig_response = process_command(&config_offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sig_response);
     assert!(sign_only.has_all_signers());
@@ -1451,6 +1475,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         destination_account_pubkey: recipient_pubkey,
         lamports: 42,
         withdraw_authority: 0,
+        custodian: None,
         sign_only: true,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_pubkey),
@@ -1466,6 +1491,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         destination_account_pubkey: recipient_pubkey,
         lamports: 42,
         withdraw_authority: 0,
+        custodian: None,
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_pubkey),

cli/tests/transfer.rs

@@ -1,5 +1,6 @@
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
+    cli_output::OutputFormat,
     nonce,
     offline::{
         blockhash_query::{self, BlockhashQuery},
@@ -59,7 +60,8 @@ fn test_transfer() {
     let sender_pubkey = config.signers[0].pubkey();
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 50_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 50_000, &config)
+        .unwrap();
     check_balance(50_000, &rpc_client, &sender_pubkey);
     check_balance(0, &rpc_client, &recipient_pubkey);
 
@@ -87,7 +89,7 @@ fn test_transfer() {
     process_command(&offline).unwrap_err();
 
     let offline_pubkey = offline.signers[0].pubkey();
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 50).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 50, &config).unwrap();
     check_balance(50, &rpc_client, &offline_pubkey);
 
     // Offline transfer
@@ -103,6 +105,7 @@ fn test_transfer() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    offline.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&offline).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_only_reply);
     assert!(sign_only.has_all_signers());
@@ -247,16 +250,24 @@ fn test_transfer_multisession_signing() {
     let offline_from_signer = keypair_from_seed(&[2u8; 32]).unwrap();
     let offline_fee_payer_signer = keypair_from_seed(&[3u8; 32]).unwrap();
     let from_null_signer = NullSigner::new(&offline_from_signer.pubkey());
+    let config = CliConfig::default();
 
     // Setup accounts
     let rpc_client = RpcClient::new_socket(leader_data.rpc);
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_from_signer.pubkey(), 43)
-        .unwrap();
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &offline_from_signer.pubkey(),
+        43,
+        &config,
+    )
+    .unwrap();
     request_and_confirm_airdrop(
         &rpc_client,
         &faucet_addr,
         &offline_fee_payer_signer.pubkey(),
         3,
+        &config,
     )
     .unwrap();
     check_balance(43, &rpc_client, &offline_from_signer.pubkey());
@@ -283,6 +294,7 @@ fn test_transfer_multisession_signing() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    fee_payer_config.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&fee_payer_config).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_only_reply);
     assert!(!sign_only.has_all_signers());
@@ -308,6 +320,7 @@ fn test_transfer_multisession_signing() {
         nonce_authority: 0,
         fee_payer: 0,
     };
+    from_config.output_format = OutputFormat::JsonCompact;
     let sign_only_reply = process_command(&from_config).unwrap();
     let sign_only = parse_sign_only_reply_string(&sign_only_reply);
     assert!(sign_only.has_all_signers());

cli/tests/vote.rs

@@ -48,6 +48,7 @@ fn test_vote_authorize_and_withdraw() {
         &faucet_addr,
         &config.signers[0].pubkey(),
         100_000,
+        &config,
     )
     .unwrap();
 

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.1.5"
+version = "1.1.17"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,10 +19,10 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.105"
 serde_derive = "1.0.103"
 serde_json = "1.0.48"
-solana-transaction-status = { path = "../transaction-status", version = "1.1.5" }
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.1.5" }
+solana-transaction-status = { path = "../transaction-status", version = "1.1.17" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.17" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -31,7 +31,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-http-server = "14.0.6"
-solana-logger = { path = "../logger", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.17" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/client_error.rs

@@ -50,29 +50,29 @@ impl Into<TransportError> for ClientErrorKind {
 #[derive(Error, Debug)]
 #[error("{kind}")]
 pub struct ClientError {
-    command: Option<&'static str>,
+    request: Option<rpc_request::RpcRequest>,
+
     #[source]
-    #[error(transparent)]
     kind: ClientErrorKind,
 }
 
 impl ClientError {
-    pub fn new_with_command(kind: ClientErrorKind, command: &'static str) -> Self {
+    pub fn new_with_request(kind: ClientErrorKind, request: rpc_request::RpcRequest) -> Self {
         Self {
-            command: Some(command),
+            request: Some(request),
             kind,
         }
     }
 
-    pub fn into_with_command(self, command: &'static str) -> Self {
+    pub fn into_with_request(self, request: rpc_request::RpcRequest) -> Self {
         Self {
-            command: Some(command),
+            request: Some(request),
             ..self
         }
     }
 
-    pub fn command(&self) -> Option<&'static str> {
-        self.command
+    pub fn request(&self) -> Option<&rpc_request::RpcRequest> {
+        self.request.as_ref()
     }
 
     pub fn kind(&self) -> &ClientErrorKind {
@@ -83,7 +83,7 @@ impl ClientError {
 impl From<ClientErrorKind> for ClientError {
     fn from(kind: ClientErrorKind) -> Self {
         Self {
-            command: None,
+            request: None,
             kind,
         }
     }
@@ -92,7 +92,7 @@ impl From<ClientErrorKind> for ClientError {
 impl From<TransportError> for ClientError {
     fn from(err: TransportError) -> Self {
         Self {
-            command: None,
+            request: None,
             kind: err.into(),
         }
     }
@@ -107,7 +107,7 @@ impl Into<TransportError> for ClientError {
 impl From<std::io::Error> for ClientError {
     fn from(err: std::io::Error) -> Self {
         Self {
-            command: None,
+            request: None,
             kind: err.into(),
         }
     }
@@ -116,7 +116,7 @@ impl From<std::io::Error> for ClientError {
 impl From<reqwest::Error> for ClientError {
     fn from(err: reqwest::Error) -> Self {
         Self {
-            command: None,
+            request: None,
             kind: err.into(),
         }
     }
@@ -125,7 +125,7 @@ impl From<reqwest::Error> for ClientError {
 impl From<rpc_request::RpcError> for ClientError {
     fn from(err: rpc_request::RpcError) -> Self {
         Self {
-            command: None,
+            request: None,
             kind: err.into(),
         }
     }
@@ -134,7 +134,7 @@ impl From<rpc_request::RpcError> for ClientError {
 impl From<serde_json::error::Error> for ClientError {
     fn from(err: serde_json::error::Error) -> Self {
         Self {
-            command: None,
+            request: None,
             kind: err.into(),
         }
     }
@@ -143,7 +143,7 @@ impl From<serde_json::error::Error> for ClientError {
 impl From<SignerError> for ClientError {
     fn from(err: SignerError) -> Self {
         Self {
-            command: None,
+            request: None,
             kind: err.into(),
         }
     }
@@ -152,7 +152,7 @@ impl From<SignerError> for ClientError {
 impl From<TransactionError> for ClientError {
     fn from(err: TransactionError) -> Self {
         Self {
-            command: None,
+            request: None,
             kind: err.into(),
         }
     }

client/src/generic_rpc_client_request.rs

@@ -1,10 +1,5 @@
 use crate::{client_error::Result, rpc_request::RpcRequest};
 
 pub(crate) trait GenericRpcClientRequest {
-    fn send(
-        &self,
-        request: &RpcRequest,
-        params: serde_json::Value,
-        retries: usize,
-    ) -> Result<serde_json::Value>;
+    fn send(&self, request: RpcRequest, params: serde_json::Value) -> Result<serde_json::Value>;
 }

client/src/lib.rs

@@ -8,6 +8,7 @@ pub mod perf_utils;
 pub mod pubsub_client;
 pub mod rpc_client;
 pub mod rpc_client_request;
+pub mod rpc_config;
 pub mod rpc_request;
 pub mod rpc_response;
 pub mod thin_client;

client/src/mock_rpc_client_request.rs

@@ -38,13 +38,8 @@ impl MockRpcClientRequest {
 }
 
 impl GenericRpcClientRequest for MockRpcClientRequest {
-    fn send(
-        &self,
-        request: &RpcRequest,
-        params: serde_json::Value,
-        _retries: usize,
-    ) -> Result<serde_json::Value> {
-        if let Some(value) = self.mocks.write().unwrap().remove(request) {
+    fn send(&self, request: RpcRequest, params: serde_json::Value) -> Result<serde_json::Value> {
+        if let Some(value) = self.mocks.write().unwrap().remove(&request) {
             return Ok(value);
         }
         if self.url == "fails" {

client/src/rpc_client_request.rs

@@ -4,8 +4,7 @@ use crate::{
     rpc_request::{RpcError, RpcRequest},
 };
 use log::*;
-use reqwest::{self, header::CONTENT_TYPE};
-use solana_sdk::clock::{DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT};
+use reqwest::{self, header::CONTENT_TYPE, StatusCode};
 use std::{thread::sleep, time::Duration};
 
 pub struct RpcClientRequest {
@@ -29,17 +28,13 @@ impl RpcClientRequest {
 }
 
 impl GenericRpcClientRequest for RpcClientRequest {
-    fn send(
-        &self,
-        request: &RpcRequest,
-        params: serde_json::Value,
-        mut retries: usize,
-    ) -> Result<serde_json::Value> {
+    fn send(&self, request: RpcRequest, params: serde_json::Value) -> Result<serde_json::Value> {
         // Concurrent requests are not supported so reuse the same request id for all requests
         let request_id = 1;
 
         let request_json = request.build_request_json(request_id, params);
 
+        let mut too_many_requests_retries = 5;
         loop {
             match self
                 .client
@@ -50,6 +45,19 @@ impl GenericRpcClientRequest for RpcClientRequest {
             {
                 Ok(response) => {
                     if !response.status().is_success() {
+                        if response.status() == StatusCode::TOO_MANY_REQUESTS
+                            && too_many_requests_retries > 0
+                        {
+                            too_many_requests_retries -= 1;
+                            debug!(
+                                "Server responded with {:?}, {} retries left",
+                                response, too_many_requests_retries
+                            );
+
+                            // Sleep for 500ms to give the server a break
+                            sleep(Duration::from_millis(500));
+                            continue;
+                        }
                         return Err(response.error_for_status().unwrap_err().into());
                     }
 
@@ -63,17 +71,8 @@ impl GenericRpcClientRequest for RpcClientRequest {
                     }
                     return Ok(json["result"].clone());
                 }
-                Err(e) => {
-                    info!("{:?} failed, {} retries left: {:?}", request, retries, e);
-                    if retries == 0 {
-                        return Err(e.into());
-                    }
-                    retries -= 1;
-
-                    // Sleep for approximately half a slot
-                    sleep(Duration::from_millis(
-                        500 * DEFAULT_TICKS_PER_SLOT / DEFAULT_TICKS_PER_SECOND,
-                    ));
+                Err(err) => {
+                    return Err(err.into());
                 }
             }
         }

client/src/rpc_config.rs

@@ -0,0 +1,45 @@
+use solana_sdk::{clock::Epoch, commitment_config::CommitmentConfig};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSignatureStatusConfig {
+    pub search_transaction_history: Option<bool>,
+    // DEPRECATED
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSendTransactionConfig {
+    pub skip_preflight: bool,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSimulateTransactionConfig {
+    pub sig_verify: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum RpcLargestAccountsFilter {
+    Circulating,
+    NonCirculating,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcLargestAccountsConfig {
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+    pub filter: Option<RpcLargestAccountsFilter>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationConfig {
+    pub epoch: Option<Epoch>,
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}

client/src/rpc_request.rs

@@ -1,7 +1,8 @@
 use serde_json::{json, Value};
+use std::fmt;
 use thiserror::Error;
 
-#[derive(Debug, PartialEq, Eq, Hash)]
+#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
 pub enum RpcRequest {
     DeregisterNode,
     ValidatorExit,
@@ -15,14 +16,18 @@ pub enum RpcRequest {
     GetConfirmedTransaction,
     GetEpochInfo,
     GetEpochSchedule,
-    GetGenesisHash,
-    GetIdentity,
-    GetInflation,
-    GetLeaderSchedule,
-    GetProgramAccounts,
-    GetRecentBlockhash,
     GetFeeCalculatorForBlockhash,
     GetFeeRateGovernor,
+    GetFees,
+    GetGenesisHash,
+    GetIdentity,
+    GetInflationGovernor,
+    GetInflationRate,
+    GetLargestAccounts,
+    GetLeaderSchedule,
+    GetMinimumBalanceForRentExemption,
+    GetProgramAccounts,
+    GetRecentBlockhash,
     GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
@@ -30,21 +35,21 @@ pub enum RpcRequest {
     GetStorageTurnRate,
     GetSlotsPerSegment,
     GetStoragePubkeysForSlot,
+    GetSupply,
     GetTotalSupply,
     GetTransactionCount,
     GetVersion,
     GetVoteAccounts,
+    MinimumLedgerSlot,
     RegisterNode,
     RequestAirdrop,
     SendTransaction,
+    SimulateTransaction,
     SignVote,
-    GetMinimumBalanceForRentExemption,
-    MinimumLedgerSlot,
 }
 
-impl RpcRequest {
-    pub(crate) fn build_request_json(&self, id: u64, params: Value) -> Value {
-        let jsonrpc = "2.0";
+impl fmt::Display for RpcRequest {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let method = match self {
             RpcRequest::DeregisterNode => "deregisterNode",
             RpcRequest::ValidatorExit => "validatorExit",
@@ -58,14 +63,18 @@ impl RpcRequest {
             RpcRequest::GetConfirmedTransaction => "getConfirmedTransaction",
             RpcRequest::GetEpochInfo => "getEpochInfo",
             RpcRequest::GetEpochSchedule => "getEpochSchedule",
-            RpcRequest::GetGenesisHash => "getGenesisHash",
-            RpcRequest::GetIdentity => "getIdentity",
-            RpcRequest::GetInflation => "getInflation",
-            RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
-            RpcRequest::GetProgramAccounts => "getProgramAccounts",
-            RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetFeeCalculatorForBlockhash => "getFeeCalculatorForBlockhash",
             RpcRequest::GetFeeRateGovernor => "getFeeRateGovernor",
+            RpcRequest::GetFees => "getFees",
+            RpcRequest::GetGenesisHash => "getGenesisHash",
+            RpcRequest::GetIdentity => "getIdentity",
+            RpcRequest::GetInflationGovernor => "getInflationGovernor",
+            RpcRequest::GetInflationRate => "getInflationRate",
+            RpcRequest::GetLargestAccounts => "getLargestAccounts",
+            RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
+            RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
+            RpcRequest::GetProgramAccounts => "getProgramAccounts",
+            RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
@@ -73,21 +82,34 @@ impl RpcRequest {
             RpcRequest::GetStorageTurnRate => "getStorageTurnRate",
             RpcRequest::GetSlotsPerSegment => "getSlotsPerSegment",
             RpcRequest::GetStoragePubkeysForSlot => "getStoragePubkeysForSlot",
+            RpcRequest::GetSupply => "getSupply",
             RpcRequest::GetTotalSupply => "getTotalSupply",
             RpcRequest::GetTransactionCount => "getTransactionCount",
             RpcRequest::GetVersion => "getVersion",
             RpcRequest::GetVoteAccounts => "getVoteAccounts",
+            RpcRequest::MinimumLedgerSlot => "minimumLedgerSlot",
             RpcRequest::RegisterNode => "registerNode",
             RpcRequest::RequestAirdrop => "requestAirdrop",
             RpcRequest::SendTransaction => "sendTransaction",
+            RpcRequest::SimulateTransaction => "simulateTransaction",
             RpcRequest::SignVote => "signVote",
-            RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
-            RpcRequest::MinimumLedgerSlot => "minimumLedgerSlot",
         };
+
+        write!(f, "{}", method)
+    }
+}
+
+pub const NUM_LARGEST_ACCOUNTS: usize = 20;
+pub const MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS: usize = 256;
+pub const MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE: u64 = 10_000;
+
+impl RpcRequest {
+    pub(crate) fn build_request_json(self, id: u64, params: Value) -> Value {
+        let jsonrpc = "2.0";
         json!({
            "jsonrpc": jsonrpc,
            "id": id,
-           "method": method,
+           "method": format!("{}", self),
            "params": params,
         })
     }
@@ -126,10 +148,6 @@ mod tests {
         let request = test_request.build_request_json(1, Value::Null);
         assert_eq!(request["method"], "getEpochInfo");
 
-        let test_request = RpcRequest::GetInflation;
-        let request = test_request.build_request_json(1, Value::Null);
-        assert_eq!(request["method"], "getInflation");
-
         let test_request = RpcRequest::GetRecentBlockhash;
         let request = test_request.build_request_json(1, Value::Null);
         assert_eq!(request["method"], "getRecentBlockhash");

client/src/rpc_response.rs

@@ -3,6 +3,7 @@ use solana_sdk::{
     account::Account,
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
+    inflation::Inflation,
     pubkey::Pubkey,
     transaction::{Result, TransactionError},
 };
@@ -35,6 +36,14 @@ pub struct RpcBlockhashFeeCalculator {
     pub fee_calculator: FeeCalculator,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcFees {
+    pub blockhash: String,
+    pub fee_calculator: FeeCalculator,
+    pub last_valid_slot: Slot,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcFeeCalculator {
@@ -47,6 +56,37 @@ pub struct RpcFeeRateGovernor {
     pub fee_rate_governor: FeeRateGovernor,
 }
 
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationGovernor {
+    pub initial: f64,
+    pub terminal: f64,
+    pub taper: f64,
+    pub foundation: f64,
+    pub foundation_term: f64,
+}
+
+impl From<Inflation> for RpcInflationGovernor {
+    fn from(inflation: Inflation) -> Self {
+        Self {
+            initial: inflation.initial,
+            terminal: inflation.terminal,
+            taper: inflation.taper,
+            foundation: inflation.foundation,
+            foundation_term: inflation.foundation_term,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationRate {
+    pub total: f64,
+    pub validator: f64,
+    pub foundation: f64,
+    pub epoch: Epoch,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcKeyedAccount {
@@ -108,6 +148,8 @@ pub struct RpcContactInfo {
     pub tpu: Option<SocketAddr>,
     /// JSON RPC port
     pub rpc: Option<SocketAddr>,
+    /// Software version
+    pub version: Option<String>,
 }
 
 /// Map of leader base58 identity pubkeys to the slot indices relative to the first epoch slot
@@ -186,9 +228,32 @@ pub struct RpcSignatureConfirmation {
     pub status: Result<()>,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSimulateTransactionResult {
+    pub err: Option<TransactionError>,
+    pub logs: Option<Vec<String>>,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcStorageTurn {
     pub blockhash: String,
     pub slot: Slot,
 }
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcAccountBalance {
+    pub address: String,
+    pub lamports: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSupply {
+    pub total: u64,
+    pub circulating: u64,
+    pub non_circulating: u64,
+    pub non_circulating_accounts: Vec<String>,
+}

client/src/thin_client.rs

@@ -440,7 +440,7 @@ impl SyncClient for ThinClient {
         match recent_blockhash {
             Ok(Response { value, .. }) => {
                 self.optimizer.report(index, duration_as_ms(&now.elapsed()));
-                Ok(value)
+                Ok((value.0, value.1))
             }
             Err(e) => {
                 self.optimizer.report(index, std::u64::MAX);

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.1.5"
+version = "1.1.17"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -21,6 +21,7 @@ byteorder = "1.3.4"
 chrono = { version = "0.4.11", features = ["serde"] }
 core_affinity = "0.5.10"
 crossbeam-channel = "0.4"
+ed25519-dalek = "=1.0.0-pre.3"
 fs_extra = "1.1.0"
 flate2 = "1.0"
 indexmap = "1.3"
@@ -34,42 +35,44 @@ jsonrpc-ws-server = "14.0.6"
 log = "0.4.8"
 num_cpus = "1.0.0"
 num-traits = "0.2"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
+rand = "0.7.0"
+rand_chacha = "0.2.2"
 rayon = "1.3.0"
 regex = "1.3.6"
 serde = "1.0.105"
 serde_derive = "1.0.103"
 serde_json = "1.0.48"
-solana-budget-program = { path = "../programs/budget", version = "1.1.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
-solana-client = { path = "../client", version = "1.1.5" }
-solana-transaction-status = { path = "../transaction-status", version = "1.1.5" }
-solana-faucet = { path = "../faucet", version = "1.1.5" }
-ed25519-dalek = "=1.0.0-pre.1"
-solana-ledger = { path = "../ledger", version = "1.1.5" }
-solana-logger = { path = "../logger", version = "1.1.5" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.1.5" }
-solana-metrics = { path = "../metrics", version = "1.1.5" }
-solana-measure = { path = "../measure", version = "1.1.5" }
-solana-net-utils = { path = "../net-utils", version = "1.1.5" }
-solana-chacha-cuda = { path = "../chacha-cuda", version = "1.1.5" }
-solana-perf = { path = "../perf", version = "1.1.5" }
-solana-runtime = { path = "../runtime", version = "1.1.5" }
-solana-sdk = { path = "../sdk", version = "1.1.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.1.5" }
-solana-storage-program = { path = "../programs/storage", version = "1.1.5" }
-solana-streamer = { path = "../streamer", version = "1.1.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.1.5" }
-solana-vote-signer = { path = "../vote-signer", version = "1.1.5" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.1.5" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.1.17" }
+solana-budget-program = { path = "../programs/budget", version = "1.1.17" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.17" }
+solana-client = { path = "../client", version = "1.1.17" }
+solana-transaction-status = { path = "../transaction-status", version = "1.1.17" }
+solana-faucet = { path = "../faucet", version = "1.1.17" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.1.17" }
+solana-ledger = { path = "../ledger", version = "1.1.17" }
+solana-logger = { path = "../logger", version = "1.1.17" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.1.17" }
+solana-metrics = { path = "../metrics", version = "1.1.17" }
+solana-measure = { path = "../measure", version = "1.1.17" }
+solana-net-utils = { path = "../net-utils", version = "1.1.17" }
+solana-chacha-cuda = { path = "../chacha-cuda", version = "1.1.17" }
+solana-perf = { path = "../perf", version = "1.1.17" }
+solana-runtime = { path = "../runtime", version = "1.1.17" }
+solana-sdk = { path = "../sdk", version = "1.1.17" }
+solana-stake-program = { path = "../programs/stake", version = "1.1.17" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.17" }
+solana-streamer = { path = "../streamer", version = "1.1.17" }
+solana-version = { path = "../version", version = "1.1.17" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.17" }
+solana-vote-signer = { path = "../vote-signer", version = "1.1.17" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.1.17" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.1.5" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.1.17" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/benches/banking_stage.rs

@@ -190,7 +190,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
         let (exit, poh_recorder, poh_service, signal_receiver) =
             create_test_recorder(&bank, &blockstore, None);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
         let _banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,

core/benches/cluster_info.rs

@@ -3,10 +3,11 @@
 extern crate test;
 
 use rand::{thread_rng, Rng};
+use solana_core::broadcast_stage::broadcast_metrics::TransmitShredsStats;
 use solana_core::broadcast_stage::{broadcast_shreds, get_broadcast_peers};
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
-use solana_ledger::shred::Shred;
+use solana_ledger::shred::{Shred, NONCE_SHRED_PAYLOAD_SIZE};
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
 use std::sync::RwLock;
@@ -26,7 +27,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
     let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
 
     const NUM_SHREDS: usize = 32;
-    let shreds = vec![Shred::new_empty_data_shred(); NUM_SHREDS];
+    let shreds = vec![Shred::new_empty_data_shred(NONCE_SHRED_PAYLOAD_SIZE); NUM_SHREDS];
     let mut stakes = HashMap::new();
     const NUM_PEERS: usize = 200;
     for _ in 0..NUM_PEERS {
@@ -36,7 +37,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
         stakes.insert(id, thread_rng().gen_range(1, NUM_PEERS) as u64);
     }
     let stakes = Arc::new(stakes);
-    let cluster_info = Arc::new(RwLock::new(cluster_info));
+    let cluster_info = Arc::new(cluster_info);
     let (peers, peers_and_stakes) = get_broadcast_peers(&cluster_info, Some(stakes.clone()));
     let shreds = Arc::new(shreds);
     let last_datapoint = Arc::new(AtomicU64::new(0));
@@ -48,7 +49,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
             &peers_and_stakes,
             &peers,
             &last_datapoint,
-            &mut 0,
+            &mut TransmitShredsStats::default(),
         )
         .unwrap();
     });

core/benches/retransmit_stage.rs

@@ -45,7 +45,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
         peer_sockets.push(socket);
     }
     let peer_sockets = Arc::new(peer_sockets);
-    let cluster_info = Arc::new(RwLock::new(cluster_info));
+    let cluster_info = Arc::new(cluster_info);
 
     let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
     let bank0 = Bank::new(&genesis_config);

core/benches/shredder.rs

@@ -5,7 +5,7 @@ extern crate test;
 use solana_ledger::entry::{create_ticks, Entry};
 use solana_ledger::shred::{
     max_entries_per_n_shred, max_ticks_per_n_shreds, Shred, Shredder, RECOMMENDED_FEC_RATE,
-    SIZE_OF_DATA_SHRED_PAYLOAD,
+    SIZE_OF_NONCE_DATA_SHRED_PAYLOAD,
 };
 use solana_perf::test_tx;
 use solana_sdk::hash::Hash;
@@ -29,10 +29,11 @@ fn make_large_unchained_entries(txs_per_entry: u64, num_entries: u64) -> Vec<Ent
 #[bench]
 fn bench_shredder_ticks(bencher: &mut Bencher) {
     let kp = Arc::new(Keypair::new());
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = SIZE_OF_NONCE_DATA_SHRED_PAYLOAD;
     let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
     // ~1Mb
-    let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
+    let num_ticks =
+        max_ticks_per_n_shreds(1, Some(SIZE_OF_NONCE_DATA_SHRED_PAYLOAD)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     bencher.iter(|| {
         let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
@@ -43,10 +44,14 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
 #[bench]
 fn bench_shredder_large_entries(bencher: &mut Bencher) {
     let kp = Arc::new(Keypair::new());
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = SIZE_OF_NONCE_DATA_SHRED_PAYLOAD;
     let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
     let txs_per_entry = 128;
-    let num_entries = max_entries_per_n_shred(&make_test_entry(txs_per_entry), num_shreds as u64);
+    let num_entries = max_entries_per_n_shred(
+        &make_test_entry(txs_per_entry),
+        num_shreds as u64,
+        Some(shred_size),
+    );
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     // 1Mb
     bencher.iter(|| {
@@ -58,10 +63,10 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
 #[bench]
 fn bench_deshredder(bencher: &mut Bencher) {
     let kp = Arc::new(Keypair::new());
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = SIZE_OF_NONCE_DATA_SHRED_PAYLOAD;
     // ~10Mb
     let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
-    let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
+    let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0, 0).unwrap();
     let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
@@ -73,7 +78,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
 
 #[bench]
 fn bench_deserialize_hdr(bencher: &mut Bencher) {
-    let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
+    let data = vec![0; SIZE_OF_NONCE_DATA_SHRED_PAYLOAD];
 
     let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0, 1);
 

core/src/accounts_hash_verifier.rs

@@ -15,7 +15,7 @@ use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
         mpsc::RecvTimeoutError,
-        Arc, RwLock,
+        Arc,
     },
     thread::{self, Builder, JoinHandle},
     time::Duration,
@@ -30,7 +30,7 @@ impl AccountsHashVerifier {
         snapshot_package_receiver: SnapshotPackageReceiver,
         snapshot_package_sender: Option<SnapshotPackageSender>,
         exit: &Arc<AtomicBool>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &Arc<ClusterInfo>,
         trusted_validators: Option<HashSet<Pubkey>>,
         halt_on_trusted_validators_accounts_hash_mismatch: bool,
         fault_injection_rate_slots: u64,
@@ -72,7 +72,7 @@ impl AccountsHashVerifier {
 
     fn process_snapshot(
         snapshot_package: SnapshotPackage,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         trusted_validators: &Option<HashSet<Pubkey>>,
         halt_on_trusted_validator_accounts_hash_mismatch: bool,
         snapshot_package_sender: &Option<SnapshotPackageSender>,
@@ -111,14 +111,11 @@ impl AccountsHashVerifier {
             if sender.send(snapshot_package).is_err() {}
         }
 
-        cluster_info
-            .write()
-            .unwrap()
-            .push_accounts_hashes(hashes.clone());
+        cluster_info.push_accounts_hashes(hashes.clone());
     }
 
     fn should_halt(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         trusted_validators: &Option<HashSet<Pubkey>>,
         slot_to_hash: &mut HashMap<Slot, Hash>,
     ) -> bool {
@@ -126,11 +123,9 @@ impl AccountsHashVerifier {
         let mut highest_slot = 0;
         if let Some(trusted_validators) = trusted_validators.as_ref() {
             for trusted_validator in trusted_validators {
-                let cluster_info_r = cluster_info.read().unwrap();
-                if let Some(accounts_hashes) =
-                    cluster_info_r.get_accounts_hash_for_node(trusted_validator)
+                let is_conflicting = cluster_info.get_accounts_hash_for_node(trusted_validator, |accounts_hashes|
                 {
-                    for (slot, hash) in accounts_hashes {
+                    accounts_hashes.iter().any(|(slot, hash)| {
                         if let Some(reference_hash) = slot_to_hash.get(slot) {
                             if *hash != *reference_hash {
                                 error!("Trusted validator {} produced conflicting hashes for slot: {} ({} != {})",
@@ -139,16 +134,21 @@ impl AccountsHashVerifier {
                                     hash,
                                     reference_hash,
                                 );
-
-                                return true;
+                                true
                             } else {
                                 verified_count += 1;
+                                false
                             }
                         } else {
                             highest_slot = std::cmp::max(*slot, highest_slot);
                             slot_to_hash.insert(*slot, *hash);
+                            false
                         }
-                    }
+                    })
+                }).unwrap_or(false);
+
+                if is_conflicting {
+                    return true;
                 }
             }
         }
@@ -181,7 +181,7 @@ mod tests {
 
         let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
 
         let mut trusted_validators = HashSet::new();
         let mut slot_to_hash = HashMap::new();
@@ -196,8 +196,7 @@ mod tests {
         let hash2 = hash(&[2]);
         {
             let message = make_accounts_hashes_message(&validator1, vec![(0, hash1)]).unwrap();
-            let mut cluster_info_w = cluster_info.write().unwrap();
-            cluster_info_w.push_message(message);
+            cluster_info.push_message(message);
         }
         slot_to_hash.insert(0, hash2);
         trusted_validators.insert(validator1.pubkey());
@@ -217,7 +216,7 @@ mod tests {
 
         let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
 
         let trusted_validators = HashSet::new();
         let exit = Arc::new(AtomicBool::new(false));
@@ -244,9 +243,8 @@ mod tests {
                 0,
             );
         }
-        let cluster_info_r = cluster_info.read().unwrap();
-        let cluster_hashes = cluster_info_r
-            .get_accounts_hash_for_node(&keypair.pubkey())
+        let cluster_hashes = cluster_info
+            .get_accounts_hash_for_node(&keypair.pubkey(), |c| c.clone())
             .unwrap();
         info!("{:?}", cluster_hashes);
         assert_eq!(hashes.len(), MAX_SNAPSHOT_HASHES);

core/src/banking_stage.rs

@@ -41,7 +41,7 @@ use std::{
     net::UdpSocket,
     sync::atomic::AtomicBool,
     sync::mpsc::Receiver,
-    sync::{Arc, Mutex, RwLock},
+    sync::{Arc, Mutex},
     thread::{self, Builder, JoinHandle},
     time::Duration,
     time::Instant,
@@ -76,7 +76,7 @@ impl BankingStage {
     /// Create the stage using `bank`. Exit when `verified_receiver` is dropped.
     #[allow(clippy::new_ret_no_self)]
     pub fn new(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &Arc<ClusterInfo>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         verified_receiver: CrossbeamReceiver<Vec<Packets>>,
         verified_vote_receiver: CrossbeamReceiver<Vec<Packets>>,
@@ -93,7 +93,7 @@ impl BankingStage {
     }
 
     fn new_num_threads(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &Arc<ClusterInfo>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         verified_receiver: CrossbeamReceiver<Vec<Packets>>,
         verified_vote_receiver: CrossbeamReceiver<Vec<Packets>>,
@@ -104,7 +104,7 @@ impl BankingStage {
         // Single thread to generate entries from many banks.
         // This thread talks to poh_service and broadcasts the entries once they have been recorded.
         // Once an entry has been recorded, its blockhash is registered with the bank.
-        let my_pubkey = cluster_info.read().unwrap().id();
+        let my_pubkey = cluster_info.id();
         // Many banks that process transactions in parallel.
         let bank_thread_hdls: Vec<JoinHandle<()>> = (0..num_threads)
             .map(|i| {
@@ -287,12 +287,12 @@ impl BankingStage {
         my_pubkey: &Pubkey,
         socket: &std::net::UdpSocket,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         buffered_packets: &mut Vec<PacketsAndOffsets>,
         enable_forwarding: bool,
         batch_limit: usize,
         transaction_status_sender: Option<TransactionStatusSender>,
-    ) {
+    ) -> BufferedPacketsDecision {
         let (leader_at_slot_offset, poh_has_bank, would_be_leader) = {
             let poh = poh_recorder.lock().unwrap();
             (
@@ -331,10 +331,7 @@ impl BankingStage {
                     next_leader.map_or((), |leader_pubkey| {
                         let leader_addr = {
                             cluster_info
-                                .read()
-                                .unwrap()
-                                .lookup(&leader_pubkey)
-                                .map(|leader| leader.tpu_forwards)
+                                .lookup_contact_info(&leader_pubkey, |leader| leader.tpu_forwards)
                         };
 
                         leader_addr.map_or((), |leader_addr| {
@@ -352,13 +349,14 @@ impl BankingStage {
             }
             _ => (),
         }
+        decision
     }
 
     pub fn process_loop(
         my_pubkey: Pubkey,
         verified_receiver: &CrossbeamReceiver<Vec<Packets>>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         recv_start: &mut Instant,
         enable_forwarding: bool,
         id: u32,
@@ -368,8 +366,8 @@ impl BankingStage {
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut buffered_packets = vec![];
         loop {
-            if !buffered_packets.is_empty() {
-                Self::process_buffered_packets(
+            while !buffered_packets.is_empty() {
+                let decision = Self::process_buffered_packets(
                     &my_pubkey,
                     &socket,
                     poh_recorder,
@@ -379,6 +377,11 @@ impl BankingStage {
                     batch_limit,
                     transaction_status_sender.clone(),
                 );
+                if decision == BufferedPacketsDecision::Hold {
+                    // If we are waiting on a new bank,
+                    // check the receiver for more transactions/for exiting
+                    break;
+                }
             }
 
             let recv_timeout = if !buffered_packets.is_empty() {
@@ -1049,7 +1052,7 @@ mod tests {
             let (exit, poh_recorder, poh_service, _entry_receiever) =
                 create_test_recorder(&bank, &blockstore, None);
             let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
-            let cluster_info = Arc::new(RwLock::new(cluster_info));
+            let cluster_info = Arc::new(cluster_info);
             let banking_stage = BankingStage::new(
                 &cluster_info,
                 &poh_recorder,
@@ -1089,7 +1092,7 @@ mod tests {
             let (exit, poh_recorder, poh_service, entry_receiver) =
                 create_test_recorder(&bank, &blockstore, Some(poh_config));
             let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
-            let cluster_info = Arc::new(RwLock::new(cluster_info));
+            let cluster_info = Arc::new(cluster_info);
             let banking_stage = BankingStage::new(
                 &cluster_info,
                 &poh_recorder,
@@ -1152,7 +1155,7 @@ mod tests {
             let (exit, poh_recorder, poh_service, entry_receiver) =
                 create_test_recorder(&bank, &blockstore, Some(poh_config));
             let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
-            let cluster_info = Arc::new(RwLock::new(cluster_info));
+            let cluster_info = Arc::new(cluster_info);
             let banking_stage = BankingStage::new(
                 &cluster_info,
                 &poh_recorder,
@@ -1293,7 +1296,7 @@ mod tests {
                     create_test_recorder(&bank, &blockstore, Some(poh_config));
                 let cluster_info =
                     ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
-                let cluster_info = Arc::new(RwLock::new(cluster_info));
+                let cluster_info = Arc::new(cluster_info);
                 let _banking_stage = BankingStage::new_num_threads(
                     &cluster_info,
                     &poh_recorder,
@@ -1991,14 +1994,14 @@ mod tests {
                             meta.err,
                             Some(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                         assert_eq!(
                             meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                     } else {

core/src/broadcast_stage.rs

@@ -29,13 +29,13 @@ use std::{
     net::UdpSocket,
     sync::atomic::{AtomicBool, Ordering},
     sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
-    sync::{Arc, Mutex, RwLock},
+    sync::{Arc, Mutex},
     thread::{self, Builder, JoinHandle},
     time::{Duration, Instant},
 };
 
 mod broadcast_fake_shreds_run;
-pub(crate) mod broadcast_metrics;
+pub mod broadcast_metrics;
 pub(crate) mod broadcast_utils;
 mod fail_entry_verification_broadcast_run;
 mod standard_broadcast_run;
@@ -62,14 +62,14 @@ impl BroadcastStageType {
     pub fn new_broadcast_stage(
         &self,
         sock: Vec<UdpSocket>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         receiver: Receiver<WorkingBankEntry>,
         retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         shred_version: u16,
     ) -> BroadcastStage {
-        let keypair = cluster_info.read().unwrap().keypair.clone();
+        let keypair = cluster_info.keypair.clone();
         match self {
             BroadcastStageType::Standard => BroadcastStage::new(
                 sock,
@@ -116,7 +116,7 @@ trait BroadcastRun {
     fn transmit(
         &mut self,
         receiver: &Arc<Mutex<TransmitReceiver>>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         sock: &UdpSocket,
     ) -> Result<()>;
     fn record(
@@ -205,7 +205,7 @@ impl BroadcastStage {
     #[allow(clippy::too_many_arguments)]
     fn new(
         socks: Vec<UdpSocket>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         receiver: Receiver<WorkingBankEntry>,
         retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
@@ -357,11 +357,11 @@ fn update_peer_stats(
 }
 
 pub fn get_broadcast_peers<S: std::hash::BuildHasher>(
-    cluster_info: &Arc<RwLock<ClusterInfo>>,
+    cluster_info: &ClusterInfo,
     stakes: Option<Arc<HashMap<Pubkey, u64, S>>>,
 ) -> (Vec<ContactInfo>, Vec<(u64, usize)>) {
     use crate::cluster_info;
-    let mut peers = cluster_info.read().unwrap().tvu_peers();
+    let mut peers = cluster_info.tvu_peers();
     let peers_and_stakes = cluster_info::stake_weight_peers(&mut peers, stakes);
     (peers, peers_and_stakes)
 }
@@ -374,13 +374,14 @@ pub fn broadcast_shreds(
     peers_and_stakes: &[(u64, usize)],
     peers: &[ContactInfo],
     last_datapoint_submit: &Arc<AtomicU64>,
-    send_mmsg_total: &mut u64,
+    transmit_stats: &mut TransmitShredsStats,
 ) -> Result<()> {
     let broadcast_len = peers_and_stakes.len();
     if broadcast_len == 0 {
         update_peer_stats(1, 1, last_datapoint_submit);
         return Ok(());
     }
+    let mut shred_select = Measure::start("shred_select");
     let packets: Vec<_> = shreds
         .iter()
         .map(|shred| {
@@ -389,6 +390,8 @@ pub fn broadcast_shreds(
             (&shred.payload, &peers[broadcast_index].tvu)
         })
         .collect();
+    shred_select.stop();
+    transmit_stats.shred_select += shred_select.as_us();
 
     let mut sent = 0;
     let mut send_mmsg_time = Measure::start("send_mmsg");
@@ -401,7 +404,7 @@ pub fn broadcast_shreds(
         }
     }
     send_mmsg_time.stop();
-    *send_mmsg_total += send_mmsg_time.as_us();
+    transmit_stats.send_mmsg_elapsed += send_mmsg_time.as_us();
 
     let num_live_peers = num_live_peers(&peers);
     update_peer_stats(
@@ -412,11 +415,19 @@ pub fn broadcast_shreds(
     Ok(())
 }
 
+fn distance(a: u64, b: u64) -> u64 {
+    if a > b {
+        a - b
+    } else {
+        b - a
+    }
+}
+
 fn num_live_peers(peers: &[ContactInfo]) -> i64 {
     let mut num_live_peers = 1i64;
     peers.iter().for_each(|p| {
         // A peer is considered live if they generated their contact info recently
-        if timestamp() - p.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
+        if distance(timestamp(), p.wallclock) <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
             num_live_peers += 1;
         }
     });
@@ -442,11 +453,7 @@ pub mod test {
         signature::{Keypair, Signer},
     };
     use std::{
-        path::Path,
-        sync::atomic::AtomicBool,
-        sync::mpsc::channel,
-        sync::{Arc, RwLock},
-        thread::sleep,
+        path::Path, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc, thread::sleep,
     };
 
     pub fn make_transmit_shreds(
@@ -459,7 +466,7 @@ pub mod test {
         Vec<TransmitShreds>,
         Vec<TransmitShreds>,
     ) {
-        let num_entries = max_ticks_per_n_shreds(num);
+        let num_entries = max_ticks_per_n_shreds(num, None);
         let (data_shreds, _) = make_slot_entries(slot, 0, num_entries);
         let keypair = Arc::new(Keypair::new());
         let shredder = Shredder::new(slot, 0, RECOMMENDED_FEC_RATE, keypair, 0, 0)
@@ -506,6 +513,17 @@ pub mod test {
         assert_eq!(num_expected_coding_shreds, coding_index);
     }
 
+    #[test]
+    fn test_num_live_peers() {
+        let mut ci = ContactInfo::default();
+        ci.wallclock = std::u64::MAX;
+        assert_eq!(num_live_peers(&[ci.clone()]), 1);
+        ci.wallclock = timestamp() - 1;
+        assert_eq!(num_live_peers(&[ci.clone()]), 2);
+        ci.wallclock = timestamp() - CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS - 1;
+        assert_eq!(num_live_peers(&[ci]), 1);
+    }
+
     #[test]
     fn test_duplicate_retransmit_signal() {
         // Setup
@@ -579,16 +597,16 @@ pub mod test {
         let broadcast_buddy = Node::new_localhost_with_pubkey(&buddy_keypair.pubkey());
 
         // Fill the cluster_info with the buddy's info
-        let mut cluster_info = ClusterInfo::new_with_invalid_keypair(leader_info.info.clone());
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(leader_info.info.clone());
         cluster_info.insert_info(broadcast_buddy.info);
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
 
         let exit_sender = Arc::new(AtomicBool::new(false));
 
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Arc::new(Bank::new(&genesis_config));
 
-        let leader_keypair = cluster_info.read().unwrap().keypair.clone();
+        let leader_keypair = cluster_info.keypair.clone();
         // Start up the broadcast stage
         let broadcast_service = BroadcastStage::new(
             leader_info.sockets.broadcast,

core/src/broadcast_stage/broadcast_fake_shreds_run.rs

@@ -104,11 +104,11 @@ impl BroadcastRun for BroadcastFakeShredsRun {
     fn transmit(
         &mut self,
         receiver: &Arc<Mutex<TransmitReceiver>>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         sock: &UdpSocket,
     ) -> Result<()> {
         for ((stakes, data_shreds), _) in receiver.lock().unwrap().iter() {
-            let peers = cluster_info.read().unwrap().tvu_peers();
+            let peers = cluster_info.tvu_peers();
             peers.iter().enumerate().for_each(|(i, peer)| {
                 if i <= self.partition && stakes.is_some() {
                     // Send fake shreds to the first N peers
@@ -145,7 +145,7 @@ mod tests {
 
     #[test]
     fn test_tvu_peers_ordering() {
-        let mut cluster = ClusterInfo::new_with_invalid_keypair(ContactInfo::new_localhost(
+        let cluster = ClusterInfo::new_with_invalid_keypair(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
             0,
         ));

core/src/broadcast_stage/broadcast_metrics.rs

@@ -29,11 +29,12 @@ impl ProcessShredsStats {
 }
 
 #[derive(Default, Clone)]
-pub(crate) struct TransmitShredsStats {
-    pub(crate) transmit_elapsed: u64,
-    pub(crate) send_mmsg_elapsed: u64,
-    pub(crate) get_peers_elapsed: u64,
-    pub(crate) num_shreds: usize,
+pub struct TransmitShredsStats {
+    pub transmit_elapsed: u64,
+    pub send_mmsg_elapsed: u64,
+    pub get_peers_elapsed: u64,
+    pub shred_select: u64,
+    pub num_shreds: usize,
 }
 
 impl BroadcastStats for TransmitShredsStats {
@@ -42,6 +43,7 @@ impl BroadcastStats for TransmitShredsStats {
         self.send_mmsg_elapsed += new_stats.send_mmsg_elapsed;
         self.get_peers_elapsed += new_stats.get_peers_elapsed;
         self.num_shreds += new_stats.num_shreds;
+        self.shred_select += new_stats.shred_select;
     }
     fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
         datapoint_info!(
@@ -58,6 +60,7 @@ impl BroadcastStats for TransmitShredsStats {
             ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
             ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
             ("num_shreds", self.num_shreds as i64, i64),
+            ("shred_select", self.shred_select as i64, i64),
         );
     }
 }
@@ -176,15 +179,16 @@ mod test {
     }
 
     #[test]
-    fn test_update() {
+    fn test_update_broadcast() {
         let start = Instant::now();
         let mut slot_broadcast_stats = SlotBroadcastStats::default();
         slot_broadcast_stats.update(
             &TransmitShredsStats {
                 transmit_elapsed: 1,
-                get_peers_elapsed: 1,
-                send_mmsg_elapsed: 1,
-                num_shreds: 1,
+                get_peers_elapsed: 2,
+                send_mmsg_elapsed: 3,
+                shred_select: 4,
+                num_shreds: 5,
             },
             &Some(BroadcastShredBatchInfo {
                 slot: 0,
@@ -198,16 +202,18 @@ mod test {
         assert_eq!(slot_0_stats.num_batches, 1);
         assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
         assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
 
         slot_broadcast_stats.update(
             &TransmitShredsStats {
-                transmit_elapsed: 1,
-                get_peers_elapsed: 1,
-                send_mmsg_elapsed: 1,
-                num_shreds: 1,
+                transmit_elapsed: 7,
+                get_peers_elapsed: 8,
+                send_mmsg_elapsed: 9,
+                shred_select: 10,
+                num_shreds: 11,
             },
             &None,
         );
@@ -217,9 +223,10 @@ mod test {
         assert_eq!(slot_0_stats.num_batches, 1);
         assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
         assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
 
         // If another batch is given, then total number of batches == num_expected_batches == 2,
         // so the batch should be purged from the HashMap
@@ -228,6 +235,7 @@ mod test {
                 transmit_elapsed: 1,
                 get_peers_elapsed: 1,
                 send_mmsg_elapsed: 1,
+                shred_select: 1,
                 num_shreds: 1,
             },
             &Some(BroadcastShredBatchInfo {

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -74,21 +74,20 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
     fn transmit(
         &mut self,
         receiver: &Arc<Mutex<TransmitReceiver>>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         sock: &UdpSocket,
     ) -> Result<()> {
         let ((stakes, shreds), _) = receiver.lock().unwrap().recv()?;
         // Broadcast data
         let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
 
-        let mut send_mmsg_total = 0;
         broadcast_shreds(
             sock,
             &shreds,
             &peers_and_stakes,
             &peers,
             &Arc::new(AtomicU64::new(0)),
-            &mut send_mmsg_total,
+            &mut TransmitShredsStats::default(),
         )?;
 
         Ok(())

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -9,6 +9,7 @@ use solana_ledger::{
 };
 use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
 use std::collections::HashMap;
+use std::sync::RwLock;
 use std::time::Duration;
 
 #[derive(Clone)]
@@ -23,6 +24,14 @@ pub struct StandardBroadcastRun {
     shred_version: u16,
     last_datapoint_submit: Arc<AtomicU64>,
     num_batches: usize,
+    broadcast_peer_cache: Arc<RwLock<BroadcastPeerCache>>,
+    last_peer_update: Arc<AtomicU64>,
+}
+
+#[derive(Default)]
+struct BroadcastPeerCache {
+    peers: Vec<ContactInfo>,
+    peers_and_stakes: Vec<(u64, usize)>,
 }
 
 impl StandardBroadcastRun {
@@ -38,6 +47,8 @@ impl StandardBroadcastRun {
             shred_version,
             last_datapoint_submit: Arc::new(AtomicU64::new(0)),
             num_batches: 0,
+            broadcast_peer_cache: Arc::new(RwLock::new(BroadcastPeerCache::default())),
+            last_peer_update: Arc::new(AtomicU64::new(0)),
         }
     }
 
@@ -120,7 +131,7 @@ impl StandardBroadcastRun {
     #[cfg(test)]
     fn test_process_receive_results(
         &mut self,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         sock: &UdpSocket,
         blockstore: &Arc<Blockstore>,
         receive_results: ReceiveResults,
@@ -288,38 +299,51 @@ impl StandardBroadcastRun {
     fn broadcast(
         &mut self,
         sock: &UdpSocket,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         stakes: Option<Arc<HashMap<Pubkey, u64>>>,
         shreds: Arc<Vec<Shred>>,
         broadcast_shred_batch_info: Option<BroadcastShredBatchInfo>,
     ) -> Result<()> {
+        const BROADCAST_PEER_UPDATE_INTERVAL_MS: u64 = 1000;
         trace!("Broadcasting {:?} shreds", shreds.len());
         // Get the list of peers to broadcast to
-        let get_peers_start = Instant::now();
-        let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
-        let get_peers_elapsed = get_peers_start.elapsed();
+        let mut get_peers_time = Measure::start("broadcast::get_peers");
+        let now = timestamp();
+        let last = self.last_peer_update.load(Ordering::Relaxed);
+        if now - last > BROADCAST_PEER_UPDATE_INTERVAL_MS
+            && self
+                .last_peer_update
+                .compare_and_swap(now, last, Ordering::Relaxed)
+                == last
+        {
+            let mut w_broadcast_peer_cache = self.broadcast_peer_cache.write().unwrap();
+            let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
+            w_broadcast_peer_cache.peers = peers;
+            w_broadcast_peer_cache.peers_and_stakes = peers_and_stakes;
+        }
+        get_peers_time.stop();
+        let r_broadcast_peer_cache = self.broadcast_peer_cache.read().unwrap();
 
+        let mut transmit_stats = TransmitShredsStats::default();
         // Broadcast the shreds
-        let transmit_start = Instant::now();
-        let mut send_mmsg_total = 0;
+        let mut transmit_time = Measure::start("broadcast_shreds");
         broadcast_shreds(
             sock,
             &shreds,
-            &peers_and_stakes,
-            &peers,
+            &r_broadcast_peer_cache.peers_and_stakes,
+            &r_broadcast_peer_cache.peers,
             &self.last_datapoint_submit,
-            &mut send_mmsg_total,
+            &mut transmit_stats,
         )?;
-        let transmit_elapsed = transmit_start.elapsed();
-        let new_transmit_shreds_stats = TransmitShredsStats {
-            transmit_elapsed: duration_as_us(&transmit_elapsed),
-            get_peers_elapsed: duration_as_us(&get_peers_elapsed),
-            send_mmsg_elapsed: send_mmsg_total,
-            num_shreds: shreds.len(),
-        };
+        drop(r_broadcast_peer_cache);
+        transmit_time.stop();
+
+        transmit_stats.transmit_elapsed = transmit_time.as_us();
+        transmit_stats.get_peers_elapsed = get_peers_time.as_us();
+        transmit_stats.num_shreds = shreds.len();
 
         // Process metrics
-        self.update_transmit_metrics(&new_transmit_shreds_stats, &broadcast_shred_batch_info);
+        self.update_transmit_metrics(&transmit_stats, &broadcast_shred_batch_info);
         Ok(())
     }
 
@@ -374,7 +398,7 @@ impl BroadcastRun for StandardBroadcastRun {
     fn transmit(
         &mut self,
         receiver: &Arc<Mutex<TransmitReceiver>>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         sock: &UdpSocket,
     ) -> Result<()> {
         let ((stakes, shreds), slot_start_ts) = receiver.lock().unwrap().recv()?;
@@ -404,7 +428,7 @@ mod test {
         genesis_config::GenesisConfig,
         signature::{Keypair, Signer},
     };
-    use std::sync::{Arc, RwLock};
+    use std::sync::Arc;
     use std::time::Duration;
 
     fn setup(
@@ -412,7 +436,7 @@ mod test {
     ) -> (
         Arc<Blockstore>,
         GenesisConfig,
-        Arc<RwLock<ClusterInfo>>,
+        Arc<ClusterInfo>,
         Arc<Bank>,
         Arc<Keypair>,
         UdpSocket,
@@ -425,12 +449,12 @@ mod test {
         let leader_keypair = Arc::new(Keypair::new());
         let leader_pubkey = leader_keypair.pubkey();
         let leader_info = Node::new_localhost_with_pubkey(&leader_pubkey);
-        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
+        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(
             leader_info.info.clone(),
-        )));
+        ));
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut genesis_config = create_genesis_config(10_000).genesis_config;
-        genesis_config.ticks_per_slot = max_ticks_per_n_shreds(num_shreds_per_slot) + 1;
+        genesis_config.ticks_per_slot = max_ticks_per_n_shreds(num_shreds_per_slot, None) + 1;
         let bank0 = Arc::new(Bank::new(&genesis_config));
         (
             blockstore,
@@ -539,7 +563,11 @@ mod test {
         // Interrupting the slot should cause the unfinished_slot and stats to reset
         let num_shreds = 1;
         assert!(num_shreds < num_shreds_per_slot);
-        let ticks1 = create_ticks(max_ticks_per_n_shreds(num_shreds), 0, genesis_config.hash());
+        let ticks1 = create_ticks(
+            max_ticks_per_n_shreds(num_shreds, None),
+            0,
+            genesis_config.hash(),
+        );
         let receive_results = ReceiveResults {
             entries: ticks1.clone(),
             time_elapsed: Duration::new(2, 0),

core/src/cluster_info_vote_listener.rs

@@ -1,8 +1,10 @@
 use crate::{
     cluster_info::{ClusterInfo, GOSSIP_SLEEP_MILLIS},
+    consensus::VOTE_THRESHOLD_SIZE,
     crds_value::CrdsValueLabel,
     poh_recorder::PohRecorder,
     result::{Error, Result},
+    rpc_subscriptions::RpcSubscriptions,
     sigverify,
     verified_vote_packets::VerifiedVotePackets,
 };
@@ -14,7 +16,10 @@ use log::*;
 use solana_ledger::bank_forks::BankForks;
 use solana_metrics::inc_new_counter_debug;
 use solana_perf::packet::{self, Packets};
-use solana_runtime::{bank::Bank, epoch_stakes::EpochAuthorizedVoters};
+use solana_runtime::{
+    bank::Bank,
+    epoch_stakes::{EpochAuthorizedVoters, EpochStakes},
+};
 use solana_sdk::{
     clock::{Epoch, Slot},
     epoch_schedule::EpochSchedule,
@@ -43,6 +48,7 @@ pub type VerifiedVoteTransactionsReceiver = CrossbeamReceiver<Vec<Transaction>>;
 pub struct SlotVoteTracker {
     voted: HashSet<Arc<Pubkey>>,
     updates: Option<Vec<Arc<Pubkey>>>,
+    total_stake: u64,
 }
 
 impl SlotVoteTracker {
@@ -197,12 +203,13 @@ pub struct ClusterInfoVoteListener {
 impl ClusterInfoVoteListener {
     pub fn new(
         exit: &Arc<AtomicBool>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         sigverify_disabled: bool,
         sender: CrossbeamSender<Vec<Packets>>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         vote_tracker: Arc<VoteTracker>,
         bank_forks: Arc<RwLock<BankForks>>,
+        subscriptions: Arc<RpcSubscriptions>,
     ) -> Self {
         let exit_ = exit.clone();
 
@@ -244,6 +251,7 @@ impl ClusterInfoVoteListener {
                     verified_vote_transactions_receiver,
                     vote_tracker,
                     &bank_forks,
+                    subscriptions,
                 );
             })
             .unwrap();
@@ -262,7 +270,7 @@ impl ClusterInfoVoteListener {
 
     fn recv_loop(
         exit: Arc<AtomicBool>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         sigverify_disabled: bool,
         verified_vote_packets_sender: VerifiedVotePacketsSender,
         verified_vote_transactions_sender: VerifiedVoteTransactionsSender,
@@ -272,40 +280,12 @@ impl ClusterInfoVoteListener {
             if exit.load(Ordering::Relaxed) {
                 return Ok(());
             }
-            let (labels, votes, new_ts) = cluster_info.read().unwrap().get_votes(last_ts);
+            let (labels, votes, new_ts) = cluster_info.get_votes(last_ts);
             inc_new_counter_debug!("cluster_info_vote_listener-recv_count", votes.len());
 
             last_ts = new_ts;
-            let msgs = packet::to_packets(&votes);
-            if !msgs.is_empty() {
-                let r = if sigverify_disabled {
-                    sigverify::ed25519_verify_disabled(&msgs)
-                } else {
-                    sigverify::ed25519_verify_cpu(&msgs)
-                };
-
-                assert_eq!(
-                    r.iter()
-                        .map(|packets_results| packets_results.len())
-                        .sum::<usize>(),
-                    votes.len()
-                );
-
-                let (vote_txs, packets) = izip!(
-                    labels.into_iter(),
-                    votes.into_iter(),
-                    r.iter().flatten(),
-                    msgs
-                )
-                .filter_map(|(label, vote, verify_result, packet)| {
-                    if *verify_result != 0 {
-                        Some((vote, (label, packet)))
-                    } else {
-                        None
-                    }
-                })
-                .unzip();
-
+            if !votes.is_empty() {
+                let (vote_txs, packets) = Self::verify_votes(votes, labels, sigverify_disabled);
                 verified_vote_transactions_sender.send(vote_txs)?;
                 verified_vote_packets_sender.send(packets)?;
             }
@@ -314,6 +294,42 @@ impl ClusterInfoVoteListener {
         }
     }
 
+    fn verify_votes(
+        votes: Vec<Transaction>,
+        labels: Vec<CrdsValueLabel>,
+        sigverify_disabled: bool,
+    ) -> (Vec<Transaction>, Vec<(CrdsValueLabel, Packets)>) {
+        let msgs = packet::to_packets_chunked(&votes, 1);
+        let r = if sigverify_disabled {
+            sigverify::ed25519_verify_disabled(&msgs)
+        } else {
+            sigverify::ed25519_verify_cpu(&msgs)
+        };
+
+        assert_eq!(
+            r.iter()
+                .map(|packets_results| packets_results.len())
+                .sum::<usize>(),
+            votes.len()
+        );
+
+        let (vote_txs, packets) = izip!(
+            labels.into_iter(),
+            votes.into_iter(),
+            r.iter().flatten(),
+            msgs,
+        )
+        .filter_map(|(label, vote, verify_result, packet)| {
+            if *verify_result != 0 {
+                Some((vote, (label, packet)))
+            } else {
+                None
+            }
+        })
+        .unzip();
+        (vote_txs, packets)
+    }
+
     fn bank_send_loop(
         exit: Arc<AtomicBool>,
         verified_vote_packets_receiver: VerifiedVotePacketsReceiver,
@@ -364,6 +380,7 @@ impl ClusterInfoVoteListener {
         vote_txs_receiver: VerifiedVoteTransactionsReceiver,
         vote_tracker: Arc<VoteTracker>,
         bank_forks: &RwLock<BankForks>,
+        subscriptions: Arc<RpcSubscriptions>,
     ) -> Result<()> {
         loop {
             if exit.load(Ordering::Relaxed) {
@@ -372,10 +389,15 @@ impl ClusterInfoVoteListener {
 
             let root_bank = bank_forks.read().unwrap().root_bank().clone();
             vote_tracker.process_new_root_bank(&root_bank);
+            let epoch_stakes = root_bank.epoch_stakes(root_bank.epoch());
 
-            if let Err(e) =
-                Self::get_and_process_votes(&vote_txs_receiver, &vote_tracker, root_bank.slot())
-            {
+            if let Err(e) = Self::get_and_process_votes(
+                &vote_txs_receiver,
+                &vote_tracker,
+                root_bank.slot(),
+                subscriptions.clone(),
+                epoch_stakes,
+            ) {
                 match e {
                     Error::CrossbeamRecvTimeoutError(RecvTimeoutError::Disconnected) => {
                         return Ok(());
@@ -389,21 +411,51 @@ impl ClusterInfoVoteListener {
         }
     }
 
+    #[cfg(test)]
+    pub fn get_and_process_votes_for_tests(
+        vote_txs_receiver: &VerifiedVoteTransactionsReceiver,
+        vote_tracker: &Arc<VoteTracker>,
+        last_root: Slot,
+        subscriptions: Arc<RpcSubscriptions>,
+    ) -> Result<()> {
+        Self::get_and_process_votes(
+            vote_txs_receiver,
+            vote_tracker,
+            last_root,
+            subscriptions,
+            None,
+        )
+    }
+
     fn get_and_process_votes(
         vote_txs_receiver: &VerifiedVoteTransactionsReceiver,
         vote_tracker: &Arc<VoteTracker>,
         last_root: Slot,
+        subscriptions: Arc<RpcSubscriptions>,
+        epoch_stakes: Option<&EpochStakes>,
     ) -> Result<()> {
         let timer = Duration::from_millis(200);
         let mut vote_txs = vote_txs_receiver.recv_timeout(timer)?;
         while let Ok(new_txs) = vote_txs_receiver.try_recv() {
             vote_txs.extend(new_txs);
         }
-        Self::process_votes(vote_tracker, vote_txs, last_root);
+        Self::process_votes(
+            vote_tracker,
+            vote_txs,
+            last_root,
+            subscriptions,
+            epoch_stakes,
+        );
         Ok(())
     }
 
-    fn process_votes(vote_tracker: &VoteTracker, vote_txs: Vec<Transaction>, root: Slot) {
+    fn process_votes(
+        vote_tracker: &VoteTracker,
+        vote_txs: Vec<Transaction>,
+        root: Slot,
+        subscriptions: Arc<RpcSubscriptions>,
+        epoch_stakes: Option<&EpochStakes>,
+    ) {
         let mut diff: HashMap<Slot, HashSet<Arc<Pubkey>>> = HashMap::new();
         {
             let all_slot_trackers = &vote_tracker.slot_vote_trackers;
@@ -455,7 +507,7 @@ impl ClusterInfoVoteListener {
                         continue;
                     }
 
-                    for slot in vote.slots {
+                    for &slot in vote.slots.iter() {
                         if slot <= root {
                             continue;
                         }
@@ -480,6 +532,8 @@ impl ClusterInfoVoteListener {
                             .or_default()
                             .insert(unduplicated_pubkey.unwrap());
                     }
+
+                    subscriptions.notify_vote(&vote);
                 }
             }
         }
@@ -496,15 +550,35 @@ impl ClusterInfoVoteListener {
                 if w_slot_tracker.updates.is_none() {
                     w_slot_tracker.updates = Some(vec![]);
                 }
-                for pk in slot_diff {
-                    w_slot_tracker.voted.insert(pk.clone());
-                    w_slot_tracker.updates.as_mut().unwrap().push(pk);
+                let mut current_stake = 0;
+                for pubkey in slot_diff {
+                    Self::sum_stake(&mut current_stake, epoch_stakes, &pubkey);
+
+                    w_slot_tracker.voted.insert(pubkey.clone());
+                    w_slot_tracker.updates.as_mut().unwrap().push(pubkey);
                 }
+                Self::notify_for_stake_change(
+                    current_stake,
+                    w_slot_tracker.total_stake,
+                    &subscriptions,
+                    epoch_stakes,
+                    slot,
+                );
+                w_slot_tracker.total_stake += current_stake;
             } else {
-                let voted: HashSet<_> = slot_diff.into_iter().collect();
+                let mut total_stake = 0;
+                let voted: HashSet<_> = slot_diff
+                    .into_iter()
+                    .map(|pubkey| {
+                        Self::sum_stake(&mut total_stake, epoch_stakes, &pubkey);
+                        pubkey
+                    })
+                    .collect();
+                Self::notify_for_stake_change(total_stake, 0, &subscriptions, epoch_stakes, slot);
                 let new_slot_tracker = SlotVoteTracker {
                     voted: voted.clone(),
                     updates: Some(voted.into_iter().collect()),
+                    total_stake,
                 };
                 vote_tracker
                     .slot_vote_trackers
@@ -514,17 +588,45 @@ impl ClusterInfoVoteListener {
             }
         }
     }
+
+    fn notify_for_stake_change(
+        current_stake: u64,
+        previous_stake: u64,
+        subscriptions: &Arc<RpcSubscriptions>,
+        epoch_stakes: Option<&EpochStakes>,
+        slot: Slot,
+    ) {
+        if let Some(stakes) = epoch_stakes {
+            let supermajority_stake = (stakes.total_stake() as f64 * VOTE_THRESHOLD_SIZE) as u64;
+            if previous_stake < supermajority_stake
+                && (previous_stake + current_stake) > supermajority_stake
+            {
+                subscriptions.notify_gossip_subscribers(slot);
+            }
+        }
+    }
+
+    fn sum_stake(sum: &mut u64, epoch_stakes: Option<&EpochStakes>, pubkey: &Pubkey) {
+        if let Some(stakes) = epoch_stakes {
+            if let Some(vote_account) = stakes.stakes().vote_accounts().get(pubkey) {
+                *sum += vote_account.0;
+            }
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::commitment::BlockCommitmentCache;
+    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
     use solana_perf::packet;
     use solana_runtime::{
         bank::Bank,
         genesis_utils::{self, GenesisConfigInfo, ValidatorVoteKeypairs},
     };
     use solana_sdk::hash::Hash;
+    use solana_sdk::signature::Signature;
     use solana_sdk::signature::{Keypair, Signer};
     use solana_vote_program::vote_transaction;
 
@@ -614,7 +716,7 @@ mod tests {
 
     #[test]
     fn test_update_new_root() {
-        let (vote_tracker, bank, _) = setup();
+        let (vote_tracker, bank, _, _) = setup();
 
         // Check outdated slots are purged with new root
         let new_voter = Arc::new(Pubkey::new_rand());
@@ -655,7 +757,7 @@ mod tests {
 
     #[test]
     fn test_update_new_leader_schedule_epoch() {
-        let (vote_tracker, bank, _) = setup();
+        let (vote_tracker, bank, _, _) = setup();
 
         // Check outdated slots are purged with new root
         let leader_schedule_epoch = bank.get_leader_schedule_epoch(bank.slot());
@@ -697,7 +799,7 @@ mod tests {
     #[test]
     fn test_process_votes() {
         // Create some voters at genesis
-        let (vote_tracker, _, validator_voting_keypairs) = setup();
+        let (vote_tracker, _, validator_voting_keypairs, subscriptions) = setup();
         let (votes_sender, votes_receiver) = unbounded();
 
         let vote_slots = vec![1, 2];
@@ -716,7 +818,14 @@ mod tests {
         });
 
         // Check that all the votes were registered for each validator correctly
-        ClusterInfoVoteListener::get_and_process_votes(&votes_receiver, &vote_tracker, 0).unwrap();
+        ClusterInfoVoteListener::get_and_process_votes(
+            &votes_receiver,
+            &vote_tracker,
+            0,
+            subscriptions,
+            None,
+        )
+        .unwrap();
         for vote_slot in vote_slots {
             let slot_vote_tracker = vote_tracker.get_slot_vote_tracker(vote_slot).unwrap();
             let r_slot_vote_tracker = slot_vote_tracker.read().unwrap();
@@ -735,7 +844,7 @@ mod tests {
     #[test]
     fn test_process_votes2() {
         // Create some voters at genesis
-        let (vote_tracker, _, validator_voting_keypairs) = setup();
+        let (vote_tracker, _, validator_voting_keypairs, subscriptions) = setup();
         // Send some votes to process
         let (votes_sender, votes_receiver) = unbounded();
 
@@ -760,7 +869,14 @@ mod tests {
         }
 
         // Check that all the votes were registered for each validator correctly
-        ClusterInfoVoteListener::get_and_process_votes(&votes_receiver, &vote_tracker, 0).unwrap();
+        ClusterInfoVoteListener::get_and_process_votes(
+            &votes_receiver,
+            &vote_tracker,
+            0,
+            subscriptions,
+            None,
+        )
+        .unwrap();
         for (i, keyset) in validator_voting_keypairs.chunks(2).enumerate() {
             let slot_vote_tracker = vote_tracker.get_slot_vote_tracker(i as u64 + 1).unwrap();
             let r_slot_vote_tracker = &slot_vote_tracker.read().unwrap();
@@ -779,7 +895,7 @@ mod tests {
     #[test]
     fn test_get_voters_by_epoch() {
         // Create some voters at genesis
-        let (vote_tracker, bank, validator_voting_keypairs) = setup();
+        let (vote_tracker, bank, validator_voting_keypairs, _) = setup();
         let last_known_epoch = bank.get_leader_schedule_epoch(bank.slot());
         let last_known_slot = bank
             .epoch_schedule()
@@ -850,11 +966,23 @@ mod tests {
                 100,
             );
         let bank = Bank::new(&genesis_config);
+        let exit = Arc::new(AtomicBool::new(false));
+        let bank_forks = BankForks::new(0, bank);
+        let bank = bank_forks.get(0).unwrap().clone();
+        let vote_tracker = VoteTracker::new(&bank);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let subscriptions = Arc::new(RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(bank_forks)),
+            Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+                blockstore.clone(),
+            ))),
+        ));
 
         // Send a vote to process, should add a reference to the pubkey for that voter
         // in the tracker
         let validator0_keypairs = &validator_voting_keypairs[0];
-        let vote_tracker = VoteTracker::new(&bank);
         let vote_tx = vec![vote_transaction::new_vote_transaction(
             // Must vote > root to be processed
             vec![bank.slot() + 1],
@@ -865,7 +993,13 @@ mod tests {
             &validator0_keypairs.vote_keypair,
         )];
 
-        ClusterInfoVoteListener::process_votes(&vote_tracker, vote_tx, 0);
+        ClusterInfoVoteListener::process_votes(
+            &vote_tracker,
+            vote_tx,
+            0,
+            subscriptions.clone(),
+            None,
+        );
         let ref_count = Arc::strong_count(
             &vote_tracker
                 .keys
@@ -915,7 +1049,7 @@ mod tests {
             })
             .collect();
 
-        ClusterInfoVoteListener::process_votes(&vote_tracker, vote_txs, 0);
+        ClusterInfoVoteListener::process_votes(&vote_tracker, vote_txs, 0, subscriptions, None);
 
         let ref_count = Arc::strong_count(
             &vote_tracker
@@ -929,7 +1063,12 @@ mod tests {
         assert_eq!(ref_count, current_ref_count);
     }
 
-    fn setup() -> (Arc<VoteTracker>, Arc<Bank>, Vec<ValidatorVoteKeypairs>) {
+    fn setup() -> (
+        Arc<VoteTracker>,
+        Arc<Bank>,
+        Vec<ValidatorVoteKeypairs>,
+        Arc<RpcSubscriptions>,
+    ) {
         let validator_voting_keypairs: Vec<_> = (0..10)
             .map(|_| ValidatorVoteKeypairs::new(Keypair::new(), Keypair::new(), Keypair::new()))
             .collect();
@@ -941,6 +1080,18 @@ mod tests {
             );
         let bank = Bank::new(&genesis_config);
         let vote_tracker = VoteTracker::new(&bank);
+        let exit = Arc::new(AtomicBool::new(false));
+        let bank_forks = BankForks::new(0, bank);
+        let bank = bank_forks.get(0).unwrap().clone();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let subscriptions = Arc::new(RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(bank_forks)),
+            Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+                blockstore.clone(),
+            ))),
+        ));
 
         // Integrity Checks
         let current_epoch = bank.epoch();
@@ -967,8 +1118,66 @@ mod tests {
         assert_eq!(*vote_tracker.current_epoch.read().unwrap(), current_epoch);
         (
             Arc::new(vote_tracker),
-            Arc::new(bank),
+            bank,
             validator_voting_keypairs,
+            subscriptions,
         )
     }
+
+    #[test]
+    fn test_verify_votes_empty() {
+        solana_logger::setup();
+        let votes = vec![];
+        let labels = vec![];
+        let (vote_txs, packets) = ClusterInfoVoteListener::verify_votes(votes, labels, false);
+        assert!(vote_txs.is_empty());
+        assert!(packets.is_empty());
+    }
+
+    fn verify_packets_len(packets: &Vec<(CrdsValueLabel, Packets)>, ref_value: usize) {
+        let num_packets: usize = packets.iter().map(|p| p.1.packets.len()).sum();
+        assert_eq!(num_packets, ref_value);
+    }
+
+    fn test_vote_tx() -> Transaction {
+        let node_keypair = Keypair::new();
+        let vote_keypair = Keypair::new();
+        let auth_voter_keypair = Keypair::new();
+        let vote_tx = vote_transaction::new_vote_transaction(
+            vec![0],
+            Hash::default(),
+            Hash::default(),
+            &node_keypair,
+            &vote_keypair,
+            &auth_voter_keypair,
+        );
+
+        vote_tx
+    }
+
+    #[test]
+    fn test_verify_votes_1_pass() {
+        let vote_tx = test_vote_tx();
+        let votes = vec![vote_tx.clone()];
+        let labels = vec![CrdsValueLabel::Vote(0, Pubkey::new_rand())];
+        let (vote_txs, packets) = ClusterInfoVoteListener::verify_votes(votes, labels, false);
+        assert_eq!(vote_txs.len(), 1);
+        verify_packets_len(&packets, 1);
+    }
+
+    #[test]
+    fn test_bad_vote() {
+        let vote_tx = test_vote_tx();
+        let mut bad_vote = vote_tx.clone();
+        bad_vote.signatures[0] = Signature::default();
+        let votes = vec![vote_tx.clone(), bad_vote, vote_tx];
+        let label = CrdsValueLabel::Vote(0, Pubkey::new_rand());
+        let labels: Vec<_> = (0..votes.len())
+            .into_iter()
+            .map(|_| label.clone())
+            .collect();
+        let (vote_txs, packets) = ClusterInfoVoteListener::verify_votes(votes, labels, false);
+        assert_eq!(vote_txs.len(), 2);
+        verify_packets_len(&packets, 2);
+    }
 }

core/src/cluster_slots.rs

@@ -27,15 +27,10 @@ impl ClusterSlots {
     pub fn lookup(&self, slot: Slot) -> Option<Arc<RwLock<SlotPubkeys>>> {
         self.cluster_slots.read().unwrap().get(&slot).cloned()
     }
-    pub fn update(
-        &self,
-        root: Slot,
-        cluster_info: &RwLock<ClusterInfo>,
-        bank_forks: &RwLock<BankForks>,
-    ) {
+    pub fn update(&self, root: Slot, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
         self.update_peers(cluster_info, bank_forks);
         let since = *self.since.read().unwrap();
-        let epoch_slots = cluster_info.read().unwrap().get_epoch_slots_since(since);
+        let epoch_slots = cluster_info.get_epoch_slots_since(since);
         self.update_internal(root, epoch_slots);
     }
     fn update_internal(&self, root: Slot, epoch_slots: (Vec<EpochSlots>, Option<u64>)) {
@@ -95,7 +90,7 @@ impl ClusterSlots {
             .collect()
     }
 
-    fn update_peers(&self, cluster_info: &RwLock<ClusterInfo>, bank_forks: &RwLock<BankForks>) {
+    fn update_peers(&self, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
         let root_bank = bank_forks.read().unwrap().root_bank().clone();
         let root_epoch = root_bank.epoch();
         let my_epoch = *self.epoch.read().unwrap();
@@ -111,7 +106,7 @@ impl ClusterSlots {
                 .clone();
 
             *self.validator_stakes.write().unwrap() = validator_stakes;
-            let id = cluster_info.read().unwrap().id();
+            let id = cluster_info.id();
             *self.self_id.write().unwrap() = id;
             *self.epoch.write().unwrap() = Some(root_epoch);
         }

core/src/commitment.rs

@@ -1,6 +1,7 @@
-use crate::consensus::VOTE_THRESHOLD_SIZE;
+use crate::{consensus::VOTE_THRESHOLD_SIZE, rpc_subscriptions::RpcSubscriptions};
+use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
-use solana_metrics::inc_new_counter_info;
+use solana_metrics::datapoint_info;
 use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
 use solana_vote_program::{vote_state::VoteState, vote_state::MAX_LOCKOUT_HISTORY};
@@ -13,9 +14,19 @@ use std::{
     time::Duration,
 };
 
+#[derive(Default)]
+pub struct CacheSlotInfo {
+    pub current_slot: Slot,
+    pub node_root: Slot,
+    pub largest_confirmed_root: Slot,
+    pub highest_confirmed_slot: Slot,
+}
+
+pub type BlockCommitmentArray = [u64; MAX_LOCKOUT_HISTORY + 1];
+
 #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
 pub struct BlockCommitment {
-    pub commitment: [u64; MAX_LOCKOUT_HISTORY],
+    pub commitment: BlockCommitmentArray,
 }
 
 impl BlockCommitment {
@@ -28,18 +39,29 @@ impl BlockCommitment {
         assert!(confirmation_count > 0 && confirmation_count <= MAX_LOCKOUT_HISTORY);
         self.commitment[confirmation_count - 1]
     }
+
+    pub fn increase_rooted_stake(&mut self, stake: u64) {
+        self.commitment[MAX_LOCKOUT_HISTORY] += stake;
+    }
+
+    pub fn get_rooted_stake(&self) -> u64 {
+        self.commitment[MAX_LOCKOUT_HISTORY]
+    }
+
     #[cfg(test)]
-    pub(crate) fn new(commitment: [u64; MAX_LOCKOUT_HISTORY]) -> Self {
+    pub(crate) fn new(commitment: BlockCommitmentArray) -> Self {
         Self { commitment }
     }
 }
 
-#[derive(Default)]
 pub struct BlockCommitmentCache {
     block_commitment: HashMap<Slot, BlockCommitment>,
+    largest_confirmed_root: Slot,
     total_stake: u64,
     bank: Arc<Bank>,
+    blockstore: Arc<Blockstore>,
     root: Slot,
+    highest_confirmed_slot: Slot,
 }
 
 impl std::fmt::Debug for BlockCommitmentCache {
@@ -59,15 +81,33 @@ impl std::fmt::Debug for BlockCommitmentCache {
 impl BlockCommitmentCache {
     pub fn new(
         block_commitment: HashMap<Slot, BlockCommitment>,
+        largest_confirmed_root: Slot,
         total_stake: u64,
         bank: Arc<Bank>,
+        blockstore: Arc<Blockstore>,
         root: Slot,
+        highest_confirmed_slot: Slot,
     ) -> Self {
         Self {
             block_commitment,
+            largest_confirmed_root,
             total_stake,
             bank,
+            blockstore,
             root,
+            highest_confirmed_slot,
+        }
+    }
+
+    pub fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        Self {
+            block_commitment: HashMap::default(),
+            largest_confirmed_root: Slot::default(),
+            total_stake: u64::default(),
+            bank: Arc::new(Bank::default()),
+            blockstore,
+            root: Slot::default(),
+            highest_confirmed_slot: Slot::default(),
         }
     }
 
@@ -75,6 +115,10 @@ impl BlockCommitmentCache {
         self.block_commitment.get(&slot)
     }
 
+    pub fn largest_confirmed_root(&self) -> Slot {
+        self.largest_confirmed_root
+    }
+
     pub fn total_stake(&self) -> u64 {
         self.total_stake
     }
@@ -91,6 +135,26 @@ impl BlockCommitmentCache {
         self.root
     }
 
+    pub fn highest_confirmed_slot(&self) -> Slot {
+        self.highest_confirmed_slot
+    }
+
+    fn highest_slot_with_confirmation_count(&self, confirmation_count: usize) -> Slot {
+        assert!(confirmation_count > 0 && confirmation_count <= MAX_LOCKOUT_HISTORY);
+        for slot in (self.root()..self.slot()).rev() {
+            if let Some(count) = self.get_confirmation_count(slot) {
+                if count >= confirmation_count {
+                    return slot;
+                }
+            }
+        }
+        self.root
+    }
+
+    fn calculate_highest_confirmed_slot(&self) -> Slot {
+        self.highest_slot_with_confirmation_count(1)
+    }
+
     pub fn get_confirmation_count(&self, slot: Slot) -> Option<usize> {
         self.get_lockout_count(slot, VOTE_THRESHOLD_SIZE)
     }
@@ -110,16 +174,49 @@ impl BlockCommitmentCache {
             0
         })
     }
+
+    pub fn is_confirmed_rooted(&self, slot: Slot) -> bool {
+        slot <= self.largest_confirmed_root()
+            && (self.blockstore.is_root(slot) || self.bank.status_cache_ancestors().contains(&slot))
+    }
+
     #[cfg(test)]
-    pub fn new_for_tests() -> Self {
+    pub fn new_for_tests_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
         let mut block_commitment: HashMap<Slot, BlockCommitment> = HashMap::new();
         block_commitment.insert(0, BlockCommitment::default());
         Self {
             block_commitment,
+            blockstore,
             total_stake: 42,
-            ..Self::default()
+            largest_confirmed_root: Slot::default(),
+            bank: Arc::new(Bank::default()),
+            root: Slot::default(),
+            highest_confirmed_slot: Slot::default(),
         }
     }
+
+    #[cfg(test)]
+    pub fn new_for_tests_with_blockstore_bank(
+        blockstore: Arc<Blockstore>,
+        bank: Arc<Bank>,
+        root: Slot,
+    ) -> Self {
+        let mut block_commitment: HashMap<Slot, BlockCommitment> = HashMap::new();
+        block_commitment.insert(0, BlockCommitment::default());
+        Self {
+            block_commitment,
+            blockstore,
+            total_stake: 42,
+            largest_confirmed_root: root,
+            bank,
+            root,
+            highest_confirmed_slot: root,
+        }
+    }
+
+    pub(crate) fn set_largest_confirmed_root(&mut self, root: Slot) {
+        self.largest_confirmed_root = root;
+    }
 }
 
 pub struct CommitmentAggregationData {
@@ -138,6 +235,18 @@ impl CommitmentAggregationData {
     }
 }
 
+fn get_largest_confirmed_root(mut rooted_stake: Vec<(Slot, u64)>, total_stake: u64) -> Slot {
+    rooted_stake.sort_by(|a, b| a.0.cmp(&b.0).reverse());
+    let mut stake_sum = 0;
+    for (root, stake) in rooted_stake {
+        stake_sum += stake;
+        if (stake_sum as f64 / total_stake as f64) > VOTE_THRESHOLD_SIZE {
+            return root;
+        }
+    }
+    0
+}
+
 pub struct AggregateCommitmentService {
     t_commitment: JoinHandle<()>,
 }
@@ -146,6 +255,7 @@ impl AggregateCommitmentService {
     pub fn new(
         exit: &Arc<AtomicBool>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        subscriptions: Arc<RpcSubscriptions>,
     ) -> (Sender<CommitmentAggregationData>, Self) {
         let (sender, receiver): (
             Sender<CommitmentAggregationData>,
@@ -163,7 +273,7 @@ impl AggregateCommitmentService {
                         }
 
                         if let Err(RecvTimeoutError::Disconnected) =
-                            Self::run(&receiver, &block_commitment_cache, &exit_)
+                            Self::run(&receiver, &block_commitment_cache, &subscriptions, &exit_)
                         {
                             break;
                         }
@@ -176,6 +286,7 @@ impl AggregateCommitmentService {
     fn run(
         receiver: &Receiver<CommitmentAggregationData>,
         block_commitment_cache: &RwLock<BlockCommitmentCache>,
+        subscriptions: &Arc<RpcSubscriptions>,
         exit: &Arc<AtomicBool>,
     ) -> Result<(), RecvTimeoutError> {
         loop {
@@ -195,27 +306,50 @@ impl AggregateCommitmentService {
             }
 
             let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
-            let block_commitment = Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+            let (block_commitment, rooted_stake) =
+                Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+
+            let largest_confirmed_root =
+                get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
 
             let mut new_block_commitment = BlockCommitmentCache::new(
                 block_commitment,
+                largest_confirmed_root,
                 aggregation_data.total_staked,
                 aggregation_data.bank,
+                block_commitment_cache.read().unwrap().blockstore.clone(),
+                aggregation_data.root,
                 aggregation_data.root,
             );
+            new_block_commitment.highest_confirmed_slot =
+                new_block_commitment.calculate_highest_confirmed_slot();
 
             let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
 
             std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
             aggregate_commitment_time.stop();
-            inc_new_counter_info!(
-                "aggregate-commitment-ms",
-                aggregate_commitment_time.as_ms() as usize
+            datapoint_info!(
+                "block-commitment-cache",
+                (
+                    "aggregate-commitment-ms",
+                    aggregate_commitment_time.as_ms() as i64,
+                    i64
+                )
             );
+
+            subscriptions.notify_subscribers(CacheSlotInfo {
+                current_slot: w_block_commitment_cache.slot(),
+                node_root: w_block_commitment_cache.root(),
+                largest_confirmed_root: w_block_commitment_cache.largest_confirmed_root(),
+                highest_confirmed_slot: w_block_commitment_cache.highest_confirmed_slot(),
+            });
         }
     }
 
-    pub fn aggregate_commitment(ancestors: &[Slot], bank: &Bank) -> HashMap<Slot, BlockCommitment> {
+    pub fn aggregate_commitment(
+        ancestors: &[Slot],
+        bank: &Bank,
+    ) -> (HashMap<Slot, BlockCommitment>, Vec<(Slot, u64)>) {
         assert!(!ancestors.is_empty());
 
         // Check ancestors is sorted
@@ -224,6 +358,7 @@ impl AggregateCommitmentService {
         }
 
         let mut commitment = HashMap::new();
+        let mut rooted_stake: Vec<(Slot, u64)> = Vec::new();
         for (_, (lamports, account)) in bank.vote_accounts().into_iter() {
             if lamports == 0 {
                 continue;
@@ -236,17 +371,19 @@ impl AggregateCommitmentService {
             let vote_state = vote_state.unwrap();
             Self::aggregate_commitment_for_vote_account(
                 &mut commitment,
+                &mut rooted_stake,
                 &vote_state,
                 ancestors,
                 lamports,
             );
         }
 
-        commitment
+        (commitment, rooted_stake)
     }
 
     fn aggregate_commitment_for_vote_account(
         commitment: &mut HashMap<Slot, BlockCommitment>,
+        rooted_stake: &mut Vec<(Slot, u64)>,
         vote_state: &VoteState,
         ancestors: &[Slot],
         lamports: u64,
@@ -259,12 +396,13 @@ impl AggregateCommitmentService {
                     commitment
                         .entry(*a)
                         .or_insert_with(BlockCommitment::default)
-                        .increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                        .increase_rooted_stake(lamports);
                 } else {
                     ancestors_index = i;
                     break;
                 }
             }
+            rooted_stake.push((root, lamports));
         }
 
         for vote in &vote_state.votes {
@@ -290,8 +428,11 @@ impl AggregateCommitmentService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use solana_sdk::pubkey::Pubkey;
+    use solana_ledger::{
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+    };
+    use solana_sdk::{genesis_config::GenesisConfig, pubkey::Pubkey};
     use solana_stake_program::stake_state;
     use solana_vote_program::vote_state::{self, VoteStateVersions};
 
@@ -308,6 +449,8 @@ mod tests {
     #[test]
     fn test_get_confirmations() {
         let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         // Build BlockCommitmentCache with votes at depths 0 and 1 for 2 slots
         let mut cache0 = BlockCommitment::default();
         cache0.increase_confirmation_stake(1, 5);
@@ -325,7 +468,8 @@ mod tests {
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
         block_commitment.entry(2).or_insert(cache2.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50, bank, 0);
+        let block_commitment_cache =
+            BlockCommitmentCache::new(block_commitment, 0, 50, bank, blockstore, 0, 0);
 
         assert_eq!(block_commitment_cache.get_confirmation_count(0), Some(2));
         assert_eq!(block_commitment_cache.get_confirmation_count(1), Some(1));
@@ -333,17 +477,177 @@ mod tests {
         assert_eq!(block_commitment_cache.get_confirmation_count(3), None,);
     }
 
+    #[test]
+    fn test_is_confirmed_rooted() {
+        let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        blockstore.set_roots(&[0, 1]).unwrap();
+        // Build BlockCommitmentCache with rooted slots
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_rooted_stake(50);
+        let mut cache1 = BlockCommitment::default();
+        cache1.increase_rooted_stake(40);
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_rooted_stake(20);
+
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache0.clone());
+        block_commitment.entry(2).or_insert(cache1.clone());
+        block_commitment.entry(3).or_insert(cache2.clone());
+        let largest_confirmed_root = 1;
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            largest_confirmed_root,
+            50,
+            bank,
+            blockstore,
+            0,
+            0,
+        );
+
+        assert!(block_commitment_cache.is_confirmed_rooted(0));
+        assert!(block_commitment_cache.is_confirmed_rooted(1));
+        assert!(!block_commitment_cache.is_confirmed_rooted(2));
+        assert!(!block_commitment_cache.is_confirmed_rooted(3));
+    }
+
+    #[test]
+    fn test_get_largest_confirmed_root() {
+        assert_eq!(get_largest_confirmed_root(vec![], 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((0, 5));
+        rooted_stake.push((1, 5));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((1, 5));
+        rooted_stake.push((0, 10));
+        rooted_stake.push((2, 5));
+        rooted_stake.push((1, 4));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 1);
+    }
+
+    #[test]
+    fn test_highest_confirmed_slot() {
+        let bank = Arc::new(Bank::new(&GenesisConfig::default()));
+        let bank_slot_5 = Arc::new(Bank::new_from_parent(&bank, &Pubkey::default(), 5));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let total_stake = 50;
+
+        // Build cache with confirmation_count 2 given total_stake
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_confirmation_stake(1, 5);
+        cache0.increase_confirmation_stake(2, 40);
+
+        // Build cache with confirmation_count 1 given total_stake
+        let mut cache1 = BlockCommitment::default();
+        cache1.increase_confirmation_stake(1, 40);
+        cache1.increase_confirmation_stake(2, 5);
+
+        // Build cache with confirmation_count 0 given total_stake
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_confirmation_stake(1, 20);
+        cache2.increase_confirmation_stake(2, 5);
+
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache0.clone()); // Slot 1, conf 2
+        block_commitment.entry(2).or_insert(cache1.clone()); // Slot 2, conf 1
+        block_commitment.entry(3).or_insert(cache2.clone()); // Slot 3, conf 0
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            total_stake,
+            bank_slot_5.clone(),
+            blockstore.clone(),
+            0,
+            0,
+        );
+
+        assert_eq!(block_commitment_cache.calculate_highest_confirmed_slot(), 2);
+
+        // Build map with multiple slots at conf 1
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache1.clone()); // Slot 1, conf 1
+        block_commitment.entry(2).or_insert(cache1.clone()); // Slot 2, conf 1
+        block_commitment.entry(3).or_insert(cache2.clone()); // Slot 3, conf 0
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            total_stake,
+            bank_slot_5.clone(),
+            blockstore.clone(),
+            0,
+            0,
+        );
+
+        assert_eq!(block_commitment_cache.calculate_highest_confirmed_slot(), 2);
+
+        // Build map with slot gaps
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache1.clone()); // Slot 1, conf 1
+        block_commitment.entry(3).or_insert(cache1.clone()); // Slot 3, conf 1
+        block_commitment.entry(5).or_insert(cache2.clone()); // Slot 5, conf 0
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            total_stake,
+            bank_slot_5.clone(),
+            blockstore.clone(),
+            0,
+            0,
+        );
+
+        assert_eq!(block_commitment_cache.calculate_highest_confirmed_slot(), 3);
+
+        // Build map with no conf 1 slots, but one higher
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache0.clone()); // Slot 1, conf 2
+        block_commitment.entry(2).or_insert(cache2.clone()); // Slot 2, conf 0
+        block_commitment.entry(3).or_insert(cache2.clone()); // Slot 3, conf 0
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            total_stake,
+            bank_slot_5.clone(),
+            blockstore.clone(),
+            0,
+            0,
+        );
+
+        assert_eq!(block_commitment_cache.calculate_highest_confirmed_slot(), 1);
+
+        // Build map with no conf 1 or higher slots
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache2.clone()); // Slot 1, conf 0
+        block_commitment.entry(2).or_insert(cache2.clone()); // Slot 2, conf 0
+        block_commitment.entry(3).or_insert(cache2.clone()); // Slot 3, conf 0
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            total_stake,
+            bank_slot_5.clone(),
+            blockstore.clone(),
+            0,
+            0,
+        );
+
+        assert_eq!(block_commitment_cache.calculate_highest_confirmed_slot(), 0);
+    }
+
     #[test]
     fn test_aggregate_commitment_for_vote_account_1() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
-        let root = ancestors.last().unwrap();
-        vote_state.root_slot = Some(*root);
+        let root = ancestors.last().unwrap().clone();
+        vote_state.root_slot = Some(root);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -351,15 +655,17 @@ mod tests {
 
         for a in ancestors {
             let mut expected = BlockCommitment::default();
-            expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+            expected.increase_rooted_stake(lamports);
             assert_eq!(*commitment.get(&a).unwrap(), expected);
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_2() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -368,6 +674,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(*ancestors.last().unwrap());
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -376,7 +683,7 @@ mod tests {
         for a in ancestors {
             if a <= root {
                 let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                expected.increase_rooted_stake(lamports);
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             } else {
                 let mut expected = BlockCommitment::default();
@@ -384,12 +691,14 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_3() {
         let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -400,6 +709,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(ancestors[6]);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -408,7 +718,7 @@ mod tests {
         for (i, a) in ancestors.iter().enumerate() {
             if *a <= root {
                 let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                expected.increase_rooted_stake(lamports);
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             } else if i <= 4 {
                 let mut expected = BlockCommitment::default();
@@ -420,6 +730,7 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
@@ -429,6 +740,8 @@ mod tests {
             mut genesis_config, ..
         } = create_genesis_config(10_000);
 
+        let rooted_stake_amount = 40;
+
         let sk1 = Pubkey::new_rand();
         let pk1 = Pubkey::new_rand();
         let mut vote_account1 = vote_state::create_account(&pk1, &Pubkey::new_rand(), 0, 100);
@@ -439,12 +752,36 @@ mod tests {
         let mut vote_account2 = vote_state::create_account(&pk2, &Pubkey::new_rand(), 0, 50);
         let stake_account2 =
             stake_state::create_account(&sk2, &pk2, &vote_account2, &genesis_config.rent, 50);
+        let sk3 = Pubkey::new_rand();
+        let pk3 = Pubkey::new_rand();
+        let mut vote_account3 = vote_state::create_account(&pk3, &Pubkey::new_rand(), 0, 1);
+        let stake_account3 = stake_state::create_account(
+            &sk3,
+            &pk3,
+            &vote_account3,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
+        let sk4 = Pubkey::new_rand();
+        let pk4 = Pubkey::new_rand();
+        let mut vote_account4 = vote_state::create_account(&pk4, &Pubkey::new_rand(), 0, 1);
+        let stake_account4 = stake_state::create_account(
+            &sk4,
+            &pk4,
+            &vote_account4,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
 
         genesis_config.accounts.extend(vec![
             (pk1, vote_account1.clone()),
             (sk1, stake_account1),
             (pk2, vote_account2.clone()),
             (sk2, stake_account2),
+            (pk3, vote_account3.clone()),
+            (sk3, stake_account3),
+            (pk4, vote_account4.clone()),
+            (sk4, stake_account4),
         ]);
 
         // Create bank
@@ -464,7 +801,20 @@ mod tests {
         VoteState::to(&versioned, &mut vote_account2).unwrap();
         bank.store_account(&pk2, &vote_account2);
 
-        let commitment = AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
+        let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
+        vote_state3.root_slot = Some(1);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
+        VoteState::to(&versioned, &mut vote_account3).unwrap();
+        bank.store_account(&pk3, &vote_account3);
+
+        let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
+        vote_state4.root_slot = Some(2);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
+        VoteState::to(&versioned, &mut vote_account4).unwrap();
+        bank.store_account(&pk4, &vote_account4);
+
+        let (commitment, rooted_stake) =
+            AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
 
         for a in ancestors {
             if a <= 3 {
@@ -488,5 +838,7 @@ mod tests {
                 assert!(commitment.get(&a).is_none());
             }
         }
+        assert_eq!(rooted_stake.len(), 2);
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 100), 1)
     }
 }

core/src/consensus.rs

@@ -637,6 +637,7 @@ pub mod test {
                     &mut self.progress,
                     &None,
                     &mut HashSet::new(),
+                    None,
                 );
             }
 

core/src/contact_info.rs

@@ -1,6 +1,8 @@
+use crate::crds_value::MAX_WALLCLOCK;
 use solana_sdk::pubkey::Pubkey;
 #[cfg(test)]
 use solana_sdk::rpc_port;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 #[cfg(test)]
 use solana_sdk::signature::{Keypair, Signer};
 use solana_sdk::timing::timestamp;
@@ -37,6 +39,15 @@ pub struct ContactInfo {
     pub shred_version: u16,
 }
 
+impl Sanitize for ContactInfo {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 impl Ord for ContactInfo {
     fn cmp(&self, other: &Self) -> Ordering {
         self.id.cmp(&other.id)
@@ -314,4 +325,12 @@ mod tests {
         ci.rpc = socketaddr!("127.0.0.1:234");
         assert!(ci.valid_client_facing_addr().is_some());
     }
+
+    #[test]
+    fn test_sanitize() {
+        let mut ci = ContactInfo::default();
+        assert_eq!(ci.sanitize(), Ok(()));
+        ci.wallclock = MAX_WALLCLOCK;
+        assert_eq!(ci.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
 }

core/src/crds.rs

@@ -93,6 +93,24 @@ impl Crds {
     pub fn new_versioned(&self, local_timestamp: u64, value: CrdsValue) -> VersionedCrdsValue {
         VersionedCrdsValue::new(local_timestamp, value)
     }
+    pub fn would_insert(
+        &self,
+        value: CrdsValue,
+        local_timestamp: u64,
+    ) -> Option<VersionedCrdsValue> {
+        let new_value = self.new_versioned(local_timestamp, value);
+        let label = new_value.value.label();
+        let would_insert = self
+            .table
+            .get(&label)
+            .map(|current| new_value > *current)
+            .unwrap_or(true);
+        if would_insert {
+            Some(new_value)
+        } else {
+            None
+        }
+    }
     /// insert the new value, returns the old value if insert succeeds
     pub fn insert_versioned(
         &mut self,

core/src/crds_gossip.rs

@@ -6,7 +6,7 @@
 use crate::{
     crds::{Crds, VersionedCrdsValue},
     crds_gossip_error::CrdsGossipError,
-    crds_gossip_pull::{CrdsFilter, CrdsGossipPull},
+    crds_gossip_pull::{CrdsFilter, CrdsGossipPull, ProcessPullStats},
     crds_gossip_push::{CrdsGossipPush, CRDS_GOSSIP_NUM_ACTIVE},
     crds_value::{CrdsValue, CrdsValueLabel},
 };
@@ -20,6 +20,7 @@ pub const CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS: usize = 500;
 pub struct CrdsGossip {
     pub crds: Crds,
     pub id: Pubkey,
+    pub shred_version: u16,
     pub push: CrdsGossipPush,
     pub pull: CrdsGossipPull,
 }
@@ -29,6 +30,7 @@ impl Default for CrdsGossip {
         CrdsGossip {
             crds: Crds::default(),
             id: Pubkey::default(),
+            shred_version: 0,
             push: CrdsGossipPush::default(),
             pull: CrdsGossipPull::default(),
         }
@@ -39,6 +41,9 @@ impl CrdsGossip {
     pub fn set_self(&mut self, id: &Pubkey) {
         self.id = *id;
     }
+    pub fn set_shred_version(&mut self, shred_version: u16) {
+        self.shred_version = shred_version;
+    }
 
     /// process a push message to the network
     pub fn process_push_message(
@@ -122,6 +127,7 @@ impl CrdsGossip {
             &self.crds,
             stakes,
             &self.id,
+            self.shred_version,
             self.pull.pull_request_time.len(),
             CRDS_GOSSIP_NUM_ACTIVE,
         )
@@ -134,8 +140,14 @@ impl CrdsGossip {
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        self.pull
-            .new_pull_request(&self.crds, &self.id, now, stakes, bloom_size)
+        self.pull.new_pull_request(
+            &self.crds,
+            &self.id,
+            self.shred_version,
+            now,
+            stakes,
+            bloom_size,
+        )
     }
 
     /// time when a request to `from` was initiated
@@ -146,24 +158,46 @@ impl CrdsGossip {
         self.pull.mark_pull_request_creation_time(from, now)
     }
     /// process a pull request and create a response
-    pub fn process_pull_requests(
-        &mut self,
-        filters: Vec<(CrdsValue, CrdsFilter)>,
-        now: u64,
-    ) -> Vec<Vec<CrdsValue>> {
+    pub fn process_pull_requests(&mut self, filters: Vec<(CrdsValue, CrdsFilter)>, now: u64) {
         self.pull
-            .process_pull_requests(&mut self.crds, filters, now)
+            .process_pull_requests(&mut self.crds, filters, now);
     }
-    /// process a pull response
-    pub fn process_pull_response(
-        &mut self,
-        from: &Pubkey,
+
+    pub fn generate_pull_responses(
+        &self,
+        filters: &[(CrdsValue, CrdsFilter)],
+    ) -> Vec<Vec<CrdsValue>> {
+        self.pull.generate_pull_responses(&self.crds, filters)
+    }
+
+    pub fn filter_pull_responses(
+        &self,
         timeouts: &HashMap<Pubkey, u64>,
         response: Vec<CrdsValue>,
         now: u64,
-    ) -> usize {
+        process_pull_stats: &mut ProcessPullStats,
+    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>) {
         self.pull
-            .process_pull_response(&mut self.crds, from, timeouts, response, now)
+            .filter_pull_responses(&self.crds, timeouts, response, now, process_pull_stats)
+    }
+
+    /// process a pull response
+    pub fn process_pull_responses(
+        &mut self,
+        from: &Pubkey,
+        responses: Vec<VersionedCrdsValue>,
+        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        now: u64,
+        process_pull_stats: &mut ProcessPullStats,
+    ) {
+        self.pull.process_pull_responses(
+            &mut self.crds,
+            from,
+            responses,
+            responses_expired_timeout,
+            now,
+            process_pull_stats,
+        )
     }
 
     pub fn make_timeouts_test(&self) -> HashMap<Pubkey, u64> {

core/src/crds_gossip_pull.rs

@@ -10,19 +10,18 @@
 //! of false positives.
 
 use crate::contact_info::ContactInfo;
-use crate::crds::Crds;
+use crate::crds::{Crds, VersionedCrdsValue};
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
-use rand;
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::Rng;
 use solana_runtime::bloom::Bloom;
 use solana_sdk::hash::Hash;
 use solana_sdk::pubkey::Pubkey;
 use std::cmp;
-use std::collections::HashMap;
 use std::collections::VecDeque;
+use std::collections::{HashMap, HashSet};
 
 pub const CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS: u64 = 15000;
 // The maximum age of a value received over pull responses
@@ -37,6 +36,13 @@ pub struct CrdsFilter {
     mask_bits: u32,
 }
 
+impl solana_sdk::sanitize::Sanitize for CrdsFilter {
+    fn sanitize(&self) -> std::result::Result<(), solana_sdk::sanitize::SanitizeError> {
+        self.filter.sanitize()?;
+        Ok(())
+    }
+}
+
 impl CrdsFilter {
     pub fn new_rand(num_items: usize, max_bytes: usize) -> Self {
         let max_bits = (max_bytes * 8) as f64;
@@ -112,6 +118,14 @@ impl CrdsFilter {
     }
 }
 
+#[derive(Default)]
+pub struct ProcessPullStats {
+    pub success: usize,
+    pub failed_insert: usize,
+    pub failed_timeout: usize,
+    pub timeout_count: usize,
+}
+
 #[derive(Clone)]
 pub struct CrdsGossipPull {
     /// timestamp of last request
@@ -138,11 +152,12 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        let options = self.pull_options(crds, &self_id, now, stakes);
+        let options = self.pull_options(crds, &self_id, self_shred_version, now, stakes);
         if options.is_empty() {
             return Err(CrdsGossipError::NoPeers);
         }
@@ -159,13 +174,20 @@ impl CrdsGossipPull {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter_map(|v| v.value.contact_info())
-            .filter(|v| v.id != *self_id && ContactInfo::is_valid_address(&v.gossip))
+            .filter(|v| {
+                v.id != *self_id
+                    && ContactInfo::is_valid_address(&v.gossip)
+                    && (self_shred_version == 0
+                        || v.shred_version == 0
+                        || self_shred_version == v.shred_version)
+            })
             .map(|item| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let req_time: u64 = *self.pull_request_time.get(&item.id).unwrap_or(&0);
@@ -190,14 +212,13 @@ impl CrdsGossipPull {
         self.purged_values.push_back((hash, timestamp))
     }
 
-    /// process a pull request and create a response
+    /// process a pull request
     pub fn process_pull_requests(
         &mut self,
         crds: &mut Crds,
         requests: Vec<(CrdsValue, CrdsFilter)>,
         now: u64,
-    ) -> Vec<Vec<CrdsValue>> {
-        let rv = self.filter_crds_values(crds, &requests);
+    ) {
         requests.into_iter().for_each(|(caller, _)| {
             let key = caller.label().pubkey();
             let old = crds.insert(caller, now);
@@ -207,19 +228,33 @@ impl CrdsGossipPull {
             }
             crds.update_record_timestamp(&key, now);
         });
-        rv
     }
-    /// process a pull response
-    pub fn process_pull_response(
-        &mut self,
-        crds: &mut Crds,
-        from: &Pubkey,
+
+    /// Create gossip responses to pull requests
+    pub fn generate_pull_responses(
+        &self,
+        crds: &Crds,
+        requests: &[(CrdsValue, CrdsFilter)],
+    ) -> Vec<Vec<CrdsValue>> {
+        self.filter_crds_values(crds, requests)
+    }
+
+    // Checks if responses should be inserted and
+    // returns those responses converted to VersionedCrdsValue
+    // Separated in two vecs as:
+    //  .0 => responses that update the owner timestamp
+    //  .1 => responses that do not update the owner timestamp
+    pub fn filter_pull_responses(
+        &self,
+        crds: &Crds,
         timeouts: &HashMap<Pubkey, u64>,
-        response: Vec<CrdsValue>,
+        responses: Vec<CrdsValue>,
         now: u64,
-    ) -> usize {
-        let mut failed = 0;
-        for r in response {
+        stats: &mut ProcessPullStats,
+    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>) {
+        let mut versioned = vec![];
+        let mut versioned_expired_timestamp = vec![];
+        for r in responses {
             let owner = r.label().pubkey();
             // Check if the crds value is older than the msg_timeout
             if now
@@ -238,11 +273,8 @@ impl CrdsGossipPull {
                         if now > r.wallclock().checked_add(timeout).unwrap_or_else(|| 0)
                             || now + timeout < r.wallclock()
                         {
-                            inc_new_counter_warn!(
-                                "cluster_info-gossip_pull_response_value_timeout",
-                                1
-                            );
-                            failed += 1;
+                            stats.timeout_count += 1;
+                            stats.failed_timeout += 1;
                             continue;
                         }
                     }
@@ -250,32 +282,62 @@ impl CrdsGossipPull {
                         // Before discarding this value, check if a ContactInfo for the owner
                         // exists in the table. If it doesn't, that implies that this value can be discarded
                         if crds.lookup(&CrdsValueLabel::ContactInfo(owner)).is_none() {
-                            inc_new_counter_warn!(
-                                "cluster_info-gossip_pull_response_value_timeout",
-                                1
-                            );
-                            failed += 1;
+                            stats.timeout_count += 1;
+                            stats.failed_timeout += 1;
                             continue;
                         } else {
                             // Silently insert this old value without bumping record timestamps
-                            failed += crds.insert(r, now).is_err() as usize;
+                            match crds.would_insert(r, now) {
+                                Some(resp) => versioned_expired_timestamp.push(resp),
+                                None => stats.failed_insert += 1,
+                            }
                             continue;
                         }
                     }
                 }
             }
-            let old = crds.insert(r, now);
-            failed += old.is_err() as usize;
+            match crds.would_insert(r, now) {
+                Some(resp) => versioned.push(resp),
+                None => stats.failed_insert += 1,
+            }
+        }
+        (versioned, versioned_expired_timestamp)
+    }
+
+    /// process a vec of pull responses
+    pub fn process_pull_responses(
+        &mut self,
+        crds: &mut Crds,
+        from: &Pubkey,
+        responses: Vec<VersionedCrdsValue>,
+        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        now: u64,
+        stats: &mut ProcessPullStats,
+    ) {
+        let mut owners = HashSet::new();
+        for r in responses_expired_timeout {
+            stats.failed_insert += crds.insert_versioned(r).is_err() as usize;
+        }
+        for r in responses {
+            let owner = r.value.label().pubkey();
+            let old = crds.insert_versioned(r);
+            if old.is_err() {
+                stats.failed_insert += 1;
+            } else {
+                stats.success += 1;
+            }
             old.ok().map(|opt| {
-                crds.update_record_timestamp(&owner, now);
+                owners.insert(owner);
                 opt.map(|val| {
                     self.purged_values
                         .push_back((val.value_hash, val.local_timestamp))
                 })
             });
         }
-        crds.update_record_timestamp(from, now);
-        failed
+        owners.insert(*from);
+        for owner in owners {
+            crds.update_record_timestamp(&owner, now);
+        }
     }
     // build a set of filters of the current crds table
     // num_filters - used to increase the likelyhood of a value in crds being added to some filter
@@ -365,6 +427,34 @@ impl CrdsGossipPull {
             .count();
         self.purged_values.drain(..cnt);
     }
+
+    /// For legacy tests
+    #[cfg(test)]
+    pub fn process_pull_response(
+        &mut self,
+        crds: &mut Crds,
+        from: &Pubkey,
+        timeouts: &HashMap<Pubkey, u64>,
+        response: Vec<CrdsValue>,
+        now: u64,
+    ) -> (usize, usize, usize) {
+        let mut stats = ProcessPullStats::default();
+        let (versioned, versioned_expired_timeout) =
+            self.filter_pull_responses(crds, timeouts, response, now, &mut stats);
+        self.process_pull_responses(
+            crds,
+            from,
+            versioned,
+            versioned_expired_timeout,
+            now,
+            &mut stats,
+        );
+        (
+            stats.failed_timeout + stats.failed_insert,
+            stats.timeout_count,
+            stats.success,
+        )
+    }
 }
 #[cfg(test)]
 mod test {
@@ -396,7 +486,7 @@ mod test {
             stakes.insert(id, i * 100);
         }
         let now = 1024;
-        let mut options = node.pull_options(&crds, &me.label().pubkey(), now, &stakes);
+        let mut options = node.pull_options(&crds, &me.label().pubkey(), 0, now, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -406,6 +496,66 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_no_pulls_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPull::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .pull_options(&crds, &me.label().pubkey(), 123, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .pull_options(&crds, &spy.label().pubkey(), 0, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
+
     #[test]
     fn test_new_pull_request() {
         let mut crds = Crds::default();
@@ -416,13 +566,13 @@ mod test {
         let id = entry.label().pubkey();
         let node = CrdsGossipPull::default();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
         crds.insert(entry.clone(), 0).unwrap();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
@@ -431,7 +581,7 @@ mod test {
             0,
         )));
         crds.insert(new.clone(), 0).unwrap();
-        let req = node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE);
+        let req = node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE);
         let (to, _, self_info) = req.unwrap();
         assert_eq!(to, new.label().pubkey());
         assert_eq!(self_info, entry);
@@ -466,6 +616,7 @@ mod test {
             let req = node.new_pull_request(
                 &crds,
                 &node_pubkey,
+                0,
                 u64::max_value(),
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
@@ -495,6 +646,7 @@ mod test {
             &node_crds,
             &node_pubkey,
             0,
+            0,
             &HashMap::new(),
             PACKET_DATA_SIZE,
         );
@@ -502,8 +654,9 @@ mod test {
         let mut dest_crds = Crds::default();
         let mut dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
-        let filters = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.process_pull_requests(&mut dest_crds, filters, 1);
+        let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
+        let rsp = dest.generate_pull_responses(&dest_crds, &filters);
+        dest.process_pull_requests(&mut dest_crds, filters, 1);
         assert!(rsp.iter().all(|rsp| rsp.is_empty()));
         assert!(dest_crds.lookup(&caller.label()).is_some());
         assert_eq!(
@@ -567,12 +720,14 @@ mod test {
                 &node_crds,
                 &node_pubkey,
                 0,
+                0,
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
             );
             let (_, filters, caller) = req.unwrap();
-            let filters = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-            let mut rsp = dest.process_pull_requests(&mut dest_crds, filters, 0);
+            let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
+            let mut rsp = dest.generate_pull_responses(&dest_crds, &filters);
+            dest.process_pull_requests(&mut dest_crds, filters, 0);
             // if there is a false positive this is empty
             // prob should be around 0.1 per iteration
             if rsp.is_empty() {
@@ -583,13 +738,15 @@ mod test {
                 continue;
             }
             assert_eq!(rsp.len(), 1);
-            let failed = node.process_pull_response(
-                &mut node_crds,
-                &node_pubkey,
-                &node.make_timeouts_def(&node_pubkey, &HashMap::new(), 0, 1),
-                rsp.pop().unwrap(),
-                1,
-            );
+            let failed = node
+                .process_pull_response(
+                    &mut node_crds,
+                    &node_pubkey,
+                    &node.make_timeouts_def(&node_pubkey, &HashMap::new(), 0, 1),
+                    rsp.pop().unwrap(),
+                    1,
+                )
+                .0;
             assert_eq!(failed, 0);
             assert_eq!(
                 node_crds
@@ -750,7 +907,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry.clone()],
                 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -766,7 +924,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry.clone(), unstaked_peer_entry],
                 node.msg_timeout + 100,
-            ),
+            )
+            .0,
             2
         );
 
@@ -779,7 +938,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry.clone()],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -795,7 +955,8 @@ mod test {
                 &timeouts,
                 vec![peer_vote.clone()],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -808,7 +969,8 @@ mod test {
                 &timeouts,
                 vec![peer_vote.clone()],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             1
         );
     }

core/src/crds_gossip_push.rs

@@ -236,13 +236,14 @@ impl CrdsGossipPush {
         crds: &Crds,
         stakes: &HashMap<Pubkey, u64>,
         self_id: &Pubkey,
+        self_shred_version: u16,
         network_size: usize,
         ratio: usize,
     ) {
         let need = Self::compute_need(self.num_active, self.active_set.len(), ratio);
         let mut new_items = HashMap::new();
 
-        let options: Vec<_> = self.push_options(crds, &self_id, stakes);
+        let options: Vec<_> = self.push_options(crds, &self_id, self_shred_version, stakes);
         if options.is_empty() {
             return;
         }
@@ -288,13 +289,20 @@ impl CrdsGossipPush {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter(|v| v.value.contact_info().is_some())
             .map(|v| (v.value.contact_info().unwrap(), v))
-            .filter(|(info, _)| info.id != *self_id && ContactInfo::is_valid_address(&info.gossip))
+            .filter(|(info, _)| {
+                info.id != *self_id
+                    && ContactInfo::is_valid_address(&info.gossip)
+                    && (self_shred_version == 0
+                        || info.shred_version == 0
+                        || self_shred_version == info.shred_version)
+            })
             .map(|(info, value)| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let last_updated: u64 = value.local_timestamp;
@@ -510,7 +518,7 @@ mod test {
         )));
 
         assert_eq!(crds.insert(value1.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         assert!(push.active_set.get(&value1.label().pubkey()).is_some());
         let value2 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -520,7 +528,7 @@ mod test {
         assert!(push.active_set.get(&value2.label().pubkey()).is_none());
         assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         for _ in 0..30 {
-            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
             if push.active_set.get(&value2.label().pubkey()).is_some() {
                 break;
             }
@@ -533,7 +541,7 @@ mod test {
             ));
             assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         }
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
         assert_eq!(push.active_set.len(), push.num_active);
     }
     #[test]
@@ -551,7 +559,7 @@ mod test {
             crds.insert(peer.clone(), time).unwrap();
             stakes.insert(id, i * 100);
         }
-        let mut options = push.push_options(&crds, &Pubkey::default(), &stakes);
+        let mut options = push.push_options(&crds, &Pubkey::default(), 0, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -560,6 +568,66 @@ mod test {
             10_000_u64
         );
     }
+
+    #[test]
+    fn test_no_pushes_to_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPush::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .push_options(&crds, &me.label().pubkey(), 123, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .push_options(&crds, &spy.label().pubkey(), 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
     #[test]
     fn test_new_push_messages() {
         let mut crds = Crds::default();
@@ -569,7 +637,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -606,7 +674,7 @@ mod test {
             push.process_push_message(&mut crds, &Pubkey::default(), peer_3.clone(), 0),
             Ok(None)
         );
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         // push 3's contact info to 1 and 2 and 3
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -628,7 +696,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -651,7 +719,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let mut ci = ContactInfo::new_localhost(&Pubkey::new_rand(), 0);
         ci.wallclock = 1;

core/src/crds_value.rs

@@ -2,6 +2,7 @@ use crate::contact_info::ContactInfo;
 use crate::deprecated;
 use crate::epoch_slots::EpochSlots;
 use bincode::{serialize, serialized_size};
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 use solana_sdk::timing::timestamp;
 use solana_sdk::{
     clock::Slot,
@@ -16,6 +17,9 @@ use std::{
     fmt,
 };
 
+pub const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000;
+pub const MAX_SLOT: u64 = 1_000_000_000_000_000;
+
 pub type VoteIndex = u8;
 pub const MAX_VOTES: VoteIndex = 32;
 
@@ -29,6 +33,13 @@ pub struct CrdsValue {
     pub data: CrdsData,
 }
 
+impl Sanitize for CrdsValue {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        self.signature.sanitize()?;
+        self.data.sanitize()
+    }
+}
+
 impl Signable for CrdsValue {
     fn pubkey(&self) -> Pubkey {
         self.pubkey()
@@ -47,15 +58,8 @@ impl Signable for CrdsValue {
     }
 
     fn verify(&self) -> bool {
-        let sig_check = self
-            .get_signature()
-            .verify(&self.pubkey().as_ref(), self.signable_data().borrow());
-        let data_check = match &self.data {
-            CrdsData::Vote(ix, _) => *ix < MAX_VOTES,
-            CrdsData::EpochSlots(ix, _) => *ix < MAX_EPOCH_SLOTS,
-            _ => true,
-        };
-        sig_check && data_check
+        self.get_signature()
+            .verify(&self.pubkey().as_ref(), self.signable_data().borrow())
     }
 }
 
@@ -71,6 +75,36 @@ pub enum CrdsData {
     SnapshotHashes(SnapshotHash),
     AccountsHashes(SnapshotHash),
     EpochSlots(EpochSlotsIndex, EpochSlots),
+    Version(Version),
+}
+
+impl Sanitize for CrdsData {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        match self {
+            CrdsData::ContactInfo(val) => val.sanitize(),
+            CrdsData::Vote(ix, val) => {
+                if *ix >= MAX_VOTES {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+            CrdsData::LowestSlot(ix, val) => {
+                if *ix as usize >= 1 {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+            CrdsData::SnapshotHashes(val) => val.sanitize(),
+            CrdsData::AccountsHashes(val) => val.sanitize(),
+            CrdsData::EpochSlots(ix, val) => {
+                if *ix as usize >= MAX_EPOCH_SLOTS as usize {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+            CrdsData::Version(version) => version.sanitize(),
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -80,6 +114,20 @@ pub struct SnapshotHash {
     pub wallclock: u64,
 }
 
+impl Sanitize for SnapshotHash {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        for (slot, _) in &self.hashes {
+            if *slot >= MAX_SLOT {
+                return Err(SanitizeError::ValueOutOfBounds);
+            }
+        }
+        self.from.sanitize()
+    }
+}
+
 impl SnapshotHash {
     pub fn new(from: Pubkey, hashes: Vec<(Slot, Hash)>) -> Self {
         Self {
@@ -112,6 +160,27 @@ impl LowestSlot {
     }
 }
 
+impl Sanitize for LowestSlot {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.lowest >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.root != 0 {
+            return Err(SanitizeError::InvalidValue);
+        }
+        if !self.slots.is_empty() {
+            return Err(SanitizeError::InvalidValue);
+        }
+        if !self.stash.is_empty() {
+            return Err(SanitizeError::InvalidValue);
+        }
+        self.from.sanitize()
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct Vote {
     pub from: Pubkey,
@@ -119,6 +188,16 @@ pub struct Vote {
     pub wallclock: u64,
 }
 
+impl Sanitize for Vote {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        self.from.sanitize()?;
+        self.transaction.sanitize()
+    }
+}
+
 impl Vote {
     pub fn new(from: &Pubkey, transaction: Transaction, wallclock: u64) -> Self {
         Self {
@@ -129,6 +208,33 @@ impl Vote {
     }
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub struct Version {
+    pub from: Pubkey,
+    pub wallclock: u64,
+    pub version: solana_version::Version,
+}
+
+impl Sanitize for Version {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        self.from.sanitize()?;
+        self.version.sanitize()
+    }
+}
+
+impl Version {
+    pub fn new(from: Pubkey) -> Self {
+        Self {
+            from,
+            wallclock: timestamp(),
+            version: solana_version::Version::default(),
+        }
+    }
+}
+
 /// Type of the replicated value
 /// These are labels for values in a record that is associated with `Pubkey`
 #[derive(PartialEq, Hash, Eq, Clone, Debug)]
@@ -139,6 +245,7 @@ pub enum CrdsValueLabel {
     SnapshotHashes(Pubkey),
     EpochSlots(EpochSlotsIndex, Pubkey),
     AccountsHashes(Pubkey),
+    Version(Pubkey),
 }
 
 impl fmt::Display for CrdsValueLabel {
@@ -150,6 +257,7 @@ impl fmt::Display for CrdsValueLabel {
             CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHash({})", self.pubkey()),
             CrdsValueLabel::EpochSlots(ix, _) => write!(f, "EpochSlots({}, {})", ix, self.pubkey()),
             CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
+            CrdsValueLabel::Version(_) => write!(f, "Version({})", self.pubkey()),
         }
     }
 }
@@ -163,6 +271,7 @@ impl CrdsValueLabel {
             CrdsValueLabel::SnapshotHashes(p) => *p,
             CrdsValueLabel::EpochSlots(_, p) => *p,
             CrdsValueLabel::AccountsHashes(p) => *p,
+            CrdsValueLabel::Version(p) => *p,
         }
     }
 }
@@ -180,7 +289,7 @@ impl CrdsValue {
         value.sign(keypair);
         value
     }
-    /// Totally unsecure unverfiable wallclock of the node that generated this message
+    /// Totally unsecure unverifiable wallclock of the node that generated this message
     /// Latest wallclock is always picked.
     /// This is used to time out push messages.
     pub fn wallclock(&self) -> u64 {
@@ -191,6 +300,7 @@ impl CrdsValue {
             CrdsData::SnapshotHashes(hash) => hash.wallclock,
             CrdsData::AccountsHashes(hash) => hash.wallclock,
             CrdsData::EpochSlots(_, p) => p.wallclock,
+            CrdsData::Version(version) => version.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
@@ -201,6 +311,7 @@ impl CrdsValue {
             CrdsData::SnapshotHashes(hash) => hash.from,
             CrdsData::AccountsHashes(hash) => hash.from,
             CrdsData::EpochSlots(_, p) => p.from,
+            CrdsData::Version(version) => version.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
@@ -211,6 +322,7 @@ impl CrdsValue {
             CrdsData::SnapshotHashes(_) => CrdsValueLabel::SnapshotHashes(self.pubkey()),
             CrdsData::AccountsHashes(_) => CrdsValueLabel::AccountsHashes(self.pubkey()),
             CrdsData::EpochSlots(ix, _) => CrdsValueLabel::EpochSlots(*ix, self.pubkey()),
+            CrdsData::Version(_) => CrdsValueLabel::Version(self.pubkey()),
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -261,6 +373,13 @@ impl CrdsValue {
         }
     }
 
+    pub fn version(&self) -> Option<&Version> {
+        match &self.data {
+            CrdsData::Version(version) => Some(version),
+            _ => None,
+        }
+    }
+
     /// Return all the possible labels for a record identified by Pubkey.
     pub fn record_labels(key: &Pubkey) -> Vec<CrdsValueLabel> {
         let mut labels = vec![
@@ -268,6 +387,7 @@ impl CrdsValue {
             CrdsValueLabel::LowestSlot(*key),
             CrdsValueLabel::SnapshotHashes(*key),
             CrdsValueLabel::AccountsHashes(*key),
+            CrdsValueLabel::Version(*key),
         ];
         labels.extend((0..MAX_VOTES).map(|ix| CrdsValueLabel::Vote(ix, *key)));
         labels.extend((0..MAX_EPOCH_SLOTS).map(|ix| CrdsValueLabel::EpochSlots(ix, *key)));
@@ -318,7 +438,7 @@ mod test {
 
     #[test]
     fn test_labels() {
-        let mut hits = [false; 4 + MAX_VOTES as usize + MAX_EPOCH_SLOTS as usize];
+        let mut hits = [false; 5 + MAX_VOTES as usize + MAX_EPOCH_SLOTS as usize];
         // this method should cover all the possible labels
         for v in &CrdsValue::record_labels(&Pubkey::default()) {
             match v {
@@ -326,9 +446,10 @@ mod test {
                 CrdsValueLabel::LowestSlot(_) => hits[1] = true,
                 CrdsValueLabel::SnapshotHashes(_) => hits[2] = true,
                 CrdsValueLabel::AccountsHashes(_) => hits[3] = true,
-                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 4] = true,
+                CrdsValueLabel::Version(_) => hits[4] = true,
+                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 5] = true,
                 CrdsValueLabel::EpochSlots(ix, _) => {
-                    hits[*ix as usize + MAX_VOTES as usize + 4] = true
+                    hits[*ix as usize + MAX_VOTES as usize + 5] = true
                 }
             }
         }
@@ -358,6 +479,32 @@ mod test {
         assert_eq!(v.label(), CrdsValueLabel::LowestSlot(key));
     }
 
+    #[test]
+    fn test_lowest_slot_sanitize() {
+        let ls = LowestSlot::new(Pubkey::default(), 0, 0);
+        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(0, ls.clone()));
+        assert_eq!(v.sanitize(), Ok(()));
+
+        let mut o = ls.clone();
+        o.root = 1;
+        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(0, o.clone()));
+        assert_eq!(v.sanitize(), Err(SanitizeError::InvalidValue));
+
+        let o = ls.clone();
+        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(1, o.clone()));
+        assert_eq!(v.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+
+        let mut o = ls.clone();
+        o.slots.insert(1);
+        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(0, o.clone()));
+        assert_eq!(v.sanitize(), Err(SanitizeError::InvalidValue));
+
+        let mut o = ls.clone();
+        o.stash.push(deprecated::EpochIncompleteSlots::default());
+        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(0, o.clone()));
+        assert_eq!(v.sanitize(), Err(SanitizeError::InvalidValue));
+    }
+
     #[test]
     fn test_signature() {
         let keypair = Keypair::new();
@@ -389,7 +536,7 @@ mod test {
             ),
             &keypair,
         );
-        assert!(!vote.verify());
+        assert!(vote.sanitize().is_err());
     }
 
     #[test]
@@ -402,7 +549,7 @@ mod test {
             ),
             &keypair,
         );
-        assert!(!item.verify());
+        assert_eq!(item.sanitize(), Err(SanitizeError::ValueOutOfBounds));
     }
     #[test]
     fn test_compute_vote_index_empty() {

core/src/epoch_slots.rs

@@ -1,10 +1,14 @@
 use crate::cluster_info::MAX_CRDS_OBJECT_SIZE;
+use crate::crds_value::MAX_SLOT;
+use crate::crds_value::MAX_WALLCLOCK;
 use bincode::serialized_size;
 use bv::BitVec;
 use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress};
 use solana_sdk::clock::Slot;
 use solana_sdk::pubkey::Pubkey;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 
+const MAX_SLOTS_PER_ENTRY: usize = 2048 * 8;
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct Uncompressed {
     pub first_slot: Slot,
@@ -12,6 +16,18 @@ pub struct Uncompressed {
     pub slots: BitVec<u8>,
 }
 
+impl Sanitize for Uncompressed {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.first_slot >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.num >= MAX_SLOTS_PER_ENTRY {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 #[derive(Deserialize, Serialize, Clone, Debug, PartialEq)]
 pub struct Flate2 {
     pub first_slot: Slot,
@@ -19,6 +35,18 @@ pub struct Flate2 {
     pub compressed: Vec<u8>,
 }
 
+impl Sanitize for Flate2 {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.first_slot >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.num >= MAX_SLOTS_PER_ENTRY {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub enum Error {
     CompressError,
@@ -84,6 +112,9 @@ impl Uncompressed {
             (min_slot - self.first_slot) as usize
         };
         for i in start..self.num {
+            if i >= self.slots.len() as usize {
+                break;
+            }
             if self.slots.get(i as u64) {
                 rv.push(self.first_slot + i as Slot);
             }
@@ -95,6 +126,9 @@ impl Uncompressed {
             if self.num == 0 {
                 self.first_slot = *s;
             }
+            if self.num >= MAX_SLOTS_PER_ENTRY {
+                return i;
+            }
             if *s < self.first_slot {
                 return i;
             }
@@ -114,6 +148,15 @@ pub enum CompressedSlots {
     Uncompressed(Uncompressed),
 }
 
+impl Sanitize for CompressedSlots {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        match self {
+            CompressedSlots::Uncompressed(a) => a.sanitize(),
+            CompressedSlots::Flate2(b) => b.sanitize(),
+        }
+    }
+}
+
 impl Default for CompressedSlots {
     fn default() -> Self {
         CompressedSlots::new(0)
@@ -168,13 +211,40 @@ impl CompressedSlots {
     }
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
+#[derive(Serialize, Deserialize, Clone, Default, PartialEq)]
 pub struct EpochSlots {
     pub from: Pubkey,
     pub slots: Vec<CompressedSlots>,
     pub wallclock: u64,
 }
 
+impl Sanitize for EpochSlots {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        self.from.sanitize()?;
+        self.slots.sanitize()
+    }
+}
+
+use std::fmt;
+impl fmt::Debug for EpochSlots {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let num_slots: usize = self.slots.iter().map(|s| s.num_slots()).sum();
+        let lowest_slot = self
+            .slots
+            .iter()
+            .map(|s| s.first_slot())
+            .fold(0, std::cmp::min);
+        write!(
+            f,
+            "EpochSlots {{ from: {} num_slots: {} lowest_slot: {} wallclock: {} }}",
+            self.from, num_slots, lowest_slot, self.wallclock
+        )
+    }
+}
+
 impl EpochSlots {
     pub fn new(from: Pubkey, now: u64) -> Self {
         Self {
@@ -256,6 +326,14 @@ mod tests {
         assert_eq!(slots.to_slots(1), vec![1]);
         assert!(slots.to_slots(2).is_empty());
     }
+
+    #[test]
+    fn test_epoch_slots_to_slots_overflow() {
+        let mut slots = Uncompressed::new(1);
+        slots.num = 100;
+        assert!(slots.to_slots(0).is_empty());
+    }
+
     #[test]
     fn test_epoch_slots_uncompressed_add_2() {
         let mut slots = Uncompressed::new(1);
@@ -299,6 +377,44 @@ mod tests {
         assert_eq!(slots.num, 701);
         assert_eq!(slots.to_slots(1), vec![1, 2, 701]);
     }
+
+    #[test]
+    fn test_epoch_slots_sanitize() {
+        let mut slots = Uncompressed::new(100);
+        slots.add(&[1, 701, 2]);
+        assert_eq!(slots.num, 701);
+        assert!(slots.sanitize().is_ok());
+
+        let mut o = slots.clone();
+        o.first_slot = MAX_SLOT;
+        assert_eq!(o.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+
+        let mut o = slots.clone();
+        o.num = MAX_SLOTS_PER_ENTRY;
+        assert_eq!(o.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+
+        let compressed = Flate2::deflate(slots).unwrap();
+        assert!(compressed.sanitize().is_ok());
+
+        let mut o = compressed.clone();
+        o.first_slot = MAX_SLOT;
+        assert_eq!(o.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+
+        let mut o = compressed.clone();
+        o.num = MAX_SLOTS_PER_ENTRY;
+        assert_eq!(o.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+
+        let mut slots = EpochSlots::default();
+        let range: Vec<Slot> = (0..5000).into_iter().collect();
+        assert_eq!(slots.fill(&range, 1), 5000);
+        assert_eq!(slots.wallclock, 1);
+        assert!(slots.sanitize().is_ok());
+
+        let mut o = slots.clone();
+        o.wallclock = MAX_WALLCLOCK;
+        assert_eq!(o.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
+
     #[test]
     fn test_epoch_slots_fill_range() {
         let range: Vec<Slot> = (0..5000).into_iter().collect();

core/src/gossip_service.rs

@@ -22,7 +22,7 @@ pub struct GossipService {
 
 impl GossipService {
     pub fn new(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &Arc<ClusterInfo>,
         bank_forks: Option<Arc<RwLock<BankForks>>>,
         gossip_socket: UdpSocket,
         exit: &Arc<AtomicBool>,
@@ -31,7 +31,7 @@ impl GossipService {
         let gossip_socket = Arc::new(gossip_socket);
         trace!(
             "GossipService: id: {}, listening on: {:?}",
-            &cluster_info.read().unwrap().my_data().id,
+            &cluster_info.id(),
             gossip_socket.local_addr().unwrap()
         );
         let t_receiver = streamer::receiver(
@@ -75,6 +75,7 @@ pub fn discover_cluster(
         None,
         None,
         None,
+        0,
     )
 }
 
@@ -85,11 +86,13 @@ pub fn discover(
     find_node_by_pubkey: Option<Pubkey>,
     find_node_by_gossip_addr: Option<&SocketAddr>,
     my_gossip_addr: Option<&SocketAddr>,
+    my_shred_version: u16,
 ) -> std::io::Result<(Vec<ContactInfo>, Vec<ContactInfo>)> {
     let exit = Arc::new(AtomicBool::new(false));
-    let (gossip_service, ip_echo, spy_ref) = make_gossip_node(entrypoint, &exit, my_gossip_addr);
+    let (gossip_service, ip_echo, spy_ref) =
+        make_gossip_node(entrypoint, &exit, my_gossip_addr, my_shred_version);
 
-    let id = spy_ref.read().unwrap().keypair.pubkey();
+    let id = spy_ref.id();
     info!("Entrypoint: {:?}", entrypoint);
     info!("Node Id: {:?}", id);
     if let Some(my_gossip_addr) = my_gossip_addr {
@@ -113,7 +116,7 @@ pub fn discover(
         info!(
             "discover success in {}s...\n{}",
             secs,
-            spy_ref.read().unwrap().contact_info_trace()
+            spy_ref.contact_info_trace()
         );
         return Ok((tvu_peers, storage_peers));
     }
@@ -121,15 +124,12 @@ pub fn discover(
     if !tvu_peers.is_empty() {
         info!(
             "discover failed to match criteria by timeout...\n{}",
-            spy_ref.read().unwrap().contact_info_trace()
+            spy_ref.contact_info_trace()
         );
         return Ok((tvu_peers, storage_peers));
     }
 
-    info!(
-        "discover failed...\n{}",
-        spy_ref.read().unwrap().contact_info_trace()
-    );
+    info!("discover failed...\n{}", spy_ref.contact_info_trace());
     Err(std::io::Error::new(
         std::io::ErrorKind::Other,
         "Discover failed",
@@ -176,7 +176,7 @@ pub fn get_multi_client(nodes: &[ContactInfo]) -> (ThinClient, usize) {
 }
 
 fn spy(
-    spy_ref: Arc<RwLock<ClusterInfo>>,
+    spy_ref: Arc<ClusterInfo>,
     num_nodes: Option<usize>,
     timeout: Option<u64>,
     find_node_by_pubkey: Option<Pubkey>,
@@ -194,13 +194,8 @@ fn spy(
             }
         }
 
-        tvu_peers = spy_ref
-            .read()
-            .unwrap()
-            .all_tvu_peers()
-            .into_iter()
-            .collect::<Vec<_>>();
-        storage_peers = spy_ref.read().unwrap().all_storage_peers();
+        tvu_peers = spy_ref.all_tvu_peers().into_iter().collect::<Vec<_>>();
+        storage_peers = spy_ref.all_storage_peers();
 
         let mut nodes: Vec<_> = tvu_peers.iter().chain(storage_peers.iter()).collect();
         nodes.sort();
@@ -232,10 +227,7 @@ fn spy(
             met_criteria = true;
         }
         if i % 20 == 0 {
-            info!(
-                "discovering...\n{}",
-                spy_ref.read().unwrap().contact_info_trace()
-            );
+            info!("discovering...\n{}", spy_ref.contact_info_trace());
         }
         sleep(Duration::from_millis(
             crate::cluster_info::GOSSIP_SLEEP_MILLIS,
@@ -256,18 +248,19 @@ fn make_gossip_node(
     entrypoint: Option<&SocketAddr>,
     exit: &Arc<AtomicBool>,
     gossip_addr: Option<&SocketAddr>,
-) -> (GossipService, Option<TcpListener>, Arc<RwLock<ClusterInfo>>) {
+    shred_version: u16,
+) -> (GossipService, Option<TcpListener>, Arc<ClusterInfo>) {
     let keypair = Arc::new(Keypair::new());
     let (node, gossip_socket, ip_echo) = if let Some(gossip_addr) = gossip_addr {
-        ClusterInfo::gossip_node(&keypair.pubkey(), gossip_addr)
+        ClusterInfo::gossip_node(&keypair.pubkey(), gossip_addr, shred_version)
     } else {
-        ClusterInfo::spy_node(&keypair.pubkey())
+        ClusterInfo::spy_node(&keypair.pubkey(), shred_version)
     };
-    let mut cluster_info = ClusterInfo::new(node, keypair);
+    let cluster_info = ClusterInfo::new(node, keypair);
     if let Some(entrypoint) = entrypoint {
         cluster_info.set_entrypoint(ContactInfo::new_gossip_entry_point(entrypoint));
     }
-    let cluster_info = Arc::new(RwLock::new(cluster_info));
+    let cluster_info = Arc::new(cluster_info);
     let gossip_service = GossipService::new(&cluster_info.clone(), None, gossip_socket, &exit);
     (gossip_service, ip_echo, cluster_info)
 }
@@ -277,7 +270,7 @@ mod tests {
     use super::*;
     use crate::cluster_info::{ClusterInfo, Node};
     use std::sync::atomic::AtomicBool;
-    use std::sync::{Arc, RwLock};
+    use std::sync::Arc;
 
     #[test]
     #[ignore]
@@ -286,7 +279,7 @@ mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let tn = Node::new_localhost();
         let cluster_info = ClusterInfo::new_with_invalid_keypair(tn.info.clone());
-        let c = Arc::new(RwLock::new(cluster_info));
+        let c = Arc::new(cluster_info);
         let d = GossipService::new(&c, None, tn.sockets.gossip, &exit);
         exit.store(true, Ordering::Relaxed);
         d.join().unwrap();
@@ -300,16 +293,16 @@ mod tests {
         let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
         let peer0_info = ContactInfo::new_localhost(&peer0, 0);
         let peer1_info = ContactInfo::new_localhost(&peer1, 0);
-        let mut cluster_info = ClusterInfo::new(contact_info.clone(), Arc::new(keypair));
+        let cluster_info = ClusterInfo::new(contact_info.clone(), Arc::new(keypair));
         cluster_info.insert_info(peer0_info.clone());
         cluster_info.insert_info(peer1_info);
 
-        let spy_ref = Arc::new(RwLock::new(cluster_info));
+        let spy_ref = Arc::new(cluster_info);
 
         let (met_criteria, secs, tvu_peers, _) = spy(spy_ref.clone(), None, Some(1), None, None);
         assert_eq!(met_criteria, false);
         assert_eq!(secs, 1);
-        assert_eq!(tvu_peers, spy_ref.read().unwrap().tvu_peers());
+        assert_eq!(tvu_peers, spy_ref.tvu_peers());
 
         // Find num_nodes
         let (met_criteria, _, _, _) = spy(spy_ref.clone(), Some(1), None, None, None);

core/src/ledger_cleanup_service.rs

@@ -3,8 +3,7 @@
 use solana_ledger::blockstore::Blockstore;
 use solana_ledger::blockstore_db::Result as BlockstoreResult;
 use solana_measure::measure::Measure;
-use solana_metrics::datapoint_debug;
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Slot, DEFAULT_TICKS_PER_SLOT, TICKS_PER_DAY};
 use std::string::ToString;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
@@ -30,9 +29,12 @@ pub const DEFAULT_MIN_MAX_LEDGER_SHREDS: u64 = 50_000_000;
 // and starve other blockstore users.
 pub const DEFAULT_PURGE_SLOT_INTERVAL: u64 = 512;
 
-// Remove a limited number of slots at a time, so the operation
-// does not take too long and block other blockstore users.
-pub const DEFAULT_PURGE_BATCH_SIZE: u64 = 256;
+// Delay between purges to cooperate with other blockstore users
+pub const DEFAULT_DELAY_BETWEEN_PURGES: Duration = Duration::from_millis(500);
+
+// Compacting at a slower interval than purging helps keep IOPS down.
+// Once a day should be ample
+const DEFAULT_COMPACTION_SLOT_INTERVAL: u64 = TICKS_PER_DAY / DEFAULT_TICKS_PER_SLOT;
 
 pub struct LedgerCleanupService {
     t_cleanup: JoinHandle<()>,
@@ -51,6 +53,8 @@ impl LedgerCleanupService {
         );
         let exit = exit.clone();
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
+
         let t_cleanup = Builder::new()
             .name("solana-ledger-cleanup".to_string())
             .spawn(move || loop {
@@ -63,6 +67,9 @@ impl LedgerCleanupService {
                     max_ledger_slots,
                     &mut last_purge_slot,
                     DEFAULT_PURGE_SLOT_INTERVAL,
+                    Some(DEFAULT_DELAY_BETWEEN_PURGES),
+                    &mut last_compaction_slot,
+                    DEFAULT_COMPACTION_SLOT_INTERVAL,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -78,8 +85,8 @@ impl LedgerCleanupService {
         blockstore: &Arc<Blockstore>,
         root: Slot,
         max_ledger_shreds: u64,
-    ) -> (u64, Slot, Slot) {
-        let mut shreds = Vec::new();
+    ) -> (bool, Slot, Slot, u64) {
+        let mut total_slots = Vec::new();
         let mut iterate_time = Measure::start("iterate_time");
         let mut total_shreds = 0;
         let mut first_slot = 0;
@@ -90,33 +97,43 @@ impl LedgerCleanupService {
             }
             // Not exact since non-full slots will have holes
             total_shreds += meta.received;
-            shreds.push((slot, meta.received));
+            total_slots.push((slot, meta.received));
             if slot > root {
                 break;
             }
         }
         iterate_time.stop();
         info!(
-            "checking for ledger purge: max_shreds: {} slots: {} total_shreds: {} {}",
-            max_ledger_shreds,
-            shreds.len(),
+            "first_slot={} total_slots={} total_shreds={} max_ledger_shreds={}, {}",
+            first_slot,
+            total_slots.len(),
             total_shreds,
+            max_ledger_shreds,
             iterate_time
         );
         if (total_shreds as u64) < max_ledger_shreds {
-            return (0, 0, 0);
+            return (false, 0, 0, total_shreds);
         }
-        let mut cur_shreds = 0;
-        let mut lowest_slot_to_clean = shreds[0].0;
-        for (slot, num_shreds) in shreds.iter().rev() {
-            cur_shreds += *num_shreds as u64;
-            if cur_shreds > max_ledger_shreds {
-                lowest_slot_to_clean = *slot;
+        let mut num_shreds_to_clean = 0;
+        let mut lowest_cleanup_slot = total_slots[0].0;
+        for (slot, num_shreds) in total_slots.iter().rev() {
+            num_shreds_to_clean += *num_shreds as u64;
+            if num_shreds_to_clean > max_ledger_shreds {
+                lowest_cleanup_slot = *slot;
                 break;
             }
         }
 
-        (cur_shreds, lowest_slot_to_clean, first_slot)
+        (true, first_slot, lowest_cleanup_slot, total_shreds)
+    }
+
+    fn receive_new_roots(new_root_receiver: &Receiver<Slot>) -> Result<Slot, RecvTimeoutError> {
+        let mut root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
+        // Get the newest root
+        while let Ok(new_root) = new_root_receiver.try_recv() {
+            root = new_root;
+        }
+        Ok(root)
     }
 
     fn cleanup_ledger(
@@ -125,68 +142,103 @@ impl LedgerCleanupService {
         max_ledger_shreds: u64,
         last_purge_slot: &mut u64,
         purge_interval: u64,
+        delay_between_purges: Option<Duration>,
+        last_compaction_slot: &mut u64,
+        compaction_interval: u64,
     ) -> Result<(), RecvTimeoutError> {
-        let mut root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
-        // Get the newest root
-        while let Ok(new_root) = new_root_receiver.try_recv() {
-            root = new_root;
+        let root = Self::receive_new_roots(new_root_receiver)?;
+        if root - *last_purge_slot <= purge_interval {
+            return Ok(());
         }
 
-        if root - *last_purge_slot > purge_interval {
-            let disk_utilization_pre = blockstore.storage_size();
-            info!(
-                "purge: new root: {} last_purge: {} purge_interval: {} disk: {:?}",
-                root, last_purge_slot, purge_interval, disk_utilization_pre
-            );
-            *last_purge_slot = root;
+        let disk_utilization_pre = blockstore.storage_size();
+        info!(
+            "purge: last_root={}, last_purge_slot={}, purge_interval={}, last_compaction_slot={}, disk_utilization={:?}",
+            root, last_purge_slot, purge_interval, last_compaction_slot, disk_utilization_pre
+        );
+        *last_purge_slot = root;
 
-            let (num_shreds_to_clean, lowest_slot_to_clean, mut first_slot) =
-                Self::find_slots_to_clean(blockstore, root, max_ledger_shreds);
+        let (slots_to_clean, purge_first_slot, lowest_cleanup_slot, total_shreds) =
+            Self::find_slots_to_clean(&blockstore, root, max_ledger_shreds);
 
-            if num_shreds_to_clean > 0 {
-                debug!(
-                    "cleaning up to: {} shreds: {} first: {}",
-                    lowest_slot_to_clean, num_shreds_to_clean, first_slot
-                );
-                loop {
-                    let current_lowest =
-                        std::cmp::min(lowest_slot_to_clean, first_slot + DEFAULT_PURGE_BATCH_SIZE);
-
-                    let mut slot_update_time = Measure::start("slot_update");
-                    *blockstore.lowest_cleanup_slot.write().unwrap() = current_lowest;
-                    slot_update_time.stop();
-
-                    let mut clean_time = Measure::start("ledger_clean");
-                    blockstore.purge_slots(first_slot, Some(current_lowest));
-                    clean_time.stop();
-
-                    debug!(
-                        "ledger purge {} -> {}: {} {}",
-                        first_slot, current_lowest, slot_update_time, clean_time
-                    );
-                    first_slot += DEFAULT_PURGE_BATCH_SIZE;
-                    if current_lowest == lowest_slot_to_clean {
-                        break;
-                    }
-                    thread::sleep(Duration::from_millis(500));
-                }
+        if slots_to_clean {
+            let mut compact_first_slot = std::u64::MAX;
+            if lowest_cleanup_slot.saturating_sub(*last_compaction_slot) > compaction_interval {
+                compact_first_slot = *last_compaction_slot;
+                *last_compaction_slot = lowest_cleanup_slot;
             }
 
-            let disk_utilization_post = blockstore.storage_size();
+            let purge_complete = Arc::new(AtomicBool::new(false));
+            let blockstore = blockstore.clone();
+            let purge_complete1 = purge_complete.clone();
+            let _t_purge = Builder::new()
+                .name("solana-ledger-purge".to_string())
+                .spawn(move || {
+                    let mut slot_update_time = Measure::start("slot_update");
+                    *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_cleanup_slot;
+                    slot_update_time.stop();
 
-            Self::report_disk_metrics(disk_utilization_pre, disk_utilization_post);
+                    info!(
+                        "purging data from slots {} to {}",
+                        purge_first_slot, lowest_cleanup_slot
+                    );
+
+                    let mut purge_time = Measure::start("purge_slots_with_delay");
+                    blockstore.purge_slots_with_delay(
+                        purge_first_slot,
+                        lowest_cleanup_slot,
+                        delay_between_purges,
+                    );
+                    purge_time.stop();
+                    info!("{}", purge_time);
+
+                    if compact_first_slot < lowest_cleanup_slot {
+                        info!(
+                            "compacting data from slots {} to {}",
+                            compact_first_slot, lowest_cleanup_slot
+                        );
+                        if let Err(err) =
+                            blockstore.compact_storage(compact_first_slot, lowest_cleanup_slot)
+                        {
+                            // This error is not fatal and indicates an internal error?
+                            error!(
+                                "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
+                                err, compact_first_slot, lowest_cleanup_slot
+                            );
+                        }
+                    }
+
+                    purge_complete1.store(true, Ordering::Relaxed);
+                })
+                .unwrap();
+
+            // Keep pulling roots off `new_root_receiver` while purging to avoid channel buildup
+            while !purge_complete.load(Ordering::Relaxed) {
+                if let Err(err) = Self::receive_new_roots(new_root_receiver) {
+                    debug!("receive_new_roots: {}", err);
+                }
+                thread::sleep(Duration::from_secs(1));
+            }
         }
 
+        let disk_utilization_post = blockstore.storage_size();
+        Self::report_disk_metrics(disk_utilization_pre, disk_utilization_post, total_shreds);
+
         Ok(())
     }
 
-    fn report_disk_metrics(pre: BlockstoreResult<u64>, post: BlockstoreResult<u64>) {
+    fn report_disk_metrics(
+        pre: BlockstoreResult<u64>,
+        post: BlockstoreResult<u64>,
+        total_shreds: u64,
+    ) {
         if let (Ok(pre), Ok(post)) = (pre, post) {
-            datapoint_debug!(
+            datapoint_info!(
                 "ledger_disk_utilization",
                 ("disk_utilization_pre", pre as i64, i64),
                 ("disk_utilization_post", post as i64, i64),
-                ("disk_utilization_delta", (pre as i64 - post as i64), i64)
+                ("disk_utilization_delta", (pre as i64 - post as i64), i64),
+                ("total_shreds", total_shreds, i64),
             );
         }
     }
@@ -214,9 +266,19 @@ mod tests {
 
         //send a signal to kill all but 5 shreds, which will be in the newest slots
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         sender.send(50).unwrap();
-        LedgerCleanupService::cleanup_ledger(&receiver, &blockstore, 5, &mut last_purge_slot, 10)
-            .unwrap();
+        LedgerCleanupService::cleanup_ledger(
+            &receiver,
+            &blockstore,
+            5,
+            &mut last_purge_slot,
+            10,
+            None,
+            &mut last_compaction_slot,
+            10,
+        )
+        .unwrap();
 
         //check that 0-40 don't exist
         blockstore
@@ -246,6 +308,7 @@ mod tests {
         info!("{}", first_insert);
 
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         let mut slot = initial_slots;
         let mut num_slots = 6;
         for _ in 0..5 {
@@ -269,6 +332,9 @@ mod tests {
                 initial_slots,
                 &mut last_purge_slot,
                 10,
+                None,
+                &mut last_compaction_slot,
+                10,
             )
             .unwrap();
             time.stop();
@@ -304,12 +370,16 @@ mod tests {
         // send signal to cleanup slots
         let (sender, receiver) = channel();
         sender.send(n).unwrap();
-        let mut next_purge_batch = 0;
+        let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         LedgerCleanupService::cleanup_ledger(
             &receiver,
             &blockstore,
             max_ledger_shreds,
-            &mut next_purge_batch,
+            &mut last_purge_slot,
+            10,
+            None,
+            &mut last_compaction_slot,
             10,
         )
         .unwrap();

core/src/lib.rs

@@ -30,19 +30,24 @@ pub mod gen_keys;
 pub mod gossip_service;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
+pub mod non_circulating_supply;
 pub mod poh_recorder;
 pub mod poh_service;
 pub mod progress_map;
+pub mod repair_response;
 pub mod repair_service;
 pub mod replay_stage;
 mod result;
 pub mod retransmit_stage;
 pub mod rewards_recorder_service;
 pub mod rpc;
+pub mod rpc_error;
+pub mod rpc_health;
 pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
+pub mod send_transaction_service;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;
@@ -58,6 +63,9 @@ pub mod verified_vote_packets;
 pub mod weighted_shuffle;
 pub mod window_service;
 
+#[macro_use]
+extern crate solana_bpf_loader_program;
+
 #[macro_use]
 extern crate solana_budget_program;
 

core/src/non_circulating_supply.rs

@@ -0,0 +1,193 @@
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use solana_stake_program::stake_state::StakeState;
+use std::{collections::HashSet, sync::Arc};
+
+pub struct NonCirculatingSupply {
+    pub lamports: u64,
+    pub accounts: Vec<Pubkey>,
+}
+
+pub fn calculate_non_circulating_supply(bank: &Arc<Bank>) -> NonCirculatingSupply {
+    debug!("Updating Bank supply, epoch: {}", bank.epoch());
+    let mut non_circulating_accounts_set: HashSet<Pubkey> = HashSet::new();
+
+    for key in non_circulating_accounts() {
+        non_circulating_accounts_set.insert(key);
+    }
+    let withdraw_authority_list = withdraw_authority();
+
+    let clock = bank.clock();
+    let stake_accounts = bank.get_program_accounts(Some(&solana_stake_program::id()));
+    for (pubkey, account) in stake_accounts.iter() {
+        let stake_account = StakeState::from(&account).unwrap_or_default();
+        match stake_account {
+            StakeState::Initialized(meta) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || withdraw_authority_list.contains(&meta.authorized.withdrawer)
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            StakeState::Stake(meta, _stake) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || withdraw_authority_list.contains(&meta.authorized.withdrawer)
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    let lamports = non_circulating_accounts_set
+        .iter()
+        .fold(0, |acc, pubkey| acc + bank.get_balance(&pubkey));
+
+    NonCirculatingSupply {
+        lamports,
+        accounts: non_circulating_accounts_set.into_iter().collect(),
+    }
+}
+
+// Mainnet-beta accounts that should be considered non-circulating
+solana_sdk::pubkeys!(
+    non_circulating_accounts,
+    [
+        "9huDUZfxoJ7wGMTffUE7vh1xePqef7gyrLJu9NApncqA",
+        "GK2zqSsXLA2rwVZk347RYhh6jJpRsCA69FjLW93ZGi3B",
+        "HCV5dGFJXRrJ3jhDYA4DCeb9TEDTwGGYXtT3wHksu2Zr",
+        "25odAafVXnd63L6Hq5Cx6xGmhKqkhE2y6UrLVuqUfWZj",
+        "14FUT96s9swbmH7ZjpDvfEDywnAYy9zaNhv4xvezySGu",
+        "HbZ5FfmKWNHC7uwk6TF1hVi6TCs7dtYfdjEcuPGgzFAg",
+        "C7C8odR8oashR5Feyrq2tJKaXL18id1dSj2zbkDGL2C2",
+        "Eyr9P5XsjK2NUKNCnfu39eqpGoiLFgVAv1LSQgMZCwiQ",
+        "DE1bawNcRJB9rVm3buyMVfr8mBEoyyu73NBovf2oXJsJ",
+        "CakcnaRDHka2gXyfbEd2d3xsvkJkqsLw2akB3zsN1D2S",
+        "7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2",
+        "GdnSyH3YtwcxFvQrVVJMm1JhTS4QVX7MFsX56uJLUfiZ",
+        "Mc5XB47H3DKJHym5RLa9mPzWv5snERsF3KNv5AauXK8",
+        "7cvkjYAkUYs4W8XcXsca7cBrEGFeSUjeZmKoNBvEwyri",
+        "AG3m2bAibcY8raMt4oXEGqRHwX4FWKPPJVjZxn1LySDX",
+        "5XdtyEDREHJXXW1CTtCsVjJRjBapAwK78ZquzvnNVRrV",
+        "6yKHERk8rsbmJxvMpPuwPs1ct3hRiP7xaJF2tvnGU6nK",
+        "CHmdL15akDcJgBkY6BP3hzs98Dqr6wbdDC5p8odvtSbq",
+        "FR84wZQy3Y3j2gWz6pgETUiUoJtreMEuWfbg6573UCj9",
+        "5q54XjQ7vDx4y6KphPeE97LUNiYGtP55spjvXAWPGBuf",
+        "3o6xgkJ9sTmDeQWyfj3sxwon18fXJB9PV5LDc8sfgR4a",
+    ]
+);
+
+// Withdraw authority for autostaked accounts on mainnet-beta
+solana_sdk::pubkeys!(
+    withdraw_authority,
+    [
+        "8CUUMKYNGxdgYio5CLHRHyzMEhhVRMcqefgE6dLqnVRK",
+        "3FFaheyqtyAXZSYxDzsr5CVKvJuvZD1WE1VEsBtDbRqB",
+    ]
+);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_sdk::{
+        account::Account, epoch_schedule::EpochSchedule, genesis_config::GenesisConfig,
+    };
+    use solana_stake_program::stake_state::{Authorized, Lockup, Meta, StakeState};
+    use std::{collections::BTreeMap, sync::Arc};
+
+    fn new_from_parent(parent: &Arc<Bank>) -> Bank {
+        Bank::new_from_parent(parent, &Pubkey::default(), parent.slot() + 1)
+    }
+
+    #[test]
+    fn test_calculate_non_circulating_supply() {
+        let mut accounts: BTreeMap<Pubkey, Account> = BTreeMap::new();
+        let balance = 10;
+        let num_genesis_accounts = 10;
+        for _ in 0..num_genesis_accounts {
+            accounts.insert(
+                Pubkey::new_rand(),
+                Account::new(balance, 0, &Pubkey::default()),
+            );
+        }
+        let non_circulating_accounts = non_circulating_accounts();
+        let num_non_circulating_accounts = non_circulating_accounts.len() as u64;
+        for key in non_circulating_accounts.clone() {
+            accounts.insert(key, Account::new(balance, 0, &Pubkey::default()));
+        }
+
+        let num_stake_accounts = 3;
+        for _ in 0..num_stake_accounts {
+            let pubkey = Pubkey::new_rand();
+            let meta = Meta {
+                authorized: Authorized::auto(&pubkey),
+                lockup: Lockup {
+                    epoch: 1,
+                    ..Lockup::default()
+                },
+                ..Meta::default()
+            };
+            let stake_account = Account::new_data_with_space(
+                balance,
+                &StakeState::Initialized(meta),
+                std::mem::size_of::<StakeState>(),
+                &solana_stake_program::id(),
+            )
+            .unwrap();
+            accounts.insert(pubkey, stake_account);
+        }
+
+        let slots_per_epoch = 32;
+        let genesis_config = GenesisConfig {
+            accounts,
+            epoch_schedule: EpochSchedule::new(slots_per_epoch),
+            ..GenesisConfig::default()
+        };
+        let mut bank = Arc::new(Bank::new(&genesis_config));
+        assert_eq!(
+            bank.capitalization(),
+            (num_genesis_accounts + num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+
+        let non_circulating_supply = calculate_non_circulating_supply(&bank);
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        bank = Arc::new(new_from_parent(&bank));
+        let new_balance = 11;
+        for key in non_circulating_accounts {
+            bank.store_account(&key, &Account::new(new_balance, 0, &Pubkey::default()));
+        }
+        let non_circulating_supply = calculate_non_circulating_supply(&bank);
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts * new_balance) + (num_stake_accounts * balance)
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        // Advance bank an epoch, which should unlock stakes
+        for _ in 0..slots_per_epoch {
+            bank = Arc::new(new_from_parent(&bank));
+        }
+        assert_eq!(bank.epoch(), 1);
+        let non_circulating_supply = calculate_non_circulating_supply(&bank);
+        assert_eq!(
+            non_circulating_supply.lamports,
+            num_non_circulating_accounts * new_balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize
+        );
+    }
+}

core/src/poh_service.rs

@@ -1,10 +1,8 @@
 //! The `poh_service` module implements a service that records the passing of
 //! "ticks", a measure of time in the PoH stream
 use crate::poh_recorder::PohRecorder;
-use core_affinity;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
 use solana_sdk::poh_config::PohConfig;
-use solana_sys_tuner;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::{self, sleep, Builder, JoinHandle};

core/src/repair_response.rs

@@ -0,0 +1,129 @@
+use solana_ledger::{
+    blockstore::Blockstore,
+    shred::{Nonce, Shred, SIZE_OF_NONCE},
+};
+use solana_perf::packet::limited_deserialize;
+use solana_sdk::{clock::Slot, packet::Packet};
+use std::{io, net::SocketAddr};
+
+pub fn repair_response_packet(
+    blockstore: &Blockstore,
+    slot: Slot,
+    shred_index: u64,
+    dest: &SocketAddr,
+    nonce: Option<Nonce>,
+) -> Option<Packet> {
+    if Shred::is_nonce_unlocked(slot) && nonce.is_none()
+        || !Shred::is_nonce_unlocked(slot) && nonce.is_some()
+    {
+        return None;
+    }
+    let shred = blockstore
+        .get_data_shred(slot, shred_index)
+        .expect("Blockstore could not get data shred");
+    shred.map(|shred| repair_response_packet_from_shred(slot, shred, dest, nonce))
+}
+
+pub fn repair_response_packet_from_shred(
+    slot: Slot,
+    shred: Vec<u8>,
+    dest: &SocketAddr,
+    nonce: Option<Nonce>,
+) -> Packet {
+    let size_of_nonce = {
+        if Shred::is_nonce_unlocked(slot) {
+            assert!(nonce.is_some());
+            SIZE_OF_NONCE
+        } else {
+            assert!(nonce.is_none());
+            0
+        }
+    };
+    let mut packet = Packet::default();
+    packet.meta.size = shred.len() + size_of_nonce;
+    packet.meta.set_addr(dest);
+    packet.data[..shred.len()].copy_from_slice(&shred);
+    let mut wr = io::Cursor::new(&mut packet.data[shred.len()..]);
+    if let Some(nonce) = nonce {
+        bincode::serialize_into(&mut wr, &nonce).expect("Buffer not large enough to fit nonce");
+    }
+    packet
+}
+
+pub fn nonce(buf: &[u8]) -> Option<Nonce> {
+    if buf.len() < SIZE_OF_NONCE {
+        None
+    } else {
+        limited_deserialize(&buf[buf.len() - SIZE_OF_NONCE..]).ok()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_ledger::{
+        shred::{Shred, Shredder, UNLOCK_NONCE_SLOT},
+        sigverify_shreds::verify_shred_cpu,
+    };
+    use solana_sdk::signature::{Keypair, Signer};
+    use std::{
+        collections::HashMap,
+        net::{IpAddr, Ipv4Addr},
+    };
+
+    fn run_test_sigverify_shred_cpu_repair(slot: Slot) {
+        solana_logger::setup();
+        let mut shred = Shred::new_from_data(
+            slot,
+            0xc0de,
+            0xdead,
+            Some(&[1, 2, 3, 4]),
+            true,
+            true,
+            0,
+            0,
+            0xc0de,
+        );
+        assert_eq!(shred.slot(), slot);
+        let keypair = Keypair::new();
+        Shredder::sign_shred(&keypair, &mut shred);
+        trace!("signature {}", shred.common_header.signature);
+        let nonce = if Shred::is_nonce_unlocked(slot) {
+            Some(9)
+        } else {
+            None
+        };
+        let mut packet = repair_response_packet_from_shred(
+            slot,
+            shred.payload,
+            &SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080),
+            nonce,
+        );
+        packet.meta.repair = true;
+
+        let leader_slots = [(slot, keypair.pubkey().to_bytes())]
+            .iter()
+            .cloned()
+            .collect();
+        let rv = verify_shred_cpu(&packet, &leader_slots);
+        assert_eq!(rv, Some(1));
+
+        let wrong_keypair = Keypair::new();
+        let leader_slots = [(slot, wrong_keypair.pubkey().to_bytes())]
+            .iter()
+            .cloned()
+            .collect();
+        let rv = verify_shred_cpu(&packet, &leader_slots);
+        assert_eq!(rv, Some(0));
+
+        let leader_slots = HashMap::new();
+        let rv = verify_shred_cpu(&packet, &leader_slots);
+        assert_eq!(rv, None);
+    }
+
+    #[test]
+    fn test_sigverify_shred_cpu_repair() {
+        run_test_sigverify_shred_cpu_repair(UNLOCK_NONCE_SLOT);
+        run_test_sigverify_shred_cpu_repair(UNLOCK_NONCE_SLOT + 1);
+    }
+}

core/src/repair_service.rs

@@ -14,7 +14,6 @@ use solana_sdk::{clock::Slot, epoch_schedule::EpochSchedule, pubkey::Pubkey};
 use std::{
     collections::HashMap,
     iter::Iterator,
-    net::SocketAddr,
     net::UdpSocket,
     sync::atomic::{AtomicBool, Ordering},
     sync::{Arc, RwLock},
@@ -81,7 +80,7 @@ impl RepairService {
         blockstore: Arc<Blockstore>,
         exit: Arc<AtomicBool>,
         repair_socket: Arc<UdpSocket>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         repair_strategy: RepairStrategy,
         cluster_slots: Arc<ClusterSlots>,
     ) -> Self {
@@ -92,7 +91,7 @@ impl RepairService {
                     &blockstore,
                     &exit,
                     &repair_socket,
-                    &cluster_info,
+                    cluster_info,
                     repair_strategy,
                     &cluster_slots,
                 )
@@ -106,14 +105,14 @@ impl RepairService {
         blockstore: &Blockstore,
         exit: &AtomicBool,
         repair_socket: &UdpSocket,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         repair_strategy: RepairStrategy,
         cluster_slots: &Arc<ClusterSlots>,
     ) {
         let serve_repair = ServeRepair::new(cluster_info.clone());
-        let id = cluster_info.read().unwrap().id();
+        let id = cluster_info.id();
         if let RepairStrategy::RepairAll { .. } = repair_strategy {
-            Self::initialize_lowest_slot(id, blockstore, cluster_info);
+            Self::initialize_lowest_slot(id, blockstore, &cluster_info);
         }
         let mut repair_stats = RepairStats::default();
         let mut last_stats = Instant::now();
@@ -122,7 +121,7 @@ impl RepairService {
             ..
         } = repair_strategy
         {
-            Self::initialize_epoch_slots(blockstore, cluster_info, completed_slots_receiver);
+            Self::initialize_epoch_slots(blockstore, &cluster_info, completed_slots_receiver);
         }
         loop {
             if exit.load(Ordering::Relaxed) {
@@ -149,7 +148,7 @@ impl RepairService {
                         let lowest_slot = blockstore.lowest_slot();
                         Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
                         Self::update_completed_slots(completed_slots_receiver, &cluster_info);
-                        cluster_slots.update(new_root, cluster_info, bank_forks);
+                        cluster_slots.update(new_root, &cluster_info, bank_forks);
                         Self::generate_repairs(blockstore, new_root, MAX_REPAIR_LENGTH)
                     }
                 }
@@ -157,27 +156,19 @@ impl RepairService {
 
             if let Ok(repairs) = repairs {
                 let mut cache = HashMap::new();
-                let reqs: Vec<((SocketAddr, Vec<u8>), RepairType)> = repairs
-                    .into_iter()
-                    .filter_map(|repair_request| {
-                        serve_repair
-                            .repair_request(
-                                &cluster_slots,
-                                &repair_request,
-                                &mut cache,
-                                &mut repair_stats,
-                            )
-                            .map(|result| (result, repair_request))
-                            .ok()
-                    })
-                    .collect();
-
-                for ((to, req), _) in reqs {
-                    repair_socket.send_to(&req, to).unwrap_or_else(|e| {
-                        info!("{} repair req send_to({}) error {:?}", id, to, e);
-                        0
-                    });
-                }
+                repairs.into_iter().for_each(|repair_request| {
+                    if let Ok((to, req)) = serve_repair.repair_request(
+                        &cluster_slots,
+                        repair_request,
+                        &mut cache,
+                        &mut repair_stats,
+                    ) {
+                        repair_socket.send_to(&req, to).unwrap_or_else(|e| {
+                            info!("{} repair req send_to({}) error {:?}", id, to, e);
+                            0
+                        });
+                    }
+                });
             }
             if last_stats.elapsed().as_secs() > 1 {
                 let repair_total = repair_stats.shred.count
@@ -308,24 +299,17 @@ impl RepairService {
         }
     }
 
-    fn initialize_lowest_slot(
-        id: Pubkey,
-        blockstore: &Blockstore,
-        cluster_info: &RwLock<ClusterInfo>,
-    ) {
+    fn initialize_lowest_slot(id: Pubkey, blockstore: &Blockstore, cluster_info: &ClusterInfo) {
         // Safe to set into gossip because by this time, the leader schedule cache should
         // also be updated with the latest root (done in blockstore_processor) and thus
         // will provide a schedule to window_service for any incoming shreds up to the
         // last_confirmed_epoch.
-        cluster_info
-            .write()
-            .unwrap()
-            .push_lowest_slot(id, blockstore.lowest_slot());
+        cluster_info.push_lowest_slot(id, blockstore.lowest_slot());
     }
 
     fn update_completed_slots(
         completed_slots_receiver: &CompletedSlotsReceiver,
-        cluster_info: &RwLock<ClusterInfo>,
+        cluster_info: &ClusterInfo,
     ) {
         let mut slots: Vec<Slot> = vec![];
         while let Ok(mut more) = completed_slots_receiver.try_recv() {
@@ -333,20 +317,17 @@ impl RepairService {
         }
         slots.sort();
         if !slots.is_empty() {
-            cluster_info.write().unwrap().push_epoch_slots(&slots);
+            cluster_info.push_epoch_slots(&slots);
         }
     }
 
-    fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &RwLock<ClusterInfo>) {
-        cluster_info
-            .write()
-            .unwrap()
-            .push_lowest_slot(*id, lowest_slot);
+    fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &ClusterInfo) {
+        cluster_info.push_lowest_slot(*id, lowest_slot);
     }
 
     fn initialize_epoch_slots(
         blockstore: &Blockstore,
-        cluster_info: &RwLock<ClusterInfo>,
+        cluster_info: &ClusterInfo,
         completed_slots_receiver: &CompletedSlotsReceiver,
     ) {
         let root = blockstore.last_root();
@@ -367,7 +348,7 @@ impl RepairService {
         slots.sort();
         slots.dedup();
         if !slots.is_empty() {
-            cluster_info.write().unwrap().push_epoch_slots(&slots);
+            cluster_info.push_epoch_slots(&slots);
         }
     }
 
@@ -514,7 +495,7 @@ mod test {
             let blockstore = Blockstore::open(&blockstore_path).unwrap();
 
             let slots: Vec<u64> = vec![1, 3, 5, 7, 8];
-            let num_entries_per_slot = max_ticks_per_n_shreds(1) + 1;
+            let num_entries_per_slot = max_ticks_per_n_shreds(1, None) + 1;
 
             let shreds = make_chaining_slot_entries(&slots, num_entries_per_slot);
             for (mut slot_shreds, _) in shreds.into_iter() {
@@ -602,17 +583,13 @@ mod test {
     #[test]
     pub fn test_update_lowest_slot() {
         let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
-        let cluster_info = RwLock::new(ClusterInfo::new_with_invalid_keypair(
-            node_info.info.clone(),
-        ));
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(node_info.info.clone());
         RepairService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
         let lowest = cluster_info
-            .read()
-            .unwrap()
-            .get_lowest_slot_for_node(&Pubkey::default(), None)
-            .unwrap()
-            .0
-            .clone();
+            .get_lowest_slot_for_node(&Pubkey::default(), None, |lowest_slot, _| {
+                lowest_slot.clone()
+            })
+            .unwrap();
         assert_eq!(lowest.lowest, 5);
     }
 }

core/src/replay_stage.rs

@@ -88,7 +88,6 @@ struct SkippedSlotsInfo {
     last_skipped_slot: u64,
 }
 
-#[derive(Default)]
 pub struct ReplayStageConfig {
     pub my_pubkey: Pubkey,
     pub vote_account: Pubkey,
@@ -114,7 +113,7 @@ impl ReplayStage {
         config: ReplayStageConfig,
         blockstore: Arc<Blockstore>,
         bank_forks: Arc<RwLock<BankForks>>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         ledger_signal_receiver: Receiver<bool>,
         poh_recorder: Arc<Mutex<PohRecorder>>,
         vote_tracker: Arc<VoteTracker>,
@@ -140,8 +139,11 @@ impl ReplayStage {
         let mut tower = Tower::new(&my_pubkey, &vote_account, &bank_forks.read().unwrap());
 
         // Start the replay stage loop
-        let (lockouts_sender, commitment_service) =
-            AggregateCommitmentService::new(&exit, block_commitment_cache.clone());
+        let (lockouts_sender, commitment_service) = AggregateCommitmentService::new(
+            &exit,
+            block_commitment_cache.clone(),
+            subscriptions.clone(),
+        );
 
         #[allow(clippy::cognitive_complexity)]
         let t_replay = Builder::new()
@@ -218,13 +220,15 @@ impl ReplayStage {
 
                     let ancestors = Arc::new(bank_forks.read().unwrap().ancestors());
                     let descendants = HashMap::new();
+                    let forks_root = bank_forks.read().unwrap().root();
                     let start = allocated.get();
                     let mut frozen_banks: Vec<_> = bank_forks
                         .read()
                         .unwrap()
                         .frozen_banks()
-                        .values()
-                        .cloned()
+                        .into_iter()
+                        .filter(|(slot, _)| *slot >= forks_root)
+                        .map(|(_, bank)| bank)
                         .collect();
                     let newly_computed_slot_stats = Self::compute_bank_stats(
                         &my_pubkey,
@@ -283,7 +287,11 @@ impl ReplayStage {
 
                         for r in failure_reasons {
                             if let HeaviestForkFailures::NoPropagatedConfirmation(slot) = r {
-                                progress.log_propagated_stats(slot, &bank_forks);
+                                if let Some(latest_leader_slot) =
+                                    progress.get_latest_leader_slot(slot)
+                                {
+                                    progress.log_propagated_stats(latest_leader_slot, &bank_forks);
+                                }
                             }
                         }
                     }
@@ -292,10 +300,6 @@ impl ReplayStage {
 
                     // Vote on a fork
                     if let Some(ref vote_bank) = vote_bank {
-                        subscriptions.notify_subscribers(
-                            block_commitment_cache.read().unwrap().slot(),
-                            &bank_forks,
-                        );
                         if let Some(votable_leader) =
                             leader_schedule_cache.slot_leader_at(vote_bank.slot(), Some(vote_bank))
                         {
@@ -323,6 +327,7 @@ impl ReplayStage {
                             &latest_root_senders,
                             &mut all_pubkeys,
                             &subscriptions,
+                            &block_commitment_cache,
                         )?;
                     };
 
@@ -685,7 +690,7 @@ impl ReplayStage {
         progress: &mut ProgressMap,
         vote_account_pubkey: &Pubkey,
         authorized_voter_keypairs: &[Arc<Keypair>],
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &Arc<ClusterInfo>,
         blockstore: &Arc<Blockstore>,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         root_bank_sender: &Sender<Vec<Arc<Bank>>>,
@@ -694,6 +699,7 @@ impl ReplayStage {
         latest_root_senders: &[Sender<Slot>],
         all_pubkeys: &mut HashSet<Rc<Pubkey>>,
         subscriptions: &Arc<RpcSubscriptions>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
     ) -> Result<()> {
         if bank.is_empty() {
             inc_new_counter_info!("replay_stage-voted_empty_bank", 1);
@@ -719,12 +725,19 @@ impl ReplayStage {
             blockstore
                 .set_roots(&rooted_slots)
                 .expect("Ledger set roots failed");
+            let largest_confirmed_root = Some(
+                block_commitment_cache
+                    .read()
+                    .unwrap()
+                    .largest_confirmed_root(),
+            );
             Self::handle_new_root(
                 new_root,
                 &bank_forks,
                 progress,
                 accounts_hash_sender,
                 all_pubkeys,
+                largest_confirmed_root,
             );
             subscriptions.notify_roots(rooted_slots);
             latest_root_senders.iter().for_each(|s| {
@@ -758,7 +771,7 @@ impl ReplayStage {
     }
 
     fn push_vote(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &ClusterInfo,
         bank: &Arc<Bank>,
         vote_account_pubkey: &Pubkey,
         authorized_voter_keypairs: &[Arc<Keypair>],
@@ -811,7 +824,7 @@ impl ReplayStage {
             }
             Some(authorized_voter_keypair) => authorized_voter_keypair,
         };
-        let node_keypair = cluster_info.read().unwrap().keypair.clone();
+        let node_keypair = cluster_info.keypair.clone();
 
         // Send our last few votes along with the new one
         let vote_ix = vote_instruction::vote(
@@ -825,10 +838,7 @@ impl ReplayStage {
         let blockhash = bank.last_blockhash();
         vote_tx.partial_sign(&[node_keypair.as_ref()], blockhash);
         vote_tx.partial_sign(&[authorized_voter_keypair.as_ref()], blockhash);
-        cluster_info
-            .write()
-            .unwrap()
-            .push_vote(tower_index, vote_tx);
+        cluster_info.push_vote(tower_index, vote_tx);
     }
 
     fn update_commitment_cache(
@@ -1482,17 +1492,19 @@ impl ReplayStage {
     }
 
     pub(crate) fn handle_new_root(
-        new_root: u64,
+        new_root: Slot,
         bank_forks: &RwLock<BankForks>,
         progress: &mut ProgressMap,
         accounts_hash_sender: &Option<SnapshotPackageSender>,
         all_pubkeys: &mut HashSet<Rc<Pubkey>>,
+        largest_confirmed_root: Option<Slot>,
     ) {
         let old_epoch = bank_forks.read().unwrap().root_bank().epoch();
-        bank_forks
-            .write()
-            .unwrap()
-            .set_root(new_root, accounts_hash_sender);
+        bank_forks.write().unwrap().set_root(
+            new_root,
+            accounts_hash_sender,
+            largest_confirmed_root,
+        );
         let r_bank_forks = bank_forks.read().unwrap();
         let new_epoch = bank_forks.read().unwrap().root_bank().epoch();
         if old_epoch != new_epoch {
@@ -1513,7 +1525,11 @@ impl ReplayStage {
         // Find the next slot that chains to the old slot
         let forks = bank_forks.read().unwrap();
         let frozen_banks = forks.frozen_banks();
-        let frozen_bank_slots: Vec<u64> = frozen_banks.keys().cloned().collect();
+        let frozen_bank_slots: Vec<u64> = frozen_banks
+            .keys()
+            .cloned()
+            .filter(|s| *s >= forks.root())
+            .collect();
         let next_slots = blockstore
             .get_slots_since(&frozen_bank_slots)
             .expect("Db error");
@@ -1977,10 +1993,6 @@ pub(crate) mod tests {
             );
             let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank0));
             let exit = Arc::new(AtomicBool::new(false));
-            let subscriptions = Arc::new(RpcSubscriptions::new(
-                &exit,
-                Arc::new(RwLock::new(BlockCommitmentCache::default())),
-            ));
             let mut bank_forks = BankForks::new(0, bank0);
 
             // Insert a non-root bank so that the propagation logic will update this
@@ -2004,7 +2016,14 @@ pub(crate) mod tests {
             assert!(progress.get_propagated_stats(1).unwrap().is_leader_slot);
             bank1.freeze();
             bank_forks.insert(bank1);
-            let bank_forks = RwLock::new(bank_forks);
+            let bank_forks = Arc::new(RwLock::new(bank_forks));
+            let subscriptions = Arc::new(RpcSubscriptions::new(
+                &exit,
+                bank_forks.clone(),
+                Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+                    blockstore.clone(),
+                ))),
+            ));
 
             // Insert shreds for slot NUM_CONSECUTIVE_LEADER_SLOTS,
             // chaining to slot 1
@@ -2095,12 +2114,66 @@ pub(crate) mod tests {
         for i in 0..=root {
             progress.insert(i, ForkProgress::new(Hash::default(), None, None, 0, 0));
         }
-        ReplayStage::handle_new_root(root, &bank_forks, &mut progress, &None, &mut HashSet::new());
+        ReplayStage::handle_new_root(
+            root,
+            &bank_forks,
+            &mut progress,
+            &None,
+            &mut HashSet::new(),
+            None,
+        );
         assert_eq!(bank_forks.read().unwrap().root(), root);
         assert_eq!(progress.len(), 1);
         assert!(progress.get(&root).is_some());
     }
 
+    #[test]
+    fn test_handle_new_root_ahead_of_largest_confirmed_root() {
+        let genesis_config = create_genesis_config(10_000).genesis_config;
+        let bank0 = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank0)));
+        let confirmed_root = 1;
+        let fork = 2;
+        let bank1 = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(0).unwrap(),
+            &Pubkey::default(),
+            confirmed_root,
+        );
+        bank_forks.write().unwrap().insert(bank1);
+        let bank2 = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(confirmed_root).unwrap(),
+            &Pubkey::default(),
+            fork,
+        );
+        bank_forks.write().unwrap().insert(bank2);
+        let root = 3;
+        let root_bank = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(confirmed_root).unwrap(),
+            &Pubkey::default(),
+            root,
+        );
+        bank_forks.write().unwrap().insert(root_bank);
+        let mut progress = ProgressMap::default();
+        for i in 0..=root {
+            progress.insert(i, ForkProgress::new(Hash::default(), None, None, 0, 0));
+        }
+        ReplayStage::handle_new_root(
+            root,
+            &bank_forks,
+            &mut progress,
+            &None,
+            &mut HashSet::new(),
+            Some(confirmed_root),
+        );
+        assert_eq!(bank_forks.read().unwrap().root(), root);
+        assert!(bank_forks.read().unwrap().get(confirmed_root).is_some());
+        assert!(bank_forks.read().unwrap().get(fork).is_none());
+        assert_eq!(progress.len(), 2);
+        assert!(progress.get(&root).is_some());
+        assert!(progress.get(&confirmed_root).is_some());
+        assert!(progress.get(&fork).is_none());
+    }
+
     #[test]
     fn test_dead_fork_transaction_error() {
         let keypair1 = Keypair::new();
@@ -2291,6 +2364,7 @@ pub(crate) mod tests {
                 ShredCommonHeader::default(),
                 data_header,
                 CodingShredHeader::default(),
+                PACKET_DATA_SIZE,
             );
             bincode::serialize_into(
                 &mut shred.payload[SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER..],
@@ -2367,11 +2441,8 @@ pub(crate) mod tests {
             bank.store_account(&pubkey, &leader_vote_account);
         }
 
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
-        let (lockouts_sender, _) = AggregateCommitmentService::new(
-            &Arc::new(AtomicBool::new(false)),
-            block_commitment_cache.clone(),
-        );
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let leader_pubkey = Pubkey::new_rand();
         let leader_lamports = 3;
@@ -2392,6 +2463,18 @@ pub(crate) mod tests {
             vec![0],
         )));
 
+        let exit = Arc::new(AtomicBool::new(false));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
+        let subscriptions = Arc::new(RpcSubscriptions::new(
+            &exit,
+            bank_forks.clone(),
+            block_commitment_cache.clone(),
+        ));
+        let (lockouts_sender, _) =
+            AggregateCommitmentService::new(&exit, block_commitment_cache.clone(), subscriptions);
+
         assert!(block_commitment_cache
             .read()
             .unwrap()
@@ -2572,14 +2655,14 @@ pub(crate) mod tests {
                             meta.err,
                             Some(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                         assert_eq!(
                             meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                     } else {
@@ -3066,7 +3149,7 @@ pub(crate) mod tests {
         bank_forks.insert(Bank::new_from_parent(&bank0, &Pubkey::default(), 9));
         let bank9 = bank_forks.get(9).unwrap().clone();
         bank_forks.insert(Bank::new_from_parent(&bank9, &Pubkey::default(), 10));
-        bank_forks.set_root(9, &None);
+        bank_forks.set_root(9, &None, None);
         let total_epoch_stake = bank0.total_epoch_stake();
 
         // Insert new ForkProgress for slot 10 and its
@@ -3159,7 +3242,7 @@ pub(crate) mod tests {
 
         let stake_per_validator = 10_000;
         let (mut bank_forks, mut progress_map) = initialize_state(&keypairs, stake_per_validator);
-        bank_forks.set_root(0, &None);
+        bank_forks.set_root(0, &None, None);
         let total_epoch_stake = bank_forks.root_bank().total_epoch_stake();
 
         // Insert new ForkProgress representing a slot for all slots 1..=num_banks. Only
@@ -3241,7 +3324,7 @@ pub(crate) mod tests {
 
         let stake_per_validator = 10_000;
         let (mut bank_forks, mut progress_map) = initialize_state(&keypairs, stake_per_validator);
-        bank_forks.set_root(0, &None);
+        bank_forks.set_root(0, &None, None);
 
         let total_epoch_stake = num_validators as u64 * stake_per_validator;
 

core/src/retransmit_stage.rs

@@ -3,6 +3,7 @@
 use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
     cluster_slots::ClusterSlots,
+    contact_info::ContactInfo,
     repair_service::RepairStrategy,
     result::{Error, Result},
     window_service::{should_retransmit_and_persist, WindowService},
@@ -17,12 +18,16 @@ use solana_ledger::{
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_error;
 use solana_perf::packet::Packets;
+use solana_sdk::clock::{Epoch, Slot};
 use solana_sdk::epoch_schedule::EpochSchedule;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::timing::timestamp;
 use solana_streamer::streamer::PacketReceiver;
 use std::{
     cmp,
+    collections::{BTreeMap, HashMap},
     net::UdpSocket,
-    sync::atomic::{AtomicBool, Ordering},
+    sync::atomic::{AtomicBool, AtomicU64, Ordering},
     sync::mpsc::channel,
     sync::mpsc::RecvTimeoutError,
     sync::Mutex,
@@ -35,13 +40,151 @@ use std::{
 // it doesn't pull up too much work.
 const MAX_PACKET_BATCH_SIZE: usize = 100;
 
+#[derive(Default)]
+struct RetransmitStats {
+    total_packets: AtomicU64,
+    total_batches: AtomicU64,
+    total_time: AtomicU64,
+    epoch_fetch: AtomicU64,
+    epoch_cache_update: AtomicU64,
+    repair_total: AtomicU64,
+    discard_total: AtomicU64,
+    retransmit_total: AtomicU64,
+    last_ts: AtomicU64,
+    compute_turbine_peers_total: AtomicU64,
+    packets_by_slot: Mutex<BTreeMap<Slot, usize>>,
+    packets_by_source: Mutex<BTreeMap<String, usize>>,
+}
+
+#[allow(clippy::too_many_arguments)]
+fn update_retransmit_stats(
+    stats: &Arc<RetransmitStats>,
+    total_time: u64,
+    total_packets: usize,
+    retransmit_total: u64,
+    discard_total: u64,
+    repair_total: u64,
+    compute_turbine_peers_total: u64,
+    peers_len: usize,
+    packets_by_slot: HashMap<Slot, usize>,
+    packets_by_source: HashMap<String, usize>,
+    epoch_fetch: u64,
+    epoch_cach_update: u64,
+) {
+    stats.total_time.fetch_add(total_time, Ordering::Relaxed);
+    stats
+        .total_packets
+        .fetch_add(total_packets as u64, Ordering::Relaxed);
+    stats
+        .retransmit_total
+        .fetch_add(retransmit_total, Ordering::Relaxed);
+    stats
+        .repair_total
+        .fetch_add(repair_total, Ordering::Relaxed);
+    stats
+        .discard_total
+        .fetch_add(discard_total, Ordering::Relaxed);
+    stats
+        .compute_turbine_peers_total
+        .fetch_add(compute_turbine_peers_total, Ordering::Relaxed);
+    stats.total_batches.fetch_add(1, Ordering::Relaxed);
+    stats.epoch_fetch.fetch_add(epoch_fetch, Ordering::Relaxed);
+    stats
+        .epoch_cache_update
+        .fetch_add(epoch_cach_update, Ordering::Relaxed);
+    {
+        let mut stats_packets_by_slot = stats.packets_by_slot.lock().unwrap();
+        for (slot, count) in packets_by_slot {
+            *stats_packets_by_slot.entry(slot).or_insert(0) += count;
+        }
+    }
+    {
+        let mut stats_packets_by_source = stats.packets_by_source.lock().unwrap();
+        for (source, count) in packets_by_source {
+            *stats_packets_by_source.entry(source).or_insert(0) += count;
+        }
+    }
+
+    let now = timestamp();
+    let last = stats.last_ts.load(Ordering::Relaxed);
+    if now - last > 2000 && stats.last_ts.compare_and_swap(last, now, Ordering::Relaxed) == last {
+        datapoint_info!("retransmit-num_nodes", ("count", peers_len, i64));
+        datapoint_info!(
+            "retransmit-stage",
+            (
+                "total_time",
+                stats.total_time.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "epoch_fetch",
+                stats.epoch_fetch.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "epoch_cache_update",
+                stats.epoch_cache_update.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "total_batches",
+                stats.total_batches.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "total_packets",
+                stats.total_packets.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "retransmit_total",
+                stats.retransmit_total.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "compute_turbine",
+                stats.compute_turbine_peers_total.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "repair_total",
+                stats.repair_total.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "discard_total",
+                stats.discard_total.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+        );
+        let mut packets_by_slot = stats.packets_by_slot.lock().unwrap();
+        info!("retransmit: packets_by_slot: {:#?}", packets_by_slot);
+        packets_by_slot.clear();
+        drop(packets_by_slot);
+        let mut packets_by_source = stats.packets_by_source.lock().unwrap();
+        info!("retransmit: packets_by_source: {:#?}", packets_by_source);
+        packets_by_source.clear();
+    }
+}
+
+#[derive(Default)]
+struct EpochStakesCache {
+    epoch: Epoch,
+    stakes: Option<Arc<HashMap<Pubkey, u64>>>,
+    peers: Vec<ContactInfo>,
+    stakes_and_index: Vec<(u64, usize)>,
+}
+
 fn retransmit(
     bank_forks: &Arc<RwLock<BankForks>>,
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
-    cluster_info: &Arc<RwLock<ClusterInfo>>,
+    cluster_info: &ClusterInfo,
     r: &Arc<Mutex<PacketReceiver>>,
     sock: &UdpSocket,
     id: u32,
+    stats: &Arc<RetransmitStats>,
+    epoch_stakes_cache: &Arc<RwLock<EpochStakesCache>>,
+    last_peer_update: &Arc<AtomicU64>,
 ) -> Result<()> {
     let timer = Duration::new(1, 0);
     let r_lock = r.lock().unwrap();
@@ -58,20 +201,49 @@ fn retransmit(
     }
     drop(r_lock);
 
+    let mut epoch_fetch = Measure::start("retransmit_epoch_fetch");
     let r_bank = bank_forks.read().unwrap().working_bank();
     let bank_epoch = r_bank.get_leader_schedule_epoch(r_bank.slot());
+    epoch_fetch.stop();
+
+    let mut epoch_cache_update = Measure::start("retransmit_epoch_cach_update");
+    let mut r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    if r_epoch_stakes_cache.epoch != bank_epoch {
+        drop(r_epoch_stakes_cache);
+        let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
+        if w_epoch_stakes_cache.epoch != bank_epoch {
+            let stakes = staking_utils::staked_nodes_at_epoch(&r_bank, bank_epoch);
+            let stakes = stakes.map(Arc::new);
+            w_epoch_stakes_cache.stakes = stakes;
+            w_epoch_stakes_cache.epoch = bank_epoch;
+        }
+        drop(w_epoch_stakes_cache);
+        r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    }
+
+    let now = timestamp();
+    let last = last_peer_update.load(Ordering::Relaxed);
+    if now - last > 1000 && last_peer_update.compare_and_swap(last, now, Ordering::Relaxed) == last
+    {
+        drop(r_epoch_stakes_cache);
+        let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
+        let (peers, stakes_and_index) =
+            cluster_info.sorted_retransmit_peers_and_stakes(w_epoch_stakes_cache.stakes.clone());
+        w_epoch_stakes_cache.peers = peers;
+        w_epoch_stakes_cache.stakes_and_index = stakes_and_index;
+        drop(w_epoch_stakes_cache);
+        r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    }
     let mut peers_len = 0;
-    let stakes = staking_utils::staked_nodes_at_epoch(&r_bank, bank_epoch);
-    let stakes = stakes.map(Arc::new);
-    let (peers, stakes_and_index) = cluster_info
-        .read()
-        .unwrap()
-        .sorted_retransmit_peers_and_stakes(stakes);
-    let me = cluster_info.read().unwrap().my_data();
+    epoch_cache_update.stop();
+
+    let my_id = cluster_info.id();
     let mut discard_total = 0;
     let mut repair_total = 0;
     let mut retransmit_total = 0;
     let mut compute_turbine_peers_total = 0;
+    let mut packets_by_slot: HashMap<Slot, usize> = HashMap::new();
+    let mut packets_by_source: HashMap<String, usize> = HashMap::new();
     for mut packets in packet_v {
         for packet in packets.packets.iter_mut() {
             // skip discarded packets and repair packets
@@ -88,9 +260,9 @@ fn retransmit(
 
             let mut compute_turbine_peers = Measure::start("turbine_start");
             let (my_index, mut shuffled_stakes_and_index) = ClusterInfo::shuffle_peers_and_index(
-                &me.id,
-                &peers,
-                &stakes_and_index,
+                &my_id,
+                &r_epoch_stakes_cache.peers,
+                &r_epoch_stakes_cache.stakes_and_index,
                 packet.meta.seed,
             );
             peers_len = cmp::max(peers_len, shuffled_stakes_and_index.len());
@@ -103,10 +275,21 @@ fn retransmit(
 
             let (neighbors, children) =
                 compute_retransmit_peers(DATA_PLANE_FANOUT, my_index, indexes);
-            let neighbors: Vec<_> = neighbors.into_iter().map(|index| &peers[index]).collect();
-            let children: Vec<_> = children.into_iter().map(|index| &peers[index]).collect();
+            let neighbors: Vec<_> = neighbors
+                .into_iter()
+                .map(|index| &r_epoch_stakes_cache.peers[index])
+                .collect();
+            let children: Vec<_> = children
+                .into_iter()
+                .map(|index| &r_epoch_stakes_cache.peers[index])
+                .collect();
             compute_turbine_peers.stop();
-            compute_turbine_peers_total += compute_turbine_peers.as_ms();
+            compute_turbine_peers_total += compute_turbine_peers.as_us();
+
+            *packets_by_slot.entry(packet.meta.slot).or_insert(0) += 1;
+            *packets_by_source
+                .entry(packet.meta.addr().to_string())
+                .or_insert(0) += 1;
 
             let leader =
                 leader_schedule_cache.slot_leader_at(packet.meta.slot, Some(r_bank.as_ref()));
@@ -118,7 +301,7 @@ fn retransmit(
                 ClusterInfo::retransmit_to(&children, packet, leader, sock, true)?;
             }
             retransmit_time.stop();
-            retransmit_total += retransmit_time.as_ms();
+            retransmit_total += retransmit_time.as_us();
         }
     }
     timer_start.stop();
@@ -129,16 +312,21 @@ fn retransmit(
         retransmit_total,
         id,
     );
-    datapoint_debug!("cluster_info-num_nodes", ("count", peers_len, i64));
-    datapoint_debug!(
-        "retransmit-stage",
-        ("total_time", timer_start.as_ms() as i64, i64),
-        ("total_packets", total_packets as i64, i64),
-        ("retransmit_total", retransmit_total as i64, i64),
-        ("compute_turbine", compute_turbine_peers_total as i64, i64),
-        ("repair_total", i64::from(repair_total), i64),
-        ("discard_total", i64::from(discard_total), i64),
+    update_retransmit_stats(
+        stats,
+        timer_start.as_us(),
+        total_packets,
+        retransmit_total,
+        discard_total,
+        repair_total,
+        compute_turbine_peers_total,
+        peers_len,
+        packets_by_slot,
+        packets_by_source,
+        epoch_fetch.as_us(),
+        epoch_cache_update.as_us(),
     );
+
     Ok(())
 }
 
@@ -154,9 +342,10 @@ pub fn retransmitter(
     sockets: Arc<Vec<UdpSocket>>,
     bank_forks: Arc<RwLock<BankForks>>,
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
-    cluster_info: Arc<RwLock<ClusterInfo>>,
+    cluster_info: Arc<ClusterInfo>,
     r: Arc<Mutex<PacketReceiver>>,
 ) -> Vec<JoinHandle<()>> {
+    let stats = Arc::new(RetransmitStats::default());
     (0..sockets.len())
         .map(|s| {
             let sockets = sockets.clone();
@@ -164,6 +353,9 @@ pub fn retransmitter(
             let leader_schedule_cache = leader_schedule_cache.clone();
             let r = r.clone();
             let cluster_info = cluster_info.clone();
+            let stats = stats.clone();
+            let epoch_stakes_cache = Arc::new(RwLock::new(EpochStakesCache::default()));
+            let last_peer_update = Arc::new(AtomicU64::new(0));
 
             Builder::new()
                 .name("solana-retransmitter".to_string())
@@ -177,6 +369,9 @@ pub fn retransmitter(
                             &r,
                             &sockets[s],
                             s as u32,
+                            &stats,
+                            &epoch_stakes_cache,
+                            &last_peer_update,
                         ) {
                             match e {
                                 Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
@@ -206,7 +401,7 @@ impl RetransmitStage {
         bank_forks: Arc<RwLock<BankForks>>,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         blockstore: Arc<Blockstore>,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        cluster_info: &Arc<ClusterInfo>,
         retransmit_sockets: Arc<Vec<UdpSocket>>,
         repair_socket: Arc<UdpSocket>,
         verified_receiver: CrossbeamReceiver<Vec<Packets>>,
@@ -316,11 +511,11 @@ mod tests {
             .unwrap();
 
         let other = ContactInfo::new_localhost(&Pubkey::new_rand(), 0);
-        let mut cluster_info = ClusterInfo::new_with_invalid_keypair(other);
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(other);
         cluster_info.insert_info(me);
 
         let retransmit_socket = Arc::new(vec![UdpSocket::bind("0.0.0.0:0").unwrap()]);
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
 
         let (retransmit_sender, retransmit_receiver) = channel();
         let t_retransmit = retransmitter(

core/src/rpc_error.rs

@@ -0,0 +1,54 @@
+use jsonrpc_core::{Error, ErrorCode};
+use solana_sdk::clock::Slot;
+
+const JSON_RPC_SERVER_ERROR_0: i64 = -32000;
+const JSON_RPC_SERVER_ERROR_1: i64 = -32001;
+const JSON_RPC_SERVER_ERROR_2: i64 = -32002;
+
+pub enum RpcCustomError {
+    NonexistentClusterRoot {
+        cluster_root: Slot,
+        node_root: Slot,
+    },
+    BlockCleanedUp {
+        slot: Slot,
+        first_available_block: Slot,
+    },
+    SendTransactionPreflightFailure {
+        message: String,
+    },
+}
+
+impl From<RpcCustomError> for Error {
+    fn from(e: RpcCustomError) -> Self {
+        match e {
+            RpcCustomError::NonexistentClusterRoot {
+                cluster_root,
+                node_root,
+            } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_0),
+                message: format!(
+                    "Cluster largest_confirmed_root {} does not exist on node. Node root: {}",
+                    cluster_root, node_root,
+                ),
+                data: None,
+            },
+            RpcCustomError::BlockCleanedUp {
+                slot,
+                first_available_block,
+            } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_1),
+                message: format!(
+                    "Block {} cleaned up, does not exist on node. First available block: {}",
+                    slot, first_available_block,
+                ),
+                data: None,
+            },
+            RpcCustomError::SendTransactionPreflightFailure { message } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_2),
+                message,
+                data: None,
+            },
+        }
+    }
+}

core/src/rpc_health.rs

@@ -0,0 +1,118 @@
+use crate::cluster_info::ClusterInfo;
+use solana_sdk::pubkey::Pubkey;
+use std::{
+    collections::HashSet,
+    sync::atomic::{AtomicBool, Ordering},
+    sync::Arc,
+};
+
+#[derive(PartialEq, Clone, Copy)]
+pub enum RpcHealthStatus {
+    Ok,
+    Behind, // Validator is behind its trusted validators
+}
+
+pub struct RpcHealth {
+    cluster_info: Arc<ClusterInfo>,
+    trusted_validators: Option<HashSet<Pubkey>>,
+    health_check_slot_distance: u64,
+    override_health_check: Arc<AtomicBool>,
+    #[cfg(test)]
+    stub_health_status: std::sync::RwLock<Option<RpcHealthStatus>>,
+}
+
+impl RpcHealth {
+    pub fn new(
+        cluster_info: Arc<ClusterInfo>,
+        trusted_validators: Option<HashSet<Pubkey>>,
+        health_check_slot_distance: u64,
+        override_health_check: Arc<AtomicBool>,
+    ) -> Self {
+        Self {
+            cluster_info,
+            trusted_validators,
+            health_check_slot_distance,
+            override_health_check,
+            #[cfg(test)]
+            stub_health_status: std::sync::RwLock::new(None),
+        }
+    }
+
+    pub fn check(&self) -> RpcHealthStatus {
+        #[cfg(test)]
+        {
+            if let Some(stub_health_status) = *self.stub_health_status.read().unwrap() {
+                return stub_health_status;
+            }
+        }
+
+        if self.override_health_check.load(Ordering::Relaxed) {
+            RpcHealthStatus::Ok
+        } else if let Some(trusted_validators) = &self.trusted_validators {
+            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
+                (
+                    self.cluster_info
+                        .get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| {
+                            hashes
+                                .iter()
+                                .max_by(|a, b| a.0.cmp(&b.0))
+                                .map(|slot_hash| slot_hash.0)
+                        })
+                        .flatten()
+                        .unwrap_or(0),
+                    trusted_validators
+                        .iter()
+                        .map(|trusted_validator| {
+                            self.cluster_info
+                                .get_accounts_hash_for_node(&trusted_validator, |hashes| {
+                                    hashes
+                                        .iter()
+                                        .max_by(|a, b| a.0.cmp(&b.0))
+                                        .map(|slot_hash| slot_hash.0)
+                                })
+                                .flatten()
+                                .unwrap_or(0)
+                        })
+                        .max()
+                        .unwrap_or(0),
+                )
+            };
+
+            // This validator is considered healthy if its latest account hash slot is within
+            // `health_check_slot_distance` of the latest trusted validator's account hash slot
+            if latest_account_hash_slot > 0
+                && latest_trusted_validator_account_hash_slot > 0
+                && latest_account_hash_slot
+                    > latest_trusted_validator_account_hash_slot
+                        .saturating_sub(self.health_check_slot_distance)
+            {
+                RpcHealthStatus::Ok
+            } else {
+                warn!(
+                    "health check: me={}, latest trusted_validator={}",
+                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
+                );
+                RpcHealthStatus::Behind
+            }
+        } else {
+            // No trusted validator point of reference available, so this validator is healthy
+            // because it's running
+            RpcHealthStatus::Ok
+        }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn stub() -> Arc<Self> {
+        Arc::new(Self::new(
+            Arc::new(ClusterInfo::default()),
+            None,
+            42,
+            Arc::new(AtomicBool::new(false)),
+        ))
+    }
+
+    #[cfg(test)]
+    pub(crate) fn stub_set_health_status(&self, stub_health_status: Option<RpcHealthStatus>) {
+        *self.stub_health_status.write().unwrap() = stub_health_status;
+    }
+}

core/src/rpc_pubsub.rs

@@ -1,14 +1,23 @@
 //! The `pubsub` module implements a threaded subscription service on client RPC request
 
-use crate::rpc_subscriptions::{Confirmations, RpcSubscriptions, SlotInfo};
+use crate::rpc_subscriptions::{RpcSubscriptions, RpcVote, SlotInfo};
 use jsonrpc_core::{Error, ErrorCode, Result};
 use jsonrpc_derive::rpc;
 use jsonrpc_pubsub::{typed::Subscriber, Session, SubscriptionId};
 use solana_client::rpc_response::{
     Response as RpcResponse, RpcAccount, RpcKeyedAccount, RpcSignatureResult,
 };
-use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature};
-use std::sync::{atomic, Arc};
+#[cfg(test)]
+use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
+use solana_sdk::{
+    clock::Slot, commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature,
+};
+#[cfg(test)]
+use std::sync::RwLock;
+use std::{
+    str::FromStr,
+    sync::{atomic, Arc},
+};
 
 // Suppress needless_return due to
 //   https://github.com/paritytech/jsonrpc/blob/2d38e6424d8461cdf72e78425ce67d51af9c6586/derive/src/lib.rs#L204
@@ -30,7 +39,7 @@ pub trait RpcSolPubSub {
         meta: Self::Metadata,
         subscriber: Subscriber<RpcResponse<RpcAccount>>,
         pubkey_str: String,
-        confirmations: Option<Confirmations>,
+        commitment: Option<CommitmentConfig>,
     );
 
     // Unsubscribe from account notification subscription.
@@ -54,7 +63,7 @@ pub trait RpcSolPubSub {
         meta: Self::Metadata,
         subscriber: Subscriber<RpcResponse<RpcKeyedAccount>>,
         pubkey_str: String,
-        confirmations: Option<Confirmations>,
+        commitment: Option<CommitmentConfig>,
     );
 
     // Unsubscribe from account notification subscription.
@@ -78,7 +87,7 @@ pub trait RpcSolPubSub {
         meta: Self::Metadata,
         subscriber: Subscriber<RpcResponse<RpcSignatureResult>>,
         signature_str: String,
-        confirmations: Option<Confirmations>,
+        commitment: Option<CommitmentConfig>,
     );
 
     // Unsubscribe from signature notification subscription.
@@ -105,6 +114,18 @@ pub trait RpcSolPubSub {
     )]
     fn slot_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
 
+    // Get notification when vote is encountered
+    #[pubsub(subscription = "voteNotification", subscribe, name = "voteSubscribe")]
+    fn vote_subscribe(&self, meta: Self::Metadata, subscriber: Subscriber<RpcVote>);
+
+    // Unsubscribe from vote notification subscription.
+    #[pubsub(
+        subscription = "voteNotification",
+        unsubscribe,
+        name = "voteUnsubscribe"
+    )]
+    fn vote_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
+
     // Get notification when a new root is set
     #[pubsub(subscription = "rootNotification", subscribe, name = "rootSubscribe")]
     fn root_subscribe(&self, meta: Self::Metadata, subscriber: Subscriber<Slot>);
@@ -118,7 +139,6 @@ pub trait RpcSolPubSub {
     fn root_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
 }
 
-#[derive(Default)]
 pub struct RpcSolPubSubImpl {
     uid: Arc<atomic::AtomicUsize>,
     subscriptions: Arc<RpcSubscriptions>,
@@ -129,9 +149,19 @@ impl RpcSolPubSubImpl {
         let uid = Arc::new(atomic::AtomicUsize::default());
         Self { uid, subscriptions }
     }
-}
 
-use std::str::FromStr;
+    #[cfg(test)]
+    fn default_with_blockstore_bank_forks(
+        blockstore: Arc<Blockstore>,
+        bank_forks: Arc<RwLock<BankForks>>,
+    ) -> Self {
+        let uid = Arc::new(atomic::AtomicUsize::default());
+        let subscriptions = Arc::new(RpcSubscriptions::default_with_blockstore_bank_forks(
+            blockstore, bank_forks,
+        ));
+        Self { uid, subscriptions }
+    }
+}
 
 fn param<T: FromStr>(param_str: &str, thing: &str) -> Result<T> {
     param_str.parse::<T>().map_err(|_e| Error {
@@ -149,19 +179,15 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         _meta: Self::Metadata,
         subscriber: Subscriber<RpcResponse<RpcAccount>>,
         pubkey_str: String,
-        confirmations: Option<Confirmations>,
+        commitment: Option<CommitmentConfig>,
     ) {
         match param::<Pubkey>(&pubkey_str, "pubkey") {
             Ok(pubkey) => {
                 let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
                 let sub_id = SubscriptionId::Number(id as u64);
                 info!("account_subscribe: account={:?} id={:?}", pubkey, sub_id);
-                self.subscriptions.add_account_subscription(
-                    pubkey,
-                    confirmations,
-                    sub_id,
-                    subscriber,
-                )
+                self.subscriptions
+                    .add_account_subscription(pubkey, commitment, sub_id, subscriber)
             }
             Err(e) => subscriber.reject(e).unwrap(),
         }
@@ -189,19 +215,15 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         _meta: Self::Metadata,
         subscriber: Subscriber<RpcResponse<RpcKeyedAccount>>,
         pubkey_str: String,
-        confirmations: Option<Confirmations>,
+        commitment: Option<CommitmentConfig>,
     ) {
         match param::<Pubkey>(&pubkey_str, "pubkey") {
             Ok(pubkey) => {
                 let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
                 let sub_id = SubscriptionId::Number(id as u64);
                 info!("program_subscribe: account={:?} id={:?}", pubkey, sub_id);
-                self.subscriptions.add_program_subscription(
-                    pubkey,
-                    confirmations,
-                    sub_id,
-                    subscriber,
-                )
+                self.subscriptions
+                    .add_program_subscription(pubkey, commitment, sub_id, subscriber)
             }
             Err(e) => subscriber.reject(e).unwrap(),
         }
@@ -229,7 +251,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         _meta: Self::Metadata,
         subscriber: Subscriber<RpcResponse<RpcSignatureResult>>,
         signature_str: String,
-        confirmations: Option<Confirmations>,
+        commitment: Option<CommitmentConfig>,
     ) {
         info!("signature_subscribe");
         match param::<Signature>(&signature_str, "signature") {
@@ -240,12 +262,8 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                     "signature_subscribe: signature={:?} id={:?}",
                     signature, sub_id
                 );
-                self.subscriptions.add_signature_subscription(
-                    signature,
-                    confirmations,
-                    sub_id,
-                    subscriber,
-                );
+                self.subscriptions
+                    .add_signature_subscription(signature, commitment, sub_id, subscriber);
             }
             Err(e) => subscriber.reject(e).unwrap(),
         }
@@ -289,6 +307,27 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         }
     }
 
+    fn vote_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber<RpcVote>) {
+        info!("vote_subscribe");
+        let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
+        let sub_id = SubscriptionId::Number(id as u64);
+        info!("vote_subscribe: id={:?}", sub_id);
+        self.subscriptions.add_vote_subscription(sub_id, subscriber);
+    }
+
+    fn vote_unsubscribe(&self, _meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool> {
+        info!("vote_unsubscribe");
+        if self.subscriptions.remove_vote_subscription(&id) {
+            Ok(true)
+        } else {
+            Err(Error {
+                code: ErrorCode::InvalidParams,
+                message: "Invalid Request: Subscription id does not exist".into(),
+                data: None,
+            })
+        }
+    }
+
     fn root_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber<Slot>) {
         info!("root_subscribe");
         let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
@@ -315,24 +354,33 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
 mod tests {
     use super::*;
     use crate::{
-        commitment::{BlockCommitment, BlockCommitmentCache},
+        cluster_info_vote_listener::{ClusterInfoVoteListener, VoteTracker},
+        commitment::{BlockCommitmentCache, CacheSlotInfo},
         rpc_subscriptions::tests::robust_poll_or_panic,
     };
+    use crossbeam_channel::unbounded;
     use jsonrpc_core::{futures::sync::mpsc, Response};
     use jsonrpc_pubsub::{PubSubHandler, Session};
     use serial_test_derive::serial;
     use solana_budget_program::{self, budget_instruction};
-    use solana_ledger::bank_forks::BankForks;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use solana_runtime::bank::Bank;
+    use solana_ledger::{
+        bank_forks::BankForks,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+    };
+    use solana_runtime::{
+        bank::Bank,
+        genesis_utils::{create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs},
+    };
     use solana_sdk::{
+        hash::Hash,
         pubkey::Pubkey,
         signature::{Keypair, Signer},
         system_program, system_transaction,
         transaction::{self, Transaction},
     };
+    use solana_vote_program::vote_transaction;
     use std::{
-        collections::HashMap,
         sync::{atomic::AtomicBool, RwLock},
         thread::sleep,
         time::Duration,
@@ -342,14 +390,17 @@ mod tests {
         bank_forks: &Arc<RwLock<BankForks>>,
         tx: &Transaction,
         subscriptions: &RpcSubscriptions,
+        current_slot: Slot,
     ) -> transaction::Result<()> {
         bank_forks
             .write()
             .unwrap()
-            .get(0)
+            .get(current_slot)
             .unwrap()
             .process_transaction(tx)?;
-        subscriptions.notify_subscribers(0, &bank_forks);
+        let mut cache_slot_info = CacheSlotInfo::default();
+        cache_slot_info.current_slot = current_slot;
+        subscriptions.notify_subscribers(cache_slot_info);
         Ok(())
     }
 
@@ -370,12 +421,17 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let rpc = RpcSolPubSubImpl {
             subscriptions: Arc::new(RpcSubscriptions::new(
                 &Arc::new(AtomicBool::new(false)),
-                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+                bank_forks.clone(),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+                )),
             )),
-            ..RpcSolPubSubImpl::default()
+            uid: Arc::new(atomic::AtomicUsize::default()),
         };
 
         // Test signature subscriptions
@@ -385,7 +441,7 @@ mod tests {
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("signatureNotification");
         rpc.signature_subscribe(session, subscriber, tx.signatures[0].to_string(), None);
 
-        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions).unwrap();
+        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 0).unwrap();
 
         // Test signature confirmation notification
         let (response, _) = robust_poll_or_panic(receiver);
@@ -414,13 +470,15 @@ mod tests {
         } = create_genesis_config(10_000);
         let bob_pubkey = Pubkey::new_rand();
         let bank = Bank::new(&genesis_config);
-        let arc_bank = Arc::new(bank);
-        let blockhash = arc_bank.last_blockhash();
+        let blockhash = bank.last_blockhash();
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let session = create_session();
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
         io.extend_with(rpc.to_delegate());
 
         let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash);
@@ -475,13 +533,25 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank0 = bank_forks.read().unwrap().get(0).unwrap().clone();
+        let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
+        bank_forks.write().unwrap().insert(bank1);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let rpc = RpcSolPubSubImpl {
             subscriptions: Arc::new(RpcSubscriptions::new(
                 &Arc::new(AtomicBool::new(false)),
-                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+                bank_forks.clone(),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore_bank(
+                        blockstore,
+                        bank_forks.read().unwrap().get(1).unwrap().clone(),
+                        1,
+                    ),
+                )),
             )),
-            ..RpcSolPubSubImpl::default()
+            uid: Arc::new(atomic::AtomicUsize::default()),
         };
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
@@ -489,11 +559,11 @@ mod tests {
             session,
             subscriber,
             contract_state.pubkey().to_string(),
-            None,
+            Some(CommitmentConfig::recent()),
         );
 
         let tx = system_transaction::transfer(&alice, &contract_funds.pubkey(), 51, blockhash);
-        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions).unwrap();
+        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap();
 
         let ixs = budget_instruction::when_signed(
             &contract_funds.pubkey(),
@@ -508,14 +578,14 @@ mod tests {
             ixs,
             blockhash,
         );
-        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions).unwrap();
+        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap();
         sleep(Duration::from_millis(200));
 
         // Test signature confirmation notification #1
         let expected_data = bank_forks
             .read()
             .unwrap()
-            .get(0)
+            .get(1)
             .unwrap()
             .get_account(&contract_state.pubkey())
             .unwrap()
@@ -525,7 +595,7 @@ mod tests {
            "method": "accountNotification",
            "params": {
                "result": {
-                   "context": { "slot": 0 },
+                   "context": { "slot": 1 },
                    "value": {
                        "owner": budget_program_id.to_string(),
                        "lamports": 51,
@@ -542,7 +612,7 @@ mod tests {
         assert_eq!(serde_json::to_string(&expected).unwrap(), response);
 
         let tx = system_transaction::transfer(&alice, &witness.pubkey(), 1, blockhash);
-        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions).unwrap();
+        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap();
         sleep(Duration::from_millis(200));
         let ix = budget_instruction::apply_signature(
             &witness.pubkey(),
@@ -550,14 +620,14 @@ mod tests {
             &bob_pubkey,
         );
         let tx = Transaction::new_signed_instructions(&[&witness], vec![ix], blockhash);
-        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions).unwrap();
+        process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap();
         sleep(Duration::from_millis(200));
 
         assert_eq!(
             bank_forks
                 .read()
                 .unwrap()
-                .get(0)
+                .get(1)
                 .unwrap()
                 .get_account(&contract_state.pubkey()),
             None
@@ -569,9 +639,14 @@ mod tests {
     fn test_account_unsubscribe() {
         let bob_pubkey = Pubkey::new_rand();
         let session = create_session();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, Bank::new(&genesis_config))));
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc =
+            RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks.clone());
 
         io.extend_with(rpc.to_delegate());
 
@@ -606,7 +681,7 @@ mod tests {
 
     #[test]
     #[should_panic]
-    fn test_account_confirmations_not_fulfilled() {
+    fn test_account_commitment_not_fulfilled() {
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair: alice,
@@ -614,36 +689,50 @@ mod tests {
         } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(1, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let mut rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(
+            blockstore.clone(),
+            bank_forks.clone(),
+        );
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            bank_forks.clone(),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
-        rpc.account_subscribe(session, subscriber, bob.pubkey().to_string(), Some(2));
+        rpc.account_subscribe(
+            session,
+            subscriber,
+            bob.pubkey().to_string(),
+            Some(CommitmentConfig::root()),
+        );
 
         let tx = system_transaction::transfer(&alice, &bob.pubkey(), 100, blockhash);
         bank_forks
             .write()
             .unwrap()
-            .get(0)
+            .get(1)
             .unwrap()
             .process_transaction(&tx)
             .unwrap();
-        rpc.subscriptions.notify_subscribers(0, &bank_forks);
+        rpc.subscriptions
+            .notify_subscribers(CacheSlotInfo::default());
         // allow 200ms for notification thread to wake
         std::thread::sleep(Duration::from_millis(200));
         let _panic = robust_poll_or_panic(receiver);
     }
 
     #[test]
-    fn test_account_confirmations() {
+    fn test_account_commitment() {
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair: alice,
@@ -652,64 +741,59 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank0 = bank_forks.read().unwrap().get(0).unwrap().clone();
+        let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
+        bank_forks.write().unwrap().insert(bank1);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let mut rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(
+            blockstore.clone(),
+            bank_forks.clone(),
+        );
         let exit = Arc::new(AtomicBool::new(false));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests()));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_blockstore(blockstore.clone()),
+        ));
 
-        let subscriptions = RpcSubscriptions::new(&exit, block_commitment_cache.clone());
+        let subscriptions =
+            RpcSubscriptions::new(&exit, bank_forks.clone(), block_commitment_cache.clone());
         rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
-        rpc.account_subscribe(session, subscriber, bob.pubkey().to_string(), Some(2));
+        rpc.account_subscribe(
+            session,
+            subscriber,
+            bob.pubkey().to_string(),
+            Some(CommitmentConfig::root()),
+        );
 
         let tx = system_transaction::transfer(&alice, &bob.pubkey(), 100, blockhash);
         bank_forks
             .write()
             .unwrap()
-            .get(0)
+            .get(1)
             .unwrap()
             .process_transaction(&tx)
             .unwrap();
-        rpc.subscriptions.notify_subscribers(0, &bank_forks);
+        let mut cache_slot_info = CacheSlotInfo::default();
+        cache_slot_info.current_slot = 1;
+        rpc.subscriptions.notify_subscribers(cache_slot_info);
 
-        let bank0 = bank_forks.read().unwrap()[0].clone();
-        let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
-        bank_forks.write().unwrap().insert(bank1);
-        let bank1 = bank_forks.read().unwrap()[1].clone();
-
-        let mut cache0 = BlockCommitment::default();
-        cache0.increase_confirmation_stake(1, 10);
-        let mut block_commitment = HashMap::new();
-        block_commitment.entry(0).or_insert(cache0.clone());
-        let mut new_block_commitment =
-            BlockCommitmentCache::new(block_commitment, 10, bank1.clone(), 0);
-        let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
-        std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
-        drop(w_block_commitment_cache);
-
-        rpc.subscriptions.notify_subscribers(1, &bank_forks);
-        let bank2 = Bank::new_from_parent(&bank1, &Pubkey::default(), 2);
-        bank_forks.write().unwrap().insert(bank2);
-        let bank2 = bank_forks.read().unwrap()[2].clone();
-
-        let mut cache0 = BlockCommitment::default();
-        cache0.increase_confirmation_stake(2, 10);
-        let mut block_commitment = HashMap::new();
-        block_commitment.entry(0).or_insert(cache0.clone());
-        let mut new_block_commitment = BlockCommitmentCache::new(block_commitment, 10, bank2, 0);
-        let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
-        std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
-        drop(w_block_commitment_cache);
-
-        rpc.subscriptions.notify_subscribers(2, &bank_forks);
+        let cache_slot_info = CacheSlotInfo {
+            current_slot: 2,
+            node_root: 1,
+            largest_confirmed_root: 1,
+            highest_confirmed_slot: 1,
+        };
+        rpc.subscriptions.notify_subscribers(cache_slot_info);
         let expected = json!({
            "jsonrpc": "2.0",
            "method": "accountNotification",
            "params": {
                "result": {
-                   "context": { "slot": 0 },
+                   "context": { "slot": 1 },
                    "value": {
                        "owner": system_program::id().to_string(),
                        "lamports": 100,
@@ -728,7 +812,12 @@ mod tests {
     #[test]
     #[serial]
     fn test_slot_subscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);
@@ -753,7 +842,12 @@ mod tests {
     #[test]
     #[serial]
     fn test_slot_unsubscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);
@@ -782,4 +876,97 @@ mod tests {
             .slot_unsubscribe(Some(session), SubscriptionId::Number(0))
             .is_ok());
     }
+
+    #[test]
+    #[serial]
+    fn test_vote_subscribe() {
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_blockstore(blockstore.clone()),
+        ));
+
+        let validator_voting_keypairs: Vec<_> = (0..10)
+            .map(|_| ValidatorVoteKeypairs::new(Keypair::new(), Keypair::new(), Keypair::new()))
+            .collect();
+        let GenesisConfigInfo { genesis_config, .. } =
+            create_genesis_config_with_vote_accounts(10_000, &validator_voting_keypairs, 100);
+        let exit = Arc::new(AtomicBool::new(false));
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = BankForks::new(0, bank);
+        let bank = bank_forks.get(0).unwrap().clone();
+        let bank_forks = Arc::new(RwLock::new(bank_forks));
+
+        // Setup RPC
+        let mut rpc =
+            RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks.clone());
+        let session = create_session();
+        let (subscriber, _id_receiver, receiver) = Subscriber::new_test("voteNotification");
+
+        // Setup Subscriptions
+        let subscriptions =
+            RpcSubscriptions::new(&exit, bank_forks.clone(), block_commitment_cache.clone());
+        rpc.subscriptions = Arc::new(subscriptions);
+        rpc.vote_subscribe(session, subscriber);
+
+        // Create some voters at genesis
+        let vote_tracker = VoteTracker::new(&bank);
+        let (votes_sender, votes_receiver) = unbounded();
+        let (vote_tracker, validator_voting_keypairs) =
+            (Arc::new(vote_tracker), validator_voting_keypairs);
+
+        let vote_slots = vec![1, 2];
+        validator_voting_keypairs.iter().for_each(|keypairs| {
+            let node_keypair = &keypairs.node_keypair;
+            let vote_keypair = &keypairs.vote_keypair;
+            let vote_tx = vote_transaction::new_vote_transaction(
+                vote_slots.clone(),
+                Hash::default(),
+                Hash::default(),
+                node_keypair,
+                vote_keypair,
+                vote_keypair,
+            );
+            votes_sender.send(vec![vote_tx]).unwrap();
+        });
+
+        // Process votes and check they were notified.
+        ClusterInfoVoteListener::get_and_process_votes_for_tests(
+            &votes_receiver,
+            &vote_tracker,
+            0,
+            rpc.subscriptions.clone(),
+        )
+        .unwrap();
+
+        let (response, _) = robust_poll_or_panic(receiver);
+        assert_eq!(
+            response,
+            r#"{"jsonrpc":"2.0","method":"voteNotification","params":{"result":{"hash":"11111111111111111111111111111111","slots":[1,2],"timestamp":null},"subscription":0}}"#
+        );
+    }
+
+    #[test]
+    #[serial]
+    fn test_vote_unsubscribe() {
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
+        let session = create_session();
+        let (subscriber, _id_receiver, _) = Subscriber::new_test("voteNotification");
+        rpc.vote_subscribe(session, subscriber);
+
+        let session = create_session();
+        assert!(rpc
+            .vote_unsubscribe(Some(session), SubscriptionId::Number(42))
+            .is_err());
+
+        let session = create_session();
+        assert!(rpc
+            .vote_unsubscribe(Some(session), SubscriptionId::Number(0))
+            .is_ok());
+    }
 }

core/src/rpc_pubsub_service.rs

@@ -73,6 +73,13 @@ impl PubSubService {
 mod tests {
     use super::*;
     use crate::commitment::BlockCommitmentCache;
+    use solana_ledger::{
+        bank_forks::BankForks,
+        blockstore::Blockstore,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+    };
+    use solana_runtime::bank::Bank;
     use std::{
         net::{IpAddr, Ipv4Addr},
         sync::RwLock,
@@ -82,9 +89,17 @@ mod tests {
     fn test_pubsub_new() {
         let pubsub_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
         let exit = Arc::new(AtomicBool::new(false));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
         let subscriptions = Arc::new(RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            bank_forks,
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         ));
         let pubsub_service = PubSubService::new(&subscriptions, pubsub_addr, &exit);
         let thread = pubsub_service.thread_hdl.thread();

core/src/rpc_service.rs

@@ -1,8 +1,9 @@
 //! The `rpc_service` module implements the Solana JSON RPC service.
 
 use crate::{
-    cluster_info::ClusterInfo, commitment::BlockCommitmentCache, rpc::*,
-    storage_stage::StorageState, validator::ValidatorExit,
+    cluster_info::ClusterInfo, commitment::BlockCommitmentCache, rpc::*, rpc_health::*,
+    send_transaction_service::SendTransactionService, storage_stage::StorageState,
+    validator::ValidatorExit,
 };
 use jsonrpc_core::MetaIoHandler;
 use jsonrpc_http_server::{
@@ -15,25 +16,22 @@ use solana_ledger::{
     blockstore::Blockstore,
     snapshot_utils,
 };
-use solana_sdk::{hash::Hash, pubkey::Pubkey};
+use solana_sdk::{hash::Hash, native_token::lamports_to_sol, pubkey::Pubkey};
 use std::{
     collections::HashSet,
     net::SocketAddr,
     path::{Path, PathBuf},
+    sync::atomic::{AtomicBool, Ordering},
     sync::{mpsc::channel, Arc, RwLock},
     thread::{self, Builder, JoinHandle},
 };
 use tokio::prelude::Future;
 
-// If trusted validators are specified, consider this validator healthy if its latest account hash
-// is no further behind than this distance from the latest trusted validator account hash
-const HEALTH_CHECK_SLOT_DISTANCE: u64 = 150;
-
 pub struct JsonRpcService {
     thread_hdl: JoinHandle<()>,
 
     #[cfg(test)]
-    pub request_processor: Arc<RwLock<JsonRpcRequestProcessor>>, // Used only by test_rpc_new()...
+    pub request_processor: JsonRpcRequestProcessor, // Used only by test_rpc_new()...
 
     close_handle: Option<CloseHandle>,
 }
@@ -42,24 +40,24 @@ struct RpcRequestMiddleware {
     ledger_path: PathBuf,
     snapshot_archive_path_regex: Regex,
     snapshot_config: Option<SnapshotConfig>,
-    cluster_info: Arc<RwLock<ClusterInfo>>,
-    trusted_validators: Option<HashSet<Pubkey>>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    health: Arc<RpcHealth>,
 }
 
 impl RpcRequestMiddleware {
     pub fn new(
         ledger_path: PathBuf,
         snapshot_config: Option<SnapshotConfig>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
-        trusted_validators: Option<HashSet<Pubkey>>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        health: Arc<RpcHealth>,
     ) -> Self {
         Self {
             ledger_path,
             snapshot_archive_path_regex: Regex::new(r"/snapshot-\d+-[[:alnum:]]+\.tar\.bz2$")
                 .unwrap(),
             snapshot_config,
-            cluster_info,
-            trusted_validators,
+            bank_forks,
+            health,
         }
     }
 
@@ -85,7 +83,7 @@ impl RpcRequestMiddleware {
             .unwrap()
     }
 
-    fn is_get_path(&self, path: &str) -> bool {
+    fn is_file_get_path(&self, path: &str) -> bool {
         match path {
             "/genesis.tar.bz2" => true,
             _ => {
@@ -98,7 +96,7 @@ impl RpcRequestMiddleware {
         }
     }
 
-    fn get(&self, path: &str) -> RequestMiddlewareAction {
+    fn process_file_get(&self, path: &str) -> RequestMiddlewareAction {
         let stem = path.split_at(1).1; // Drop leading '/' from path
         let filename = {
             match path {
@@ -130,53 +128,10 @@ impl RpcRequestMiddleware {
     }
 
     fn health_check(&self) -> &'static str {
-        let response = if let Some(trusted_validators) = &self.trusted_validators {
-            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
-                let cluster_info = self.cluster_info.read().unwrap();
-                (
-                    cluster_info
-                        .get_accounts_hash_for_node(&cluster_info.id())
-                        .map(|hashes| hashes.iter().max_by(|a, b| a.0.cmp(&b.0)))
-                        .flatten()
-                        .map(|slot_hash| slot_hash.0)
-                        .unwrap_or(0),
-                    trusted_validators
-                        .iter()
-                        .map(|trusted_validator| {
-                            cluster_info
-                                .get_accounts_hash_for_node(&trusted_validator)
-                                .map(|hashes| hashes.iter().max_by(|a, b| a.0.cmp(&b.0)))
-                                .flatten()
-                                .map(|slot_hash| slot_hash.0)
-                                .unwrap_or(0)
-                        })
-                        .max()
-                        .unwrap_or(0),
-                )
-            };
-
-            // This validator is considered healthy if its latest account hash slot is within
-            // `HEALTH_CHECK_SLOT_DISTANCE` of the latest trusted validator's account hash slot
-            if latest_account_hash_slot > 0
-                && latest_trusted_validator_account_hash_slot > 0
-                && latest_account_hash_slot
-                    > latest_trusted_validator_account_hash_slot
-                        .saturating_sub(HEALTH_CHECK_SLOT_DISTANCE)
-            {
-                "ok"
-            } else {
-                warn!(
-                    "health check: me={}, latest trusted_validator={}",
-                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
-                );
-                "behind"
-            }
-        } else {
-            // No trusted validator point of reference available, so this validator is healthy
-            // because it's running
-            "ok"
+        let response = match self.health.check() {
+            RpcHealthStatus::Ok => "ok",
+            RpcHealthStatus::Behind => "behind",
         };
-
         info!("health check: {}", response);
         response
     }
@@ -212,8 +167,19 @@ impl RequestMiddleware for RpcRequestMiddleware {
                 };
             }
         }
-        if self.is_get_path(request.uri().path()) {
-            self.get(request.uri().path())
+
+        if let Some(result) = process_rest(&self.bank_forks, request.uri().path()) {
+            RequestMiddlewareAction::Respond {
+                should_validate_hosts: true,
+                response: Box::new(jsonrpc_core::futures::future::ok(
+                    hyper::Response::builder()
+                        .status(hyper::StatusCode::OK)
+                        .body(hyper::Body::from(result))
+                        .unwrap(),
+                )),
+            }
+        } else if self.is_file_get_path(request.uri().path()) {
+            self.process_file_get(request.uri().path())
         } else if request.uri().path() == "/health" {
             RequestMiddlewareAction::Respond {
                 should_validate_hosts: true,
@@ -233,6 +199,29 @@ impl RequestMiddleware for RpcRequestMiddleware {
     }
 }
 
+fn process_rest(bank_forks: &Arc<RwLock<BankForks>>, path: &str) -> Option<String> {
+    match path {
+        "/v0/circulating-supply" => {
+            let r_bank_forks = bank_forks.read().unwrap();
+            let bank = r_bank_forks.root_bank();
+            let total_supply = bank.capitalization();
+            let non_circulating_supply =
+                crate::non_circulating_supply::calculate_non_circulating_supply(&bank).lamports;
+            Some(format!(
+                "{}",
+                lamports_to_sol(total_supply - non_circulating_supply)
+            ))
+        }
+        "/v0/total-supply" => {
+            let r_bank_forks = bank_forks.read().unwrap();
+            let bank = r_bank_forks.root_bank();
+            let total_supply = bank.capitalization();
+            Some(format!("{}", lamports_to_sol(total_supply)))
+        }
+        _ => None,
+    }
+}
+
 impl JsonRpcService {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
@@ -242,23 +231,43 @@ impl JsonRpcService {
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         blockstore: Arc<Blockstore>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
+        cluster_info: Arc<ClusterInfo>,
         genesis_hash: Hash,
         ledger_path: &Path,
         storage_state: StorageState,
         validator_exit: Arc<RwLock<Option<ValidatorExit>>>,
         trusted_validators: Option<HashSet<Pubkey>>,
+        override_health_check: Arc<AtomicBool>,
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
-        let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new(
+
+        let health = Arc::new(RpcHealth::new(
+            cluster_info.clone(),
+            trusted_validators,
+            config.health_check_slot_distance,
+            override_health_check,
+        ));
+
+        let exit_send_transaction_service = Arc::new(AtomicBool::new(false));
+        let send_transaction_service = Arc::new(SendTransactionService::new(
+            &cluster_info,
+            &bank_forks,
+            &exit_send_transaction_service,
+        ));
+
+        let request_processor = JsonRpcRequestProcessor::new(
             config,
-            bank_forks,
+            bank_forks.clone(),
             block_commitment_cache,
             blockstore,
             storage_state,
             validator_exit.clone(),
-        )));
+            health.clone(),
+            cluster_info,
+            genesis_hash,
+            send_transaction_service,
+        );
 
         #[cfg(test)]
         let test_request_processor = request_processor.clone();
@@ -276,16 +285,12 @@ impl JsonRpcService {
                 let request_middleware = RpcRequestMiddleware::new(
                     ledger_path,
                     snapshot_config,
-                    cluster_info.clone(),
-                    trusted_validators,
+                    bank_forks.clone(),
+                    health.clone(),
                 );
                 let server = ServerBuilder::with_meta_extractor(
                     io,
-                    move |_req: &hyper::Request<hyper::Body>| Meta {
-                        request_processor: request_processor.clone(),
-                        cluster_info: cluster_info.clone(),
-                        genesis_hash,
-                    },
+                    move |_req: &hyper::Request<hyper::Body>| request_processor.clone(),
                 )
                 .threads(num_cpus::get())
                 .cors(DomainsValidation::AllowOnly(vec![
@@ -308,6 +313,7 @@ impl JsonRpcService {
                 let server = server.unwrap();
                 close_handle_sender.send(server.close_handle()).unwrap();
                 server.wait();
+                exit_send_transaction_service.store(true, Ordering::Relaxed);
             })
             .unwrap();
 
@@ -341,7 +347,6 @@ impl JsonRpcService {
 mod tests {
     use super::*;
     use crate::{
-        contact_info::ContactInfo,
         crds_value::{CrdsData, CrdsValue, SnapshotHash},
         rpc::tests::create_validator_exit,
     };
@@ -351,8 +356,7 @@ mod tests {
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::signature::Signer;
-    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
-    use std::sync::atomic::AtomicBool;
+    use std::net::{IpAddr, Ipv4Addr};
 
     #[test]
     fn test_rpc_new() {
@@ -364,31 +368,32 @@ mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let validator_exit = create_validator_exit(&exit);
         let bank = Bank::new(&genesis_config);
-        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
-            ContactInfo::default(),
-        )));
+        let cluster_info = Arc::new(ClusterInfo::default());
         let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
         let rpc_addr = SocketAddr::new(
             ip_addr,
             solana_net_utils::find_available_port_in_range(ip_addr, (10000, 65535)).unwrap(),
         );
         let bank_forks = Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let ledger_path = get_tmp_ledger_path!();
-        let blockstore = Blockstore::open(&ledger_path).unwrap();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let mut rpc_service = JsonRpcService::new(
             rpc_addr,
             JsonRpcConfig::default(),
             None,
             bank_forks,
             block_commitment_cache,
-            Arc::new(blockstore),
+            blockstore,
             cluster_info,
             Hash::default(),
             &PathBuf::from("farf"),
             StorageState::default(),
             validator_exit,
             None,
+            Arc::new(AtomicBool::new(false)),
         );
         let thread = rpc_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
@@ -397,8 +402,6 @@ mod tests {
             10_000,
             rpc_service
                 .request_processor
-                .read()
-                .unwrap()
                 .get_balance(Ok(mint_keypair.pubkey()), None)
                 .unwrap()
                 .value
@@ -407,13 +410,36 @@ mod tests {
         rpc_service.join().unwrap();
     }
 
-    #[test]
-    fn test_is_get_path() {
-        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
-            ContactInfo::default(),
-        )));
+    fn create_bank_forks() -> Arc<RwLock<BankForks>> {
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+        let bank = Bank::new(&genesis_config);
+        Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)))
+    }
 
-        let rrm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
+    #[test]
+    fn test_process_rest_api() {
+        let bank_forks = create_bank_forks();
+
+        assert_eq!(None, process_rest(&bank_forks, "not-a-supported-rest-api"));
+        assert_eq!(
+            Some("0.000010127".to_string()),
+            process_rest(&bank_forks, "/v0/circulating-supply")
+        );
+        assert_eq!(
+            Some("0.000010127".to_string()),
+            process_rest(&bank_forks, "/v0/total-supply")
+        );
+    }
+
+    #[test]
+    fn test_is_file_get_path() {
+        let bank_forks = create_bank_forks();
+        let rrm = RpcRequestMiddleware::new(
+            PathBuf::from("/"),
+            None,
+            bank_forks.clone(),
+            RpcHealth::stub(),
+        );
         let rrm_with_snapshot_config = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             Some(SnapshotConfig {
@@ -421,118 +447,118 @@ mod tests {
                 snapshot_package_output_path: PathBuf::from("/"),
                 snapshot_path: PathBuf::from("/"),
             }),
-            cluster_info,
-            None,
+            bank_forks,
+            RpcHealth::stub(),
         );
 
-        assert!(rrm.is_get_path("/genesis.tar.bz2"));
-        assert!(!rrm.is_get_path("genesis.tar.bz2"));
+        assert!(rrm.is_file_get_path("/genesis.tar.bz2"));
+        assert!(!rrm.is_file_get_path("genesis.tar.bz2"));
 
-        assert!(!rrm.is_get_path("/snapshot.tar.bz2")); // This is a redirect
+        assert!(!rrm.is_file_get_path("/snapshot.tar.bz2")); // This is a redirect
 
-        assert!(
-            !rrm.is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2")
-        );
-        assert!(rrm_with_snapshot_config
-            .is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"));
+        assert!(!rrm.is_file_get_path(
+            "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
+        ));
+        assert!(rrm_with_snapshot_config.is_file_get_path(
+            "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
+        ));
 
-        assert!(!rrm.is_get_path(
+        assert!(!rrm.is_file_get_path(
             "/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
         ));
 
-        assert!(!rrm.is_get_path("/"));
-        assert!(!rrm.is_get_path(".."));
-        assert!(!rrm.is_get_path("🎣"));
+        assert!(!rrm.is_file_get_path("/"));
+        assert!(!rrm.is_file_get_path(".."));
+        assert!(!rrm.is_file_get_path("🎣"));
     }
 
     #[test]
     fn test_health_check_with_no_trusted_validators() {
-        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
-            ContactInfo::default(),
-        )));
-
-        let rm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
+        let rm = RpcRequestMiddleware::new(
+            PathBuf::from("/"),
+            None,
+            create_bank_forks(),
+            RpcHealth::stub(),
+        );
         assert_eq!(rm.health_check(), "ok");
     }
 
     #[test]
     fn test_health_check_with_trusted_validators() {
-        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
-            ContactInfo::default(),
-        )));
-
+        let cluster_info = Arc::new(ClusterInfo::default());
+        let health_check_slot_distance = 123;
+        let override_health_check = Arc::new(AtomicBool::new(false));
         let trusted_validators = vec![Pubkey::new_rand(), Pubkey::new_rand(), Pubkey::new_rand()];
-        let rm = RpcRequestMiddleware::new(
-            PathBuf::from("/"),
-            None,
+
+        let health = Arc::new(RpcHealth::new(
             cluster_info.clone(),
             Some(trusted_validators.clone().into_iter().collect()),
-        );
+            health_check_slot_distance,
+            override_health_check.clone(),
+        ));
+
+        let rm = RpcRequestMiddleware::new(PathBuf::from("/"), None, create_bank_forks(), health);
 
         // No account hashes for this node or any trusted validators == "behind"
         assert_eq!(rm.health_check(), "behind");
 
         // No account hashes for any trusted validators == "behind"
-        {
-            let mut cluster_info = cluster_info.write().unwrap();
-            cluster_info
-                .push_accounts_hashes(vec![(1000, Hash::default()), (900, Hash::default())]);
-        }
+        cluster_info.push_accounts_hashes(vec![(1000, Hash::default()), (900, Hash::default())]);
         assert_eq!(rm.health_check(), "behind");
+        override_health_check.store(true, Ordering::Relaxed);
+        assert_eq!(rm.health_check(), "ok");
+        override_health_check.store(false, Ordering::Relaxed);
 
         // This node is ahead of the trusted validators == "ok"
-        {
-            let mut cluster_info = cluster_info.write().unwrap();
-            cluster_info
-                .gossip
-                .crds
-                .insert(
-                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
-                        trusted_validators[0].clone(),
-                        vec![
-                            (1, Hash::default()),
-                            (1001, Hash::default()),
-                            (2, Hash::default()),
-                        ],
-                    ))),
-                    1,
-                )
-                .unwrap();
-        }
+        cluster_info
+            .gossip
+            .write()
+            .unwrap()
+            .crds
+            .insert(
+                CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                    trusted_validators[0].clone(),
+                    vec![
+                        (1, Hash::default()),
+                        (1001, Hash::default()),
+                        (2, Hash::default()),
+                    ],
+                ))),
+                1,
+            )
+            .unwrap();
         assert_eq!(rm.health_check(), "ok");
 
         // Node is slightly behind the trusted validators == "ok"
-        {
-            let mut cluster_info = cluster_info.write().unwrap();
-            cluster_info
-                .gossip
-                .crds
-                .insert(
-                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
-                        trusted_validators[1].clone(),
-                        vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE - 1, Hash::default())],
-                    ))),
-                    1,
-                )
-                .unwrap();
-        }
+        cluster_info
+            .gossip
+            .write()
+            .unwrap()
+            .crds
+            .insert(
+                CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                    trusted_validators[1].clone(),
+                    vec![(1000 + health_check_slot_distance - 1, Hash::default())],
+                ))),
+                1,
+            )
+            .unwrap();
         assert_eq!(rm.health_check(), "ok");
 
         // Node is far behind the trusted validators == "behind"
-        {
-            let mut cluster_info = cluster_info.write().unwrap();
-            cluster_info
-                .gossip
-                .crds
-                .insert(
-                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
-                        trusted_validators[2].clone(),
-                        vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE, Hash::default())],
-                    ))),
-                    1,
-                )
-                .unwrap();
-        }
+        cluster_info
+            .gossip
+            .write()
+            .unwrap()
+            .crds
+            .insert(
+                CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                    trusted_validators[2].clone(),
+                    vec![(1000 + health_check_slot_distance, Hash::default())],
+                ))),
+                1,
+            )
+            .unwrap();
         assert_eq!(rm.health_check(), "behind");
     }
 }