chore: conflate use statement

(cherry picked from commit 6d11d5dd9f)

.buildkite/env/secrets.ejson

@@ -2,6 +2,6 @@
     "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
     "_comment": "These credentials are encrypted and pose no risk",
     "environment": {
-      "CODECOV_TOKEN": "EJ[1:Z7OneT3RdJJ0DipCHQ7rC84snQ+FPbgHwZADQiz54wk=:3K68mE38LJ2RB98VWmjuNLFBNn1XTGR4:cR4r05/TOZQKmEZp1v4CSgUJtC6QJiOaL85QjXW0qZ061fMnsBA8AtAPMDoDq4WCGOZM1A==]"
+      "CODECOV_TOKEN": "EJ[1:KToenD1Sr3w82lHGxz1n+j3hwNlLk/1pYrjZHlvY6kE=:hN1Q25omtJ+4yYVn+qzIsPLKT3O6J9XN:DMLNLXi/pkWgvwF6gNIcNF222sgsRR9LnwLZYj0P0wGj7q6w8YQnd1Rskj+sRroI/z5pQg==]"
     }
 }

Cargo.toml

@@ -77,3 +77,6 @@ members = [
 exclude = [
     "programs/bpf",
 ]
+
+[profile.dev]
+split-debuginfo = "unpacked"

SECURITY.md

@@ -42,6 +42,14 @@ RPC DoS/Crashes:
 $5,000 USD in locked SOL tokens (locked for 12 months)
 * RPC attacks
 
+Out of Scope:
+The following components are out of scope for the bounty program
+* Metrics: `/metrics` in the monorepo as well as https://metrics.solana.com
+* Explorer: `/explorer` in the monorepo as well as https://explorer.solana.com
+* Any encrypted credentials, auth tokens, etc. checked into the repo
+* Bugs in dependencies. Please take them upstream!
+* Attacks that require social engineering
+
 Eligibility:
 * The participant submitting the bug bounty shall follow the process outlined within this document
 * Valid exploits can be eligible even if they are not successfully executed on the cluster

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.6.0"
+version = "1.6.7"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,10 +19,10 @@ lazy_static = "1.4.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.6.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.6.0" }
+solana-config-program = { path = "../programs/config", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.7" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.7" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"

account-decoder/src/parse_nonce.rs

@@ -9,7 +9,13 @@ pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
         .map_err(|_| ParseAccountError::from(InstructionError::InvalidAccountData))?;
     let nonce_state = nonce_state.convert_to_current();
     match nonce_state {
-        State::Uninitialized => Ok(UiNonceState::Uninitialized),
+        // This prevents parsing an allocated System-owned account with empty data of any non-zero
+        // length as `uninitialized` nonce. An empty account of the wrong length can never be
+        // initialized as a nonce account, and an empty account of the correct length may not be an
+        // uninitialized nonce account, since it can be assigned to another program.
+        State::Uninitialized => Err(ParseAccountError::from(
+            InstructionError::InvalidAccountData,
+        )),
         State::Initialized(data) => Ok(UiNonceState::Initialized(UiNonceData {
             authority: data.authority.to_string(),
             blockhash: data.blockhash.to_string(),

account-decoder/src/parse_sysvar.rs

@@ -214,13 +214,13 @@ pub struct UiStakeHistoryEntry {
 mod test {
     use super::*;
     use solana_sdk::{
-        account::create_account, fee_calculator::FeeCalculator, hash::Hash,
+        account::create_account_for_test, fee_calculator::FeeCalculator, hash::Hash,
         sysvar::recent_blockhashes::IterItem,
     };
 
     #[test]
     fn test_parse_sysvars() {
-        let clock_sysvar = create_account(&Clock::default(), 1);
+        let clock_sysvar = create_account_for_test(&Clock::default());
         assert_eq!(
             parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
             SysvarAccountType::Clock(UiClock::default()),
@@ -233,13 +233,13 @@ mod test {
             first_normal_epoch: 1,
             first_normal_slot: 12,
         };
-        let epoch_schedule_sysvar = create_account(&epoch_schedule, 1);
+        let epoch_schedule_sysvar = create_account_for_test(&epoch_schedule);
         assert_eq!(
             parse_sysvar(&epoch_schedule_sysvar.data, &sysvar::epoch_schedule::id()).unwrap(),
             SysvarAccountType::EpochSchedule(epoch_schedule),
         );
 
-        let fees_sysvar = create_account(&Fees::default(), 1);
+        let fees_sysvar = create_account_for_test(&Fees::default());
         assert_eq!(
             parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
             SysvarAccountType::Fees(UiFees::default()),
@@ -252,7 +252,7 @@ mod test {
         let recent_blockhashes: RecentBlockhashes = vec![IterItem(0, &hash, &fee_calculator)]
             .into_iter()
             .collect();
-        let recent_blockhashes_sysvar = create_account(&recent_blockhashes, 1);
+        let recent_blockhashes_sysvar = create_account_for_test(&recent_blockhashes);
         assert_eq!(
             parse_sysvar(
                 &recent_blockhashes_sysvar.data,
@@ -270,13 +270,13 @@ mod test {
             exemption_threshold: 2.0,
             burn_percent: 5,
         };
-        let rent_sysvar = create_account(&rent, 1);
+        let rent_sysvar = create_account_for_test(&rent);
         assert_eq!(
             parse_sysvar(&rent_sysvar.data, &sysvar::rent::id()).unwrap(),
             SysvarAccountType::Rent(rent.into()),
         );
 
-        let rewards_sysvar = create_account(&Rewards::default(), 1);
+        let rewards_sysvar = create_account_for_test(&Rewards::default());
         assert_eq!(
             parse_sysvar(&rewards_sysvar.data, &sysvar::rewards::id()).unwrap(),
             SysvarAccountType::Rewards(UiRewards::default()),
@@ -284,7 +284,7 @@ mod test {
 
         let mut slot_hashes = SlotHashes::default();
         slot_hashes.add(1, hash);
-        let slot_hashes_sysvar = create_account(&slot_hashes, 1);
+        let slot_hashes_sysvar = create_account_for_test(&slot_hashes);
         assert_eq!(
             parse_sysvar(&slot_hashes_sysvar.data, &sysvar::slot_hashes::id()).unwrap(),
             SysvarAccountType::SlotHashes(vec![UiSlotHashEntry {
@@ -295,7 +295,7 @@ mod test {
 
         let mut slot_history = SlotHistory::default();
         slot_history.add(42);
-        let slot_history_sysvar = create_account(&slot_history, 1);
+        let slot_history_sysvar = create_account_for_test(&slot_history);
         assert_eq!(
             parse_sysvar(&slot_history_sysvar.data, &sysvar::slot_history::id()).unwrap(),
             SysvarAccountType::SlotHistory(UiSlotHistory {
@@ -311,7 +311,7 @@ mod test {
             deactivating: 3,
         };
         stake_history.add(1, stake_history_entry.clone());
-        let stake_history_sysvar = create_account(&stake_history, 1);
+        let stake_history_sysvar = create_account_for_test(&stake_history);
         assert_eq!(
             parse_sysvar(&stake_history_sysvar.data, &sysvar::stake_history::id()).unwrap(),
             SysvarAccountType::StakeHistory(vec![UiStakeHistoryEntry {

account-decoder/src/parse_token.rs

@@ -172,10 +172,12 @@ pub fn real_number_string(amount: u64, decimals: u8) -> StringDecimals {
 }
 
 pub fn real_number_string_trimmed(amount: u64, decimals: u8) -> StringDecimals {
-    let s = real_number_string(amount, decimals);
-    let zeros_trimmed = s.trim_end_matches('0');
-    let decimal_trimmed = zeros_trimmed.trim_end_matches('.');
-    decimal_trimmed.to_string()
+    let mut s = real_number_string(amount, decimals);
+    if decimals > 0 {
+        let zeros_trimmed = s.trim_end_matches('0');
+        s = zeros_trimmed.trim_end_matches('.').to_string();
+    }
+    s
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -363,6 +365,14 @@ mod test {
             real_number_string_trimmed(1, 0)
         );
         assert_eq!(token_amount.ui_amount, Some(1.0));
+        assert_eq!(&real_number_string(10, 0), "10");
+        assert_eq!(&real_number_string_trimmed(10, 0), "10");
+        let token_amount = token_amount_to_ui_amount(10, 0);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(10, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(10.0));
         assert_eq!(&real_number_string(1, 9), "0.000000001");
         assert_eq!(&real_number_string_trimmed(1, 9), "0.000000001");
         let token_amount = token_amount_to_ui_amount(1, 9);
@@ -402,4 +412,32 @@ mod test {
         );
         assert_eq!(token_amount.ui_amount, None);
     }
+
+    #[test]
+    fn test_ui_token_amount_real_string_zero() {
+        assert_eq!(&real_number_string(0, 0), "0");
+        assert_eq!(&real_number_string_trimmed(0, 0), "0");
+        let token_amount = token_amount_to_ui_amount(0, 0);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.0));
+        assert_eq!(&real_number_string(0, 9), "0.000000000");
+        assert_eq!(&real_number_string_trimmed(0, 9), "0");
+        let token_amount = token_amount_to_ui_amount(0, 9);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 9)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.0));
+        assert_eq!(&real_number_string(0, 25), "0.0000000000000000000000000");
+        assert_eq!(&real_number_string_trimmed(0, 25), "0");
+        let token_amount = token_amount_to_ui_amount(0, 20);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 20)
+        );
+        assert_eq!(token_amount.ui_amount, None);
+    }
 }

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-measure = { path = "../measure", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-measure = { path = "../measure", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-cluster-bench/.gitignore

@@ -0,0 +1 @@
+/farf/

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-cluster-bench"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,22 +13,22 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.1"
-solana-account-decoder = { path = "../account-decoder", version = "1.6.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-client = { path = "../client", version = "1.6.0" }
-solana-core = { path = "../core", version = "1.6.0" }
-solana-measure = { path = "../measure", version = "1.6.0" }
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-net-utils = { path = "../net-utils", version = "1.6.0" }
-solana-faucet = { path = "../faucet", version = "1.6.0" }
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.7" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-client = { path = "../client", version = "=1.6.7" }
+solana-core = { path = "../core", version = "=1.6.7" }
+solana-measure = { path = "../measure", version = "=1.6.7" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.7" }
+solana-faucet = { path = "../faucet", version = "=1.6.7" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.6.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.7" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/src/main.rs

@@ -1,5 +1,5 @@
 #![allow(clippy::integer_arithmetic)]
-use clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg};
+use clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg};
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
@@ -21,7 +21,6 @@ use solana_sdk::{
     transaction::Transaction,
 };
 use solana_transaction_status::parse_token::spl_token_v2_0_instruction;
-use spl_token_v2_0::solana_program::pubkey::Pubkey as SplPubkey;
 use std::{
     net::SocketAddr,
     process::exit,
@@ -33,6 +32,10 @@ use std::{
     time::{Duration, Instant},
 };
 
+// Create and close messages both require 2 signatures; if transaction construction changes, update
+// this magic number
+const NUM_SIGNATURES: u64 = 2;
+
 pub fn airdrop_lamports(
     client: &RpcClient,
     faucet_addr: &SocketAddr,
@@ -186,14 +189,13 @@ impl TransactionExecutor {
                         let mut start = Measure::start("sig_status");
                         let statuses: Vec<_> = sigs_w
                             .chunks(200)
-                            .map(|sig_chunk| {
+                            .flat_map(|sig_chunk| {
                                 let only_sigs: Vec<_> = sig_chunk.iter().map(|s| s.0).collect();
                                 client
                                     .get_signature_statuses(&only_sigs)
                                     .expect("status fail")
                                     .value
                             })
-                            .flatten()
                             .collect();
                         let mut num_cleared = 0;
                         let start_len = sigs_w.len();
@@ -251,28 +253,38 @@ impl TransactionExecutor {
     }
 }
 
-fn make_message(
+struct SeedTracker {
+    max_created: Arc<AtomicU64>,
+    max_closed: Arc<AtomicU64>,
+}
+
+fn make_create_message(
     keypair: &Keypair,
+    base_keypair: &Keypair,
+    max_created_seed: Arc<AtomicU64>,
     num_instructions: usize,
     balance: u64,
     maybe_space: Option<u64>,
     mint: Option<Pubkey>,
-) -> (Message, Vec<Keypair>) {
+) -> Message {
     let space = maybe_space.unwrap_or_else(|| thread_rng().gen_range(0, 1000));
 
-    let (instructions, new_keypairs): (Vec<_>, Vec<_>) = (0..num_instructions)
+    let instructions: Vec<_> = (0..num_instructions)
         .into_iter()
         .map(|_| {
-            let new_keypair = Keypair::new();
-
             let program_id = if mint.is_some() {
                 inline_spl_token_v2_0::id()
             } else {
                 system_program::id()
             };
-            let mut instructions = vec![system_instruction::create_account(
+            let seed = max_created_seed.fetch_add(1, Ordering::Relaxed).to_string();
+            let to_pubkey =
+                Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
+            let mut instructions = vec![system_instruction::create_account_with_seed(
                 &keypair.pubkey(),
-                &new_keypair.pubkey(),
+                &to_pubkey,
+                &base_keypair.pubkey(),
+                &seed,
                 balance,
                 space,
                 &program_id,
@@ -281,32 +293,77 @@ fn make_message(
                 instructions.push(spl_token_v2_0_instruction(
                     spl_token_v2_0::instruction::initialize_account(
                         &spl_token_v2_0::id(),
-                        &spl_token_v2_0_pubkey(&new_keypair.pubkey()),
+                        &spl_token_v2_0_pubkey(&to_pubkey),
                         &spl_token_v2_0_pubkey(&mint_address),
-                        &SplPubkey::new_unique(),
+                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
                     )
                     .unwrap(),
                 ));
             }
 
-            (instructions, new_keypair)
+            instructions
         })
-        .unzip();
+        .collect();
     let instructions: Vec<_> = instructions.into_iter().flatten().collect();
 
-    (
-        Message::new(&instructions, Some(&keypair.pubkey())),
-        new_keypairs,
-    )
+    Message::new(&instructions, Some(&keypair.pubkey()))
 }
 
+fn make_close_message(
+    keypair: &Keypair,
+    base_keypair: &Keypair,
+    max_closed_seed: Arc<AtomicU64>,
+    num_instructions: usize,
+    balance: u64,
+    spl_token: bool,
+) -> Message {
+    let instructions: Vec<_> = (0..num_instructions)
+        .into_iter()
+        .map(|_| {
+            let program_id = if spl_token {
+                inline_spl_token_v2_0::id()
+            } else {
+                system_program::id()
+            };
+            let seed = max_closed_seed.fetch_add(1, Ordering::Relaxed).to_string();
+            let address =
+                Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
+            if spl_token {
+                spl_token_v2_0_instruction(
+                    spl_token_v2_0::instruction::close_account(
+                        &spl_token_v2_0::id(),
+                        &spl_token_v2_0_pubkey(&address),
+                        &spl_token_v2_0_pubkey(&keypair.pubkey()),
+                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
+                        &[],
+                    )
+                    .unwrap(),
+                )
+            } else {
+                system_instruction::transfer_with_seed(
+                    &address,
+                    &base_keypair.pubkey(),
+                    seed,
+                    &program_id,
+                    &keypair.pubkey(),
+                    balance,
+                )
+            }
+        })
+        .collect();
+
+    Message::new(&instructions, Some(&keypair.pubkey()))
+}
+
+#[allow(clippy::too_many_arguments)]
 fn run_accounts_bench(
     entrypoint_addr: SocketAddr,
     faucet_addr: SocketAddr,
-    keypair: &Keypair,
+    payer_keypairs: &[&Keypair],
     iterations: usize,
     maybe_space: Option<u64>,
     batch_size: usize,
+    close_nth: u64,
     maybe_lamports: Option<u64>,
     num_instructions: usize,
     mint: Option<Pubkey>,
@@ -315,15 +372,19 @@ fn run_accounts_bench(
     let client =
         RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed());
 
-    info!("Targetting {}", entrypoint_addr);
+    info!("Targeting {}", entrypoint_addr);
 
     let mut last_blockhash = Instant::now();
     let mut last_log = Instant::now();
     let mut count = 0;
     let mut recent_blockhash = client.get_recent_blockhash().expect("blockhash");
     let mut tx_sent_count = 0;
-    let mut total_account_count = 0;
-    let mut balance = client.get_balance(&keypair.pubkey()).unwrap_or(0);
+    let mut total_accounts_created = 0;
+    let mut total_accounts_closed = 0;
+    let mut balances: Vec<_> = payer_keypairs
+        .iter()
+        .map(|keypair| client.get_balance(&keypair.pubkey()).unwrap_or(0))
+        .collect();
     let mut last_balance = Instant::now();
 
     let default_max_lamports = 1000;
@@ -334,7 +395,13 @@ fn run_accounts_bench(
             .expect("min balance")
     });
 
-    info!("Starting balance: {}", balance);
+    let base_keypair = Keypair::new();
+    let seed_tracker = SeedTracker {
+        max_created: Arc::new(AtomicU64::default()),
+        max_closed: Arc::new(AtomicU64::default()),
+    };
+
+    info!("Starting balance(s): {:?}", balances);
 
     let executor = TransactionExecutor::new(entrypoint_addr);
 
@@ -344,24 +411,32 @@ fn run_accounts_bench(
             last_blockhash = Instant::now();
         }
 
-        let (message, _keypairs) =
-            make_message(keypair, num_instructions, min_balance, maybe_space, mint);
-        let fee = recent_blockhash.1.calculate_fee(&message);
+        let fee = recent_blockhash
+            .1
+            .lamports_per_signature
+            .saturating_mul(NUM_SIGNATURES);
         let lamports = min_balance + fee;
 
-        if balance < lamports || last_balance.elapsed().as_millis() > 2000 {
-            if let Ok(b) = client.get_balance(&keypair.pubkey()) {
-                balance = b;
-            }
-            last_balance = Instant::now();
-            if balance < lamports {
-                info!(
-                    "Balance {} is less than needed: {}, doing aidrop...",
-                    balance, lamports
-                );
-                if !airdrop_lamports(&client, &faucet_addr, keypair, lamports * 100_000) {
-                    warn!("failed airdrop, exiting");
-                    return;
+        for (i, balance) in balances.iter_mut().enumerate() {
+            if *balance < lamports || last_balance.elapsed().as_millis() > 2000 {
+                if let Ok(b) = client.get_balance(&payer_keypairs[i].pubkey()) {
+                    *balance = b;
+                }
+                last_balance = Instant::now();
+                if *balance < lamports * 2 {
+                    info!(
+                        "Balance {} is less than needed: {}, doing aidrop...",
+                        balance, lamports
+                    );
+                    if !airdrop_lamports(
+                        &client,
+                        &faucet_addr,
+                        &payer_keypairs[i],
+                        lamports * 100_000,
+                    ) {
+                        warn!("failed airdrop, exiting");
+                        return;
+                    }
                 }
             }
         }
@@ -369,28 +444,63 @@ fn run_accounts_bench(
         let sigs_len = executor.num_outstanding();
         if sigs_len < batch_size {
             let num_to_create = batch_size - sigs_len;
-            info!("creating {} new", num_to_create);
-            let (txs, _new_keypairs): (Vec<_>, Vec<_>) = (0..num_to_create)
-                .into_par_iter()
-                .map(|_| {
-                    let (message, new_keypairs) =
-                        make_message(keypair, num_instructions, min_balance, maybe_space, mint);
-                    let signers: Vec<&Keypair> = new_keypairs
-                        .iter()
-                        .chain(std::iter::once(keypair))
+            if num_to_create >= payer_keypairs.len() {
+                info!("creating {} new", num_to_create);
+                let chunk_size = num_to_create / payer_keypairs.len();
+                if chunk_size > 0 {
+                    for (i, keypair) in payer_keypairs.iter().enumerate() {
+                        let txs: Vec<_> = (0..chunk_size)
+                            .into_par_iter()
+                            .map(|_| {
+                                let message = make_create_message(
+                                    keypair,
+                                    &base_keypair,
+                                    seed_tracker.max_created.clone(),
+                                    num_instructions,
+                                    min_balance,
+                                    maybe_space,
+                                    mint,
+                                );
+                                let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                                Transaction::new(&signers, message, recent_blockhash.0)
+                            })
+                            .collect();
+                        balances[i] = balances[i].saturating_sub(lamports * txs.len() as u64);
+                        info!("txs: {}", txs.len());
+                        let new_ids = executor.push_transactions(txs);
+                        info!("ids: {}", new_ids.len());
+                        tx_sent_count += new_ids.len();
+                        total_accounts_created += num_instructions * new_ids.len();
+                    }
+                }
+            }
+
+            if close_nth > 0 {
+                let expected_closed = total_accounts_created as u64 / close_nth;
+                if expected_closed > total_accounts_closed {
+                    let txs: Vec<_> = (0..expected_closed - total_accounts_closed)
+                        .into_par_iter()
+                        .map(|_| {
+                            let message = make_close_message(
+                                &payer_keypairs[0],
+                                &base_keypair,
+                                seed_tracker.max_closed.clone(),
+                                1,
+                                min_balance,
+                                mint.is_some(),
+                            );
+                            let signers: Vec<&Keypair> = vec![&payer_keypairs[0], &base_keypair];
+                            Transaction::new(&signers, message, recent_blockhash.0)
+                        })
                         .collect();
-                    (
-                        Transaction::new(&signers, message, recent_blockhash.0),
-                        new_keypairs,
-                    )
-                })
-                .unzip();
-            balance = balance.saturating_sub(lamports * txs.len() as u64);
-            info!("txs: {}", txs.len());
-            let new_ids = executor.push_transactions(txs);
-            info!("ids: {}", new_ids.len());
-            tx_sent_count += new_ids.len();
-            total_account_count += num_instructions * new_ids.len();
+                    balances[0] = balances[0].saturating_sub(fee * txs.len() as u64);
+                    info!("close txs: {}", txs.len());
+                    let new_ids = executor.push_transactions(txs);
+                    info!("close ids: {}", new_ids.len());
+                    tx_sent_count += new_ids.len();
+                    total_accounts_closed += new_ids.len() as u64;
+                }
+            }
         } else {
             let _ = executor.drain_cleared();
         }
@@ -398,8 +508,8 @@ fn run_accounts_bench(
         count += 1;
         if last_log.elapsed().as_millis() > 3000 {
             info!(
-                "total_accounts: {} tx_sent_count: {} loop_count: {} balance: {}",
-                total_account_count, tx_sent_count, count, balance
+                "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
+                total_accounts_created, total_accounts_closed, tx_sent_count, count, balances
             );
             last_log = Instant::now();
         }
@@ -450,19 +560,33 @@ fn main() {
             Arg::with_name("identity")
                 .long("identity")
                 .takes_value(true)
+                .multiple(true)
                 .value_name("FILE")
                 .help("keypair file"),
         )
         .arg(
             Arg::with_name("batch_size")
-                .long("batch_size")
+                .long("batch-size")
                 .takes_value(true)
                 .value_name("BYTES")
-                .help("Size of accounts to create"),
+                .help("Number of transactions to send per batch"),
+        )
+        .arg(
+            Arg::with_name("close_nth")
+                .long("close-frequency")
+                .takes_value(true)
+                .value_name("BYTES")
+                .help(
+                    "Send close transactions after this many accounts created. \
+                    Note: a `close-frequency` value near or below `batch-size` \
+                    may result in transaction-simulation errors, as the close \
+                    transactions will be submitted before the corresponding \
+                    create transactions have been confirmed",
+                ),
         )
         .arg(
             Arg::with_name("num_instructions")
-                .long("num_instructions")
+                .long("num-instructions")
                 .takes_value(true)
                 .value_name("NUM")
                 .help("Number of accounts to create on each transaction"),
@@ -508,6 +632,7 @@ fn main() {
     let space = value_t!(matches, "space", u64).ok();
     let lamports = value_t!(matches, "lamports", u64).ok();
     let batch_size = value_t!(matches, "batch_size", usize).unwrap_or(4);
+    let close_nth = value_t!(matches, "close_nth", u64).unwrap_or(0);
     let iterations = value_t!(matches, "iterations", usize).unwrap_or(10);
     let num_instructions = value_t!(matches, "num_instructions", usize).unwrap_or(1);
     if num_instructions == 0 || num_instructions > 500 {
@@ -517,8 +642,17 @@ fn main() {
 
     let mint = pubkey_of(&matches, "mint");
 
-    let keypair =
-        read_keypair_file(&value_t_or_exit!(matches, "identity", String)).expect("bad keypair");
+    let payer_keypairs: Vec<_> = values_t_or_exit!(matches, "identity", String)
+        .iter()
+        .map(|keypair_string| {
+            read_keypair_file(keypair_string)
+                .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_string))
+        })
+        .collect();
+    let mut payer_keypair_refs: Vec<&Keypair> = vec![];
+    for keypair in payer_keypairs.iter() {
+        payer_keypair_refs.push(keypair);
+    }
 
     let rpc_addr = if !skip_gossip {
         info!("Finding cluster entry: {:?}", entrypoint_addr);
@@ -547,10 +681,11 @@ fn main() {
     run_accounts_bench(
         rpc_addr,
         faucet_addr,
-        &keypair,
+        &payer_keypair_refs,
         iterations,
         space,
         batch_size,
+        close_nth,
         lamports,
         num_instructions,
         mint,
@@ -585,16 +720,18 @@ pub mod test {
         let iterations = 10;
         let maybe_space = None;
         let batch_size = 100;
+        let close_nth = 100;
         let maybe_lamports = None;
         let num_instructions = 2;
         let mut start = Measure::start("total accounts run");
         run_accounts_bench(
             cluster.entry_point_info.rpc,
             faucet_addr,
-            &cluster.funding_keypair,
+            &[&cluster.funding_keypair],
             iterations,
             maybe_space,
             batch_size,
+            close_nth,
             maybe_lamports,
             num_instructions,
             None,

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-core = { path = "../core", version = "1.6.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-streamer = { path = "../streamer", version = "1.6.0" }
-solana-perf = { path = "../perf", version = "1.6.0" }
-solana-ledger = { path = "../ledger", version = "1.6.0" }
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-measure = { path = "../measure", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
+solana-core = { path = "../core", version = "=1.6.7" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-streamer = { path = "../streamer", version = "=1.6.7" }
+solana-perf = { path = "../perf", version = "=1.6.7" }
+solana-ledger = { path = "../ledger", version = "=1.6.7" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-measure = { path = "../measure", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.6.0"
+version = "1.6.7"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,16 +15,16 @@ borsh = "0.8.1"
 borsh-derive = "0.8.1"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.6.0" }
-solana-program = { path = "../sdk/program", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.7" }
+solana-program = { path = "../sdk/program", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
 tarpc = { version = "0.24.1", features = ["full"] }
-tokio = { version = "1.1", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-banks-server = { path = "../banks-server", version = "1.6.0" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-banks-server = { path = "../banks-server", version = "=1.6.7" }
 
 [lib]
 crate-type = ["lib"]

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.6.0"
+version = "1.6.7"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,11 +12,11 @@ edition = "2018"
 [dependencies]
 mio = "0.7.6"
 serde = { version = "1.0.122", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
 tarpc = { version = "0.24.1", features = ["full"] }
 
 [dev-dependencies]
-tokio = { version = "1.1", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.6.0"
+version = "1.6.7"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,12 +14,12 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.6.0" }
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-metrics = { path = "../metrics", version = "1.6.0" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.7" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-metrics = { path = "../metrics", version = "=1.6.7" }
 tarpc = { version = "0.24.1", features = ["full"] }
-tokio = { version = "1.1", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 tokio-stream = "0.1"
 

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-core = { path = "../core", version = "1.6.0" }
-solana-genesis = { path = "../genesis", version = "1.6.0" }
-solana-client = { path = "../client", version = "1.6.0" }
-solana-faucet = { path = "../faucet", version = "1.6.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.6.0" }
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-metrics = { path = "../metrics", version = "1.6.0" }
-solana-net-utils = { path = "../net-utils", version = "1.6.0" }
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-core = { path = "../core", version = "=1.6.7" }
+solana-genesis = { path = "../genesis", version = "=1.6.7" }
+solana-client = { path = "../client", version = "=1.6.7" }
+solana-faucet = { path = "../faucet", version = "=1.6.7" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.6.7" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-metrics = { path = "../metrics", version = "=1.6.7" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.7" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.6.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.7" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-streamer = { path = "../streamer", version = "1.6.0" }
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-net-utils = { path = "../net-utils", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-streamer = { path = "../streamer", version = "=1.6.7" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,22 +15,22 @@ log = "0.4.11"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-core = { path = "../core", version = "1.6.0" }
-solana-genesis = { path = "../genesis", version = "1.6.0" }
-solana-client = { path = "../client", version = "1.6.0" }
-solana-faucet = { path = "../faucet", version = "1.6.0" }
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-metrics = { path = "../metrics", version = "1.6.0" }
-solana-measure = { path = "../measure", version = "1.6.0" }
-solana-net-utils = { path = "../net-utils", version = "1.6.0" }
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-core = { path = "../core", version = "=1.6.7" }
+solana-genesis = { path = "../genesis", version = "=1.6.7" }
+solana-client = { path = "../client", version = "=1.6.7" }
+solana-faucet = { path = "../faucet", version = "=1.6.7" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-metrics = { path = "../metrics", version = "=1.6.7" }
+solana-measure = { path = "../measure", version = "=1.6.7" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.7" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.6.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.7" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cargo

@@ -3,25 +3,22 @@
 # shellcheck source=ci/rust-version.sh
 here=$(dirname "$0")
 
-source "${here}"/ci/rust-version.sh all
-
 toolchain=
 case "$1" in
   stable)
+    source "${here}"/ci/rust-version.sh stable
     # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
     toolchain="$rust_stable"
     shift
     ;;
   nightly)
+    source "${here}"/ci/rust-version.sh nightly
     # shellcheck disable=SC2054 # rust_nightly is sourced from rust-version.sh
     toolchain="$rust_nightly"
     shift
     ;;
-  +*)
-    toolchain="${1#+}"
-    shift
-    ;;
   *)
+    source "${here}"/ci/rust-version.sh stable
     # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
     toolchain="$rust_stable"
     ;;

ci/channel-info.sh

@@ -105,11 +105,18 @@ if [[ -z "$CHANNEL" ]]; then
   fi
 fi
 
+if [[ $CHANNEL = beta ]]; then
+  CHANNEL_LATEST_TAG="$BETA_CHANNEL_LATEST_TAG"
+elif [[ $CHANNEL = stable ]]; then
+  CHANNEL_LATEST_TAG="$STABLE_CHANNEL_LATEST_TAG"
+fi
+
 echo EDGE_CHANNEL="$EDGE_CHANNEL"
 echo BETA_CHANNEL="$BETA_CHANNEL"
 echo BETA_CHANNEL_LATEST_TAG="$BETA_CHANNEL_LATEST_TAG"
 echo STABLE_CHANNEL="$STABLE_CHANNEL"
 echo STABLE_CHANNEL_LATEST_TAG="$STABLE_CHANNEL_LATEST_TAG"
 echo CHANNEL="$CHANNEL"
+echo CHANNEL_LATEST_TAG="$CHANNEL_LATEST_TAG"
 
 exit 0

ci/do-audit.sh

@@ -7,8 +7,6 @@ src_root="$(readlink -f "${here}/..")"
 
 cd "${src_root}"
 
-source ci/rust-version.sh stable
-
 cargo_audit_ignores=(
   # failure is officially deprecated/unmaintained
   #
@@ -42,4 +40,4 @@ cargo_audit_ignores=(
   --ignore RUSTSEC-2020-0146
 
 )
-scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"
+scripts/cargo-for-all-lock-files.sh stable audit "${cargo_audit_ignores[@]}"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.50.0
+FROM solanalabs/rust:1.51.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.50.0
+FROM rust:1.51.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/localnet-sanity.sh

@@ -70,7 +70,7 @@ done
 
 source ci/upload-ci-artifact.sh
 source scripts/configure-metrics.sh
-source multinode-demo/common.sh
+source multinode-demo/common.sh --prebuild
 
 nodes=(
   "multinode-demo/bootstrap-validator.sh \

ci/publish-tarball.sh

@@ -83,7 +83,7 @@ echo --- Creating release tarball
   export CHANNEL
 
   source ci/rust-version.sh stable
-  scripts/cargo-install-all.sh +"$rust_stable" "${RELEASE_BASENAME}"
+  scripts/cargo-install-all.sh stable "${RELEASE_BASENAME}"
 
   tar cvf "${TARBALL_BASENAME}"-$TARGET.tar "${RELEASE_BASENAME}"
   bzip2 "${TARBALL_BASENAME}"-$TARGET.tar

ci/run-sanity.sh

@@ -22,7 +22,7 @@ done
 snapshot_slot=1
 
 # wait a bit longer than snapshot_slot
-while [[ $($solana_cli --url http://localhost:8899 slot --commitment recent) -le $((snapshot_slot + 1)) ]]; do
+while [[ $($solana_cli --url http://localhost:8899 slot --commitment processed) -le $((snapshot_slot + 1)) ]]; do
   sleep 1
 done
 

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.50.0
+  stable_version=1.51.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2021-02-18
+  nightly_version=2021-04-18
 fi
 
 

ci/test-checks.sh

@@ -45,7 +45,7 @@ export RUSTFLAGS="-D warnings -A incomplete_features"
 # Only force up-to-date lock files on edge
 if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
   # Exclude --benches as it's not available in rust stable yet
-  if _ scripts/cargo-for-all-lock-files.sh +"$rust_stable" check --locked --tests --bins --examples; then
+  if _ scripts/cargo-for-all-lock-files.sh stable check --locked --tests --bins --examples; then
     true
   else
     check_status=$?
@@ -56,7 +56,7 @@ if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
   fi
 
   # Ensure nightly and --benches
-  _ scripts/cargo-for-all-lock-files.sh +"$rust_nightly" check --locked --all-targets
+  _ scripts/cargo-for-all-lock-files.sh nightly check --locked --all-targets
 else
   echo "Note: cargo-for-all-lock-files.sh skipped because $CI_BASE_BRANCH != $EDGE_CHANNEL"
 fi
@@ -79,7 +79,6 @@ _ ci/do-audit.sh
       cd "$project"
       _ "$cargo" nightly clippy -- --deny=warnings --allow=clippy::missing_safety_doc
       _ "$cargo" stable fmt -- --check
-      _ "$cargo" nightly test
     )
   done
 }

ci/test-sanity.sh

@@ -25,4 +25,29 @@ echo
 _ ci/nits.sh
 _ ci/check-ssh-keys.sh
 
+
+# Ensure the current channel version is not equal ("greater") than
+# the version of the latest tag
+if [[ -z $CI_TAG ]]; then
+  echo "--- channel version check"
+  (
+    eval "$(ci/channel-info.sh)"
+
+    if [[ -n $CHANNEL_LATEST_TAG ]]; then
+      source scripts/read-cargo-variable.sh
+
+      version=$(readCargoVariable version "version/Cargo.toml")
+      echo "version: v$version"
+      echo "latest channel tag: $CHANNEL_LATEST_TAG"
+
+      if [[ $CHANNEL_LATEST_TAG = v$version ]]; then
+        echo "Error: please run ./scripts/increment-cargo-version.sh"
+        exit 1
+      fi
+    else
+      echo "Skipped. CHANNEL_LATEST_TAG (CHANNEL=$CHANNEL) unset"
+    fi
+  )
+fi
+
 echo --- ok

ci/test-stable.sh

@@ -25,9 +25,6 @@ source scripts/ulimit-n.sh
 test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 
-# Clear the BPF sysroot files, they are not automatically rebuilt
-rm -rf target/xargo # Issue #3105
-
 # Limit compiler jobs to reduce memory usage
 # on machines with 2gb/thread of memory
 NPROC=$(nproc)
@@ -46,7 +43,11 @@ test-stable-perf)
   # BPF solana-sdk legacy compile test
   ./cargo-build-bpf --manifest-path sdk/Cargo.toml
 
-  # BPF program tests
+  # BPF Program unit tests
+  "$cargo" stable test --manifest-path programs/bpf/Cargo.toml
+  cargo-build-bpf --manifest-path programs/bpf/Cargo.toml --bpf-sdk sdk/bpf
+
+  # BPF program system tests
   _ make -C programs/bpf/c tests
   _ "$cargo" stable test \
     --manifest-path programs/bpf/Cargo.toml \

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.6.0"
+version = "1.6.7"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,13 +12,17 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.8.0"
+uriparse = "0.6.3"
 url = "2.1.0"
 chrono = "0.4"
 
+[dev-dependencies]
+tempfile = "3.1.0"
+
 [lib]
 name = "solana_clap_utils"
 

clap-utils/src/fee_payer.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators, ArgConstant};
-use clap::Arg;
+use {
+    crate::{input_validators, ArgConstant},
+    clap::Arg,
+};
 
 pub const FEE_PAYER_ARG: ArgConstant<'static> = ArgConstant {
     name: "fee_payer",

clap-utils/src/input_parsers.rs

@@ -1,19 +1,21 @@
-use crate::keypair::{
-    keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
-    ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
+use {
+    crate::keypair::{
+        keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
+        ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
+    },
+    chrono::DateTime,
+    clap::ArgMatches,
+    solana_remote_wallet::remote_wallet::RemoteWalletManager,
+    solana_sdk::{
+        clock::UnixTimestamp,
+        commitment_config::CommitmentConfig,
+        genesis_config::ClusterType,
+        native_token::sol_to_lamports,
+        pubkey::Pubkey,
+        signature::{read_keypair_file, Keypair, Signature, Signer},
+    },
+    std::{str::FromStr, sync::Arc},
 };
-use chrono::DateTime;
-use clap::ArgMatches;
-use solana_remote_wallet::remote_wallet::RemoteWalletManager;
-use solana_sdk::{
-    clock::UnixTimestamp,
-    commitment_config::CommitmentConfig,
-    genesis_config::ClusterType,
-    native_token::sol_to_lamports,
-    pubkey::Pubkey,
-    signature::{read_keypair_file, Keypair, Signature, Signer},
-};
-use std::{str::FromStr, sync::Arc};
 
 // Return parsed values from matches at `name`
 pub fn values_of<T>(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<T>>
@@ -55,7 +57,7 @@ pub fn keypair_of(matches: &ArgMatches<'_>, name: &str) -> Option<Keypair> {
     if let Some(value) = matches.value_of(name) {
         if value == ASK_KEYWORD {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-            keypair_from_seed_phrase(name, skip_validation, true).ok()
+            keypair_from_seed_phrase(name, skip_validation, true, None).ok()
         } else {
             read_keypair_file(value).ok()
         }
@@ -70,7 +72,7 @@ pub fn keypairs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Keypair>>
             .filter_map(|value| {
                 if value == ASK_KEYWORD {
                     let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-                    keypair_from_seed_phrase(name, skip_validation, true).ok()
+                    keypair_from_seed_phrase(name, skip_validation, true, None).ok()
                 } else {
                     read_keypair_file(value).ok()
                 }

clap-utils/src/input_validators.rs

@@ -1,13 +1,15 @@
-use crate::keypair::{parse_keypair_path, KeypairUrl, ASK_KEYWORD};
-use chrono::DateTime;
-use solana_sdk::{
-    clock::{Epoch, Slot},
-    hash::Hash,
-    pubkey::{Pubkey, MAX_SEED_LEN},
-    signature::{read_keypair_file, Signature},
+use {
+    crate::keypair::{parse_signer_source, SignerSourceKind, ASK_KEYWORD},
+    chrono::DateTime,
+    solana_sdk::{
+        clock::{Epoch, Slot},
+        hash::Hash,
+        pubkey::{Pubkey, MAX_SEED_LEN},
+        signature::{read_keypair_file, Signature},
+    },
+    std::fmt::Display,
+    std::str::FromStr,
 };
-use std::fmt::Display;
-use std::str::FromStr;
 
 fn is_parsable_generic<U, T>(string: T) -> Result<(), String>
 where
@@ -32,6 +34,29 @@ where
     is_parsable_generic::<T, String>(string)
 }
 
+// Return an error if string cannot be parsed as numeric type T, and value not within specified
+// range
+pub fn is_within_range<T>(string: String, range_min: T, range_max: T) -> Result<(), String>
+where
+    T: FromStr + Copy + std::fmt::Debug + PartialOrd + std::ops::Add<Output = T> + From<usize>,
+    T::Err: Display,
+{
+    match string.parse::<T>() {
+        Ok(input) => {
+            let range = range_min..range_max + 1.into();
+            if !range.contains(&input) {
+                Err(format!(
+                    "input '{:?}' out of range ({:?}..{:?}]",
+                    input, range_min, range_max
+                ))
+            } else {
+                Ok(())
+            }
+        }
+        Err(err) => Err(format!("error parsing '{}': {}", string, err)),
+    }
+}
+
 // Return an error if a pubkey cannot be parsed.
 pub fn is_pubkey<T>(string: T) -> Result<(), String>
 where
@@ -85,8 +110,11 @@ pub fn is_valid_pubkey<T>(string: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,
 {
-    match parse_keypair_path(string.as_ref()) {
-        KeypairUrl::Filepath(path) => is_keypair(path),
+    match parse_signer_source(string.as_ref())
+        .map_err(|err| format!("{}", err))?
+        .kind
+    {
+        SignerSourceKind::Filepath(path) => is_keypair(path),
         _ => Ok(()),
     }
 }

clap-utils/src/keypair.rs

@@ -1,33 +1,41 @@
-use crate::{
-    input_parsers::pubkeys_sigs_of,
-    offline::{SIGNER_ARG, SIGN_ONLY_ARG},
-    ArgConstant,
-};
-use bip39::{Language, Mnemonic, Seed};
-use clap::ArgMatches;
-use rpassword::prompt_password_stderr;
-use solana_remote_wallet::{
-    remote_keypair::generate_remote_keypair,
-    remote_wallet::{maybe_wallet_manager, RemoteWalletError, RemoteWalletManager},
-};
-use solana_sdk::{
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::{
-        keypair_from_seed, keypair_from_seed_phrase_and_passphrase, read_keypair,
-        read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
+use {
+    crate::{
+        input_parsers::pubkeys_sigs_of,
+        offline::{SIGNER_ARG, SIGN_ONLY_ARG},
+        ArgConstant,
     },
-};
-use std::{
-    error,
-    io::{stdin, stdout, Write},
-    process::exit,
-    str::FromStr,
-    sync::Arc,
+    bip39::{Language, Mnemonic, Seed},
+    clap::ArgMatches,
+    rpassword::prompt_password_stderr,
+    solana_remote_wallet::{
+        locator::{Locator as RemoteWalletLocator, LocatorError as RemoteWalletLocatorError},
+        remote_keypair::generate_remote_keypair,
+        remote_wallet::{maybe_wallet_manager, RemoteWalletError, RemoteWalletManager},
+    },
+    solana_sdk::{
+        derivation_path::{DerivationPath, DerivationPathError},
+        hash::Hash,
+        message::Message,
+        pubkey::Pubkey,
+        signature::{
+            generate_seed_from_seed_phrase_and_passphrase, keypair_from_seed_and_derivation_path,
+            read_keypair, read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
+        },
+    },
+    std::{
+        convert::TryFrom,
+        error,
+        io::{stdin, stdout, Write},
+        process::exit,
+        str::FromStr,
+        sync::Arc,
+    },
+    thiserror::Error,
 };
 
 pub struct SignOnly {
     pub blockhash: Hash,
+    pub message: Option<String>,
     pub present_signers: Vec<(Pubkey, Signature)>,
     pub absent_signers: Vec<Pubkey>,
     pub bad_signers: Vec<Pubkey>,
@@ -67,6 +75,18 @@ impl CliSignerInfo {
             None
         }
     }
+    pub fn signers_for_message(&self, message: &Message) -> Vec<&dyn Signer> {
+        self.signers
+            .iter()
+            .filter_map(|k| {
+                if message.signer_keys().contains(&&k.pubkey()) {
+                    Some(k.as_ref())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
 }
 
 pub struct DefaultSigner {
@@ -89,11 +109,9 @@ impl DefaultSigner {
             unique_signers.push(default_signer);
         }
 
-        for signer in bulk_signers.into_iter() {
-            if let Some(signer) = signer {
-                if !unique_signers.iter().any(|s| s == &signer) {
-                    unique_signers.push(signer);
-                }
+        for signer in bulk_signers.into_iter().flatten() {
+            if !unique_signers.iter().any(|s| s == &signer) {
+                unique_signers.push(signer);
             }
         }
         Ok(CliSignerInfo {
@@ -108,27 +126,90 @@ impl DefaultSigner {
     ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
         signer_from_path(matches, &self.path, &self.arg_name, wallet_manager)
     }
+
+    pub fn signer_from_path_with_config(
+        &self,
+        matches: &ArgMatches,
+        wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+        config: &SignerFromPathConfig,
+    ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
+        signer_from_path_with_config(matches, &self.path, &self.arg_name, wallet_manager, config)
+    }
 }
 
-pub enum KeypairUrl {
+pub(crate) struct SignerSource {
+    pub kind: SignerSourceKind,
+    pub derivation_path: Option<DerivationPath>,
+}
+
+impl SignerSource {
+    fn new(kind: SignerSourceKind) -> Self {
+        Self {
+            kind,
+            derivation_path: None,
+        }
+    }
+}
+
+pub(crate) enum SignerSourceKind {
     Ask,
     Filepath(String),
-    Usb(String),
+    Usb(RemoteWalletLocator),
     Stdin,
     Pubkey(Pubkey),
 }
 
-pub fn parse_keypair_path(path: &str) -> KeypairUrl {
-    if path == "-" {
-        KeypairUrl::Stdin
-    } else if path == ASK_KEYWORD {
-        KeypairUrl::Ask
-    } else if path.starts_with("usb://") {
-        KeypairUrl::Usb(path.to_string())
-    } else if let Ok(pubkey) = Pubkey::from_str(path) {
-        KeypairUrl::Pubkey(pubkey)
-    } else {
-        KeypairUrl::Filepath(path.to_string())
+#[derive(Debug, Error)]
+pub(crate) enum SignerSourceError {
+    #[error("unrecognized signer source")]
+    UnrecognizedSource,
+    #[error(transparent)]
+    RemoteWalletLocatorError(#[from] RemoteWalletLocatorError),
+    #[error(transparent)]
+    DerivationPathError(#[from] DerivationPathError),
+    #[error(transparent)]
+    IoError(#[from] std::io::Error),
+}
+
+pub(crate) fn parse_signer_source<S: AsRef<str>>(
+    source: S,
+) -> Result<SignerSource, SignerSourceError> {
+    let source = source.as_ref();
+    match uriparse::URIReference::try_from(source) {
+        Err(_) => Err(SignerSourceError::UnrecognizedSource),
+        Ok(uri) => {
+            if let Some(scheme) = uri.scheme() {
+                let scheme = scheme.as_str().to_ascii_lowercase();
+                match scheme.as_str() {
+                    "ask" => Ok(SignerSource {
+                        kind: SignerSourceKind::Ask,
+                        derivation_path: DerivationPath::from_uri_any_query(&uri)?,
+                    }),
+                    "file" => Ok(SignerSource::new(SignerSourceKind::Filepath(
+                        uri.path().to_string(),
+                    ))),
+                    "stdin" => Ok(SignerSource::new(SignerSourceKind::Stdin)),
+                    "usb" => Ok(SignerSource {
+                        kind: SignerSourceKind::Usb(RemoteWalletLocator::new_from_uri(&uri)?),
+                        derivation_path: DerivationPath::from_uri_key_query(&uri)?,
+                    }),
+                    _ => Err(SignerSourceError::UnrecognizedSource),
+                }
+            } else {
+                match source {
+                    "-" => Ok(SignerSource::new(SignerSourceKind::Stdin)),
+                    ASK_KEYWORD => Ok(SignerSource::new(SignerSourceKind::Ask)),
+                    _ => match Pubkey::from_str(source) {
+                        Ok(pubkey) => Ok(SignerSource::new(SignerSourceKind::Pubkey(pubkey))),
+                        Err(_) => std::fs::metadata(source)
+                            .map(|_| {
+                                SignerSource::new(SignerSourceKind::Filepath(source.to_string()))
+                            })
+                            .map_err(|err| err.into()),
+                    },
+                }
+            }
+        }
     }
 }
 
@@ -145,22 +226,51 @@ pub fn presigner_from_pubkey_sigs(
     })
 }
 
+#[derive(Debug)]
+pub struct SignerFromPathConfig {
+    pub allow_null_signer: bool,
+}
+
+impl Default for SignerFromPathConfig {
+    fn default() -> Self {
+        Self {
+            allow_null_signer: false,
+        }
+    }
+}
+
 pub fn signer_from_path(
     matches: &ArgMatches,
     path: &str,
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Box<dyn Signer>, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Ask => {
+    let config = SignerFromPathConfig::default();
+    signer_from_path_with_config(matches, path, keypair_name, wallet_manager, &config)
+}
+
+pub fn signer_from_path_with_config(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+    config: &SignerFromPathConfig,
+) -> Result<Box<dyn Signer>, Box<dyn error::Error>> {
+    let SignerSource {
+        kind,
+        derivation_path,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Ask => {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
             Ok(Box::new(keypair_from_seed_phrase(
                 keypair_name,
                 skip_validation,
                 false,
+                derivation_path,
             )?))
         }
-        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
             Err(e) => Err(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 format!("could not read keypair file \"{}\". Run \"solana-keygen new\" to create a keypair file: {}", path, e),
@@ -168,17 +278,18 @@ pub fn signer_from_path(
             .into()),
             Ok(file) => Ok(Box::new(file)),
         },
-        KeypairUrl::Stdin => {
+        SignerSourceKind::Stdin => {
             let mut stdin = std::io::stdin();
             Ok(Box::new(read_keypair(&mut stdin)?))
         }
-        KeypairUrl::Usb(path) => {
+        SignerSourceKind::Usb(locator) => {
             if wallet_manager.is_none() {
                 *wallet_manager = maybe_wallet_manager()?;
             }
             if let Some(wallet_manager) = wallet_manager {
                 Ok(Box::new(generate_remote_keypair(
-                    path,
+                    locator,
+                    derivation_path.unwrap_or_default(),
                     wallet_manager,
                     matches.is_present("confirm_key"),
                     keypair_name,
@@ -187,13 +298,13 @@ pub fn signer_from_path(
                 Err(RemoteWalletError::NoDeviceFound.into())
             }
         }
-        KeypairUrl::Pubkey(pubkey) => {
+        SignerSourceKind::Pubkey(pubkey) => {
             let presigner = pubkeys_sigs_of(matches, SIGNER_ARG.name)
                 .as_ref()
                 .and_then(|presigners| presigner_from_pubkey_sigs(&pubkey, presigners));
             if let Some(presigner) = presigner {
                 Ok(Box::new(presigner))
-            } else if matches.is_present(SIGN_ONLY_ARG.name) {
+            } else if config.allow_null_signer || matches.is_present(SIGN_ONLY_ARG.name) {
                 Ok(Box::new(NullSigner::new(&pubkey)))
             } else {
                 Err(std::io::Error::new(
@@ -212,8 +323,9 @@ pub fn pubkey_from_path(
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Pubkey, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Pubkey(pubkey) => Ok(pubkey),
+    let SignerSource { kind, .. } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Pubkey(pubkey) => Ok(pubkey),
         _ => Ok(signer_from_path(matches, path, keypair_name, wallet_manager)?.pubkey()),
     }
 }
@@ -224,14 +336,18 @@ pub fn resolve_signer_from_path(
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Option<String>, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Ask => {
+    let SignerSource {
+        kind,
+        derivation_path,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Ask => {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
             // This method validates the seed phrase, but returns `None` because there is no path
             // on disk or to a device
-            keypair_from_seed_phrase(keypair_name, skip_validation, false).map(|_| None)
+            keypair_from_seed_phrase(keypair_name, skip_validation, false, derivation_path).map(|_| None)
         }
-        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
             Err(e) => Err(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 format!("could not read keypair file \"{}\". Run \"solana-keygen new\" to create a keypair file: {}", path, e),
@@ -239,19 +355,20 @@ pub fn resolve_signer_from_path(
             .into()),
             Ok(_) => Ok(Some(path.to_string())),
         },
-        KeypairUrl::Stdin => {
+        SignerSourceKind::Stdin => {
             let mut stdin = std::io::stdin();
             // This method validates the keypair from stdin, but returns `None` because there is no
             // path on disk or to a device
             read_keypair(&mut stdin).map(|_| None)
         }
-        KeypairUrl::Usb(path) => {
+        SignerSourceKind::Usb(locator) => {
             if wallet_manager.is_none() {
                 *wallet_manager = maybe_wallet_manager()?;
             }
             if let Some(wallet_manager) = wallet_manager {
                 let path = generate_remote_keypair(
-                    path,
+                    locator,
+                    derivation_path.unwrap_or_default(),
                     wallet_manager,
                     matches.is_present("confirm_key"),
                     keypair_name,
@@ -294,6 +411,7 @@ pub fn keypair_from_seed_phrase(
     keypair_name: &str,
     skip_validation: bool,
     confirm_pubkey: bool,
+    derivation_path: Option<DerivationPath>,
 ) -> Result<Keypair, Box<dyn error::Error>> {
     let seed_phrase = prompt_password_stderr(&format!("[{}] seed phrase: ", keypair_name))?;
     let seed_phrase = seed_phrase.trim();
@@ -304,7 +422,8 @@ pub fn keypair_from_seed_phrase(
 
     let keypair = if skip_validation {
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
-        keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
+        let seed = generate_seed_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase);
+        keypair_from_seed_and_derivation_path(&seed, derivation_path)?
     } else {
         let sanitized = sanitize_seed_phrase(seed_phrase);
         let parse_language_fn = || {
@@ -327,7 +446,7 @@ pub fn keypair_from_seed_phrase(
         let mnemonic = parse_language_fn()?;
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
         let seed = Seed::new(&mnemonic, &passphrase);
-        keypair_from_seed(seed.as_bytes())?
+        keypair_from_seed_and_derivation_path(&seed.as_bytes(), derivation_path)?
     };
 
     if confirm_pubkey {
@@ -355,6 +474,9 @@ fn sanitize_seed_phrase(seed_phrase: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use solana_remote_wallet::locator::Manufacturer;
+    use solana_sdk::system_instruction;
+    use tempfile::NamedTempFile;
 
     #[test]
     fn test_sanitize_seed_phrase() {
@@ -364,4 +486,142 @@ mod tests {
             sanitize_seed_phrase(seed_phrase)
         );
     }
+
+    #[test]
+    fn test_signer_info_signers_for_message() {
+        let source = Keypair::new();
+        let fee_payer = Keypair::new();
+        let nonsigner1 = Keypair::new();
+        let nonsigner2 = Keypair::new();
+        let recipient = Pubkey::new_unique();
+        let message = Message::new(
+            &[system_instruction::transfer(
+                &source.pubkey(),
+                &recipient,
+                42,
+            )],
+            Some(&fee_payer.pubkey()),
+        );
+        let signers = vec![
+            Box::new(fee_payer) as Box<dyn Signer>,
+            Box::new(source) as Box<dyn Signer>,
+            Box::new(nonsigner1) as Box<dyn Signer>,
+            Box::new(nonsigner2) as Box<dyn Signer>,
+        ];
+        let signer_info = CliSignerInfo { signers };
+        let msg_signers = signer_info.signers_for_message(&message);
+        let signer_pubkeys = msg_signers.iter().map(|s| s.pubkey()).collect::<Vec<_>>();
+        let expect = vec![
+            signer_info.signers[0].pubkey(),
+            signer_info.signers[1].pubkey(),
+        ];
+        assert_eq!(signer_pubkeys, expect);
+    }
+
+    #[test]
+    fn test_parse_signer_source() {
+        assert!(matches!(
+            parse_signer_source("-").unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Stdin,
+                derivation_path: None,
+            }
+        ));
+        let ask = "stdin:".to_string();
+        assert!(matches!(
+            parse_signer_source(&ask).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Stdin,
+                derivation_path: None,
+            }
+        ));
+        assert!(matches!(
+            parse_signer_source(ASK_KEYWORD).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Ask,
+                derivation_path: None,
+            }
+        ));
+        let pubkey = Pubkey::new_unique();
+        assert!(
+            matches!(parse_signer_source(&pubkey.to_string()).unwrap(), SignerSource {
+                kind: SignerSourceKind::Pubkey(p),
+                derivation_path: None,
+            }
+            if p == pubkey)
+        );
+
+        // Set up absolute and relative path strs
+        let file0 = NamedTempFile::new().unwrap();
+        let path = file0.path();
+        assert!(path.is_absolute());
+        let absolute_path_str = path.to_str().unwrap();
+
+        let file1 = NamedTempFile::new_in(std::env::current_dir().unwrap()).unwrap();
+        let path = file1.path().file_name().unwrap().to_str().unwrap();
+        let path = std::path::Path::new(path);
+        assert!(path.is_relative());
+        let relative_path_str = path.to_str().unwrap();
+
+        assert!(
+            matches!(parse_signer_source(absolute_path_str).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+            } if p == absolute_path_str)
+        );
+        assert!(
+            matches!(parse_signer_source(&relative_path_str).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+            } if p == relative_path_str)
+        );
+
+        let usb = "usb://ledger".to_string();
+        let expected_locator = RemoteWalletLocator {
+            manufacturer: Manufacturer::Ledger,
+            pubkey: None,
+        };
+        assert!(matches!(parse_signer_source(&usb).unwrap(), SignerSource {
+                kind: SignerSourceKind::Usb(u),
+                derivation_path: None,
+            } if u == expected_locator));
+        let usb = "usb://ledger?key=0/0".to_string();
+        let expected_locator = RemoteWalletLocator {
+            manufacturer: Manufacturer::Ledger,
+            pubkey: None,
+        };
+        let expected_derivation_path = Some(DerivationPath::new_bip44(Some(0), Some(0)));
+        assert!(matches!(parse_signer_source(&usb).unwrap(), SignerSource {
+                kind: SignerSourceKind::Usb(u),
+                derivation_path: d,
+            } if u == expected_locator && d == expected_derivation_path));
+        // Catchall into SignerSource::Filepath fails
+        let junk = "sometextthatisnotapubkeyorfile".to_string();
+        assert!(Pubkey::from_str(&junk).is_err());
+        assert!(matches!(
+            parse_signer_source(&junk),
+            Err(SignerSourceError::IoError(_))
+        ));
+
+        let ask = "ask:".to_string();
+        assert!(matches!(
+            parse_signer_source(&ask).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Ask,
+                derivation_path: None,
+            }
+        ));
+        assert!(
+            matches!(parse_signer_source(&format!("file:{}", absolute_path_str)).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+            } if p == absolute_path_str)
+        );
+        assert!(
+            matches!(parse_signer_source(&format!("file:{}", relative_path_str)).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+            } if p == relative_path_str)
+        );
+    }
 }

clap-utils/src/lib.rs

@@ -27,5 +27,6 @@ pub mod fee_payer;
 pub mod input_parsers;
 pub mod input_validators;
 pub mod keypair;
+pub mod memo;
 pub mod nonce;
 pub mod offline;

clap-utils/src/memo.rs

@@ -0,0 +1,15 @@
+use {crate::ArgConstant, clap::Arg};
+
+pub const MEMO_ARG: ArgConstant<'static> = ArgConstant {
+    name: "memo",
+    long: "--with-memo",
+    help: "Specify a memo string to include in the transaction.",
+};
+
+pub fn memo_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(MEMO_ARG.name)
+        .long(MEMO_ARG.long)
+        .takes_value(true)
+        .value_name("MEMO")
+        .help(MEMO_ARG.help)
+}

clap-utils/src/nonce.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators::*, offline::BLOCKHASH_ARG, ArgConstant};
-use clap::{App, Arg};
+use {
+    crate::{input_validators::*, offline::BLOCKHASH_ARG, ArgConstant},
+    clap::{App, Arg},
+};
 
 pub const NONCE_ARG: ArgConstant<'static> = ArgConstant {
     name: "nonce",

clap-utils/src/offline.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators::*, ArgConstant};
-use clap::{App, Arg};
+use {
+    crate::{input_validators::*, ArgConstant},
+    clap::{App, Arg},
+};
 
 pub const BLOCKHASH_ARG: ArgConstant<'static> = ArgConstant {
     name: "blockhash",
@@ -19,6 +21,12 @@ pub const SIGNER_ARG: ArgConstant<'static> = ArgConstant {
     help: "Provide a public-key/signature pair for the transaction",
 };
 
+pub const DUMP_TRANSACTION_MESSAGE: ArgConstant<'static> = ArgConstant {
+    name: "dump_transaction_message",
+    long: "dump-transaction-message",
+    help: "Display the base64 encoded binary transaction message in sign-only mode",
+};
+
 pub fn blockhash_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name(BLOCKHASH_ARG.name)
         .long(BLOCKHASH_ARG.long)
@@ -47,6 +55,14 @@ fn signer_arg<'a, 'b>() -> Arg<'a, 'b> {
         .help(SIGNER_ARG.help)
 }
 
+pub fn dump_transaction_message<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(DUMP_TRANSACTION_MESSAGE.name)
+        .long(DUMP_TRANSACTION_MESSAGE.long)
+        .takes_value(false)
+        .requires(SIGN_ONLY_ARG.name)
+        .help(DUMP_TRANSACTION_MESSAGE.help)
+}
+
 pub trait ArgsConfig {
     fn blockhash_arg<'a, 'b>(&self, arg: Arg<'a, 'b>) -> Arg<'a, 'b> {
         arg
@@ -57,6 +73,9 @@ pub trait ArgsConfig {
     fn signer_arg<'a, 'b>(&self, arg: Arg<'a, 'b>) -> Arg<'a, 'b> {
         arg
     }
+    fn dump_transaction_message_arg<'a, 'b>(&self, arg: Arg<'a, 'b>) -> Arg<'a, 'b> {
+        arg
+    }
 }
 
 pub trait OfflineArgs {
@@ -69,6 +88,7 @@ impl OfflineArgs for App<'_, '_> {
         self.arg(config.blockhash_arg(blockhash_arg()))
             .arg(config.sign_only_arg(sign_only_arg()))
             .arg(config.signer_arg(signer_arg()))
+            .arg(config.dump_transaction_message_arg(dump_transaction_message()))
     }
     fn offline_args(self) -> Self {
         struct NullArgsConfig {}

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-output/Cargo.toml

@@ -3,13 +3,14 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-cli-output"
 
 [dependencies]
+base64 = "0.13.0"
 chrono = { version = "0.4.11", features = ["serde"] }
 console = "0.11.3"
 humantime = "2.0.1"
@@ -18,13 +19,14 @@ indicatif = "0.15.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.6.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-client = { path = "../client", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.6.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.6.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.6.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.7" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-client = { path = "../client", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.7" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.7" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.7" }
+spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -19,7 +19,7 @@ use {
         RpcVoteAccountInfo,
     },
     solana_sdk::{
-        clock::{self, Epoch, Slot, UnixTimestamp},
+        clock::{Epoch, Slot, UnixTimestamp},
         epoch_info::EpochInfo,
         hash::Hash,
         native_token::lamports_to_sol,
@@ -231,12 +231,8 @@ pub struct CliSlotStatus {
 pub struct CliEpochInfo {
     #[serde(flatten)]
     pub epoch_info: EpochInfo,
-}
-
-impl From<EpochInfo> for CliEpochInfo {
-    fn from(epoch_info: EpochInfo) -> Self {
-        Self { epoch_info }
-    }
+    #[serde(skip)]
+    pub average_slot_time_ms: u64,
 }
 
 impl QuietDisplay for CliEpochInfo {}
@@ -286,16 +282,16 @@ impl fmt::Display for CliEpochInfo {
             "Epoch Completed Time:",
             &format!(
                 "{}/{} ({} remaining)",
-                slot_to_human_time(self.epoch_info.slot_index),
-                slot_to_human_time(self.epoch_info.slots_in_epoch),
-                slot_to_human_time(remaining_slots_in_epoch)
+                slot_to_human_time(self.epoch_info.slot_index, self.average_slot_time_ms),
+                slot_to_human_time(self.epoch_info.slots_in_epoch, self.average_slot_time_ms),
+                slot_to_human_time(remaining_slots_in_epoch, self.average_slot_time_ms)
             ),
         )
     }
 }
 
-fn slot_to_human_time(slot: Slot) -> String {
-    humantime::format_duration(Duration::from_millis(slot * clock::DEFAULT_MS_PER_SLOT)).to_string()
+fn slot_to_human_time(slot: Slot, slot_time_ms: u64) -> String {
+    humantime::format_duration(Duration::from_secs((slot * slot_time_ms) / 1000)).to_string()
 }
 
 #[derive(Serialize, Deserialize, Default)]
@@ -307,14 +303,32 @@ pub struct CliValidatorsStakeByVersion {
     pub delinquent_active_stake: u64,
 }
 
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
+pub enum CliValidatorsSortOrder {
+    Delinquent,
+    Commission,
+    EpochCredits,
+    Identity,
+    LastVote,
+    Root,
+    SkipRate,
+    Stake,
+    VoteAccount,
+}
+
 #[derive(Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct CliValidators {
     pub total_active_stake: u64,
     pub total_current_stake: u64,
     pub total_delinquent_stake: u64,
-    pub current_validators: Vec<CliValidator>,
-    pub delinquent_validators: Vec<CliValidator>,
+    pub validators: Vec<CliValidator>,
+    #[serde(skip_serializing)]
+    pub validators_sort_order: CliValidatorsSortOrder,
+    #[serde(skip_serializing)]
+    pub validators_reverse_sort: bool,
+    #[serde(skip_serializing)]
+    pub number_validators: bool,
     pub stake_by_version: BTreeMap<String, CliValidatorsStakeByVersion>,
     #[serde(skip_serializing)]
     pub use_lamports_unit: bool,
@@ -330,30 +344,40 @@ impl fmt::Display for CliValidators {
             validator: &CliValidator,
             total_active_stake: u64,
             use_lamports_unit: bool,
-            delinquent: bool,
+            highest_last_vote: u64,
+            highest_root: u64,
         ) -> fmt::Result {
-            fn non_zero_or_dash(v: u64) -> String {
+            fn non_zero_or_dash(v: u64, max_v: u64) -> String {
                 if v == 0 {
-                    "-".into()
+                    "-         ".into()
+                } else if v == max_v {
+                    format!("{:>8} (  0)", v)
+                } else if v > max_v.saturating_sub(100) {
+                    format!("{:>8} ({:>3})", v, -(max_v.saturating_sub(v) as isize))
                 } else {
-                    format!("{}", v)
+                    format!("{:>8}      ", v)
                 }
             }
 
             writeln!(
                 f,
-                "{} {:<44}  {:<44}  {:>3}%   {:>8}  {:>10}  {:>10}  {:>8}  {}",
-                if delinquent {
+                "{} {:<44}  {:<44}  {:>3}%  {:>14}  {:>14} {:>7} {:>8}  {:>7}  {}",
+                if validator.delinquent {
                     WARNING.to_string()
                 } else {
-                    " ".to_string()
+                    "\u{a0}".to_string()
                 },
                 validator.identity_pubkey,
                 validator.vote_account_pubkey,
                 validator.commission,
-                non_zero_or_dash(validator.last_vote),
-                non_zero_or_dash(validator.root_slot),
-                validator.credits,
+                non_zero_or_dash(validator.last_vote, highest_last_vote),
+                non_zero_or_dash(validator.root_slot, highest_root),
+                if let Some(skip_rate) = validator.skip_rate {
+                    format!("{:.2}%", skip_rate)
+                } else {
+                    "- ".to_string()
+                },
+                validator.epoch_credits,
                 validator.version,
                 if validator.activated_stake > 0 {
                     format!(
@@ -366,39 +390,100 @@ impl fmt::Display for CliValidators {
                 },
             )
         }
-        writeln!(
-            f,
-            "{}",
-            style(format!(
-                "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
-                "Identity",
-                "Vote Account",
-                "Commission",
-                "Last Vote",
-                "Root Block",
-                "Credits",
-                "Version",
-                "Active Stake",
-            ))
-            .bold()
-        )?;
-        for validator in &self.current_validators {
+
+        let padding = if self.number_validators {
+            ((self.validators.len() + 1) as f64).log10().floor() as usize + 1
+        } else {
+            0
+        };
+        let header = style(format!(
+            "{:padding$} {:<44}  {:<38}  {}  {}  {} {}  {}  {}  {}",
+            " ",
+            "Identity",
+            "Vote Account",
+            "Commission",
+            "Last Vote     ",
+            "Root Slot   ",
+            "Skip Rate",
+            "Credits",
+            "Version",
+            "Active Stake",
+            padding = padding + 1
+        ))
+        .bold();
+        writeln!(f, "{}", header)?;
+
+        let mut sorted_validators = self.validators.clone();
+        match self.validators_sort_order {
+            CliValidatorsSortOrder::Delinquent => {
+                sorted_validators.sort_by_key(|a| a.delinquent);
+            }
+            CliValidatorsSortOrder::Commission => {
+                sorted_validators.sort_by_key(|a| a.commission);
+            }
+            CliValidatorsSortOrder::EpochCredits => {
+                sorted_validators.sort_by_key(|a| a.epoch_credits);
+            }
+            CliValidatorsSortOrder::Identity => {
+                sorted_validators.sort_by(|a, b| a.identity_pubkey.cmp(&b.identity_pubkey));
+            }
+            CliValidatorsSortOrder::LastVote => {
+                sorted_validators.sort_by_key(|a| a.last_vote);
+            }
+            CliValidatorsSortOrder::Root => {
+                sorted_validators.sort_by_key(|a| a.root_slot);
+            }
+            CliValidatorsSortOrder::VoteAccount => {
+                sorted_validators.sort_by(|a, b| a.vote_account_pubkey.cmp(&b.vote_account_pubkey));
+            }
+            CliValidatorsSortOrder::SkipRate => {
+                sorted_validators.sort_by(|a, b| {
+                    use std::cmp::Ordering;
+                    match (a.skip_rate, b.skip_rate) {
+                        (None, None) => Ordering::Equal,
+                        (None, Some(_)) => Ordering::Greater,
+                        (Some(_), None) => Ordering::Less,
+                        (Some(a), Some(b)) => a.partial_cmp(&b).unwrap_or(Ordering::Equal),
+                    }
+                });
+            }
+            CliValidatorsSortOrder::Stake => {
+                sorted_validators.sort_by_key(|a| a.activated_stake);
+            }
+        }
+
+        if self.validators_reverse_sort {
+            sorted_validators.reverse();
+        }
+
+        let highest_root = sorted_validators
+            .iter()
+            .map(|v| v.root_slot)
+            .max()
+            .unwrap_or_default();
+        let highest_last_vote = sorted_validators
+            .iter()
+            .map(|v| v.last_vote)
+            .max()
+            .unwrap_or_default();
+
+        for (i, validator) in sorted_validators.iter().enumerate() {
+            if padding > 0 {
+                write!(f, "{:padding$}", i + 1, padding = padding)?;
+            }
             write_vote_account(
                 f,
                 validator,
                 self.total_active_stake,
                 self.use_lamports_unit,
-                false,
+                highest_last_vote,
+                highest_root,
             )?;
         }
-        for validator in &self.delinquent_validators {
-            write_vote_account(
-                f,
-                validator,
-                self.total_active_stake,
-                self.use_lamports_unit,
-                true,
-            )?;
+
+        // The actual header has long scrolled away.  Print the header once more as a footer
+        if self.validators.len() > 100 {
+            writeln!(f, "{}", header)?;
         }
 
         writeln!(f)?;
@@ -457,7 +542,7 @@ impl fmt::Display for CliValidators {
     }
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct CliValidator {
     pub identity_pubkey: String,
@@ -465,9 +550,12 @@ pub struct CliValidator {
     pub commission: u8,
     pub last_vote: u64,
     pub root_slot: u64,
-    pub credits: u64,
+    pub credits: u64,       // lifetime credits
+    pub epoch_credits: u64, // credits earned in the current epoch
     pub activated_stake: u64,
     pub version: String,
+    pub delinquent: bool,
+    pub skip_rate: Option<f64>,
 }
 
 impl CliValidator {
@@ -475,27 +563,67 @@ impl CliValidator {
         vote_account: &RpcVoteAccountInfo,
         current_epoch: Epoch,
         version: String,
+        skip_rate: Option<f64>,
         address_labels: &HashMap<String, String>,
     ) -> Self {
+        Self::_new(
+            vote_account,
+            current_epoch,
+            version,
+            skip_rate,
+            address_labels,
+            false,
+        )
+    }
+
+    pub fn new_delinquent(
+        vote_account: &RpcVoteAccountInfo,
+        current_epoch: Epoch,
+        version: String,
+        skip_rate: Option<f64>,
+        address_labels: &HashMap<String, String>,
+    ) -> Self {
+        Self::_new(
+            vote_account,
+            current_epoch,
+            version,
+            skip_rate,
+            address_labels,
+            true,
+        )
+    }
+
+    fn _new(
+        vote_account: &RpcVoteAccountInfo,
+        current_epoch: Epoch,
+        version: String,
+        skip_rate: Option<f64>,
+        address_labels: &HashMap<String, String>,
+        delinquent: bool,
+    ) -> Self {
+        let (credits, epoch_credits) = vote_account
+            .epoch_credits
+            .iter()
+            .find_map(|(epoch, credits, pre_credits)| {
+                if *epoch == current_epoch {
+                    Some((*credits, credits.saturating_sub(*pre_credits)))
+                } else {
+                    None
+                }
+            })
+            .unwrap_or((0, 0));
         Self {
             identity_pubkey: format_labeled_address(&vote_account.node_pubkey, address_labels),
             vote_account_pubkey: format_labeled_address(&vote_account.vote_pubkey, address_labels),
             commission: vote_account.commission,
             last_vote: vote_account.last_vote,
             root_slot: vote_account.root_slot,
-            credits: vote_account
-                .epoch_credits
-                .iter()
-                .find_map(|(epoch, credits, _)| {
-                    if *epoch == current_epoch {
-                        Some(*credits)
-                    } else {
-                        None
-                    }
-                })
-                .unwrap_or(0),
+            credits,
+            epoch_credits,
             activated_stake: vote_account.activated_stake,
             version,
+            delinquent,
+            skip_rate,
         }
     }
 }
@@ -607,6 +735,76 @@ pub struct CliEpochReward {
     pub apr: Option<f64>,
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliKeyedEpochReward {
+    pub address: String,
+    pub reward: Option<CliEpochReward>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliEpochRewardshMetadata {
+    pub epoch: Epoch,
+    pub effective_slot: Slot,
+    pub block_time: UnixTimestamp,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliKeyedEpochRewards {
+    #[serde(flatten, skip_serializing_if = "Option::is_none")]
+    pub epoch_metadata: Option<CliEpochRewardshMetadata>,
+    pub rewards: Vec<CliKeyedEpochReward>,
+}
+
+impl QuietDisplay for CliKeyedEpochRewards {}
+impl VerboseDisplay for CliKeyedEpochRewards {}
+
+impl fmt::Display for CliKeyedEpochRewards {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        if self.rewards.is_empty() {
+            writeln!(f, "No rewards found in epoch")?;
+            return Ok(());
+        }
+
+        if let Some(metadata) = &self.epoch_metadata {
+            writeln!(f, "Epoch: {}", metadata.epoch)?;
+            writeln!(f, "Reward Slot: {}", metadata.effective_slot)?;
+            let timestamp = metadata.block_time;
+            writeln!(f, "Block Time: {}", unix_timestamp_to_string(timestamp))?;
+        }
+        writeln!(f, "Epoch Rewards:")?;
+        writeln!(
+            f,
+            "  {:<44}  {:<18}  {:<18}  {:>14}  {:>14}",
+            "Address", "Amount", "New Balance", "Percent Change", "APR"
+        )?;
+        for keyed_reward in &self.rewards {
+            match &keyed_reward.reward {
+                Some(reward) => {
+                    writeln!(
+                        f,
+                        "  {:<44}  ◎{:<17.9}  ◎{:<17.9}  {:>13.2}%  {}",
+                        keyed_reward.address,
+                        lamports_to_sol(reward.amount),
+                        lamports_to_sol(reward.post_balance),
+                        reward.percent_change,
+                        reward
+                            .apr
+                            .map(|apr| format!("{:>13.2}%", apr))
+                            .unwrap_or_default(),
+                    )?;
+                }
+                None => {
+                    writeln!(f, "  {:<44}  No rewards in epoch", keyed_reward.address,)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
 fn show_votes_and_credits(
     f: &mut fmt::Formatter,
     votes: &[CliLockout],
@@ -712,13 +910,13 @@ fn show_epoch_rewards(
         writeln!(f, "Epoch Rewards:")?;
         writeln!(
             f,
-            "  {:<6}  {:<11}  {:<16}  {:<16}  {:>14}  {:>14}",
+            "  {:<6}  {:<11}  {:<18}  {:<18}  {:>14}  {:>14}",
             "Epoch", "Reward Slot", "Amount", "New Balance", "Percent Change", "APR"
         )?;
         for reward in epoch_rewards {
             writeln!(
                 f,
-                "  {:<6}  {:<11}  ◎{:<16.9}  ◎{:<14.9}  {:>13.2}%  {}",
+                "  {:<6}  {:<11}  ◎{:<17.9}  ◎{:<17.9}  {:>13.2}%  {}",
                 reward.epoch,
                 reward.effective_slot,
                 lamports_to_sol(reward.amount),
@@ -1263,18 +1461,18 @@ impl fmt::Display for CliInflation {
         if (self.governor.initial - self.governor.terminal).abs() < f64::EPSILON {
             writeln!(
                 f,
-                "Fixed APR:               {:>5.2}%",
+                "Fixed rate:              {:>5.2}%",
                 self.governor.terminal * 100.
             )?;
         } else {
             writeln!(
                 f,
-                "Initial APR:             {:>5.2}%",
+                "Initial rate:            {:>5.2}%",
                 self.governor.initial * 100.
             )?;
             writeln!(
                 f,
-                "Terminal APR:            {:>5.2}%",
+                "Terminal rate:           {:>5.2}%",
                 self.governor.terminal * 100.
             )?;
             writeln!(
@@ -1282,6 +1480,10 @@ impl fmt::Display for CliInflation {
                 "Rate reduction per year: {:>5.2}%",
                 self.governor.taper * 100.
             )?;
+            writeln!(
+                f,
+                "* Rate reduction is derived using the target slot time in genesis config"
+            )?;
         }
         if self.governor.foundation_term > 0. {
             writeln!(
@@ -1303,17 +1505,17 @@ impl fmt::Display for CliInflation {
         )?;
         writeln!(
             f,
-            "Total APR:               {:>5.2}%",
+            "Total rate:              {:>5.2}%",
             self.current_rate.total * 100.
         )?;
         writeln!(
             f,
-            "Staking APR:             {:>5.2}%",
+            "Staking rate:            {:>5.2}%",
             self.current_rate.validator * 100.
         )?;
         writeln!(
             f,
-            "Foundation APR:          {:>5.2}%",
+            "Foundation rate:         {:>5.2}%",
             self.current_rate.foundation * 100.
         )
     }
@@ -1323,6 +1525,8 @@ impl fmt::Display for CliInflation {
 #[serde(rename_all = "camelCase")]
 pub struct CliSignOnlyData {
     pub blockhash: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub message: Option<String>,
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
     pub signers: Vec<String>,
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
@@ -1338,6 +1542,9 @@ impl fmt::Display for CliSignOnlyData {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln!(f)?;
         writeln_name_value(f, "Blockhash:", &self.blockhash)?;
+        if let Some(message) = self.message.as_ref() {
+            writeln_name_value(f, "Transaction Message:", message)?;
+        }
         if !self.signers.is_empty() {
             writeln!(f, "{}", style("Signers (Pubkey=Signature):").bold())?;
             for signer in self.signers.iter() {
@@ -1391,7 +1598,7 @@ impl fmt::Display for CliAccountBalances {
         writeln!(
             f,
             "{}",
-            style(format!("{:<44}  {}", "Address", "Balance",)).bold()
+            style(format!("{:<44}  {}", "Address", "Balance")).bold()
         )?;
         for account in &self.accounts {
             writeln!(
@@ -1683,6 +1890,9 @@ pub struct CliUpgradeableBuffer {
     pub address: String,
     pub authority: String,
     pub data_len: usize,
+    pub lamports: u64,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
 }
 impl QuietDisplay for CliUpgradeableBuffer {}
 impl VerboseDisplay for CliUpgradeableBuffer {}
@@ -1691,18 +1901,74 @@ impl fmt::Display for CliUpgradeableBuffer {
         writeln!(f)?;
         writeln_name_value(f, "Buffer Address:", &self.address)?;
         writeln_name_value(f, "Authority:", &self.authority)?;
+        writeln_name_value(
+            f,
+            "Balance:",
+            &build_balance_message(self.lamports, self.use_lamports_unit, true),
+        )?;
         writeln_name_value(
             f,
             "Data Length:",
             &format!("{:?} ({:#x?}) bytes", self.data_len, self.data_len),
         )?;
+
         Ok(())
     }
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliUpgradeableBuffers {
+    pub buffers: Vec<CliUpgradeableBuffer>,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
+}
+impl QuietDisplay for CliUpgradeableBuffers {}
+impl VerboseDisplay for CliUpgradeableBuffers {}
+impl fmt::Display for CliUpgradeableBuffers {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln!(
+            f,
+            "{}",
+            style(format!(
+                "{:<44} | {:<44} | {}",
+                "Buffer Address", "Authority", "Balance"
+            ))
+            .bold()
+        )?;
+        for buffer in self.buffers.iter() {
+            writeln!(
+                f,
+                "{}",
+                &format!(
+                    "{:<44} | {:<44} | {}",
+                    buffer.address,
+                    buffer.authority,
+                    build_balance_message(buffer.lamports, self.use_lamports_unit, true)
+                )
+            )?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct ReturnSignersConfig {
+    pub dump_transaction_message: bool,
+}
+
 pub fn return_signers(
     tx: &Transaction,
     output_format: &OutputFormat,
+) -> Result<String, Box<dyn std::error::Error>> {
+    return_signers_with_config(tx, output_format, &ReturnSignersConfig::default())
+}
+
+pub fn return_signers_with_config(
+    tx: &Transaction,
+    output_format: &OutputFormat,
+    config: &ReturnSignersConfig,
 ) -> Result<String, Box<dyn std::error::Error>> {
     let verify_results = tx.verify_with_results();
     let mut signers = Vec::new();
@@ -1721,9 +1987,16 @@ pub fn return_signers(
                 bad_sig.push(key.to_string());
             }
         });
+    let message = if config.dump_transaction_message {
+        let message_data = tx.message_data();
+        Some(base64::encode(&message_data))
+    } else {
+        None
+    };
 
     let cli_command = CliSignOnlyData {
         blockhash: tx.message.recent_blockhash.to_string(),
+        message,
         signers,
         absent,
         bad_sig,
@@ -1778,8 +2051,14 @@ pub fn parse_sign_only_reply_string(reply: &str) -> SignOnly {
             .collect();
     }
 
+    let message = object
+        .get("message")
+        .and_then(|o| o.as_str())
+        .map(|m| m.to_string());
+
     SignOnly {
         blockhash,
+        message,
         present_signers,
         absent_signers,
         bad_signers,
@@ -2058,6 +2337,25 @@ mod tests {
         let res = return_signers(&tx, &OutputFormat::JsonCompact).unwrap();
         let sign_only = parse_sign_only_reply_string(&res);
         assert_eq!(sign_only.blockhash, blockhash);
+        assert_eq!(sign_only.message, None);
+        assert_eq!(sign_only.present_signers[0].0, present.pubkey());
+        assert_eq!(sign_only.absent_signers[0], absent.pubkey());
+        assert_eq!(sign_only.bad_signers[0], bad.pubkey());
+
+        let expected_msg = "AwECBwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDgTl3Dqh9\
+            F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE\
+            BAQEBAYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBQUFBQUFBQUFBQUFBQUFBQUF\
+            BQUFBQUFBQUFBQUFBQUGp9UXGSxWjuCKhF9z0peIzwNcMUWyGrNE2AYuqUAAAAAAAAAAAAAA\
+            AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcH\
+            BwcCBgMDBQIEBAAAAAYCAQQMAgAAACoAAAAAAAAA"
+            .to_string();
+        let config = ReturnSignersConfig {
+            dump_transaction_message: true,
+        };
+        let res = return_signers_with_config(&tx, &OutputFormat::JsonCompact, &config).unwrap();
+        let sign_only = parse_sign_only_reply_string(&res);
+        assert_eq!(sign_only.blockhash, blockhash);
+        assert_eq!(sign_only.message, Some(expected_msg));
         assert_eq!(sign_only.present_signers[0].0, present.pubkey());
         assert_eq!(sign_only.absent_signers[0], absent.pubkey());
         assert_eq!(sign_only.bad_signers[0], bad.pubkey());

cli-output/src/display.rs

@@ -4,10 +4,12 @@ use {
     console::style,
     indicatif::{ProgressBar, ProgressStyle},
     solana_sdk::{
-        clock::UnixTimestamp, hash::Hash, native_token::lamports_to_sol,
-        program_utils::limited_deserialize, transaction::Transaction,
+        clock::UnixTimestamp, hash::Hash, message::Message, native_token::lamports_to_sol,
+        program_utils::limited_deserialize, pubkey::Pubkey, transaction::Transaction,
     },
     solana_transaction_status::UiTransactionStatusMeta,
+    spl_memo::id as spl_memo_id,
+    spl_memo::v1::id as spl_memo_v1_id,
     std::{collections::HashMap, fmt, io},
 };
 
@@ -28,6 +30,11 @@ impl Default for BuildBalanceMessageConfig {
     }
 }
 
+fn is_memo_program(k: &Pubkey) -> bool {
+    let k_str = k.to_string();
+    (k_str == spl_memo_v1_id().to_string()) || (k_str == spl_memo_id().to_string())
+}
+
 pub fn build_balance_message_with_config(
     lamports: u64,
     config: &BuildBalanceMessageConfig,
@@ -125,6 +132,31 @@ pub fn println_signers(
     println!();
 }
 
+fn format_account_mode(message: &Message, index: usize) -> String {
+    format!(
+        "{}r{}{}", // accounts are always readable...
+        if message.is_signer(index) {
+            "s" // stands for signer
+        } else {
+            "-"
+        },
+        if message.is_writable(index, /*demote_sysvar_write_locks=*/ true) {
+            "w" // comment for consistent rust fmt (no joking; lol)
+        } else {
+            "-"
+        },
+        // account may be executable on-chain while not being
+        // designated as a program-id in the message
+        if message.maybe_executable(index) {
+            "x"
+        } else {
+            // programs to be executed via CPI cannot be identified as
+            // executable from the message
+            "-"
+        },
+    )
+}
+
 pub fn write_transaction<W: io::Write>(
     w: &mut W,
     transaction: &Transaction,
@@ -167,16 +199,31 @@ pub fn write_transaction<W: io::Write>(
             prefix, signature_index, signature, sigverify_status,
         )?;
     }
-    writeln!(w, "{}{:?}", prefix, message.header)?;
+    let mut fee_payer_index = None;
     for (account_index, account) in message.account_keys.iter().enumerate() {
-        writeln!(w, "{}Account {}: {:?}", prefix, account_index, account)?;
+        if fee_payer_index.is_none() && message.is_non_loader_key(account, account_index) {
+            fee_payer_index = Some(account_index)
+        }
+        writeln!(
+            w,
+            "{}Account {}: {} {}{}",
+            prefix,
+            account_index,
+            format_account_mode(message, account_index),
+            account,
+            if Some(account_index) == fee_payer_index {
+                " (fee payer)"
+            } else {
+                ""
+            },
+        )?;
     }
     for (instruction_index, instruction) in message.instructions.iter().enumerate() {
         let program_pubkey = message.account_keys[instruction.program_id_index as usize];
         writeln!(w, "{}Instruction {}", prefix, instruction_index)?;
         writeln!(
             w,
-            "{}  Program: {} ({})",
+            "{}  Program:   {} ({})",
             prefix, program_pubkey, instruction.program_id_index
         )?;
         for (account_index, account) in instruction.accounts.iter().enumerate() {
@@ -213,6 +260,11 @@ pub fn write_transaction<W: io::Write>(
                 writeln!(w, "{}  {:?}", prefix, system_instruction)?;
                 raw = false;
             }
+        } else if is_memo_program(&program_pubkey) {
+            if let Ok(s) = std::str::from_utf8(&instruction.data) {
+                writeln!(w, "{}  Data: \"{}\"", prefix, s)?;
+                raw = false;
+            }
         }
 
         if raw {

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.0"
+version = "1.6.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,33 +24,34 @@ indicatif = "0.15.0"
 humantime = "2.0.1"
 num-traits = "0.2"
 pretty-hex = "0.2.1"
-reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.6.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.6.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-cli-config = { path = "../cli-config", version = "1.6.0" }
-solana-cli-output = { path = "../cli-output", version = "1.6.0" }
-solana-client = { path = "../client", version = "1.6.0" }
-solana-config-program = { path = "../programs/config", version = "1.6.0" }
-solana-faucet = { path = "../faucet", version = "1.6.0" }
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-net-utils = { path = "../net-utils", version = "1.6.0" }
-solana_rbpf = "=0.2.5"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.6.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.6.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.7" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.6.7" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-cli-config = { path = "../cli-config", version = "=1.6.7" }
+solana-cli-output = { path = "../cli-output", version = "=1.6.7" }
+solana-client = { path = "../client", version = "=1.6.7" }
+solana-config-program = { path = "../programs/config", version = "=1.6.7" }
+solana-faucet = { path = "../faucet", version = "=1.6.7" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.7" }
+solana_rbpf = "=0.2.8"
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.7" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.7" }
+spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.6.0" }
+solana-core = { path = "../core", version = "=1.6.7" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cluster_query.rs

@@ -24,8 +24,9 @@ use solana_client::{
     pubsub_client::PubsubClient,
     rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClient},
     rpc_config::{
-        RpcAccountInfoConfig, RpcLargestAccountsConfig, RpcLargestAccountsFilter,
-        RpcProgramAccountsConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter,
+        RpcAccountInfoConfig, RpcConfirmedBlockConfig, RpcConfirmedTransactionConfig,
+        RpcLargestAccountsConfig, RpcLargestAccountsFilter, RpcProgramAccountsConfig,
+        RpcTransactionLogsConfig, RpcTransactionLogsFilter,
     },
     rpc_filter,
     rpc_response::SlotInfo,
@@ -40,23 +41,28 @@ use solana_sdk::{
     hash::Hash,
     message::Message,
     native_token::lamports_to_sol,
+    nonce::State as NonceState,
     pubkey::{self, Pubkey},
     rent::Rent,
     rpc_port::DEFAULT_RPC_PORT_STR,
     signature::Signature,
-    system_instruction, system_program,
+    slot_history, system_instruction, system_program,
     sysvar::{
         self,
+        slot_history::SlotHistory,
         stake_history::{self},
     },
     timing,
     transaction::Transaction,
 };
+use solana_stake_program::stake_state::StakeState;
 use solana_transaction_status::UiTransactionEncoding;
+use solana_vote_program::vote_state::VoteState;
 use std::{
     collections::{BTreeMap, HashMap, VecDeque},
     fmt,
     net::SocketAddr,
+    str::FromStr,
     sync::{
         atomic::{AtomicBool, Ordering},
         Arc,
@@ -64,6 +70,7 @@ use std::{
     thread::sleep,
     time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
+use thiserror::Error;
 
 static CHECK_MARK: Emoji = Emoji("✅ ", "");
 static CROSS_MARK: Emoji = Emoji("❌ ", "");
@@ -343,6 +350,38 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
+                )
+                .arg(
+                    Arg::with_name("number")
+                        .long("number")
+                        .short("n")
+                        .takes_value(false)
+                        .help("Number the validators"),
+                )
+                .arg(
+                    Arg::with_name("reverse")
+                        .long("reverse")
+                        .short("r")
+                        .takes_value(false)
+                        .help("Reverse order while sorting"),
+                )
+                .arg(
+                    Arg::with_name("sort")
+                        .long("sort")
+                        .takes_value(true)
+                        .possible_values(&[
+                            "delinquent",
+                            "commission",
+                            "credits",
+                            "identity",
+                            "last-vote",
+                            "root",
+                            "skip-rate",
+                            "stake",
+                            "vote-account",
+                        ])
+                        .default_value("stake")
+                        .help("Sort order (does not affect JSON output)"),
                 ),
         )
         .subcommand(
@@ -396,9 +435,14 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("data_length")
                         .index(1)
-                        .value_name("DATA_LENGTH")
+                        .value_name("DATA_LENGTH_OR_MONIKER")
                         .required(true)
-                        .help("Length of data in the account to calculate rent for"),
+                        .validator(|s| {
+                            RentLengthValue::from_str(&s)
+                                .map(|_| ())
+                                .map_err(|e| e.to_string())
+                        })
+                        .help("Length of data in the account to calculate rent for, or moniker: [nonce, stake, system, vote]"),
                 )
                 .arg(
                     Arg::with_name("lamports")
@@ -571,9 +615,29 @@ pub fn parse_show_stakes(
 
 pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
+    let number_validators = matches.is_present("number");
+    let reverse_sort = matches.is_present("reverse");
+
+    let sort_order = match value_t_or_exit!(matches, "sort", String).as_str() {
+        "delinquent" => CliValidatorsSortOrder::Delinquent,
+        "commission" => CliValidatorsSortOrder::Commission,
+        "credits" => CliValidatorsSortOrder::EpochCredits,
+        "identity" => CliValidatorsSortOrder::Identity,
+        "last-vote" => CliValidatorsSortOrder::LastVote,
+        "root" => CliValidatorsSortOrder::Root,
+        "skip-rate" => CliValidatorsSortOrder::SkipRate,
+        "stake" => CliValidatorsSortOrder::Stake,
+        "vote-account" => CliValidatorsSortOrder::VoteAccount,
+        _ => unreachable!(),
+    };
 
     Ok(CliCommandInfo {
-        command: CliCommand::ShowValidators { use_lamports_unit },
+        command: CliCommand::ShowValidators {
+            use_lamports_unit,
+            sort_order,
+            reverse_sort,
+            number_validators,
+        },
         signers: vec![],
     })
 }
@@ -963,8 +1027,16 @@ pub fn process_get_block(
         rpc_client.get_slot_with_commitment(CommitmentConfig::finalized())?
     };
 
-    let encoded_confirmed_block =
-        rpc_client.get_confirmed_block_with_encoding(slot, UiTransactionEncoding::Base64)?;
+    let encoded_confirmed_block = rpc_client
+        .get_confirmed_block_with_config(
+            slot,
+            RpcConfirmedBlockConfig {
+                encoding: Some(UiTransactionEncoding::Base64),
+                commitment: Some(CommitmentConfig::confirmed()),
+                ..RpcConfirmedBlockConfig::default()
+            },
+        )?
+        .into();
     let cli_block = CliBlock {
         encoded_confirmed_block,
         slot,
@@ -993,7 +1065,21 @@ pub fn process_get_epoch(rpc_client: &RpcClient, _config: &CliConfig) -> Process
 }
 
 pub fn process_get_epoch_info(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
-    let epoch_info: CliEpochInfo = rpc_client.get_epoch_info()?.into();
+    let epoch_info = rpc_client.get_epoch_info()?;
+    let average_slot_time_ms = rpc_client
+        .get_recent_performance_samples(Some(60))
+        .ok()
+        .and_then(|samples| {
+            let (slots, secs) = samples.iter().fold((0, 0), |(slots, secs), sample| {
+                (slots + sample.num_slots, secs + sample.sample_period_secs)
+            });
+            (secs as u64).saturating_mul(1000).checked_div(slots)
+        })
+        .unwrap_or(clock::DEFAULT_MS_PER_SLOT);
+    let epoch_info = CliEpochInfo {
+        epoch_info,
+        average_slot_time_ms,
+    };
     Ok(config.output_format.formatted_string(&epoch_info))
 }
 
@@ -1008,8 +1094,8 @@ pub fn process_get_slot(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessR
 }
 
 pub fn process_get_block_height(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
-    let epoch_info: CliEpochInfo = rpc_client.get_epoch_info()?.into();
-    Ok(epoch_info.epoch_info.block_height.to_string())
+    let epoch_info = rpc_client.get_epoch_info()?;
+    Ok(epoch_info.block_height.to_string())
 }
 
 pub fn parse_show_block_production(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
@@ -1036,8 +1122,6 @@ pub fn process_show_block_production(
         return Err(format!("Epoch {} is in the future", epoch).into());
     }
 
-    let minimum_ledger_slot = rpc_client.minimum_ledger_slot()?;
-
     let first_slot_in_epoch = epoch_schedule.get_first_slot_in_epoch(epoch);
     let end_slot = std::cmp::min(
         epoch_info.absolute_slot,
@@ -1050,32 +1134,60 @@ pub fn process_show_block_production(
         first_slot_in_epoch
     };
 
-    if minimum_ledger_slot > end_slot {
-        return Err(format!(
-            "Ledger data not available for slots {} to {} (minimum ledger slot is {})",
-            start_slot, end_slot, minimum_ledger_slot
-        )
-        .into());
-    }
-
-    if minimum_ledger_slot > start_slot {
-        println!(
-            "\n{}",
-            style(format!(
-                "Note: Requested start slot was {} but minimum ledger slot is {}",
-                start_slot, minimum_ledger_slot
-            ))
-            .italic(),
-        );
-        start_slot = minimum_ledger_slot;
-    }
-
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message(&format!(
         "Fetching confirmed blocks between slots {} and {}...",
         start_slot, end_slot
     ));
-    let confirmed_blocks = rpc_client.get_confirmed_blocks(start_slot, Some(end_slot))?;
+
+    let slot_history_account = rpc_client
+        .get_account_with_commitment(&sysvar::slot_history::id(), CommitmentConfig::finalized())?
+        .value
+        .unwrap();
+
+    let slot_history: SlotHistory = from_account(&slot_history_account).ok_or_else(|| {
+        CliError::RpcRequestError("Failed to deserialize slot history".to_string())
+    })?;
+
+    let (confirmed_blocks, start_slot) =
+        if start_slot >= slot_history.oldest() && end_slot <= slot_history.newest() {
+            // Fast, more reliable path using the SlotHistory sysvar
+
+            let confirmed_blocks: Vec<_> = (start_slot..=end_slot)
+                .filter(|slot| slot_history.check(*slot) == slot_history::Check::Found)
+                .collect();
+            (confirmed_blocks, start_slot)
+        } else {
+            // Slow, less reliable path using `getBlocks`.
+            //
+            // "less reliable" because if the RPC node has holds in its ledger then the block production data will be
+            // incorrect.  This condition currently can't be detected over RPC
+            //
+
+            let minimum_ledger_slot = rpc_client.minimum_ledger_slot()?;
+            if minimum_ledger_slot > end_slot {
+                return Err(format!(
+                    "Ledger data not available for slots {} to {} (minimum ledger slot is {})",
+                    start_slot, end_slot, minimum_ledger_slot
+                )
+                .into());
+            }
+
+            if minimum_ledger_slot > start_slot {
+                progress_bar.println(format!(
+                    "{}",
+                    style(format!(
+                        "Note: Requested start slot was {} but minimum ledger slot is {}",
+                        start_slot, minimum_ledger_slot
+                    ))
+                    .italic(),
+                ));
+                start_slot = minimum_ledger_slot;
+            }
+
+            let confirmed_blocks = rpc_client.get_confirmed_blocks(start_slot, Some(end_slot))?;
+            (confirmed_blocks, start_slot)
+        };
 
     let start_slot_index = (start_slot - first_slot_in_epoch) as usize;
     let end_slot_index = (end_slot - first_slot_in_epoch) as usize;
@@ -1201,8 +1313,8 @@ pub fn process_supply(
 }
 
 pub fn process_total_supply(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
-    let total_supply = rpc_client.total_supply()?;
-    Ok(format!("{} SOL", lamports_to_sol(total_supply)))
+    let supply = rpc_client.supply()?.value;
+    Ok(format!("{} SOL", lamports_to_sol(supply.total)))
 }
 
 pub fn process_get_transaction_count(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
@@ -1586,7 +1698,6 @@ pub fn process_show_stakes(
     vote_account_pubkeys: Option<&[Pubkey]>,
 ) -> ProcessResult {
     use crate::stake::build_stake_state;
-    use solana_stake_program::stake_state::StakeState;
 
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message("Fetching stake accounts...");
@@ -1698,10 +1809,36 @@ pub fn process_show_validators(
     rpc_client: &RpcClient,
     config: &CliConfig,
     use_lamports_unit: bool,
+    validators_sort_order: CliValidatorsSortOrder,
+    validators_reverse_sort: bool,
+    number_validators: bool,
 ) -> ProcessResult {
+    let progress_bar = new_spinner_progress_bar();
+    progress_bar.set_message("Fetching vote accounts...");
     let epoch_info = rpc_client.get_epoch_info()?;
     let vote_accounts = rpc_client.get_vote_accounts()?;
 
+    progress_bar.set_message("Fetching block production...");
+    let skip_rate: HashMap<_, _> = rpc_client
+        .get_block_production()
+        .ok()
+        .map(|result| {
+            result
+                .value
+                .by_identity
+                .into_iter()
+                .map(|(identity, (leader_slots, blocks_produced))| {
+                    (
+                        identity,
+                        100. * (leader_slots.saturating_sub(blocks_produced)) as f64
+                            / leader_slots as f64,
+                    )
+                })
+                .collect()
+        })
+        .unwrap_or_default();
+
+    progress_bar.set_message("Fetching version information...");
     let mut node_version = HashMap::new();
     let unknown_version = "unknown".to_string();
     for contact_info in rpc_client.get_cluster_nodes()? {
@@ -1713,6 +1850,8 @@ pub fn process_show_validators(
         );
     }
 
+    progress_bar.finish_and_clear();
+
     let total_active_stake = vote_accounts
         .current
         .iter()
@@ -1727,9 +1866,8 @@ pub fn process_show_validators(
         .sum();
     let total_current_stake = total_active_stake - total_delinquent_stake;
 
-    let mut current = vote_accounts.current;
-    current.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
-    let current_validators: Vec<CliValidator> = current
+    let current_validators: Vec<CliValidator> = vote_accounts
+        .current
         .iter()
         .map(|vote_account| {
             CliValidator::new(
@@ -1739,22 +1877,23 @@ pub fn process_show_validators(
                     .get(&vote_account.node_pubkey)
                     .unwrap_or(&unknown_version)
                     .clone(),
+                skip_rate.get(&vote_account.node_pubkey).cloned(),
                 &config.address_labels,
             )
         })
         .collect();
-    let mut delinquent = vote_accounts.delinquent;
-    delinquent.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
-    let delinquent_validators: Vec<CliValidator> = delinquent
+    let delinquent_validators: Vec<CliValidator> = vote_accounts
+        .delinquent
         .iter()
         .map(|vote_account| {
-            CliValidator::new(
+            CliValidator::new_delinquent(
                 vote_account,
                 epoch_info.epoch,
                 node_version
                     .get(&vote_account.node_pubkey)
                     .unwrap_or(&unknown_version)
                     .clone(),
+                skip_rate.get(&vote_account.node_pubkey).cloned(),
                 &config.address_labels,
             )
         })
@@ -1780,8 +1919,13 @@ pub fn process_show_validators(
         total_active_stake,
         total_current_stake,
         total_delinquent_stake,
-        current_validators,
-        delinquent_validators,
+        validators: current_validators
+            .into_iter()
+            .chain(delinquent_validators.into_iter())
+            .collect(),
+        validators_sort_order,
+        validators_reverse_sort,
+        number_validators,
         stake_by_version,
         use_lamports_unit,
     };
@@ -1803,6 +1947,7 @@ pub fn process_transaction_history(
             before,
             until,
             limit: Some(limit),
+            commitment: Some(CommitmentConfig::confirmed()),
         },
     )?;
 
@@ -1819,9 +1964,13 @@ pub fn process_transaction_history(
                     Some(block_time) =>
                         format!("timestamp={} ", unix_timestamp_to_string(block_time)),
                 },
-                match result.err {
-                    None => "Confirmed".to_string(),
-                    Some(err) => format!("Failed: {:?}", err),
+                if let Some(err) = result.err {
+                    format!("Failed: {:?}", err)
+                } else {
+                    match result.confirmation_status {
+                        None => "Finalized".to_string(),
+                        Some(status) => format!("{:?}", status),
+                    }
                 },
                 result.memo.unwrap_or_else(|| "".to_string()),
             );
@@ -1831,9 +1980,13 @@ pub fn process_transaction_history(
 
         if show_transactions {
             if let Ok(signature) = result.signature.parse::<Signature>() {
-                match rpc_client
-                    .get_confirmed_transaction(&signature, UiTransactionEncoding::Base64)
-                {
+                match rpc_client.get_confirmed_transaction_with_config(
+                    &signature,
+                    RpcConfirmedTransactionConfig {
+                        encoding: Some(UiTransactionEncoding::Base64),
+                        commitment: Some(CommitmentConfig::confirmed()),
+                    },
+                ) {
                     Ok(confirmed_transaction) => {
                         println_transaction(
                             &confirmed_transaction
@@ -1886,6 +2039,47 @@ impl fmt::Display for CliRentCalculation {
 impl QuietDisplay for CliRentCalculation {}
 impl VerboseDisplay for CliRentCalculation {}
 
+#[derive(Debug, PartialEq)]
+pub enum RentLengthValue {
+    Nonce,
+    Stake,
+    System,
+    Vote,
+    Bytes(usize),
+}
+
+impl RentLengthValue {
+    pub fn length(&self) -> usize {
+        match self {
+            Self::Nonce => NonceState::size(),
+            Self::Stake => std::mem::size_of::<StakeState>(),
+            Self::System => 0,
+            Self::Vote => VoteState::size_of(),
+            Self::Bytes(l) => *l,
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+#[error("expected number or moniker, got \"{0}\"")]
+pub struct RentLengthValueError(pub String);
+
+impl FromStr for RentLengthValue {
+    type Err = RentLengthValueError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let s = s.to_ascii_lowercase();
+        match s.as_str() {
+            "nonce" => Ok(Self::Nonce),
+            "stake" => Ok(Self::Stake),
+            "system" => Ok(Self::System),
+            "vote" => Ok(Self::Vote),
+            _ => usize::from_str(&s)
+                .map(Self::Bytes)
+                .map_err(|_| RentLengthValueError(s)),
+        }
+    }
+}
+
 pub fn process_calculate_rent(
     rpc_client: &RpcClient,
     config: &CliConfig,

cli/src/inflation.rs

@@ -1,14 +1,22 @@
 use crate::cli::{CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult};
-use clap::{App, ArgMatches, SubCommand};
-use solana_clap_utils::keypair::*;
-use solana_cli_output::CliInflation;
+use clap::{App, Arg, ArgMatches, SubCommand};
+use solana_clap_utils::{
+    input_parsers::{pubkeys_of, value_of},
+    input_validators::is_valid_pubkey,
+    keypair::*,
+};
+use solana_cli_output::{
+    CliEpochRewardshMetadata, CliInflation, CliKeyedEpochReward, CliKeyedEpochRewards,
+};
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
+use solana_sdk::{clock::Epoch, pubkey::Pubkey};
 use std::sync::Arc;
 
 #[derive(Debug, PartialEq)]
 pub enum InflationCliCommand {
     Show,
+    Rewards(Vec<Pubkey>, Option<Epoch>),
 }
 
 pub trait InflationSubCommands {
@@ -17,17 +25,47 @@ pub trait InflationSubCommands {
 
 impl InflationSubCommands for App<'_, '_> {
     fn inflation_subcommands(self) -> Self {
-        self.subcommand(SubCommand::with_name("inflation").about("Show inflation information"))
+        self.subcommand(
+            SubCommand::with_name("inflation")
+                .about("Show inflation information")
+                .subcommand(
+                    SubCommand::with_name("rewards")
+                        .about("Show inflation rewards for a set of addresses")
+                        .arg(pubkey!(
+                            Arg::with_name("addresses")
+                                .value_name("ADDRESS")
+                                .index(1)
+                                .multiple(true)
+                                .required(true),
+                            "Address of account to query for rewards. "
+                        ))
+                        .arg(
+                            Arg::with_name("rewards_epoch")
+                                .long("rewards-epoch")
+                                .takes_value(true)
+                                .value_name("EPOCH")
+                                .help("Display rewards for specific epoch [default: latest epoch]"),
+                        ),
+                ),
+        )
     }
 }
 
 pub fn parse_inflation_subcommand(
-    _matches: &ArgMatches<'_>,
+    matches: &ArgMatches<'_>,
     _default_signer: &DefaultSigner,
     _wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
+    let command = match matches.subcommand() {
+        ("rewards", Some(matches)) => {
+            let addresses = pubkeys_of(matches, "addresses").unwrap();
+            let rewards_epoch = value_of(matches, "rewards_epoch");
+            InflationCliCommand::Rewards(addresses, rewards_epoch)
+        }
+        _ => InflationCliCommand::Show,
+    };
     Ok(CliCommandInfo {
-        command: CliCommand::Inflation(InflationCliCommand::Show),
+        command: CliCommand::Inflation(command),
         signers: vec![],
     })
 }
@@ -37,8 +75,15 @@ pub fn process_inflation_subcommand(
     config: &CliConfig,
     inflation_subcommand: &InflationCliCommand,
 ) -> ProcessResult {
-    assert_eq!(*inflation_subcommand, InflationCliCommand::Show);
+    match inflation_subcommand {
+        InflationCliCommand::Show => process_show(rpc_client, config),
+        InflationCliCommand::Rewards(ref addresses, rewards_epoch) => {
+            process_rewards(rpc_client, config, addresses, *rewards_epoch)
+        }
+    }
+}
 
+fn process_show(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
     let governor = rpc_client.get_inflation_governor()?;
     let current_rate = rpc_client.get_inflation_rate()?;
 
@@ -49,3 +94,49 @@ pub fn process_inflation_subcommand(
 
     Ok(config.output_format.formatted_string(&inflation))
 }
+
+fn process_rewards(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    addresses: &[Pubkey],
+    rewards_epoch: Option<Epoch>,
+) -> ProcessResult {
+    let rewards = rpc_client
+        .get_inflation_reward(&addresses, rewards_epoch)
+        .map_err(|err| {
+            if let Some(epoch) = rewards_epoch {
+                format!("Rewards not available for epoch {}", epoch)
+            } else {
+                format!("Rewards not available {}", err)
+            }
+        })?;
+    let epoch_schedule = rpc_client.get_epoch_schedule()?;
+
+    let mut epoch_rewards: Vec<CliKeyedEpochReward> = vec![];
+    let epoch_metadata = if let Some(Some(first_reward)) = rewards.iter().find(|&v| v.is_some()) {
+        let (epoch_start_time, epoch_end_time) =
+            crate::stake::get_epoch_boundary_timestamps(rpc_client, first_reward, &epoch_schedule)?;
+        for (reward, address) in rewards.iter().zip(addresses) {
+            let cli_reward = reward.as_ref().and_then(|reward| {
+                crate::stake::make_cli_reward(reward, epoch_start_time, epoch_end_time)
+            });
+            epoch_rewards.push(CliKeyedEpochReward {
+                address: address.to_string(),
+                reward: cli_reward,
+            });
+        }
+        let block_time = rpc_client.get_block_time(first_reward.effective_slot)?;
+        Some(CliEpochRewardshMetadata {
+            epoch: first_reward.epoch,
+            effective_slot: first_reward.effective_slot,
+            block_time,
+        })
+    } else {
+        None
+    };
+    let cli_rewards = CliKeyedEpochRewards {
+        epoch_metadata,
+        rewards: epoch_rewards,
+    };
+    Ok(config.output_format.formatted_string(&cli_rewards))
+}

cli/src/lib.rs

@@ -26,9 +26,9 @@ pub mod cli;
 pub mod cluster_query;
 pub mod feature;
 pub mod inflation;
+pub mod memo;
 pub mod nonce;
 pub mod program;
-pub mod send_tpu;
 pub mod spend_utils;
 pub mod stake;
 pub mod test_utils;

cli/src/memo.rs

@@ -0,0 +1,22 @@
+use solana_sdk::instruction::Instruction;
+use solana_sdk::pubkey::Pubkey;
+use spl_memo::id;
+
+pub trait WithMemo {
+    fn with_memo<T: AsRef<str>>(self, memo: Option<T>) -> Self;
+}
+
+impl WithMemo for Vec<Instruction> {
+    fn with_memo<T: AsRef<str>>(mut self, memo: Option<T>) -> Self {
+        if let Some(memo) = &memo {
+            let memo = memo.as_ref();
+            let memo_ix = Instruction {
+                program_id: Pubkey::new(&id().to_bytes()),
+                accounts: vec![],
+                data: memo.as_bytes().to_vec(),
+            };
+            self.push(memo_ix);
+        }
+        self
+    }
+}

cli/src/nonce.rs

@@ -4,6 +4,7 @@ use crate::{
         log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError,
         ProcessResult,
     },
+    memo::WithMemo,
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
 };
 use clap::{App, Arg, ArgMatches, SubCommand};
@@ -11,6 +12,7 @@ use solana_clap_utils::{
     input_parsers::*,
     input_validators::*,
     keypair::{DefaultSigner, SignerIndex},
+    memo::MEMO_ARG,
     nonce::*,
 };
 use solana_cli_output::CliNonceAccount;
@@ -171,6 +173,7 @@ pub fn parse_authorize_nonce_account(
 ) -> Result<CliCommandInfo, CliError> {
     let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
     let new_authority = pubkey_of_signer(matches, "new_authority", wallet_manager)?.unwrap();
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
@@ -185,6 +188,7 @@ pub fn parse_authorize_nonce_account(
         command: CliCommand::AuthorizeNonceAccount {
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
+            memo,
             new_authority,
         },
         signers: signer_info.signers,
@@ -201,6 +205,7 @@ pub fn parse_nonce_create_account(
     let seed = matches.value_of("seed").map(|s| s.to_string());
     let amount = SpendAmount::new_from_matches(matches, "amount");
     let nonce_authority = pubkey_of_signer(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
 
     let payer_provided = None;
     let signer_info = default_signer.generate_unique_signers(
@@ -214,6 +219,7 @@ pub fn parse_nonce_create_account(
             nonce_account: signer_info.index_of(nonce_account_pubkey).unwrap(),
             seed,
             nonce_authority,
+            memo,
             amount,
         },
         signers: signer_info.signers,
@@ -239,6 +245,7 @@ pub fn parse_new_nonce(
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
@@ -253,6 +260,7 @@ pub fn parse_new_nonce(
         command: CliCommand::NewNonce {
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
+            memo,
         },
         signers: signer_info.signers,
     })
@@ -284,6 +292,7 @@ pub fn parse_withdraw_from_nonce_account(
     let destination_account_pubkey =
         pubkey_of_signer(matches, "destination_account_pubkey", wallet_manager)?.unwrap();
     let lamports = lamports_of_sol(matches, "amount").unwrap();
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
@@ -298,6 +307,7 @@ pub fn parse_withdraw_from_nonce_account(
         command: CliCommand::WithdrawFromNonceAccount {
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
+            memo,
             destination_account_pubkey,
             lamports,
         },
@@ -330,13 +340,19 @@ pub fn process_authorize_nonce_account(
     config: &CliConfig,
     nonce_account: &Pubkey,
     nonce_authority: SignerIndex,
+    memo: Option<&String>,
     new_authority: &Pubkey,
 ) -> ProcessResult {
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let nonce_authority = config.signers[nonce_authority];
-    let ix = authorize_nonce_account(nonce_account, &nonce_authority.pubkey(), new_authority);
-    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
+    let ixs = vec![authorize_nonce_account(
+        nonce_account,
+        &nonce_authority.pubkey(),
+        new_authority,
+    )]
+    .with_memo(memo);
+    let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
 
@@ -357,6 +373,7 @@ pub fn process_create_nonce_account(
     nonce_account: SignerIndex,
     seed: Option<String>,
     nonce_authority: Option<Pubkey>,
+    memo: Option<&String>,
     amount: SpendAmount,
 ) -> ProcessResult {
     let nonce_account_pubkey = config.signers[nonce_account].pubkey();
@@ -383,6 +400,7 @@ pub fn process_create_nonce_account(
                 &nonce_authority,
                 lamports,
             )
+            .with_memo(memo)
         } else {
             create_nonce_account(
                 &config.signers[0].pubkey(),
@@ -390,6 +408,7 @@ pub fn process_create_nonce_account(
                 &nonce_authority,
                 lamports,
             )
+            .with_memo(memo)
         };
         Message::new(&ixs, Some(&config.signers[0].pubkey()))
     };
@@ -451,6 +470,7 @@ pub fn process_new_nonce(
     config: &CliConfig,
     nonce_account: &Pubkey,
     nonce_authority: SignerIndex,
+    memo: Option<&String>,
 ) -> ProcessResult {
     check_unique_pubkeys(
         (&config.signers[0].pubkey(), "cli keypair".to_string()),
@@ -466,9 +486,13 @@ pub fn process_new_nonce(
     }
 
     let nonce_authority = config.signers[nonce_authority];
-    let ix = advance_nonce_account(&nonce_account, &nonce_authority.pubkey());
+    let ixs = vec![advance_nonce_account(
+        &nonce_account,
+        &nonce_authority.pubkey(),
+    )]
+    .with_memo(memo);
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee_with_commitment(
@@ -517,19 +541,21 @@ pub fn process_withdraw_from_nonce_account(
     config: &CliConfig,
     nonce_account: &Pubkey,
     nonce_authority: SignerIndex,
+    memo: Option<&String>,
     destination_account_pubkey: &Pubkey,
     lamports: u64,
 ) -> ProcessResult {
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let nonce_authority = config.signers[nonce_authority];
-    let ix = withdraw_nonce_account(
+    let ixs = vec![withdraw_nonce_account(
         nonce_account,
         &nonce_authority.pubkey(),
         destination_account_pubkey,
         lamports,
-    );
-    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
+    )]
+    .with_memo(memo);
+    let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee_with_commitment(
@@ -597,6 +623,7 @@ mod tests {
                 command: CliCommand::AuthorizeNonceAccount {
                     nonce_account: nonce_account_pubkey,
                     nonce_authority: 0,
+                    memo: None,
                     new_authority: Pubkey::default(),
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
@@ -618,6 +645,7 @@ mod tests {
                 command: CliCommand::AuthorizeNonceAccount {
                     nonce_account: read_keypair_file(&keypair_file).unwrap().pubkey(),
                     nonce_authority: 1,
+                    memo: None,
                     new_authority: Pubkey::default(),
                 },
                 signers: vec![
@@ -641,6 +669,7 @@ mod tests {
                     nonce_account: 1,
                     seed: None,
                     nonce_authority: None,
+                    memo: None,
                     amount: SpendAmount::Some(50_000_000_000),
                 },
                 signers: vec![
@@ -666,6 +695,7 @@ mod tests {
                     nonce_account: 1,
                     seed: None,
                     nonce_authority: Some(nonce_authority_keypair.pubkey()),
+                    memo: None,
                     amount: SpendAmount::Some(50_000_000_000),
                 },
                 signers: vec![
@@ -701,6 +731,7 @@ mod tests {
                 command: CliCommand::NewNonce {
                     nonce_account: nonce_account.pubkey(),
                     nonce_authority: 0,
+                    memo: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -721,6 +752,7 @@ mod tests {
                 command: CliCommand::NewNonce {
                     nonce_account: nonce_account.pubkey(),
                     nonce_authority: 1,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -765,6 +797,7 @@ mod tests {
                 command: CliCommand::WithdrawFromNonceAccount {
                     nonce_account: read_keypair_file(&keypair_file).unwrap().pubkey(),
                     nonce_authority: 0,
+                    memo: None,
                     destination_account_pubkey: nonce_account_pubkey,
                     lamports: 42_000_000_000
                 },
@@ -793,6 +826,7 @@ mod tests {
                 command: CliCommand::WithdrawFromNonceAccount {
                     nonce_account: read_keypair_file(&keypair_file).unwrap().pubkey(),
                     nonce_authority: 1,
+                    memo: None,
                     destination_account_pubkey: nonce_account_pubkey,
                     lamports: 42_000_000_000
                 },

cli/src/send_tpu.rs

@@ -1,46 +0,0 @@
-use log::*;
-use solana_client::rpc_response::{RpcContactInfo, RpcLeaderSchedule};
-use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS;
-use std::net::{SocketAddr, UdpSocket};
-
-pub fn get_leader_tpus(
-    slot_index: u64,
-    num_leaders: u64,
-    leader_schedule: Option<&RpcLeaderSchedule>,
-    cluster_nodes: Option<&Vec<RpcContactInfo>>,
-) -> Vec<SocketAddr> {
-    let leaders: Vec<_> = (0..num_leaders)
-        .filter_map(|i| {
-            leader_schedule?
-                .iter()
-                .find(|(_pubkey, slots)| {
-                    slots.iter().any(|slot| {
-                        *slot as u64 == (slot_index + (i * NUM_CONSECUTIVE_LEADER_SLOTS))
-                    })
-                })
-                .and_then(|(pubkey, _)| {
-                    cluster_nodes?
-                        .iter()
-                        .find(|contact_info| contact_info.pubkey == *pubkey)
-                        .and_then(|contact_info| contact_info.tpu)
-                })
-        })
-        .collect();
-    let mut unique_leaders = vec![];
-    for leader in leaders.into_iter() {
-        if !unique_leaders.contains(&leader) {
-            unique_leaders.push(leader);
-        }
-    }
-    unique_leaders
-}
-
-pub fn send_transaction_tpu(
-    send_socket: &UdpSocket,
-    tpu_address: &SocketAddr,
-    wire_transaction: &[u8],
-) {
-    if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
-        warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
-    }
-}

cli/src/vote.rs

@@ -4,6 +4,7 @@ use crate::{
         log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError,
         ProcessResult,
     },
+    memo::WithMemo,
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
@@ -11,6 +12,7 @@ use solana_clap_utils::{
     input_parsers::*,
     input_validators::*,
     keypair::{DefaultSigner, SignerIndex},
+    memo::{memo_arg, MEMO_ARG},
 };
 use solana_cli_output::{CliEpochVotingHistory, CliLockout, CliVoteAccount};
 use solana_client::rpc_client::RpcClient;
@@ -79,7 +81,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("STRING")
                         .takes_value(true)
                         .help("Seed for address generation; if specified, the resulting account will be at a derived address of the VOTE ACCOUNT pubkey")
-                ),
+                )
+		.arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-authorize-voter")
@@ -105,7 +108,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("NEW_AUTHORIZED_PUBKEY")
                         .required(true),
                         "New authorized vote signer. "),
-                ),
+                )
+		.arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-authorize-withdrawer")
@@ -131,7 +135,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("AUTHORIZED_PUBKEY")
                         .required(true),
                         "New authorized withdrawer. "),
-                ),
+                )
+		.arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-update-validator")
@@ -161,6 +166,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .validator(is_valid_signer)
                         .help("Authorized withdrawer keypair"),
                 )
+		.arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-update-commission")
@@ -190,6 +196,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .validator(is_valid_signer)
                         .help("Authorized withdrawer keypair"),
                 )
+		.arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-account")
@@ -207,6 +214,22 @@ impl VoteSubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
+                )
+                .arg(
+                    Arg::with_name("with_rewards")
+                        .long("with-rewards")
+                        .takes_value(false)
+                        .help("Display inflation rewards"),
+                )
+                .arg(
+                    Arg::with_name("num_rewards_epochs")
+                        .long("num-rewards-epochs")
+                        .takes_value(true)
+                        .value_name("NUM")
+                        .validator(|s| is_within_range(s, 1, 10))
+                        .default_value_if("with_rewards", None, "1")
+                        .requires("with_rewards")
+                        .help("Display rewards for NUM recent epochs, max 10 [default: latest epoch only]"),
                 ),
         )
         .subcommand(
@@ -243,6 +266,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .validator(is_valid_signer)
                         .help("Authorized withdrawer [default: cli config keypair]"),
                 )
+		.arg(memo_arg())
         )
     }
 }
@@ -259,6 +283,7 @@ pub fn parse_create_vote_account(
     let commission = value_t_or_exit!(matches, "commission", u8);
     let authorized_voter = pubkey_of_signer(matches, "authorized_voter", wallet_manager)?;
     let authorized_withdrawer = pubkey_of_signer(matches, "authorized_withdrawer", wallet_manager)?;
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
 
     let payer_provided = None;
     let signer_info = default_signer.generate_unique_signers(
@@ -275,6 +300,7 @@ pub fn parse_create_vote_account(
             authorized_voter,
             authorized_withdrawer,
             commission,
+            memo,
         },
         signers: signer_info.signers,
     })
@@ -298,12 +324,14 @@ pub fn parse_vote_authorize(
         matches,
         wallet_manager,
     )?;
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
 
     Ok(CliCommandInfo {
         command: CliCommand::VoteAuthorize {
             vote_account_pubkey,
             new_authorized_pubkey,
             vote_authorize,
+            memo,
         },
         signers: signer_info.signers,
     })
@@ -327,12 +355,14 @@ pub fn parse_vote_update_validator(
         matches,
         wallet_manager,
     )?;
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
 
     Ok(CliCommandInfo {
         command: CliCommand::VoteUpdateValidator {
             vote_account_pubkey,
             new_identity_account: signer_info.index_of(new_identity_pubkey).unwrap(),
             withdraw_authority: signer_info.index_of(authorized_withdrawer_pubkey).unwrap(),
+            memo,
         },
         signers: signer_info.signers,
     })
@@ -355,12 +385,14 @@ pub fn parse_vote_update_commission(
         matches,
         wallet_manager,
     )?;
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
 
     Ok(CliCommandInfo {
         command: CliCommand::VoteUpdateCommission {
             vote_account_pubkey,
             commission,
             withdraw_authority: signer_info.index_of(authorized_withdrawer_pubkey).unwrap(),
+            memo,
         },
         signers: signer_info.signers,
     })
@@ -373,10 +405,16 @@ pub fn parse_vote_get_account_command(
     let vote_account_pubkey =
         pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
+    let with_rewards = if matches.is_present("with_rewards") {
+        Some(value_of(matches, "num_rewards_epochs").unwrap())
+    } else {
+        None
+    };
     Ok(CliCommandInfo {
         command: CliCommand::ShowVoteAccount {
             pubkey: vote_account_pubkey,
             use_lamports_unit,
+            with_rewards,
         },
         signers: vec![],
     })
@@ -402,6 +440,7 @@ pub fn parse_withdraw_from_vote_account(
         matches,
         wallet_manager,
     )?;
+    let memo = matches.value_of(MEMO_ARG.name).map(String::from);
 
     Ok(CliCommandInfo {
         command: CliCommand::WithdrawFromVoteAccount {
@@ -409,6 +448,7 @@ pub fn parse_withdraw_from_vote_account(
             destination_account_pubkey,
             withdraw_authority: signer_info.index_of(withdraw_authority_pubkey).unwrap(),
             withdraw_amount,
+            memo,
         },
         signers: signer_info.signers,
     })
@@ -423,6 +463,7 @@ pub fn process_create_vote_account(
     authorized_voter: &Option<Pubkey>,
     authorized_withdrawer: &Option<Pubkey>,
     commission: u8,
+    memo: Option<&String>,
 ) -> ProcessResult {
     let vote_account = config.signers[vote_account];
     let vote_account_pubkey = vote_account.pubkey();
@@ -465,6 +506,7 @@ pub fn process_create_vote_account(
                 &vote_init,
                 lamports,
             )
+            .with_memo(memo)
         } else {
             vote_instruction::create_account(
                 &config.signers[0].pubkey(),
@@ -472,6 +514,7 @@ pub fn process_create_vote_account(
                 &vote_init,
                 lamports,
             )
+            .with_memo(memo)
         };
         Message::new(&ixs, Some(&config.signers[0].pubkey()))
     };
@@ -515,6 +558,7 @@ pub fn process_vote_authorize(
     vote_account_pubkey: &Pubkey,
     new_authorized_pubkey: &Pubkey,
     vote_authorize: VoteAuthorize,
+    memo: Option<&String>,
 ) -> ProcessResult {
     // If the `authorized_account` is also the fee payer, `config.signers` will only have one
     // keypair in it
@@ -534,7 +578,8 @@ pub fn process_vote_authorize(
         &authorized.pubkey(),  // current authorized
         new_authorized_pubkey, // new vote signer/withdrawer
         vote_authorize,        // vote or withdraw
-    )];
+    )]
+    .with_memo(memo);
 
     let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
@@ -556,6 +601,7 @@ pub fn process_vote_update_validator(
     vote_account_pubkey: &Pubkey,
     new_identity_account: SignerIndex,
     withdraw_authority: SignerIndex,
+    memo: Option<&String>,
 ) -> ProcessResult {
     let authorized_withdrawer = config.signers[withdraw_authority];
     let new_identity_account = config.signers[new_identity_account];
@@ -569,7 +615,8 @@ pub fn process_vote_update_validator(
         vote_account_pubkey,
         &authorized_withdrawer.pubkey(),
         &new_identity_pubkey,
-    )];
+    )]
+    .with_memo(memo);
 
     let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
@@ -591,6 +638,7 @@ pub fn process_vote_update_commission(
     vote_account_pubkey: &Pubkey,
     commission: u8,
     withdraw_authority: SignerIndex,
+    memo: Option<&String>,
 ) -> ProcessResult {
     let authorized_withdrawer = config.signers[withdraw_authority];
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
@@ -598,7 +646,8 @@ pub fn process_vote_update_commission(
         vote_account_pubkey,
         &authorized_withdrawer.pubkey(),
         commission,
-    )];
+    )]
+    .with_memo(memo);
 
     let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
@@ -647,6 +696,7 @@ pub fn process_show_vote_account(
     config: &CliConfig,
     vote_account_address: &Pubkey,
     use_lamports_unit: bool,
+    with_rewards: Option<usize>,
 ) -> ProcessResult {
     let (vote_account, vote_state) =
         get_vote_account(rpc_client, vote_account_address, config.commitment)?;
@@ -672,14 +722,16 @@ pub fn process_show_vote_account(
         }
     }
 
-    let epoch_rewards = match crate::stake::fetch_epoch_rewards(rpc_client, vote_account_address, 1)
-    {
-        Ok(rewards) => Some(rewards),
-        Err(error) => {
-            eprintln!("Failed to fetch epoch rewards: {:?}", error);
-            None
-        }
-    };
+    let epoch_rewards =
+        with_rewards.and_then(|num_epochs| {
+            match crate::stake::fetch_epoch_rewards(rpc_client, vote_account_address, num_epochs) {
+                Ok(rewards) => Some(rewards),
+                Err(error) => {
+                    eprintln!("Failed to fetch epoch rewards: {:?}", error);
+                    None
+                }
+            }
+        });
 
     let vote_account_data = CliVoteAccount {
         account_balance: vote_account.lamports,
@@ -706,6 +758,7 @@ pub fn process_withdraw_from_vote_account(
     withdraw_authority: SignerIndex,
     withdraw_amount: SpendAmount,
     destination_account_pubkey: &Pubkey,
+    memo: Option<&String>,
 ) -> ProcessResult {
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let withdraw_authority = config.signers[withdraw_authority];
@@ -726,14 +779,15 @@ pub fn process_withdraw_from_vote_account(
         }
     };
 
-    let ix = withdraw(
+    let ixs = vec![withdraw(
         vote_account_pubkey,
         &withdraw_authority.pubkey(),
         lamports,
         destination_account_pubkey,
-    );
+    )]
+    .with_memo(memo);
 
-    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut transaction = Transaction::new_unsigned(message);
     transaction.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee_with_commitment(
@@ -790,7 +844,8 @@ mod tests {
                 command: CliCommand::VoteAuthorize {
                     vote_account_pubkey: pubkey,
                     new_authorized_pubkey: pubkey2,
-                    vote_authorize: VoteAuthorize::Voter
+                    vote_authorize: VoteAuthorize::Voter,
+                    memo: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -813,7 +868,8 @@ mod tests {
                 command: CliCommand::VoteAuthorize {
                     vote_account_pubkey: pubkey,
                     new_authorized_pubkey: pubkey2,
-                    vote_authorize: VoteAuthorize::Voter
+                    vote_authorize: VoteAuthorize::Voter,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -847,6 +903,7 @@ mod tests {
                     authorized_voter: None,
                     authorized_withdrawer: None,
                     commission: 10,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -876,6 +933,7 @@ mod tests {
                     authorized_voter: None,
                     authorized_withdrawer: None,
                     commission: 100,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -908,7 +966,8 @@ mod tests {
                     identity_account: 2,
                     authorized_voter: Some(authed),
                     authorized_withdrawer: None,
-                    commission: 100
+                    commission: 100,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -939,7 +998,8 @@ mod tests {
                     identity_account: 2,
                     authorized_voter: None,
                     authorized_withdrawer: Some(authed),
-                    commission: 100
+                    commission: 100,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -963,6 +1023,7 @@ mod tests {
                     vote_account_pubkey: pubkey,
                     new_identity_account: 2,
                     withdraw_authority: 1,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -986,6 +1047,7 @@ mod tests {
                     vote_account_pubkey: pubkey,
                     commission: 42,
                     withdraw_authority: 1,
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
@@ -1010,6 +1072,7 @@ mod tests {
                     destination_account_pubkey: pubkey,
                     withdraw_authority: 0,
                     withdraw_amount: SpendAmount::Some(42_000_000_000),
+                    memo: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -1031,6 +1094,7 @@ mod tests {
                     destination_account_pubkey: pubkey,
                     withdraw_authority: 0,
                     withdraw_amount: SpendAmount::All,
+                    memo: None,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -1057,6 +1121,7 @@ mod tests {
                     destination_account_pubkey: pubkey,
                     withdraw_authority: 1,
                     withdraw_amount: SpendAmount::Some(42_000_000_000),
+                    memo: None,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),

cli/tests/nonce.rs

@@ -22,44 +22,38 @@ use solana_sdk::{
 #[test]
 fn test_nonce() {
     let mint_keypair = Keypair::new();
-    full_battery_tests(
-        TestValidator::with_no_fees(mint_keypair.pubkey()),
-        mint_keypair,
-        None,
-        false,
-    );
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
+
+    full_battery_tests(test_validator, None, false);
 }
 
 #[test]
 fn test_nonce_with_seed() {
     let mint_keypair = Keypair::new();
-    full_battery_tests(
-        TestValidator::with_no_fees(mint_keypair.pubkey()),
-        mint_keypair,
-        Some(String::from("seed")),
-        false,
-    );
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
+
+    full_battery_tests(test_validator, Some(String::from("seed")), false);
 }
 
 #[test]
 fn test_nonce_with_authority() {
     let mint_keypair = Keypair::new();
-    full_battery_tests(
-        TestValidator::with_no_fees(mint_keypair.pubkey()),
-        mint_keypair,
-        None,
-        true,
-    );
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
+
+    full_battery_tests(test_validator, None, true);
 }
 
 fn full_battery_tests(
     test_validator: TestValidator,
-    mint_keypair: Keypair,
     seed: Option<String>,
     use_nonce_authority: bool,
 ) {
-    let faucet_addr = run_local_faucet(mint_keypair, None);
-
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let json_rpc_url = test_validator.rpc_url();
@@ -71,10 +65,9 @@ fn full_battery_tests(
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_payer,
         &config_payer.signers[0].pubkey(),
         2000,
-        &config_payer,
     )
     .unwrap();
     check_recent_balance(2000, &rpc_client, &config_payer.signers[0].pubkey());
@@ -108,6 +101,7 @@ fn full_battery_tests(
         nonce_account: 1,
         seed,
         nonce_authority: optional_authority,
+        memo: None,
         amount: SpendAmount::Some(1000),
     };
 
@@ -141,6 +135,7 @@ fn full_battery_tests(
     config_payer.command = CliCommand::NewNonce {
         nonce_account,
         nonce_authority: index,
+        memo: None,
     };
     process_command(&config_payer).unwrap();
 
@@ -158,6 +153,7 @@ fn full_battery_tests(
     config_payer.command = CliCommand::WithdrawFromNonceAccount {
         nonce_account,
         nonce_authority: index,
+        memo: None,
         destination_account_pubkey: payee_pubkey,
         lamports: 100,
     };
@@ -178,6 +174,7 @@ fn full_battery_tests(
     config_payer.command = CliCommand::AuthorizeNonceAccount {
         nonce_account,
         nonce_authority: index,
+        memo: None,
         new_authority: new_authority.pubkey(),
     };
     process_command(&config_payer).unwrap();
@@ -186,6 +183,7 @@ fn full_battery_tests(
     config_payer.command = CliCommand::NewNonce {
         nonce_account,
         nonce_authority: index,
+        memo: None,
     };
     process_command(&config_payer).unwrap_err();
 
@@ -194,6 +192,7 @@ fn full_battery_tests(
     config_payer.command = CliCommand::NewNonce {
         nonce_account,
         nonce_authority: 1,
+        memo: None,
     };
     process_command(&config_payer).unwrap();
 
@@ -201,6 +200,7 @@ fn full_battery_tests(
     config_payer.command = CliCommand::WithdrawFromNonceAccount {
         nonce_account,
         nonce_authority: 1,
+        memo: None,
         destination_account_pubkey: payee_pubkey,
         lamports: 100,
     };
@@ -214,32 +214,29 @@ fn full_battery_tests(
 fn test_create_account_with_seed() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let offline_nonce_authority_signer = keypair_from_seed(&[1u8; 32]).unwrap();
     let online_nonce_creator_signer = keypair_from_seed(&[2u8; 32]).unwrap();
     let to_address = Pubkey::new(&[3u8; 32]);
-    let config = CliConfig::recent_for_tests();
 
     // Setup accounts
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &CliConfig::recent_for_tests(),
         &offline_nonce_authority_signer.pubkey(),
         42,
-        &config,
     )
     .unwrap();
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &CliConfig::recent_for_tests(),
         &online_nonce_creator_signer.pubkey(),
         4242,
-        &config,
     )
     .unwrap();
     check_recent_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
@@ -263,6 +260,7 @@ fn test_create_account_with_seed() {
         nonce_account: 0,
         seed: Some(seed),
         nonce_authority: Some(authority_pubkey),
+        memo: None,
         amount: SpendAmount::Some(241),
     };
     process_command(&creator_config).unwrap();
@@ -293,10 +291,13 @@ fn test_create_account_with_seed() {
         to: to_address,
         from: 0,
         sign_only: true,
+        dump_transaction_message: true,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_address),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -316,6 +317,8 @@ fn test_create_account_with_seed() {
         to: to_address,
         from: 0,
         sign_only: false,
+        dump_transaction_message: true,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_address),
@@ -323,6 +326,7 @@ fn test_create_account_with_seed() {
         ),
         nonce_account: Some(nonce_address),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,

cli/tests/program.rs

@@ -28,8 +28,9 @@ fn test_cli_program_deploy_non_upgradeable() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -46,8 +47,6 @@ fn test_cli_program_deploy_non_upgradeable() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 4 * minimum_balance_for_rent_exemption, // min balance for rent exemption for three programs + leftover for tx processing
     };
@@ -104,8 +103,6 @@ fn test_cli_program_deploy_non_upgradeable() {
     let custom_address_keypair = Keypair::new();
     config.signers = vec![&custom_address_keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 2 * minimum_balance_for_rent_exemption, // Anything over minimum_balance_for_rent_exemption should trigger err
     };
@@ -147,8 +144,9 @@ fn test_cli_program_deploy_no_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -171,8 +169,6 @@ fn test_cli_program_deploy_no_authority() {
     let keypair = Keypair::new();
     config.json_rpc_url = test_validator.rpc_url();
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_programdata + minimum_balance_for_program,
     };
@@ -231,8 +227,9 @@ fn test_cli_program_deploy_with_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -256,8 +253,6 @@ fn test_cli_program_deploy_with_authority() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_programdata + minimum_balance_for_program,
     };
@@ -442,6 +437,9 @@ fn test_cli_program_deploy_with_authority() {
     config.signers = vec![&keypair];
     config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(program_pubkey),
+        authority_pubkey: keypair.pubkey(),
+        all: false,
+        use_lamports_unit: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -530,6 +528,9 @@ fn test_cli_program_deploy_with_authority() {
     config.signers = vec![&keypair];
     config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(program_pubkey),
+        authority_pubkey: keypair.pubkey(),
+        all: false,
+        use_lamports_unit: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -554,8 +555,9 @@ fn test_cli_program_write_buffer() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -580,8 +582,6 @@ fn test_cli_program_write_buffer() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -657,9 +657,12 @@ fn test_cli_program_write_buffer() {
     );
 
     // Get buffer authority
-    config.signers = vec![&keypair];
+    config.signers = vec![];
     config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(buffer_keypair.pubkey()),
+        authority_pubkey: keypair.pubkey(),
+        all: false,
+        use_lamports_unit: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -747,9 +750,12 @@ fn test_cli_program_write_buffer() {
     );
 
     // Get buffer authority
-    config.signers = vec![&keypair];
+    config.signers = vec![];
     config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(buffer_pubkey),
+        authority_pubkey: keypair.pubkey(),
+        all: false,
+        use_lamports_unit: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -764,6 +770,60 @@ fn test_cli_program_write_buffer() {
         authority_keypair.pubkey(),
         Pubkey::from_str(&authority_pubkey_str).unwrap()
     );
+
+    // Close buffer
+    let close_account = rpc_client.get_account(&buffer_pubkey).unwrap();
+    assert_eq!(minimum_balance_for_buffer, close_account.lamports);
+    let recipient_pubkey = Pubkey::new_unique();
+    config.signers = vec![&keypair, &authority_keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Close {
+        account_pubkey: Some(buffer_pubkey),
+        recipient_pubkey,
+        authority_index: 1,
+        use_lamports_unit: false,
+    });
+    process_command(&config).unwrap();
+    rpc_client.get_account(&buffer_pubkey).unwrap_err();
+    let recipient_account = rpc_client.get_account(&recipient_pubkey).unwrap();
+    assert_eq!(minimum_balance_for_buffer, recipient_account.lamports);
+
+    // Write a buffer with default params
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
+        program_location: pathbuf.to_str().unwrap().to_string(),
+        buffer_signer_index: None,
+        buffer_pubkey: None,
+        buffer_authority_signer_index: None,
+        max_len: None,
+    });
+    config.output_format = OutputFormat::JsonCompact;
+    let response = process_command(&config);
+    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
+    let buffer_pubkey_str = json
+        .as_object()
+        .unwrap()
+        .get("buffer")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    let new_buffer_pubkey = Pubkey::from_str(&buffer_pubkey_str).unwrap();
+
+    // Close buffers and deposit default keypair
+    let pre_lamports = rpc_client.get_account(&keypair.pubkey()).unwrap().lamports;
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Close {
+        account_pubkey: Some(new_buffer_pubkey),
+        recipient_pubkey: keypair.pubkey(),
+        authority_index: 0,
+        use_lamports_unit: false,
+    });
+    process_command(&config).unwrap();
+    rpc_client.get_account(&new_buffer_pubkey).unwrap_err();
+    let recipient_account = rpc_client.get_account(&keypair.pubkey()).unwrap();
+    assert_eq!(
+        pre_lamports + minimum_balance_for_buffer,
+        recipient_account.lamports
+    );
 }
 
 #[test]
@@ -777,8 +837,9 @@ fn test_cli_program_set_buffer_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -798,8 +859,6 @@ fn test_cli_program_set_buffer_authority() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -891,8 +950,9 @@ fn test_cli_program_mismatch_buffer_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -912,8 +972,6 @@ fn test_cli_program_mismatch_buffer_authority() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -981,8 +1039,9 @@ fn test_cli_program_show() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1005,8 +1064,6 @@ fn test_cli_program_show() {
     // Airdrop
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -1029,6 +1086,9 @@ fn test_cli_program_show() {
     config.signers = vec![&keypair];
     config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(buffer_keypair.pubkey()),
+        authority_pubkey: keypair.pubkey(),
+        all: false,
+        use_lamports_unit: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -1086,6 +1146,9 @@ fn test_cli_program_show() {
     config.signers = vec![&keypair];
     config.command = CliCommand::Program(ProgramCliCommand::Show {
         account_pubkey: Some(program_keypair.pubkey()),
+        authority_pubkey: keypair.pubkey(),
+        all: false,
+        use_lamports_unit: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -1156,8 +1219,9 @@ fn test_cli_program_dump() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1180,8 +1244,6 @@ fn test_cli_program_dump() {
     // Airdrop
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };

cli/tests/request_airdrop.rs

@@ -10,15 +10,13 @@ use solana_sdk::{
 #[test]
 fn test_cli_request_airdrop() {
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let mut bob_config = CliConfig::recent_for_tests();
     bob_config.json_rpc_url = test_validator.rpc_url();
     bob_config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 50,
     };

cli/tests/stake.rs

@@ -26,8 +26,9 @@ use solana_stake_program::{
 #[test]
 fn test_stake_delegation_force() {
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -37,14 +38,8 @@ fn test_stake_delegation_force() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-        &config,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     // Create vote account
     let vote_keypair = Keypair::new();
@@ -56,6 +51,7 @@ fn test_stake_delegation_force() {
         authorized_voter: None,
         authorized_withdrawer: None,
         commission: 0,
+        memo: None,
     };
     process_command(&config).unwrap();
 
@@ -70,9 +66,11 @@ fn test_stake_delegation_force() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -86,9 +84,11 @@ fn test_stake_delegation_force() {
         stake_authority: 0,
         force: false,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap_err();
@@ -100,9 +100,11 @@ fn test_stake_delegation_force() {
         stake_authority: 0,
         force: true,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -113,8 +115,9 @@ fn test_seed_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -126,10 +129,9 @@ fn test_seed_stake_delegation_and_deactivation() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_validator,
         &config_validator.signers[0].pubkey(),
         100_000,
-        &config_validator,
     )
     .unwrap();
     check_recent_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
@@ -151,9 +153,11 @@ fn test_seed_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -166,9 +170,11 @@ fn test_seed_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: true,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
@@ -178,9 +184,12 @@ fn test_seed_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_address,
         stake_authority: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
@@ -191,8 +200,9 @@ fn test_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -206,10 +216,9 @@ fn test_stake_delegation_and_deactivation() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_validator,
         &config_validator.signers[0].pubkey(),
         100_000,
-        &config_validator,
     )
     .unwrap();
     check_recent_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
@@ -224,9 +233,11 @@ fn test_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -240,9 +251,11 @@ fn test_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: true,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
@@ -252,9 +265,12 @@ fn test_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
@@ -265,8 +281,9 @@ fn test_offline_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -291,20 +308,18 @@ fn test_offline_stake_delegation_and_deactivation() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_validator,
         &config_validator.signers[0].pubkey(),
         100_000,
-        &config_offline,
     )
     .unwrap();
     check_recent_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_offline,
         &config_offline.signers[0].pubkey(),
         100_000,
-        &config_validator,
     )
     .unwrap();
     check_recent_balance(100_000, &rpc_client, &config_offline.signers[0].pubkey());
@@ -319,9 +334,11 @@ fn test_offline_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -335,9 +352,11 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: true,
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     config_offline.output_format = OutputFormat::JsonCompact;
@@ -354,9 +373,11 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: true,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config_payer).unwrap();
@@ -367,9 +388,12 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
+        seed: None,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
@@ -383,9 +407,12 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config_payer).unwrap();
@@ -396,8 +423,9 @@ fn test_nonced_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -411,14 +439,8 @@ fn test_nonced_stake_delegation_and_deactivation() {
         .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-        &config,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     // Create stake account
     let stake_keypair = Keypair::new();
@@ -431,9 +453,11 @@ fn test_nonced_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -446,6 +470,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
         nonce_account: 1,
         seed: None,
         nonce_authority: Some(config.signers[0].pubkey()),
+        memo: None,
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
@@ -468,12 +493,14 @@ fn test_nonced_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: true,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
             nonce_hash,
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -493,12 +520,15 @@ fn test_nonced_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
             nonce_hash,
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -509,8 +539,9 @@ fn test_stake_authorize() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -520,14 +551,8 @@ fn test_stake_authorize() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-        &config,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     let offline_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let mut config_offline = CliConfig::recent_for_tests();
@@ -540,10 +565,9 @@ fn test_stake_authorize() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_offline,
         &config_offline.signers[0].pubkey(),
         100_000,
-        &config,
     )
     .unwrap();
 
@@ -559,9 +583,11 @@ fn test_stake_authorize() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -575,9 +601,11 @@ fn test_stake_authorize() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 0)],
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -603,9 +631,11 @@ fn test_stake_authorize() {
             (StakeAuthorize::Withdrawer, withdraw_authority_pubkey, 0),
         ],
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -626,9 +656,11 @@ fn test_stake_authorize() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, offline_authority_pubkey, 1)],
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -649,9 +681,11 @@ fn test_stake_authorize() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, nonced_authority_pubkey, 0)],
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -665,9 +699,11 @@ fn test_stake_authorize() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, nonced_authority_pubkey, 0)],
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -690,6 +726,7 @@ fn test_stake_authorize() {
         nonce_account: 1,
         seed: None,
         nonce_authority: Some(offline_authority_pubkey),
+        memo: None,
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
@@ -712,9 +749,11 @@ fn test_stake_authorize() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 1)],
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -729,12 +768,14 @@ fn test_stake_authorize() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 1)],
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
             sign_only.blockhash,
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -764,8 +805,9 @@ fn test_stake_authorize_with_fee_payer() {
     const SIG_FEE: u64 = 42;
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), SIG_FEE);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, SIG_FEE, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -791,16 +833,13 @@ fn test_stake_authorize_with_fee_payer() {
     config_offline.command = CliCommand::ClusterVersion;
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &default_pubkey, 100_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &default_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &payer_pubkey, 100_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_payer, &payer_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &payer_pubkey);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     check_ready(&rpc_client);
@@ -817,9 +856,11 @@ fn test_stake_authorize_with_fee_payer() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -833,9 +874,11 @@ fn test_stake_authorize_with_fee_payer() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, offline_pubkey, 0)],
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 1,
         custodian: None,
     };
@@ -852,9 +895,11 @@ fn test_stake_authorize_with_fee_payer() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, payer_pubkey, 0)],
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -868,9 +913,11 @@ fn test_stake_authorize_with_fee_payer() {
         stake_account_pubkey,
         new_authorizations: vec![(StakeAuthorize::Staker, payer_pubkey, 0)],
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         custodian: None,
     };
@@ -887,8 +934,9 @@ fn test_stake_split() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -907,18 +955,11 @@ fn test_stake_split() {
     config_offline.command = CliCommand::ClusterVersion;
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        500_000,
-        &config,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 500_000)
+        .unwrap();
     check_recent_balance(500_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
@@ -936,9 +977,11 @@ fn test_stake_split() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(10 * minimum_stake_balance),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -959,6 +1002,7 @@ fn test_stake_split() {
         nonce_account: 1,
         seed: None,
         nonce_authority: Some(offline_pubkey),
+        memo: None,
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
@@ -982,9 +1026,11 @@ fn test_stake_split() {
         stake_account_pubkey,
         stake_authority: 0,
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         split_stake_account: 1,
         seed: None,
         lamports: 2 * minimum_stake_balance,
@@ -1000,12 +1046,14 @@ fn test_stake_split() {
         stake_account_pubkey,
         stake_authority: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
             sign_only.blockhash,
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         split_stake_account: 1,
         seed: None,
         lamports: 2 * minimum_stake_balance,
@@ -1029,8 +1077,9 @@ fn test_stake_set_lockup() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1049,18 +1098,11 @@ fn test_stake_set_lockup() {
     config_offline.command = CliCommand::ClusterVersion;
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        500_000,
-        &config,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 500_000)
+        .unwrap();
     check_recent_balance(500_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
@@ -1085,9 +1127,11 @@ fn test_stake_set_lockup() {
         lockup,
         amount: SpendAmount::Some(10 * minimum_stake_balance),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -1110,9 +1154,11 @@ fn test_stake_set_lockup() {
         lockup,
         custodian: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -1143,9 +1189,11 @@ fn test_stake_set_lockup() {
         lockup,
         custodian: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -1161,9 +1209,11 @@ fn test_stake_set_lockup() {
         lockup,
         custodian: 1,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -1191,9 +1241,11 @@ fn test_stake_set_lockup() {
         lockup,
         custodian: 1,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -1209,6 +1261,7 @@ fn test_stake_set_lockup() {
         nonce_account: 1,
         seed: None,
         nonce_authority: Some(offline_pubkey),
+        memo: None,
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
@@ -1235,9 +1288,11 @@ fn test_stake_set_lockup() {
         lockup,
         custodian: 0,
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account_pubkey),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     config_offline.output_format = OutputFormat::JsonCompact;
@@ -1251,12 +1306,14 @@ fn test_stake_set_lockup() {
         lockup,
         custodian: 0,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account_pubkey),
             sign_only.blockhash,
         ),
         nonce_account: Some(nonce_account_pubkey),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -1279,8 +1336,9 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1298,18 +1356,11 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        200_000,
-        &config,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 200_000)
+        .unwrap();
     check_recent_balance(200_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create nonce account
@@ -1323,6 +1374,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         nonce_account: 1,
         seed: None,
         nonce_authority: Some(offline_pubkey),
+        memo: None,
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
@@ -1349,9 +1401,11 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -1370,12 +1424,14 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_pubkey),
             sign_only.blockhash,
         ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -1399,13 +1455,16 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     config_offline.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
         destination_account_pubkey: recipient_pubkey,
-        lamports: 42,
+        amount: SpendAmount::Some(42),
         withdraw_authority: 0,
         custodian: None,
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
+        memo: None,
+        seed: None,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
@@ -1415,16 +1474,19 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     config.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
         destination_account_pubkey: recipient_pubkey,
-        lamports: 42,
+        amount: SpendAmount::Some(42),
         withdraw_authority: 0,
         custodian: None,
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_pubkey),
             sign_only.blockhash,
         ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
+        memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -1451,9 +1513,11 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: true,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };
@@ -1470,12 +1534,14 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         amount: SpendAmount::Some(50_000),
         sign_only: false,
+        dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_pubkey),
             sign_only.blockhash,
         ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         from: 0,
     };

cli/tests/transfer.rs

@@ -22,8 +22,9 @@ use solana_sdk::{
 fn test_transfer() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -38,8 +39,7 @@ fn test_transfer() {
     let sender_pubkey = config.signers[0].pubkey();
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 50_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
     check_recent_balance(50_000, &rpc_client, &sender_pubkey);
     check_recent_balance(0, &rpc_client, &recipient_pubkey);
 
@@ -51,10 +51,13 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -69,10 +72,13 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -89,7 +95,7 @@ fn test_transfer() {
     process_command(&offline).unwrap_err();
 
     let offline_pubkey = offline.signers[0].pubkey();
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 50, &config).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &offline, &offline_pubkey, 50).unwrap();
     check_recent_balance(50, &rpc_client, &offline_pubkey);
 
     // Offline transfer
@@ -99,10 +105,13 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -118,10 +127,13 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -140,6 +152,7 @@ fn test_transfer() {
         nonce_account: 1,
         seed: None,
         nonce_authority: None,
+        memo: None,
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
@@ -162,6 +175,8 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
@@ -169,6 +184,7 @@ fn test_transfer() {
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -191,6 +207,7 @@ fn test_transfer() {
     config.command = CliCommand::AuthorizeNonceAccount {
         nonce_account: nonce_account.pubkey(),
         nonce_authority: 0,
+        memo: None,
         new_authority: offline_pubkey,
     };
     process_command(&config).unwrap();
@@ -213,10 +230,13 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -231,6 +251,8 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
@@ -238,6 +260,7 @@ fn test_transfer() {
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -251,32 +274,30 @@ fn test_transfer() {
 fn test_transfer_multisession_signing() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let to_pubkey = Pubkey::new(&[1u8; 32]);
     let offline_from_signer = keypair_from_seed(&[2u8; 32]).unwrap();
     let offline_fee_payer_signer = keypair_from_seed(&[3u8; 32]).unwrap();
     let from_null_signer = NullSigner::new(&offline_from_signer.pubkey());
-    let config = CliConfig::recent_for_tests();
 
     // Setup accounts
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &CliConfig::recent_for_tests(),
         &offline_from_signer.pubkey(),
         43,
-        &config,
     )
     .unwrap();
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &CliConfig::recent_for_tests(),
         &offline_fee_payer_signer.pubkey(),
         3,
-        &config,
     )
     .unwrap();
     check_recent_balance(43, &rpc_client, &offline_from_signer.pubkey());
@@ -299,10 +320,13 @@ fn test_transfer_multisession_signing() {
         to: to_pubkey,
         from: 1,
         sign_only: true,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -327,10 +351,13 @@ fn test_transfer_multisession_signing() {
         to: to_pubkey,
         from: 1,
         sign_only: true,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -352,10 +379,13 @@ fn test_transfer_multisession_signing() {
         to: to_pubkey,
         from: 1,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -371,8 +401,9 @@ fn test_transfer_multisession_signing() {
 fn test_transfer_all() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -386,8 +417,7 @@ fn test_transfer_all() {
     let sender_pubkey = config.signers[0].pubkey();
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 50_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
     check_recent_balance(50_000, &rpc_client, &sender_pubkey);
     check_recent_balance(0, &rpc_client, &recipient_pubkey);
 
@@ -399,10 +429,13 @@ fn test_transfer_all() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -412,12 +445,61 @@ fn test_transfer_all() {
     check_recent_balance(49_999, &rpc_client, &recipient_pubkey);
 }
 
+#[test]
+fn test_transfer_unfunded_recipient() {
+    solana_logger::setup();
+    let mint_keypair = Keypair::new();
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
+
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+    let default_signer = Keypair::new();
+
+    let mut config = CliConfig::recent_for_tests();
+    config.json_rpc_url = test_validator.rpc_url();
+    config.signers = vec![&default_signer];
+
+    let sender_pubkey = config.signers[0].pubkey();
+    let recipient_pubkey = Pubkey::new(&[1u8; 32]);
+
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
+    check_recent_balance(50_000, &rpc_client, &sender_pubkey);
+    check_recent_balance(0, &rpc_client, &recipient_pubkey);
+
+    check_ready(&rpc_client);
+
+    // Plain ole transfer
+    config.command = CliCommand::Transfer {
+        amount: SpendAmount::All,
+        to: recipient_pubkey,
+        from: 0,
+        sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: false,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
+    };
+
+    // Expect failure due to unfunded recipient and the lack of the `allow_unfunded_recipient` flag
+    process_command(&config).unwrap_err();
+}
+
 #[test]
 fn test_transfer_with_seed() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -439,9 +521,8 @@ fn test_transfer_with_seed() {
     )
     .unwrap();
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 1, &config).unwrap();
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &derived_address, 50_000, &config)
-        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 1).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &derived_address, 50_000).unwrap();
     check_recent_balance(1, &rpc_client, &sender_pubkey);
     check_recent_balance(50_000, &rpc_client, &derived_address);
     check_recent_balance(0, &rpc_client, &recipient_pubkey);
@@ -454,10 +535,13 @@ fn test_transfer_with_seed() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: Some(derived_address_seed),
         derived_address_program_id: Some(derived_address_program_id),

cli/tests/vote.rs

@@ -19,8 +19,9 @@ use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersion
 #[test]
 fn test_vote_authorize_and_withdraw() {
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -30,14 +31,8 @@ fn test_vote_authorize_and_withdraw() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-        &config,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     // Create vote account
     let vote_account_keypair = Keypair::new();
@@ -50,6 +45,7 @@ fn test_vote_authorize_and_withdraw() {
         authorized_voter: None,
         authorized_withdrawer: Some(config.signers[0].pubkey()),
         commission: 0,
+        memo: None,
     };
     process_command(&config).unwrap();
     let vote_account = rpc_client
@@ -71,10 +67,13 @@ fn test_vote_authorize_and_withdraw() {
         to: vote_account_pubkey,
         from: 0,
         sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
         no_wait: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
+        memo: None,
         fee_payer: 0,
         derived_address_seed: None,
         derived_address_program_id: None,
@@ -90,6 +89,7 @@ fn test_vote_authorize_and_withdraw() {
         vote_account_pubkey,
         new_authorized_pubkey: withdraw_authority.pubkey(),
         vote_authorize: VoteAuthorize::Withdrawer,
+        memo: None,
     };
     process_command(&config).unwrap();
     let vote_account = rpc_client
@@ -107,6 +107,7 @@ fn test_vote_authorize_and_withdraw() {
         withdraw_authority: 1,
         withdraw_amount: SpendAmount::Some(100),
         destination_account_pubkey: destination_account,
+        memo: None,
     };
     process_command(&config).unwrap();
     check_recent_balance(expected_balance - 100, &rpc_client, &vote_account_pubkey);
@@ -119,6 +120,7 @@ fn test_vote_authorize_and_withdraw() {
         vote_account_pubkey,
         new_identity_account: 2,
         withdraw_authority: 1,
+        memo: None,
     };
     process_command(&config).unwrap();
 }

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.6.0"
+version = "1.6.7"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,26 +19,28 @@ jsonrpc-core = "17.0.0"
 log = "0.4.11"
 net2 = "0.2.37"
 rayon = "1.5.0"
-reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 semver = "0.11.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.6.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-net-utils = { path = "../net-utils", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.6.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.7" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-faucet = { path = "../faucet", version = "=1.6.7" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.7" }
 thiserror = "1.0"
+tokio = { version = "1", features = ["full"] }
 tungstenite = "0.10.1"
 url = "2.1.1"
 
 [dev-dependencies]
 assert_matches = "1.3.0"
 jsonrpc-http-server = "17.0.0"
-solana-logger = { path = "../logger", version = "1.6.0" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/blockhash_query.rs

@@ -1,12 +1,15 @@
-use crate::{nonce_utils, rpc_client::RpcClient};
-use clap::ArgMatches;
-use solana_clap_utils::{
-    input_parsers::{pubkey_of, value_of},
-    nonce::*,
-    offline::*,
-};
-use solana_sdk::{
-    commitment_config::CommitmentConfig, fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey,
+use {
+    crate::{nonce_utils, rpc_client::RpcClient},
+    clap::ArgMatches,
+    solana_clap_utils::{
+        input_parsers::{pubkey_of, value_of},
+        nonce::*,
+        offline::*,
+    },
+    solana_sdk::{
+        commitment_config::CommitmentConfig, fee_calculator::FeeCalculator, hash::Hash,
+        pubkey::Pubkey,
+    },
 };
 
 #[derive(Debug, PartialEq)]

client/src/client_error.rs

@@ -1,9 +1,12 @@
-use crate::rpc_request;
-use solana_sdk::{
-    signature::SignerError, transaction::TransactionError, transport::TransportError,
+use {
+    crate::rpc_request,
+    solana_faucet::faucet::FaucetError,
+    solana_sdk::{
+        signature::SignerError, transaction::TransactionError, transport::TransportError,
+    },
+    std::io,
+    thiserror::Error,
 };
-use std::io;
-use thiserror::Error;
 
 pub use reqwest; // export `reqwest` for clients
 
@@ -21,6 +24,8 @@ pub enum ClientErrorKind {
     SigningError(#[from] SignerError),
     #[error(transparent)]
     TransactionError(#[from] TransactionError),
+    #[error(transparent)]
+    FaucetError(#[from] FaucetError),
     #[error("Custom: {0}")]
     Custom(String),
 }
@@ -44,6 +49,7 @@ impl From<ClientErrorKind> for TransportError {
             ClientErrorKind::RpcError(err) => Self::Custom(format!("{:?}", err)),
             ClientErrorKind::SerdeJson(err) => Self::Custom(format!("{:?}", err)),
             ClientErrorKind::SigningError(err) => Self::Custom(format!("{:?}", err)),
+            ClientErrorKind::FaucetError(err) => Self::Custom(format!("{:?}", err)),
             ClientErrorKind::Custom(err) => Self::Custom(format!("{:?}", err)),
         }
     }
@@ -160,4 +166,13 @@ impl From<TransactionError> for ClientError {
     }
 }
 
+impl From<FaucetError> for ClientError {
+    fn from(err: FaucetError) -> Self {
+        Self {
+            request: None,
+            kind: err.into(),
+        }
+    }
+}
+
 pub type Result<T> = std::result::Result<T, ClientError>;

client/src/http_sender.rs

@@ -1,17 +1,27 @@
-use crate::{
-    client_error::Result,
-    rpc_custom_error,
-    rpc_request::{RpcError, RpcRequest, RpcResponseErrorData},
-    rpc_response::RpcSimulateTransactionResult,
-    rpc_sender::RpcSender,
+use {
+    crate::{
+        client_error::Result,
+        rpc_custom_error,
+        rpc_request::{RpcError, RpcRequest, RpcResponseErrorData},
+        rpc_response::RpcSimulateTransactionResult,
+        rpc_sender::RpcSender,
+    },
+    log::*,
+    reqwest::{self, header::CONTENT_TYPE, StatusCode},
+    std::{
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc,
+        },
+        thread::sleep,
+        time::Duration,
+    },
 };
-use log::*;
-use reqwest::{self, header::CONTENT_TYPE, StatusCode};
-use std::{thread::sleep, time::Duration};
 
 pub struct HttpSender {
-    client: reqwest::blocking::Client,
+    client: Arc<reqwest::blocking::Client>,
     url: String,
+    request_id: AtomicU64,
 }
 
 impl HttpSender {
@@ -20,12 +30,22 @@ impl HttpSender {
     }
 
     pub fn new_with_timeout(url: String, timeout: Duration) -> Self {
-        let client = reqwest::blocking::Client::builder()
-            .timeout(timeout)
-            .build()
-            .expect("build rpc client");
+        // `reqwest::blocking::Client` panics if run in a tokio async context.  Shuttle the
+        // request to a different tokio thread to avoid this
+        let client = Arc::new(
+            tokio::task::block_in_place(move || {
+                reqwest::blocking::Client::builder()
+                    .timeout(timeout)
+                    .build()
+            })
+            .expect("build rpc client"),
+        );
 
-        Self { client, url }
+        Self {
+            client,
+            url,
+            request_id: AtomicU64::new(0),
+        }
     }
 }
 
@@ -38,20 +58,26 @@ struct RpcErrorObject {
 
 impl RpcSender for HttpSender {
     fn send(&self, request: RpcRequest, params: serde_json::Value) -> Result<serde_json::Value> {
-        // Concurrent requests are not supported so reuse the same request id for all requests
-        let request_id = 1;
-
-        let request_json = request.build_request_json(request_id, params);
+        let request_id = self.request_id.fetch_add(1, Ordering::Relaxed);
+        let request_json = request.build_request_json(request_id, params).to_string();
 
         let mut too_many_requests_retries = 5;
         loop {
-            match self
-                .client
-                .post(&self.url)
-                .header(CONTENT_TYPE, "application/json")
-                .body(request_json.to_string())
-                .send()
-            {
+            // `reqwest::blocking::Client` panics if run in a tokio async context.  Shuttle the
+            // request to a different tokio thread to avoid this
+            let response = {
+                let client = self.client.clone();
+                let request_json = request_json.clone();
+                tokio::task::block_in_place(move || {
+                    client
+                        .post(&self.url)
+                        .header(CONTENT_TYPE, "application/json")
+                        .body(request_json)
+                        .send()
+                })
+            };
+
+            match response {
                 Ok(response) => {
                     if !response.status().is_success() {
                         if response.status() == StatusCode::TOO_MANY_REQUESTS
@@ -70,7 +96,9 @@ impl RpcSender for HttpSender {
                         return Err(response.error_for_status().unwrap_err().into());
                     }
 
-                    let json: serde_json::Value = serde_json::from_str(&response.text()?)?;
+                    let response_text = tokio::task::block_in_place(move || response.text())?;
+
+                    let json: serde_json::Value = serde_json::from_str(&response_text)?;
                     if json["error"].is_object() {
                         return match serde_json::from_value::<RpcErrorObject>(json["error"].clone())
                         {
@@ -120,3 +148,22 @@ impl RpcSender for HttpSender {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn http_sender_on_tokio_multi_thread() {
+        let http_sender = HttpSender::new("http://localhost:1234".to_string());
+        let _ = http_sender.send(RpcRequest::GetVersion, serde_json::Value::Null);
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    #[should_panic(expected = "can call blocking only when running on the multi-threaded runtime")]
+    async fn http_sender_ontokio_current_thread_should_panic() {
+        // RpcClient::new() will panic in the tokio current-thread runtime due to `tokio::task::block_in_place()` usage, and there
+        // doesn't seem to be a way to detect whether the tokio runtime is multi_thread or current_thread...
+        let _ = HttpSender::new("http://localhost:1234".to_string());
+    }
+}

client/src/lib.rs

@@ -18,3 +18,4 @@ pub mod rpc_request;
 pub mod rpc_response;
 pub mod rpc_sender;
 pub mod thin_client;
+pub mod tpu_client;

client/src/mock_sender.rs

@@ -1,20 +1,22 @@
-use crate::{
-    client_error::Result,
-    rpc_request::RpcRequest,
-    rpc_response::{Response, RpcResponseContext, RpcVersionInfo},
-    rpc_sender::RpcSender,
+use {
+    crate::{
+        client_error::Result,
+        rpc_request::RpcRequest,
+        rpc_response::{Response, RpcResponseContext, RpcVersionInfo},
+        rpc_sender::RpcSender,
+    },
+    serde_json::{json, Number, Value},
+    solana_sdk::{
+        epoch_info::EpochInfo,
+        fee_calculator::{FeeCalculator, FeeRateGovernor},
+        instruction::InstructionError,
+        signature::Signature,
+        transaction::{self, Transaction, TransactionError},
+    },
+    solana_transaction_status::{TransactionConfirmationStatus, TransactionStatus},
+    solana_version::Version,
+    std::{collections::HashMap, sync::RwLock},
 };
-use serde_json::{json, Number, Value};
-use solana_sdk::{
-    epoch_info::EpochInfo,
-    fee_calculator::{FeeCalculator, FeeRateGovernor},
-    instruction::InstructionError,
-    signature::Signature,
-    transaction::{self, Transaction, TransactionError},
-};
-use solana_transaction_status::{TransactionConfirmationStatus, TransactionStatus};
-use solana_version::Version;
-use std::{collections::HashMap, sync::RwLock};
 
 pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
 pub const SIGNATURE: &str =
@@ -122,6 +124,8 @@ impl RpcSender for MockSender {
             }
             RpcRequest::GetTransactionCount => Value::Number(Number::from(1234)),
             RpcRequest::GetSlot => Value::Number(Number::from(0)),
+            RpcRequest::GetMaxShredInsertSlot => Value::Number(Number::from(0)),
+            RpcRequest::RequestAirdrop => Value::String(Signature::new(&[8; 64]).to_string()),
             RpcRequest::SendTransaction => {
                 let signature = if self.url == "malicious" {
                     Signature::new(&[8; 64]).to_string()

client/src/nonce_utils.rs

@@ -1,14 +1,16 @@
-use crate::rpc_client::RpcClient;
-use solana_sdk::{
-    account::{Account, ReadableAccount},
-    account_utils::StateMut,
-    commitment_config::CommitmentConfig,
-    nonce::{
-        state::{Data, Versions},
-        State,
+use {
+    crate::rpc_client::RpcClient,
+    solana_sdk::{
+        account::{Account, ReadableAccount},
+        account_utils::StateMut,
+        commitment_config::CommitmentConfig,
+        nonce::{
+            state::{Data, Versions},
+            State,
+        },
+        pubkey::Pubkey,
+        system_program,
     },
-    pubkey::Pubkey,
-    system_program,
 };
 
 #[derive(Debug, thiserror::Error, PartialEq)]
@@ -46,10 +48,7 @@ pub fn get_account_with_commitment(
                 .value
                 .ok_or_else(|| Error::Client(format!("AccountNotFound: pubkey={}", nonce_pubkey)))
         })
-        .and_then(|a| match account_identity_ok(&a) {
-            Ok(()) => Ok(a),
-            Err(e) => Err(e),
-        })
+        .and_then(|a| account_identity_ok(&a).map(|()| a))
 }
 
 pub fn account_identity_ok<T: ReadableAccount>(account: &T) -> Result<(), Error> {

client/src/perf_utils.rs

@@ -1,12 +1,14 @@
-use log::*;
-use solana_sdk::{client::Client, commitment_config::CommitmentConfig, timing::duration_as_s};
-use std::{
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc, RwLock,
+use {
+    log::*,
+    solana_sdk::{client::Client, commitment_config::CommitmentConfig, timing::duration_as_s},
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc, RwLock,
+        },
+        thread::sleep,
+        time::{Duration, Instant},
     },
-    thread::sleep,
-    time::{Duration, Instant},
 };
 
 #[derive(Default)]

client/src/pubsub_client.rs

@@ -1,27 +1,33 @@
-use crate::{
-    rpc_config::{RpcSignatureSubscribeConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter},
-    rpc_response::{Response as RpcResponse, RpcLogsResponse, RpcSignatureResult, SlotInfo},
-};
-use log::*;
-use serde::de::DeserializeOwned;
-use serde_json::{
-    json,
-    value::Value::{Number, Object},
-    Map, Value,
-};
-use solana_sdk::signature::Signature;
-use std::{
-    marker::PhantomData,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        mpsc::{channel, Receiver},
-        Arc, RwLock,
+use {
+    crate::{
+        rpc_config::{
+            RpcSignatureSubscribeConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter,
+        },
+        rpc_response::{
+            Response as RpcResponse, RpcLogsResponse, RpcSignatureResult, SlotInfo, SlotUpdate,
+        },
     },
-    thread::JoinHandle,
+    log::*,
+    serde::de::DeserializeOwned,
+    serde_json::{
+        json,
+        value::Value::{Number, Object},
+        Map, Value,
+    },
+    solana_sdk::signature::Signature,
+    std::{
+        marker::PhantomData,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            mpsc::{channel, Receiver},
+            Arc, RwLock,
+        },
+        thread::JoinHandle,
+    },
+    thiserror::Error,
+    tungstenite::{client::AutoStream, connect, Message, WebSocket},
+    url::{ParseError, Url},
 };
-use thiserror::Error;
-use tungstenite::{client::AutoStream, connect, Message, WebSocket};
-use url::{ParseError, Url};
 
 #[derive(Debug, Error)]
 pub enum PubsubClientError {
@@ -336,6 +342,54 @@ impl PubsubClient {
 
         Ok((result, receiver))
     }
+
+    pub fn slot_updates_subscribe(
+        url: &str,
+        handler: impl Fn(SlotUpdate) + Send + 'static,
+    ) -> Result<PubsubClientSubscription<SlotUpdate>, PubsubClientError> {
+        let url = Url::parse(url)?;
+        let (socket, _response) = connect(url)?;
+
+        let socket = Arc::new(RwLock::new(socket));
+        let exit = Arc::new(AtomicBool::new(false));
+        let exit_clone = exit.clone();
+        let subscription_id = PubsubClientSubscription::<SlotUpdate>::send_subscribe(
+            &socket,
+            json!({
+                "jsonrpc":"2.0","id":1,"method":"slotsUpdatesSubscribe","params":[]
+            })
+            .to_string(),
+        )?;
+
+        let t_cleanup = {
+            let socket = socket.clone();
+            std::thread::spawn(move || {
+                loop {
+                    if exit_clone.load(Ordering::Relaxed) {
+                        break;
+                    }
+                    match PubsubClientSubscription::read_message(&socket) {
+                        Ok(message) => handler(message),
+                        Err(err) => {
+                            info!("receive error: {:?}", err);
+                            break;
+                        }
+                    }
+                }
+
+                info!("websocket - exited receive loop");
+            })
+        };
+
+        Ok(PubsubClientSubscription {
+            message_type: PhantomData,
+            operation: "slotsUpdates",
+            socket,
+            subscription_id,
+            t_cleanup: Some(t_cleanup),
+            exit,
+        })
+    }
 }
 
 #[cfg(test)]

client/src/rpc_cache.rs

@@ -1,7 +1,9 @@
-use crate::{rpc_config::RpcLargestAccountsFilter, rpc_response::RpcAccountBalance};
-use std::{
-    collections::HashMap,
-    time::{Duration, SystemTime},
+use {
+    crate::{rpc_config::RpcLargestAccountsFilter, rpc_response::RpcAccountBalance},
+    std::{
+        collections::HashMap,
+        time::{Duration, SystemTime},
+    },
 };
 
 #[derive(Debug, Clone)]

client/src/rpc_client.rs

@@ -1,47 +1,47 @@
-use crate::{
-    client_error::{ClientError, ClientErrorKind, Result as ClientResult},
-    http_sender::HttpSender,
-    mock_sender::{MockSender, Mocks},
-    rpc_config::RpcAccountInfoConfig,
-    rpc_config::{
-        RpcGetConfirmedSignaturesForAddress2Config, RpcLargestAccountsConfig,
-        RpcProgramAccountsConfig, RpcSendTransactionConfig, RpcSimulateTransactionConfig,
-        RpcTokenAccountsFilter,
+use {
+    crate::{
+        client_error::{ClientError, ClientErrorKind, Result as ClientResult},
+        http_sender::HttpSender,
+        mock_sender::{MockSender, Mocks},
+        rpc_config::RpcAccountInfoConfig,
+        rpc_config::*,
+        rpc_request::{RpcError, RpcRequest, RpcResponseErrorData, TokenAccountsFilter},
+        rpc_response::*,
+        rpc_sender::RpcSender,
+    },
+    bincode::serialize,
+    indicatif::{ProgressBar, ProgressStyle},
+    log::*,
+    serde_json::{json, Value},
+    solana_account_decoder::{
+        parse_token::{TokenAccountType, UiTokenAccount, UiTokenAmount},
+        UiAccount, UiAccountData, UiAccountEncoding,
+    },
+    solana_sdk::{
+        account::Account,
+        clock::{Epoch, Slot, UnixTimestamp, DEFAULT_MS_PER_SLOT, MAX_HASH_AGE_IN_SECONDS},
+        commitment_config::{CommitmentConfig, CommitmentLevel},
+        epoch_info::EpochInfo,
+        epoch_schedule::EpochSchedule,
+        fee_calculator::{FeeCalculator, FeeRateGovernor},
+        hash::Hash,
+        pubkey::Pubkey,
+        signature::Signature,
+        transaction::{self, uses_durable_nonce, Transaction},
+    },
+    solana_transaction_status::{
+        EncodedConfirmedBlock, EncodedConfirmedTransaction, TransactionStatus, UiConfirmedBlock,
+        UiTransactionEncoding,
+    },
+    solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY,
+    std::{
+        cmp::min,
+        net::SocketAddr,
+        str::FromStr,
+        sync::RwLock,
+        thread::sleep,
+        time::{Duration, Instant},
     },
-    rpc_request::{RpcError, RpcRequest, RpcResponseErrorData, TokenAccountsFilter},
-    rpc_response::*,
-    rpc_sender::RpcSender,
-};
-use bincode::serialize;
-use indicatif::{ProgressBar, ProgressStyle};
-use log::*;
-use serde_json::{json, Value};
-use solana_account_decoder::{
-    parse_token::{TokenAccountType, UiTokenAccount, UiTokenAmount},
-    UiAccount, UiAccountData, UiAccountEncoding,
-};
-use solana_sdk::{
-    account::Account,
-    clock::{Slot, UnixTimestamp, DEFAULT_MS_PER_SLOT, MAX_HASH_AGE_IN_SECONDS},
-    commitment_config::{CommitmentConfig, CommitmentLevel},
-    epoch_info::EpochInfo,
-    epoch_schedule::EpochSchedule,
-    fee_calculator::{FeeCalculator, FeeRateGovernor},
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::Signature,
-    transaction::{self, uses_durable_nonce, Transaction},
-};
-use solana_transaction_status::{
-    EncodedConfirmedBlock, EncodedConfirmedTransaction, TransactionStatus, UiTransactionEncoding,
-};
-use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
-use std::{
-    cmp::min,
-    net::SocketAddr,
-    sync::RwLock,
-    thread::sleep,
-    time::{Duration, Instant},
 };
 
 pub struct RpcClient {
@@ -404,6 +404,53 @@ impl RpcClient {
         )
     }
 
+    pub fn get_slot_leaders(&self, start_slot: Slot, limit: u64) -> ClientResult<Vec<Pubkey>> {
+        self.send(RpcRequest::GetSlotLeaders, json!([start_slot, limit]))
+            .and_then(|slot_leaders: Vec<String>| {
+                slot_leaders
+                    .iter()
+                    .map(|slot_leader| {
+                        Pubkey::from_str(slot_leader).map_err(|err| {
+                            ClientErrorKind::Custom(format!(
+                                "pubkey deserialization failed: {}",
+                                err
+                            ))
+                            .into()
+                        })
+                    })
+                    .collect()
+            })
+    }
+
+    /// Get block production for the current epoch
+    pub fn get_block_production(&self) -> RpcResult<RpcBlockProduction> {
+        self.send(RpcRequest::GetBlockProduction, Value::Null)
+    }
+
+    pub fn get_block_production_with_config(
+        &self,
+        config: RpcBlockProductionConfig,
+    ) -> RpcResult<RpcBlockProduction> {
+        self.send(RpcRequest::GetBlockProduction, json!(config))
+    }
+
+    pub fn get_stake_activation(
+        &self,
+        stake_account: Pubkey,
+        epoch: Option<Epoch>,
+    ) -> ClientResult<RpcStakeActivation> {
+        self.send(
+            RpcRequest::GetStakeActivation,
+            json!([
+                stake_account.to_string(),
+                RpcEpochConfig {
+                    epoch,
+                    commitment: Some(self.commitment_config),
+                }
+            ]),
+        )
+    }
+
     pub fn supply(&self) -> RpcResult<RpcSupply> {
         self.supply_with_commitment(self.commitment_config)
     }
@@ -418,10 +465,17 @@ impl RpcClient {
         )
     }
 
+    #[deprecated(since = "1.5.19", note = "Please use RpcClient::supply() instead")]
+    #[allow(deprecated)]
     pub fn total_supply(&self) -> ClientResult<u64> {
         self.total_supply_with_commitment(self.commitment_config)
     }
 
+    #[deprecated(
+        since = "1.5.19",
+        note = "Please use RpcClient::supply_with_commitment() instead"
+    )]
+    #[allow(deprecated)]
     pub fn total_supply_with_commitment(
         &self,
         commitment_config: CommitmentConfig,
@@ -453,10 +507,17 @@ impl RpcClient {
         &self,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<RpcVoteAccountStatus> {
-        self.send(
-            RpcRequest::GetVoteAccounts,
-            json!([self.maybe_map_commitment(commitment_config)?]),
-        )
+        self.get_vote_accounts_with_config(RpcGetVoteAccountsConfig {
+            commitment: Some(self.maybe_map_commitment(commitment_config)?),
+            ..RpcGetVoteAccountsConfig::default()
+        })
+    }
+
+    pub fn get_vote_accounts_with_config(
+        &self,
+        config: RpcGetVoteAccountsConfig,
+    ) -> ClientResult<RpcVoteAccountStatus> {
+        self.send(RpcRequest::GetVoteAccounts, json!([config]))
     }
 
     pub fn wait_for_max_stake(
@@ -507,6 +568,14 @@ impl RpcClient {
         self.send(RpcRequest::GetConfirmedBlock, json!([slot, encoding]))
     }
 
+    pub fn get_confirmed_block_with_config(
+        &self,
+        slot: Slot,
+        config: RpcConfirmedBlockConfig,
+    ) -> ClientResult<UiConfirmedBlock> {
+        self.send(RpcRequest::GetConfirmedBlock, json!([slot, config]))
+    }
+
     pub fn get_confirmed_blocks(
         &self,
         start_slot: Slot,
@@ -518,6 +587,24 @@ impl RpcClient {
         )
     }
 
+    pub fn get_confirmed_blocks_with_commitment(
+        &self,
+        start_slot: Slot,
+        end_slot: Option<Slot>,
+        commitment_config: CommitmentConfig,
+    ) -> ClientResult<Vec<Slot>> {
+        let json = if end_slot.is_some() {
+            json!([
+                start_slot,
+                end_slot,
+                self.maybe_map_commitment(commitment_config)?
+            ])
+        } else {
+            json!([start_slot, self.maybe_map_commitment(commitment_config)?])
+        };
+        self.send(RpcRequest::GetConfirmedBlocks, json)
+    }
+
     pub fn get_confirmed_blocks_with_limit(
         &self,
         start_slot: Slot,
@@ -529,6 +616,27 @@ impl RpcClient {
         )
     }
 
+    pub fn get_confirmed_blocks_with_limit_and_commitment(
+        &self,
+        start_slot: Slot,
+        limit: usize,
+        commitment_config: CommitmentConfig,
+    ) -> ClientResult<Vec<Slot>> {
+        self.send(
+            RpcRequest::GetConfirmedBlocksWithLimit,
+            json!([
+                start_slot,
+                limit,
+                self.maybe_map_commitment(commitment_config)?
+            ]),
+        )
+    }
+
+    #[deprecated(
+        since = "1.5.19",
+        note = "Please use RpcClient::get_confirmed_signatures_for_address2() instead"
+    )]
+    #[allow(deprecated)]
     pub fn get_confirmed_signatures_for_address(
         &self,
         address: &Pubkey,
@@ -570,6 +678,7 @@ impl RpcClient {
             before: config.before.map(|signature| signature.to_string()),
             until: config.until.map(|signature| signature.to_string()),
             limit: config.limit,
+            commitment: config.commitment,
         };
 
         let result: Vec<RpcConfirmedTransactionStatusWithSignature> = self.send(
@@ -591,6 +700,17 @@ impl RpcClient {
         )
     }
 
+    pub fn get_confirmed_transaction_with_config(
+        &self,
+        signature: &Signature,
+        config: RpcConfirmedTransactionConfig,
+    ) -> ClientResult<EncodedConfirmedTransaction> {
+        self.send(
+            RpcRequest::GetConfirmedTransaction,
+            json!([signature.to_string(), config]),
+        )
+    }
+
     pub fn get_block_time(&self, slot: Slot) -> ClientResult<UnixTimestamp> {
         let request = RpcRequest::GetBlockTime;
         let response = self.sender.send(request, json!([slot]));
@@ -634,16 +754,34 @@ impl RpcClient {
         slot: Option<Slot>,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<Option<RpcLeaderSchedule>> {
-        self.send(
-            RpcRequest::GetLeaderSchedule,
-            json!([slot, self.maybe_map_commitment(commitment_config)?]),
+        self.get_leader_schedule_with_config(
+            slot,
+            RpcLeaderScheduleConfig {
+                commitment: Some(self.maybe_map_commitment(commitment_config)?),
+                ..RpcLeaderScheduleConfig::default()
+            },
         )
     }
 
+    pub fn get_leader_schedule_with_config(
+        &self,
+        slot: Option<Slot>,
+        config: RpcLeaderScheduleConfig,
+    ) -> ClientResult<Option<RpcLeaderSchedule>> {
+        self.send(RpcRequest::GetLeaderSchedule, json!([slot, config]))
+    }
+
     pub fn get_epoch_schedule(&self) -> ClientResult<EpochSchedule> {
         self.send(RpcRequest::GetEpochSchedule, Value::Null)
     }
 
+    pub fn get_recent_performance_samples(
+        &self,
+        limit: Option<usize>,
+    ) -> ClientResult<Vec<RpcPerfSample>> {
+        self.send(RpcRequest::GetRecentPerformanceSamples, json!([limit]))
+    }
+
     pub fn get_identity(&self) -> ClientResult<Pubkey> {
         let rpc_identity: RpcIdentity = self.send(RpcRequest::GetIdentity, Value::Null)?;
 
@@ -663,6 +801,27 @@ impl RpcClient {
         self.send(RpcRequest::GetInflationRate, Value::Null)
     }
 
+    pub fn get_inflation_reward(
+        &self,
+        addresses: &[Pubkey],
+        epoch: Option<Epoch>,
+    ) -> ClientResult<Vec<Option<RpcInflationReward>>> {
+        let addresses: Vec<_> = addresses
+            .iter()
+            .map(|address| address.to_string())
+            .collect();
+        self.send(
+            RpcRequest::GetInflationReward,
+            json!([
+                addresses,
+                RpcEpochConfig {
+                    epoch,
+                    commitment: Some(self.commitment_config),
+                }
+            ]),
+        )
+    }
+
     pub fn get_version(&self) -> ClientResult<RpcVersionInfo> {
         self.send(RpcRequest::GetVersion, Value::Null)
     }
@@ -769,6 +928,14 @@ impl RpcClient {
             })?
     }
 
+    pub fn get_max_retransmit_slot(&self) -> ClientResult<Slot> {
+        self.send(RpcRequest::GetMaxRetransmitSlot, Value::Null)
+    }
+
+    pub fn get_max_shred_insert_slot(&self) -> ClientResult<Slot> {
+        self.send(RpcRequest::GetMaxShredInsertSlot, Value::Null)
+    }
+
     pub fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> ClientResult<Vec<Option<Account>>> {
         Ok(self
             .get_multiple_accounts_with_commitment(pubkeys, self.commitment_config)?
@@ -1214,6 +1381,64 @@ impl RpcClient {
         )
     }
 
+    pub fn request_airdrop(&self, pubkey: &Pubkey, lamports: u64) -> ClientResult<Signature> {
+        self.request_airdrop_with_config(
+            pubkey,
+            lamports,
+            RpcRequestAirdropConfig {
+                commitment: Some(self.commitment_config),
+                ..RpcRequestAirdropConfig::default()
+            },
+        )
+    }
+
+    pub fn request_airdrop_with_blockhash(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        recent_blockhash: &Hash,
+    ) -> ClientResult<Signature> {
+        self.request_airdrop_with_config(
+            pubkey,
+            lamports,
+            RpcRequestAirdropConfig {
+                commitment: Some(self.commitment_config),
+                recent_blockhash: Some(recent_blockhash.to_string()),
+            },
+        )
+    }
+
+    pub fn request_airdrop_with_config(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        config: RpcRequestAirdropConfig,
+    ) -> ClientResult<Signature> {
+        let commitment = config.commitment.unwrap_or_default();
+        let commitment = self.maybe_map_commitment(commitment)?;
+        let config = RpcRequestAirdropConfig {
+            commitment: Some(commitment),
+            ..config
+        };
+        self.send(
+            RpcRequest::RequestAirdrop,
+            json!([pubkey.to_string(), lamports, config]),
+        )
+        .and_then(|signature: String| {
+            Signature::from_str(&signature).map_err(|err| {
+                ClientErrorKind::Custom(format!("signature deserialization failed: {}", err)).into()
+            })
+        })
+        .map_err(|_| {
+            RpcError::ForUser(
+                "airdrop request failed. \
+                This can happen when the rate limit is reached."
+                    .to_string(),
+            )
+            .into()
+        })
+    }
+
     fn poll_balance_with_timeout_and_commitment(
         &self,
         pubkey: &Pubkey,
@@ -1415,6 +1640,24 @@ impl RpcClient {
         commitment: CommitmentConfig,
         config: RpcSendTransactionConfig,
     ) -> ClientResult<Signature> {
+        let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
+            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
+                .value
+                .0
+        } else {
+            transaction.message.recent_blockhash
+        };
+        let signature = self.send_transaction_with_config(transaction, config)?;
+        self.confirm_transaction_with_spinner(&signature, &recent_blockhash, commitment)?;
+        Ok(signature)
+    }
+
+    pub fn confirm_transaction_with_spinner(
+        &self,
+        signature: &Signature,
+        recent_blockhash: &Hash,
+        commitment: CommitmentConfig,
+    ) -> ClientResult<()> {
         let desired_confirmations = if commitment.is_finalized() {
             MAX_LOCKOUT_HISTORY + 1
         } else {
@@ -1426,16 +1669,8 @@ impl RpcClient {
 
         progress_bar.set_message(&format!(
             "[{}/{}] Finalizing transaction {}",
-            confirmations, desired_confirmations, transaction.signatures[0],
+            confirmations, desired_confirmations, signature,
         ));
-        let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
-            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
-                .value
-                .0
-        } else {
-            transaction.message.recent_blockhash
-        };
-        let signature = self.send_transaction_with_config(transaction, config)?;
         let (signature, status) = loop {
             // Get recent commitment in order to count confirmations for successful transactions
             let status = self
@@ -1482,7 +1717,7 @@ impl RpcClient {
             {
                 progress_bar.set_message("Transaction confirmed");
                 progress_bar.finish_and_clear();
-                return Ok(signature);
+                return Ok(());
             }
 
             progress_bar.set_message(&format!(
@@ -1524,6 +1759,7 @@ pub struct GetConfirmedSignaturesForAddress2Config {
     pub before: Option<Signature>,
     pub until: Option<Signature>,
     pub limit: Option<usize>,
+    pub commitment: Option<CommitmentConfig>,
 }
 
 fn new_spinner_progress_bar() -> ProgressBar {
@@ -1583,6 +1819,21 @@ mod tests {
 
     #[test]
     fn test_send() {
+        _test_send();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    #[should_panic(expected = "can call blocking only when running on the multi-threaded runtime")]
+    async fn test_send_async_current_thread_should_panic() {
+        _test_send();
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_send_async_multi_thread() {
+        _test_send();
+    }
+
+    fn _test_send() {
         let (sender, receiver) = channel();
         thread::spawn(move || {
             let rpc_addr = "0.0.0.0:0".parse().unwrap();

client/src/rpc_config.rs

@@ -1,10 +1,12 @@
-use crate::rpc_filter::RpcFilterType;
-use solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig};
-use solana_sdk::{
-    clock::Epoch,
-    commitment_config::{CommitmentConfig, CommitmentLevel},
+use {
+    crate::rpc_filter::RpcFilterType,
+    solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig},
+    solana_sdk::{
+        clock::{Epoch, Slot},
+        commitment_config::{CommitmentConfig, CommitmentLevel},
+    },
+    solana_transaction_status::{TransactionDetails, UiTransactionEncoding},
 };
-use solana_transaction_status::UiTransactionEncoding;
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -31,6 +33,62 @@ pub struct RpcSimulateTransactionConfig {
     pub encoding: Option<UiTransactionEncoding>,
 }
 
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcRequestAirdropConfig {
+    pub recent_blockhash: Option<String>, // base-58 encoded blockhash
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcLeaderScheduleConfig {
+    pub identity: Option<String>, // validator identity, as a base-58 encoded string
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProductionConfigRange {
+    pub first_slot: Slot,
+    pub last_slot: Option<Slot>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProductionConfig {
+    pub identity: Option<String>, // validator identity, as a base-58 encoded string
+    pub range: Option<RpcBlockProductionConfigRange>, // current epoch if `None`
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcGetVoteAccountsConfig {
+    pub vote_pubkey: Option<String>, // validator vote address, as a base-58 encoded string
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum RpcLeaderScheduleConfigWrapper {
+    SlotOnly(Option<Slot>),
+    ConfigOnly(Option<RpcLeaderScheduleConfig>),
+}
+
+impl RpcLeaderScheduleConfigWrapper {
+    pub fn unzip(&self) -> (Option<Slot>, Option<RpcLeaderScheduleConfig>) {
+        match &self {
+            RpcLeaderScheduleConfigWrapper::SlotOnly(slot) => (*slot, None),
+            RpcLeaderScheduleConfigWrapper::ConfigOnly(config) => (None, config.clone()),
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum RpcLargestAccountsFilter {
@@ -48,7 +106,7 @@ pub struct RpcLargestAccountsConfig {
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct RpcStakeConfig {
+pub struct RpcEpochConfig {
     pub epoch: Option<Epoch>,
     #[serde(flatten)]
     pub commitment: Option<CommitmentConfig>,
@@ -107,6 +165,8 @@ pub struct RpcGetConfirmedSignaturesForAddress2Config {
     pub before: Option<String>, // Signature as base-58 string
     pub until: Option<String>,  // Signature as base-58 string
     pub limit: Option<usize>,
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -133,26 +193,73 @@ pub trait EncodingConfig {
 #[serde(rename_all = "camelCase")]
 pub struct RpcConfirmedBlockConfig {
     pub encoding: Option<UiTransactionEncoding>,
+    pub transaction_details: Option<TransactionDetails>,
+    pub rewards: Option<bool>,
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
 }
 
 impl EncodingConfig for RpcConfirmedBlockConfig {
     fn new_with_encoding(encoding: &Option<UiTransactionEncoding>) -> Self {
         Self {
             encoding: *encoding,
+            ..Self::default()
         }
     }
 }
 
+impl RpcConfirmedBlockConfig {
+    pub fn rewards_only() -> Self {
+        Self {
+            transaction_details: Some(TransactionDetails::None),
+            ..Self::default()
+        }
+    }
+
+    pub fn rewards_with_commitment(commitment: Option<CommitmentConfig>) -> Self {
+        Self {
+            transaction_details: Some(TransactionDetails::None),
+            commitment,
+            ..Self::default()
+        }
+    }
+}
+
+impl From<RpcConfirmedBlockConfig> for RpcEncodingConfigWrapper<RpcConfirmedBlockConfig> {
+    fn from(config: RpcConfirmedBlockConfig) -> Self {
+        RpcEncodingConfigWrapper::Current(Some(config))
+    }
+}
+
 #[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcConfirmedTransactionConfig {
     pub encoding: Option<UiTransactionEncoding>,
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
 }
 
 impl EncodingConfig for RpcConfirmedTransactionConfig {
     fn new_with_encoding(encoding: &Option<UiTransactionEncoding>) -> Self {
         Self {
             encoding: *encoding,
+            ..Self::default()
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum RpcConfirmedBlocksConfigWrapper {
+    EndSlotOnly(Option<Slot>),
+    CommitmentOnly(Option<CommitmentConfig>),
+}
+
+impl RpcConfirmedBlocksConfigWrapper {
+    pub fn unzip(&self) -> (Option<Slot>, Option<CommitmentConfig>) {
+        match &self {
+            RpcConfirmedBlocksConfigWrapper::EndSlotOnly(end_slot) => (*end_slot, None),
+            RpcConfirmedBlocksConfigWrapper::CommitmentOnly(commitment) => (None, *commitment),
         }
     }
 }

client/src/rpc_custom_error.rs

@@ -1,8 +1,10 @@
 //! Implementation defined RPC server errors
 
-use crate::rpc_response::RpcSimulateTransactionResult;
-use jsonrpc_core::{Error, ErrorCode};
-use solana_sdk::clock::Slot;
+use {
+    crate::rpc_response::RpcSimulateTransactionResult,
+    jsonrpc_core::{Error, ErrorCode},
+    solana_sdk::clock::Slot,
+};
 
 pub const JSON_RPC_SERVER_ERROR_BLOCK_CLEANED_UP: i64 = -32001;
 pub const JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE: i64 = -32002;

client/src/rpc_request.rs

@@ -1,20 +1,29 @@
-use crate::rpc_response::RpcSimulateTransactionResult;
-use serde_json::{json, Value};
-use solana_sdk::{clock::Slot, pubkey::Pubkey};
-use std::fmt;
-use thiserror::Error;
+use {
+    crate::rpc_response::RpcSimulateTransactionResult,
+    serde_json::{json, Value},
+    solana_sdk::{clock::Slot, pubkey::Pubkey},
+    std::fmt,
+    thiserror::Error,
+};
 
 #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
 pub enum RpcRequest {
     DeregisterNode,
     GetAccountInfo,
     GetBalance,
+    GetBlockProduction,
     GetBlockTime,
     GetClusterNodes,
     GetConfirmedBlock,
     GetConfirmedBlocks,
     GetConfirmedBlocksWithLimit,
+
+    #[deprecated(
+        since = "1.5.19",
+        note = "Please use RpcRequest::GetConfirmedSignaturesForAddress2 instead"
+    )]
     GetConfirmedSignaturesForAddress,
+
     GetConfirmedSignaturesForAddress2,
     GetConfirmedTransaction,
     GetEpochInfo,
@@ -28,26 +37,35 @@ pub enum RpcRequest {
     GetIdentity,
     GetInflationGovernor,
     GetInflationRate,
+    GetInflationReward,
     GetLargestAccounts,
     GetLeaderSchedule,
+    GetMaxRetransmitSlot,
+    GetMaxShredInsertSlot,
     GetMinimumBalanceForRentExemption,
     GetMultipleAccounts,
     GetProgramAccounts,
     GetRecentBlockhash,
+    GetRecentPerformanceSamples,
     GetSnapshotSlot,
     GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
+    GetSlotLeaders,
     GetStorageTurn,
     GetStorageTurnRate,
     GetSlotsPerSegment,
+    GetStakeActivation,
     GetStoragePubkeysForSlot,
     GetSupply,
     GetTokenAccountBalance,
     GetTokenAccountsByDelegate,
     GetTokenAccountsByOwner,
     GetTokenSupply,
+
+    #[deprecated(since = "1.5.19", note = "Please use RpcRequest::GetSupply instead")]
     GetTotalSupply,
+
     GetTransactionCount,
     GetVersion,
     GetVoteAccounts,
@@ -59,12 +77,14 @@ pub enum RpcRequest {
     SignVote,
 }
 
+#[allow(deprecated)]
 impl fmt::Display for RpcRequest {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let method = match self {
             RpcRequest::DeregisterNode => "deregisterNode",
             RpcRequest::GetAccountInfo => "getAccountInfo",
             RpcRequest::GetBalance => "getBalance",
+            RpcRequest::GetBlockProduction => "getBlockProduction",
             RpcRequest::GetBlockTime => "getBlockTime",
             RpcRequest::GetClusterNodes => "getClusterNodes",
             RpcRequest::GetConfirmedBlock => "getConfirmedBlock",
@@ -84,16 +104,22 @@ impl fmt::Display for RpcRequest {
             RpcRequest::GetIdentity => "getIdentity",
             RpcRequest::GetInflationGovernor => "getInflationGovernor",
             RpcRequest::GetInflationRate => "getInflationRate",
+            RpcRequest::GetInflationReward => "getInflationReward",
             RpcRequest::GetLargestAccounts => "getLargestAccounts",
             RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
+            RpcRequest::GetMaxRetransmitSlot => "getMaxRetransmitSlot",
+            RpcRequest::GetMaxShredInsertSlot => "getMaxShredInsertSlot",
             RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
             RpcRequest::GetMultipleAccounts => "getMultipleAccounts",
             RpcRequest::GetProgramAccounts => "getProgramAccounts",
             RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
+            RpcRequest::GetRecentPerformanceSamples => "getRecentPerformanceSamples",
             RpcRequest::GetSnapshotSlot => "getSnapshotSlot",
             RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
+            RpcRequest::GetSlotLeaders => "getSlotLeaders",
+            RpcRequest::GetStakeActivation => "getStakeActivation",
             RpcRequest::GetStorageTurn => "getStorageTurn",
             RpcRequest::GetStorageTurnRate => "getStorageTurnRate",
             RpcRequest::GetSlotsPerSegment => "getSlotsPerSegment",
@@ -126,6 +152,7 @@ pub const MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS2_LIMIT: usize = 1_000;
 pub const MAX_MULTIPLE_ACCOUNTS: usize = 100;
 pub const NUM_LARGEST_ACCOUNTS: usize = 20;
 pub const MAX_GET_PROGRAM_ACCOUNT_FILTERS: usize = 4;
+pub const MAX_GET_SLOT_LEADERS: usize = 5000;
 
 // Validators that are this number of slots behind are considered delinquent
 pub const DELINQUENT_VALIDATOR_SLOT_DISTANCE: u64 = 128;

client/src/rpc_response.rs

@@ -1,13 +1,17 @@
-use crate::client_error;
-use solana_account_decoder::{parse_token::UiTokenAmount, UiAccount};
-use solana_sdk::{
-    clock::{Epoch, Slot, UnixTimestamp},
-    fee_calculator::{FeeCalculator, FeeRateGovernor},
-    inflation::Inflation,
-    transaction::{Result, TransactionError},
+use {
+    crate::client_error,
+    solana_account_decoder::{parse_token::UiTokenAmount, UiAccount},
+    solana_sdk::{
+        clock::{Epoch, Slot, UnixTimestamp},
+        fee_calculator::{FeeCalculator, FeeRateGovernor},
+        inflation::Inflation,
+        transaction::{Result, TransactionError},
+    },
+    solana_transaction_status::{
+        ConfirmedTransactionStatusWithSignature, TransactionConfirmationStatus,
+    },
+    std::{collections::HashMap, fmt, net::SocketAddr},
 };
-use solana_transaction_status::ConfirmedTransactionStatusWithSignature;
-use std::{collections::HashMap, fmt, net::SocketAddr};
 
 pub type RpcResult<T> = client_error::Result<Response<T>>;
 
@@ -207,6 +211,21 @@ pub struct RpcContactInfo {
 /// Map of leader base58 identity pubkeys to the slot indices relative to the first epoch slot
 pub type RpcLeaderSchedule = HashMap<String, Vec<usize>>;
 
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProductionRange {
+    pub first_slot: Slot,
+    pub last_slot: Slot,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProduction {
+    /// Map of leader base58 identity pubkeys to a tuple of `(number of leader slots, number of blocks produced)`
+    pub by_identity: HashMap<String, (usize, usize)>,
+    pub range: RpcBlockProductionRange,
+}
+
 #[derive(Serialize, Deserialize, Clone)]
 #[serde(rename_all = "kebab-case")]
 pub struct RpcVersionInfo {
@@ -250,10 +269,10 @@ pub struct RpcVoteAccountStatus {
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcVoteAccountInfo {
-    /// Vote account pubkey as base-58 encoded string
+    /// Vote account address, as base-58 encoded string
     pub vote_pubkey: String,
 
-    /// The pubkey of the node that votes using this account
+    /// The validator identity, as base-58 encoded string
     pub node_pubkey: String,
 
     /// The current stake, in lamports, delegated to this vote account
@@ -313,7 +332,7 @@ pub struct RpcSupply {
     pub non_circulating_accounts: Vec<String>,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub enum StakeActivationState {
     Activating,
@@ -322,7 +341,7 @@ pub enum StakeActivationState {
     Inactive,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcStakeActivation {
     pub state: StakeActivationState,
@@ -346,6 +365,7 @@ pub struct RpcConfirmedTransactionStatusWithSignature {
     pub err: Option<TransactionError>,
     pub memo: Option<String>,
     pub block_time: Option<UnixTimestamp>,
+    pub confirmation_status: Option<TransactionConfirmationStatus>,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -357,6 +377,15 @@ pub struct RpcPerfSample {
     pub sample_period_secs: u16,
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationReward {
+    pub epoch: Epoch,
+    pub effective_slot: Slot,
+    pub amount: u64,       // lamports
+    pub post_balance: u64, // lamports
+}
+
 impl From<ConfirmedTransactionStatusWithSignature> for RpcConfirmedTransactionStatusWithSignature {
     fn from(value: ConfirmedTransactionStatusWithSignature) -> Self {
         let ConfirmedTransactionStatusWithSignature {
@@ -372,6 +401,7 @@ impl From<ConfirmedTransactionStatusWithSignature> for RpcConfirmedTransactionSt
             err,
             memo,
             block_time,
+            confirmation_status: None,
         }
     }
 }

client/src/thin_client.rs

@@ -3,36 +3,38 @@
 //! messages to the network directly. The binary encoding of its messages are
 //! unstable and may change in future releases.
 
-use crate::{rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response};
-use bincode::{serialize_into, serialized_size};
-use log::*;
-use solana_sdk::{
-    account::Account,
-    client::{AsyncClient, Client, SyncClient},
-    clock::{Slot, MAX_PROCESSING_AGE},
-    commitment_config::CommitmentConfig,
-    epoch_info::EpochInfo,
-    fee_calculator::{FeeCalculator, FeeRateGovernor},
-    hash::Hash,
-    instruction::Instruction,
-    message::Message,
-    packet::PACKET_DATA_SIZE,
-    pubkey::Pubkey,
-    signature::{Keypair, Signature, Signer},
-    signers::Signers,
-    system_instruction,
-    timing::duration_as_ms,
-    transaction::{self, Transaction},
-    transport::Result as TransportResult,
-};
-use std::{
-    io,
-    net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
-    sync::{
-        atomic::{AtomicBool, AtomicUsize, Ordering},
-        RwLock,
+use {
+    crate::{rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response},
+    bincode::{serialize_into, serialized_size},
+    log::*,
+    solana_sdk::{
+        account::Account,
+        client::{AsyncClient, Client, SyncClient},
+        clock::{Slot, MAX_PROCESSING_AGE},
+        commitment_config::CommitmentConfig,
+        epoch_info::EpochInfo,
+        fee_calculator::{FeeCalculator, FeeRateGovernor},
+        hash::Hash,
+        instruction::Instruction,
+        message::Message,
+        packet::PACKET_DATA_SIZE,
+        pubkey::Pubkey,
+        signature::{Keypair, Signature, Signer},
+        signers::Signers,
+        system_instruction,
+        timing::duration_as_ms,
+        transaction::{self, Transaction},
+        transport::Result as TransportResult,
+    },
+    std::{
+        io,
+        net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
+        sync::{
+            atomic::{AtomicBool, AtomicUsize, Ordering},
+            RwLock,
+        },
+        time::{Duration, Instant},
     },
-    time::{Duration, Instant},
 };
 
 struct ClientOptimizer {
@@ -167,8 +169,8 @@ impl ThinClient {
         let rpc_clients: Vec<_> = rpc_addrs.into_iter().map(RpcClient::new_socket).collect();
         let optimizer = ClientOptimizer::new(rpc_clients.len());
         Self {
-            tpu_addrs,
             transactions_socket,
+            tpu_addrs,
             rpc_clients,
             optimizer,
         }

client/src/tpu_client.rs

@@ -0,0 +1,393 @@
+use crate::{
+    pubsub_client::{PubsubClient, PubsubClientError, PubsubClientSubscription},
+    rpc_client::RpcClient,
+    rpc_response::SlotUpdate,
+};
+use bincode::serialize;
+use log::*;
+use solana_sdk::{clock::Slot, pubkey::Pubkey, transaction::Transaction};
+use std::{
+    collections::{HashMap, HashSet, VecDeque},
+    net::{SocketAddr, UdpSocket},
+    str::FromStr,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, RwLock,
+    },
+    thread::JoinHandle,
+    time::{Duration, Instant},
+};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum TpuSenderError {
+    #[error("Pubsub error: {0:?}")]
+    PubsubError(#[from] PubsubClientError),
+    #[error("RPC error: {0:?}")]
+    RpcError(#[from] crate::client_error::ClientError),
+    #[error("IO error: {0:?}")]
+    IoError(#[from] std::io::Error),
+}
+
+type Result<T> = std::result::Result<T, TpuSenderError>;
+
+/// Default number of slots used to build TPU socket fanout set
+pub const DEFAULT_FANOUT_SLOTS: u64 = 12;
+
+/// Maximum number of slots used to build TPU socket fanout set
+pub const MAX_FANOUT_SLOTS: u64 = 100;
+
+/// Config params for `TpuClient`
+#[derive(Clone, Debug)]
+pub struct TpuClientConfig {
+    /// The range of upcoming slots to include when determining which
+    /// leaders to send transactions to (min: 1, max: 100)
+    pub fanout_slots: u64,
+}
+
+impl Default for TpuClientConfig {
+    fn default() -> Self {
+        Self {
+            fanout_slots: DEFAULT_FANOUT_SLOTS,
+        }
+    }
+}
+
+/// Client which sends transactions directly to the current leader's TPU port over UDP.
+/// The client uses RPC to determine the current leader and fetch node contact info
+pub struct TpuClient {
+    send_socket: UdpSocket,
+    fanout_slots: u64,
+    leader_tpu_service: LeaderTpuService,
+    exit: Arc<AtomicBool>,
+}
+
+impl TpuClient {
+    /// Serializes and sends a transaction to the current leader's TPU port
+    pub fn send_transaction(&self, transaction: &Transaction) -> bool {
+        let wire_transaction = serialize(transaction).expect("serialization should succeed");
+        self.send_wire_transaction(&wire_transaction)
+    }
+
+    /// Sends a transaction to the current leader's TPU port
+    pub fn send_wire_transaction(&self, wire_transaction: &[u8]) -> bool {
+        let mut sent = false;
+        for tpu_address in self
+            .leader_tpu_service
+            .leader_tpu_sockets(self.fanout_slots)
+        {
+            if self
+                .send_socket
+                .send_to(wire_transaction, tpu_address)
+                .is_ok()
+            {
+                sent = true;
+            }
+        }
+        sent
+    }
+
+    /// Create a new client that disconnects when dropped
+    pub fn new(
+        rpc_client: Arc<RpcClient>,
+        websocket_url: &str,
+        config: TpuClientConfig,
+    ) -> Result<Self> {
+        let exit = Arc::new(AtomicBool::new(false));
+        let leader_tpu_service = LeaderTpuService::new(rpc_client, websocket_url, exit.clone())?;
+
+        Ok(Self {
+            send_socket: UdpSocket::bind("0.0.0.0:0").unwrap(),
+            fanout_slots: config.fanout_slots.min(MAX_FANOUT_SLOTS).max(1),
+            leader_tpu_service,
+            exit,
+        })
+    }
+}
+
+impl Drop for TpuClient {
+    fn drop(&mut self) {
+        self.exit.store(true, Ordering::Relaxed);
+        self.leader_tpu_service.join();
+    }
+}
+
+struct LeaderTpuCache {
+    first_slot: Slot,
+    leaders: Vec<Pubkey>,
+    leader_tpu_map: HashMap<Pubkey, SocketAddr>,
+}
+
+impl LeaderTpuCache {
+    fn new(rpc_client: &RpcClient, first_slot: Slot) -> Self {
+        let leaders = Self::fetch_slot_leaders(rpc_client, first_slot).unwrap_or_default();
+        let leader_tpu_map = Self::fetch_cluster_tpu_sockets(&rpc_client).unwrap_or_default();
+        Self {
+            first_slot,
+            leaders,
+            leader_tpu_map,
+        }
+    }
+
+    // Last slot that has a cached leader pubkey
+    fn last_slot(&self) -> Slot {
+        self.first_slot + self.leaders.len().saturating_sub(1) as u64
+    }
+
+    // Get the TPU sockets for the current leader and upcoming leaders according to fanout size
+    fn get_leader_sockets(&self, current_slot: Slot, fanout_slots: u64) -> Vec<SocketAddr> {
+        let mut leader_set = HashSet::new();
+        let mut leader_sockets = Vec::new();
+        for leader_slot in current_slot..current_slot + fanout_slots {
+            if let Some(leader) = self.get_slot_leader(leader_slot) {
+                if let Some(tpu_socket) = self.leader_tpu_map.get(leader) {
+                    if leader_set.insert(*leader) {
+                        leader_sockets.push(*tpu_socket);
+                    }
+                }
+            }
+        }
+        leader_sockets
+    }
+
+    fn get_slot_leader(&self, slot: Slot) -> Option<&Pubkey> {
+        if slot >= self.first_slot {
+            let index = slot - self.first_slot;
+            self.leaders.get(index as usize)
+        } else {
+            None
+        }
+    }
+
+    fn fetch_cluster_tpu_sockets(rpc_client: &RpcClient) -> Result<HashMap<Pubkey, SocketAddr>> {
+        let cluster_contact_info = rpc_client.get_cluster_nodes()?;
+        Ok(cluster_contact_info
+            .into_iter()
+            .filter_map(|contact_info| {
+                Some((
+                    Pubkey::from_str(&contact_info.pubkey).ok()?,
+                    contact_info.tpu?,
+                ))
+            })
+            .collect())
+    }
+
+    fn fetch_slot_leaders(rpc_client: &RpcClient, start_slot: Slot) -> Result<Vec<Pubkey>> {
+        Ok(rpc_client.get_slot_leaders(start_slot, 2 * MAX_FANOUT_SLOTS)?)
+    }
+}
+
+// 48 chosen because it's unlikely that 12 leaders in a row will miss their slots
+const MAX_SLOT_SKIP_DISTANCE: u64 = 48;
+
+#[derive(Clone, Debug)]
+struct RecentLeaderSlots(Arc<RwLock<VecDeque<Slot>>>);
+impl RecentLeaderSlots {
+    fn new(current_slot: Slot) -> Self {
+        let mut recent_slots = VecDeque::new();
+        recent_slots.push_back(current_slot);
+        Self(Arc::new(RwLock::new(recent_slots)))
+    }
+
+    fn record_slot(&self, current_slot: Slot) {
+        let mut recent_slots = self.0.write().unwrap();
+        recent_slots.push_back(current_slot);
+        // 12 recent slots should be large enough to avoid a misbehaving
+        // validator from affecting the median recent slot
+        while recent_slots.len() > 12 {
+            recent_slots.pop_front();
+        }
+    }
+
+    // Estimate the current slot from recent slot notifications.
+    fn estimated_current_slot(&self) -> Slot {
+        let mut recent_slots: Vec<Slot> = self.0.read().unwrap().iter().cloned().collect();
+        assert!(!recent_slots.is_empty());
+        recent_slots.sort_unstable();
+
+        // Validators can broadcast invalid blocks that are far in the future
+        // so check if the current slot is in line with the recent progression.
+        let max_index = recent_slots.len() - 1;
+        let median_index = max_index / 2;
+        let median_recent_slot = recent_slots[median_index];
+        let expected_current_slot = median_recent_slot + (max_index - median_index) as u64;
+        let max_reasonable_current_slot = expected_current_slot + MAX_SLOT_SKIP_DISTANCE;
+
+        // Return the highest slot that doesn't exceed what we believe is a
+        // reasonable slot.
+        recent_slots
+            .into_iter()
+            .rev()
+            .find(|slot| *slot <= max_reasonable_current_slot)
+            .unwrap()
+    }
+}
+
+#[cfg(test)]
+impl From<Vec<Slot>> for RecentLeaderSlots {
+    fn from(recent_slots: Vec<Slot>) -> Self {
+        assert!(!recent_slots.is_empty());
+        Self(Arc::new(RwLock::new(recent_slots.into_iter().collect())))
+    }
+}
+
+/// Service that tracks upcoming leaders and maintains an up-to-date mapping
+/// of leader id to TPU socket address.
+struct LeaderTpuService {
+    recent_slots: RecentLeaderSlots,
+    leader_tpu_cache: Arc<RwLock<LeaderTpuCache>>,
+    subscription: Option<PubsubClientSubscription<SlotUpdate>>,
+    t_leader_tpu_service: Option<JoinHandle<()>>,
+}
+
+impl LeaderTpuService {
+    fn new(rpc_client: Arc<RpcClient>, websocket_url: &str, exit: Arc<AtomicBool>) -> Result<Self> {
+        let start_slot = rpc_client.get_max_shred_insert_slot()?;
+
+        let recent_slots = RecentLeaderSlots::new(start_slot);
+        let leader_tpu_cache = Arc::new(RwLock::new(LeaderTpuCache::new(&rpc_client, start_slot)));
+
+        let subscription = if !websocket_url.is_empty() {
+            let recent_slots = recent_slots.clone();
+            Some(PubsubClient::slot_updates_subscribe(
+                websocket_url,
+                move |update| {
+                    let current_slot = match update {
+                        // This update indicates that a full slot was received by the connected
+                        // node so we can stop sending transactions to the leader for that slot
+                        SlotUpdate::Completed { slot, .. } => slot.saturating_add(1),
+                        // This update indicates that we have just received the first shred from
+                        // the leader for this slot and they are probably still accepting transactions.
+                        SlotUpdate::FirstShredReceived { slot, .. } => slot,
+                        _ => return,
+                    };
+
+                    recent_slots.record_slot(current_slot);
+                },
+            )?)
+        } else {
+            None
+        };
+
+        let t_leader_tpu_service = Some({
+            let recent_slots = recent_slots.clone();
+            let leader_tpu_cache = leader_tpu_cache.clone();
+            std::thread::Builder::new()
+                .name("ldr-tpu-srv".to_string())
+                .spawn(move || Self::run(rpc_client, recent_slots, leader_tpu_cache, exit))
+                .unwrap()
+        });
+
+        Ok(LeaderTpuService {
+            recent_slots,
+            leader_tpu_cache,
+            subscription,
+            t_leader_tpu_service,
+        })
+    }
+
+    fn join(&mut self) {
+        if let Some(mut subscription) = self.subscription.take() {
+            let _ = subscription.send_unsubscribe();
+            let _ = subscription.shutdown();
+        }
+        if let Some(t_handle) = self.t_leader_tpu_service.take() {
+            t_handle.join().unwrap();
+        }
+    }
+
+    fn leader_tpu_sockets(&self, fanout_slots: u64) -> Vec<SocketAddr> {
+        let current_slot = self.recent_slots.estimated_current_slot();
+        self.leader_tpu_cache
+            .read()
+            .unwrap()
+            .get_leader_sockets(current_slot, fanout_slots)
+    }
+
+    fn run(
+        rpc_client: Arc<RpcClient>,
+        recent_slots: RecentLeaderSlots,
+        leader_tpu_cache: Arc<RwLock<LeaderTpuCache>>,
+        exit: Arc<AtomicBool>,
+    ) {
+        let mut last_cluster_refresh = Instant::now();
+        let mut sleep_ms = 1000;
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            // Refresh cluster TPU ports every 5min in case validators restart with new port configuration
+            // or new validators come online
+            if last_cluster_refresh.elapsed() > Duration::from_secs(5 * 60) {
+                if let Ok(leader_tpu_map) = LeaderTpuCache::fetch_cluster_tpu_sockets(&rpc_client) {
+                    leader_tpu_cache.write().unwrap().leader_tpu_map = leader_tpu_map;
+                    last_cluster_refresh = Instant::now();
+                } else {
+                    sleep_ms = 100;
+                    continue;
+                }
+            }
+
+            // Sleep a few slots before checking if leader cache needs to be refreshed again
+            std::thread::sleep(Duration::from_millis(sleep_ms));
+
+            let current_slot = recent_slots.estimated_current_slot();
+            if current_slot
+                >= leader_tpu_cache
+                    .read()
+                    .unwrap()
+                    .last_slot()
+                    .saturating_sub(MAX_FANOUT_SLOTS)
+            {
+                if let Ok(slot_leaders) =
+                    LeaderTpuCache::fetch_slot_leaders(&rpc_client, current_slot)
+                {
+                    let mut leader_tpu_cache = leader_tpu_cache.write().unwrap();
+                    leader_tpu_cache.first_slot = current_slot;
+                    leader_tpu_cache.leaders = slot_leaders;
+                } else {
+                    sleep_ms = 100;
+                    continue;
+                }
+            }
+
+            sleep_ms = 1000;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_slot(recent_slots: RecentLeaderSlots, expected_slot: Slot) {
+        assert_eq!(recent_slots.estimated_current_slot(), expected_slot);
+    }
+
+    #[test]
+    fn test_recent_leader_slots() {
+        assert_slot(RecentLeaderSlots::new(0), 0);
+
+        let mut recent_slots: Vec<Slot> = (1..=12).collect();
+        assert_slot(RecentLeaderSlots::from(recent_slots.clone()), 12);
+
+        recent_slots.reverse();
+        assert_slot(RecentLeaderSlots::from(recent_slots), 12);
+
+        assert_slot(
+            RecentLeaderSlots::from(vec![0, 1 + MAX_SLOT_SKIP_DISTANCE]),
+            1 + MAX_SLOT_SKIP_DISTANCE,
+        );
+        assert_slot(
+            RecentLeaderSlots::from(vec![0, 2 + MAX_SLOT_SKIP_DISTANCE]),
+            0,
+        );
+
+        assert_slot(RecentLeaderSlots::from(vec![1]), 1);
+        assert_slot(RecentLeaderSlots::from(vec![1, 100]), 1);
+        assert_slot(RecentLeaderSlots::from(vec![1, 2, 100]), 2);
+        assert_slot(RecentLeaderSlots::from(vec![1, 2, 3, 100]), 3);
+        assert_slot(RecentLeaderSlots::from(vec![1, 2, 3, 99, 100]), 3);
+    }
+}

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.0"
+version = "1.6.7"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -17,6 +17,7 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
 ahash = "0.6.1"
 base64 = "0.12.3"
 bincode = "1.3.1"
+blake3 = "0.3.7"
 bv = { version = "0.11.1", features = ["serde"] }
 bs58 = "0.3.1"
 byteorder = "1.3.4"
@@ -47,48 +48,47 @@ raptorq = "1.4.2"
 rayon = "1.5.0"
 regex = "1.3.9"
 retain_mut = "0.1.2"
-rustversion = "1.0.4"
 serde = "1.0.122"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.6.0" }
-solana-banks-server = { path = "../banks-server", version = "1.6.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.6.0" }
-solana-client = { path = "../client", version = "1.6.0" }
-solana-faucet = { path = "../faucet", version = "1.6.0" }
-solana-ledger = { path = "../ledger", version = "1.6.0" }
-solana-logger = { path = "../logger", version = "1.6.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.6.0" }
-solana-metrics = { path = "../metrics", version = "1.6.0" }
-solana-measure = { path = "../measure", version = "1.6.0" }
-solana-net-utils = { path = "../net-utils", version = "1.6.0" }
-solana-perf = { path = "../perf", version = "1.6.0" }
-solana-program-test = { path = "../program-test", version = "1.6.0" }
-solana-runtime = { path = "../runtime", version = "1.6.0" }
-solana-sdk = { path = "../sdk", version = "1.6.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "1.6.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.6.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.6.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.6.0" }
-solana-streamer = { path = "../streamer", version = "1.6.0" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.6.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.6.0" }
-solana-version = { path = "../version", version = "1.6.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.6.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.7" }
+solana-banks-server = { path = "../banks-server", version = "=1.6.7" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.7" }
+solana-client = { path = "../client", version = "=1.6.7" }
+solana-faucet = { path = "../faucet", version = "=1.6.7" }
+solana-ledger = { path = "../ledger", version = "=1.6.7" }
+solana-logger = { path = "../logger", version = "=1.6.7" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.6.7" }
+solana-metrics = { path = "../metrics", version = "=1.6.7" }
+solana-measure = { path = "../measure", version = "=1.6.7" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.7" }
+solana-perf = { path = "../perf", version = "=1.6.7" }
+solana-program-test = { path = "../program-test", version = "=1.6.7" }
+solana-runtime = { path = "../runtime", version = "=1.6.7" }
+solana-sdk = { path = "../sdk", version = "=1.6.7" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.6.7" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.6.7" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.7" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.6.7" }
+solana-streamer = { path = "../streamer", version = "=1.6.7" }
+solana-sys-tuner = { path = "../sys-tuner", version = "=1.6.7" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.7" }
+solana-version = { path = "../version", version = "=1.6.7" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.7" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
-tokio = { version = "1.1", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
 tokio_02 = { version = "0.2", package = "tokio", features = ["full"] }
 tokio-util = { version = "0.3", features = ["codec"] } # This crate needs to stay in sync with tokio_02, until that dependency can be removed
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.6.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.6.7" }
 trees = "0.2.1"
 
 [dev-dependencies]
 matches = "0.1.6"
 num_cpus = "1.13.0"
-reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serial_test = "0.4.0"
 symlink = "0.1.0"
 systemstat = "0.1.5"

core/benches/banking_stage.rs

@@ -7,7 +7,7 @@ use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::banking_stage::{create_test_recorder, BankingStage};
+use solana_core::banking_stage::{create_test_recorder, BankingStage, BankingStageStats};
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
 use solana_core::poh_recorder::WorkingBankEntry;
@@ -66,6 +66,8 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
         let (exit, poh_recorder, poh_service, _signal_receiver) =
             create_test_recorder(&bank, &blockstore, None);
 
+        let recorder = poh_recorder.lock().unwrap().recorder();
+
         let tx = test_tx();
         let len = 4096;
         let chunk_size = 1024;
@@ -81,12 +83,14 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
         bencher.iter(move || {
             let _ignored = BankingStage::consume_buffered_packets(
                 &my_pubkey,
+                std::u128::MAX,
                 &poh_recorder,
                 &mut packets,
                 None,
                 &s,
                 None::<Box<dyn Fn()>>,
-                None,
+                &BankingStageStats::default(),
+                &recorder,
             );
         });
 
@@ -154,6 +158,9 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
 
     let (verified_sender, verified_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
+    let mut bank = Bank::new(&genesis_config);
+    // Allow arbitrary transaction processing time for the purposes of this bench
+    bank.ns_per_slot = std::u128::MAX;
     let bank = Arc::new(Bank::new(&genesis_config));
 
     debug!("threads: {} txs: {}", num_threads, txes);
@@ -295,7 +302,7 @@ fn simulate_process_entries(
         hash: next_hash(&bank.last_blockhash(), 1, &tx_vector),
         transactions: tx_vector,
     };
-    process_entries(&bank, &[entry], randomize_txs, None, None).unwrap();
+    process_entries(&bank, &mut [entry], randomize_txs, None, None).unwrap();
 }
 
 #[allow(clippy::same_item_push)]

core/benches/crds_shards.rs

@@ -3,30 +3,34 @@
 extern crate test;
 
 use rand::{thread_rng, Rng};
-use solana_core::contact_info::ContactInfo;
-use solana_core::crds::VersionedCrdsValue;
-use solana_core::crds_shards::CrdsShards;
-use solana_core::crds_value::{CrdsData, CrdsValue};
-use solana_sdk::pubkey;
+use solana_core::{
+    crds::{Crds, VersionedCrdsValue},
+    crds_shards::CrdsShards,
+    crds_value::CrdsValue,
+};
 use solana_sdk::timing::timestamp;
+use std::iter::repeat_with;
 use test::Bencher;
 
 const CRDS_SHARDS_BITS: u32 = 8;
 
-fn new_test_crds_value() -> VersionedCrdsValue {
-    let data = CrdsData::ContactInfo(ContactInfo::new_localhost(&pubkey::new_rand(), timestamp()));
-    VersionedCrdsValue::new(timestamp(), CrdsValue::new_unsigned(data))
+fn new_test_crds_value<R: Rng>(rng: &mut R) -> VersionedCrdsValue {
+    let value = CrdsValue::new_rand(rng, None);
+    let label = value.label();
+    let mut crds = Crds::default();
+    crds.insert(value, timestamp()).unwrap();
+    crds.remove(&label).unwrap()
 }
 
 fn bench_crds_shards_find(bencher: &mut Bencher, num_values: usize, mask_bits: u32) {
-    let values: Vec<VersionedCrdsValue> = std::iter::repeat_with(new_test_crds_value)
+    let mut rng = thread_rng();
+    let values: Vec<_> = repeat_with(|| new_test_crds_value(&mut rng))
         .take(num_values)
         .collect();
     let mut shards = CrdsShards::new(CRDS_SHARDS_BITS);
     for (index, value) in values.iter().enumerate() {
         assert!(shards.insert(index, value));
     }
-    let mut rng = thread_rng();
     bencher.iter(|| {
         let mask = rng.gen();
         let _hits = shards.find(mask, mask_bits).count();

core/benches/retransmit_stage.rs

@@ -81,8 +81,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     let keypair = Arc::new(Keypair::new());
     let slot = 0;
     let parent = 0;
-    let shredder =
-        Shredder::new(slot, parent, 0.0, keypair, 0, 0).expect("Failed to create entry shredder");
+    let shredder = Shredder::new(slot, parent, keypair, 0, 0).unwrap();
     let mut data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
 
     let num_packets = data_shreds.len();

core/benches/shredder.rs

@@ -8,8 +8,8 @@ use raptorq::{Decoder, Encoder};
 use solana_ledger::entry::{create_ticks, Entry};
 use solana_ledger::shred::{
     max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, Shredder,
-    MAX_DATA_SHREDS_PER_FEC_BLOCK, RECOMMENDED_FEC_RATE, SHRED_PAYLOAD_SIZE,
-    SIZE_OF_DATA_SHRED_IGNORED_TAIL, SIZE_OF_DATA_SHRED_PAYLOAD,
+    MAX_DATA_SHREDS_PER_FEC_BLOCK, SHRED_PAYLOAD_SIZE, SIZE_OF_DATA_SHRED_IGNORED_TAIL,
+    SIZE_OF_DATA_SHRED_PAYLOAD,
 };
 use solana_perf::test_tx;
 use solana_sdk::hash::Hash;
@@ -39,10 +39,15 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
         Some(shred_size),
     );
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
-    let shredder =
-        Shredder::new(1, 0, RECOMMENDED_FEC_RATE, Arc::new(Keypair::new()), 0, 0).unwrap();
+    let shredder = Shredder::new(1, 0, Arc::new(Keypair::new()), 0, 0).unwrap();
     let data_shreds = shredder
-        .entries_to_data_shreds(&entries, true, 0, &mut ProcessShredsStats::default())
+        .entries_to_data_shreds(
+            &entries,
+            true, // is_last_in_slot
+            0,    // next_shred_index
+            0,    // fec_set_offset
+            &mut ProcessShredsStats::default(),
+        )
         .0;
     assert!(data_shreds.len() >= num_shreds);
     data_shreds
@@ -69,7 +74,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
     let num_ticks = max_ticks_per_n_shreds(1, Some(SIZE_OF_DATA_SHRED_PAYLOAD)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
+        let shredder = Shredder::new(1, 0, kp.clone(), 0, 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -88,7 +93,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     // 1Mb
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
+        let shredder = Shredder::new(1, 0, kp.clone(), 0, 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -101,7 +106,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
     let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
     let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
-    let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0, 0).unwrap();
+    let shredder = Shredder::new(1, 0, kp, 0, 0).unwrap();
     let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
     bencher.iter(|| {
         let raw = &mut Shredder::deshred(&data_shreds).unwrap();
@@ -127,11 +132,8 @@ fn bench_shredder_coding(bencher: &mut Bencher) {
     let data_shreds = make_shreds(symbol_count);
     bencher.iter(|| {
         Shredder::generate_coding_shreds(
-            0,
-            RECOMMENDED_FEC_RATE,
             &data_shreds[..symbol_count],
-            0,
-            symbol_count,
+            true, // is_last_in_slot
         )
         .len();
     })
@@ -142,20 +144,16 @@ fn bench_shredder_decoding(bencher: &mut Bencher) {
     let symbol_count = MAX_DATA_SHREDS_PER_FEC_BLOCK as usize;
     let data_shreds = make_shreds(symbol_count);
     let coding_shreds = Shredder::generate_coding_shreds(
-        0,
-        RECOMMENDED_FEC_RATE,
         &data_shreds[..symbol_count],
-        0,
-        symbol_count,
+        true, // is_last_in_slot
     );
     bencher.iter(|| {
         Shredder::try_recovery(
             coding_shreds[..].to_vec(),
             symbol_count,
             symbol_count,
-            0,
-            0,
-            1,
+            0, // first index
+            1, // slot
         )
         .unwrap();
     })

core/src/broadcast_stage.rs

@@ -373,7 +373,7 @@ pub fn get_broadcast_peers(
 /// # Remarks
 pub fn broadcast_shreds(
     s: &UdpSocket,
-    shreds: &Arc<Vec<Shred>>,
+    shreds: &[Shred],
     peers_and_stakes: &[(u64, usize)],
     peers: &[ContactInfo],
     last_datapoint_submit: &Arc<AtomicU64>,
@@ -447,7 +447,7 @@ pub mod test {
         entry::create_ticks,
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
-        shred::{max_ticks_per_n_shreds, ProcessShredsStats, Shredder, RECOMMENDED_FEC_RATE},
+        shred::{max_ticks_per_n_shreds, ProcessShredsStats, Shredder},
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::{
@@ -472,12 +472,14 @@ pub mod test {
     ) {
         let num_entries = max_ticks_per_n_shreds(num, None);
         let (data_shreds, _) = make_slot_entries(slot, 0, num_entries);
-        let keypair = Arc::new(Keypair::new());
-        let shredder = Shredder::new(slot, 0, RECOMMENDED_FEC_RATE, keypair, 0, 0)
-            .expect("Expected to create a new shredder");
-
-        let coding_shreds = shredder
-            .data_shreds_to_coding_shreds(&data_shreds[0..], &mut ProcessShredsStats::default());
+        let keypair = Keypair::new();
+        let coding_shreds = Shredder::data_shreds_to_coding_shreds(
+            &keypair,
+            &data_shreds[0..],
+            true, // is_last_in_slot
+            &mut ProcessShredsStats::default(),
+        )
+        .unwrap();
         (
             data_shreds.clone(),
             coding_shreds.clone(),

core/src/broadcast_stage/broadcast_fake_shreds_run.rs

@@ -1,6 +1,6 @@
 use super::*;
 use solana_ledger::entry::Entry;
-use solana_ledger::shred::{Shredder, RECOMMENDED_FEC_RATE};
+use solana_ledger::shred::Shredder;
 use solana_sdk::hash::Hash;
 use solana_sdk::signature::Keypair;
 
@@ -47,7 +47,6 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         let shredder = Shredder::new(
             bank.slot(),
             bank.parent().unwrap().slot(),
-            RECOMMENDED_FEC_RATE,
             self.keypair.clone(),
             (bank.tick_height() % bank.ticks_per_slot()) as u8,
             self.shred_version,

core/src/broadcast_stage/broadcast_utils.rs

@@ -1,6 +1,6 @@
 use crate::poh_recorder::WorkingBankEntry;
 use crate::result::Result;
-use solana_ledger::entry::Entry;
+use solana_ledger::{entry::Entry, shred::Shred};
 use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
 use std::{
@@ -16,11 +16,15 @@ pub(super) struct ReceiveResults {
     pub last_tick_height: u64,
 }
 
-#[derive(Copy, Clone)]
+#[derive(Clone)]
 pub struct UnfinishedSlotInfo {
     pub next_shred_index: u32,
     pub slot: Slot,
     pub parent: Slot,
+    // Data shreds buffered to make a batch of size
+    // MAX_DATA_SHREDS_PER_FEC_BLOCK.
+    pub(crate) data_shreds_buffer: Vec<Shred>,
+    pub(crate) fec_set_offset: u32, // See Shredder::fec_set_index.
 }
 
 /// This parameter tunes how many entries are received in one iteration of recv loop

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -71,7 +71,6 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         let shredder = Shredder::new(
             bank.slot(),
             bank.parent().unwrap().slot(),
-            0.0,
             self.keypair.clone(),
             (bank.tick_height() % bank.ticks_per_slot()) as u8,
             self.shred_version,

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -7,12 +7,13 @@ use super::{
 use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
 use solana_ledger::{
     entry::Entry,
-    shred::{ProcessShredsStats, Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK},
+    shred::{
+        ProcessShredsStats, Shred, Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK,
+        SHRED_TICK_REFERENCE_MASK,
+    },
 };
 use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
-use std::collections::HashMap;
-use std::sync::RwLock;
-use std::time::Duration;
+use std::{collections::HashMap, ops::Deref, sync::RwLock, time::Duration};
 
 #[derive(Clone)]
 pub struct StandardBroadcastRun {
@@ -40,94 +41,113 @@ impl StandardBroadcastRun {
     pub(super) fn new(keypair: Arc<Keypair>, shred_version: u16) -> Self {
         Self {
             process_shreds_stats: ProcessShredsStats::default(),
-            transmit_shreds_stats: Arc::new(Mutex::new(SlotBroadcastStats::default())),
-            insert_shreds_stats: Arc::new(Mutex::new(SlotBroadcastStats::default())),
+            transmit_shreds_stats: Arc::default(),
+            insert_shreds_stats: Arc::default(),
             unfinished_slot: None,
             current_slot_and_parent: None,
             slot_broadcast_start: None,
             keypair,
             shred_version,
-            last_datapoint_submit: Arc::new(AtomicU64::new(0)),
+            last_datapoint_submit: Arc::default(),
             num_batches: 0,
-            broadcast_peer_cache: Arc::new(RwLock::new(BroadcastPeerCache::default())),
-            last_peer_update: Arc::new(AtomicU64::new(0)),
+            broadcast_peer_cache: Arc::default(),
+            last_peer_update: Arc::default(),
         }
     }
 
-    fn check_for_interrupted_slot(&mut self, max_ticks_in_slot: u8) -> Option<Shred> {
-        let (slot, _) = self.current_slot_and_parent.unwrap();
-        let mut last_unfinished_slot_shred = self
-            .unfinished_slot
-            .map(|last_unfinished_slot| {
-                if last_unfinished_slot.slot != slot {
-                    self.report_and_reset_stats();
-                    Some(Shred::new_from_data(
-                        last_unfinished_slot.slot,
-                        last_unfinished_slot.next_shred_index,
-                        (last_unfinished_slot.slot - last_unfinished_slot.parent) as u16,
-                        None,
-                        true,
-                        true,
-                        max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK,
-                        self.shred_version,
-                        last_unfinished_slot.next_shred_index,
-                    ))
-                } else {
-                    None
-                }
-            })
-            .unwrap_or(None);
-
-        // This shred should only be Some if the previous slot was interrupted
-        if let Some(ref mut shred) = last_unfinished_slot_shred {
-            Shredder::sign_shred(&self.keypair, shred);
-            self.unfinished_slot = None;
+    // If the current slot has changed, generates an empty shred indicating
+    // last shred in the previous slot, along with coding shreds for the data
+    // shreds buffered.
+    fn finish_prev_slot(
+        &mut self,
+        max_ticks_in_slot: u8,
+        stats: &mut ProcessShredsStats,
+    ) -> Vec<Shred> {
+        let (current_slot, _) = self.current_slot_and_parent.unwrap();
+        match self.unfinished_slot {
+            None => Vec::default(),
+            Some(ref state) if state.slot == current_slot => Vec::default(),
+            Some(ref mut state) => {
+                let parent_offset = state.slot - state.parent;
+                let reference_tick = max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK;
+                let fec_set_index =
+                    Shredder::fec_set_index(state.next_shred_index, state.fec_set_offset);
+                let mut shred = Shred::new_from_data(
+                    state.slot,
+                    state.next_shred_index,
+                    parent_offset as u16,
+                    None, // data
+                    true, // is_last_in_fec_set
+                    true, // is_last_in_slot
+                    reference_tick,
+                    self.shred_version,
+                    fec_set_index.unwrap(),
+                );
+                Shredder::sign_shred(self.keypair.deref(), &mut shred);
+                state.data_shreds_buffer.push(shred.clone());
+                let mut shreds = make_coding_shreds(
+                    self.keypair.deref(),
+                    &mut self.unfinished_slot,
+                    true, // is_last_in_slot
+                    stats,
+                );
+                shreds.insert(0, shred);
+                self.report_and_reset_stats();
+                self.unfinished_slot = None;
+                shreds
+            }
         }
+    }
 
-        last_unfinished_slot_shred
-    }
-    fn init_shredder(&self, blockstore: &Blockstore, reference_tick: u8) -> (Shredder, u32) {
-        let (slot, parent_slot) = self.current_slot_and_parent.unwrap();
-        let next_shred_index = self
-            .unfinished_slot
-            .map(|s| s.next_shred_index)
-            .unwrap_or_else(|| {
-                blockstore
-                    .meta(slot)
-                    .expect("Database error")
-                    .map(|meta| meta.consumed)
-                    .unwrap_or(0) as u32
-            });
-        (
-            Shredder::new(
-                slot,
-                parent_slot,
-                RECOMMENDED_FEC_RATE,
-                self.keypair.clone(),
-                reference_tick,
-                self.shred_version,
-            )
-            .expect("Expected to create a new shredder"),
-            next_shred_index,
-        )
-    }
     fn entries_to_data_shreds(
         &mut self,
-        shredder: &Shredder,
-        next_shred_index: u32,
         entries: &[Entry],
+        blockstore: &Blockstore,
+        reference_tick: u8,
         is_slot_end: bool,
         process_stats: &mut ProcessShredsStats,
     ) -> Vec<Shred> {
-        let (data_shreds, new_next_shred_index) =
-            shredder.entries_to_data_shreds(entries, is_slot_end, next_shred_index, process_stats);
-
+        let (slot, parent_slot) = self.current_slot_and_parent.unwrap();
+        let (next_shred_index, fec_set_offset) = match &self.unfinished_slot {
+            Some(state) => (state.next_shred_index, state.fec_set_offset),
+            None => match blockstore.meta(slot).unwrap() {
+                Some(slot_meta) => {
+                    let shreds_consumed = slot_meta.consumed as u32;
+                    (shreds_consumed, shreds_consumed)
+                }
+                None => (0, 0),
+            },
+        };
+        let (data_shreds, next_shred_index) = Shredder::new(
+            slot,
+            parent_slot,
+            self.keypair.clone(),
+            reference_tick,
+            self.shred_version,
+        )
+        .unwrap()
+        .entries_to_data_shreds(
+            entries,
+            is_slot_end,
+            next_shred_index,
+            fec_set_offset,
+            process_stats,
+        );
+        let mut data_shreds_buffer = match &mut self.unfinished_slot {
+            Some(state) => {
+                assert_eq!(state.slot, slot);
+                std::mem::take(&mut state.data_shreds_buffer)
+            }
+            None => Vec::default(),
+        };
+        data_shreds_buffer.extend(data_shreds.clone());
         self.unfinished_slot = Some(UnfinishedSlotInfo {
-            next_shred_index: new_next_shred_index,
-            slot: shredder.slot,
-            parent: shredder.parent_slot,
+            next_shred_index,
+            slot,
+            parent: parent_slot,
+            data_shreds_buffer,
+            fec_set_offset,
         });
-
         data_shreds
     }
 
@@ -184,19 +204,16 @@ impl StandardBroadcastRun {
         let mut to_shreds_time = Measure::start("broadcast_to_shreds");
 
         // 1) Check if slot was interrupted
-        let last_unfinished_slot_shred =
-            self.check_for_interrupted_slot(bank.ticks_per_slot() as u8);
+        let prev_slot_shreds =
+            self.finish_prev_slot(bank.ticks_per_slot() as u8, &mut process_stats);
 
         // 2) Convert entries to shreds and coding shreds
-        let (shredder, next_shred_index) = self.init_shredder(
-            blockstore,
-            (bank.tick_height() % bank.ticks_per_slot()) as u8,
-        );
         let is_last_in_slot = last_tick_height == bank.max_tick_height();
+        let reference_tick = bank.tick_height() % bank.ticks_per_slot();
         let data_shreds = self.entries_to_data_shreds(
-            &shredder,
-            next_shred_index,
             &receive_results.entries,
+            blockstore,
+            reference_tick as u8,
             is_last_in_slot,
             &mut process_stats,
         );
@@ -208,27 +225,25 @@ impl StandardBroadcastRun {
                 .insert_shreds(first, None, true)
                 .expect("Failed to insert shreds in blockstore");
         }
-        let last_data_shred = data_shreds.len();
         to_shreds_time.stop();
 
         let mut get_leader_schedule_time = Measure::start("broadcast_get_leader_schedule");
         let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-        let stakes = bank.epoch_staked_nodes(bank_epoch);
-        let stakes = stakes.map(Arc::new);
+        let stakes = bank.epoch_staked_nodes(bank_epoch).map(Arc::new);
 
         // Broadcast the last shred of the interrupted slot if necessary
-        if let Some(last_shred) = last_unfinished_slot_shred {
+        if !prev_slot_shreds.is_empty() {
             let batch_info = Some(BroadcastShredBatchInfo {
-                slot: last_shred.slot(),
+                slot: prev_slot_shreds[0].slot(),
                 num_expected_batches: Some(old_num_batches + 1),
                 slot_start_ts: old_broadcast_start.expect(
                     "Old broadcast start time for previous slot must exist if the previous slot
                  was interrupted",
                 ),
             });
-            let last_shred = Arc::new(vec![last_shred]);
-            socket_sender.send(((stakes.clone(), last_shred.clone()), batch_info.clone()))?;
-            blockstore_sender.send((last_shred, batch_info))?;
+            let shreds = Arc::new(prev_slot_shreds);
+            socket_sender.send(((stakes.clone(), shreds.clone()), batch_info.clone()))?;
+            blockstore_sender.send((shreds, batch_info))?;
         }
 
         // Increment by two batches, one for the data batch, one for the coding batch.
@@ -245,7 +260,6 @@ impl StandardBroadcastRun {
             num_expected_batches,
             slot_start_ts: self
                 .slot_broadcast_start
-                .clone()
                 .expect("Start timestamp must exist for a slot if we're broadcasting the slot"),
         });
         get_leader_schedule_time.stop();
@@ -255,11 +269,15 @@ impl StandardBroadcastRun {
         // Send data shreds
         let data_shreds = Arc::new(data_shreds);
         socket_sender.send(((stakes.clone(), data_shreds.clone()), batch_info.clone()))?;
-        blockstore_sender.send((data_shreds.clone(), batch_info.clone()))?;
+        blockstore_sender.send((data_shreds, batch_info.clone()))?;
 
         // Create and send coding shreds
-        let coding_shreds = shredder
-            .data_shreds_to_coding_shreds(&data_shreds[0..last_data_shred], &mut process_stats);
+        let coding_shreds = make_coding_shreds(
+            self.keypair.deref(),
+            &mut self.unfinished_slot,
+            is_last_in_slot,
+            &mut process_stats,
+        );
         let coding_shreds = Arc::new(coding_shreds);
         socket_sender.send(((stakes, coding_shreds.clone()), batch_info.clone()))?;
         blockstore_sender.send((coding_shreds, batch_info))?;
@@ -378,15 +396,15 @@ impl StandardBroadcastRun {
 
     fn report_and_reset_stats(&mut self) {
         let stats = &self.process_shreds_stats;
-        assert!(self.unfinished_slot.is_some());
+        let unfinished_slot = self.unfinished_slot.as_ref().unwrap();
         datapoint_info!(
             "broadcast-process-shreds-stats",
-            ("slot", self.unfinished_slot.unwrap().slot as i64, i64),
+            ("slot", unfinished_slot.slot as i64, i64),
             ("shredding_time", stats.shredding_elapsed, i64),
             ("receive_time", stats.receive_elapsed, i64),
             (
                 "num_data_shreds",
-                i64::from(self.unfinished_slot.unwrap().next_shred_index),
+                unfinished_slot.next_shred_index as i64,
                 i64
             ),
             (
@@ -409,6 +427,32 @@ impl StandardBroadcastRun {
     }
 }
 
+// Consumes data_shreds_buffer returning corresponding coding shreds.
+fn make_coding_shreds(
+    keypair: &Keypair,
+    unfinished_slot: &mut Option<UnfinishedSlotInfo>,
+    is_slot_end: bool,
+    stats: &mut ProcessShredsStats,
+) -> Vec<Shred> {
+    let data_shreds = match unfinished_slot {
+        None => Vec::default(),
+        Some(unfinished_slot) => {
+            let size = unfinished_slot.data_shreds_buffer.len();
+            // Consume a multiple of 32, unless this is the slot end.
+            let offset = if is_slot_end {
+                0
+            } else {
+                size % MAX_DATA_SHREDS_PER_FEC_BLOCK as usize
+            };
+            unfinished_slot
+                .data_shreds_buffer
+                .drain(0..size - offset)
+                .collect()
+        }
+    };
+    Shredder::data_shreds_to_coding_shreds(keypair, &data_shreds, is_slot_end, stats).unwrap()
+}
+
 impl BroadcastRun for StandardBroadcastRun {
     fn run(
         &mut self,
@@ -418,6 +462,8 @@ impl BroadcastRun for StandardBroadcastRun {
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         let receive_results = broadcast_utils::recv_slot_entries(receiver)?;
+        // TODO: Confirm that last chunk of coding shreds
+        // will not be lost or delayed for too long.
         self.process_receive_results(
             blockstore,
             socket_sender,
@@ -508,6 +554,8 @@ mod test {
             next_shred_index,
             slot,
             parent,
+            data_shreds_buffer: Vec::default(),
+            fec_set_offset: next_shred_index,
         });
         run.slot_broadcast_start = Some(Instant::now());
 
@@ -515,8 +563,9 @@ mod test {
         run.current_slot_and_parent = Some((4, 2));
 
         // Slot 2 interrupted slot 1
-        let shred = run
-            .check_for_interrupted_slot(0)
+        let shreds = run.finish_prev_slot(0, &mut ProcessShredsStats::default());
+        let shred = shreds
+            .get(0)
             .expect("Expected a shred that signals an interrupt");
 
         // Validate the shred
@@ -642,6 +691,50 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_buffer_data_shreds() {
+        let num_shreds_per_slot = 2;
+        let (blockstore, genesis_config, _cluster_info, bank, leader_keypair, _socket) =
+            setup(num_shreds_per_slot);
+        let (bsend, brecv) = channel();
+        let (ssend, _srecv) = channel();
+        let mut last_tick_height = 0;
+        let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair, 0);
+        let mut process_ticks = |num_ticks| {
+            let ticks = create_ticks(num_ticks, 0, genesis_config.hash());
+            last_tick_height += (ticks.len() - 1) as u64;
+            let receive_results = ReceiveResults {
+                entries: ticks,
+                time_elapsed: Duration::new(1, 0),
+                bank: bank.clone(),
+                last_tick_height,
+            };
+            standard_broadcast_run
+                .process_receive_results(&blockstore, &ssend, &bsend, receive_results)
+                .unwrap();
+        };
+        for i in 0..3 {
+            process_ticks((i + 1) * 100);
+        }
+        let mut shreds = Vec::<Shred>::new();
+        while let Ok((recv_shreds, _)) = brecv.recv_timeout(Duration::from_secs(1)) {
+            shreds.extend(recv_shreds.deref().clone());
+        }
+        assert!(shreds.len() < 32, "shreds.len(): {}", shreds.len());
+        assert!(shreds.iter().all(|shred| shred.is_data()));
+        process_ticks(75);
+        while let Ok((recv_shreds, _)) = brecv.recv_timeout(Duration::from_secs(1)) {
+            shreds.extend(recv_shreds.deref().clone());
+        }
+        assert!(shreds.len() > 64, "shreds.len(): {}", shreds.len());
+        let num_coding_shreds = shreds.iter().filter(|shred| shred.is_code()).count();
+        assert_eq!(
+            num_coding_shreds, 32,
+            "num coding shreds: {}",
+            num_coding_shreds
+        );
+    }
+
     #[test]
     fn test_slot_finish() {
         // Setup

core/src/cluster_info_metrics.rs

@@ -0,0 +1,394 @@
+use crate::crds_gossip::CrdsGossip;
+use solana_measure::measure::Measure;
+use std::{
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        RwLock,
+    },
+    time::Instant,
+};
+
+#[derive(Default)]
+pub(crate) struct Counter(AtomicU64);
+
+impl Counter {
+    pub(crate) fn add_measure(&self, x: &mut Measure) {
+        x.stop();
+        self.0.fetch_add(x.as_us(), Ordering::Relaxed);
+    }
+    pub(crate) fn add_relaxed(&self, x: u64) {
+        self.0.fetch_add(x, Ordering::Relaxed);
+    }
+    fn clear(&self) -> u64 {
+        self.0.swap(0, Ordering::Relaxed)
+    }
+}
+
+pub(crate) struct ScopedTimer<'a> {
+    clock: Instant,
+    metric: &'a AtomicU64,
+}
+
+impl<'a> From<&'a Counter> for ScopedTimer<'a> {
+    // Output should be assigned to a *named* variable, otherwise it is
+    // immediately dropped.
+    #[must_use]
+    fn from(counter: &'a Counter) -> Self {
+        Self {
+            clock: Instant::now(),
+            metric: &counter.0,
+        }
+    }
+}
+
+impl Drop for ScopedTimer<'_> {
+    fn drop(&mut self) {
+        let micros = self.clock.elapsed().as_micros();
+        self.metric.fetch_add(micros as u64, Ordering::Relaxed);
+    }
+}
+
+#[derive(Default)]
+pub(crate) struct GossipStats {
+    pub(crate) all_tvu_peers: Counter,
+    pub(crate) entrypoint2: Counter,
+    pub(crate) entrypoint: Counter,
+    pub(crate) epoch_slots_lookup: Counter,
+    pub(crate) filter_pull_response: Counter,
+    pub(crate) generate_pull_responses: Counter,
+    pub(crate) get_accounts_hash: Counter,
+    pub(crate) get_votes: Counter,
+    pub(crate) gossip_packets_dropped_count: Counter,
+    pub(crate) handle_batch_ping_messages_time: Counter,
+    pub(crate) handle_batch_pong_messages_time: Counter,
+    pub(crate) handle_batch_prune_messages_time: Counter,
+    pub(crate) handle_batch_pull_requests_time: Counter,
+    pub(crate) handle_batch_pull_responses_time: Counter,
+    pub(crate) handle_batch_push_messages_time: Counter,
+    pub(crate) mark_pull_request: Counter,
+    pub(crate) new_pull_requests: Counter,
+    pub(crate) new_pull_requests_count: Counter,
+    pub(crate) new_pull_requests_pings_count: Counter,
+    pub(crate) new_push_requests2: Counter,
+    pub(crate) new_push_requests: Counter,
+    pub(crate) new_push_requests_num: Counter,
+    pub(crate) packets_received_count: Counter,
+    pub(crate) packets_received_prune_messages_count: Counter,
+    pub(crate) packets_received_pull_requests_count: Counter,
+    pub(crate) packets_received_pull_responses_count: Counter,
+    pub(crate) packets_received_push_messages_count: Counter,
+    pub(crate) packets_received_verified_count: Counter,
+    pub(crate) packets_sent_gossip_requests_count: Counter,
+    pub(crate) packets_sent_prune_messages_count: Counter,
+    pub(crate) packets_sent_pull_requests_count: Counter,
+    pub(crate) packets_sent_pull_responses_count: Counter,
+    pub(crate) packets_sent_push_messages_count: Counter,
+    pub(crate) process_gossip_packets_time: Counter,
+    pub(crate) process_prune: Counter,
+    pub(crate) process_pull_requests: Counter,
+    pub(crate) process_pull_response: Counter,
+    pub(crate) process_pull_response_count: Counter,
+    pub(crate) process_pull_response_fail_insert: Counter,
+    pub(crate) process_pull_response_fail_timeout: Counter,
+    pub(crate) process_pull_response_len: Counter,
+    pub(crate) process_pull_response_success: Counter,
+    pub(crate) process_pull_response_timeout: Counter,
+    pub(crate) process_push_message: Counter,
+    pub(crate) prune_message_count: Counter,
+    pub(crate) prune_message_len: Counter,
+    pub(crate) prune_received_cache: Counter,
+    pub(crate) pull_from_entrypoint_count: Counter,
+    pub(crate) pull_request_ping_pong_check_failed_count: Counter,
+    pub(crate) pull_requests_count: Counter,
+    pub(crate) purge: Counter,
+    pub(crate) push_message_count: Counter,
+    pub(crate) push_message_value_count: Counter,
+    pub(crate) push_response_count: Counter,
+    pub(crate) push_vote_read: Counter,
+    pub(crate) repair_peers: Counter,
+    pub(crate) require_stake_for_gossip_unknown_feature_set: Counter,
+    pub(crate) require_stake_for_gossip_unknown_stakes: Counter,
+    pub(crate) skip_pull_response_shred_version: Counter,
+    pub(crate) skip_pull_shred_version: Counter,
+    pub(crate) skip_push_message_shred_version: Counter,
+    pub(crate) trim_crds_table_failed: Counter,
+    pub(crate) trim_crds_table_purged_values_count: Counter,
+    pub(crate) tvu_peers: Counter,
+}
+
+pub(crate) fn submit_gossip_stats(stats: &GossipStats, gossip: &RwLock<CrdsGossip>) {
+    let (table_size, purged_values_size, failed_inserts_size) = {
+        let gossip = gossip.read().unwrap();
+        (
+            gossip.crds.len(),
+            gossip.pull.purged_values.len(),
+            gossip.pull.failed_inserts.len(),
+        )
+    };
+    datapoint_info!(
+        "cluster_info_stats",
+        ("entrypoint", stats.entrypoint.clear(), i64),
+        ("entrypoint2", stats.entrypoint2.clear(), i64),
+        ("push_vote_read", stats.push_vote_read.clear(), i64),
+        ("get_votes", stats.get_votes.clear(), i64),
+        ("get_accounts_hash", stats.get_accounts_hash.clear(), i64),
+        ("all_tvu_peers", stats.all_tvu_peers.clear(), i64),
+        ("tvu_peers", stats.tvu_peers.clear(), i64),
+        (
+            "new_push_requests_num",
+            stats.new_push_requests_num.clear(),
+            i64
+        ),
+        ("table_size", table_size as i64, i64),
+        ("purged_values_size", purged_values_size as i64, i64),
+        ("failed_inserts_size", failed_inserts_size as i64, i64),
+    );
+    datapoint_info!(
+        "cluster_info_stats2",
+        (
+            "gossip_packets_dropped_count",
+            stats.gossip_packets_dropped_count.clear(),
+            i64
+        ),
+        ("repair_peers", stats.repair_peers.clear(), i64),
+        ("new_push_requests", stats.new_push_requests.clear(), i64),
+        ("new_push_requests2", stats.new_push_requests2.clear(), i64),
+        ("purge", stats.purge.clear(), i64),
+        (
+            "process_gossip_packets_time",
+            stats.process_gossip_packets_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_ping_messages_time",
+            stats.handle_batch_ping_messages_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_pong_messages_time",
+            stats.handle_batch_pong_messages_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_prune_messages_time",
+            stats.handle_batch_prune_messages_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_pull_requests_time",
+            stats.handle_batch_pull_requests_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_pull_responses_time",
+            stats.handle_batch_pull_responses_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_push_messages_time",
+            stats.handle_batch_push_messages_time.clear(),
+            i64
+        ),
+        (
+            "process_pull_resp",
+            stats.process_pull_response.clear(),
+            i64
+        ),
+        ("filter_pull_resp", stats.filter_pull_response.clear(), i64),
+        (
+            "process_pull_resp_count",
+            stats.process_pull_response_count.clear(),
+            i64
+        ),
+        (
+            "pull_response_fail_insert",
+            stats.process_pull_response_fail_insert.clear(),
+            i64
+        ),
+        (
+            "pull_response_fail_timeout",
+            stats.process_pull_response_fail_timeout.clear(),
+            i64
+        ),
+        (
+            "pull_response_success",
+            stats.process_pull_response_success.clear(),
+            i64
+        ),
+        (
+            "process_pull_resp_timeout",
+            stats.process_pull_response_timeout.clear(),
+            i64
+        ),
+        (
+            "push_response_count",
+            stats.push_response_count.clear(),
+            i64
+        ),
+    );
+    datapoint_info!(
+        "cluster_info_stats3",
+        (
+            "process_pull_resp_len",
+            stats.process_pull_response_len.clear(),
+            i64
+        ),
+        (
+            "process_pull_requests",
+            stats.process_pull_requests.clear(),
+            i64
+        ),
+        (
+            "pull_request_ping_pong_check_failed_count",
+            stats.pull_request_ping_pong_check_failed_count.clear(),
+            i64
+        ),
+        (
+            "new_pull_requests_pings_count",
+            stats.new_pull_requests_pings_count.clear(),
+            i64
+        ),
+        (
+            "generate_pull_responses",
+            stats.generate_pull_responses.clear(),
+            i64
+        ),
+        ("process_prune", stats.process_prune.clear(), i64),
+        (
+            "process_push_message",
+            stats.process_push_message.clear(),
+            i64
+        ),
+        (
+            "prune_received_cache",
+            stats.prune_received_cache.clear(),
+            i64
+        ),
+        ("epoch_slots_lookup", stats.epoch_slots_lookup.clear(), i64),
+        ("new_pull_requests", stats.new_pull_requests.clear(), i64),
+        ("mark_pull_request", stats.mark_pull_request.clear(), i64),
+    );
+    datapoint_info!(
+        "cluster_info_stats4",
+        (
+            "skip_push_message_shred_version",
+            stats.skip_push_message_shred_version.clear(),
+            i64
+        ),
+        (
+            "skip_pull_response_shred_version",
+            stats.skip_pull_response_shred_version.clear(),
+            i64
+        ),
+        (
+            "skip_pull_shred_version",
+            stats.skip_pull_shred_version.clear(),
+            i64
+        ),
+        ("push_message_count", stats.push_message_count.clear(), i64),
+        (
+            "push_message_value_count",
+            stats.push_message_value_count.clear(),
+            i64
+        ),
+        (
+            "new_pull_requests_count",
+            stats.new_pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "pull_from_entrypoint_count",
+            stats.pull_from_entrypoint_count.clear(),
+            i64
+        ),
+        (
+            "prune_message_count",
+            stats.prune_message_count.clear(),
+            i64
+        ),
+        ("prune_message_len", stats.prune_message_len.clear(), i64),
+    );
+    datapoint_info!(
+        "cluster_info_stats5",
+        (
+            "pull_requests_count",
+            stats.pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_count",
+            stats.packets_received_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_prune_messages_count",
+            stats.packets_received_prune_messages_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_pull_requests_count",
+            stats.packets_received_pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_pull_responses_count",
+            stats.packets_received_pull_responses_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_push_messages_count",
+            stats.packets_received_push_messages_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_verified_count",
+            stats.packets_received_verified_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_gossip_requests_count",
+            stats.packets_sent_gossip_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_prune_messages_count",
+            stats.packets_sent_prune_messages_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_pull_requests_count",
+            stats.packets_sent_pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_pull_responses_count",
+            stats.packets_sent_pull_responses_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_push_messages_count",
+            stats.packets_sent_push_messages_count.clear(),
+            i64
+        ),
+        (
+            "require_stake_for_gossip_unknown_feature_set",
+            stats.require_stake_for_gossip_unknown_feature_set.clear(),
+            i64
+        ),
+        (
+            "require_stake_for_gossip_unknown_stakes",
+            stats.require_stake_for_gossip_unknown_stakes.clear(),
+            i64
+        ),
+        (
+            "trim_crds_table_failed",
+            stats.trim_crds_table_failed.clear(),
+            i64
+        ),
+        (
+            "trim_crds_table_purged_values_count",
+            stats.trim_crds_table_purged_values_count.clear(),
+            i64
+        ),
+    );
+}

core/src/cluster_info_vote_listener.rs

@@ -4,6 +4,7 @@ use crate::{
     optimistic_confirmation_verifier::OptimisticConfirmationVerifier,
     optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender},
     poh_recorder::PohRecorder,
+    replay_stage::DUPLICATE_THRESHOLD,
     result::{Error, Result},
     rpc_subscriptions::RpcSubscriptions,
     sigverify,
@@ -21,6 +22,7 @@ use solana_perf::packet::{self, Packets};
 use solana_runtime::{
     bank::Bank,
     bank_forks::BankForks,
+    commitment::VOTE_THRESHOLD_SIZE,
     epoch_stakes::{EpochAuthorizedVoters, EpochStakes},
     stakes::Stakes,
     vote_sender_types::{ReplayVoteReceiver, ReplayedVote},
@@ -44,12 +46,20 @@ use std::{
 };
 
 // Map from a vote account to the authorized voter for an epoch
+pub type ThresholdConfirmedSlots = Vec<(Slot, Hash)>;
+pub type VotedHashUpdates = HashMap<Hash, Vec<Pubkey>>;
 pub type VerifiedLabelVotePacketsSender = CrossbeamSender<Vec<(CrdsValueLabel, Slot, Packets)>>;
 pub type VerifiedLabelVotePacketsReceiver = CrossbeamReceiver<Vec<(CrdsValueLabel, Slot, Packets)>>;
 pub type VerifiedVoteTransactionsSender = CrossbeamSender<Vec<Transaction>>;
 pub type VerifiedVoteTransactionsReceiver = CrossbeamReceiver<Vec<Transaction>>;
 pub type VerifiedVoteSender = CrossbeamSender<(Pubkey, Vec<Slot>)>;
 pub type VerifiedVoteReceiver = CrossbeamReceiver<(Pubkey, Vec<Slot>)>;
+pub type GossipVerifiedVoteHashSender = CrossbeamSender<(Pubkey, Slot, Hash)>;
+pub type GossipVerifiedVoteHashReceiver = CrossbeamReceiver<(Pubkey, Slot, Hash)>;
+pub type GossipDuplicateConfirmedSlotsSender = CrossbeamSender<ThresholdConfirmedSlots>;
+pub type GossipDuplicateConfirmedSlotsReceiver = CrossbeamReceiver<ThresholdConfirmedSlots>;
+
+const THRESHOLDS_TO_CHECK: [f64; 2] = [DUPLICATE_THRESHOLD, VOTE_THRESHOLD_SIZE];
 
 #[derive(Default)]
 pub struct SlotVoteTracker {
@@ -58,14 +68,13 @@ pub struct SlotVoteTracker {
     // True if seen on gossip, false if only seen in replay.
     voted: HashMap<Pubkey, bool>,
     optimistic_votes_tracker: HashMap<Hash, VoteStakeTracker>,
-    updates: Option<Vec<Pubkey>>,
+    voted_slot_updates: Option<Vec<Pubkey>>,
     gossip_only_stake: u64,
 }
 
 impl SlotVoteTracker {
-    #[allow(dead_code)]
-    pub fn get_updates(&mut self) -> Option<Vec<Pubkey>> {
-        self.updates.take()
+    pub fn get_voted_slot_updates(&mut self) -> Option<Vec<Pubkey>> {
+        self.voted_slot_updates.take()
     }
 
     pub fn get_or_insert_optimistic_votes_tracker(&mut self, hash: Hash) -> &mut VoteStakeTracker {
@@ -112,7 +121,7 @@ impl VoteTracker {
             let new_slot_tracker = Arc::new(RwLock::new(SlotVoteTracker {
                 voted: HashMap::new(),
                 optimistic_votes_tracker: HashMap::default(),
-                updates: None,
+                voted_slot_updates: None,
                 gossip_only_stake: 0,
             }));
             self.slot_vote_trackers
@@ -163,10 +172,10 @@ impl VoteTracker {
         let mut w_slot_vote_tracker = slot_vote_tracker.write().unwrap();
 
         w_slot_vote_tracker.voted.insert(pubkey, true);
-        if let Some(ref mut updates) = w_slot_vote_tracker.updates {
-            updates.push(pubkey)
+        if let Some(ref mut voted_slot_updates) = w_slot_vote_tracker.voted_slot_updates {
+            voted_slot_updates.push(pubkey)
         } else {
-            w_slot_vote_tracker.updates = Some(vec![pubkey]);
+            w_slot_vote_tracker.voted_slot_updates = Some(vec![pubkey]);
         }
     }
 
@@ -242,9 +251,11 @@ impl ClusterInfoVoteListener {
         bank_forks: Arc<RwLock<BankForks>>,
         subscriptions: Arc<RpcSubscriptions>,
         verified_vote_sender: VerifiedVoteSender,
+        gossip_verified_vote_hash_sender: GossipVerifiedVoteHashSender,
         replay_votes_receiver: ReplayVoteReceiver,
         blockstore: Arc<Blockstore>,
         bank_notification_sender: Option<BankNotificationSender>,
+        cluster_confirmed_slot_sender: GossipDuplicateConfirmedSlotsSender,
     ) -> Self {
         let exit_ = exit.clone();
 
@@ -287,10 +298,12 @@ impl ClusterInfoVoteListener {
                     vote_tracker,
                     bank_forks,
                     subscriptions,
+                    gossip_verified_vote_hash_sender,
                     verified_vote_sender,
                     replay_votes_receiver,
                     blockstore,
                     bank_notification_sender,
+                    cluster_confirmed_slot_sender,
                 );
             })
             .unwrap();
@@ -406,20 +419,24 @@ impl ClusterInfoVoteListener {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn process_votes_loop(
         exit: Arc<AtomicBool>,
         gossip_vote_txs_receiver: VerifiedVoteTransactionsReceiver,
         vote_tracker: Arc<VoteTracker>,
         bank_forks: Arc<RwLock<BankForks>>,
         subscriptions: Arc<RpcSubscriptions>,
+        gossip_verified_vote_hash_sender: GossipVerifiedVoteHashSender,
         verified_vote_sender: VerifiedVoteSender,
         replay_votes_receiver: ReplayVoteReceiver,
         blockstore: Arc<Blockstore>,
         bank_notification_sender: Option<BankNotificationSender>,
+        cluster_confirmed_slot_sender: GossipDuplicateConfirmedSlotsSender,
     ) -> Result<()> {
         let mut confirmation_verifier =
             OptimisticConfirmationVerifier::new(bank_forks.read().unwrap().root());
         let mut last_process_root = Instant::now();
+        let cluster_confirmed_slot_sender = Some(cluster_confirmed_slot_sender);
         loop {
             if exit.load(Ordering::Relaxed) {
                 return Ok(());
@@ -445,13 +462,16 @@ impl ClusterInfoVoteListener {
                 &vote_tracker,
                 &root_bank,
                 &subscriptions,
+                &gossip_verified_vote_hash_sender,
                 &verified_vote_sender,
                 &replay_votes_receiver,
                 &bank_notification_sender,
+                &cluster_confirmed_slot_sender,
             );
             match confirmed_slots {
                 Ok(confirmed_slots) => {
-                    confirmation_verifier.add_new_optimistic_confirmed_slots(confirmed_slots);
+                    confirmation_verifier
+                        .add_new_optimistic_confirmed_slots(confirmed_slots.clone());
                 }
                 Err(e) => match e {
                     Error::CrossbeamRecvTimeoutError(RecvTimeoutError::Timeout)
@@ -470,17 +490,20 @@ impl ClusterInfoVoteListener {
         vote_tracker: &VoteTracker,
         root_bank: &Bank,
         subscriptions: &RpcSubscriptions,
+        gossip_verified_vote_hash_sender: &GossipVerifiedVoteHashSender,
         verified_vote_sender: &VerifiedVoteSender,
         replay_votes_receiver: &ReplayVoteReceiver,
-    ) -> Result<Vec<(Slot, Hash)>> {
+    ) -> Result<ThresholdConfirmedSlots> {
         Self::listen_and_confirm_votes(
             gossip_vote_txs_receiver,
             vote_tracker,
             root_bank,
             subscriptions,
+            gossip_verified_vote_hash_sender,
             verified_vote_sender,
             replay_votes_receiver,
             &None,
+            &None,
         )
     }
 
@@ -489,10 +512,12 @@ impl ClusterInfoVoteListener {
         vote_tracker: &VoteTracker,
         root_bank: &Bank,
         subscriptions: &RpcSubscriptions,
+        gossip_verified_vote_hash_sender: &GossipVerifiedVoteHashSender,
         verified_vote_sender: &VerifiedVoteSender,
         replay_votes_receiver: &ReplayVoteReceiver,
         bank_notification_sender: &Option<BankNotificationSender>,
-    ) -> Result<Vec<(Slot, Hash)>> {
+        cluster_confirmed_slot_sender: &Option<GossipDuplicateConfirmedSlotsSender>,
+    ) -> Result<ThresholdConfirmedSlots> {
         let mut sel = Select::new();
         sel.recv(gossip_vote_txs_receiver);
         sel.recv(replay_votes_receiver);
@@ -519,8 +544,10 @@ impl ClusterInfoVoteListener {
                     replay_votes,
                     root_bank,
                     subscriptions,
+                    gossip_verified_vote_hash_sender,
                     verified_vote_sender,
                     bank_notification_sender,
+                    cluster_confirmed_slot_sender,
                 ));
             } else {
                 remaining_wait_time = remaining_wait_time
@@ -538,10 +565,12 @@ impl ClusterInfoVoteListener {
         root_bank: &Bank,
         subscriptions: &RpcSubscriptions,
         verified_vote_sender: &VerifiedVoteSender,
+        gossip_verified_vote_hash_sender: &GossipVerifiedVoteHashSender,
         diff: &mut HashMap<Slot, HashMap<Pubkey, bool>>,
-        new_optimistic_confirmed_slots: &mut Vec<(Slot, Hash)>,
+        new_optimistic_confirmed_slots: &mut ThresholdConfirmedSlots,
         is_gossip_vote: bool,
         bank_notification_sender: &Option<BankNotificationSender>,
+        cluster_confirmed_slot_sender: &Option<GossipDuplicateConfirmedSlotsSender>,
     ) {
         if vote.slots.is_empty() {
             return;
@@ -577,7 +606,7 @@ impl ClusterInfoVoteListener {
                 // Fast track processing of the last slot in a vote transactions
                 // so that notifications for optimistic confirmation can be sent
                 // as soon as possible.
-                let (is_confirmed, is_new) = Self::track_optimistic_confirmation_vote(
+                let (reached_threshold_results, is_new) = Self::track_optimistic_confirmation_vote(
                     vote_tracker,
                     last_vote_slot,
                     last_vote_hash,
@@ -586,7 +615,20 @@ impl ClusterInfoVoteListener {
                     total_stake,
                 );
 
-                if is_confirmed {
+                if is_gossip_vote && is_new && stake > 0 {
+                    let _ = gossip_verified_vote_hash_sender.send((
+                        *vote_pubkey,
+                        last_vote_slot,
+                        last_vote_hash,
+                    ));
+                }
+
+                if reached_threshold_results[0] {
+                    if let Some(sender) = cluster_confirmed_slot_sender {
+                        let _ = sender.send(vec![(last_vote_slot, last_vote_hash)]);
+                    }
+                }
+                if reached_threshold_results[1] {
                     new_optimistic_confirmed_slots.push((last_vote_slot, last_vote_hash));
                     // Notify subscribers about new optimistic confirmation
                     if let Some(sender) = bank_notification_sender {
@@ -668,9 +710,11 @@ impl ClusterInfoVoteListener {
         replayed_votes: Vec<ReplayedVote>,
         root_bank: &Bank,
         subscriptions: &RpcSubscriptions,
+        gossip_verified_vote_hash_sender: &GossipVerifiedVoteHashSender,
         verified_vote_sender: &VerifiedVoteSender,
         bank_notification_sender: &Option<BankNotificationSender>,
-    ) -> Vec<(Slot, Hash)> {
+        cluster_confirmed_slot_sender: &Option<GossipDuplicateConfirmedSlotsSender>,
+    ) -> ThresholdConfirmedSlots {
         let mut diff: HashMap<Slot, HashMap<Pubkey, bool>> = HashMap::new();
         let mut new_optimistic_confirmed_slots = vec![];
 
@@ -693,10 +737,12 @@ impl ClusterInfoVoteListener {
                 root_bank,
                 subscriptions,
                 verified_vote_sender,
+                gossip_verified_vote_hash_sender,
                 &mut diff,
                 &mut new_optimistic_confirmed_slots,
                 is_gossip,
                 bank_notification_sender,
+                cluster_confirmed_slot_sender,
             );
         }
 
@@ -717,8 +763,8 @@ impl ClusterInfoVoteListener {
                 });
             }
             let mut w_slot_tracker = slot_tracker.write().unwrap();
-            if w_slot_tracker.updates.is_none() {
-                w_slot_tracker.updates = Some(vec![]);
+            if w_slot_tracker.voted_slot_updates.is_none() {
+                w_slot_tracker.voted_slot_updates = Some(vec![]);
             }
             let mut gossip_only_stake = 0;
             let epoch = root_bank.epoch_schedule().get_epoch(slot);
@@ -739,7 +785,11 @@ impl ClusterInfoVoteListener {
                 // `is_new || is_new_from_gossip`. In both cases we want to record
                 // `is_new_from_gossip` for the `pubkey` entry.
                 w_slot_tracker.voted.insert(pubkey, seen_in_gossip_above);
-                w_slot_tracker.updates.as_mut().unwrap().push(pubkey);
+                w_slot_tracker
+                    .voted_slot_updates
+                    .as_mut()
+                    .unwrap()
+                    .push(pubkey);
             }
 
             w_slot_tracker.gossip_only_stake += gossip_only_stake
@@ -756,14 +806,14 @@ impl ClusterInfoVoteListener {
         pubkey: Pubkey,
         stake: u64,
         total_epoch_stake: u64,
-    ) -> (bool, bool) {
+    ) -> (Vec<bool>, bool) {
         let slot_tracker = vote_tracker.get_or_insert_slot_tracker(slot);
         // Insert vote and check for optimistic confirmation
         let mut w_slot_tracker = slot_tracker.write().unwrap();
 
         w_slot_tracker
             .get_or_insert_optimistic_votes_tracker(hash)
-            .add_vote_pubkey(pubkey, stake, total_epoch_stake)
+            .add_vote_pubkey(pubkey, stake, total_epoch_stake, &THRESHOLDS_TO_CHECK)
     }
 
     fn sum_stake(sum: &mut u64, epoch_stakes: Option<&EpochStakes>, pubkey: &Pubkey) {
@@ -972,6 +1022,7 @@ mod tests {
         let (vote_tracker, _, validator_voting_keypairs, subscriptions) = setup();
         let (votes_sender, votes_receiver) = unbounded();
         let (verified_vote_sender, _verified_vote_receiver) = unbounded();
+        let (gossip_verified_vote_hash_sender, _gossip_verified_vote_hash_receiver) = unbounded();
         let (replay_votes_sender, replay_votes_receiver) = unbounded();
 
         let GenesisConfigInfo { genesis_config, .. } =
@@ -1002,9 +1053,11 @@ mod tests {
             &vote_tracker,
             &bank3,
             &subscriptions,
+            &gossip_verified_vote_hash_sender,
             &verified_vote_sender,
             &replay_votes_receiver,
             &None,
+            &None,
         )
         .unwrap();
 
@@ -1031,9 +1084,11 @@ mod tests {
             &vote_tracker,
             &bank3,
             &subscriptions,
+            &gossip_verified_vote_hash_sender,
             &verified_vote_sender,
             &replay_votes_receiver,
             &None,
+            &None,
         )
         .unwrap();
 
@@ -1082,6 +1137,7 @@ mod tests {
         let (vote_tracker, _, validator_voting_keypairs, subscriptions) = setup();
         let (votes_txs_sender, votes_txs_receiver) = unbounded();
         let (replay_votes_sender, replay_votes_receiver) = unbounded();
+        let (gossip_verified_vote_hash_sender, gossip_verified_vote_hash_receiver) = unbounded();
         let (verified_vote_sender, verified_vote_receiver) = unbounded();
 
         let GenesisConfigInfo { genesis_config, .. } =
@@ -1109,17 +1165,52 @@ mod tests {
             &vote_tracker,
             &bank0,
             &subscriptions,
+            &gossip_verified_vote_hash_sender,
             &verified_vote_sender,
             &replay_votes_receiver,
             &None,
+            &None,
         )
         .unwrap();
 
+        let mut gossip_verified_votes: HashMap<Slot, HashMap<Hash, Vec<Pubkey>>> = HashMap::new();
+        for (pubkey, slot, hash) in gossip_verified_vote_hash_receiver.try_iter() {
+            // send_vote_txs() will send each vote twice, but we should only get a notification
+            // once for each via this channel
+            let exists = gossip_verified_votes
+                .get(&slot)
+                .and_then(|slot_hashes| slot_hashes.get(&hash))
+                .map(|slot_hash_voters| slot_hash_voters.contains(&pubkey))
+                .unwrap_or(false);
+            assert!(!exists);
+            gossip_verified_votes
+                .entry(slot)
+                .or_default()
+                .entry(hash)
+                .or_default()
+                .push(pubkey);
+        }
+
+        // Only the last vote in the `gossip_vote` set should count towards
+        // the `voted_hash_updates` set. Important to note here that replay votes
+        // should not count
+        let last_gossip_vote_slot = *gossip_vote_slots.last().unwrap();
+        assert_eq!(gossip_verified_votes.len(), 1);
+        let slot_hashes = gossip_verified_votes.get(&last_gossip_vote_slot).unwrap();
+        assert_eq!(slot_hashes.len(), 1);
+        let slot_hash_votes = slot_hashes.get(&Hash::default()).unwrap();
+        assert_eq!(slot_hash_votes.len(), validator_voting_keypairs.len());
+        for voting_keypairs in &validator_voting_keypairs {
+            let pubkey = voting_keypairs.vote_keypair.pubkey();
+            assert!(slot_hash_votes.contains(&pubkey));
+        }
+
         // Check that the received votes were pushed to other commponents
         // subscribing via `verified_vote_receiver`
         let all_expected_slots: BTreeSet<_> = gossip_vote_slots
+            .clone()
             .into_iter()
-            .chain(replay_vote_slots.into_iter())
+            .chain(replay_vote_slots.clone().into_iter())
             .collect();
         let mut pubkey_to_votes: HashMap<Pubkey, BTreeSet<Slot>> = HashMap::new();
         for (received_pubkey, new_votes) in verified_vote_receiver.try_iter() {
@@ -1147,15 +1238,17 @@ mod tests {
                 let pubkey = voting_keypairs.vote_keypair.pubkey();
                 assert!(r_slot_vote_tracker.voted.contains_key(&pubkey));
                 assert!(r_slot_vote_tracker
-                    .updates
+                    .voted_slot_updates
                     .as_ref()
                     .unwrap()
                     .contains(&Arc::new(pubkey)));
-                // Only the last vote in the stack of `gossip_votes` should count towards
-                // the `optimistic` vote set.
+                // Only the last vote in the stack of `gossip_vote` and `replay_vote_slots`
+                // should count towards the `optimistic` vote set,
                 let optimistic_votes_tracker =
                     r_slot_vote_tracker.optimistic_votes_tracker(&Hash::default());
-                if vote_slot == 2 || vote_slot == 4 {
+                if vote_slot == *gossip_vote_slots.last().unwrap()
+                    || vote_slot == *replay_vote_slots.last().unwrap()
+                {
                     let optimistic_votes_tracker = optimistic_votes_tracker.unwrap();
                     assert!(optimistic_votes_tracker.voted().contains(&pubkey));
                     assert_eq!(
@@ -1192,6 +1285,7 @@ mod tests {
 
         // Send some votes to process
         let (votes_txs_sender, votes_txs_receiver) = unbounded();
+        let (gossip_verified_vote_hash_sender, _gossip_verified_vote_hash_receiver) = unbounded();
         let (verified_vote_sender, verified_vote_receiver) = unbounded();
         let (_replay_votes_sender, replay_votes_receiver) = unbounded();
 
@@ -1228,9 +1322,11 @@ mod tests {
             &vote_tracker,
             &bank0,
             &subscriptions,
+            &gossip_verified_vote_hash_sender,
             &verified_vote_sender,
             &replay_votes_receiver,
             &None,
+            &None,
         )
         .unwrap();
 
@@ -1252,7 +1348,7 @@ mod tests {
                 let pubkey = voting_keypairs.vote_keypair.pubkey();
                 assert!(r_slot_vote_tracker.voted.contains_key(&pubkey));
                 assert!(r_slot_vote_tracker
-                    .updates
+                    .voted_slot_updates
                     .as_ref()
                     .unwrap()
                     .contains(&Arc::new(pubkey)));
@@ -1273,6 +1369,7 @@ mod tests {
     fn run_test_process_votes3(switch_proof_hash: Option<Hash>) {
         let (votes_sender, votes_receiver) = unbounded();
         let (verified_vote_sender, _verified_vote_receiver) = unbounded();
+        let (gossip_verified_vote_hash_sender, _gossip_verified_vote_hash_receiver) = unbounded();
         let (replay_votes_sender, replay_votes_receiver) = unbounded();
 
         let vote_slot = 1;
@@ -1323,34 +1420,29 @@ mod tests {
                     &vote_tracker,
                     &bank,
                     &subscriptions,
+                    &gossip_verified_vote_hash_sender,
                     &verified_vote_sender,
                     &replay_votes_receiver,
                     &None,
+                    &None,
                 );
             }
             let slot_vote_tracker = vote_tracker.get_slot_vote_tracker(vote_slot).unwrap();
             let r_slot_vote_tracker = &slot_vote_tracker.read().unwrap();
 
+            assert_eq!(
+                r_slot_vote_tracker
+                    .optimistic_votes_tracker(&vote_bank_hash)
+                    .unwrap()
+                    .stake(),
+                100
+            );
             if events == vec![1] {
                 // Check `gossip_only_stake` is not incremented
-                assert_eq!(
-                    r_slot_vote_tracker
-                        .optimistic_votes_tracker(&vote_bank_hash)
-                        .unwrap()
-                        .stake(),
-                    100
-                );
                 assert_eq!(r_slot_vote_tracker.gossip_only_stake, 0);
             } else {
                 // Check that both the `gossip_only_stake` and `total_voted_stake` both
                 // increased
-                assert_eq!(
-                    r_slot_vote_tracker
-                        .optimistic_votes_tracker(&vote_bank_hash)
-                        .unwrap()
-                        .stake(),
-                    100
-                );
                 assert_eq!(r_slot_vote_tracker.gossip_only_stake, 100);
             }
         }
@@ -1457,6 +1549,7 @@ mod tests {
         )];
 
         let (verified_vote_sender, _verified_vote_receiver) = unbounded();
+        let (gossip_verified_vote_hash_sender, _gossip_verified_vote_hash_receiver) = unbounded();
         ClusterInfoVoteListener::filter_and_confirm_with_new_votes(
             &vote_tracker,
             vote_tx,
@@ -1468,8 +1561,10 @@ mod tests {
             )],
             &bank,
             &subscriptions,
+            &gossip_verified_vote_hash_sender,
             &verified_vote_sender,
             &None,
+            &None,
         );
 
         // Setup next epoch
@@ -1522,8 +1617,10 @@ mod tests {
             )],
             &new_root_bank,
             &subscriptions,
+            &gossip_verified_vote_hash_sender,
             &verified_vote_sender,
             &None,
+            &None,
         );
     }
 

core/src/cluster_slot_state_verifier.rs

@@ -0,0 +1,806 @@
+use crate::{
+    fork_choice::ForkChoice, heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
+    progress_map::ProgressMap,
+};
+use solana_sdk::{clock::Slot, hash::Hash};
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+pub(crate) type GossipDuplicateConfirmedSlots = BTreeMap<Slot, Hash>;
+type SlotStateHandler = fn(Slot, &Hash, Option<&Hash>, bool, bool) -> Vec<ResultingStateChange>;
+
+#[derive(PartialEq, Debug)]
+pub enum SlotStateUpdate {
+    Frozen,
+    DuplicateConfirmed,
+    Dead,
+    Duplicate,
+}
+
+#[derive(PartialEq, Debug)]
+pub enum ResultingStateChange {
+    // Hash of our current frozen version of the slot
+    MarkSlotDuplicate(Hash),
+    // Hash of the cluster confirmed slot that is not equivalent
+    // to our frozen version of the slot
+    RepairDuplicateConfirmedVersion(Hash),
+    // Hash of our current frozen version of the slot
+    DuplicateConfirmedSlotMatchesCluster(Hash),
+}
+
+impl SlotStateUpdate {
+    fn to_handler(&self) -> SlotStateHandler {
+        match self {
+            SlotStateUpdate::Dead => on_dead_slot,
+            SlotStateUpdate::Frozen => on_frozen_slot,
+            SlotStateUpdate::DuplicateConfirmed => on_cluster_update,
+            SlotStateUpdate::Duplicate => on_cluster_update,
+        }
+    }
+}
+
+fn repair_correct_version(_slot: Slot, _hash: &Hash) {}
+
+fn on_dead_slot(
+    slot: Slot,
+    bank_frozen_hash: &Hash,
+    cluster_duplicate_confirmed_hash: Option<&Hash>,
+    _is_slot_duplicate: bool,
+    is_dead: bool,
+) -> Vec<ResultingStateChange> {
+    assert!(is_dead);
+    // Bank should not have been frozen if the slot was marked dead
+    assert_eq!(*bank_frozen_hash, Hash::default());
+    if let Some(cluster_duplicate_confirmed_hash) = cluster_duplicate_confirmed_hash {
+        // If the cluster duplicate_confirmed some version of this slot, then
+        // there's another version
+        warn!(
+            "Cluster duplicate_confirmed slot {} with hash {}, but we marked slot dead",
+            slot, cluster_duplicate_confirmed_hash
+        );
+        // No need to check `is_slot_duplicate` and modify fork choice as dead slots
+        // are never frozen, and thus never added to fork choice. The state change for
+        // `MarkSlotDuplicate` will try to modify fork choice, but won't find the slot
+        // in the fork choice tree, so is equivalent to a no-op
+        return vec![
+            ResultingStateChange::MarkSlotDuplicate(Hash::default()),
+            ResultingStateChange::RepairDuplicateConfirmedVersion(
+                *cluster_duplicate_confirmed_hash,
+            ),
+        ];
+    }
+
+    vec![]
+}
+
+fn on_frozen_slot(
+    slot: Slot,
+    bank_frozen_hash: &Hash,
+    cluster_duplicate_confirmed_hash: Option<&Hash>,
+    is_slot_duplicate: bool,
+    is_dead: bool,
+) -> Vec<ResultingStateChange> {
+    // If a slot is marked frozen, the bank hash should not be default,
+    // and the slot should not be dead
+    assert!(*bank_frozen_hash != Hash::default());
+    assert!(!is_dead);
+
+    if let Some(cluster_duplicate_confirmed_hash) = cluster_duplicate_confirmed_hash {
+        // If the cluster duplicate_confirmed some version of this slot, then
+        // confirm our version agrees with the cluster,
+        if cluster_duplicate_confirmed_hash != bank_frozen_hash {
+            // If the versions do not match, modify fork choice rule
+            // to exclude our version from being voted on and also
+            // repair correct version
+            warn!(
+                "Cluster duplicate_confirmed slot {} with hash {}, but we froze slot with hash {}",
+                slot, cluster_duplicate_confirmed_hash, bank_frozen_hash
+            );
+            return vec![
+                ResultingStateChange::MarkSlotDuplicate(*bank_frozen_hash),
+                ResultingStateChange::RepairDuplicateConfirmedVersion(
+                    *cluster_duplicate_confirmed_hash,
+                ),
+            ];
+        } else {
+            // If the versions match, then add the slot to the candidate
+            // set to account for the case where it was removed earlier
+            // by the `on_duplicate_slot()` handler
+            return vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(
+                *bank_frozen_hash,
+            )];
+        }
+    }
+
+    if is_slot_duplicate {
+        // If we detected a duplicate, but have not yet seen any version
+        // of the slot duplicate_confirmed (i.e. block above did not execute), then
+        // remove the slot from fork choice until we get confirmation.
+
+        // If we get here, we either detected duplicate from
+        // 1) WindowService
+        // 2) A gossip duplicate_confirmed version that didn't match our frozen
+        // version.
+        // In both cases, mark the progress map for this slot as duplicate
+        return vec![ResultingStateChange::MarkSlotDuplicate(*bank_frozen_hash)];
+    }
+
+    vec![]
+}
+
+// Called when we receive either:
+// 1) A duplicate slot signal from WindowStage,
+// 2) Confirmation of a slot by observing votes from replay or gossip.
+//
+// This signals external information about this slot, which affects
+// this validator's understanding of the validity of this slot
+fn on_cluster_update(
+    slot: Slot,
+    bank_frozen_hash: &Hash,
+    cluster_duplicate_confirmed_hash: Option<&Hash>,
+    is_slot_duplicate: bool,
+    is_dead: bool,
+) -> Vec<ResultingStateChange> {
+    if is_dead {
+        on_dead_slot(
+            slot,
+            bank_frozen_hash,
+            cluster_duplicate_confirmed_hash,
+            is_slot_duplicate,
+            is_dead,
+        )
+    } else if *bank_frozen_hash != Hash::default() {
+        // This case is mutually exclusive with is_dead case above because if a slot is dead,
+        // it cannot have  been frozen, and thus cannot have a non-default bank hash.
+        on_frozen_slot(
+            slot,
+            bank_frozen_hash,
+            cluster_duplicate_confirmed_hash,
+            is_slot_duplicate,
+            is_dead,
+        )
+    } else {
+        vec![]
+    }
+}
+
+fn get_cluster_duplicate_confirmed_hash<'a>(
+    slot: Slot,
+    gossip_duplicate_confirmed_hash: Option<&'a Hash>,
+    local_frozen_hash: &'a Hash,
+    is_local_replay_duplicate_confirmed: bool,
+) -> Option<&'a Hash> {
+    let local_duplicate_confirmed_hash = if is_local_replay_duplicate_confirmed {
+        // If local replay has duplicate_confirmed this slot, this slot must have
+        // descendants with votes for this slot, hence this slot must be
+        // frozen.
+        assert!(*local_frozen_hash != Hash::default());
+        Some(local_frozen_hash)
+    } else {
+        None
+    };
+
+    match (
+        local_duplicate_confirmed_hash,
+        gossip_duplicate_confirmed_hash,
+    ) {
+        (Some(local_duplicate_confirmed_hash), Some(gossip_duplicate_confirmed_hash)) => {
+            if local_duplicate_confirmed_hash != gossip_duplicate_confirmed_hash {
+                error!(
+                    "For slot {}, the gossip duplicate confirmed hash {}, is not equal
+                to the confirmed hash we replayed: {}",
+                    slot, gossip_duplicate_confirmed_hash, local_duplicate_confirmed_hash
+                );
+            }
+            Some(&local_frozen_hash)
+        }
+        (Some(local_frozen_hash), None) => Some(local_frozen_hash),
+        _ => gossip_duplicate_confirmed_hash,
+    }
+}
+
+fn apply_state_changes(
+    slot: Slot,
+    progress: &mut ProgressMap,
+    fork_choice: &mut HeaviestSubtreeForkChoice,
+    ancestors: &HashMap<Slot, HashSet<Slot>>,
+    descendants: &HashMap<Slot, HashSet<Slot>>,
+    state_changes: Vec<ResultingStateChange>,
+) {
+    for state_change in state_changes {
+        match state_change {
+            ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash) => {
+                progress.set_unconfirmed_duplicate_slot(
+                    slot,
+                    descendants.get(&slot).unwrap_or(&HashSet::default()),
+                );
+                fork_choice.mark_fork_invalid_candidate(&(slot, bank_frozen_hash));
+            }
+            ResultingStateChange::RepairDuplicateConfirmedVersion(
+                cluster_duplicate_confirmed_hash,
+            ) => {
+                // TODO: Should consider moving the updating of the duplicate slots in the
+                // progress map from ReplayStage::confirm_forks to here.
+                repair_correct_version(slot, &cluster_duplicate_confirmed_hash);
+            }
+            ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(bank_frozen_hash) => {
+                progress.set_confirmed_duplicate_slot(
+                    slot,
+                    ancestors.get(&slot).unwrap_or(&HashSet::default()),
+                    descendants.get(&slot).unwrap_or(&HashSet::default()),
+                );
+                fork_choice.mark_fork_valid_candidate(&(slot, bank_frozen_hash));
+            }
+        }
+    }
+}
+
+pub(crate) fn check_slot_agrees_with_cluster(
+    slot: Slot,
+    root: Slot,
+    frozen_hash: Option<Hash>,
+    gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots,
+    ancestors: &HashMap<Slot, HashSet<Slot>>,
+    descendants: &HashMap<Slot, HashSet<Slot>>,
+    progress: &mut ProgressMap,
+    fork_choice: &mut HeaviestSubtreeForkChoice,
+    slot_state_update: SlotStateUpdate,
+) {
+    info!(
+        "check_slot_agrees_with_cluster()
+        slot: {},
+        root: {},
+        frozen_hash: {:?},
+        update: {:?}",
+        slot, root, frozen_hash, slot_state_update
+    );
+
+    if slot <= root {
+        return;
+    }
+
+    if frozen_hash.is_none() {
+        // If the bank doesn't even exist in BankForks yet,
+        // then there's nothing to do as replay of the slot
+        // hasn't even started
+        return;
+    }
+
+    let frozen_hash = frozen_hash.unwrap();
+    let gossip_duplicate_confirmed_hash = gossip_duplicate_confirmed_slots.get(&slot);
+
+    let is_local_replay_duplicate_confirmed = progress.is_duplicate_confirmed(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map");
+    let cluster_duplicate_confirmed_hash = get_cluster_duplicate_confirmed_hash(
+        slot,
+        gossip_duplicate_confirmed_hash,
+        &frozen_hash,
+        is_local_replay_duplicate_confirmed,
+    );
+    let mut is_slot_duplicate =
+        progress.is_unconfirmed_duplicate(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map");
+    if matches!(slot_state_update, SlotStateUpdate::Duplicate) {
+        if is_slot_duplicate {
+            // Already processed duplicate signal for this slot, no need to continue
+            return;
+        } else {
+            // Otherwise, mark the slot as duplicate so the appropriate state changes
+            // will trigger
+            is_slot_duplicate = true;
+        }
+    }
+    let is_dead = progress.is_dead(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map");
+
+    info!(
+        "check_slot_agrees_with_cluster() state
+        is_local_replay_duplicate_confirmed: {:?},
+        cluster_duplicate_confirmed_hash: {:?},
+        is_slot_duplicate: {:?},
+        is_dead: {:?}",
+        is_local_replay_duplicate_confirmed,
+        cluster_duplicate_confirmed_hash,
+        is_slot_duplicate,
+        is_dead,
+    );
+
+    let state_handler = slot_state_update.to_handler();
+    let state_changes = state_handler(
+        slot,
+        &frozen_hash,
+        cluster_duplicate_confirmed_hash,
+        is_slot_duplicate,
+        is_dead,
+    );
+    apply_state_changes(
+        slot,
+        progress,
+        fork_choice,
+        ancestors,
+        descendants,
+        state_changes,
+    );
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::consensus::test::VoteSimulator;
+    use solana_runtime::bank_forks::BankForks;
+    use std::sync::RwLock;
+    use trees::tr;
+
+    struct InitialState {
+        heaviest_subtree_fork_choice: HeaviestSubtreeForkChoice,
+        progress: ProgressMap,
+        ancestors: HashMap<Slot, HashSet<Slot>>,
+        descendants: HashMap<Slot, HashSet<Slot>>,
+        slot: Slot,
+        bank_forks: RwLock<BankForks>,
+    }
+
+    fn setup() -> InitialState {
+        // Create simple fork 0 -> 1 -> 2 -> 3
+        let forks = tr(0) / (tr(1) / (tr(2) / tr(3)));
+        let mut vote_simulator = VoteSimulator::new(1);
+        vote_simulator.fill_bank_forks(forks, &HashMap::new());
+        let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
+
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
+
+        InitialState {
+            heaviest_subtree_fork_choice: vote_simulator.heaviest_subtree_fork_choice,
+            progress: vote_simulator.progress,
+            ancestors,
+            descendants,
+            slot: 0,
+            bank_forks: vote_simulator.bank_forks,
+        }
+    }
+
+    #[test]
+    fn test_frozen_duplicate() {
+        // Common state
+        let slot = 0;
+        let cluster_duplicate_confirmed_hash = None;
+        let is_dead = false;
+
+        // Slot is not detected as duplicate yet
+        let mut is_slot_duplicate = false;
+
+        // Simulate freezing the bank, add a
+        // new non-default hash, should return
+        // no actionable state changes yet
+        let bank_frozen_hash = Hash::new_unique();
+        assert!(on_frozen_slot(
+            slot,
+            &bank_frozen_hash,
+            cluster_duplicate_confirmed_hash,
+            is_slot_duplicate,
+            is_dead
+        )
+        .is_empty());
+
+        // Now mark the slot as duplicate, should
+        // trigger marking the slot as a duplicate
+        is_slot_duplicate = true;
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash)]
+        );
+    }
+
+    #[test]
+    fn test_frozen_duplicate_confirmed() {
+        // Common state
+        let slot = 0;
+        let is_slot_duplicate = false;
+        let is_dead = false;
+
+        // No cluster duplicate_confirmed hash yet
+        let mut cluster_duplicate_confirmed_hash = None;
+
+        // Simulate freezing the bank, add a
+        // new non-default hash, should return
+        // no actionable state changes
+        let bank_frozen_hash = Hash::new_unique();
+        assert!(on_frozen_slot(
+            slot,
+            &bank_frozen_hash,
+            cluster_duplicate_confirmed_hash,
+            is_slot_duplicate,
+            is_dead
+        )
+        .is_empty());
+
+        // Now mark the same frozen slot hash as duplicate_confirmed by the cluster,
+        // should just confirm the slot
+        cluster_duplicate_confirmed_hash = Some(&bank_frozen_hash);
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(
+                bank_frozen_hash
+            ),]
+        );
+
+        // If the cluster_duplicate_confirmed_hash does not match, then we
+        // should trigger marking the slot as a duplicate, and also
+        // try to repair correct version
+        let mismatched_hash = Hash::new_unique();
+        cluster_duplicate_confirmed_hash = Some(&mismatched_hash);
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![
+                ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash),
+                ResultingStateChange::RepairDuplicateConfirmedVersion(mismatched_hash),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_duplicate_frozen_duplicate_confirmed() {
+        // Common state
+        let slot = 0;
+        let is_dead = false;
+        let is_slot_duplicate = true;
+
+        // Bank is not frozen yet
+        let mut cluster_duplicate_confirmed_hash = None;
+        let mut bank_frozen_hash = Hash::default();
+
+        // Mark the slot as duplicate. Because our version of the slot is not
+        // frozen yet, we don't know which version we have, so no action is
+        // taken.
+        assert!(on_cluster_update(
+            slot,
+            &bank_frozen_hash,
+            cluster_duplicate_confirmed_hash,
+            is_slot_duplicate,
+            is_dead
+        )
+        .is_empty());
+
+        // Freeze the bank, should now mark the slot as duplicate since we have
+        // not seen confirmation yet.
+        bank_frozen_hash = Hash::new_unique();
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash),]
+        );
+
+        // If the cluster_duplicate_confirmed_hash matches, we just confirm
+        // the slot
+        cluster_duplicate_confirmed_hash = Some(&bank_frozen_hash);
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(
+                bank_frozen_hash
+            ),]
+        );
+
+        // If the cluster_duplicate_confirmed_hash does not match, then we
+        // should trigger marking the slot as a duplicate, and also
+        // try to repair correct version
+        let mismatched_hash = Hash::new_unique();
+        cluster_duplicate_confirmed_hash = Some(&mismatched_hash);
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![
+                ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash),
+                ResultingStateChange::RepairDuplicateConfirmedVersion(mismatched_hash),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_duplicate_duplicate_confirmed() {
+        let slot = 0;
+        let correct_hash = Hash::new_unique();
+        let cluster_duplicate_confirmed_hash = Some(&correct_hash);
+        let is_dead = false;
+        // Bank is not frozen yet
+        let bank_frozen_hash = Hash::default();
+
+        // Because our version of the slot is not frozen yet, then even though
+        // the cluster has duplicate_confirmed a hash, we don't know which version we
+        // have, so no action is taken.
+        let is_slot_duplicate = true;
+        assert!(on_cluster_update(
+            slot,
+            &bank_frozen_hash,
+            cluster_duplicate_confirmed_hash,
+            is_slot_duplicate,
+            is_dead
+        )
+        .is_empty());
+    }
+
+    #[test]
+    fn test_duplicate_dead() {
+        let slot = 0;
+        let cluster_duplicate_confirmed_hash = None;
+        let is_dead = true;
+        // Bank is not frozen yet
+        let bank_frozen_hash = Hash::default();
+
+        // Even though our version of the slot is dead, the cluster has not
+        // duplicate_confirmed a hash, we don't know which version we have, so no action
+        // is taken.
+        let is_slot_duplicate = true;
+        assert!(on_cluster_update(
+            slot,
+            &bank_frozen_hash,
+            cluster_duplicate_confirmed_hash,
+            is_slot_duplicate,
+            is_dead
+        )
+        .is_empty());
+    }
+
+    #[test]
+    fn test_duplicate_confirmed_dead_duplicate() {
+        let slot = 0;
+        let correct_hash = Hash::new_unique();
+        // Cluster has duplicate_confirmed some version of the slot
+        let cluster_duplicate_confirmed_hash = Some(&correct_hash);
+        // Our version of the slot is dead
+        let is_dead = true;
+        let bank_frozen_hash = Hash::default();
+
+        // Even if the duplicate signal hasn't come in yet,
+        // we can deduce the slot is duplicate AND we have,
+        // the wrong version, so should mark the slot as duplicate,
+        // and repair the correct version
+        let mut is_slot_duplicate = false;
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![
+                ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash),
+                ResultingStateChange::RepairDuplicateConfirmedVersion(correct_hash),
+            ]
+        );
+
+        // If the duplicate signal comes in, nothing should change
+        is_slot_duplicate = true;
+        assert_eq!(
+            on_cluster_update(
+                slot,
+                &bank_frozen_hash,
+                cluster_duplicate_confirmed_hash,
+                is_slot_duplicate,
+                is_dead
+            ),
+            vec![
+                ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash),
+                ResultingStateChange::RepairDuplicateConfirmedVersion(correct_hash),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_apply_state_changes() {
+        // Common state
+        let InitialState {
+            mut heaviest_subtree_fork_choice,
+            mut progress,
+            ancestors,
+            descendants,
+            slot,
+            bank_forks,
+        } = setup();
+
+        // MarkSlotDuplicate should mark progress map and remove
+        // the slot from fork choice
+        let slot_hash = bank_forks.read().unwrap().get(slot).unwrap().hash();
+        apply_state_changes(
+            slot,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            &ancestors,
+            &descendants,
+            vec![ResultingStateChange::MarkSlotDuplicate(slot_hash)],
+        );
+        assert!(!heaviest_subtree_fork_choice
+            .is_candidate_slot(&(slot, slot_hash))
+            .unwrap());
+        for child_slot in descendants
+            .get(&slot)
+            .unwrap()
+            .iter()
+            .chain(std::iter::once(&slot))
+        {
+            assert_eq!(
+                progress
+                    .latest_unconfirmed_duplicate_ancestor(*child_slot)
+                    .unwrap(),
+                slot
+            );
+        }
+
+        // DuplicateConfirmedSlotMatchesCluster should re-enable fork choice
+        apply_state_changes(
+            slot,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            &ancestors,
+            &descendants,
+            vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(
+                slot_hash,
+            )],
+        );
+        for child_slot in descendants
+            .get(&slot)
+            .unwrap()
+            .iter()
+            .chain(std::iter::once(&slot))
+        {
+            assert!(progress
+                .latest_unconfirmed_duplicate_ancestor(*child_slot)
+                .is_none());
+        }
+        assert!(heaviest_subtree_fork_choice
+            .is_candidate_slot(&(slot, slot_hash))
+            .unwrap());
+    }
+
+    #[test]
+    fn test_state_ancestor_confirmed_descendant_duplicate() {
+        // Common state
+        let InitialState {
+            mut heaviest_subtree_fork_choice,
+            mut progress,
+            ancestors,
+            descendants,
+            bank_forks,
+            ..
+        } = setup();
+
+        let slot3_hash = bank_forks.read().unwrap().get(3).unwrap().hash();
+        assert_eq!(
+            heaviest_subtree_fork_choice.best_overall_slot(),
+            (3, slot3_hash)
+        );
+        let root = 0;
+        let mut gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default();
+
+        // Mark slot 2 as duplicate confirmed
+        let slot2_hash = bank_forks.read().unwrap().get(2).unwrap().hash();
+        gossip_duplicate_confirmed_slots.insert(2, slot2_hash);
+        check_slot_agrees_with_cluster(
+            2,
+            root,
+            Some(slot2_hash),
+            &gossip_duplicate_confirmed_slots,
+            &ancestors,
+            &descendants,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            SlotStateUpdate::DuplicateConfirmed,
+        );
+
+        assert_eq!(
+            heaviest_subtree_fork_choice.best_overall_slot(),
+            (3, slot3_hash)
+        );
+
+        // Mark 3 as duplicate, should not remove slot 2 from fork choice
+        check_slot_agrees_with_cluster(
+            3,
+            root,
+            Some(slot3_hash),
+            &gossip_duplicate_confirmed_slots,
+            &ancestors,
+            &descendants,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            SlotStateUpdate::Duplicate,
+        );
+
+        assert_eq!(
+            heaviest_subtree_fork_choice.best_overall_slot(),
+            (2, slot2_hash)
+        );
+    }
+
+    #[test]
+    fn test_state_ancestor_duplicate_descendant_confirmed() {
+        // Common state
+        let InitialState {
+            mut heaviest_subtree_fork_choice,
+            mut progress,
+            ancestors,
+            descendants,
+            bank_forks,
+            ..
+        } = setup();
+
+        let slot3_hash = bank_forks.read().unwrap().get(3).unwrap().hash();
+        assert_eq!(
+            heaviest_subtree_fork_choice.best_overall_slot(),
+            (3, slot3_hash)
+        );
+        let root = 0;
+        let mut gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default();
+        // Mark 2 as duplicate confirmed
+        check_slot_agrees_with_cluster(
+            2,
+            root,
+            Some(bank_forks.read().unwrap().get(2).unwrap().hash()),
+            &gossip_duplicate_confirmed_slots,
+            &ancestors,
+            &descendants,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            SlotStateUpdate::Duplicate,
+        );
+
+        let slot1_hash = bank_forks.read().unwrap().get(1).unwrap().hash();
+        assert_eq!(
+            heaviest_subtree_fork_choice.best_overall_slot(),
+            (1, slot1_hash)
+        );
+
+        // Mark slot 3 as duplicate confirmed, should mark slot 2 as duplicate confirmed as well
+        gossip_duplicate_confirmed_slots.insert(3, slot3_hash);
+        check_slot_agrees_with_cluster(
+            3,
+            root,
+            Some(slot3_hash),
+            &gossip_duplicate_confirmed_slots,
+            &ancestors,
+            &descendants,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            SlotStateUpdate::DuplicateConfirmed,
+        );
+
+        assert_eq!(
+            heaviest_subtree_fork_choice.best_overall_slot(),
+            (3, slot3_hash)
+        );
+    }
+}

core/src/cluster_slots.rs

@@ -191,8 +191,7 @@ impl ClusterSlots {
             .read()
             .unwrap()
             .keys()
-            .filter(|x| **x > root)
-            .filter(|x| !my_slots.contains(*x))
+            .filter(|x| **x > root && !my_slots.contains(*x))
             .map(|x| RepairType::HighestShred(*x, 0))
             .collect()
     }

core/src/commitment_service.rs

@@ -315,15 +315,13 @@ mod tests {
         );
 
         for a in ancestors {
+            let mut expected = BlockCommitment::default();
             if a <= root {
-                let mut expected = BlockCommitment::default();
                 expected.increase_rooted_stake(lamports);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
             } else {
-                let mut expected = BlockCommitment::default();
                 expected.increase_confirmation_stake(1, lamports);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
+            assert_eq!(*commitment.get(&a).unwrap(), expected);
         }
         assert_eq!(rooted_stake[0], (root, lamports));
     }

core/src/consensus.rs

@@ -1,4 +1,7 @@
-use crate::progress_map::{LockoutIntervals, ProgressMap};
+use crate::{
+    latest_validator_votes_for_frozen_banks::LatestValidatorVotesForFrozenBanks,
+    progress_map::{LockoutIntervals, ProgressMap},
+};
 use chrono::prelude::*;
 use solana_ledger::{ancestor_iterator::AncestorIterator, blockstore::Blockstore, blockstore_db};
 use solana_measure::measure::Measure;
@@ -37,6 +40,7 @@ pub enum SwitchForkDecision {
     SwitchProof(Hash),
     SameFork,
     FailedSwitchThreshold(u64, u64),
+    FailedSwitchDuplicateRollback(Slot),
 }
 
 impl SwitchForkDecision {
@@ -51,6 +55,7 @@ impl SwitchForkDecision {
                 assert_ne!(*total_stake, 0);
                 None
             }
+            SwitchForkDecision::FailedSwitchDuplicateRollback(_) => None,
             SwitchForkDecision::SameFork => Some(vote_instruction::vote(
                 vote_account_pubkey,
                 authorized_voter_pubkey,
@@ -68,7 +73,12 @@ impl SwitchForkDecision {
     }
 
     pub fn can_vote(&self) -> bool {
-        !matches!(self, SwitchForkDecision::FailedSwitchThreshold(_, _))
+        match self {
+            SwitchForkDecision::FailedSwitchThreshold(_, _) => false,
+            SwitchForkDecision::FailedSwitchDuplicateRollback(_) => false,
+            SwitchForkDecision::SameFork => true,
+            SwitchForkDecision::SwitchProof(_) => true,
+        }
     }
 }
 
@@ -88,7 +98,7 @@ pub(crate) struct ComputedBankState {
     // Tree of intervals of lockouts of the form [slot, slot + slot.lockout],
     // keyed by end of the range
     pub lockout_intervals: LockoutIntervals,
-    pub pubkey_votes: Arc<PubkeyVotes>,
+    pub my_latest_landed_vote: Option<Slot>,
 }
 
 #[frozen_abi(digest = "Eay84NBbJqiMBfE7HHH2o6e51wcvoU79g8zCi5sw6uj3")]
@@ -99,6 +109,12 @@ pub struct Tower {
     threshold_size: f64,
     lockouts: VoteState,
     last_vote: Vote,
+    #[serde(skip)]
+    // The blockhash used in the last vote transaction, may or may not equal the
+    // blockhash of the voted block itself, depending if the vote slot was refreshed.
+    // For instance, a vote for slot 5, may be refreshed/resubmitted for inclusion in
+    //  block 10, in  which case `last_vote_tx_blockhash` equals the blockhash of 10, not 5.
+    last_vote_tx_blockhash: Hash,
     last_timestamp: BlockTimestamp,
     #[serde(skip)]
     path: PathBuf,
@@ -125,6 +141,7 @@ impl Default for Tower {
             lockouts: VoteState::default(),
             last_vote: Vote::default(),
             last_timestamp: BlockTimestamp::default(),
+            last_vote_tx_blockhash: Hash::default(),
             path: PathBuf::default(),
             tmp_path: PathBuf::default(),
             stray_restored_slot: Option::default(),
@@ -190,8 +207,9 @@ impl Tower {
             );
         let root = root_bank.slot();
 
+        let (best_slot, best_hash) = heaviest_subtree_fork_choice.best_overall_slot();
         let heaviest_bank = bank_forks
-            .get(heaviest_subtree_fork_choice.best_overall_slot())
+            .get_with_checked_hash((best_slot, best_hash))
             .expect(
                 "The best overall slot must be one of `frozen_banks` which all exist in bank_forks",
             )
@@ -207,10 +225,12 @@ impl Tower {
     }
 
     pub(crate) fn collect_vote_lockouts<F>(
-        node_pubkey: &Pubkey,
+        vote_account_pubkey: &Pubkey,
         bank_slot: Slot,
         vote_accounts: F,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
+        get_frozen_hash: impl Fn(Slot) -> Option<Hash>,
+        latest_validator_votes_for_frozen_banks: &mut LatestValidatorVotesForFrozenBanks,
     ) -> ComputedBankState
     where
         F: IntoIterator<Item = (Pubkey, (u64, ArcVoteAccount))>,
@@ -222,12 +242,12 @@ impl Tower {
         // Tree of intervals of lockouts of the form [slot, slot + slot.lockout],
         // keyed by end of the range
         let mut lockout_intervals = LockoutIntervals::new();
-        let mut pubkey_votes = vec![];
+        let mut my_latest_landed_vote = None;
         for (key, (voted_stake, account)) in vote_accounts {
             if voted_stake == 0 {
                 continue;
             }
-            trace!("{} {} with stake {}", node_pubkey, key, voted_stake);
+            trace!("{} {} with stake {}", vote_account_pubkey, key, voted_stake);
             let mut vote_state = match account.vote_state().as_ref() {
                 Err(_) => {
                     datapoint_warn!(
@@ -249,7 +269,8 @@ impl Tower {
                     .push((vote.slot, key));
             }
 
-            if key == *node_pubkey || vote_state.node_pubkey == *node_pubkey {
+            if key == *vote_account_pubkey {
+                my_latest_landed_vote = vote_state.nth_recent_vote(0).map(|v| v.slot);
                 debug!("vote state {:?}", vote_state);
                 debug!(
                     "observed slot {}",
@@ -269,8 +290,13 @@ impl Tower {
             let start_root = vote_state.root_slot;
 
             // Add the last vote to update the `heaviest_subtree_fork_choice`
-            if let Some(last_voted_slot) = vote_state.last_voted_slot() {
-                pubkey_votes.push((key, last_voted_slot));
+            if let Some(last_landed_voted_slot) = vote_state.last_voted_slot() {
+                latest_validator_votes_for_frozen_banks.check_add_vote(
+                    key,
+                    last_landed_voted_slot,
+                    get_frozen_hash(last_landed_voted_slot),
+                    true,
+                );
             }
 
             vote_state.process_slot_vote_unchecked(bank_slot);
@@ -333,7 +359,7 @@ impl Tower {
             total_stake,
             bank_weight,
             lockout_intervals,
-            pubkey_votes: Arc::new(pubkey_votes),
+            my_latest_landed_vote,
         }
     }
 
@@ -349,13 +375,24 @@ impl Tower {
             .unwrap_or(false)
     }
 
-    fn new_vote(
-        local_vote_state: &VoteState,
+    pub fn tower_slots(&self) -> Vec<Slot> {
+        self.lockouts.tower()
+    }
+
+    pub fn last_vote_tx_blockhash(&self) -> Hash {
+        self.last_vote_tx_blockhash
+    }
+
+    pub fn refresh_last_vote_tx_blockhash(&mut self, new_vote_tx_blockhash: Hash) {
+        self.last_vote_tx_blockhash = new_vote_tx_blockhash;
+    }
+
+    fn apply_vote_and_generate_vote_diff(
+        local_vote_state: &mut VoteState,
         slot: Slot,
         hash: Hash,
         last_voted_slot_in_bank: Option<Slot>,
-    ) -> (Vote, Vec<Slot> /*VoteState.tower*/) {
-        let mut local_vote_state = local_vote_state.clone();
+    ) -> Vote {
         let vote = Vote::new(vec![slot], hash);
         local_vote_state.process_vote_unchecked(&vote);
         let slots = if let Some(last_voted_slot_in_bank) = last_voted_slot_in_bank {
@@ -374,33 +411,48 @@ impl Tower {
             slots,
             local_vote_state.votes
         );
-        (Vote::new(slots, hash), local_vote_state.tower())
+        Vote::new(slots, hash)
     }
 
-    fn last_voted_slot_in_bank(bank: &Bank, vote_account_pubkey: &Pubkey) -> Option<Slot> {
+    pub fn last_voted_slot_in_bank(bank: &Bank, vote_account_pubkey: &Pubkey) -> Option<Slot> {
         let (_stake, vote_account) = bank.get_vote_account(vote_account_pubkey)?;
         let slot = vote_account.vote_state().as_ref().ok()?.last_voted_slot();
         slot
     }
 
-    pub fn new_vote_from_bank(
-        &self,
-        bank: &Bank,
-        vote_account_pubkey: &Pubkey,
-    ) -> (Vote, Vec<Slot> /*VoteState.tower*/) {
-        let voted_slot = Self::last_voted_slot_in_bank(bank, vote_account_pubkey);
-        Self::new_vote(&self.lockouts, bank.slot(), bank.hash(), voted_slot)
+    pub fn record_bank_vote(&mut self, bank: &Bank, vote_account_pubkey: &Pubkey) -> Option<Slot> {
+        let last_voted_slot_in_bank = Self::last_voted_slot_in_bank(bank, vote_account_pubkey);
+
+        // Returns the new root if one is made after applying a vote for the given bank to
+        // `self.lockouts`
+        self.record_bank_vote_and_update_lockouts(bank.slot(), bank.hash(), last_voted_slot_in_bank)
     }
 
-    pub fn record_bank_vote(&mut self, vote: Vote) -> Option<Slot> {
-        let slot = vote.last_voted_slot().unwrap_or(0);
-        trace!("{} record_vote for {}", self.node_pubkey, slot);
+    fn record_bank_vote_and_update_lockouts(
+        &mut self,
+        vote_slot: Slot,
+        vote_hash: Hash,
+        last_voted_slot_in_bank: Option<Slot>,
+    ) -> Option<Slot> {
+        trace!("{} record_vote for {}", self.node_pubkey, vote_slot);
         let old_root = self.root();
-        self.lockouts.process_vote_unchecked(&vote);
-        self.last_vote = vote;
+        let mut new_vote = Self::apply_vote_and_generate_vote_diff(
+            &mut self.lockouts,
+            vote_slot,
+            vote_hash,
+            last_voted_slot_in_bank,
+        );
+
+        new_vote.timestamp = self.maybe_timestamp(self.last_vote.last_voted_slot().unwrap_or(0));
+        self.last_vote = new_vote;
+
         let new_root = self.root();
 
-        datapoint_info!("tower-vote", ("latest", slot, i64), ("root", new_root, i64));
+        datapoint_info!(
+            "tower-vote",
+            ("latest", vote_slot, i64),
+            ("root", new_root, i64)
+        );
         if old_root != new_root {
             Some(new_root)
         } else {
@@ -410,22 +462,23 @@ impl Tower {
 
     #[cfg(test)]
     pub fn record_vote(&mut self, slot: Slot, hash: Hash) -> Option<Slot> {
-        let vote = Vote::new(vec![slot], hash);
-        self.record_bank_vote(vote)
+        self.record_bank_vote_and_update_lockouts(slot, hash, self.last_voted_slot())
     }
 
     pub fn last_voted_slot(&self) -> Option<Slot> {
         self.last_vote.last_voted_slot()
     }
 
+    pub fn last_voted_slot_hash(&self) -> Option<(Slot, Hash)> {
+        self.last_vote.last_voted_slot_hash()
+    }
+
     pub fn stray_restored_slot(&self) -> Option<Slot> {
         self.stray_restored_slot
     }
 
-    pub fn last_vote_and_timestamp(&mut self) -> Vote {
-        let mut last_vote = self.last_vote.clone();
-        last_vote.timestamp = self.maybe_timestamp(last_vote.last_voted_slot().unwrap_or(0));
-        last_vote
+    pub fn last_vote(&mut self) -> Vote {
+        self.last_vote.clone()
     }
 
     fn maybe_timestamp(&mut self, current_slot: Slot) -> Option<UnixTimestamp> {
@@ -575,9 +628,13 @@ impl Tower {
                     SwitchForkDecision::FailedSwitchThreshold(0, total_stake)
                 };
 
+                let rollback_due_to_to_to_duplicate_ancestor = |latest_duplicate_ancestor| {
+                    SwitchForkDecision::FailedSwitchDuplicateRollback(latest_duplicate_ancestor)
+                };
+
                 let last_vote_ancestors =
                     ancestors.get(&last_voted_slot).unwrap_or_else(|| {
-                        if !self.is_stray_last_vote() {
+                        if self.is_stray_last_vote() {
                             // Unless last vote is stray and stale, ancestors.get(last_voted_slot) must
                             // return Some(_), justifying to panic! here.
                             // Also, adjust_lockouts_after_replay() correctly makes last_voted_slot None,
@@ -586,9 +643,9 @@ impl Tower {
                             // In other words, except being stray, all other slots have been voted on while
                             // this validator has been running, so we must be able to fetch ancestors for
                             // all of them.
-                            panic!("no ancestors found with slot: {}", last_voted_slot);
-                        } else {
                             empty_ancestors_due_to_minor_unsynced_ledger()
+                        } else {
+                            panic!("no ancestors found with slot: {}", last_voted_slot);
                         }
                     });
 
@@ -601,15 +658,23 @@ impl Tower {
                 }
 
                 if last_vote_ancestors.contains(&switch_slot) {
-                    if !self.is_stray_last_vote() {
-                        panic!(
-                            "Should never consider switching to slot ({}), which is ancestors({:?}) of last vote: {}",
-                            switch_slot,
-                            last_vote_ancestors,
-                            last_voted_slot
-                        );
-                    } else {
+                    if self.is_stray_last_vote() {
                         return suspended_decision_due_to_major_unsynced_ledger();
+                    } else if let Some(latest_duplicate_ancestor) = progress.latest_unconfirmed_duplicate_ancestor(last_voted_slot) {
+                        // We're rolling back because one of the ancestors of the last vote was a duplicate. In this
+                        // case, it's acceptable if the switch candidate is one of ancestors of the previous vote,
+                        // just fail the switch check because there's no point in voting on an ancestor. ReplayStage
+                        // should then have a special case continue building an alternate fork from this ancestor, NOT
+                        // the `last_voted_slot`. This is in contrast to usual SwitchFailure where ReplayStage continues to build blocks
+                        // on latest vote. See `select_vote_and_reset_forks()` for more details.
+                        return rollback_due_to_to_to_duplicate_ancestor(latest_duplicate_ancestor);
+                    } else {
+                        panic!(
+                            "Should never consider switching to ancestor ({}) of last vote: {}, ancestors({:?})",
+                            switch_slot,
+                            last_voted_slot,
+                            last_vote_ancestors,
+                        );
                     }
                 }
 
@@ -1175,7 +1240,7 @@ impl SavedTower {
     pub fn new<T: Signer>(tower: &Tower, keypair: &Arc<T>) -> Result<Self> {
         let data = bincode::serialize(tower)?;
         let signature = keypair.sign_message(&data);
-        Ok(Self { data, signature })
+        Ok(Self { signature, data })
     }
 
     pub fn verify(&self, pubkey: &Pubkey) -> bool {
@@ -1237,11 +1302,13 @@ pub mod test {
     use super::*;
     use crate::{
         cluster_info_vote_listener::VoteTracker,
+        cluster_slot_state_verifier::GossipDuplicateConfirmedSlots,
         cluster_slots::ClusterSlots,
         fork_choice::SelectVoteAndResetForkResult,
-        heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
-        progress_map::ForkProgress,
+        heaviest_subtree_fork_choice::{HeaviestSubtreeForkChoice, SlotHashKey},
+        progress_map::{DuplicateStats, ForkProgress},
         replay_stage::{HeaviestForkFailures, ReplayStage},
+        unfrozen_gossip_verified_vote_hashes::UnfrozenGossipVerifiedVoteHashes,
     };
     use solana_ledger::{blockstore::make_slot_entries, get_tmp_ledger_path};
     use solana_runtime::{
@@ -1280,6 +1347,7 @@ pub mod test {
         pub bank_forks: RwLock<BankForks>,
         pub progress: ProgressMap,
         pub heaviest_subtree_fork_choice: HeaviestSubtreeForkChoice,
+        pub latest_validator_votes_for_frozen_banks: LatestValidatorVotesForFrozenBanks,
     }
 
     impl VoteSimulator {
@@ -1299,6 +1367,8 @@ pub mod test {
                 bank_forks: RwLock::new(bank_forks),
                 progress,
                 heaviest_subtree_fork_choice,
+                latest_validator_votes_for_frozen_banks:
+                    LatestValidatorVotesForFrozenBanks::default(),
             }
         }
         pub(crate) fn fill_bank_forks(
@@ -1313,9 +1383,9 @@ pub mod test {
 
             while let Some(visit) = walk.get() {
                 let slot = visit.node().data;
-                self.progress
-                    .entry(slot)
-                    .or_insert_with(|| ForkProgress::new(Hash::default(), None, None, 0, 0));
+                self.progress.entry(slot).or_insert_with(|| {
+                    ForkProgress::new(Hash::default(), None, DuplicateStats::default(), None, 0, 0)
+                });
                 if self.bank_forks.read().unwrap().get(slot).is_some() {
                     walk.forward();
                     continue;
@@ -1342,8 +1412,10 @@ pub mod test {
                     }
                 }
                 new_bank.freeze();
-                self.heaviest_subtree_fork_choice
-                    .add_new_leaf_slot(new_bank.slot(), Some(new_bank.parent_slot()));
+                self.heaviest_subtree_fork_choice.add_new_leaf_slot(
+                    (new_bank.slot(), new_bank.hash()),
+                    Some((new_bank.parent_slot(), new_bank.parent_hash())),
+                );
                 self.bank_forks.write().unwrap().insert(new_bank);
                 walk.forward();
             }
@@ -1378,6 +1450,7 @@ pub mod test {
                 &ClusterSlots::default(),
                 &self.bank_forks,
                 &mut self.heaviest_subtree_fork_choice,
+                &mut self.latest_validator_votes_for_frozen_banks,
             );
 
             let vote_bank = self
@@ -1395,7 +1468,7 @@ pub mod test {
                 ..
             } = ReplayStage::select_vote_and_reset_forks(
                 &vote_bank,
-                &None,
+                None,
                 &ancestors,
                 &descendants,
                 &self.progress,
@@ -1407,8 +1480,9 @@ pub mod test {
             if !heaviest_fork_failures.is_empty() {
                 return heaviest_fork_failures;
             }
-            let vote = tower.new_vote_from_bank(&vote_bank, &my_vote_pubkey).0;
-            if let Some(new_root) = tower.record_bank_vote(vote) {
+
+            let new_root = tower.record_bank_vote(&vote_bank, &my_vote_pubkey);
+            if let Some(new_root) = new_root {
                 self.set_root(new_root);
             }
 
@@ -1423,6 +1497,10 @@ pub mod test {
                 &AbsRequestSender::default(),
                 None,
                 &mut self.heaviest_subtree_fork_choice,
+                &mut GossipDuplicateConfirmedSlots::default(),
+                &mut UnfrozenGossipVerifiedVoteHashes::default(),
+                &mut true,
+                &mut Vec::new(),
             )
         }
 
@@ -1457,7 +1535,9 @@ pub mod test {
         ) {
             self.progress
                 .entry(slot)
-                .or_insert_with(|| ForkProgress::new(Hash::default(), None, None, 0, 0))
+                .or_insert_with(|| {
+                    ForkProgress::new(Hash::default(), None, DuplicateStats::default(), None, 0, 0)
+                })
                 .fork_stats
                 .lockout_intervals
                 .entry(lockout_interval.1)
@@ -1564,7 +1644,14 @@ pub mod test {
         let mut progress = ProgressMap::default();
         progress.insert(
             0,
-            ForkProgress::new(bank0.last_blockhash(), None, None, 0, 0),
+            ForkProgress::new(
+                bank0.last_blockhash(),
+                None,
+                DuplicateStats::default(),
+                None,
+                0,
+                0,
+            ),
         );
         let bank_forks = BankForks::new(bank0);
         let heaviest_subtree_fork_choice =
@@ -1604,6 +1691,12 @@ pub mod test {
         assert!(decision
             .to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default())
             .is_none());
+
+        decision = SwitchForkDecision::FailedSwitchDuplicateRollback(0);
+        assert!(decision
+            .to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default())
+            .is_none());
+
         decision = SwitchForkDecision::SameFork;
         assert_eq!(
             decision.to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default()),
@@ -1613,6 +1706,7 @@ pub mod test {
                 vote.clone(),
             ))
         );
+
         decision = SwitchForkDecision::SwitchProof(Hash::default());
         assert_eq!(
             decision.to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default()),
@@ -1655,11 +1749,20 @@ pub mod test {
     }
 
     #[test]
-    fn test_switch_threshold() {
+    fn test_switch_threshold_duplicate_rollback() {
+        run_test_switch_threshold_duplicate_rollback(false);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_switch_threshold_duplicate_rollback_panic() {
+        run_test_switch_threshold_duplicate_rollback(true);
+    }
+
+    fn setup_switch_test(num_accounts: usize) -> (Arc<Bank>, VoteSimulator, u64) {
         // Init state
-        let mut vote_simulator = VoteSimulator::new(2);
-        let my_pubkey = vote_simulator.node_pubkeys[0];
-        let other_vote_account = vote_simulator.vote_pubkeys[1];
+        assert!(num_accounts > 1);
+        let mut vote_simulator = VoteSimulator::new(num_accounts);
         let bank0 = vote_simulator
             .bank_forks
             .read()
@@ -1690,6 +1793,82 @@ pub mod test {
         for (_, fork_progress) in vote_simulator.progress.iter_mut() {
             fork_progress.fork_stats.computed = true;
         }
+
+        (bank0, vote_simulator, total_stake)
+    }
+
+    fn run_test_switch_threshold_duplicate_rollback(should_panic: bool) {
+        let (bank0, mut vote_simulator, total_stake) = setup_switch_test(2);
+        let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
+        let descendants = vote_simulator
+            .bank_forks
+            .read()
+            .unwrap()
+            .descendants()
+            .clone();
+        let mut tower = Tower::new_with_key(&vote_simulator.node_pubkeys[0]);
+
+        // Last vote is 47
+        tower.record_vote(47, Hash::default());
+
+        // Trying to switch to an ancestor of last vote should only not panic
+        // if the current vote has a duplicate ancestor
+        let ancestor_of_voted_slot = 43;
+        let duplicate_ancestor1 = 44;
+        let duplicate_ancestor2 = 45;
+        vote_simulator.progress.set_unconfirmed_duplicate_slot(
+            duplicate_ancestor1,
+            &descendants.get(&duplicate_ancestor1).unwrap(),
+        );
+        vote_simulator.progress.set_unconfirmed_duplicate_slot(
+            duplicate_ancestor2,
+            &descendants.get(&duplicate_ancestor2).unwrap(),
+        );
+        assert_eq!(
+            tower.check_switch_threshold(
+                ancestor_of_voted_slot,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::FailedSwitchDuplicateRollback(duplicate_ancestor2)
+        );
+        let mut confirm_ancestors = vec![duplicate_ancestor1];
+        if should_panic {
+            // Adding the last duplicate ancestor will
+            // 1) Cause loop below to confirm last ancestor
+            // 2) Check switch threshold on a vote ancestor when there
+            // are no duplicates on that fork, which will cause a panic
+            confirm_ancestors.push(duplicate_ancestor2);
+        }
+        for (i, duplicate_ancestor) in confirm_ancestors.into_iter().enumerate() {
+            vote_simulator.progress.set_confirmed_duplicate_slot(
+                duplicate_ancestor,
+                ancestors.get(&duplicate_ancestor).unwrap(),
+                &descendants.get(&duplicate_ancestor).unwrap(),
+            );
+            let res = tower.check_switch_threshold(
+                ancestor_of_voted_slot,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            );
+            if i == 0 {
+                assert_eq!(
+                    res,
+                    SwitchForkDecision::FailedSwitchDuplicateRollback(duplicate_ancestor2)
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_switch_threshold() {
+        let (bank0, mut vote_simulator, total_stake) = setup_switch_test(2);
         let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
         let mut descendants = vote_simulator
             .bank_forks
@@ -1697,7 +1876,8 @@ pub mod test {
             .unwrap()
             .descendants()
             .clone();
-        let mut tower = Tower::new_with_key(&my_pubkey);
+        let mut tower = Tower::new_with_key(&vote_simulator.node_pubkeys[0]);
+        let other_vote_account = vote_simulator.vote_pubkeys[1];
 
         // Last vote is 47
         tower.record_vote(47, Hash::default());
@@ -1986,24 +2166,34 @@ pub mod test {
         //two accounts voting for slot 0 with 1 token staked
         let mut accounts = gen_stakes(&[(1, &[0]), (1, &[0])]);
         accounts.sort_by_key(|(pk, _)| *pk);
-        let account_latest_votes: PubkeyVotes =
-            accounts.iter().map(|(pubkey, _)| (*pubkey, 0)).collect();
+        let account_latest_votes: Vec<(Pubkey, SlotHashKey)> = accounts
+            .iter()
+            .map(|(pubkey, _)| (*pubkey, (0, Hash::default())))
+            .collect();
 
         let ancestors = vec![(1, vec![0].into_iter().collect()), (0, HashSet::new())]
             .into_iter()
             .collect();
+        let mut latest_validator_votes_for_frozen_banks =
+            LatestValidatorVotesForFrozenBanks::default();
         let ComputedBankState {
             voted_stakes,
             total_stake,
             bank_weight,
-            pubkey_votes,
             ..
-        } = Tower::collect_vote_lockouts(&Pubkey::default(), 1, accounts.into_iter(), &ancestors);
+        } = Tower::collect_vote_lockouts(
+            &Pubkey::default(),
+            1,
+            accounts.into_iter(),
+            &ancestors,
+            |_| Some(Hash::default()),
+            &mut latest_validator_votes_for_frozen_banks,
+        );
         assert_eq!(voted_stakes[&0], 2);
         assert_eq!(total_stake, 2);
-        let mut pubkey_votes = Arc::try_unwrap(pubkey_votes).unwrap();
-        pubkey_votes.sort();
-        assert_eq!(pubkey_votes, account_latest_votes);
+        let mut new_votes = latest_validator_votes_for_frozen_banks.take_votes_dirty_set(0);
+        new_votes.sort();
+        assert_eq!(new_votes, account_latest_votes);
 
         // Each account has 1 vote in it. After simulating a vote in collect_vote_lockouts,
         // the account will have 2 votes, with lockout 2 + 4 = 6. So expected weight for
@@ -2016,9 +2206,14 @@ pub mod test {
         //two accounts voting for slots 0..MAX_LOCKOUT_HISTORY with 1 token staked
         let mut accounts = gen_stakes(&[(1, &votes), (1, &votes)]);
         accounts.sort_by_key(|(pk, _)| *pk);
-        let account_latest_votes: PubkeyVotes = accounts
+        let account_latest_votes: Vec<(Pubkey, SlotHashKey)> = accounts
             .iter()
-            .map(|(pubkey, _)| (*pubkey, (MAX_LOCKOUT_HISTORY - 1) as Slot))
+            .map(|(pubkey, _)| {
+                (
+                    *pubkey,
+                    ((MAX_LOCKOUT_HISTORY - 1) as Slot, Hash::default()),
+                )
+            })
             .collect();
         let mut tower = Tower::new_for_tests(0, 0.67);
         let mut ancestors = HashMap::new();
@@ -2040,16 +2235,19 @@ pub mod test {
             + root_weight;
         let expected_bank_weight = 2 * vote_account_expected_weight;
         assert_eq!(tower.lockouts.root_slot, Some(0));
+        let mut latest_validator_votes_for_frozen_banks =
+            LatestValidatorVotesForFrozenBanks::default();
         let ComputedBankState {
             voted_stakes,
             bank_weight,
-            pubkey_votes,
             ..
         } = Tower::collect_vote_lockouts(
             &Pubkey::default(),
             MAX_LOCKOUT_HISTORY as u64,
             accounts.into_iter(),
             &ancestors,
+            |_| Some(Hash::default()),
+            &mut latest_validator_votes_for_frozen_banks,
         );
         for i in 0..MAX_LOCKOUT_HISTORY {
             assert_eq!(voted_stakes[&(i as u64)], 2);
@@ -2057,9 +2255,9 @@ pub mod test {
 
         // should be the sum of all the weights for root
         assert_eq!(bank_weight, expected_bank_weight);
-        let mut pubkey_votes = Arc::try_unwrap(pubkey_votes).unwrap();
-        pubkey_votes.sort();
-        assert_eq!(pubkey_votes, account_latest_votes);
+        let mut new_votes = latest_validator_votes_for_frozen_banks.take_votes_dirty_set(root.slot);
+        new_votes.sort();
+        assert_eq!(new_votes, account_latest_votes);
     }
 
     #[test]
@@ -2268,23 +2466,26 @@ pub mod test {
     }
 
     #[test]
-    fn test_new_vote() {
-        let local = VoteState::default();
-        let (vote, tower_slots) = Tower::new_vote(&local, 0, Hash::default(), None);
-        assert_eq!(local.votes.len(), 0);
+    fn test_apply_vote_and_generate_vote_diff() {
+        let mut local = VoteState::default();
+        let vote = Tower::apply_vote_and_generate_vote_diff(&mut local, 0, Hash::default(), None);
+        assert_eq!(local.votes.len(), 1);
         assert_eq!(vote.slots, vec![0]);
-        assert_eq!(tower_slots, vec![0]);
+        assert_eq!(local.tower(), vec![0]);
     }
 
     #[test]
-    fn test_new_vote_dup_vote() {
-        let local = VoteState::default();
-        let vote = Tower::new_vote(&local, 0, Hash::default(), Some(0));
-        assert!(vote.0.slots.is_empty());
+    fn test_apply_vote_and_generate_vote_diff_dup_vote() {
+        let mut local = VoteState::default();
+        // If `latest_voted_slot_in_bank == Some(0)`, then we already have a vote for 0. Adding
+        // another vote for slot 0 should return an empty vote as the diff.
+        let vote =
+            Tower::apply_vote_and_generate_vote_diff(&mut local, 0, Hash::default(), Some(0));
+        assert!(vote.slots.is_empty());
     }
 
     #[test]
-    fn test_new_vote_next_vote() {
+    fn test_apply_vote_and_generate_vote_diff_next_vote() {
         let mut local = VoteState::default();
         let vote = Vote {
             slots: vec![0],
@@ -2293,13 +2494,14 @@ pub mod test {
         };
         local.process_vote_unchecked(&vote);
         assert_eq!(local.votes.len(), 1);
-        let (vote, tower_slots) = Tower::new_vote(&local, 1, Hash::default(), Some(0));
+        let vote =
+            Tower::apply_vote_and_generate_vote_diff(&mut local, 1, Hash::default(), Some(0));
         assert_eq!(vote.slots, vec![1]);
-        assert_eq!(tower_slots, vec![0, 1]);
+        assert_eq!(local.tower(), vec![0, 1]);
     }
 
     #[test]
-    fn test_new_vote_next_after_expired_vote() {
+    fn test_apply_vote_and_generate_vote_diff_next_after_expired_vote() {
         let mut local = VoteState::default();
         let vote = Vote {
             slots: vec![0],
@@ -2308,10 +2510,14 @@ pub mod test {
         };
         local.process_vote_unchecked(&vote);
         assert_eq!(local.votes.len(), 1);
-        let (vote, tower_slots) = Tower::new_vote(&local, 3, Hash::default(), Some(0));
+
+        // First vote expired, so should be evicted from tower. Thus even with
+        // `latest_voted_slot_in_bank == Some(0)`, the first vote slot won't be
+        // observable in any of the results.
+        let vote =
+            Tower::apply_vote_and_generate_vote_diff(&mut local, 3, Hash::default(), Some(0));
         assert_eq!(vote.slots, vec![3]);
-        // First vote expired, so should be evicted from tower.
-        assert_eq!(tower_slots, vec![3]);
+        assert_eq!(local.tower(), vec![3]);
     }
 
     #[test]
@@ -2356,6 +2562,8 @@ pub mod test {
             vote_to_evaluate,
             accounts.clone().into_iter(),
             &ancestors,
+            |_| None,
+            &mut LatestValidatorVotesForFrozenBanks::default(),
         );
         assert!(tower.check_vote_stake_threshold(vote_to_evaluate, &voted_stakes, total_stake,));
 
@@ -2372,6 +2580,8 @@ pub mod test {
             vote_to_evaluate,
             accounts.into_iter(),
             &ancestors,
+            |_| None,
+            &mut LatestValidatorVotesForFrozenBanks::default(),
         );
         assert!(!tower.check_vote_stake_threshold(vote_to_evaluate, &voted_stakes, total_stake,));
     }
@@ -2383,10 +2593,12 @@ pub mod test {
         } else {
             vec![]
         };
-        let expected = Vote::new(slots, Hash::default());
+        let mut expected = Vote::new(slots, Hash::default());
         for i in 0..num_votes {
             tower.record_vote(i as u64, Hash::default());
         }
+
+        expected.timestamp = tower.last_vote.timestamp;
         assert_eq!(expected, tower.last_vote)
     }
 

core/src/crds.rs

@@ -33,11 +33,11 @@ use indexmap::set::IndexSet;
 use rayon::{prelude::*, ThreadPool};
 use solana_sdk::hash::{hash, Hash};
 use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::Keypair;
-use solana_sdk::timing::timestamp;
-use std::cmp;
-use std::collections::{hash_map, BTreeSet, HashMap};
-use std::ops::{Bound, Index, IndexMut};
+use std::{
+    cmp::Ordering,
+    collections::{hash_map, BTreeSet, HashMap},
+    ops::{Bound, Index, IndexMut},
+};
 
 const CRDS_SHARDS_BITS: u32 = 8;
 // Limit number of crds values associated with each unique pubkey. This
@@ -60,7 +60,9 @@ pub struct Crds {
 
 #[derive(PartialEq, Debug)]
 pub enum CrdsError {
-    InsertFailed,
+    // Hash of the crds value which failed to insert should be recorded in
+    // failed_inserts to be excluded from the next pull-request.
+    InsertFailed(Hash),
     UnknownStakes,
 }
 
@@ -71,26 +73,15 @@ pub enum CrdsError {
 pub struct VersionedCrdsValue {
     pub value: CrdsValue,
     /// local time when inserted
-    pub insert_timestamp: u64,
+    pub(crate) insert_timestamp: u64,
     /// local time when updated
-    pub local_timestamp: u64,
+    pub(crate) local_timestamp: u64,
     /// value hash
-    pub value_hash: Hash,
+    pub(crate) value_hash: Hash,
 }
 
-impl PartialOrd for VersionedCrdsValue {
-    fn partial_cmp(&self, other: &VersionedCrdsValue) -> Option<cmp::Ordering> {
-        if self.value.label() != other.value.label() {
-            None
-        } else if self.value.wallclock() == other.value.wallclock() {
-            Some(self.value_hash.cmp(&other.value_hash))
-        } else {
-            Some(self.value.wallclock().cmp(&other.value.wallclock()))
-        }
-    }
-}
 impl VersionedCrdsValue {
-    pub fn new(local_timestamp: u64, value: CrdsValue) -> Self {
+    fn new(local_timestamp: u64, value: CrdsValue) -> Self {
         let value_hash = hash(&serialize(&value).unwrap());
         VersionedCrdsValue {
             value,
@@ -99,13 +90,6 @@ impl VersionedCrdsValue {
             value_hash,
         }
     }
-
-    /// New random VersionedCrdsValue for tests and simulations.
-    pub fn new_rand<R: rand::Rng>(rng: &mut R, keypair: Option<&Keypair>) -> Self {
-        let delay = 10 * 60 * 1000; // 10 minutes
-        let now = timestamp() - delay + rng.gen_range(0, 2 * delay);
-        Self::new(now, CrdsValue::new_rand(rng, keypair))
-    }
 }
 
 impl Default for Crds {
@@ -122,34 +106,46 @@ impl Default for Crds {
     }
 }
 
-impl Crds {
-    /// must be called atomically with `insert_versioned`
-    pub fn new_versioned(&self, local_timestamp: u64, value: CrdsValue) -> VersionedCrdsValue {
-        VersionedCrdsValue::new(local_timestamp, value)
+// Returns true if the first value updates the 2nd one.
+// Both values should have the same key/label.
+fn overrides(value: &CrdsValue, other: &VersionedCrdsValue) -> bool {
+    assert_eq!(value.label(), other.value.label(), "labels mismatch!");
+    match value.wallclock().cmp(&other.value.wallclock()) {
+        Ordering::Less => false,
+        Ordering::Greater => true,
+        // Ties should be broken in a deterministic way across the cluster.
+        // For backward compatibility this is done by comparing hash of
+        // serialized values.
+        Ordering::Equal => {
+            let value_hash = hash(&serialize(&value).unwrap());
+            other.value_hash < value_hash
+        }
     }
-    pub fn would_insert(
-        &self,
+}
+
+impl Crds {
+    /// Returns true if the given value updates an existing one in the table.
+    /// The value is outdated and fails to insert, if it already exists in the
+    /// table with a more recent wallclock.
+    pub(crate) fn upserts(&self, value: &CrdsValue) -> bool {
+        match self.table.get(&value.label()) {
+            Some(other) => overrides(value, other),
+            None => true,
+        }
+    }
+
+    pub fn insert(
+        &mut self,
         value: CrdsValue,
         local_timestamp: u64,
-    ) -> (bool, VersionedCrdsValue) {
-        let new_value = self.new_versioned(local_timestamp, value);
-        let label = new_value.value.label();
-        // New value is outdated and fails to insert, if it already exists in
-        // the table with a more recent wallclock.
-        let outdated = matches!(self.table.get(&label), Some(current) if new_value <= *current);
-        (!outdated, new_value)
-    }
-    /// insert the new value, returns the old value if insert succeeds
-    pub fn insert_versioned(
-        &mut self,
-        new_value: VersionedCrdsValue,
     ) -> Result<Option<VersionedCrdsValue>, CrdsError> {
-        let label = new_value.value.label();
+        let label = value.label();
+        let value = VersionedCrdsValue::new(local_timestamp, value);
         match self.table.entry(label) {
             Entry::Vacant(entry) => {
                 let entry_index = entry.index();
-                self.shards.insert(entry_index, &new_value);
-                match new_value.value.data {
+                self.shards.insert(entry_index, &value);
+                match value.value.data {
                     CrdsData::ContactInfo(_) => {
                         self.nodes.insert(entry_index);
                     }
@@ -158,52 +154,45 @@ impl Crds {
                     }
                     CrdsData::EpochSlots(_, _) => {
                         self.epoch_slots
-                            .insert((new_value.insert_timestamp, entry_index));
+                            .insert((value.insert_timestamp, entry_index));
                     }
                     _ => (),
                 };
                 self.records
-                    .entry(new_value.value.pubkey())
+                    .entry(value.value.pubkey())
                     .or_default()
                     .insert(entry_index);
-                entry.insert(new_value);
+                entry.insert(value);
                 self.num_inserts += 1;
                 Ok(None)
             }
-            Entry::Occupied(mut entry) if *entry.get() < new_value => {
+            Entry::Occupied(mut entry) if overrides(&value.value, entry.get()) => {
                 let entry_index = entry.index();
                 self.shards.remove(entry_index, entry.get());
-                self.shards.insert(entry_index, &new_value);
-                if let CrdsData::EpochSlots(_, _) = new_value.value.data {
+                self.shards.insert(entry_index, &value);
+                if let CrdsData::EpochSlots(_, _) = value.value.data {
                     self.epoch_slots
                         .remove(&(entry.get().insert_timestamp, entry_index));
                     self.epoch_slots
-                        .insert((new_value.insert_timestamp, entry_index));
+                        .insert((value.insert_timestamp, entry_index));
                 }
                 self.num_inserts += 1;
                 // As long as the pubkey does not change, self.records
                 // does not need to be updated.
-                debug_assert_eq!(entry.get().value.pubkey(), new_value.value.pubkey());
-                Ok(Some(entry.insert(new_value)))
+                debug_assert_eq!(entry.get().value.pubkey(), value.value.pubkey());
+                Ok(Some(entry.insert(value)))
             }
             _ => {
                 trace!(
                     "INSERT FAILED data: {} new.wallclock: {}",
-                    new_value.value.label(),
-                    new_value.value.wallclock(),
+                    value.value.label(),
+                    value.value.wallclock(),
                 );
-                Err(CrdsError::InsertFailed)
+                Err(CrdsError::InsertFailed(value.value_hash))
             }
         }
     }
-    pub fn insert(
-        &mut self,
-        value: CrdsValue,
-        local_timestamp: u64,
-    ) -> Result<Option<VersionedCrdsValue>, CrdsError> {
-        let new_value = self.new_versioned(local_timestamp, value);
-        self.insert_versioned(new_value)
-    }
+
     pub fn lookup(&self, label: &CrdsValueLabel) -> Option<&CrdsValue> {
         self.table.get(label).map(|x| &x.value)
     }
@@ -264,6 +253,11 @@ impl Crds {
             .map(move |i| self.table.index(*i))
     }
 
+    /// Returns number of known pubkeys (network size).
+    pub(crate) fn num_nodes(&self) -> usize {
+        self.records.len()
+    }
+
     pub fn len(&self) -> usize {
         self.table.len()
     }
@@ -295,7 +289,15 @@ impl Crds {
 
     /// Update the timestamp's of all the labels that are associated with Pubkey
     pub fn update_record_timestamp(&mut self, pubkey: &Pubkey, now: u64) {
-        if let Some(indices) = self.records.get(pubkey) {
+        // It suffices to only overwrite the origin's timestamp since that is
+        // used when purging old values. If the origin does not exist in the
+        // table, fallback to exhaustive update on all associated records.
+        let origin = CrdsValueLabel::ContactInfo(*pubkey);
+        if let Some(origin) = self.table.get_mut(&origin) {
+            if origin.local_timestamp < now {
+                origin.local_timestamp = now;
+            }
+        } else if let Some(indices) = self.records.get(pubkey) {
             for index in indices {
                 let entry = self.table.index_mut(*index);
                 if entry.local_timestamp < now {
@@ -313,28 +315,31 @@ impl Crds {
         now: u64,
         timeouts: &HashMap<Pubkey, u64>,
     ) -> Vec<CrdsValueLabel> {
-        #[rustversion::before(1.49.0)]
-        fn select_nth<T: Ord>(xs: &mut Vec<T>, _nth: usize) {
-            xs.sort_unstable();
-        }
-        #[rustversion::since(1.49.0)]
-        fn select_nth<T: Ord>(xs: &mut Vec<T>, nth: usize) {
-            xs.select_nth_unstable(nth);
-        }
         let default_timeout = *timeouts
             .get(&Pubkey::default())
             .expect("must have default timeout");
         // Given an index of all crd values associated with a pubkey,
         // returns crds labels of old values to be evicted.
         let evict = |pubkey, index: &IndexSet<usize>| {
-            let timeout = *timeouts.get(pubkey).unwrap_or(&default_timeout);
+            let timeout = timeouts.get(pubkey).copied().unwrap_or(default_timeout);
+            let local_timestamp = {
+                let origin = CrdsValueLabel::ContactInfo(*pubkey);
+                match self.table.get(&origin) {
+                    Some(origin) => origin.local_timestamp,
+                    None => 0,
+                }
+            };
             let mut old_labels = Vec::new();
             // Buffer of crds values to be evicted based on their wallclock.
             let mut recent_unlimited_labels: Vec<(u64 /*wallclock*/, usize /*index*/)> = index
                 .into_iter()
                 .filter_map(|ix| {
                     let (label, value) = self.table.get_index(*ix).unwrap();
-                    if value.local_timestamp.saturating_add(timeout) <= now {
+                    let expiry_timestamp = value
+                        .local_timestamp
+                        .max(local_timestamp)
+                        .saturating_add(timeout);
+                    if expiry_timestamp <= now {
                         old_labels.push(label.clone());
                         None
                     } else {
@@ -351,7 +356,7 @@ impl Crds {
                 .saturating_sub(MAX_CRDS_VALUES_PER_PUBKEY);
             // Partition on wallclock to discard the older ones.
             if nth > 0 && nth < recent_unlimited_labels.len() {
-                select_nth(&mut recent_unlimited_labels, nth);
+                recent_unlimited_labels.select_nth_unstable(nth);
             }
             old_labels.extend(
                 recent_unlimited_labels
@@ -488,10 +493,13 @@ impl Crds {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::{contact_info::ContactInfo, crds_value::NodeInstance};
+    use crate::{
+        contact_info::ContactInfo,
+        crds_value::{new_rand_timestamp, NodeInstance},
+    };
     use rand::{thread_rng, Rng};
     use rayon::ThreadPoolBuilder;
-    use solana_sdk::signature::Signer;
+    use solana_sdk::signature::{Keypair, Signer};
     use std::{collections::HashSet, iter::repeat_with};
 
     #[test]
@@ -507,8 +515,12 @@ mod test {
     fn test_update_old() {
         let mut crds = Crds::default();
         let val = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::default()));
+        let value_hash = hash(&serialize(&val).unwrap());
         assert_eq!(crds.insert(val.clone(), 0), Ok(None));
-        assert_eq!(crds.insert(val.clone(), 1), Err(CrdsError::InsertFailed));
+        assert_eq!(
+            crds.insert(val.clone(), 1),
+            Err(CrdsError::InsertFailed(value_hash))
+        );
         assert_eq!(crds.table[&val.label()].local_timestamp, 0);
     }
     #[test]
@@ -702,8 +714,9 @@ mod test {
         let mut num_overrides = 0;
         for _ in 0..4096 {
             let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
-            let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
-            match crds.insert_versioned(value) {
+            let value = CrdsValue::new_rand(&mut rng, Some(keypair));
+            let local_timestamp = new_rand_timestamp(&mut rng);
+            match crds.insert(value, local_timestamp) {
                 Ok(None) => {
                     num_inserts += 1;
                     check_crds_shards(&crds);
@@ -742,8 +755,10 @@ mod test {
                 let num_epoch_slots = crds
                     .table
                     .values()
-                    .filter(|value| value.insert_timestamp >= since)
-                    .filter(|value| matches!(value.value.data, CrdsData::EpochSlots(_, _)))
+                    .filter(|value| {
+                        value.insert_timestamp >= since
+                            && matches!(value.value.data, CrdsData::EpochSlots(_, _))
+                    })
                     .count();
                 assert_eq!(num_epoch_slots, crds.get_epoch_slots_since(since).count());
                 for value in crds.get_epoch_slots_since(since) {
@@ -793,8 +808,9 @@ mod test {
         let mut num_overrides = 0;
         for k in 0..4096 {
             let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
-            let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
-            match crds.insert_versioned(value) {
+            let value = CrdsValue::new_rand(&mut rng, Some(keypair));
+            let local_timestamp = new_rand_timestamp(&mut rng);
+            match crds.insert(value, local_timestamp) {
                 Ok(None) => {
                     num_inserts += 1;
                 }
@@ -852,8 +868,9 @@ mod test {
         let mut crds = Crds::default();
         for k in 0..4096 {
             let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
-            let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
-            let _ = crds.insert_versioned(value);
+            let value = CrdsValue::new_rand(&mut rng, Some(keypair));
+            let local_timestamp = new_rand_timestamp(&mut rng);
+            let _ = crds.insert(value, local_timestamp);
             if k % 64 == 0 {
                 check_crds_records(&crds);
             }
@@ -893,8 +910,9 @@ mod test {
         let mut crds = Crds::default();
         for _ in 0..2048 {
             let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
-            let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
-            let _ = crds.insert_versioned(value);
+            let value = CrdsValue::new_rand(&mut rng, Some(keypair));
+            let local_timestamp = new_rand_timestamp(&mut rng);
+            let _ = crds.insert(value, local_timestamp);
         }
         let num_values = crds.table.len();
         let num_pubkeys = num_unique_pubkeys(crds.table.values());
@@ -949,8 +967,8 @@ mod test {
         let v2 = VersionedCrdsValue::new(1, val);
         assert_eq!(v1, v2);
         assert!(!(v1 != v2));
-        assert_eq!(v1.partial_cmp(&v2), Some(cmp::Ordering::Equal));
-        assert_eq!(v2.partial_cmp(&v1), Some(cmp::Ordering::Equal));
+        assert!(!overrides(&v1.value, &v2));
+        assert!(!overrides(&v2.value, &v1));
     }
     #[test]
     #[allow(clippy::neg_cmp_op_on_partial_ord)]
@@ -973,18 +991,12 @@ mod test {
         assert_ne!(v1.value_hash, v2.value_hash);
         assert!(v1 != v2);
         assert!(!(v1 == v2));
-        if v1 > v2 {
-            assert!(v1 > v2);
-            assert!(v2 < v1);
-            assert_eq!(v1.partial_cmp(&v2), Some(cmp::Ordering::Greater));
-            assert_eq!(v2.partial_cmp(&v1), Some(cmp::Ordering::Less));
-        } else if v2 > v1 {
-            assert!(v1 < v2);
-            assert!(v2 > v1);
-            assert_eq!(v1.partial_cmp(&v2), Some(cmp::Ordering::Less));
-            assert_eq!(v2.partial_cmp(&v1), Some(cmp::Ordering::Greater));
+        if v1.value_hash > v2.value_hash {
+            assert!(overrides(&v1.value, &v2));
+            assert!(!overrides(&v2.value, &v1));
         } else {
-            panic!("bad PartialOrd implementation?");
+            assert!(overrides(&v2.value, &v1));
+            assert!(!overrides(&v1.value, &v2));
         }
     }
     #[test]
@@ -1005,14 +1017,13 @@ mod test {
             ))),
         );
         assert_eq!(v1.value.label(), v2.value.label());
-        assert!(v1 > v2);
-        assert!(!(v1 < v2));
+        assert!(overrides(&v1.value, &v2));
+        assert!(!overrides(&v2.value, &v1));
         assert!(v1 != v2);
         assert!(!(v1 == v2));
-        assert_eq!(v1.partial_cmp(&v2), Some(cmp::Ordering::Greater));
-        assert_eq!(v2.partial_cmp(&v1), Some(cmp::Ordering::Less));
     }
     #[test]
+    #[should_panic(expected = "labels mismatch!")]
     #[allow(clippy::neg_cmp_op_on_partial_ord)]
     fn test_label_order() {
         let v1 = VersionedCrdsValue::new(
@@ -1031,11 +1042,6 @@ mod test {
         );
         assert_ne!(v1, v2);
         assert!(!(v1 == v2));
-        assert!(!(v1 < v2));
-        assert!(!(v1 > v2));
-        assert!(!(v2 < v1));
-        assert!(!(v2 > v1));
-        assert_eq!(v1.partial_cmp(&v2), None);
-        assert_eq!(v2.partial_cmp(&v1), None);
+        assert!(!overrides(&v2.value, &v1));
     }
 }

core/src/crds_gossip.rs

@@ -4,12 +4,15 @@
 //! packet::PACKET_DATA_SIZE size.
 
 use crate::{
+    cluster_info::Ping,
+    contact_info::ContactInfo,
     crds::{Crds, VersionedCrdsValue},
     crds_gossip_error::CrdsGossipError,
     crds_gossip_pull::{CrdsFilter, CrdsGossipPull, ProcessPullStats},
     crds_gossip_push::{CrdsGossipPush, CRDS_GOSSIP_NUM_ACTIVE},
     crds_value::{CrdsData, CrdsValue, CrdsValueLabel},
     duplicate_shred::{self, DuplicateShredIndex, LeaderScheduleFn, MAX_DUPLICATE_SHREDS},
+    ping_pong::PingCache,
 };
 use rayon::ThreadPool;
 use solana_ledger::shred::Shred;
@@ -19,7 +22,11 @@ use solana_sdk::{
     signature::{Keypair, Signer},
     timing::timestamp,
 };
-use std::collections::{HashMap, HashSet};
+use std::{
+    collections::{HashMap, HashSet},
+    net::SocketAddr,
+    sync::Mutex,
+};
 
 ///The min size for bloom filters
 pub const CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS: usize = 500;
@@ -62,16 +69,12 @@ impl CrdsGossip {
         values
             .into_iter()
             .filter_map(|val| {
-                let res = self
+                let old = self
                     .push
-                    .process_push_message(&mut self.crds, from, val, now);
-                if let Ok(Some(val)) = res {
-                    self.pull
-                        .record_old_hash(val.value_hash, val.local_timestamp);
-                    Some(val)
-                } else {
-                    None
-                }
+                    .process_push_message(&mut self.crds, from, val, now)
+                    .ok()?;
+                self.pull.record_old_hash(old.as_ref()?.value_hash, now);
+                old
             })
             .collect()
     }
@@ -94,20 +97,13 @@ impl CrdsGossip {
         prune_map
     }
 
-    pub fn process_push_messages(&mut self, pending_push_messages: Vec<(CrdsValue, u64)>) {
-        for (push_message, timestamp) in pending_push_messages {
-            let _ =
-                self.push
-                    .process_push_message(&mut self.crds, &self.id, push_message, timestamp);
-        }
-    }
-
     pub fn new_push_messages(
         &mut self,
-        pending_push_messages: Vec<(CrdsValue, u64)>,
+        pending_push_messages: Vec<CrdsValue>,
         now: u64,
     ) -> HashMap<Pubkey, Vec<CrdsValue>> {
-        self.process_push_messages(pending_push_messages);
+        let self_pubkey = self.id;
+        self.process_push_message(&self_pubkey, pending_push_messages, now);
         self.push.new_push_messages(&self.crds, now)
     }
 
@@ -207,7 +203,7 @@ impl CrdsGossip {
             gossip_validators,
             &self.id,
             self.shred_version,
-            self.pull.pull_request_time.len(),
+            self.crds.num_nodes(),
             CRDS_GOSSIP_NUM_ACTIVE,
         )
     }
@@ -216,20 +212,25 @@ impl CrdsGossip {
     pub fn new_pull_request(
         &self,
         thread_pool: &ThreadPool,
+        self_keypair: &Keypair,
         now: u64,
         gossip_validators: Option<&HashSet<Pubkey>>,
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
-    ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
+        ping_cache: &Mutex<PingCache>,
+        pings: &mut Vec<(SocketAddr, Ping)>,
+    ) -> Result<(ContactInfo, Vec<CrdsFilter>), CrdsGossipError> {
         self.pull.new_pull_request(
             thread_pool,
             &self.crds,
-            &self.id,
+            self_keypair,
             self.shred_version,
             now,
             gossip_validators,
             stakes,
             bloom_size,
+            ping_cache,
+            pings,
         )
     }
 
@@ -237,7 +238,7 @@ impl CrdsGossip {
     /// This is used for weighted random selection during `new_pull_request`
     /// It's important to use the local nodes request creation time as the weight
     /// instead of the response received time otherwise failed nodes will increase their weight.
-    pub fn mark_pull_request_creation_time(&mut self, from: &Pubkey, now: u64) {
+    pub fn mark_pull_request_creation_time(&mut self, from: Pubkey, now: u64) {
         self.pull.mark_pull_request_creation_time(from, now)
     }
     /// process a pull request and create a response
@@ -265,7 +266,11 @@ impl CrdsGossip {
         response: Vec<CrdsValue>,
         now: u64,
         process_pull_stats: &mut ProcessPullStats,
-    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>, Vec<Hash>) {
+    ) -> (
+        Vec<CrdsValue>, // valid responses.
+        Vec<CrdsValue>, // responses with expired timestamps.
+        Vec<Hash>,      // hash of outdated values.
+    ) {
         self.pull
             .filter_pull_responses(&self.crds, timeouts, response, now, process_pull_stats)
     }
@@ -274,8 +279,8 @@ impl CrdsGossip {
     pub fn process_pull_responses(
         &mut self,
         from: &Pubkey,
-        responses: Vec<VersionedCrdsValue>,
-        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        responses: Vec<CrdsValue>,
+        responses_expired_timeout: Vec<CrdsValue>,
         failed_inserts: Vec<Hash>,
         now: u64,
         process_pull_stats: &mut ProcessPullStats,
@@ -341,7 +346,7 @@ impl CrdsGossip {
         Self {
             crds: self.crds.clone(),
             push: self.push.mock_clone(),
-            pull: self.pull.clone(),
+            pull: self.pull.mock_clone(),
             ..*self
         }
     }

core/src/crds_gossip_pull.rs

@@ -9,22 +9,34 @@
 //! with random hash functions.  So each subsequent request will have a different distribution
 //! of false positives.
 
-use crate::contact_info::ContactInfo;
-use crate::crds::{Crds, VersionedCrdsValue};
-use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
-use crate::crds_gossip_error::CrdsGossipError;
-use crate::crds_value::{CrdsValue, CrdsValueLabel};
+use crate::{
+    cluster_info::{Ping, CRDS_UNIQUE_PUBKEY_CAPACITY},
+    contact_info::ContactInfo,
+    crds::{Crds, CrdsError},
+    crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS},
+    crds_gossip_error::CrdsGossipError,
+    crds_value::{CrdsValue, CrdsValueLabel},
+    ping_pong::PingCache,
+};
 use itertools::Itertools;
+use lru::LruCache;
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::Rng;
 use rayon::{prelude::*, ThreadPool};
 use solana_runtime::bloom::{AtomicBloom, Bloom};
-use solana_sdk::hash::Hash;
-use solana_sdk::pubkey::Pubkey;
-use std::cmp;
-use std::collections::VecDeque;
-use std::collections::{HashMap, HashSet};
-use std::convert::TryInto;
+use solana_sdk::{
+    hash::{hash, Hash},
+    pubkey::Pubkey,
+    signature::{Keypair, Signer},
+};
+use std::{
+    cmp,
+    collections::{HashMap, HashSet, VecDeque},
+    convert::TryInto,
+    net::SocketAddr,
+    sync::Mutex,
+    time::Instant,
+};
 
 pub const CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS: u64 = 15000;
 // The maximum age of a value received over pull responses
@@ -168,10 +180,9 @@ pub struct ProcessPullStats {
     pub timeout_count: usize,
 }
 
-#[derive(Clone)]
 pub struct CrdsGossipPull {
     /// timestamp of last request
-    pub pull_request_time: HashMap<Pubkey, u64>,
+    pub(crate) pull_request_time: LruCache<Pubkey, u64>,
     /// hash and insert time
     pub purged_values: VecDeque<(Hash, u64)>,
     // Hash value and record time (ms) of the pull responses which failed to be
@@ -188,7 +199,7 @@ impl Default for CrdsGossipPull {
     fn default() -> Self {
         Self {
             purged_values: VecDeque::new(),
-            pull_request_time: HashMap::new(),
+            pull_request_time: LruCache::new(CRDS_UNIQUE_PUBKEY_CAPACITY),
             failed_inserts: VecDeque::new(),
             crds_timeout: CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS,
             msg_timeout: CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS,
@@ -198,35 +209,62 @@ impl Default for CrdsGossipPull {
 }
 impl CrdsGossipPull {
     /// generate a random request
+    #[allow(clippy::too_many_arguments)]
     pub fn new_pull_request(
         &self,
         thread_pool: &ThreadPool,
         crds: &Crds,
-        self_id: &Pubkey,
+        self_keypair: &Keypair,
         self_shred_version: u16,
         now: u64,
         gossip_validators: Option<&HashSet<Pubkey>>,
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
-    ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        let options = self.pull_options(
-            crds,
-            &self_id,
-            self_shred_version,
-            now,
-            gossip_validators,
-            stakes,
-        );
-        if options.is_empty() {
+        ping_cache: &Mutex<PingCache>,
+        pings: &mut Vec<(SocketAddr, Ping)>,
+    ) -> Result<(ContactInfo, Vec<CrdsFilter>), CrdsGossipError> {
+        let (weights, peers): (Vec<_>, Vec<_>) = self
+            .pull_options(
+                crds,
+                &self_keypair.pubkey(),
+                self_shred_version,
+                now,
+                gossip_validators,
+                stakes,
+            )
+            .into_iter()
+            .unzip();
+        if peers.is_empty() {
             return Err(CrdsGossipError::NoPeers);
         }
-        let filters = self.build_crds_filters(thread_pool, crds, bloom_size);
-        let index = WeightedIndex::new(options.iter().map(|weighted| weighted.0)).unwrap();
-        let random = index.sample(&mut rand::thread_rng());
-        let self_info = crds
-            .lookup(&CrdsValueLabel::ContactInfo(*self_id))
-            .unwrap_or_else(|| panic!("self_id invalid {}", self_id));
-        Ok((options[random].1.id, filters, self_info.clone()))
+        let mut peers = {
+            let mut rng = rand::thread_rng();
+            let num_samples = peers.len() * 2;
+            let index = WeightedIndex::new(weights).unwrap();
+            let sample_peer = move || peers[index.sample(&mut rng)];
+            std::iter::repeat_with(sample_peer).take(num_samples)
+        };
+        let peer = {
+            let mut rng = rand::thread_rng();
+            let mut ping_cache = ping_cache.lock().unwrap();
+            let mut pingf = move || Ping::new_rand(&mut rng, self_keypair).ok();
+            let now = Instant::now();
+            peers.find(|peer| {
+                let node = (peer.id, peer.gossip);
+                let (check, ping) = ping_cache.check(now, node, &mut pingf);
+                if let Some(ping) = ping {
+                    pings.push((peer.gossip, ping));
+                }
+                check
+            })
+        };
+        match peer {
+            None => Err(CrdsGossipError::NoPeers),
+            Some(peer) => {
+                let filters = self.build_crds_filters(thread_pool, crds, bloom_size);
+                Ok((peer.clone(), filters))
+            }
+        }
     }
 
     fn pull_options<'a>(
@@ -263,8 +301,12 @@ impl CrdsGossipPull {
             })
             .map(|item| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
-                let req_time: u64 = *self.pull_request_time.get(&item.id).unwrap_or(&0);
-                let since = ((now - req_time) / 1024) as u32;
+                let req_time: u64 = self
+                    .pull_request_time
+                    .peek(&item.id)
+                    .copied()
+                    .unwrap_or_default();
+                let since = (now.saturating_sub(req_time).min(3600 * 1000) / 1024) as u32;
                 let stake = get_stake(&item.id, stakes);
                 let weight = get_weight(max_weight, since, stake);
                 (weight, item)
@@ -276,8 +318,8 @@ impl CrdsGossipPull {
     /// This is used for weighted random selection during `new_pull_request`
     /// It's important to use the local nodes request creation time as the weight
     /// instead of the response received time otherwise failed nodes will increase their weight.
-    pub fn mark_pull_request_creation_time(&mut self, from: &Pubkey, now: u64) {
-        self.pull_request_time.insert(*from, now);
+    pub fn mark_pull_request_creation_time(&mut self, from: Pubkey, now: u64) {
+        self.pull_request_time.put(from, now);
     }
 
     /// Store an old hash in the purged values set
@@ -293,8 +335,7 @@ impl CrdsGossipPull {
         for caller in callers {
             let key = caller.label().pubkey();
             if let Ok(Some(val)) = crds.insert(caller, now) {
-                self.purged_values
-                    .push_back((val.value_hash, val.local_timestamp));
+                self.purged_values.push_back((val.value_hash, now));
             }
             crds.update_record_timestamp(&key, now);
         }
@@ -324,56 +365,41 @@ impl CrdsGossipPull {
         responses: Vec<CrdsValue>,
         now: u64,
         stats: &mut ProcessPullStats,
-    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>, Vec<Hash>) {
-        let mut versioned = vec![];
-        let mut versioned_expired_timestamp = vec![];
+    ) -> (Vec<CrdsValue>, Vec<CrdsValue>, Vec<Hash>) {
+        let mut active_values = vec![];
+        let mut expired_values = vec![];
         let mut failed_inserts = vec![];
-        let mut maybe_push = |response, values: &mut Vec<VersionedCrdsValue>| {
-            let (push, value) = crds.would_insert(response, now);
-            if push {
-                values.push(value);
+        let mut maybe_push = |response, values: &mut Vec<CrdsValue>| {
+            if crds.upserts(&response) {
+                values.push(response);
             } else {
-                failed_inserts.push(value.value_hash)
+                let response = bincode::serialize(&response).unwrap();
+                failed_inserts.push(hash(&response));
             }
         };
-        for r in responses {
-            let owner = r.label().pubkey();
+        let default_timeout = timeouts
+            .get(&Pubkey::default())
+            .copied()
+            .unwrap_or(self.msg_timeout);
+        for response in responses {
+            let owner = response.label().pubkey();
             // Check if the crds value is older than the msg_timeout
-            if now > r.wallclock().checked_add(self.msg_timeout).unwrap_or(0)
-                || now + self.msg_timeout < r.wallclock()
-            {
-                match &r.label() {
-                    CrdsValueLabel::ContactInfo(_) => {
-                        // Check if this ContactInfo is actually too old, it's possible that it has
-                        // stake and so might have a longer effective timeout
-                        let timeout = *timeouts
-                            .get(&owner)
-                            .unwrap_or_else(|| timeouts.get(&Pubkey::default()).unwrap());
-                        if now > r.wallclock().checked_add(timeout).unwrap_or(0)
-                            || now + timeout < r.wallclock()
-                        {
-                            stats.timeout_count += 1;
-                            stats.failed_timeout += 1;
-                            continue;
-                        }
-                    }
-                    _ => {
-                        // Before discarding this value, check if a ContactInfo for the owner
-                        // exists in the table. If it doesn't, that implies that this value can be discarded
-                        if crds.lookup(&CrdsValueLabel::ContactInfo(owner)).is_none() {
-                            stats.timeout_count += 1;
-                            stats.failed_timeout += 1;
-                        } else {
-                            // Silently insert this old value without bumping record timestamps
-                            maybe_push(r, &mut versioned_expired_timestamp);
-                        }
-                        continue;
-                    }
-                }
+            let timeout = timeouts.get(&owner).copied().unwrap_or(default_timeout);
+            // Before discarding this value, check if a ContactInfo for the
+            // owner exists in the table. If it doesn't, that implies that this
+            // value can be discarded
+            if now <= response.wallclock().saturating_add(timeout) {
+                maybe_push(response, &mut active_values);
+            } else if crds.get_contact_info(owner).is_some() {
+                // Silently insert this old value without bumping record
+                // timestamps
+                maybe_push(response, &mut expired_values);
+            } else {
+                stats.timeout_count += 1;
+                stats.failed_timeout += 1;
             }
-            maybe_push(r, &mut versioned);
         }
-        (versioned, versioned_expired_timestamp, failed_inserts)
+        (active_values, expired_values, failed_inserts)
     }
 
     /// process a vec of pull responses
@@ -381,34 +407,36 @@ impl CrdsGossipPull {
         &mut self,
         crds: &mut Crds,
         from: &Pubkey,
-        responses: Vec<VersionedCrdsValue>,
-        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        responses: Vec<CrdsValue>,
+        responses_expired_timeout: Vec<CrdsValue>,
         mut failed_inserts: Vec<Hash>,
         now: u64,
         stats: &mut ProcessPullStats,
     ) -> Vec<(CrdsValueLabel, Hash, u64)> {
         let mut success = vec![];
         let mut owners = HashSet::new();
-        for r in responses_expired_timeout {
-            let value_hash = r.value_hash;
-            if crds.insert_versioned(r).is_err() {
-                failed_inserts.push(value_hash);
+        for response in responses_expired_timeout {
+            match crds.insert(response, now) {
+                Ok(None) => (),
+                Ok(Some(old)) => self.purged_values.push_back((old.value_hash, now)),
+                Err(CrdsError::InsertFailed(value_hash)) => failed_inserts.push(value_hash),
+                Err(CrdsError::UnknownStakes) => (),
             }
         }
-        for r in responses {
-            let label = r.value.label();
-            let wc = r.value.wallclock();
-            let hash = r.value_hash;
-            match crds.insert_versioned(r) {
-                Err(_) => failed_inserts.push(hash),
+        for response in responses {
+            let label = response.label();
+            let wallclock = response.wallclock();
+            match crds.insert(response, now) {
+                Err(CrdsError::InsertFailed(value_hash)) => failed_inserts.push(value_hash),
+                Err(CrdsError::UnknownStakes) => (),
                 Ok(old) => {
                     stats.success += 1;
                     self.num_pulls += 1;
                     owners.insert(label.pubkey());
-                    success.push((label, hash, wc));
+                    let value_hash = crds.get(&label).unwrap().value_hash;
+                    success.push((label, value_hash, wallclock));
                     if let Some(val) = old {
-                        self.purged_values
-                            .push_back((val.value_hash, val.local_timestamp))
+                        self.purged_values.push_back((val.value_hash, now))
                     }
                 }
             }
@@ -563,7 +591,7 @@ impl CrdsGossipPull {
                 .into_iter()
                 .filter_map(|label| {
                     let val = crds.remove(&label)?;
-                    Some((val.value_hash, val.local_timestamp))
+                    Some((val.value_hash, now))
                 }),
         );
         self.purged_values.len() - num_purged_values
@@ -606,6 +634,20 @@ impl CrdsGossipPull {
             stats.success,
         )
     }
+
+    // Only for tests and simulations.
+    pub(crate) fn mock_clone(&self) -> Self {
+        let mut pull_request_time = LruCache::new(self.pull_request_time.cap());
+        for (k, v) in self.pull_request_time.iter().rev() {
+            pull_request_time.put(*k, *v);
+        }
+        Self {
+            pull_request_time,
+            purged_values: self.purged_values.clone(),
+            failed_inserts: self.failed_inserts.clone(),
+            ..*self
+        }
+    }
 }
 #[cfg(test)]
 mod test {
@@ -617,8 +659,12 @@ mod test {
     use rand::thread_rng;
     use rayon::ThreadPoolBuilder;
     use solana_perf::test_tx::test_tx;
-    use solana_sdk::hash::{hash, HASH_BYTES};
-    use solana_sdk::packet::PACKET_DATA_SIZE;
+    use solana_sdk::{
+        hash::{hash, HASH_BYTES},
+        packet::PACKET_DATA_SIZE,
+        timing::timestamp,
+    };
+    use std::{iter::repeat_with, time::Duration};
 
     #[test]
     fn test_hash_as_u64() {
@@ -908,137 +954,195 @@ mod test {
     fn test_new_pull_request() {
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
         let mut crds = Crds::default();
+        let node_keypair = Keypair::new();
         let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
+            &node_keypair.pubkey(),
             0,
         )));
-        let id = entry.label().pubkey();
         let node = CrdsGossipPull::default();
+        let mut pings = Vec::new();
+        let ping_cache = Mutex::new(PingCache::new(
+            Duration::from_secs(20 * 60), // ttl
+            128,                          // capacity
+        ));
         assert_eq!(
             node.new_pull_request(
                 &thread_pool,
                 &crds,
-                &id,
+                &node_keypair,
                 0,
                 0,
                 None,
                 &HashMap::new(),
-                PACKET_DATA_SIZE
+                PACKET_DATA_SIZE,
+                &ping_cache,
+                &mut pings,
             ),
             Err(CrdsGossipError::NoPeers)
         );
 
-        crds.insert(entry.clone(), 0).unwrap();
+        crds.insert(entry, 0).unwrap();
         assert_eq!(
             node.new_pull_request(
                 &thread_pool,
                 &crds,
-                &id,
+                &node_keypair,
                 0,
                 0,
                 None,
                 &HashMap::new(),
-                PACKET_DATA_SIZE
+                PACKET_DATA_SIZE,
+                &ping_cache,
+                &mut pings,
             ),
             Err(CrdsGossipError::NoPeers)
         );
-
-        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
-            0,
-        )));
+        let new = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 0);
+        ping_cache
+            .lock()
+            .unwrap()
+            .mock_pong(new.id, new.gossip, Instant::now());
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(new));
         crds.insert(new.clone(), 0).unwrap();
         let req = node.new_pull_request(
             &thread_pool,
             &crds,
-            &id,
+            &node_keypair,
             0,
             0,
             None,
             &HashMap::new(),
             PACKET_DATA_SIZE,
+            &ping_cache,
+            &mut pings,
         );
-        let (to, _, self_info) = req.unwrap();
-        assert_eq!(to, new.label().pubkey());
-        assert_eq!(self_info, entry);
+        let (peer, _) = req.unwrap();
+        assert_eq!(peer, *new.contact_info().unwrap());
     }
 
     #[test]
     fn test_new_mark_creation_time() {
         let now: u64 = 1_605_127_770_789;
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
+        let mut ping_cache = PingCache::new(
+            Duration::from_secs(20 * 60), // ttl
+            128,                          // capacity
+        );
         let mut crds = Crds::default();
+        let node_keypair = Keypair::new();
         let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
+            &node_keypair.pubkey(),
             0,
         )));
-        let node_pubkey = entry.label().pubkey();
         let mut node = CrdsGossipPull::default();
-        crds.insert(entry.clone(), now).unwrap();
-        let old = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
-            0,
-        )));
+        crds.insert(entry, now).unwrap();
+        let old = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 0);
+        ping_cache.mock_pong(old.id, old.gossip, Instant::now());
+        let old = CrdsValue::new_unsigned(CrdsData::ContactInfo(old));
         crds.insert(old.clone(), now).unwrap();
-        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
-            0,
-        )));
+        let new = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 0);
+        ping_cache.mock_pong(new.id, new.gossip, Instant::now());
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(new));
         crds.insert(new.clone(), now).unwrap();
 
         // set request creation time to now.
         let now = now + 50_000;
-        node.mark_pull_request_creation_time(&new.label().pubkey(), now);
+        node.mark_pull_request_creation_time(new.label().pubkey(), now);
 
         // odds of getting the other request should be close to 1.
         let now = now + 1_000;
+        let mut pings = Vec::new();
+        let ping_cache = Mutex::new(ping_cache);
         for _ in 0..10 {
             let req = node.new_pull_request(
                 &thread_pool,
                 &crds,
-                &node_pubkey,
+                &node_keypair,
                 0,
                 now,
                 None,
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
+                &ping_cache,
+                &mut pings,
             );
-            let (to, _, self_info) = req.unwrap();
-            assert_eq!(to, old.label().pubkey());
-            assert_eq!(self_info, entry);
+            let (peer, _) = req.unwrap();
+            assert_eq!(peer, *old.contact_info().unwrap());
         }
     }
 
+    #[test]
+    fn test_pull_request_time() {
+        const NUM_REPS: usize = 2 * CRDS_UNIQUE_PUBKEY_CAPACITY;
+        let mut rng = rand::thread_rng();
+        let pubkeys: Vec<_> = repeat_with(Pubkey::new_unique).take(NUM_REPS).collect();
+        let mut node = CrdsGossipPull::default();
+        let mut requests = HashMap::new();
+        let now = timestamp();
+        for k in 0..NUM_REPS {
+            let pubkey = pubkeys[rng.gen_range(0, pubkeys.len())];
+            let now = now + k as u64;
+            node.mark_pull_request_creation_time(pubkey, now);
+            *requests.entry(pubkey).or_default() = now;
+        }
+        assert!(node.pull_request_time.len() <= CRDS_UNIQUE_PUBKEY_CAPACITY);
+        // Assert that timestamps match most recent request.
+        for (pk, ts) in &node.pull_request_time {
+            assert_eq!(*ts, requests[pk]);
+        }
+        // Assert that most recent pull timestamps are maintained.
+        let max_ts = requests
+            .iter()
+            .filter(|(pk, _)| !node.pull_request_time.contains(*pk))
+            .map(|(_, ts)| *ts)
+            .max()
+            .unwrap();
+        let min_ts = requests
+            .iter()
+            .filter(|(pk, _)| node.pull_request_time.contains(*pk))
+            .map(|(_, ts)| *ts)
+            .min()
+            .unwrap();
+        assert!(max_ts <= min_ts);
+    }
+
     #[test]
     fn test_generate_pull_responses() {
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
+        let node_keypair = Keypair::new();
         let mut node_crds = Crds::default();
+        let mut ping_cache = PingCache::new(
+            Duration::from_secs(20 * 60), // ttl
+            128,                          // capacity
+        );
         let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
+            &node_keypair.pubkey(),
             0,
         )));
-        let node_pubkey = entry.label().pubkey();
+        let caller = entry.clone();
         let node = CrdsGossipPull::default();
         node_crds.insert(entry, 0).unwrap();
-        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
-            0,
-        )));
+        let new = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 0);
+        ping_cache.mock_pong(new.id, new.gossip, Instant::now());
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(new));
         node_crds.insert(new, 0).unwrap();
+        let mut pings = Vec::new();
         let req = node.new_pull_request(
             &thread_pool,
             &node_crds,
-            &node_pubkey,
+            &node_keypair,
             0,
             0,
             None,
             &HashMap::new(),
             PACKET_DATA_SIZE,
+            &Mutex::new(ping_cache),
+            &mut pings,
         );
 
         let mut dest_crds = Crds::default();
         let dest = CrdsGossipPull::default();
-        let (_, filters, caller) = req.unwrap();
+        let (_, filters) = req.unwrap();
         let mut filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
         let rsp = dest.generate_pull_responses(
             &dest_crds,
@@ -1088,33 +1192,40 @@ mod test {
     #[test]
     fn test_process_pull_request() {
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
+        let node_keypair = Keypair::new();
         let mut node_crds = Crds::default();
         let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
+            &node_keypair.pubkey(),
             0,
         )));
-        let node_pubkey = entry.label().pubkey();
+        let caller = entry.clone();
         let node = CrdsGossipPull::default();
         node_crds.insert(entry, 0).unwrap();
-        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
-            0,
-        )));
+        let mut ping_cache = PingCache::new(
+            Duration::from_secs(20 * 60), // ttl
+            128,                          // capacity
+        );
+        let new = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 0);
+        ping_cache.mock_pong(new.id, new.gossip, Instant::now());
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(new));
         node_crds.insert(new, 0).unwrap();
+        let mut pings = Vec::new();
         let req = node.new_pull_request(
             &thread_pool,
             &node_crds,
-            &node_pubkey,
+            &node_keypair,
             0,
             0,
             None,
             &HashMap::new(),
             PACKET_DATA_SIZE,
+            &Mutex::new(ping_cache),
+            &mut pings,
         );
 
         let mut dest_crds = Crds::default();
         let mut dest = CrdsGossipPull::default();
-        let (_, filters, caller) = req.unwrap();
+        let (_, filters) = req.unwrap();
         let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
         let rsp = dest.generate_pull_responses(
             &dest_crds,
@@ -1147,33 +1258,37 @@ mod test {
     #[test]
     fn test_process_pull_request_response() {
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
+        let node_keypair = Keypair::new();
         let mut node_crds = Crds::default();
         let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
+            &node_keypair.pubkey(),
             1,
         )));
+        let caller = entry.clone();
         let node_pubkey = entry.label().pubkey();
         let mut node = CrdsGossipPull::default();
         node_crds.insert(entry, 0).unwrap();
-
-        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &solana_sdk::pubkey::new_rand(),
-            1,
-        )));
+        let mut ping_cache = PingCache::new(
+            Duration::from_secs(20 * 60), // ttl
+            128,                          // capacity
+        );
+        let new = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 1);
+        ping_cache.mock_pong(new.id, new.gossip, Instant::now());
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(new));
         node_crds.insert(new, 0).unwrap();
 
         let mut dest = CrdsGossipPull::default();
         let mut dest_crds = Crds::default();
         let new_id = solana_sdk::pubkey::new_rand();
-        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &new_id, 1,
-        )));
+        let new = ContactInfo::new_localhost(&new_id, 1);
+        ping_cache.mock_pong(new.id, new.gossip, Instant::now());
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(new));
         dest_crds.insert(new.clone(), 0).unwrap();
 
         // node contains a key from the dest node, but at an older local timestamp
-        let same_key = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-            &new_id, 0,
-        )));
+        let same_key = ContactInfo::new_localhost(&new_id, 0);
+        ping_cache.mock_pong(same_key.id, same_key.gossip, Instant::now());
+        let same_key = CrdsValue::new_unsigned(CrdsData::ContactInfo(same_key));
         assert_eq!(same_key.label(), new.label());
         assert!(same_key.wallclock() < new.wallclock());
         node_crds.insert(same_key.clone(), 0).unwrap();
@@ -1185,19 +1300,23 @@ mod test {
             0
         );
         let mut done = false;
+        let mut pings = Vec::new();
+        let ping_cache = Mutex::new(ping_cache);
         for _ in 0..30 {
             // there is a chance of a false positive with bloom filters
             let req = node.new_pull_request(
                 &thread_pool,
                 &node_crds,
-                &node_pubkey,
+                &node_keypair,
                 0,
                 0,
                 None,
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
+                &ping_cache,
+                &mut pings,
             );
-            let (_, filters, caller) = req.unwrap();
+            let (_, filters) = req.unwrap();
             let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
             let mut rsp = dest.generate_pull_responses(
                 &dest_crds,
@@ -1290,7 +1409,7 @@ mod test {
         }
 
         // purge the value
-        node.purge_purged(1);
+        node.purge_purged(3);
         assert_eq!(node.purged_values.len(), 0);
     }
     #[test]
@@ -1451,7 +1570,7 @@ mod test {
                 &peer_pubkey,
                 &timeouts,
                 vec![peer_vote],
-                node.msg_timeout + 1,
+                node.msg_timeout + 2,
             )
             .0,
             1

core/src/crds_gossip_push.rs

@@ -9,6 +9,7 @@
 //! 2. The prune set is stored in a Bloom filter.
 
 use crate::{
+    cluster_info::CRDS_UNIQUE_PUBKEY_CAPACITY,
     contact_info::ContactInfo,
     crds::{Crds, VersionedCrdsValue},
     crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS},
@@ -19,6 +20,7 @@ use crate::{
 use bincode::serialized_size;
 use indexmap::map::IndexMap;
 use itertools::Itertools;
+use lru::LruCache;
 use rand::{seq::SliceRandom, Rng};
 use solana_runtime::bloom::{AtomicBloom, Bloom};
 use solana_sdk::{hash::Hash, packet::PACKET_DATA_SIZE, pubkey::Pubkey, timing::timestamp};
@@ -39,9 +41,6 @@ pub const CRDS_GOSSIP_PRUNE_MIN_INGRESS_NODES: usize = 3;
 // Do not push to peers which have not been updated for this long.
 const PUSH_ACTIVE_TIMEOUT_MS: u64 = 60_000;
 
-// 10 minutes
-const MAX_PUSHED_TO_TIMEOUT_MS: u64 = 10 * 60 * 1000;
-
 pub struct CrdsGossipPush {
     /// max bytes per message
     pub max_bytes: usize,
@@ -54,8 +53,7 @@ pub struct CrdsGossipPush {
     /// This cache represents a lagging view of which validators
     /// currently have this node in their `active_set`
     received_cache: HashMap<Pubkey, HashMap<Pubkey, (bool, u64)>>,
-    last_pushed_to: HashMap<Pubkey, u64>,
-    last_pushed_to_cleanup_ts: u64,
+    last_pushed_to: LruCache<Pubkey, u64>,
     pub num_active: usize,
     pub push_fanout: usize,
     pub msg_timeout: u64,
@@ -73,8 +71,7 @@ impl Default for CrdsGossipPush {
             active_set: IndexMap::new(),
             push_messages: HashMap::new(),
             received_cache: HashMap::new(),
-            last_pushed_to: HashMap::new(),
-            last_pushed_to_cleanup_ts: 0,
+            last_pushed_to: LruCache::new(CRDS_UNIQUE_PUBKEY_CAPACITY),
             num_active: CRDS_GOSSIP_NUM_ACTIVE,
             push_fanout: CRDS_GOSSIP_PUSH_FANOUT,
             msg_timeout: CRDS_GOSSIP_PUSH_MSG_TIMEOUT_MS,
@@ -129,7 +126,7 @@ impl CrdsGossipPush {
         let mut seed = [0; 32];
         rand::thread_rng().fill(&mut seed[..]);
         let shuffle = weighted_shuffle(
-            staked_peers.iter().map(|(_, stake)| *stake).collect_vec(),
+            &staked_peers.iter().map(|(_, stake)| *stake).collect_vec(),
             seed,
         );
 
@@ -175,29 +172,29 @@ impl CrdsGossipPush {
         now: u64,
     ) -> Result<Option<VersionedCrdsValue>, CrdsGossipError> {
         self.num_total += 1;
-        if now > value.wallclock().checked_add(self.msg_timeout).unwrap_or(0) {
-            return Err(CrdsGossipError::PushMessageTimeout);
-        }
-        if now + self.msg_timeout < value.wallclock() {
+        let range = now.saturating_sub(self.msg_timeout)..=now.saturating_add(self.msg_timeout);
+        if !range.contains(&value.wallclock()) {
             return Err(CrdsGossipError::PushMessageTimeout);
         }
         let label = value.label();
         let origin = label.pubkey();
-        let new_value = crds.new_versioned(now, value);
-        let value_hash = new_value.value_hash;
-        let received_set = self
-            .received_cache
+        self.received_cache
             .entry(origin)
-            .or_insert_with(HashMap::new);
-        received_set.entry(*from).or_insert((false, 0)).1 = now;
-
-        let old = crds.insert_versioned(new_value);
-        if old.is_err() {
-            self.num_old += 1;
-            return Err(CrdsGossipError::PushMessageOldVersion);
+            .or_default()
+            .entry(*from)
+            .and_modify(|(_pruned, timestamp)| *timestamp = now)
+            .or_insert((/*pruned:*/ false, now));
+        match crds.insert(value, now) {
+            Err(_) => {
+                self.num_old += 1;
+                Err(CrdsGossipError::PushMessageOldVersion)
+            }
+            Ok(old) => {
+                let value_hash = crds.get(&label).unwrap().value_hash;
+                self.push_messages.insert(label, value_hash);
+                Ok(old)
+            }
         }
-        self.push_messages.insert(label, value_hash);
-        Ok(old.unwrap())
     }
 
     /// push pull responses
@@ -269,13 +266,8 @@ impl CrdsGossipPush {
         for label in labels {
             self.push_messages.remove(&label);
         }
-        for target_pubkey in push_messages.keys() {
-            *self.last_pushed_to.entry(*target_pubkey).or_insert(0) = now;
-        }
-        if now - self.last_pushed_to_cleanup_ts > MAX_PUSHED_TO_TIMEOUT_MS {
-            self.last_pushed_to
-                .retain(|_id, timestamp| now - *timestamp > MAX_PUSHED_TO_TIMEOUT_MS);
-            self.last_pushed_to_cleanup_ts = now;
+        for target_pubkey in push_messages.keys().copied() {
+            self.last_pushed_to.put(target_pubkey, now);
         }
         push_messages
     }
@@ -326,7 +318,7 @@ impl CrdsGossipPush {
         let mut seed = [0; 32];
         rng.fill(&mut seed[..]);
         let mut shuffle = weighted_shuffle(
-            options.iter().map(|weighted| weighted.0).collect_vec(),
+            &options.iter().map(|weighted| weighted.0).collect_vec(),
             seed,
         )
         .into_iter();
@@ -395,8 +387,12 @@ impl CrdsGossipPush {
                     })
             })
             .map(|info| {
-                let last_pushed_to: u64 = *self.last_pushed_to.get(&info.id).unwrap_or(&0);
-                let since = (now.saturating_sub(last_pushed_to) / 1024) as u32;
+                let last_pushed_to = self
+                    .last_pushed_to
+                    .peek(&info.id)
+                    .copied()
+                    .unwrap_or_default();
+                let since = (now.saturating_sub(last_pushed_to).min(3600 * 1000) / 1024) as u32;
                 let stake = get_stake(&info.id, stakes);
                 let weight = get_weight(max_weight, since, stake);
                 (weight, info)
@@ -423,15 +419,20 @@ impl CrdsGossipPush {
 
     // Only for tests and simulations.
     pub(crate) fn mock_clone(&self) -> Self {
-        let mut active_set = IndexMap::<Pubkey, AtomicBloom<Pubkey>>::new();
-        for (k, v) in &self.active_set {
-            active_set.insert(*k, v.mock_clone());
+        let active_set = self
+            .active_set
+            .iter()
+            .map(|(k, v)| (*k, v.mock_clone()))
+            .collect();
+        let mut last_pushed_to = LruCache::new(self.last_pushed_to.cap());
+        for (k, v) in self.last_pushed_to.iter().rev() {
+            last_pushed_to.put(*k, *v);
         }
         Self {
             active_set,
             push_messages: self.push_messages.clone(),
             received_cache: self.received_cache.clone(),
-            last_pushed_to: self.last_pushed_to.clone(),
+            last_pushed_to,
             ..*self
         }
     }
@@ -641,7 +642,7 @@ mod test {
             let id = peer.label().pubkey();
             crds.insert(peer.clone(), time).unwrap();
             stakes.insert(id, i * 100);
-            push.last_pushed_to.insert(id, time);
+            push.last_pushed_to.put(id, time);
         }
         let mut options = push.push_options(&crds, &Pubkey::default(), 0, &stakes, None);
         assert!(!options.is_empty());