Sigverify refactor (#18873)

(cherry picked from commit d6f5945653)

Co-authored-by: Jon Cinque <jon.cinque@gmail.com>

.buildkite/env/secrets.ejson

@@ -2,6 +2,6 @@
     "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
     "_comment": "These credentials are encrypted and pose no risk",
     "environment": {
-      "CODECOV_TOKEN": "EJ[1:Z7OneT3RdJJ0DipCHQ7rC84snQ+FPbgHwZADQiz54wk=:3K68mE38LJ2RB98VWmjuNLFBNn1XTGR4:cR4r05/TOZQKmEZp1v4CSgUJtC6QJiOaL85QjXW0qZ061fMnsBA8AtAPMDoDq4WCGOZM1A==]"
+      "CODECOV_TOKEN": "EJ[1:KToenD1Sr3w82lHGxz1n+j3hwNlLk/1pYrjZHlvY6kE=:hN1Q25omtJ+4yYVn+qzIsPLKT3O6J9XN:DMLNLXi/pkWgvwF6gNIcNF222sgsRR9LnwLZYj0P0wGj7q6w8YQnd1Rskj+sRroI/z5pQg==]"
     }
 }

.buildkite/hooks/pre-command

@@ -36,4 +36,7 @@ export CARGO_TARGET_CACHE=$HOME/cargo-target-cache/"$CHANNEL"-"$BUILDKITE_LABEL"
   # `std:
   #    "found possibly newer version of crate `std` which `xyz` depends on
   rm -rf target/bpfel-unknown-unknown
+  if [[ $BUILDKITE_LABEL = "stable-perf" ]]; then
+    rm -rf target/release
+  fi
 )

.travis.yml

@@ -38,6 +38,7 @@ jobs:
         - readlink -f .
       script:
         - source ci/env.sh
+        - rustup set profile default
         - ci/publish-tarball.sh
       deploy:
         - provider: s3

Cargo.toml

@@ -61,7 +61,6 @@ members = [
     "sdk/cargo-test-bpf",
     "scripts",
     "stake-accounts",
-    "stake-monitor",
     "sys-tuner",
     "tokens",
     "transaction-status",
@@ -77,3 +76,6 @@ members = [
 exclude = [
     "programs/bpf",
 ]
+
+[profile.dev]
+split-debuginfo = "unpacked"

README.md

@@ -19,7 +19,7 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt
 ```
 
-Please sure you are always using the latest stable rust version by running:
+Please make sure you are always using the latest stable rust version by running:
 
 ```bash
 $ rustup update

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.6.5"
+version = "1.6.20"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,11 +19,11 @@ lazy_static = "1.4.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-stake-program = { path = "../programs/stake", version = "=1.6.5" }
-solana-vote-program = { path = "../programs/vote", version = "=1.6.5" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
+solana-config-program = { path = "../programs/config", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.20" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.20" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"
 

account-decoder/src/lib.rs

@@ -48,7 +48,7 @@ pub enum UiAccountData {
     Binary(String, UiAccountEncoding),
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub enum UiAccountEncoding {
     Binary, // Legacy. Retained for RPC backwards compatibility
@@ -62,7 +62,7 @@ pub enum UiAccountEncoding {
 impl UiAccount {
     pub fn encode<T: ReadableAccount>(
         pubkey: &Pubkey,
-        account: T,
+        account: &T,
         encoding: UiAccountEncoding,
         additional_data: Option<AccountAdditionalData>,
         data_slice_config: Option<UiDataSliceConfig>,
@@ -224,7 +224,7 @@ mod test {
     fn test_base64_zstd() {
         let encoded_account = UiAccount::encode(
             &Pubkey::default(),
-            AccountSharedData::from(Account {
+            &AccountSharedData::from(Account {
                 data: vec![0; 1024],
                 ..Account::default()
             }),

account-decoder/src/parse_nonce.rs

@@ -9,7 +9,13 @@ pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
         .map_err(|_| ParseAccountError::from(InstructionError::InvalidAccountData))?;
     let nonce_state = nonce_state.convert_to_current();
     match nonce_state {
-        State::Uninitialized => Ok(UiNonceState::Uninitialized),
+        // This prevents parsing an allocated System-owned account with empty data of any non-zero
+        // length as `uninitialized` nonce. An empty account of the wrong length can never be
+        // initialized as a nonce account, and an empty account of the correct length may not be an
+        // uninitialized nonce account, since it can be assigned to another program.
+        State::Uninitialized => Err(ParseAccountError::from(
+            InstructionError::InvalidAccountData,
+        )),
         State::Initialized(data) => Ok(UiNonceState::Initialized(UiNonceData {
             authority: data.authority.to_string(),
             blockhash: data.blockhash.to_string(),

account-decoder/src/parse_token.rs

@@ -14,23 +14,23 @@ use std::str::FromStr;
 // A helper function to convert spl_token_v2_0::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
 pub fn spl_token_id_v2_0() -> Pubkey {
-    Pubkey::from_str(&spl_token_v2_0::id().to_string()).unwrap()
+    Pubkey::new_from_array(spl_token_v2_0::id().to_bytes())
 }
 
 // A helper function to convert spl_token_v2_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
 pub fn spl_token_v2_0_native_mint() -> Pubkey {
-    Pubkey::from_str(&spl_token_v2_0::native_mint::id().to_string()).unwrap()
+    Pubkey::new_from_array(spl_token_v2_0::native_mint::id().to_bytes())
 }
 
 // A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
 pub fn spl_token_v2_0_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
-    SplTokenPubkey::from_str(&pubkey.to_string()).unwrap()
+    SplTokenPubkey::new_from_array(pubkey.to_bytes())
 }
 
 // A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
 pub fn pubkey_from_spl_token_v2_0(pubkey: &SplTokenPubkey) -> Pubkey {
-    Pubkey::from_str(&pubkey.to_string()).unwrap()
+    Pubkey::new_from_array(pubkey.to_bytes())
 }
 
 pub fn parse_token(

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-measure = { path = "../measure", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-measure = { path = "../measure", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -6,10 +6,10 @@ use rayon::prelude::*;
 use solana_measure::measure::Measure;
 use solana_runtime::{
     accounts::{create_test_accounts, update_accounts_bench, Accounts},
-    accounts_index::Ancestors,
+    accounts_index::{AccountSecondaryIndexes, Ancestors},
 };
 use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
-use std::{collections::HashSet, env, fs, path::PathBuf};
+use std::{env, fs, path::PathBuf};
 
 fn main() {
     solana_logger::setup();
@@ -58,8 +58,12 @@ fn main() {
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
-    let accounts =
-        Accounts::new_with_config(vec![path], &ClusterType::Testnet, HashSet::new(), false);
+    let accounts = Accounts::new_with_config(
+        vec![path],
+        &ClusterType::Testnet,
+        AccountSecondaryIndexes::default(),
+        false,
+    );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");
     let pubkeys: Vec<_> = (0..num_slots)

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-cluster-bench"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,22 +13,22 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.6.5" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-client = { path = "../client", version = "=1.6.5" }
-solana-core = { path = "../core", version = "=1.6.5" }
-solana-measure = { path = "../measure", version = "=1.6.5" }
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-net-utils = { path = "../net-utils", version = "=1.6.5" }
-solana-faucet = { path = "../faucet", version = "=1.6.5" }
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.20" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-client = { path = "../client", version = "=1.6.20" }
+solana-core = { path = "../core", version = "=1.6.20" }
+solana-measure = { path = "../measure", version = "=1.6.20" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.20" }
+solana-faucet = { path = "../faucet", version = "=1.6.20" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.6.5" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.20" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/src/main.rs

@@ -1,5 +1,5 @@
 #![allow(clippy::integer_arithmetic)]
-use clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg};
+use clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg};
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
@@ -359,7 +359,7 @@ fn make_close_message(
 fn run_accounts_bench(
     entrypoint_addr: SocketAddr,
     faucet_addr: SocketAddr,
-    keypair: &Keypair,
+    payer_keypairs: &[&Keypair],
     iterations: usize,
     maybe_space: Option<u64>,
     batch_size: usize,
@@ -372,7 +372,7 @@ fn run_accounts_bench(
     let client =
         RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed());
 
-    info!("Targetting {}", entrypoint_addr);
+    info!("Targeting {}", entrypoint_addr);
 
     let mut last_blockhash = Instant::now();
     let mut last_log = Instant::now();
@@ -381,7 +381,10 @@ fn run_accounts_bench(
     let mut tx_sent_count = 0;
     let mut total_accounts_created = 0;
     let mut total_accounts_closed = 0;
-    let mut balance = client.get_balance(&keypair.pubkey()).unwrap_or(0);
+    let mut balances: Vec<_> = payer_keypairs
+        .iter()
+        .map(|keypair| client.get_balance(&keypair.pubkey()).unwrap_or(0))
+        .collect();
     let mut last_balance = Instant::now();
 
     let default_max_lamports = 1000;
@@ -398,7 +401,7 @@ fn run_accounts_bench(
         max_closed: Arc::new(AtomicU64::default()),
     };
 
-    info!("Starting balance: {}", balance);
+    info!("Starting balance(s): {:?}", balances);
 
     let executor = TransactionExecutor::new(entrypoint_addr);
 
@@ -414,19 +417,26 @@ fn run_accounts_bench(
             .saturating_mul(NUM_SIGNATURES);
         let lamports = min_balance + fee;
 
-        if balance < lamports || last_balance.elapsed().as_millis() > 2000 {
-            if let Ok(b) = client.get_balance(&keypair.pubkey()) {
-                balance = b;
-            }
-            last_balance = Instant::now();
-            if balance < lamports {
-                info!(
-                    "Balance {} is less than needed: {}, doing aidrop...",
-                    balance, lamports
-                );
-                if !airdrop_lamports(&client, &faucet_addr, keypair, lamports * 100_000) {
-                    warn!("failed airdrop, exiting");
-                    return;
+        for (i, balance) in balances.iter_mut().enumerate() {
+            if *balance < lamports || last_balance.elapsed().as_millis() > 2000 {
+                if let Ok(b) = client.get_balance(&payer_keypairs[i].pubkey()) {
+                    *balance = b;
+                }
+                last_balance = Instant::now();
+                if *balance < lamports * 2 {
+                    info!(
+                        "Balance {} is less than needed: {}, doing aidrop...",
+                        balance, lamports
+                    );
+                    if !airdrop_lamports(
+                        &client,
+                        &faucet_addr,
+                        &payer_keypairs[i],
+                        lamports * 100_000,
+                    ) {
+                        warn!("failed airdrop, exiting");
+                        return;
+                    }
                 }
             }
         }
@@ -434,29 +444,36 @@ fn run_accounts_bench(
         let sigs_len = executor.num_outstanding();
         if sigs_len < batch_size {
             let num_to_create = batch_size - sigs_len;
-            info!("creating {} new", num_to_create);
-            let txs: Vec<_> = (0..num_to_create)
-                .into_par_iter()
-                .map(|_| {
-                    let message = make_create_message(
-                        keypair,
-                        &base_keypair,
-                        seed_tracker.max_created.clone(),
-                        num_instructions,
-                        min_balance,
-                        maybe_space,
-                        mint,
-                    );
-                    let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
-                    Transaction::new(&signers, message, recent_blockhash.0)
-                })
-                .collect();
-            balance = balance.saturating_sub(lamports * txs.len() as u64);
-            info!("txs: {}", txs.len());
-            let new_ids = executor.push_transactions(txs);
-            info!("ids: {}", new_ids.len());
-            tx_sent_count += new_ids.len();
-            total_accounts_created += num_instructions * new_ids.len();
+            if num_to_create >= payer_keypairs.len() {
+                info!("creating {} new", num_to_create);
+                let chunk_size = num_to_create / payer_keypairs.len();
+                if chunk_size > 0 {
+                    for (i, keypair) in payer_keypairs.iter().enumerate() {
+                        let txs: Vec<_> = (0..chunk_size)
+                            .into_par_iter()
+                            .map(|_| {
+                                let message = make_create_message(
+                                    keypair,
+                                    &base_keypair,
+                                    seed_tracker.max_created.clone(),
+                                    num_instructions,
+                                    min_balance,
+                                    maybe_space,
+                                    mint,
+                                );
+                                let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                                Transaction::new(&signers, message, recent_blockhash.0)
+                            })
+                            .collect();
+                        balances[i] = balances[i].saturating_sub(lamports * txs.len() as u64);
+                        info!("txs: {}", txs.len());
+                        let new_ids = executor.push_transactions(txs);
+                        info!("ids: {}", new_ids.len());
+                        tx_sent_count += new_ids.len();
+                        total_accounts_created += num_instructions * new_ids.len();
+                    }
+                }
+            }
 
             if close_nth > 0 {
                 let expected_closed = total_accounts_created as u64 / close_nth;
@@ -465,18 +482,18 @@ fn run_accounts_bench(
                         .into_par_iter()
                         .map(|_| {
                             let message = make_close_message(
-                                keypair,
+                                &payer_keypairs[0],
                                 &base_keypair,
                                 seed_tracker.max_closed.clone(),
                                 1,
                                 min_balance,
                                 mint.is_some(),
                             );
-                            let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                            let signers: Vec<&Keypair> = vec![&payer_keypairs[0], &base_keypair];
                             Transaction::new(&signers, message, recent_blockhash.0)
                         })
                         .collect();
-                    balance = balance.saturating_sub(fee * txs.len() as u64);
+                    balances[0] = balances[0].saturating_sub(fee * txs.len() as u64);
                     info!("close txs: {}", txs.len());
                     let new_ids = executor.push_transactions(txs);
                     info!("close ids: {}", new_ids.len());
@@ -491,8 +508,8 @@ fn run_accounts_bench(
         count += 1;
         if last_log.elapsed().as_millis() > 3000 {
             info!(
-                "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance: {}",
-                total_accounts_created, total_accounts_closed, tx_sent_count, count, balance
+                "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
+                total_accounts_created, total_accounts_closed, tx_sent_count, count, balances
             );
             last_log = Instant::now();
         }
@@ -543,6 +560,7 @@ fn main() {
             Arg::with_name("identity")
                 .long("identity")
                 .takes_value(true)
+                .multiple(true)
                 .value_name("FILE")
                 .help("keypair file"),
         )
@@ -624,20 +642,29 @@ fn main() {
 
     let mint = pubkey_of(&matches, "mint");
 
-    let keypair =
-        read_keypair_file(&value_t_or_exit!(matches, "identity", String)).expect("bad keypair");
+    let payer_keypairs: Vec<_> = values_t_or_exit!(matches, "identity", String)
+        .iter()
+        .map(|keypair_string| {
+            read_keypair_file(keypair_string)
+                .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_string))
+        })
+        .collect();
+    let mut payer_keypair_refs: Vec<&Keypair> = vec![];
+    for keypair in payer_keypairs.iter() {
+        payer_keypair_refs.push(keypair);
+    }
 
     let rpc_addr = if !skip_gossip {
         info!("Finding cluster entry: {:?}", entrypoint_addr);
         let (gossip_nodes, _validators) = discover(
-            None,
+            None, // keypair
             Some(&entrypoint_addr),
-            None,
-            Some(60),
-            None,
-            Some(&entrypoint_addr),
-            None,
-            0,
+            None,                    // num_nodes
+            Duration::from_secs(60), // timeout
+            None,                    // find_node_by_pubkey
+            Some(&entrypoint_addr),  // find_node_by_gossip_addr
+            None,                    // my_gossip_addr
+            0,                       // my_shred_version
         )
         .unwrap_or_else(|err| {
             eprintln!("Failed to discover {} node: {:?}", entrypoint_addr, err);
@@ -654,7 +681,7 @@ fn main() {
     run_accounts_bench(
         rpc_addr,
         faucet_addr,
-        &keypair,
+        &payer_keypair_refs,
         iterations,
         space,
         batch_size,
@@ -700,7 +727,7 @@ pub mod test {
         run_accounts_bench(
             cluster.entry_point_info.rpc,
             faucet_addr,
-            &cluster.funding_keypair,
+            &[&cluster.funding_keypair],
             iterations,
             maybe_space,
             batch_size,

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-core = { path = "../core", version = "=1.6.5" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-streamer = { path = "../streamer", version = "=1.6.5" }
-solana-perf = { path = "../perf", version = "=1.6.5" }
-solana-ledger = { path = "../ledger", version = "=1.6.5" }
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-measure = { path = "../measure", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
+solana-core = { path = "../core", version = "=1.6.20" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-streamer = { path = "../streamer", version = "=1.6.20" }
+solana-perf = { path = "../perf", version = "=1.6.20" }
+solana-ledger = { path = "../ledger", version = "=1.6.20" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-measure = { path = "../measure", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.6.5"
+version = "1.6.20"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,20 +11,20 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.3.1"
-borsh = "0.8.1"
-borsh-derive = "0.8.1"
+borsh = "0.9.0"
+borsh-derive = "0.9.0"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.6.5" }
-solana-program = { path = "../sdk/program", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.20" }
+solana-program = { path = "../sdk/program", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-banks-server = { path = "../banks-server", version = "=1.6.5" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-banks-server = { path = "../banks-server", version = "=1.6.20" }
 
 [lib]
 crate-type = ["lib"]

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.6.5"
+version = "1.6.20"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ edition = "2018"
 [dependencies]
 mio = "0.7.6"
 serde = { version = "1.0.122", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
 tarpc = { version = "0.24.1", features = ["full"] }
 
 [dev-dependencies]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.6.5"
+version = "1.6.20"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,10 +14,10 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.6.5" }
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-metrics = { path = "../metrics", version = "=1.6.5" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.20" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-metrics = { path = "../metrics", version = "=1.6.20" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -131,10 +131,13 @@ impl BanksServer {
     }
 }
 
-fn verify_transaction(transaction: &Transaction) -> transaction::Result<()> {
+fn verify_transaction(
+    transaction: &Transaction,
+    libsecp256k1_0_5_upgrade_enabled: bool,
+) -> transaction::Result<()> {
     if let Err(err) = transaction.verify() {
         Err(err)
-    } else if let Err(err) = transaction.verify_precompiles() {
+    } else if let Err(err) = transaction.verify_precompiles(libsecp256k1_0_5_upgrade_enabled) {
         Err(err)
     } else {
         Ok(())
@@ -215,7 +218,10 @@ impl Banks for BanksServer {
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        if let Err(err) = verify_transaction(&transaction) {
+        if let Err(err) = verify_transaction(
+            &transaction,
+            self.bank(commitment).libsecp256k1_0_5_upgrade_enabled(),
+        ) {
             return Some(Err(err));
         }
 

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-core = { path = "../core", version = "=1.6.5" }
-solana-genesis = { path = "../genesis", version = "=1.6.5" }
-solana-client = { path = "../client", version = "=1.6.5" }
-solana-faucet = { path = "../faucet", version = "=1.6.5" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.6.5" }
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-metrics = { path = "../metrics", version = "=1.6.5" }
-solana-net-utils = { path = "../net-utils", version = "=1.6.5" }
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-core = { path = "../core", version = "=1.6.20" }
+solana-genesis = { path = "../genesis", version = "=1.6.20" }
+solana-client = { path = "../client", version = "=1.6.20" }
+solana-faucet = { path = "../faucet", version = "=1.6.20" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.6.20" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-metrics = { path = "../metrics", version = "=1.6.20" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.20" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.6.5" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.20" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-streamer = { path = "../streamer", version = "=1.6.5" }
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-net-utils = { path = "../net-utils", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-streamer = { path = "../streamer", version = "=1.6.20" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,22 +15,22 @@ log = "0.4.11"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-core = { path = "../core", version = "=1.6.5" }
-solana-genesis = { path = "../genesis", version = "=1.6.5" }
-solana-client = { path = "../client", version = "=1.6.5" }
-solana-faucet = { path = "../faucet", version = "=1.6.5" }
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-metrics = { path = "../metrics", version = "=1.6.5" }
-solana-measure = { path = "../measure", version = "=1.6.5" }
-solana-net-utils = { path = "../net-utils", version = "=1.6.5" }
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-core = { path = "../core", version = "=1.6.20" }
+solana-genesis = { path = "../genesis", version = "=1.6.20" }
+solana-client = { path = "../client", version = "=1.6.20" }
+solana-faucet = { path = "../faucet", version = "=1.6.20" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-metrics = { path = "../metrics", version = "=1.6.20" }
+solana-measure = { path = "../measure", version = "=1.6.20" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.20" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "=1.6.5" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.20" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

ci/buildkite-pipeline.sh

@@ -148,6 +148,33 @@ all_test_steps() {
   command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
   wait_step
 
+  # BPF test suite
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-bpf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "ci/test-stable-bpf.sh"
+    name: "stable-bpf"
+    timeout_in_minutes: 20
+    artifact_paths: "bpf-dumps.tar.bz2"
+    agents:
+      - "queue=default"
+EOF
+  else
+    annotate --style info \
+      "Stable-BPF skipped as no relevant files were modified"
+  fi
+
   # Perf test suite
   if affects \
              .rs$ \
@@ -165,7 +192,7 @@ all_test_steps() {
     cat >> "$output_file" <<"EOF"
   - command: "ci/test-stable-perf.sh"
     name: "stable-perf"
-    timeout_in_minutes: 40
+    timeout_in_minutes: 20
     artifact_paths: "log-*.txt"
     agents:
       - "queue=cuda"

ci/buildkite-secondary.yml

@@ -3,16 +3,24 @@
 # Pull requests to not run these steps.
 steps:
   - command: "ci/publish-tarball.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 60
     name: "publish tarball"
   - command: "ci/publish-bpf-sdk.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 5
     name: "publish bpf sdk"
   - wait
   - command: "sdk/docker-solana/build.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 60
     name: "publish docker"
   - command: "ci/publish-crate.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.50.0
+FROM solanalabs/rust:1.51.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.50.0
+FROM rust:1.51.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/env.sh

@@ -74,10 +74,13 @@ else
   export CI_BUILD_ID=
   export CI_COMMIT=
   export CI_JOB_ID=
-  export CI_OS_NAME=
   export CI_PULL_REQUEST=
   export CI_REPO_SLUG=
   export CI_TAG=
+  # Don't override ci/run-local.sh
+  if [[ -z $CI_LOCAL_RUN ]]; then
+    export CI_OS_NAME=
+  fi
 fi
 
 cat <<EOF

ci/localnet-sanity.sh

@@ -70,7 +70,7 @@ done
 
 source ci/upload-ci-artifact.sh
 source scripts/configure-metrics.sh
-source multinode-demo/common.sh
+source multinode-demo/common.sh --prebuild
 
 nodes=(
   "multinode-demo/bootstrap-validator.sh \
@@ -127,7 +127,7 @@ startNode() {
 waitForNodeToInit() {
   declare initCompleteFile=$1
   while [[ ! -r $initCompleteFile ]]; do
-    if [[ $SECONDS -ge 240 ]]; then
+    if [[ $SECONDS -ge 300 ]]; then
       echo "^^^ +++"
       echo "Error: $initCompleteFile not found in $SECONDS seconds"
       exit 1

ci/order-crates-for-publishing.py

@@ -12,10 +12,14 @@ import json
 import subprocess
 import sys;
 
+real_file = os.path.realpath(__file__)
+ci_path = os.path.dirname(real_file)
+src_root = os.path.dirname(ci_path)
+
 def load_metadata():
+    cmd = f'{src_root}/cargo metadata --no-deps --format-version=1'
     return json.loads(subprocess.Popen(
-        'cargo metadata --no-deps --format-version=1',
-        shell=True, stdout=subprocess.PIPE).communicate()[0])
+        cmd, shell=True, stdout=subprocess.PIPE).communicate()[0])
 
 def get_packages():
     metadata = load_metadata()

ci/run-local.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")/.."
+
+export CI_LOCAL_RUN=true
+
+set -e
+
+case $(uname -o) in
+  */Linux)
+    export CI_OS_NAME=linux
+    ;;
+  *)
+    echo "local CI runs are only supported on Linux" 1>&2
+    exit 1
+    ;;
+esac
+
+steps=()
+steps+=(test-sanity)
+steps+=(shellcheck)
+steps+=(test-checks)
+steps+=(test-coverage)
+steps+=(test-stable)
+steps+=(test-stable-bpf)
+steps+=(test-stable-perf)
+steps+=(test-downstream-builds)
+steps+=(test-bench)
+steps+=(test-local-cluster)
+
+step_index=0
+if [[ -n "$1" ]]; then
+  start_step="$1"
+  while [[ $step_index -lt ${#steps[@]} ]]; do
+    step="${steps[$step_index]}"
+    if [[ "$step" = "$start_step" ]]; then
+      break
+    fi
+    step_index=$((step_index + 1))
+  done
+  if [[ $step_index -eq ${#steps[@]} ]]; then
+    echo "unexpected start step: \"$start_step\"" 1>&2
+    exit 1
+  else
+    echo "** starting at step: \"$start_step\" **"
+    echo
+  fi
+fi
+
+while [[ $step_index -lt ${#steps[@]} ]]; do
+  step="${steps[$step_index]}"
+  cmd="ci/${step}.sh"
+  $cmd
+  step_index=$((step_index + 1))
+done

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.50.0
+  stable_version=1.51.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2021-02-18
+  nightly_version=2021-04-18
 fi
 
 

ci/test-checks.sh

@@ -14,7 +14,7 @@ scripts/increment-cargo-version.sh check
 
 # Disallow uncommitted Cargo.lock changes
 (
-  _ scripts/cargo-for-all-lock-files.sh tree
+  _ scripts/cargo-for-all-lock-files.sh tree >/dev/null
   set +e
   if ! _ git diff --exit-code; then
     echo -e "\nError: Uncommitted Cargo.lock changes" 1>&2
@@ -35,8 +35,10 @@ echo --- build environment
   "$cargo" stable clippy --version --verbose
   "$cargo" nightly clippy --version --verbose
 
-  # audit is done only with stable
+  # audit is done only with "$cargo stable"
   "$cargo" stable audit --version
+
+  grcov --version
 )
 
 export RUST_BACKTRACE=1
@@ -65,7 +67,8 @@ _ ci/order-crates-for-publishing.py
 
 # -Z... is needed because of clippy bug: https://github.com/rust-lang/rust-clippy/issues/4612
 # run nightly clippy for `sdk/` as there's a moderate amount of nightly-only code there
-_ "$cargo" nightly clippy -Zunstable-options --workspace --all-targets -- --deny=warnings --deny=clippy::integer_arithmetic
+_ "$cargo" nightly clippy -Zunstable-options --workspace --all-targets -- \
+  --deny=warnings --deny=clippy::integer_arithmetic --allow=clippy::inconsistent_struct_constructor
 
 _ "$cargo" stable fmt --all -- --check
 

ci/test-downstream-builds.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")/.."
+
+export CI_LOCAL_RUN=true
+
+set -ex
+
+scripts/build-downstream-projects.sh

ci/test-stable-bpf.sh

@@ -0,0 +1 @@
+test-stable.sh

ci/test-stable.sh

@@ -21,10 +21,6 @@ export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 source scripts/ulimit-n.sh
 
-# Clear the C dependency files, if dependency moves these files are not regenerated
-test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
-test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
-
 # Limit compiler jobs to reduce memory usage
 # on machines with 2gb/thread of memory
 NPROC=$(nproc)
@@ -35,17 +31,25 @@ case $testName in
 test-stable)
   _ "$cargo" stable test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
   ;;
-test-stable-perf)
+test-stable-bpf)
+  # Clear the C dependency files, if dependency moves these files are not regenerated
+  test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
+  test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
+
+  # rustfilt required for dumping BPF assembly listings
+  "$cargo" install rustfilt
+
   # solana-keygen required when building C programs
   _ "$cargo" build --manifest-path=keygen/Cargo.toml
   export PATH="$PWD/target/debug":$PATH
+  cargo_build_bpf="$(realpath ./cargo-build-bpf)"
 
   # BPF solana-sdk legacy compile test
-  ./cargo-build-bpf --manifest-path sdk/Cargo.toml
+  "$cargo_build_bpf" --manifest-path sdk/Cargo.toml
 
   # BPF Program unit tests
   "$cargo" stable test --manifest-path programs/bpf/Cargo.toml
-  cargo-build-bpf --manifest-path programs/bpf/Cargo.toml --bpf-sdk sdk/bpf
+  "$cargo_build_bpf" --manifest-path programs/bpf/Cargo.toml --bpf-sdk sdk/bpf
 
   # BPF program system tests
   _ make -C programs/bpf/c tests
@@ -53,11 +57,31 @@ test-stable-perf)
     --manifest-path programs/bpf/Cargo.toml \
     --no-default-features --features=bpf_c,bpf_rust -- --nocapture
 
+  # Dump BPF program assembly listings
+  for bpf_test in programs/bpf/rust/*; do
+    if pushd "$bpf_test"; then
+      "$cargo_build_bpf" --dump
+      popd
+    fi
+  done
+
+  # BPF program instruction count assertion
+  bpf_target_path=programs/bpf/target
+  _ "$cargo" stable test \
+    --manifest-path programs/bpf/Cargo.toml \
+    --no-default-features --features=bpf_c,bpf_rust assert_instruction_count \
+    -- --nocapture &> "${bpf_target_path}"/deploy/instuction_counts.txt
+
+  bpf_dump_archive="bpf-dumps.tar.bz2"
+  rm -f "$bpf_dump_archive"
+  tar cjvf "$bpf_dump_archive" "${bpf_target_path}"/{deploy/*.txt,bpfel-unknown-unknown/release/*.so}
+  ;;
+test-stable-perf)
   if [[ $(uname) = Linux ]]; then
     # Enable persistence mode to keep the CUDA kernel driver loaded, avoiding a
     # lengthy and unexpected delay the first time CUDA is involved when the driver
     # is not yet loaded.
-    sudo --non-interactive ./net/scripts/enable-nvidia-persistence-mode.sh
+    sudo --non-interactive ./net/scripts/enable-nvidia-persistence-mode.sh || true
 
     rm -rf target/perf-libs
     ./fetch-perf-libs.sh

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.6.5"
+version = "1.6.20"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,13 +12,17 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.8.0"
+uriparse = "0.6.3"
 url = "2.1.0"
 chrono = "0.4"
 
+[dev-dependencies]
+tempfile = "3.1.0"
+
 [lib]
 name = "solana_clap_utils"
 

clap-utils/src/fee_payer.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators, ArgConstant};
-use clap::Arg;
+use {
+    crate::{input_validators, ArgConstant},
+    clap::Arg,
+};
 
 pub const FEE_PAYER_ARG: ArgConstant<'static> = ArgConstant {
     name: "fee_payer",

clap-utils/src/input_parsers.rs

@@ -1,19 +1,21 @@
-use crate::keypair::{
-    keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
-    ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
+use {
+    crate::keypair::{
+        keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
+        ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
+    },
+    chrono::DateTime,
+    clap::ArgMatches,
+    solana_remote_wallet::remote_wallet::RemoteWalletManager,
+    solana_sdk::{
+        clock::UnixTimestamp,
+        commitment_config::CommitmentConfig,
+        genesis_config::ClusterType,
+        native_token::sol_to_lamports,
+        pubkey::Pubkey,
+        signature::{read_keypair_file, Keypair, Signature, Signer},
+    },
+    std::{str::FromStr, sync::Arc},
 };
-use chrono::DateTime;
-use clap::ArgMatches;
-use solana_remote_wallet::remote_wallet::RemoteWalletManager;
-use solana_sdk::{
-    clock::UnixTimestamp,
-    commitment_config::CommitmentConfig,
-    genesis_config::ClusterType,
-    native_token::sol_to_lamports,
-    pubkey::Pubkey,
-    signature::{read_keypair_file, Keypair, Signature, Signer},
-};
-use std::{str::FromStr, sync::Arc};
 
 // Return parsed values from matches at `name`
 pub fn values_of<T>(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<T>>
@@ -55,7 +57,7 @@ pub fn keypair_of(matches: &ArgMatches<'_>, name: &str) -> Option<Keypair> {
     if let Some(value) = matches.value_of(name) {
         if value == ASK_KEYWORD {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-            keypair_from_seed_phrase(name, skip_validation, true).ok()
+            keypair_from_seed_phrase(name, skip_validation, true, None, true).ok()
         } else {
             read_keypair_file(value).ok()
         }
@@ -70,7 +72,7 @@ pub fn keypairs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Keypair>>
             .filter_map(|value| {
                 if value == ASK_KEYWORD {
                     let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-                    keypair_from_seed_phrase(name, skip_validation, true).ok()
+                    keypair_from_seed_phrase(name, skip_validation, true, None, true).ok()
                 } else {
                     read_keypair_file(value).ok()
                 }

clap-utils/src/input_validators.rs

@@ -1,13 +1,15 @@
-use crate::keypair::{parse_keypair_path, KeypairUrl, ASK_KEYWORD};
-use chrono::DateTime;
-use solana_sdk::{
-    clock::{Epoch, Slot},
-    hash::Hash,
-    pubkey::{Pubkey, MAX_SEED_LEN},
-    signature::{read_keypair_file, Signature},
+use {
+    crate::keypair::{parse_signer_source, SignerSourceKind, ASK_KEYWORD},
+    chrono::DateTime,
+    solana_sdk::{
+        clock::{Epoch, Slot},
+        hash::Hash,
+        pubkey::{Pubkey, MAX_SEED_LEN},
+        signature::{read_keypair_file, Signature},
+    },
+    std::fmt::Display,
+    std::str::FromStr,
 };
-use std::fmt::Display;
-use std::str::FromStr;
 
 fn is_parsable_generic<U, T>(string: T) -> Result<(), String>
 where
@@ -94,6 +96,26 @@ where
         .map_err(|err| format!("{}", err))
 }
 
+// Return an error if a `SignerSourceKind::Prompt` cannot be parsed
+pub fn is_prompt_signer_source<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    if string.as_ref() == ASK_KEYWORD {
+        return Ok(());
+    }
+    match parse_signer_source(string.as_ref())
+        .map_err(|err| format!("{}", err))?
+        .kind
+    {
+        SignerSourceKind::Prompt => Ok(()),
+        _ => Err(format!(
+            "Unable to parse input as `prompt:` URI scheme or `ASK` keyword: {}",
+            string
+        )),
+    }
+}
+
 // Return an error if string cannot be parsed as pubkey string or keypair file location
 pub fn is_pubkey_or_keypair<T>(string: T) -> Result<(), String>
 where
@@ -108,8 +130,11 @@ pub fn is_valid_pubkey<T>(string: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,
 {
-    match parse_keypair_path(string.as_ref()) {
-        KeypairUrl::Filepath(path) => is_keypair(path),
+    match parse_signer_source(string.as_ref())
+        .map_err(|err| format!("{}", err))?
+        .kind
+    {
+        SignerSourceKind::Filepath(path) => is_keypair(path),
         _ => Ok(()),
     }
 }
@@ -190,8 +215,8 @@ where
 pub fn normalize_to_url_if_moniker<T: AsRef<str>>(url_or_moniker: T) -> String {
     match url_or_moniker.as_ref() {
         "m" | "mainnet-beta" => "https://api.mainnet-beta.solana.com",
-        "t" | "testnet" => "https://testnet.solana.com",
-        "d" | "devnet" => "https://devnet.solana.com",
+        "t" | "testnet" => "https://api.testnet.solana.com",
+        "d" | "devnet" => "https://api.devnet.solana.com",
         "l" | "localhost" => "http://localhost:8899",
         url => url,
     }

clap-utils/src/keypair.rs

@@ -1,30 +1,39 @@
-use crate::{
-    input_parsers::pubkeys_sigs_of,
-    offline::{SIGNER_ARG, SIGN_ONLY_ARG},
-    ArgConstant,
-};
-use bip39::{Language, Mnemonic, Seed};
-use clap::ArgMatches;
-use rpassword::prompt_password_stderr;
-use solana_remote_wallet::{
-    remote_keypair::generate_remote_keypair,
-    remote_wallet::{maybe_wallet_manager, RemoteWalletError, RemoteWalletManager},
-};
-use solana_sdk::{
-    hash::Hash,
-    message::Message,
-    pubkey::Pubkey,
-    signature::{
-        keypair_from_seed, keypair_from_seed_phrase_and_passphrase, read_keypair,
-        read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
+use {
+    crate::{
+        input_parsers::pubkeys_sigs_of,
+        offline::{SIGNER_ARG, SIGN_ONLY_ARG},
+        ArgConstant,
     },
-};
-use std::{
-    error,
-    io::{stdin, stdout, Write},
-    process::exit,
-    str::FromStr,
-    sync::Arc,
+    bip39::{Language, Mnemonic, Seed},
+    clap::ArgMatches,
+    rpassword::prompt_password_stderr,
+    solana_remote_wallet::{
+        locator::{Locator as RemoteWalletLocator, LocatorError as RemoteWalletLocatorError},
+        remote_keypair::generate_remote_keypair,
+        remote_wallet::{maybe_wallet_manager, RemoteWalletError, RemoteWalletManager},
+    },
+    solana_sdk::{
+        derivation_path::{DerivationPath, DerivationPathError},
+        hash::Hash,
+        message::Message,
+        pubkey::Pubkey,
+        signature::{
+            generate_seed_from_seed_phrase_and_passphrase, keypair_from_seed,
+            keypair_from_seed_and_derivation_path, keypair_from_seed_phrase_and_passphrase,
+            read_keypair, read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
+        },
+    },
+    std::{
+        cell::RefCell,
+        convert::TryFrom,
+        error,
+        io::{stdin, stdout, Write},
+        ops::Deref,
+        process::exit,
+        str::FromStr,
+        sync::Arc,
+    },
+    thiserror::Error,
 };
 
 pub struct SignOnly {
@@ -83,12 +92,48 @@ impl CliSignerInfo {
     }
 }
 
+#[derive(Debug, Default)]
 pub struct DefaultSigner {
     pub arg_name: String,
     pub path: String,
+    is_path_checked: RefCell<bool>,
 }
 
 impl DefaultSigner {
+    pub fn new<AN: AsRef<str>, P: AsRef<str>>(arg_name: AN, path: P) -> Self {
+        let arg_name = arg_name.as_ref().to_string();
+        let path = path.as_ref().to_string();
+        Self {
+            arg_name,
+            path,
+            ..Self::default()
+        }
+    }
+
+    fn path(&self) -> Result<&str, Box<dyn std::error::Error>> {
+        if !self.is_path_checked.borrow().deref() {
+            parse_signer_source(&self.path)
+                .and_then(|s| {
+                    if let SignerSourceKind::Filepath(path) = &s.kind {
+                        std::fs::metadata(path).map(|_| ()).map_err(|e| e.into())
+                    } else {
+                        Ok(())
+                    }
+                })
+                .map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        format!(
+                        "No default signer found, run \"solana-keygen new -o {}\" to create a new one",
+                        self.path
+                    ),
+                    )
+                })?;
+            *self.is_path_checked.borrow_mut() = true;
+        }
+        Ok(&self.path)
+    }
+
     pub fn generate_unique_signers(
         &self,
         bulk_signers: Vec<Option<Box<dyn Signer>>>,
@@ -103,11 +148,9 @@ impl DefaultSigner {
             unique_signers.push(default_signer);
         }
 
-        for signer in bulk_signers.into_iter() {
-            if let Some(signer) = signer {
-                if !unique_signers.iter().any(|s| s == &signer) {
-                    unique_signers.push(signer);
-                }
+        for signer in bulk_signers.into_iter().flatten() {
+            if !unique_signers.iter().any(|s| s == &signer) {
+                unique_signers.push(signer);
             }
         }
         Ok(CliSignerInfo {
@@ -120,7 +163,7 @@ impl DefaultSigner {
         matches: &ArgMatches,
         wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
     ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
-        signer_from_path(matches, &self.path, &self.arg_name, wallet_manager)
+        signer_from_path(matches, self.path()?, &self.arg_name, wallet_manager)
     }
 
     pub fn signer_from_path_with_config(
@@ -129,29 +172,142 @@ impl DefaultSigner {
         wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
         config: &SignerFromPathConfig,
     ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
-        signer_from_path_with_config(matches, &self.path, &self.arg_name, wallet_manager, config)
+        signer_from_path_with_config(
+            matches,
+            self.path()?,
+            &self.arg_name,
+            wallet_manager,
+            config,
+        )
     }
 }
 
-pub enum KeypairUrl {
-    Ask,
+pub(crate) struct SignerSource {
+    pub kind: SignerSourceKind,
+    pub derivation_path: Option<DerivationPath>,
+    pub legacy: bool,
+}
+
+impl SignerSource {
+    fn new(kind: SignerSourceKind) -> Self {
+        Self {
+            kind,
+            derivation_path: None,
+            legacy: false,
+        }
+    }
+
+    fn new_legacy(kind: SignerSourceKind) -> Self {
+        Self {
+            kind,
+            derivation_path: None,
+            legacy: true,
+        }
+    }
+}
+
+const SIGNER_SOURCE_PROMPT: &str = "prompt";
+const SIGNER_SOURCE_FILEPATH: &str = "file";
+const SIGNER_SOURCE_USB: &str = "usb";
+const SIGNER_SOURCE_STDIN: &str = "stdin";
+const SIGNER_SOURCE_PUBKEY: &str = "pubkey";
+
+pub(crate) enum SignerSourceKind {
+    Prompt,
     Filepath(String),
-    Usb(String),
+    Usb(RemoteWalletLocator),
     Stdin,
     Pubkey(Pubkey),
 }
 
-pub fn parse_keypair_path(path: &str) -> KeypairUrl {
-    if path == "-" {
-        KeypairUrl::Stdin
-    } else if path == ASK_KEYWORD {
-        KeypairUrl::Ask
-    } else if path.starts_with("usb://") {
-        KeypairUrl::Usb(path.to_string())
-    } else if let Ok(pubkey) = Pubkey::from_str(path) {
-        KeypairUrl::Pubkey(pubkey)
-    } else {
-        KeypairUrl::Filepath(path.to_string())
+impl AsRef<str> for SignerSourceKind {
+    fn as_ref(&self) -> &str {
+        match self {
+            Self::Prompt => SIGNER_SOURCE_PROMPT,
+            Self::Filepath(_) => SIGNER_SOURCE_FILEPATH,
+            Self::Usb(_) => SIGNER_SOURCE_USB,
+            Self::Stdin => SIGNER_SOURCE_STDIN,
+            Self::Pubkey(_) => SIGNER_SOURCE_PUBKEY,
+        }
+    }
+}
+
+impl std::fmt::Debug for SignerSourceKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        let s: &str = self.as_ref();
+        write!(f, "{}", s)
+    }
+}
+
+#[derive(Debug, Error)]
+pub(crate) enum SignerSourceError {
+    #[error("unrecognized signer source")]
+    UnrecognizedSource,
+    #[error(transparent)]
+    RemoteWalletLocatorError(#[from] RemoteWalletLocatorError),
+    #[error(transparent)]
+    DerivationPathError(#[from] DerivationPathError),
+    #[error(transparent)]
+    IoError(#[from] std::io::Error),
+}
+
+pub(crate) fn parse_signer_source<S: AsRef<str>>(
+    source: S,
+) -> Result<SignerSource, SignerSourceError> {
+    let source = source.as_ref();
+    let source = {
+        #[cfg(target_family = "windows")]
+        {
+            source.replace("\\", "/")
+        }
+        #[cfg(not(target_family = "windows"))]
+        {
+            source.to_string()
+        }
+    };
+    match uriparse::URIReference::try_from(source.as_str()) {
+        Err(_) => Err(SignerSourceError::UnrecognizedSource),
+        Ok(uri) => {
+            if let Some(scheme) = uri.scheme() {
+                let scheme = scheme.as_str().to_ascii_lowercase();
+                match scheme.as_str() {
+                    SIGNER_SOURCE_PROMPT => Ok(SignerSource {
+                        kind: SignerSourceKind::Prompt,
+                        derivation_path: DerivationPath::from_uri_any_query(&uri)?,
+                        legacy: false,
+                    }),
+                    SIGNER_SOURCE_FILEPATH => Ok(SignerSource::new(SignerSourceKind::Filepath(
+                        uri.path().to_string(),
+                    ))),
+                    SIGNER_SOURCE_USB => Ok(SignerSource {
+                        kind: SignerSourceKind::Usb(RemoteWalletLocator::new_from_uri(&uri)?),
+                        derivation_path: DerivationPath::from_uri_key_query(&uri)?,
+                        legacy: false,
+                    }),
+                    SIGNER_SOURCE_STDIN => Ok(SignerSource::new(SignerSourceKind::Stdin)),
+                    _ => {
+                        #[cfg(target_family = "windows")]
+                        // On Windows, an absolute path's drive letter will be parsed as the URI
+                        // scheme. Assume a filepath source in case of a single character shceme.
+                        if scheme.len() == 1 {
+                            return Ok(SignerSource::new(SignerSourceKind::Filepath(source)));
+                        }
+                        Err(SignerSourceError::UnrecognizedSource)
+                    }
+                }
+            } else {
+                match source.as_str() {
+                    "-" => Ok(SignerSource::new(SignerSourceKind::Stdin)),
+                    ASK_KEYWORD => Ok(SignerSource::new_legacy(SignerSourceKind::Prompt)),
+                    _ => match Pubkey::from_str(source.as_str()) {
+                        Ok(pubkey) => Ok(SignerSource::new(SignerSourceKind::Pubkey(pubkey))),
+                        Err(_) => std::fs::metadata(source.as_str())
+                            .map(|_| SignerSource::new(SignerSourceKind::Filepath(source)))
+                            .map_err(|err| err.into()),
+                    },
+                }
+            }
+        }
     }
 }
 
@@ -198,16 +354,23 @@ pub fn signer_from_path_with_config(
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
     config: &SignerFromPathConfig,
 ) -> Result<Box<dyn Signer>, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Ask => {
+    let SignerSource {
+        kind,
+        derivation_path,
+        legacy,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Prompt => {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
             Ok(Box::new(keypair_from_seed_phrase(
                 keypair_name,
                 skip_validation,
                 false,
+                derivation_path,
+                legacy,
             )?))
         }
-        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
             Err(e) => Err(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 format!("could not read keypair file \"{}\". Run \"solana-keygen new\" to create a keypair file: {}", path, e),
@@ -215,17 +378,18 @@ pub fn signer_from_path_with_config(
             .into()),
             Ok(file) => Ok(Box::new(file)),
         },
-        KeypairUrl::Stdin => {
+        SignerSourceKind::Stdin => {
             let mut stdin = std::io::stdin();
             Ok(Box::new(read_keypair(&mut stdin)?))
         }
-        KeypairUrl::Usb(path) => {
+        SignerSourceKind::Usb(locator) => {
             if wallet_manager.is_none() {
                 *wallet_manager = maybe_wallet_manager()?;
             }
             if let Some(wallet_manager) = wallet_manager {
                 Ok(Box::new(generate_remote_keypair(
-                    path,
+                    locator,
+                    derivation_path.unwrap_or_default(),
                     wallet_manager,
                     matches.is_present("confirm_key"),
                     keypair_name,
@@ -234,7 +398,7 @@ pub fn signer_from_path_with_config(
                 Err(RemoteWalletError::NoDeviceFound.into())
             }
         }
-        KeypairUrl::Pubkey(pubkey) => {
+        SignerSourceKind::Pubkey(pubkey) => {
             let presigner = pubkeys_sigs_of(matches, SIGNER_ARG.name)
                 .as_ref()
                 .and_then(|presigners| presigner_from_pubkey_sigs(&pubkey, presigners));
@@ -259,8 +423,9 @@ pub fn pubkey_from_path(
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Pubkey, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Pubkey(pubkey) => Ok(pubkey),
+    let SignerSource { kind, .. } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Pubkey(pubkey) => Ok(pubkey),
         _ => Ok(signer_from_path(matches, path, keypair_name, wallet_manager)?.pubkey()),
     }
 }
@@ -271,34 +436,51 @@ pub fn resolve_signer_from_path(
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Option<String>, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Ask => {
+    let SignerSource {
+        kind,
+        derivation_path,
+        legacy,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Prompt => {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
             // This method validates the seed phrase, but returns `None` because there is no path
             // on disk or to a device
-            keypair_from_seed_phrase(keypair_name, skip_validation, false).map(|_| None)
+            keypair_from_seed_phrase(
+                keypair_name,
+                skip_validation,
+                false,
+                derivation_path,
+                legacy,
+            )
+            .map(|_| None)
         }
-        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
             Err(e) => Err(std::io::Error::new(
                 std::io::ErrorKind::Other,
-                format!("could not read keypair file \"{}\". Run \"solana-keygen new\" to create a keypair file: {}", path, e),
+                format!(
+                    "could not read keypair file \"{}\". \
+                    Run \"solana-keygen new\" to create a keypair file: {}",
+                    path, e
+                ),
             )
             .into()),
             Ok(_) => Ok(Some(path.to_string())),
         },
-        KeypairUrl::Stdin => {
+        SignerSourceKind::Stdin => {
             let mut stdin = std::io::stdin();
             // This method validates the keypair from stdin, but returns `None` because there is no
             // path on disk or to a device
             read_keypair(&mut stdin).map(|_| None)
         }
-        KeypairUrl::Usb(path) => {
+        SignerSourceKind::Usb(locator) => {
             if wallet_manager.is_none() {
                 *wallet_manager = maybe_wallet_manager()?;
             }
             if let Some(wallet_manager) = wallet_manager {
                 let path = generate_remote_keypair(
-                    path,
+                    locator,
+                    derivation_path.unwrap_or_default(),
                     wallet_manager,
                     matches.is_present("confirm_key"),
                     keypair_name,
@@ -313,7 +495,7 @@ pub fn resolve_signer_from_path(
     }
 }
 
-// Keyword used to indicate that the user should be asked for a keypair seed phrase
+// Keyword used to indicate that the user should be prompted for a keypair seed phrase
 pub const ASK_KEYWORD: &str = "ASK";
 
 pub const SKIP_SEED_PHRASE_VALIDATION_ARG: ArgConstant<'static> = ArgConstant {
@@ -334,6 +516,56 @@ pub fn prompt_passphrase(prompt: &str) -> Result<String, Box<dyn error::Error>>
     Ok(passphrase)
 }
 
+/// Parses a path into a SignerSource and returns a Keypair for supporting SignerSourceKinds
+pub fn keypair_from_path(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    confirm_pubkey: bool,
+) -> Result<Keypair, Box<dyn error::Error>> {
+    let SignerSource {
+        kind,
+        derivation_path,
+        legacy,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Prompt => {
+            let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+            Ok(keypair_from_seed_phrase(
+                keypair_name,
+                skip_validation,
+                confirm_pubkey,
+                derivation_path,
+                legacy,
+            )?)
+        }
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
+            Err(e) => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!(
+                    "could not read keypair file \"{}\". \
+                    Run \"solana-keygen new\" to create a keypair file: {}",
+                    path, e
+                ),
+            )
+            .into()),
+            Ok(file) => Ok(file),
+        },
+        SignerSourceKind::Stdin => {
+            let mut stdin = std::io::stdin();
+            Ok(read_keypair(&mut stdin)?)
+        }
+        _ => Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            format!(
+                "signer of type `{:?}` does not support Keypair output",
+                kind
+            ),
+        )
+        .into()),
+    }
+}
+
 /// Reads user input from stdin to retrieve a seed phrase and passphrase for keypair derivation
 /// Optionally skips validation of seed phrase
 /// Optionally confirms recovered public key
@@ -341,6 +573,8 @@ pub fn keypair_from_seed_phrase(
     keypair_name: &str,
     skip_validation: bool,
     confirm_pubkey: bool,
+    derivation_path: Option<DerivationPath>,
+    legacy: bool,
 ) -> Result<Keypair, Box<dyn error::Error>> {
     let seed_phrase = prompt_password_stderr(&format!("[{}] seed phrase: ", keypair_name))?;
     let seed_phrase = seed_phrase.trim();
@@ -351,7 +585,12 @@ pub fn keypair_from_seed_phrase(
 
     let keypair = if skip_validation {
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
-        keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
+        if legacy {
+            keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
+        } else {
+            let seed = generate_seed_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase);
+            keypair_from_seed_and_derivation_path(&seed, derivation_path)?
+        }
     } else {
         let sanitized = sanitize_seed_phrase(seed_phrase);
         let parse_language_fn = || {
@@ -374,7 +613,11 @@ pub fn keypair_from_seed_phrase(
         let mnemonic = parse_language_fn()?;
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
         let seed = Seed::new(&mnemonic, &passphrase);
-        keypair_from_seed(seed.as_bytes())?
+        if legacy {
+            keypair_from_seed(seed.as_bytes())?
+        } else {
+            keypair_from_seed_and_derivation_path(&seed.as_bytes(), derivation_path)?
+        }
     };
 
     if confirm_pubkey {
@@ -402,7 +645,9 @@ fn sanitize_seed_phrase(seed_phrase: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use solana_remote_wallet::locator::Manufacturer;
     use solana_sdk::system_instruction;
+    use tempfile::NamedTempFile;
 
     #[test]
     fn test_sanitize_seed_phrase() {
@@ -443,4 +688,122 @@ mod tests {
         ];
         assert_eq!(signer_pubkeys, expect);
     }
+
+    #[test]
+    fn test_parse_signer_source() {
+        assert!(matches!(
+            parse_signer_source("-").unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Stdin,
+                derivation_path: None,
+                legacy: false,
+            }
+        ));
+        let stdin = "stdin:".to_string();
+        assert!(matches!(
+            parse_signer_source(&stdin).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Stdin,
+                derivation_path: None,
+                legacy: false,
+            }
+        ));
+        assert!(matches!(
+            parse_signer_source(ASK_KEYWORD).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Prompt,
+                derivation_path: None,
+                legacy: true,
+            }
+        ));
+        let pubkey = Pubkey::new_unique();
+        assert!(
+            matches!(parse_signer_source(&pubkey.to_string()).unwrap(), SignerSource {
+                kind: SignerSourceKind::Pubkey(p),
+                derivation_path: None,
+                legacy: false,
+            }
+            if p == pubkey)
+        );
+
+        // Set up absolute and relative path strs
+        let file0 = NamedTempFile::new().unwrap();
+        let path = file0.path();
+        assert!(path.is_absolute());
+        let absolute_path_str = path.to_str().unwrap();
+
+        let file1 = NamedTempFile::new_in(std::env::current_dir().unwrap()).unwrap();
+        let path = file1.path().file_name().unwrap().to_str().unwrap();
+        let path = std::path::Path::new(path);
+        assert!(path.is_relative());
+        let relative_path_str = path.to_str().unwrap();
+
+        assert!(
+            matches!(parse_signer_source(absolute_path_str).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == absolute_path_str)
+        );
+        assert!(
+            matches!(parse_signer_source(&relative_path_str).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == relative_path_str)
+        );
+
+        let usb = "usb://ledger".to_string();
+        let expected_locator = RemoteWalletLocator {
+            manufacturer: Manufacturer::Ledger,
+            pubkey: None,
+        };
+        assert!(matches!(parse_signer_source(&usb).unwrap(), SignerSource {
+                kind: SignerSourceKind::Usb(u),
+                derivation_path: None,
+                legacy: false,
+            } if u == expected_locator));
+        let usb = "usb://ledger?key=0/0".to_string();
+        let expected_locator = RemoteWalletLocator {
+            manufacturer: Manufacturer::Ledger,
+            pubkey: None,
+        };
+        let expected_derivation_path = Some(DerivationPath::new_bip44(Some(0), Some(0)));
+        assert!(matches!(parse_signer_source(&usb).unwrap(), SignerSource {
+                kind: SignerSourceKind::Usb(u),
+                derivation_path: d,
+                legacy: false,
+            } if u == expected_locator && d == expected_derivation_path));
+        // Catchall into SignerSource::Filepath fails
+        let junk = "sometextthatisnotapubkeyorfile".to_string();
+        assert!(Pubkey::from_str(&junk).is_err());
+        assert!(matches!(
+            parse_signer_source(&junk),
+            Err(SignerSourceError::IoError(_))
+        ));
+
+        let prompt = "prompt:".to_string();
+        assert!(matches!(
+            parse_signer_source(&prompt).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Prompt,
+                derivation_path: None,
+                legacy: false,
+            }
+        ));
+        assert!(
+            matches!(parse_signer_source(&format!("file:{}", absolute_path_str)).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == absolute_path_str)
+        );
+        assert!(
+            matches!(parse_signer_source(&format!("file:{}", relative_path_str)).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == relative_path_str)
+        );
+    }
 }

clap-utils/src/memo.rs

@@ -1,5 +1,4 @@
-use crate::ArgConstant;
-use clap::Arg;
+use {crate::ArgConstant, clap::Arg};
 
 pub const MEMO_ARG: ArgConstant<'static> = ArgConstant {
     name: "memo",

clap-utils/src/nonce.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators::*, offline::BLOCKHASH_ARG, ArgConstant};
-use clap::{App, Arg};
+use {
+    crate::{input_validators::*, offline::BLOCKHASH_ARG, ArgConstant},
+    clap::{App, Arg},
+};
 
 pub const NONCE_ARG: ArgConstant<'static> = ArgConstant {
     name: "nonce",

clap-utils/src/offline.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators::*, ArgConstant};
-use clap::{App, Arg};
+use {
+    crate::{input_validators::*, ArgConstant},
+    clap::{App, Arg},
+};
 
 pub const BLOCKHASH_ARG: ArgConstant<'static> = ArgConstant {
     name: "blockhash",

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-config/src/config.rs

@@ -107,13 +107,13 @@ mod test {
     #[test]
     fn compute_websocket_url() {
         assert_eq!(
-            Config::compute_websocket_url(&"http://devnet.solana.com"),
-            "ws://devnet.solana.com/".to_string()
+            Config::compute_websocket_url(&"http://api.devnet.solana.com"),
+            "ws://api.devnet.solana.com/".to_string()
         );
 
         assert_eq!(
-            Config::compute_websocket_url(&"https://devnet.solana.com"),
-            "wss://devnet.solana.com/".to_string()
+            Config::compute_websocket_url(&"https://api.devnet.solana.com"),
+            "wss://api.devnet.solana.com/".to_string()
         );
 
         assert_eq!(

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -19,13 +19,13 @@ indicatif = "0.15.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.6.5" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-client = { path = "../client", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-stake-program = { path = "../programs/stake", version = "=1.6.5" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.6.5" }
-solana-vote-program = { path = "../programs/vote", version = "=1.6.5" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.20" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-client = { path = "../client", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.20" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.20" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.20" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [package.metadata.docs.rs]

cli-output/src/cli_output.rs

@@ -15,8 +15,8 @@ use {
     solana_account_decoder::parse_token::UiTokenAccount,
     solana_clap_utils::keypair::SignOnly,
     solana_client::rpc_response::{
-        RpcAccountBalance, RpcInflationGovernor, RpcInflationRate, RpcKeyedAccount, RpcSupply,
-        RpcVoteAccountInfo,
+        RpcAccountBalance, RpcContactInfo, RpcInflationGovernor, RpcInflationRate, RpcKeyedAccount,
+        RpcSupply, RpcVoteAccountInfo,
     },
     solana_sdk::{
         clock::{Epoch, Slot, UnixTimestamp},
@@ -233,6 +233,10 @@ pub struct CliEpochInfo {
     pub epoch_info: EpochInfo,
     #[serde(skip)]
     pub average_slot_time_ms: u64,
+    #[serde(skip)]
+    pub start_block_time: Option<UnixTimestamp>,
+    #[serde(skip)]
+    pub current_block_time: Option<UnixTimestamp>,
 }
 
 impl QuietDisplay for CliEpochInfo {}
@@ -277,21 +281,41 @@ impl fmt::Display for CliEpochInfo {
                 remaining_slots_in_epoch
             ),
         )?;
+        let (time_elapsed, annotation) = if let (Some(start_block_time), Some(current_block_time)) =
+            (self.start_block_time, self.current_block_time)
+        {
+            (
+                Duration::from_secs((current_block_time - start_block_time) as u64),
+                None,
+            )
+        } else {
+            (
+                slot_to_duration(self.epoch_info.slot_index, self.average_slot_time_ms),
+                Some("* estimated based on current slot durations"),
+            )
+        };
+        let time_remaining = slot_to_duration(remaining_slots_in_epoch, self.average_slot_time_ms);
         writeln_name_value(
             f,
             "Epoch Completed Time:",
             &format!(
-                "{}/{} ({} remaining)",
-                slot_to_human_time(self.epoch_info.slot_index, self.average_slot_time_ms),
-                slot_to_human_time(self.epoch_info.slots_in_epoch, self.average_slot_time_ms),
-                slot_to_human_time(remaining_slots_in_epoch, self.average_slot_time_ms)
+                "{}{}/{} ({} remaining)",
+                humantime::format_duration(time_elapsed).to_string(),
+                if annotation.is_some() { "*" } else { "" },
+                humantime::format_duration(time_elapsed + time_remaining).to_string(),
+                humantime::format_duration(time_remaining).to_string(),
             ),
-        )
+        )?;
+        if let Some(annotation) = annotation {
+            writeln!(f)?;
+            writeln!(f, "{}", annotation)?;
+        }
+        Ok(())
     }
 }
 
-fn slot_to_human_time(slot: Slot, slot_time_ms: u64) -> String {
-    humantime::format_duration(Duration::from_secs((slot * slot_time_ms) / 1000)).to_string()
+fn slot_to_duration(slot: Slot, slot_time_ms: u64) -> Duration {
+    Duration::from_secs((slot * slot_time_ms) / 1000)
 }
 
 #[derive(Serialize, Deserialize, Default)]
@@ -303,14 +327,34 @@ pub struct CliValidatorsStakeByVersion {
     pub delinquent_active_stake: u64,
 }
 
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
+pub enum CliValidatorsSortOrder {
+    Delinquent,
+    Commission,
+    EpochCredits,
+    Identity,
+    LastVote,
+    Root,
+    SkipRate,
+    Stake,
+    VoteAccount,
+}
+
 #[derive(Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct CliValidators {
     pub total_active_stake: u64,
     pub total_current_stake: u64,
     pub total_delinquent_stake: u64,
-    pub current_validators: Vec<CliValidator>,
-    pub delinquent_validators: Vec<CliValidator>,
+    pub validators: Vec<CliValidator>,
+    pub average_skip_rate: f64,
+    pub average_stake_weighted_skip_rate: f64,
+    #[serde(skip_serializing)]
+    pub validators_sort_order: CliValidatorsSortOrder,
+    #[serde(skip_serializing)]
+    pub validators_reverse_sort: bool,
+    #[serde(skip_serializing)]
+    pub number_validators: bool,
     pub stake_by_version: BTreeMap<String, CliValidatorsStakeByVersion>,
     #[serde(skip_serializing)]
     pub use_lamports_unit: bool,
@@ -326,30 +370,40 @@ impl fmt::Display for CliValidators {
             validator: &CliValidator,
             total_active_stake: u64,
             use_lamports_unit: bool,
-            delinquent: bool,
+            highest_last_vote: u64,
+            highest_root: u64,
         ) -> fmt::Result {
-            fn non_zero_or_dash(v: u64) -> String {
+            fn non_zero_or_dash(v: u64, max_v: u64) -> String {
                 if v == 0 {
-                    "-".into()
+                    "-         ".into()
+                } else if v == max_v {
+                    format!("{:>8} (  0)", v)
+                } else if v > max_v.saturating_sub(100) {
+                    format!("{:>8} ({:>3})", v, -(max_v.saturating_sub(v) as isize))
                 } else {
-                    format!("{}", v)
+                    format!("{:>8}      ", v)
                 }
             }
 
             writeln!(
                 f,
-                "{} {:<44}  {:<44}  {:>3}%   {:>8}  {:>10}  {:>10}  {:>8}  {}",
-                if delinquent {
+                "{} {:<44}  {:<44}  {:>3}%  {:>14}  {:>14} {:>7} {:>8}  {:>7}  {}",
+                if validator.delinquent {
                     WARNING.to_string()
                 } else {
-                    " ".to_string()
+                    "\u{a0}".to_string()
                 },
                 validator.identity_pubkey,
                 validator.vote_account_pubkey,
                 validator.commission,
-                non_zero_or_dash(validator.last_vote),
-                non_zero_or_dash(validator.root_slot),
-                validator.credits,
+                non_zero_or_dash(validator.last_vote, highest_last_vote),
+                non_zero_or_dash(validator.root_slot, highest_root),
+                if let Some(skip_rate) = validator.skip_rate {
+                    format!("{:.2}%", skip_rate)
+                } else {
+                    "- ".to_string()
+                },
+                validator.epoch_credits,
                 validator.version,
                 if validator.activated_stake > 0 {
                     format!(
@@ -362,40 +416,113 @@ impl fmt::Display for CliValidators {
                 },
             )
         }
-        writeln!(
+
+        let padding = if self.number_validators {
+            ((self.validators.len() + 1) as f64).log10().floor() as usize + 1
+        } else {
+            0
+        };
+        let header = style(format!(
+            "{:padding$} {:<44}  {:<38}  {}  {}  {} {}  {}  {}  {}",
+            " ",
+            "Identity",
+            "Vote Account",
+            "Commission",
+            "Last Vote     ",
+            "Root Slot   ",
+            "Skip Rate",
+            "Credits",
+            "Version",
+            "Active Stake",
+            padding = padding + 1
+        ))
+        .bold();
+        writeln!(f, "{}", header)?;
+
+        let mut sorted_validators = self.validators.clone();
+        match self.validators_sort_order {
+            CliValidatorsSortOrder::Delinquent => {
+                sorted_validators.sort_by_key(|a| a.delinquent);
+            }
+            CliValidatorsSortOrder::Commission => {
+                sorted_validators.sort_by_key(|a| a.commission);
+            }
+            CliValidatorsSortOrder::EpochCredits => {
+                sorted_validators.sort_by_key(|a| a.epoch_credits);
+            }
+            CliValidatorsSortOrder::Identity => {
+                sorted_validators.sort_by(|a, b| a.identity_pubkey.cmp(&b.identity_pubkey));
+            }
+            CliValidatorsSortOrder::LastVote => {
+                sorted_validators.sort_by_key(|a| a.last_vote);
+            }
+            CliValidatorsSortOrder::Root => {
+                sorted_validators.sort_by_key(|a| a.root_slot);
+            }
+            CliValidatorsSortOrder::VoteAccount => {
+                sorted_validators.sort_by(|a, b| a.vote_account_pubkey.cmp(&b.vote_account_pubkey));
+            }
+            CliValidatorsSortOrder::SkipRate => {
+                sorted_validators.sort_by(|a, b| {
+                    use std::cmp::Ordering;
+                    match (a.skip_rate, b.skip_rate) {
+                        (None, None) => Ordering::Equal,
+                        (None, Some(_)) => Ordering::Greater,
+                        (Some(_), None) => Ordering::Less,
+                        (Some(a), Some(b)) => a.partial_cmp(&b).unwrap_or(Ordering::Equal),
+                    }
+                });
+            }
+            CliValidatorsSortOrder::Stake => {
+                sorted_validators.sort_by_key(|a| a.activated_stake);
+            }
+        }
+
+        if self.validators_reverse_sort {
+            sorted_validators.reverse();
+        }
+
+        let highest_root = sorted_validators
+            .iter()
+            .map(|v| v.root_slot)
+            .max()
+            .unwrap_or_default();
+        let highest_last_vote = sorted_validators
+            .iter()
+            .map(|v| v.last_vote)
+            .max()
+            .unwrap_or_default();
+
+        for (i, validator) in sorted_validators.iter().enumerate() {
+            if padding > 0 {
+                write!(f, "{:padding$}", i + 1, padding = padding)?;
+            }
+            write_vote_account(
+                f,
+                validator,
+                self.total_active_stake,
+                self.use_lamports_unit,
+                highest_last_vote,
+                highest_root,
+            )?;
+        }
+
+        // The actual header has long scrolled away.  Print the header once more as a footer
+        if self.validators.len() > 100 {
+            writeln!(f, "{}", header)?;
+        }
+
+        writeln!(f)?;
+        writeln_name_value(
             f,
-            "{}",
-            style(format!(
-                "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
-                "Identity",
-                "Vote Account",
-                "Commission",
-                "Last Vote",
-                "Root Block",
-                "Credits",
-                "Version",
-                "Active Stake",
-            ))
-            .bold()
+            "Average Stake-Weighted Skip Rate:",
+            &format!("{:.2}%", self.average_stake_weighted_skip_rate,),
+        )?;
+        writeln_name_value(
+            f,
+            "Average Unweighted Skip Rate:    ",
+            &format!("{:.2}%", self.average_skip_rate),
         )?;
-        for validator in &self.current_validators {
-            write_vote_account(
-                f,
-                validator,
-                self.total_active_stake,
-                self.use_lamports_unit,
-                false,
-            )?;
-        }
-        for validator in &self.delinquent_validators {
-            write_vote_account(
-                f,
-                validator,
-                self.total_active_stake,
-                self.use_lamports_unit,
-                true,
-            )?;
-        }
 
         writeln!(f)?;
         writeln_name_value(
@@ -453,7 +580,7 @@ impl fmt::Display for CliValidators {
     }
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct CliValidator {
     pub identity_pubkey: String,
@@ -461,9 +588,12 @@ pub struct CliValidator {
     pub commission: u8,
     pub last_vote: u64,
     pub root_slot: u64,
-    pub credits: u64,
+    pub credits: u64,       // lifetime credits
+    pub epoch_credits: u64, // credits earned in the current epoch
     pub activated_stake: u64,
     pub version: String,
+    pub delinquent: bool,
+    pub skip_rate: Option<f64>,
 }
 
 impl CliValidator {
@@ -471,27 +601,67 @@ impl CliValidator {
         vote_account: &RpcVoteAccountInfo,
         current_epoch: Epoch,
         version: String,
+        skip_rate: Option<f64>,
         address_labels: &HashMap<String, String>,
     ) -> Self {
+        Self::_new(
+            vote_account,
+            current_epoch,
+            version,
+            skip_rate,
+            address_labels,
+            false,
+        )
+    }
+
+    pub fn new_delinquent(
+        vote_account: &RpcVoteAccountInfo,
+        current_epoch: Epoch,
+        version: String,
+        skip_rate: Option<f64>,
+        address_labels: &HashMap<String, String>,
+    ) -> Self {
+        Self::_new(
+            vote_account,
+            current_epoch,
+            version,
+            skip_rate,
+            address_labels,
+            true,
+        )
+    }
+
+    fn _new(
+        vote_account: &RpcVoteAccountInfo,
+        current_epoch: Epoch,
+        version: String,
+        skip_rate: Option<f64>,
+        address_labels: &HashMap<String, String>,
+        delinquent: bool,
+    ) -> Self {
+        let (credits, epoch_credits) = vote_account
+            .epoch_credits
+            .iter()
+            .find_map(|(epoch, credits, pre_credits)| {
+                if *epoch == current_epoch {
+                    Some((*credits, credits.saturating_sub(*pre_credits)))
+                } else {
+                    None
+                }
+            })
+            .unwrap_or((0, 0));
         Self {
             identity_pubkey: format_labeled_address(&vote_account.node_pubkey, address_labels),
             vote_account_pubkey: format_labeled_address(&vote_account.vote_pubkey, address_labels),
             commission: vote_account.commission,
             last_vote: vote_account.last_vote,
             root_slot: vote_account.root_slot,
-            credits: vote_account
-                .epoch_credits
-                .iter()
-                .find_map(|(epoch, credits, _)| {
-                    if *epoch == current_epoch {
-                        Some(*credits)
-                    } else {
-                        None
-                    }
-                })
-                .unwrap_or(0),
+            credits,
+            epoch_credits,
             activated_stake: vote_account.activated_stake,
             version,
+            delinquent,
+            skip_rate,
         }
     }
 }
@@ -601,6 +771,7 @@ pub struct CliEpochReward {
     pub post_balance: u64, // lamports
     pub percent_change: f64,
     pub apr: Option<f64>,
+    pub commission: Option<u8>,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -645,23 +816,27 @@ impl fmt::Display for CliKeyedEpochRewards {
         writeln!(f, "Epoch Rewards:")?;
         writeln!(
             f,
-            "  {:<44}  {:<18}  {:<18}  {:>14}  {:>14}",
-            "Address", "Amount", "New Balance", "Percent Change", "APR"
+            "  {:<44}  {:<18}  {:<18}  {:>14}  {:>14}  {:>10}",
+            "Address", "Amount", "New Balance", "Percent Change", "APR", "Commission"
         )?;
         for keyed_reward in &self.rewards {
             match &keyed_reward.reward {
                 Some(reward) => {
                     writeln!(
                         f,
-                        "  {:<44}  ◎{:<17.9}  ◎{:<17.9}  {:>13.2}%  {}",
+                        "  {:<44}  ◎{:<17.9}  ◎{:<17.9}  {:>13.9}%  {:>14}  {:>10}",
                         keyed_reward.address,
                         lamports_to_sol(reward.amount),
                         lamports_to_sol(reward.post_balance),
                         reward.percent_change,
                         reward
                             .apr
-                            .map(|apr| format!("{:>13.2}%", apr))
+                            .map(|apr| format!("{:.2}%", apr))
                             .unwrap_or_default(),
+                        reward
+                            .commission
+                            .map(|commission| format!("{}%", commission))
+                            .unwrap_or_else(|| "-".to_string())
                     )?;
                 }
                 None => {
@@ -778,13 +953,13 @@ fn show_epoch_rewards(
         writeln!(f, "Epoch Rewards:")?;
         writeln!(
             f,
-            "  {:<6}  {:<11}  {:<18}  {:<18}  {:>14}  {:>14}",
-            "Epoch", "Reward Slot", "Amount", "New Balance", "Percent Change", "APR"
+            "  {:<6}  {:<11}  {:<18}  {:<18}  {:>14}  {:>14}  {:>10}",
+            "Epoch", "Reward Slot", "Amount", "New Balance", "Percent Change", "APR", "Commission"
         )?;
         for reward in epoch_rewards {
             writeln!(
                 f,
-                "  {:<6}  {:<11}  ◎{:<17.9}  ◎{:<17.9}  {:>13.2}%  {}",
+                "  {:<6}  {:<11}  ◎{:<17.9}  ◎{:<17.9}  {:>13.9}%  {:>14}  {:>10}",
                 reward.epoch,
                 reward.effective_slot,
                 lamports_to_sol(reward.amount),
@@ -792,8 +967,12 @@ fn show_epoch_rewards(
                 reward.percent_change,
                 reward
                     .apr
-                    .map(|apr| format!("{:>13.2}%", apr))
+                    .map(|apr| format!("{:.2}%", apr))
                     .unwrap_or_default(),
+                reward
+                    .commission
+                    .map(|commission| format!("{}%", commission))
+                    .unwrap_or_else(|| "-".to_string())
             )?;
         }
     }
@@ -1329,18 +1508,18 @@ impl fmt::Display for CliInflation {
         if (self.governor.initial - self.governor.terminal).abs() < f64::EPSILON {
             writeln!(
                 f,
-                "Fixed APR:               {:>5.2}%",
+                "Fixed rate:              {:>5.2}%",
                 self.governor.terminal * 100.
             )?;
         } else {
             writeln!(
                 f,
-                "Initial APR:             {:>5.2}%",
+                "Initial rate:            {:>5.2}%",
                 self.governor.initial * 100.
             )?;
             writeln!(
                 f,
-                "Terminal APR:            {:>5.2}%",
+                "Terminal rate:           {:>5.2}%",
                 self.governor.terminal * 100.
             )?;
             writeln!(
@@ -1348,6 +1527,10 @@ impl fmt::Display for CliInflation {
                 "Rate reduction per year: {:>5.2}%",
                 self.governor.taper * 100.
             )?;
+            writeln!(
+                f,
+                "* Rate reduction is derived using the target slot time in genesis config"
+            )?;
         }
         if self.governor.foundation_term > 0. {
             writeln!(
@@ -1369,17 +1552,17 @@ impl fmt::Display for CliInflation {
         )?;
         writeln!(
             f,
-            "Total APR:               {:>5.2}%",
+            "Total rate:              {:>5.2}%",
             self.current_rate.total * 100.
         )?;
         writeln!(
             f,
-            "Staking APR:             {:>5.2}%",
+            "Staking rate:            {:>5.2}%",
             self.current_rate.validator * 100.
         )?;
         writeln!(
             f,
-            "Foundation APR:          {:>5.2}%",
+            "Foundation rate:         {:>5.2}%",
             self.current_rate.foundation * 100.
         )
     }
@@ -1533,6 +1716,7 @@ pub struct CliFeesInner {
     pub blockhash: String,
     pub lamports_per_signature: u64,
     pub last_valid_slot: Option<Slot>,
+    pub last_valid_block_height: Option<Slot>,
 }
 
 impl QuietDisplay for CliFeesInner {}
@@ -1546,11 +1730,11 @@ impl fmt::Display for CliFeesInner {
             "Lamports per signature:",
             &self.lamports_per_signature.to_string(),
         )?;
-        let last_valid_slot = self
-            .last_valid_slot
+        let last_valid_block_height = self
+            .last_valid_block_height
             .map(|s| s.to_string())
             .unwrap_or_default();
-        writeln_name_value(f, "Last valid slot:", &last_valid_slot)
+        writeln_name_value(f, "Last valid block height:", &last_valid_block_height)
     }
 }
 
@@ -1579,6 +1763,7 @@ impl CliFees {
         blockhash: Hash,
         lamports_per_signature: u64,
         last_valid_slot: Option<Slot>,
+        last_valid_block_height: Option<Slot>,
     ) -> Self {
         Self {
             inner: Some(CliFeesInner {
@@ -1586,6 +1771,7 @@ impl CliFees {
                 blockhash: blockhash.to_string(),
                 lamports_per_signature,
                 last_valid_slot,
+                last_valid_block_height,
             }),
         }
     }
@@ -1990,6 +2176,9 @@ impl fmt::Display for CliBlock {
         if let Some(block_time) = self.encoded_confirmed_block.block_time {
             writeln!(f, "Block Time: {:?}", Local.timestamp(block_time, 0))?;
         }
+        if let Some(block_height) = self.encoded_confirmed_block.block_height {
+            writeln!(f, "Block Height: {:?}", block_height)?;
+        }
         if !self.encoded_confirmed_block.rewards.is_empty() {
             let mut rewards = self.encoded_confirmed_block.rewards.clone();
             rewards.sort_by(|a, b| a.pubkey.cmp(&b.pubkey));
@@ -1997,8 +2186,8 @@ impl fmt::Display for CliBlock {
             writeln!(f, "Rewards:")?;
             writeln!(
                 f,
-                "  {:<44}  {:^15}  {:<15}  {:<20}  {:>14}",
-                "Address", "Type", "Amount", "New Balance", "Percent Change"
+                "  {:<44}  {:^15}  {:<15}  {:<20}  {:>14}  {:>10}",
+                "Address", "Type", "Amount", "New Balance", "Percent Change", "Commission"
             )?;
             for reward in rewards {
                 let sign = if reward.lamports < 0 { "-" } else { "" };
@@ -2006,7 +2195,7 @@ impl fmt::Display for CliBlock {
                 total_rewards += reward.lamports;
                 writeln!(
                     f,
-                    "  {:<44}  {:^15}  {:>15}  {}",
+                    "  {:<44}  {:^15}  {:>15}  {}  {}",
                     reward.pubkey,
                     if let Some(reward_type) = reward.reward_type {
                         format!("{}", reward_type)
@@ -2028,7 +2217,11 @@ impl fmt::Display for CliBlock {
                                 / (reward.post_balance as f64 - reward.lamports as f64))
                                 * 100.0
                         )
-                    }
+                    },
+                    reward
+                        .commission
+                        .map(|commission| format!("{:>9}%", commission))
+                        .unwrap_or_else(|| "    -".to_string())
                 )?;
             }
 
@@ -2147,6 +2340,97 @@ impl fmt::Display for CliTransactionConfirmation {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliGossipNode {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ip_address: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub identity_label: Option<String>,
+    pub identity_pubkey: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub gossip_port: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tpu_port: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rpc_host: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub version: Option<String>,
+}
+
+impl CliGossipNode {
+    pub fn new(info: RpcContactInfo, labels: &HashMap<String, String>) -> Self {
+        Self {
+            ip_address: info.gossip.map(|addr| addr.ip().to_string()),
+            identity_label: labels.get(&info.pubkey).cloned(),
+            identity_pubkey: info.pubkey,
+            gossip_port: info.gossip.map(|addr| addr.port()),
+            tpu_port: info.tpu.map(|addr| addr.port()),
+            rpc_host: info.rpc.map(|addr| addr.to_string()),
+            version: info.version,
+        }
+    }
+}
+
+fn unwrap_to_string_or_none<T>(option: Option<T>) -> String
+where
+    T: std::string::ToString,
+{
+    unwrap_to_string_or_default(option, "none")
+}
+
+fn unwrap_to_string_or_default<T>(option: Option<T>, default: &str) -> String
+where
+    T: std::string::ToString,
+{
+    option
+        .as_ref()
+        .map(|v| v.to_string())
+        .unwrap_or_else(|| default.to_string())
+}
+
+impl fmt::Display for CliGossipNode {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "{:15} | {:44} | {:6} | {:5} | {:21} | {}",
+            unwrap_to_string_or_none(self.ip_address.as_ref()),
+            self.identity_label
+                .as_ref()
+                .unwrap_or(&self.identity_pubkey),
+            unwrap_to_string_or_none(self.gossip_port.as_ref()),
+            unwrap_to_string_or_none(self.tpu_port.as_ref()),
+            unwrap_to_string_or_none(self.rpc_host.as_ref()),
+            unwrap_to_string_or_default(self.version.as_ref(), "unknown"),
+        )
+    }
+}
+
+impl QuietDisplay for CliGossipNode {}
+impl VerboseDisplay for CliGossipNode {}
+
+#[derive(Serialize, Deserialize)]
+pub struct CliGossipNodes(pub Vec<CliGossipNode>);
+
+impl fmt::Display for CliGossipNodes {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(
+            f,
+            "IP Address      | Node identifier                              \
+             | Gossip | TPU   | RPC Address           | Version\n\
+             ----------------+----------------------------------------------+\
+             --------+-------+-----------------------+----------------",
+        )?;
+        for node in self.0.iter() {
+            writeln!(f, "{}", node)?;
+        }
+        writeln!(f, "Nodes: {}", self.0.len())
+    }
+}
+
+impl QuietDisplay for CliGossipNodes {}
+impl VerboseDisplay for CliGossipNodes {}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -2178,6 +2462,10 @@ mod tests {
             fn try_sign_message(&self, _message: &[u8]) -> Result<Signature, SignerError> {
                 Ok(Signature::new(&[1u8; 64]))
             }
+
+            fn is_interactive(&self) -> bool {
+                false
+            }
         }
 
         let present: Box<dyn Signer> = Box::new(keypair_from_seed(&[2u8; 32]).unwrap());

cli-output/src/display.rs

@@ -322,7 +322,38 @@ pub fn write_transaction<W: io::Write>(
             if !log_messages.is_empty() {
                 writeln!(w, "{}Log Messages:", prefix,)?;
                 for log_message in log_messages {
-                    writeln!(w, "{}  {}", prefix, log_message,)?;
+                    writeln!(w, "{}  {}", prefix, log_message)?;
+                }
+            }
+        }
+
+        if let Some(rewards) = &transaction_status.rewards {
+            if !rewards.is_empty() {
+                writeln!(w, "{}Rewards:", prefix,)?;
+                writeln!(
+                    w,
+                    "{}  {:<44}  {:^15}  {:<15}  {:<20}",
+                    prefix, "Address", "Type", "Amount", "New Balance"
+                )?;
+                for reward in rewards {
+                    let sign = if reward.lamports < 0 { "-" } else { "" };
+                    writeln!(
+                        w,
+                        "{}  {:<44}  {:^15}  {:<15}  {}",
+                        prefix,
+                        reward.pubkey,
+                        if let Some(reward_type) = reward.reward_type {
+                            format!("{}", reward_type)
+                        } else {
+                            "-".to_string()
+                        },
+                        format!(
+                            "{}◎{:<14.9}",
+                            sign,
+                            lamports_to_sol(reward.lamports.abs() as u64)
+                        ),
+                        format!("◎{:<18.9}", lamports_to_sol(reward.post_balance),)
+                    )?;
                 }
             }
         }

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.5"
+version = "1.6.20"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,6 +17,7 @@ clap = "2.33.1"
 criterion-stats = "0.3.0"
 ctrlc = { version = "3.1.5", features = ["termination"] }
 console = "0.11.3"
+const_format = "0.2.14"
 dirs-next = "2.0.0"
 log = "0.4.11"
 Inflector = "0.11.4"
@@ -28,30 +29,30 @@ reqwest = { version = "0.11.2", default-features = false, features = ["blocking"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.6.5" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.6.5" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-cli-config = { path = "../cli-config", version = "=1.6.5" }
-solana-cli-output = { path = "../cli-output", version = "=1.6.5" }
-solana-client = { path = "../client", version = "=1.6.5" }
-solana-config-program = { path = "../programs/config", version = "=1.6.5" }
-solana-faucet = { path = "../faucet", version = "=1.6.5" }
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-net-utils = { path = "../net-utils", version = "=1.6.5" }
-solana_rbpf = "=0.2.7"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-stake-program = { path = "../programs/stake", version = "=1.6.5" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
-solana-vote-program = { path = "../programs/vote", version = "=1.6.5" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.20" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.6.20" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-cli-config = { path = "../cli-config", version = "=1.6.20" }
+solana-cli-output = { path = "../cli-output", version = "=1.6.20" }
+solana-client = { path = "../client", version = "=1.6.20" }
+solana-config-program = { path = "../programs/config", version = "=1.6.20" }
+solana-faucet = { path = "../faucet", version = "=1.6.20" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.20" }
+solana_rbpf = "=0.2.9"
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.20" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.20" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "=1.6.5" }
+solana-core = { path = "../core", version = "=1.6.20" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -20,11 +20,12 @@ use solana_clap_utils::{
 use solana_cli_output::{
     display::{build_balance_message, println_name_value},
     return_signers_with_config, CliAccount, CliSignature, CliSignatureVerificationStatus,
-    CliTransaction, CliTransactionConfirmation, OutputFormat, ReturnSignersConfig,
+    CliTransaction, CliTransactionConfirmation, CliValidatorsSortOrder, OutputFormat,
+    ReturnSignersConfig,
 };
 use solana_client::{
     blockhash_query::BlockhashQuery,
-    client_error::{ClientError, ClientErrorKind, Result as ClientResult},
+    client_error::{ClientError, Result as ClientResult},
     nonce_utils,
     rpc_client::RpcClient,
     rpc_config::{
@@ -33,11 +34,6 @@ use solana_client::{
     },
     rpc_response::RpcKeyedAccount,
 };
-#[cfg(not(test))]
-use solana_faucet::faucet::request_airdrop_transaction;
-use solana_faucet::faucet::FaucetError;
-#[cfg(test)]
-use solana_faucet::faucet_mock::request_airdrop_transaction;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     clock::{Epoch, Slot},
@@ -59,21 +55,13 @@ use solana_stake_program::{
 use solana_transaction_status::{EncodedTransaction, UiTransactionEncoding};
 use solana_vote_program::vote_state::VoteAuthorize;
 use std::{
-    collections::HashMap,
-    error,
-    fmt::Write as FmtWrite,
-    fs::File,
-    io::Write,
-    net::{IpAddr, SocketAddr},
-    str::FromStr,
-    sync::Arc,
-    thread::sleep,
-    time::Duration,
+    collections::HashMap, error, fmt::Write as FmtWrite, fs::File, io::Write, str::FromStr,
+    sync::Arc, time::Duration,
 };
 use thiserror::Error;
-use url::Url;
 
 pub const DEFAULT_RPC_TIMEOUT_SECONDS: &str = "30";
+pub const DEFAULT_CONFIRM_TX_TIMEOUT_SECONDS: &str = "5";
 
 #[derive(Debug, PartialEq)]
 #[allow(clippy::large_enum_variant)]
@@ -144,6 +132,11 @@ pub enum CliCommand {
     },
     ShowValidators {
         use_lamports_unit: bool,
+        sort_order: CliValidatorsSortOrder,
+        reverse_sort: bool,
+        number_validators: bool,
+        keep_unstaked_delinquents: bool,
+        delinquent_slot_distance: Option<Slot>,
     },
     Supply {
         print_accounts: bool,
@@ -224,6 +217,7 @@ pub enum CliCommand {
         nonce_account: Option<Pubkey>,
         nonce_authority: SignerIndex,
         memo: Option<String>,
+        seed: Option<String>,
         fee_payer: SignerIndex,
     },
     DelegateStake {
@@ -267,6 +261,7 @@ pub enum CliCommand {
     },
     ShowStakeHistory {
         use_lamports_unit: bool,
+        limit_results: usize,
     },
     ShowStakeAccount {
         pubkey: Pubkey,
@@ -300,7 +295,7 @@ pub enum CliCommand {
     WithdrawStake {
         stake_account_pubkey: Pubkey,
         destination_account_pubkey: Pubkey,
-        lamports: u64,
+        amount: SpendAmount,
         withdraw_authority: SignerIndex,
         custodian: Option<SignerIndex>,
         sign_only: bool,
@@ -309,6 +304,7 @@ pub enum CliCommand {
         nonce_account: Option<Pubkey>,
         nonce_authority: SignerIndex,
         memo: Option<String>,
+        seed: Option<String>,
         fee_payer: SignerIndex,
     },
     // Validator Info Commands
@@ -361,8 +357,6 @@ pub enum CliCommand {
     // Wallet Commands
     Address,
     Airdrop {
-        faucet_host: Option<IpAddr>,
-        faucet_port: u16,
         pubkey: Option<Pubkey>,
         lamports: u64,
     },
@@ -453,12 +447,13 @@ pub struct CliConfig<'a> {
     pub websocket_url: String,
     pub signers: Vec<&'a dyn Signer>,
     pub keypair_path: String,
-    pub rpc_client: Option<RpcClient>,
+    pub rpc_client: Option<Arc<RpcClient>>,
     pub rpc_timeout: Duration,
     pub verbose: bool,
     pub output_format: OutputFormat,
     pub commitment: CommitmentConfig,
     pub send_transaction_config: RpcSendTransactionConfig,
+    pub confirm_transaction_initial_timeout: Duration,
     pub address_labels: HashMap<String, String>,
 }
 
@@ -603,6 +598,9 @@ impl Default for CliConfig<'_> {
             output_format: OutputFormat::Display,
             commitment: CommitmentConfig::confirmed(),
             send_transaction_config: RpcSendTransactionConfig::default(),
+            confirm_transaction_initial_timeout: Duration::from_secs(
+                u64::from_str(DEFAULT_CONFIRM_TX_TIMEOUT_SECONDS).unwrap(),
+            ),
             address_labels: HashMap::new(),
         }
     }
@@ -719,7 +717,7 @@ pub fn parse_command(
         }
         // Stake Commands
         ("create-stake-account", Some(matches)) => {
-            parse_stake_create_account(matches, default_signer, wallet_manager)
+            parse_create_stake_account(matches, default_signer, wallet_manager)
         }
         ("delegate-stake", Some(matches)) => {
             parse_stake_delegate_stake(matches, default_signer, wallet_manager)
@@ -784,20 +782,6 @@ pub fn parse_command(
             signers: vec![default_signer.signer_from_path(matches, wallet_manager)?],
         }),
         ("airdrop", Some(matches)) => {
-            let faucet_port = matches
-                .value_of("faucet_port")
-                .ok_or_else(|| CliError::BadParameter("Missing faucet port".to_string()))?
-                .parse()
-                .map_err(|err| CliError::BadParameter(format!("Invalid faucet port: {}", err)))?;
-
-            let faucet_host = matches
-                .value_of("faucet_host")
-                .map(|faucet_host| {
-                    solana_net_utils::parse_host(faucet_host).map_err(|err| {
-                        CliError::BadParameter(format!("Invalid faucet host: {}", err))
-                    })
-                })
-                .transpose()?;
             let pubkey = pubkey_of_signer(matches, "to", wallet_manager)?;
             let signers = if pubkey.is_some() {
                 vec![]
@@ -806,12 +790,7 @@ pub fn parse_command(
             };
             let lamports = lamports_of_sol(matches, "amount").unwrap();
             Ok(CliCommandInfo {
-                command: CliCommand::Airdrop {
-                    faucet_host,
-                    faucet_port,
-                    pubkey,
-                    lamports,
-                },
+                command: CliCommand::Airdrop { pubkey, lamports },
                 signers,
             })
         }
@@ -878,7 +857,7 @@ pub fn parse_command(
                 signers: vec![],
             })
         }
-        ("pay", Some(matches)) | ("transfer", Some(matches)) => {
+        ("transfer", Some(matches)) => {
             let amount = SpendAmount::new_from_matches(matches, "amount");
             let to = pubkey_of_signer(matches, "to", wallet_manager)?.unwrap();
             let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
@@ -1008,7 +987,6 @@ fn process_create_address_with_seed(
 fn process_airdrop(
     rpc_client: &RpcClient,
     config: &CliConfig,
-    faucet_addr: &SocketAddr,
     pubkey: &Option<Pubkey>,
     lamports: u64,
 ) -> ProcessResult {
@@ -1018,14 +996,13 @@ fn process_airdrop(
         config.pubkey()?
     };
     println!(
-        "Requesting airdrop of {} from {}",
+        "Requesting airdrop of {}",
         build_balance_message(lamports, false, true),
-        faucet_addr
     );
 
     let pre_balance = rpc_client.get_balance(&pubkey)?;
 
-    let result = request_and_confirm_airdrop(&rpc_client, faucet_addr, &pubkey, lamports);
+    let result = request_and_confirm_airdrop(rpc_client, config, &pubkey, lamports);
     if let Ok(signature) = result {
         let signature_cli_message = log_instruction_custom_error::<SystemError>(result, &config)?;
         println!("{}", signature_cli_message);
@@ -1152,7 +1129,7 @@ fn process_show_account(
             pubkey: account_pubkey.to_string(),
             account: UiAccount::encode(
                 account_pubkey,
-                account,
+                &account,
                 UiAccountEncoding::Base64,
                 None,
                 None,
@@ -1314,17 +1291,16 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         println_name_value("Commitment:", &config.commitment.commitment.to_string());
     }
 
-    let mut _rpc_client;
     let rpc_client = if config.rpc_client.is_none() {
-        _rpc_client = RpcClient::new_with_timeout_and_commitment(
+        Arc::new(RpcClient::new_with_timeouts_and_commitment(
             config.json_rpc_url.to_string(),
             config.rpc_timeout,
             config.commitment,
-        );
-        &_rpc_client
+            config.confirm_transaction_initial_timeout,
+        ))
     } else {
         // Primarily for testing
-        config.rpc_client.as_ref().unwrap()
+        config.rpc_client.as_ref().unwrap().clone()
     };
 
     match &config.command {
@@ -1415,9 +1391,23 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::WaitForMaxStake { max_stake_percent } => {
             process_wait_for_max_stake(&rpc_client, config, *max_stake_percent)
         }
-        CliCommand::ShowValidators { use_lamports_unit } => {
-            process_show_validators(&rpc_client, config, *use_lamports_unit)
-        }
+        CliCommand::ShowValidators {
+            use_lamports_unit,
+            sort_order,
+            reverse_sort,
+            number_validators,
+            keep_unstaked_delinquents,
+            delinquent_slot_distance,
+        } => process_show_validators(
+            &rpc_client,
+            config,
+            *use_lamports_unit,
+            *sort_order,
+            *reverse_sort,
+            *number_validators,
+            *keep_unstaked_delinquents,
+            *delinquent_slot_distance,
+        ),
         CliCommand::Supply { print_accounts } => {
             process_supply(&rpc_client, config, *print_accounts)
         }
@@ -1522,7 +1512,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             use_deprecated_loader,
             allow_excessive_balance,
         } => process_deploy(
-            &rpc_client,
+            rpc_client,
             config,
             program_location,
             *address,
@@ -1530,7 +1520,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *allow_excessive_balance,
         ),
         CliCommand::Program(program_subcommand) => {
-            process_program_subcommand(&rpc_client, config, program_subcommand)
+            process_program_subcommand(rpc_client, config, program_subcommand)
         }
 
         // Stake Commands
@@ -1578,6 +1568,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             nonce_account,
             nonce_authority,
             memo,
+            seed,
             fee_payer,
         } => process_deactivate_stake_account(
             &rpc_client,
@@ -1590,6 +1581,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *nonce_account,
             *nonce_authority,
             memo.as_ref(),
+            seed.as_ref(),
             *fee_payer,
         ),
         CliCommand::DelegateStake {
@@ -1684,9 +1676,10 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *use_lamports_unit,
             *with_rewards,
         ),
-        CliCommand::ShowStakeHistory { use_lamports_unit } => {
-            process_show_stake_history(&rpc_client, config, *use_lamports_unit)
-        }
+        CliCommand::ShowStakeHistory {
+            use_lamports_unit,
+            limit_results,
+        } => process_show_stake_history(&rpc_client, config, *use_lamports_unit, *limit_results),
         CliCommand::StakeAuthorize {
             stake_account_pubkey,
             ref new_authorizations,
@@ -1740,7 +1733,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::WithdrawStake {
             stake_account_pubkey,
             destination_account_pubkey,
-            lamports,
+            amount,
             withdraw_authority,
             custodian,
             sign_only,
@@ -1749,13 +1742,14 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             ref nonce_account,
             nonce_authority,
             memo,
+            seed,
             fee_payer,
         } => process_withdraw_stake(
             &rpc_client,
             config,
             &stake_account_pubkey,
             &destination_account_pubkey,
-            *lamports,
+            *amount,
             *withdraw_authority,
             *custodian,
             *sign_only,
@@ -1764,6 +1758,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             nonce_account.as_ref(),
             *nonce_authority,
             memo.as_ref(),
+            seed.as_ref(),
             *fee_payer,
         ),
 
@@ -1877,27 +1872,8 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         // Wallet Commands
 
         // Request an airdrop from Solana Faucet;
-        CliCommand::Airdrop {
-            faucet_host,
-            faucet_port,
-            pubkey,
-            lamports,
-        } => {
-            let faucet_addr = SocketAddr::new(
-                faucet_host.unwrap_or_else(|| {
-                    let faucet_host = Url::parse(&config.json_rpc_url)
-                        .unwrap()
-                        .host()
-                        .unwrap()
-                        .to_string();
-                    solana_net_utils::parse_host(&faucet_host).unwrap_or_else(|err| {
-                        panic!("Unable to resolve {}: {}", faucet_host, err);
-                    })
-                }),
-                *faucet_port,
-            );
-
-            process_airdrop(&rpc_client, config, &faucet_addr, pubkey, *lamports)
+        CliCommand::Airdrop { pubkey, lamports } => {
+            process_airdrop(&rpc_client, config, pubkey, *lamports)
         }
         // Check client balance
         CliCommand::Balance {
@@ -1963,67 +1939,32 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
     }
 }
 
-// Quick and dirty Keypair that assumes the client will do retries but not update the
-// blockhash. If the client updates the blockhash, the signature will be invalid.
-struct FaucetKeypair {
-    transaction: Transaction,
-}
-
-impl FaucetKeypair {
-    fn new_keypair(
-        faucet_addr: &SocketAddr,
-        to_pubkey: &Pubkey,
-        lamports: u64,
-        blockhash: Hash,
-    ) -> Result<Self, FaucetError> {
-        let transaction = request_airdrop_transaction(faucet_addr, to_pubkey, lamports, blockhash)?;
-        Ok(Self { transaction })
-    }
-
-    fn airdrop_transaction(&self) -> Transaction {
-        self.transaction.clone()
-    }
-}
-
-impl Signer for FaucetKeypair {
-    /// Return the public key of the keypair used to sign votes
-    fn pubkey(&self) -> Pubkey {
-        self.transaction.message().account_keys[0]
-    }
-
-    fn try_pubkey(&self) -> Result<Pubkey, SignerError> {
-        Ok(self.pubkey())
-    }
-
-    fn sign_message(&self, _msg: &[u8]) -> Signature {
-        self.transaction.signatures[0]
-    }
-
-    fn try_sign_message(&self, message: &[u8]) -> Result<Signature, SignerError> {
-        Ok(self.sign_message(message))
-    }
-}
-
 pub fn request_and_confirm_airdrop(
     rpc_client: &RpcClient,
-    faucet_addr: &SocketAddr,
+    config: &CliConfig,
     to_pubkey: &Pubkey,
     lamports: u64,
 ) -> ClientResult<Signature> {
-    let (blockhash, _fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let keypair = {
-        let mut retries = 5;
-        loop {
-            let result = FaucetKeypair::new_keypair(faucet_addr, to_pubkey, lamports, blockhash);
-            if result.is_ok() || retries == 0 {
-                break result;
-            }
-            retries -= 1;
-            sleep(Duration::from_secs(1));
-        }
-    }?;
-    let tx = keypair.airdrop_transaction();
-    rpc_client.send_and_confirm_transaction_with_spinner(&tx)
+    let (recent_blockhash, _fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let signature =
+        rpc_client.request_airdrop_with_blockhash(to_pubkey, lamports, &recent_blockhash)?;
+    rpc_client.confirm_transaction_with_spinner(
+        &signature,
+        &recent_blockhash,
+        config.commitment,
+    )?;
+    Ok(signature)
+}
+
+fn common_error_adapter<E>(ix_error: &InstructionError) -> Option<E>
+where
+    E: 'static + std::error::Error + DecodeError<E> + FromPrimitive,
+{
+    if let InstructionError::Custom(code) = ix_error {
+        E::decode_custom_error_to_enum(*code)
+    } else {
+        None
+    }
 }
 
 pub fn log_instruction_custom_error<E>(
@@ -2032,15 +1973,24 @@ pub fn log_instruction_custom_error<E>(
 ) -> ProcessResult
 where
     E: 'static + std::error::Error + DecodeError<E> + FromPrimitive,
+{
+    log_instruction_custom_error_ex::<E, _>(result, config, common_error_adapter)
+}
+
+pub fn log_instruction_custom_error_ex<E, F>(
+    result: ClientResult<Signature>,
+    config: &CliConfig,
+    error_adapter: F,
+) -> ProcessResult
+where
+    E: 'static + std::error::Error + DecodeError<E> + FromPrimitive,
+    F: Fn(&InstructionError) -> Option<E>,
 {
     match result {
         Err(err) => {
-            if let ClientErrorKind::TransactionError(TransactionError::InstructionError(
-                _,
-                InstructionError::Custom(code),
-            )) = err.kind()
-            {
-                if let Some(specific_error) = E::decode_custom_error_to_enum(*code) {
+            let maybe_tx_err = err.get_transaction_error();
+            if let Some(TransactionError::InstructionError(_, ix_error)) = maybe_tx_err {
+                if let Some(specific_error) = error_adapter(&ix_error) {
                     return Err(specific_error.into());
                 }
             }
@@ -2078,22 +2028,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
         .stake_subcommands()
         .subcommand(
             SubCommand::with_name("airdrop")
-                .about("Request lamports")
-                .arg(
-                    Arg::with_name("faucet_host")
-                        .long("faucet-host")
-                        .value_name("URL")
-                        .takes_value(true)
-                        .help("Faucet host to use [default: the --url host]"),
-                )
-                .arg(
-                    Arg::with_name("faucet_port")
-                        .long("faucet-port")
-                        .value_name("PORT_NUMBER")
-                        .takes_value(true)
-                        .default_value(solana_faucet::faucet::FAUCET_PORT_STR)
-                        .help("Faucet port to use"),
-                )
+                .about("Request SOL from a faucet")
                 .arg(
                     Arg::with_name("amount")
                         .index(1)
@@ -2235,28 +2170,6 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .help("Use the designated program id, even if the account already holds a large balance of SOL")
                 ),
         )
-        .subcommand(
-            SubCommand::with_name("pay")
-                .about("Deprecated alias for the transfer command")
-                .arg(
-                    pubkey!(Arg::with_name("to")
-                        .index(1)
-                        .value_name("RECIPIENT_ADDRESS")
-                        .required(true),
-                        "The account address of recipient. "),
-                )
-                .arg(
-                    Arg::with_name("amount")
-                        .index(2)
-                        .value_name("AMOUNT")
-                        .takes_value(true)
-                        .validator(is_amount_or_all)
-                        .required(true)
-                        .help("The amount to send, in SOL; accepts keyword ALL"),
-                )
-                .offline_args()
-                .nonce_args(false)
-        )
         .subcommand(
             SubCommand::with_name("resolve-signer")
                 .about("Checks that a signer is valid, and returns its specific path; useful for signers that may be specified generally, eg. usb://ledger")
@@ -2273,6 +2186,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
         .subcommand(
             SubCommand::with_name("transfer")
                 .about("Transfer funds between system accounts")
+                .alias("pay")
                 .arg(
                     pubkey!(Arg::with_name("to")
                         .index(1)
@@ -2326,7 +2240,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                 )
                 .offline_args()
                 .nonce_args(false)
-		.arg(memo_arg())
+                .arg(memo_arg())
                 .arg(fee_payer_arg()),
         )
         .subcommand(
@@ -2399,10 +2313,7 @@ mod tests {
         let default_keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &default_keypair_file).unwrap();
 
-        let default_signer = DefaultSigner {
-            arg_name: "keypair".to_string(),
-            path: default_keypair_file,
-        };
+        let default_signer = DefaultSigner::new("keypair", &default_keypair_file);
 
         let signer_info = default_signer
             .generate_unique_signers(vec![], &matches, &mut None)
@@ -2480,10 +2391,7 @@ mod tests {
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
         let keypair = read_keypair_file(&keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: keypair_file.clone(),
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &keypair_file);
         // Test Airdrop Subcommand
         let test_airdrop =
             test_commands
@@ -2493,8 +2401,6 @@ mod tests {
             parse_command(&test_airdrop, &default_signer, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::Airdrop {
-                    faucet_host: None,
-                    faucet_port: solana_faucet::faucet::FAUCET_PORT,
                     pubkey: Some(pubkey),
                     lamports: 50_000_000_000,
                 },
@@ -2657,7 +2563,7 @@ mod tests {
             }
         );
 
-        // Test ResolveSigner Subcommand, KeypairUrl::Filepath
+        // Test ResolveSigner Subcommand, SignerSource::Filepath
         let test_resolve_signer =
             test_commands
                 .clone()
@@ -2669,7 +2575,7 @@ mod tests {
                 signers: vec![],
             }
         );
-        // Test ResolveSigner Subcommand, KeypairUrl::Pubkey (Presigner)
+        // Test ResolveSigner Subcommand, SignerSource::Pubkey (Presigner)
         let test_resolve_signer =
             test_commands
                 .clone()
@@ -2688,7 +2594,7 @@ mod tests {
     fn test_cli_process_command() {
         // Success cases
         let mut config = CliConfig {
-            rpc_client: Some(RpcClient::new_mock("succeeds".to_string())),
+            rpc_client: Some(Arc::new(RpcClient::new_mock("succeeds".to_string()))),
             json_rpc_url: "http://127.0.0.1:8899".to_string(),
             ..CliConfig::default()
         };
@@ -2788,7 +2694,7 @@ mod tests {
         config.command = CliCommand::WithdrawStake {
             stake_account_pubkey,
             destination_account_pubkey: to_pubkey,
-            lamports: 100,
+            amount: SpendAmount::All,
             withdraw_authority: 0,
             custodian: None,
             sign_only: false,
@@ -2797,6 +2703,7 @@ mod tests {
             nonce_account: None,
             nonce_authority: 0,
             memo: None,
+            seed: None,
             fee_payer: 0,
         };
         config.signers = vec![&keypair];
@@ -2813,6 +2720,7 @@ mod tests {
             nonce_account: None,
             nonce_authority: 0,
             memo: None,
+            seed: None,
             fee_payer: 0,
         };
         let result = process_command(&config);
@@ -2855,7 +2763,7 @@ mod tests {
         };
         config.signers = vec![&keypair, &merge_stake_account];
         let result = process_command(&config);
-        assert!(dbg!(result).is_ok());
+        assert!(result.is_ok());
 
         config.command = CliCommand::GetSlot;
         assert_eq!(process_command(&config).unwrap(), "0");
@@ -2880,21 +2788,19 @@ mod tests {
         let to = solana_sdk::pubkey::new_rand();
         config.signers = vec![&keypair];
         config.command = CliCommand::Airdrop {
-            faucet_host: None,
-            faucet_port: 1234,
             pubkey: Some(to),
             lamports: 50,
         };
         assert!(process_command(&config).is_ok());
 
         // sig_not_found case
-        config.rpc_client = Some(RpcClient::new_mock("sig_not_found".to_string()));
+        config.rpc_client = Some(Arc::new(RpcClient::new_mock("sig_not_found".to_string())));
         let missing_signature = Signature::new(&bs58::decode("5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW").into_vec().unwrap());
         config.command = CliCommand::Confirm(missing_signature);
         assert_eq!(process_command(&config).unwrap(), "Not found");
 
         // Tx error case
-        config.rpc_client = Some(RpcClient::new_mock("account_in_use".to_string()));
+        config.rpc_client = Some(Arc::new(RpcClient::new_mock("account_in_use".to_string())));
         let any_signature = Signature::new(&bs58::decode(SIGNATURE).into_vec().unwrap());
         config.command = CliCommand::Confirm(any_signature);
         assert_eq!(
@@ -2903,11 +2809,9 @@ mod tests {
         );
 
         // Failure cases
-        config.rpc_client = Some(RpcClient::new_mock("fails".to_string()));
+        config.rpc_client = Some(Arc::new(RpcClient::new_mock("fails".to_string())));
 
         config.command = CliCommand::Airdrop {
-            faucet_host: None,
-            faucet_port: 1234,
             pubkey: None,
             lamports: 50,
         };
@@ -2975,7 +2879,7 @@ mod tests {
         mocks.insert(RpcRequest::GetAccountInfo, account_info_response);
         let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
 
-        config.rpc_client = Some(rpc_client);
+        config.rpc_client = Some(Arc::new(rpc_client));
         let default_keypair = Keypair::new();
         config.signers = vec![&default_keypair];
 
@@ -3015,10 +2919,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let default_keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &default_keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: default_keypair_file.clone(),
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
 
         //Test Transfer Subcommand, SOL
         let from_keypair = keypair_from_seed(&[0u8; 32]).unwrap();

cli/src/cluster_query.rs

@@ -25,10 +25,11 @@ use solana_client::{
     rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClient},
     rpc_config::{
         RpcAccountInfoConfig, RpcConfirmedBlockConfig, RpcConfirmedTransactionConfig,
-        RpcLargestAccountsConfig, RpcLargestAccountsFilter, RpcProgramAccountsConfig,
-        RpcTransactionLogsConfig, RpcTransactionLogsFilter,
+        RpcGetVoteAccountsConfig, RpcLargestAccountsConfig, RpcLargestAccountsFilter,
+        RpcProgramAccountsConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter,
     },
     rpc_filter,
+    rpc_request::DELINQUENT_VALIDATOR_SLOT_DISTANCE,
     rpc_response::SlotInfo,
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
@@ -46,9 +47,10 @@ use solana_sdk::{
     rent::Rent,
     rpc_port::DEFAULT_RPC_PORT_STR,
     signature::Signature,
-    system_instruction, system_program,
+    slot_history, system_instruction, system_program,
     sysvar::{
         self,
+        slot_history::SlotHistory,
         stake_history::{self},
     },
     timing,
@@ -60,7 +62,6 @@ use solana_vote_program::vote_state::VoteState;
 use std::{
     collections::{BTreeMap, HashMap, VecDeque},
     fmt,
-    net::SocketAddr,
     str::FromStr,
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -175,7 +176,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .takes_value(true)
                     .value_name("EPOCH")
                     .validator(is_epoch)
-                    .help("Epoch to show leader schedule for. (default: current)")
+                    .help("Epoch to show leader schedule for. [default: current]")
             )
         )
         .subcommand(
@@ -349,6 +350,57 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
+                )
+                .arg(
+                    Arg::with_name("number")
+                        .long("number")
+                        .short("n")
+                        .takes_value(false)
+                        .help("Number the validators"),
+                )
+                .arg(
+                    Arg::with_name("reverse")
+                        .long("reverse")
+                        .short("r")
+                        .takes_value(false)
+                        .help("Reverse order while sorting"),
+                )
+                .arg(
+                    Arg::with_name("sort")
+                        .long("sort")
+                        .takes_value(true)
+                        .possible_values(&[
+                            "delinquent",
+                            "commission",
+                            "credits",
+                            "identity",
+                            "last-vote",
+                            "root",
+                            "skip-rate",
+                            "stake",
+                            "vote-account",
+                        ])
+                        .default_value("stake")
+                        .help("Sort order (does not affect JSON output)"),
+                )
+                .arg(
+                    Arg::with_name("keep_unstaked_delinquents")
+                        .long("keep-unstaked-delinquents")
+                        .takes_value(false)
+                        .help("Don't discard unstaked, delinquent validators")
+                )
+                .arg(
+                    Arg::with_name("delinquent_slot_distance")
+                        .long("delinquent-slot-distance")
+                        .takes_value(true)
+                        .value_name("SLOT_DISTANCE")
+                        .validator(is_slot)
+                        .help(
+                            concatcp!(
+                                "Minimum slot distance from the tip to consider a validator delinquent. [default: ",
+                                DELINQUENT_VALIDATOR_SLOT_DISTANCE,
+                                "]",
+                        ))
                 ),
         )
         .subcommand(
@@ -582,9 +634,33 @@ pub fn parse_show_stakes(
 
 pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
+    let number_validators = matches.is_present("number");
+    let reverse_sort = matches.is_present("reverse");
+    let keep_unstaked_delinquents = matches.is_present("keep_unstaked_delinquents");
+    let delinquent_slot_distance = value_of(matches, "delinquent_slot_distance");
+
+    let sort_order = match value_t_or_exit!(matches, "sort", String).as_str() {
+        "delinquent" => CliValidatorsSortOrder::Delinquent,
+        "commission" => CliValidatorsSortOrder::Commission,
+        "credits" => CliValidatorsSortOrder::EpochCredits,
+        "identity" => CliValidatorsSortOrder::Identity,
+        "last-vote" => CliValidatorsSortOrder::LastVote,
+        "root" => CliValidatorsSortOrder::Root,
+        "skip-rate" => CliValidatorsSortOrder::SkipRate,
+        "stake" => CliValidatorsSortOrder::Stake,
+        "vote-account" => CliValidatorsSortOrder::VoteAccount,
+        _ => unreachable!(),
+    };
 
     Ok(CliCommandInfo {
-        command: CliCommand::ShowValidators { use_lamports_unit },
+        command: CliCommand::ShowValidators {
+            use_lamports_unit,
+            sort_order,
+            reverse_sort,
+            number_validators,
+            keep_unstaked_delinquents,
+            delinquent_slot_distance,
+        },
         signers: vec![],
     })
 }
@@ -885,18 +961,19 @@ pub fn process_fees(
                 *recent_blockhash,
                 fee_calculator.lamports_per_signature,
                 None,
+                None,
             )
         } else {
             CliFees::none()
         }
     } else {
-        let result = rpc_client.get_recent_blockhash_with_commitment(config.commitment)?;
-        let (recent_blockhash, fee_calculator, last_valid_slot) = result.value;
+        let result = rpc_client.get_fees_with_commitment(config.commitment)?;
         CliFees::some(
             result.context.slot,
-            recent_blockhash,
-            fee_calculator.lamports_per_signature,
-            Some(last_valid_slot),
+            result.value.blockhash,
+            result.value.fee_calculator.lamports_per_signature,
+            None,
+            Some(result.value.last_valid_block_height),
         )
     };
     Ok(config.output_format.formatted_string(&fees))
@@ -1023,9 +1100,15 @@ pub fn process_get_epoch_info(rpc_client: &RpcClient, config: &CliConfig) -> Pro
             (secs as u64).saturating_mul(1000).checked_div(slots)
         })
         .unwrap_or(clock::DEFAULT_MS_PER_SLOT);
+    let start_block_time = rpc_client
+        .get_block_time(epoch_info.absolute_slot - epoch_info.slot_index)
+        .ok();
+    let current_block_time = rpc_client.get_block_time(epoch_info.absolute_slot).ok();
     let epoch_info = CliEpochInfo {
         epoch_info,
         average_slot_time_ms,
+        start_block_time,
+        current_block_time,
     };
     Ok(config.output_format.formatted_string(&epoch_info))
 }
@@ -1041,8 +1124,8 @@ pub fn process_get_slot(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessR
 }
 
 pub fn process_get_block_height(rpc_client: &RpcClient, _config: &CliConfig) -> ProcessResult {
-    let epoch_info = rpc_client.get_epoch_info()?;
-    Ok(epoch_info.block_height.to_string())
+    let block_height = rpc_client.get_block_height()?;
+    Ok(block_height.to_string())
 }
 
 pub fn parse_show_block_production(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
@@ -1069,8 +1152,6 @@ pub fn process_show_block_production(
         return Err(format!("Epoch {} is in the future", epoch).into());
     }
 
-    let minimum_ledger_slot = rpc_client.minimum_ledger_slot()?;
-
     let first_slot_in_epoch = epoch_schedule.get_first_slot_in_epoch(epoch);
     let end_slot = std::cmp::min(
         epoch_info.absolute_slot,
@@ -1083,32 +1164,60 @@ pub fn process_show_block_production(
         first_slot_in_epoch
     };
 
-    if minimum_ledger_slot > end_slot {
-        return Err(format!(
-            "Ledger data not available for slots {} to {} (minimum ledger slot is {})",
-            start_slot, end_slot, minimum_ledger_slot
-        )
-        .into());
-    }
-
-    if minimum_ledger_slot > start_slot {
-        println!(
-            "\n{}",
-            style(format!(
-                "Note: Requested start slot was {} but minimum ledger slot is {}",
-                start_slot, minimum_ledger_slot
-            ))
-            .italic(),
-        );
-        start_slot = minimum_ledger_slot;
-    }
-
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message(&format!(
         "Fetching confirmed blocks between slots {} and {}...",
         start_slot, end_slot
     ));
-    let confirmed_blocks = rpc_client.get_confirmed_blocks(start_slot, Some(end_slot))?;
+
+    let slot_history_account = rpc_client
+        .get_account_with_commitment(&sysvar::slot_history::id(), CommitmentConfig::finalized())?
+        .value
+        .unwrap();
+
+    let slot_history: SlotHistory = from_account(&slot_history_account).ok_or_else(|| {
+        CliError::RpcRequestError("Failed to deserialize slot history".to_string())
+    })?;
+
+    let (confirmed_blocks, start_slot) =
+        if start_slot >= slot_history.oldest() && end_slot <= slot_history.newest() {
+            // Fast, more reliable path using the SlotHistory sysvar
+
+            let confirmed_blocks: Vec<_> = (start_slot..=end_slot)
+                .filter(|slot| slot_history.check(*slot) == slot_history::Check::Found)
+                .collect();
+            (confirmed_blocks, start_slot)
+        } else {
+            // Slow, less reliable path using `getBlocks`.
+            //
+            // "less reliable" because if the RPC node has holds in its ledger then the block production data will be
+            // incorrect.  This condition currently can't be detected over RPC
+            //
+
+            let minimum_ledger_slot = rpc_client.minimum_ledger_slot()?;
+            if minimum_ledger_slot > end_slot {
+                return Err(format!(
+                    "Ledger data not available for slots {} to {} (minimum ledger slot is {})",
+                    start_slot, end_slot, minimum_ledger_slot
+                )
+                .into());
+            }
+
+            if minimum_ledger_slot > start_slot {
+                progress_bar.println(format!(
+                    "{}",
+                    style(format!(
+                        "Note: Requested start slot was {} but minimum ledger slot is {}",
+                        start_slot, minimum_ledger_slot
+                    ))
+                    .italic(),
+                ));
+                start_slot = minimum_ledger_slot;
+            }
+
+            let confirmed_blocks = rpc_client.get_confirmed_blocks(start_slot, Some(end_slot))?;
+            (confirmed_blocks, start_slot)
+        };
 
     let start_slot_index = (start_slot - first_slot_in_epoch) as usize;
     let end_slot_index = (end_slot - first_slot_in_epoch) as usize;
@@ -1576,40 +1685,14 @@ pub fn process_live_slots(config: &CliConfig) -> ProcessResult {
 pub fn process_show_gossip(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
     let cluster_nodes = rpc_client.get_cluster_nodes()?;
 
-    fn format_port(addr: Option<SocketAddr>) -> String {
-        addr.map(|addr| addr.port().to_string())
-            .unwrap_or_else(|| "none".to_string())
-    }
-
-    let s: Vec<_> = cluster_nodes
+    let nodes: Vec<_> = cluster_nodes
         .into_iter()
-        .map(|node| {
-            format!(
-                "{:15} | {:44} | {:6} | {:5} | {:21} | {}",
-                node.gossip
-                    .map(|addr| addr.ip().to_string())
-                    .unwrap_or_else(|| "none".to_string()),
-                format_labeled_address(&node.pubkey, &config.address_labels),
-                format_port(node.gossip),
-                format_port(node.tpu),
-                node.rpc
-                    .map(|addr| addr.to_string())
-                    .unwrap_or_else(|| "none".to_string()),
-                node.version.unwrap_or_else(|| "unknown".to_string()),
-            )
-        })
+        .map(|node| CliGossipNode::new(node, &config.address_labels))
         .collect();
 
-    Ok(format!(
-        "IP Address      | Node identifier                              \
-         | Gossip | TPU   | RPC Address           | Version\n\
-         ----------------+----------------------------------------------+\
-         --------+-------+-----------------------+----------------\n\
-         {}\n\
-         Nodes: {}",
-        s.join("\n"),
-        s.len(),
-    ))
+    Ok(config
+        .output_format
+        .formatted_string(&CliGossipNodes(nodes)))
 }
 
 pub fn process_show_stakes(
@@ -1624,11 +1707,11 @@ pub fn process_show_stakes(
     progress_bar.set_message("Fetching stake accounts...");
 
     let mut program_accounts_config = RpcProgramAccountsConfig {
-        filters: None,
         account_config: RpcAccountInfoConfig {
             encoding: Some(solana_account_decoder::UiAccountEncoding::Base64),
             ..RpcAccountInfoConfig::default()
         },
+        ..RpcProgramAccountsConfig::default()
     };
 
     if let Some(vote_account_pubkeys) = vote_account_pubkeys {
@@ -1730,10 +1813,42 @@ pub fn process_show_validators(
     rpc_client: &RpcClient,
     config: &CliConfig,
     use_lamports_unit: bool,
+    validators_sort_order: CliValidatorsSortOrder,
+    validators_reverse_sort: bool,
+    number_validators: bool,
+    keep_unstaked_delinquents: bool,
+    delinquent_slot_distance: Option<Slot>,
 ) -> ProcessResult {
+    let progress_bar = new_spinner_progress_bar();
+    progress_bar.set_message("Fetching vote accounts...");
     let epoch_info = rpc_client.get_epoch_info()?;
-    let vote_accounts = rpc_client.get_vote_accounts()?;
+    let vote_accounts = rpc_client.get_vote_accounts_with_config(RpcGetVoteAccountsConfig {
+        keep_unstaked_delinquents: Some(keep_unstaked_delinquents),
+        delinquent_slot_distance,
+        ..RpcGetVoteAccountsConfig::default()
+    })?;
 
+    progress_bar.set_message("Fetching block production...");
+    let skip_rate: HashMap<_, _> = rpc_client
+        .get_block_production()
+        .ok()
+        .map(|result| {
+            result
+                .value
+                .by_identity
+                .into_iter()
+                .map(|(identity, (leader_slots, blocks_produced))| {
+                    (
+                        identity,
+                        100. * (leader_slots.saturating_sub(blocks_produced)) as f64
+                            / leader_slots as f64,
+                    )
+                })
+                .collect()
+        })
+        .unwrap_or_default();
+
+    progress_bar.set_message("Fetching version information...");
     let mut node_version = HashMap::new();
     let unknown_version = "unknown".to_string();
     for contact_info in rpc_client.get_cluster_nodes()? {
@@ -1745,6 +1860,8 @@ pub fn process_show_validators(
         );
     }
 
+    progress_bar.finish_and_clear();
+
     let total_active_stake = vote_accounts
         .current
         .iter()
@@ -1759,9 +1876,8 @@ pub fn process_show_validators(
         .sum();
     let total_current_stake = total_active_stake - total_delinquent_stake;
 
-    let mut current = vote_accounts.current;
-    current.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
-    let current_validators: Vec<CliValidator> = current
+    let current_validators: Vec<CliValidator> = vote_accounts
+        .current
         .iter()
         .map(|vote_account| {
             CliValidator::new(
@@ -1771,22 +1887,23 @@ pub fn process_show_validators(
                     .get(&vote_account.node_pubkey)
                     .unwrap_or(&unknown_version)
                     .clone(),
+                skip_rate.get(&vote_account.node_pubkey).cloned(),
                 &config.address_labels,
             )
         })
         .collect();
-    let mut delinquent = vote_accounts.delinquent;
-    delinquent.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
-    let delinquent_validators: Vec<CliValidator> = delinquent
+    let delinquent_validators: Vec<CliValidator> = vote_accounts
+        .delinquent
         .iter()
         .map(|vote_account| {
-            CliValidator::new(
+            CliValidator::new_delinquent(
                 vote_account,
                 epoch_info.epoch,
                 node_version
                     .get(&vote_account.node_pubkey)
                     .unwrap_or(&unknown_version)
                     .clone(),
+                skip_rate.get(&vote_account.node_pubkey).cloned(),
                 &config.address_labels,
             )
         })
@@ -1808,12 +1925,43 @@ pub fn process_show_validators(
         entry.delinquent_active_stake += validator.activated_stake;
     }
 
+    let validators: Vec<_> = current_validators
+        .into_iter()
+        .chain(delinquent_validators.into_iter())
+        .collect();
+
+    let (average_skip_rate, average_stake_weighted_skip_rate) = {
+        let mut skip_rate_len = 0;
+        let mut skip_rate_sum = 0.;
+        let mut skip_rate_weighted_sum = 0.;
+        for validator in validators.iter() {
+            if let Some(skip_rate) = validator.skip_rate {
+                skip_rate_sum += skip_rate;
+                skip_rate_len += 1;
+                skip_rate_weighted_sum += skip_rate * validator.activated_stake as f64;
+            }
+        }
+
+        if skip_rate_len > 0 && total_active_stake > 0 {
+            (
+                skip_rate_sum / skip_rate_len as f64,
+                skip_rate_weighted_sum / total_active_stake as f64,
+            )
+        } else {
+            (100., 100.) // Impossible?
+        }
+    };
+
     let cli_validators = CliValidators {
         total_active_stake,
         total_current_stake,
         total_delinquent_stake,
-        current_validators,
-        delinquent_validators,
+        validators,
+        average_skip_rate,
+        average_stake_weighted_skip_rate,
+        validators_sort_order,
+        validators_reverse_sort,
+        number_validators,
         stake_by_version,
         use_lamports_unit,
     };
@@ -2013,10 +2161,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner {
-            path: default_keypair_file,
-            arg_name: String::new(),
-        };
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
 
         let test_cluster_version = test_commands
             .clone()

cli/src/feature.rs

@@ -10,6 +10,7 @@ use solana_cli_output::{QuietDisplay, VerboseDisplay};
 use solana_client::{client_error::ClientError, rpc_client::RpcClient};
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
+    account::Account,
     clock::Slot,
     feature::{self, Feature},
     feature_set::FEATURE_NAMES,
@@ -312,6 +313,31 @@ fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<boo
     Ok(feature_activation_allowed)
 }
 
+fn status_from_account(account: Account) -> Option<CliFeatureStatus> {
+    feature::from_account(&account).map(|feature| match feature.activated_at {
+        None => CliFeatureStatus::Pending,
+        Some(activation_slot) => CliFeatureStatus::Active(activation_slot),
+    })
+}
+
+fn get_feature_status(
+    rpc_client: &RpcClient,
+    feature_id: &Pubkey,
+) -> Result<Option<CliFeatureStatus>, Box<dyn std::error::Error>> {
+    rpc_client
+        .get_account(feature_id)
+        .map(status_from_account)
+        .map_err(|e| e.into())
+}
+
+pub fn get_feature_is_active(
+    rpc_client: &RpcClient,
+    feature_id: &Pubkey,
+) -> Result<bool, Box<dyn std::error::Error>> {
+    get_feature_status(rpc_client, feature_id)
+        .map(|status| matches!(status, Some(CliFeatureStatus::Active(_))))
+}
+
 fn process_status(
     rpc_client: &RpcClient,
     config: &CliConfig,
@@ -327,11 +353,7 @@ fn process_status(
         let feature_id = &feature_ids[i];
         let feature_name = FEATURE_NAMES.get(feature_id).unwrap();
         if let Some(account) = account {
-            if let Some(feature) = feature::from_account(&account) {
-                let feature_status = match feature.activated_at {
-                    None => CliFeatureStatus::Pending,
-                    Some(activation_slot) => CliFeatureStatus::Active(activation_slot),
-                };
+            if let Some(feature_status) = status_from_account(account) {
                 features.push(CliFeature {
                     id: feature_id.to_string(),
                     description: feature_name.to_string(),

cli/src/lib.rs

@@ -19,6 +19,9 @@ macro_rules! pubkey {
     };
 }
 
+#[macro_use]
+extern crate const_format;
+
 extern crate serde_derive;
 
 pub mod checks;
@@ -29,7 +32,6 @@ pub mod inflation;
 pub mod memo;
 pub mod nonce;
 pub mod program;
-pub mod send_tpu;
 pub mod spend_utils;
 pub mod stake;
 pub mod test_utils;

cli/src/main.rs

@@ -10,7 +10,7 @@ use solana_clap_utils::{
 };
 use solana_cli::cli::{
     app, parse_command, process_command, CliCommandInfo, CliConfig, SettingType,
-    DEFAULT_RPC_TIMEOUT_SECONDS,
+    DEFAULT_CONFIRM_TX_TIMEOUT_SECONDS, DEFAULT_RPC_TIMEOUT_SECONDS,
 };
 use solana_cli_config::{Config, CONFIG_FILE};
 use solana_cli_output::{display::println_name_value, OutputFormat};
@@ -167,6 +167,11 @@ pub fn parse_args<'a>(
     let rpc_timeout = value_t_or_exit!(matches, "rpc_timeout", u64);
     let rpc_timeout = Duration::from_secs(rpc_timeout);
 
+    let confirm_transaction_initial_timeout =
+        value_t_or_exit!(matches, "confirm_transaction_initial_timeout", u64);
+    let confirm_transaction_initial_timeout =
+        Duration::from_secs(confirm_transaction_initial_timeout);
+
     let (_, websocket_url) = CliConfig::compute_websocket_url_setting(
         matches.value_of("websocket_url").unwrap_or(""),
         &config.websocket_url,
@@ -179,10 +184,7 @@ pub fn parse_args<'a>(
         &config.keypair_path,
     );
 
-    let default_signer = DefaultSigner {
-        arg_name: default_signer_arg_name,
-        path: default_signer_path.clone(),
-    };
+    let default_signer = DefaultSigner::new(default_signer_arg_name, &default_signer_path);
 
     let CliCommandInfo {
         command,
@@ -238,6 +240,7 @@ pub fn parse_args<'a>(
                 preflight_commitment: Some(commitment.commitment),
                 ..RpcSendTransactionConfig::default()
             },
+            confirm_transaction_initial_timeout,
             address_labels,
         },
         signers,
@@ -353,6 +356,16 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             .hidden(true)
             .help("Timeout value for RPC requests"),
     )
+    .arg(
+        Arg::with_name("confirm_transaction_initial_timeout")
+            .long("confirm-timeout")
+            .value_name("SECONDS")
+            .takes_value(true)
+            .default_value(DEFAULT_CONFIRM_TX_TIMEOUT_SECONDS)
+            .global(true)
+            .hidden(true)
+            .help("Timeout value for initial transaction status"),
+    )
     .subcommand(
         SubCommand::with_name("config")
             .about("Solana command-line tool configuration settings")

cli/src/nonce.rs

@@ -1,9 +1,10 @@
 use crate::{
     checks::{check_account_for_fee_with_commitment, check_unique_pubkeys},
     cli::{
-        log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError,
-        ProcessResult,
+        log_instruction_custom_error, log_instruction_custom_error_ex, CliCommand, CliCommandInfo,
+        CliConfig, CliError, ProcessResult,
     },
+    feature::get_feature_is_active,
     memo::WithMemo,
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
 };
@@ -12,7 +13,7 @@ use solana_clap_utils::{
     input_parsers::*,
     input_validators::*,
     keypair::{DefaultSigner, SignerIndex},
-    memo::MEMO_ARG,
+    memo::{memo_arg, MEMO_ARG},
     nonce::*,
 };
 use solana_cli_output::CliNonceAccount;
@@ -20,16 +21,19 @@ use solana_client::{nonce_utils::*, rpc_client::RpcClient};
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     account::Account,
+    feature_set::merge_nonce_error_into_system_error,
     hash::Hash,
+    instruction::InstructionError,
     message::Message,
     nonce::{self, State},
     pubkey::Pubkey,
     system_instruction::{
         advance_nonce_account, authorize_nonce_account, create_nonce_account,
-        create_nonce_account_with_seed, withdraw_nonce_account, NonceError, SystemError,
+        create_nonce_account_with_seed, instruction_to_nonce_error, withdraw_nonce_account,
+        NonceError, SystemError,
     },
     system_program,
-    transaction::Transaction,
+    transaction::{Transaction, TransactionError},
 };
 use std::sync::Arc;
 
@@ -56,7 +60,8 @@ impl NonceSubCommands for App<'_, '_> {
                         .required(true),
                         "Account to be granted authority of the nonce account. "),
                 )
-                .arg(nonce_authority_arg()),
+                .arg(nonce_authority_arg())
+                .arg(memo_arg()),
         )
         .subcommand(
             SubCommand::with_name("create-nonce-account")
@@ -91,7 +96,8 @@ impl NonceSubCommands for App<'_, '_> {
                         .value_name("STRING")
                         .takes_value(true)
                         .help("Seed for address generation; if specified, the resulting account will be at a derived address of the NONCE_ACCOUNT pubkey")
-                ),
+                )
+                .arg(memo_arg()),
         )
         .subcommand(
             SubCommand::with_name("nonce")
@@ -115,7 +121,8 @@ impl NonceSubCommands for App<'_, '_> {
                         .required(true),
                         "Address of the nonce account. "),
                 )
-                .arg(nonce_authority_arg()),
+                .arg(nonce_authority_arg())
+                .arg(memo_arg()),
         )
         .subcommand(
             SubCommand::with_name("nonce-account")
@@ -161,7 +168,8 @@ impl NonceSubCommands for App<'_, '_> {
                         .validator(is_amount)
                         .help("The amount to withdraw from the nonce account, in SOL"),
                 )
-                .arg(nonce_authority_arg()),
+                .arg(nonce_authority_arg())
+                .arg(memo_arg()),
         )
     }
 }
@@ -363,8 +371,21 @@ pub fn process_authorize_nonce_account(
         &tx.message,
         config.commitment,
     )?;
+    let merge_errors =
+        get_feature_is_active(rpc_client, &merge_nonce_error_into_system_error::id())?;
     let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
-    log_instruction_custom_error::<NonceError>(result, &config)
+
+    if merge_errors {
+        log_instruction_custom_error::<SystemError>(result, config)
+    } else {
+        log_instruction_custom_error_ex::<NonceError, _>(result, config, |ix_error| {
+            if let InstructionError::Custom(_) = ix_error {
+                instruction_to_nonce_error(ix_error, merge_errors)
+            } else {
+                None
+            }
+        })
+    }
 }
 
 pub fn process_create_nonce_account(
@@ -448,8 +469,40 @@ pub fn process_create_nonce_account(
 
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
+    let merge_errors =
+        get_feature_is_active(rpc_client, &merge_nonce_error_into_system_error::id())?;
     let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
-    log_instruction_custom_error::<SystemError>(result, &config)
+
+    let err_ix_index = if let Err(err) = &result {
+        err.get_transaction_error().and_then(|tx_err| {
+            if let TransactionError::InstructionError(ix_index, _) = tx_err {
+                Some(ix_index)
+            } else {
+                None
+            }
+        })
+    } else {
+        None
+    };
+
+    match err_ix_index {
+        // SystemInstruction::InitializeNonceAccount failed
+        Some(1) => {
+            if merge_errors {
+                log_instruction_custom_error::<SystemError>(result, config)
+            } else {
+                log_instruction_custom_error_ex::<NonceError, _>(result, config, |ix_error| {
+                    if let InstructionError::Custom(_) = ix_error {
+                        instruction_to_nonce_error(ix_error, merge_errors)
+                    } else {
+                        None
+                    }
+                })
+            }
+        }
+        // SystemInstruction::CreateAccount{,WithSeed} failed
+        _ => log_instruction_custom_error::<SystemError>(result, config),
+    }
 }
 
 pub fn process_get_nonce(
@@ -502,8 +555,21 @@ pub fn process_new_nonce(
         &tx.message,
         config.commitment,
     )?;
+    let merge_errors =
+        get_feature_is_active(rpc_client, &merge_nonce_error_into_system_error::id())?;
     let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
-    log_instruction_custom_error::<SystemError>(result, &config)
+
+    if merge_errors {
+        log_instruction_custom_error::<SystemError>(result, config)
+    } else {
+        log_instruction_custom_error_ex::<NonceError, _>(result, config, |ix_error| {
+            if let InstructionError::Custom(_) = ix_error {
+                instruction_to_nonce_error(ix_error, merge_errors)
+            } else {
+                None
+            }
+        })
+    }
 }
 
 pub fn process_show_nonce_account(
@@ -565,8 +631,21 @@ pub fn process_withdraw_from_nonce_account(
         &tx.message,
         config.commitment,
     )?;
+    let merge_errors =
+        get_feature_is_active(rpc_client, &merge_nonce_error_into_system_error::id())?;
     let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
-    log_instruction_custom_error::<NonceError>(result, &config)
+
+    if merge_errors {
+        log_instruction_custom_error::<SystemError>(result, config)
+    } else {
+        log_instruction_custom_error_ex::<NonceError, _>(result, config, |ix_error| {
+            if let InstructionError::Custom(_) = ix_error {
+                instruction_to_nonce_error(ix_error, merge_errors)
+            } else {
+                None
+            }
+        })
+    }
 }
 
 #[cfg(test)]
@@ -596,10 +675,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner {
-            path: default_keypair_file.clone(),
-            arg_name: String::new(),
-        };
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let nonce_account_keypair = Keypair::new();
         write_keypair(&nonce_account_keypair, tmp_file.as_file_mut()).unwrap();

cli/src/program.rs

@@ -1,4 +1,3 @@
-use crate::send_tpu::{get_leader_tpus, send_transaction_tpu};
 use crate::{
     checks::*,
     cli::{
@@ -6,7 +5,6 @@ use crate::{
         ProcessResult,
     },
 };
-use bincode::serialize;
 use bip39::{Language, Mnemonic, MnemonicType, Seed};
 use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
 use log::*;
@@ -25,7 +23,7 @@ use solana_client::{
     rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig},
     rpc_filter::{Memcmp, MemcmpEncodedBytes, RpcFilterType},
     rpc_request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS,
-    rpc_response::RpcLeaderSchedule,
+    tpu_client::{TpuClient, TpuClientConfig},
 };
 use solana_rbpf::vm::{Config, Executable};
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
@@ -51,20 +49,17 @@ use solana_sdk::{
 };
 use solana_transaction_status::TransactionConfirmationStatus;
 use std::{
-    cmp::min,
     collections::HashMap,
     error,
     fs::File,
     io::{Read, Write},
-    net::UdpSocket,
     path::PathBuf,
     sync::Arc,
     thread::sleep,
-    time::{Duration, Instant},
+    time::Duration,
 };
 
 const DATA_CHUNK_SIZE: usize = 229; // Keep program chunks under PACKET_DATA_SIZE
-const NUM_TPU_LEADERS: u64 = 2;
 
 #[derive(Debug, PartialEq)]
 pub enum ProgramCliCommand {
@@ -622,7 +617,7 @@ pub fn parse_program_subcommand(
 }
 
 pub fn process_program_subcommand(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
     config: &CliConfig,
     program_subcommand: &ProgramCliCommand,
 ) -> ProcessResult {
@@ -638,7 +633,7 @@ pub fn process_program_subcommand(
             max_len,
             allow_excessive_balance,
         } => process_program_deploy(
-            &rpc_client,
+            rpc_client,
             config,
             program_location,
             *program_signer_index,
@@ -657,7 +652,7 @@ pub fn process_program_subcommand(
             buffer_authority_signer_index,
             max_len,
         } => process_write_buffer(
-            &rpc_client,
+            rpc_client,
             config,
             program_location,
             *buffer_signer_index,
@@ -746,7 +741,7 @@ fn get_default_program_keypair(program_location: &Option<String>) -> Keypair {
 /// Deploy using upgradeable loader
 #[allow(clippy::too_many_arguments)]
 fn process_program_deploy(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
     config: &CliConfig,
     program_location: &Option<String>,
     program_signer_index: Option<SignerIndex>,
@@ -892,7 +887,7 @@ fn process_program_deploy(
 
     let result = if do_deploy {
         do_process_program_write_and_deploy(
-            rpc_client,
+            rpc_client.clone(),
             config,
             &program_data,
             buffer_data_len,
@@ -907,7 +902,7 @@ fn process_program_deploy(
         )
     } else {
         do_process_program_upgrade(
-            rpc_client,
+            rpc_client.clone(),
             config,
             &program_data,
             &program_pubkey,
@@ -918,7 +913,7 @@ fn process_program_deploy(
     };
     if result.is_ok() && is_final {
         process_set_authority(
-            rpc_client,
+            &rpc_client,
             config,
             Some(program_pubkey),
             None,
@@ -933,7 +928,7 @@ fn process_program_deploy(
 }
 
 fn process_write_buffer(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
     config: &CliConfig,
     program_location: &str,
     buffer_signer_index: Option<SignerIndex>,
@@ -1107,6 +1102,7 @@ fn get_buffers(
                 data_slice: Some(UiDataSliceConfig { offset: 0, length }),
                 ..RpcAccountInfoConfig::default()
             },
+            ..RpcProgramAccountsConfig::default()
         },
     )?;
     Ok(results)
@@ -1411,6 +1407,7 @@ fn process_close(
                     data_slice: Some(UiDataSliceConfig { offset: 0, length }),
                     ..RpcAccountInfoConfig::default()
                 },
+                ..RpcProgramAccountsConfig::default()
             },
         )?;
 
@@ -1450,7 +1447,7 @@ fn process_close(
 
 /// Deploy using non-upgradeable loader
 pub fn process_deploy(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
     config: &CliConfig,
     program_location: &str,
     buffer_signer_index: Option<SignerIndex>,
@@ -1495,7 +1492,7 @@ pub fn process_deploy(
 
 #[allow(clippy::too_many_arguments)]
 fn do_process_program_write_and_deploy(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
     config: &CliConfig,
     program_data: &[u8],
     buffer_data_len: usize,
@@ -1633,7 +1630,7 @@ fn do_process_program_write_and_deploy(
         messages.push(message);
     }
 
-    check_payer(rpc_client, config, balance_needed, &messages)?;
+    check_payer(&rpc_client, config, balance_needed, &messages)?;
 
     send_deploy_messages(
         rpc_client,
@@ -1660,7 +1657,7 @@ fn do_process_program_write_and_deploy(
 }
 
 fn do_process_program_upgrade(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
     config: &CliConfig,
     program_data: &[u8],
     program_id: &Pubkey,
@@ -1756,7 +1753,7 @@ fn do_process_program_upgrade(
     );
     messages.push(&final_message);
 
-    check_payer(rpc_client, config, balance_needed, &messages)?;
+    check_payer(&rpc_client, config, balance_needed, &messages)?;
     send_deploy_messages(
         rpc_client,
         config,
@@ -1782,7 +1779,7 @@ fn read_and_verify_elf(program_location: &str) -> Result<Vec<u8>, Box<dyn std::e
         .map_err(|err| format!("Unable to read program file: {}", err))?;
 
     // Verify the program
-    Executable::<BpfError, ThisInstructionMeter>::from_elf(
+    <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
         &program_data,
         Some(|x| bpf_verifier::check(x)),
         Config::default(),
@@ -1861,7 +1858,7 @@ fn check_payer(
 }
 
 fn send_deploy_messages(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
     config: &CliConfig,
     initial_message: &Option<Message>,
     write_messages: &Option<Vec<Message>>,
@@ -1909,7 +1906,8 @@ fn send_deploy_messages(
             }
 
             send_and_confirm_transactions_with_spinner(
-                &rpc_client,
+                rpc_client.clone(),
+                &config.websocket_url,
                 write_transactions,
                 &[payer_signer, write_signer],
                 config.commitment,
@@ -1978,7 +1976,8 @@ fn report_ephemeral_mnemonic(words: usize, mnemonic: bip39::Mnemonic) {
 }
 
 fn send_and_confirm_transactions_with_spinner<T: Signers>(
-    rpc_client: &RpcClient,
+    rpc_client: Arc<RpcClient>,
+    websocket_url: &str,
     mut transactions: Vec<Transaction>,
     signer_keys: &T,
     commitment: CommitmentConfig,
@@ -1986,39 +1985,19 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
 ) -> Result<(), Box<dyn error::Error>> {
     let progress_bar = new_spinner_progress_bar();
     let mut send_retries = 5;
-    let mut leader_schedule: Option<RpcLeaderSchedule> = None;
-    let mut leader_schedule_epoch = 0;
-    let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-    let cluster_nodes = rpc_client.get_cluster_nodes().ok();
 
+    progress_bar.set_message("Finding leader nodes...");
+    let tpu_client = TpuClient::new(
+        rpc_client.clone(),
+        websocket_url,
+        TpuClientConfig::default(),
+    )?;
     loop {
-        progress_bar.set_message("Finding leader nodes...");
-        let epoch_info = rpc_client.get_epoch_info()?;
-        let mut slot = epoch_info.absolute_slot;
-        let mut last_epoch_fetch = Instant::now();
-        if epoch_info.epoch > leader_schedule_epoch || leader_schedule.is_none() {
-            leader_schedule = rpc_client.get_leader_schedule(Some(epoch_info.absolute_slot))?;
-            leader_schedule_epoch = epoch_info.epoch;
-        }
-
-        let mut tpu_addresses = get_leader_tpus(
-            min(epoch_info.slot_index + 1, epoch_info.slots_in_epoch),
-            NUM_TPU_LEADERS,
-            leader_schedule.as_ref(),
-            cluster_nodes.as_ref(),
-        );
-
         // Send all transactions
         let mut pending_transactions = HashMap::new();
         let num_transactions = transactions.len();
         for transaction in transactions {
-            if !tpu_addresses.is_empty() {
-                let wire_transaction =
-                    serialize(&transaction).expect("serialization should succeed");
-                for tpu_address in &tpu_addresses {
-                    send_transaction_tpu(&send_socket, &tpu_address, &wire_transaction);
-                }
-            } else {
+            if !tpu_client.send_transaction(&transaction) {
                 let _result = rpc_client
                     .send_transaction_with_config(
                         &transaction,
@@ -2038,22 +2017,11 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
 
             // Throttle transactions to about 100 TPS
             sleep(Duration::from_millis(10));
-
-            // Update leader periodically
-            if last_epoch_fetch.elapsed() > Duration::from_millis(400) {
-                let epoch_info = rpc_client.get_epoch_info()?;
-                last_epoch_fetch = Instant::now();
-                tpu_addresses = get_leader_tpus(
-                    min(epoch_info.slot_index + 1, epoch_info.slots_in_epoch),
-                    NUM_TPU_LEADERS,
-                    leader_schedule.as_ref(),
-                    cluster_nodes.as_ref(),
-                );
-            }
         }
 
         // Collect statuses for all the transactions, drop those that are confirmed
         loop {
+            let mut slot = 0;
             let pending_signatures = pending_transactions.keys().cloned().collect::<Vec<_>>();
             for pending_signatures_chunk in
                 pending_signatures.chunks(MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS)
@@ -2095,22 +2063,8 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
                 break;
             }
 
-            let epoch_info = rpc_client.get_epoch_info()?;
-            tpu_addresses = get_leader_tpus(
-                min(epoch_info.slot_index + 1, epoch_info.slots_in_epoch),
-                NUM_TPU_LEADERS,
-                leader_schedule.as_ref(),
-                cluster_nodes.as_ref(),
-            );
-
             for transaction in pending_transactions.values() {
-                if !tpu_addresses.is_empty() {
-                    let wire_transaction =
-                        serialize(&transaction).expect("serialization should succeed");
-                    for tpu_address in &tpu_addresses {
-                        send_transaction_tpu(&send_socket, &tpu_address, &wire_transaction);
-                    }
-                } else {
+                if !tpu_client.send_transaction(transaction) {
                     let _result = rpc_client
                         .send_transaction_with_config(
                             transaction,
@@ -2177,10 +2131,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: keypair_file.clone(),
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         let test_command = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2388,10 +2339,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: keypair_file.clone(),
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         // defaults
         let test_command = test_commands.clone().get_matches_from(vec![
@@ -2539,10 +2487,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: keypair_file.clone(),
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         let program_pubkey = Pubkey::new_unique();
         let new_authority_pubkey = Pubkey::new_unique();
@@ -2650,10 +2595,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: keypair_file.clone(),
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         let buffer_pubkey = Pubkey::new_unique();
         let new_authority_pubkey = Pubkey::new_unique();
@@ -2710,10 +2652,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: keypair_file,
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         // defaults
         let buffer_pubkey = Pubkey::new_unique();
@@ -2812,10 +2751,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner {
-            path: keypair_file.clone(),
-            arg_name: "".to_string(),
-        };
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         // defaults
         let buffer_pubkey = Pubkey::new_unique();
@@ -2933,7 +2869,7 @@ mod tests {
         write_keypair_file(&program_pubkey, &program_keypair_location).unwrap();
 
         let config = CliConfig {
-            rpc_client: Some(RpcClient::new_mock("".to_string())),
+            rpc_client: Some(Arc::new(RpcClient::new_mock("".to_string()))),
             command: CliCommand::Program(ProgramCliCommand::Deploy {
                 program_location: Some(program_location.to_str().unwrap().to_string()),
                 buffer_signer_index: None,

cli/src/send_tpu.rs

@@ -1,46 +0,0 @@
-use log::*;
-use solana_client::rpc_response::{RpcContactInfo, RpcLeaderSchedule};
-use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS;
-use std::net::{SocketAddr, UdpSocket};
-
-pub fn get_leader_tpus(
-    slot_index: u64,
-    num_leaders: u64,
-    leader_schedule: Option<&RpcLeaderSchedule>,
-    cluster_nodes: Option<&Vec<RpcContactInfo>>,
-) -> Vec<SocketAddr> {
-    let leaders: Vec<_> = (0..num_leaders)
-        .filter_map(|i| {
-            leader_schedule?
-                .iter()
-                .find(|(_pubkey, slots)| {
-                    slots.iter().any(|slot| {
-                        *slot as u64 == (slot_index + (i * NUM_CONSECUTIVE_LEADER_SLOTS))
-                    })
-                })
-                .and_then(|(pubkey, _)| {
-                    cluster_nodes?
-                        .iter()
-                        .find(|contact_info| contact_info.pubkey == *pubkey)
-                        .and_then(|contact_info| contact_info.tpu)
-                })
-        })
-        .collect();
-    let mut unique_leaders = vec![];
-    for leader in leaders.into_iter() {
-        if !unique_leaders.contains(&leader) {
-            unique_leaders.push(leader);
-        }
-    }
-    unique_leaders
-}
-
-pub fn send_transaction_tpu(
-    send_socket: &UdpSocket,
-    tpu_address: &SocketAddr,
-    wire_transaction: &[u8],
-) {
-    if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
-        warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
-    }
-}

cli/src/stake.rs

@@ -8,20 +8,20 @@ use crate::{
     nonce::check_nonce_account,
     spend_utils::{resolve_spend_tx_and_check_account_balances, SpendAmount},
 };
-use clap::{App, Arg, ArgGroup, ArgMatches, SubCommand};
+use clap::{value_t, App, Arg, ArgGroup, ArgMatches, SubCommand};
 use solana_clap_utils::{
     fee_payer::{fee_payer_arg, FEE_PAYER_ARG},
     input_parsers::*,
     input_validators::*,
     keypair::{DefaultSigner, SignerIndex},
-    memo::MEMO_ARG,
+    memo::{memo_arg, MEMO_ARG},
     nonce::*,
     offline::*,
     ArgConstant,
 };
 use solana_cli_output::{
     return_signers_with_config, CliEpochReward, CliStakeHistory, CliStakeHistoryEntry,
-    CliStakeState, CliStakeType, ReturnSignersConfig,
+    CliStakeState, CliStakeType, OutputFormat, ReturnSignersConfig,
 };
 use solana_client::{
     blockhash_query::BlockhashQuery, nonce_utils, rpc_client::RpcClient,
@@ -107,7 +107,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("stake_account")
                         .index(1)
-                        .value_name("ACCOUNT_KEYPAIR")
+                        .value_name("STAKE_ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
                         .validator(is_valid_signer)
@@ -133,7 +133,8 @@ impl StakeSubCommands for App<'_, '_> {
                         .long("seed")
                         .value_name("STRING")
                         .takes_value(true)
-                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the stake_account pubkey")
+                        .help("Seed for address generation; if specified, the resulting account \
+                               will be at a derived address of the STAKE_ACCOUNT_KEYPAIR pubkey")
                 )
                 .arg(
                     Arg::with_name("lockup_epoch")
@@ -177,6 +178,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("delegate-stake")
@@ -206,6 +208,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("stake-authorize")
@@ -237,6 +240,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .nonce_args(false)
                 .arg(fee_payer_arg())
                 .arg(custodian_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("deactivate-stake")
@@ -246,12 +250,21 @@ impl StakeSubCommands for App<'_, '_> {
                         .index(1)
                         .value_name("STAKE_ACCOUNT_ADDRESS")
                         .required(true),
-                        "Stake account to be deactivated. ")
+                        "Stake account to be deactivated (or base of derived address if --seed is used). ")
+                )
+                .arg(
+                    Arg::with_name("seed")
+                        .long("seed")
+                        .value_name("STRING")
+                        .takes_value(true)
+                        .help("Seed for address generation; if specified, the resulting account \
+                               will be at a derived address of STAKE_ACCOUNT_ADDRESS")
                 )
                 .arg(stake_authority_arg())
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("split-stake")
@@ -286,12 +299,14 @@ impl StakeSubCommands for App<'_, '_> {
                         .long("seed")
                         .value_name("STRING")
                         .takes_value(true)
-                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the SPLIT_STAKE_ACCOUNT pubkey")
+                        .help("Seed for address generation; if specified, the resulting account \
+                               will be at a derived address of SPLIT_STAKE_ACCOUNT")
                 )
                 .arg(stake_authority_arg())
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("merge-stake")
@@ -308,12 +323,14 @@ impl StakeSubCommands for App<'_, '_> {
                         .index(2)
                         .value_name("SOURCE_STAKE_ACCOUNT_ADDRESS")
                         .required(true),
-                        "Source stake account for the merge.  If successful, this stake account will no longer exist after the merge")
+                        "Source stake account for the merge.  If successful, this stake account \
+                         will no longer exist after the merge")
                 )
                 .arg(stake_authority_arg())
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("withdraw-stake")
@@ -323,7 +340,7 @@ impl StakeSubCommands for App<'_, '_> {
                         .index(1)
                         .value_name("STAKE_ACCOUNT_ADDRESS")
                         .required(true),
-                        "Stake account from which to withdraw")
+                        "Stake account from which to withdraw (or base of derived address if --seed is used). ")
                 )
                 .arg(
                     pubkey!(Arg::with_name("destination_account_pubkey")
@@ -337,15 +354,24 @@ impl StakeSubCommands for App<'_, '_> {
                         .index(3)
                         .value_name("AMOUNT")
                         .takes_value(true)
-                        .validator(is_amount)
+                        .validator(is_amount_or_all)
                         .required(true)
-                        .help("The amount to withdraw from the stake account, in SOL")
+                        .help("The amount to withdraw from the stake account, in SOL; accepts keyword ALL")
+                )
+                .arg(
+                    Arg::with_name("seed")
+                        .long("seed")
+                        .value_name("STRING")
+                        .takes_value(true)
+                        .help("Seed for address generation; if specified, the resulting account \
+                               will be at a derived address of STAKE_ACCOUNT_ADDRESS")
                 )
                 .arg(withdraw_authority_arg())
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
                 .arg(custodian_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("stake-set-lockup")
@@ -393,6 +419,7 @@ impl StakeSubCommands for App<'_, '_> {
                 .offline_args()
                 .nonce_args(false)
                 .arg(fee_payer_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("stake-account")
@@ -438,11 +465,24 @@ impl StakeSubCommands for App<'_, '_> {
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL")
                 )
+                .arg(
+                    Arg::with_name("limit")
+                        .long("limit")
+                        .takes_value(true)
+                        .value_name("NUM")
+                        .default_value("10")
+                        .validator(|s| {
+                            s.parse::<usize>()
+                                .map(|_| ())
+                                .map_err(|e| e.to_string())
+                        })
+                        .help("Display NUM recent epochs worth of stake history in text mode. 0 for all")
+                )
         )
     }
 }
 
-pub fn parse_stake_create_account(
+pub fn parse_create_stake_account(
     matches: &ArgMatches<'_>,
     default_signer: &DefaultSigner,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
@@ -480,9 +520,9 @@ pub fn parse_stake_create_account(
             staker,
             withdrawer,
             lockup: Lockup {
-                custodian,
-                epoch,
                 unix_timestamp,
+                epoch,
+                custodian,
             },
             amount,
             sign_only,
@@ -742,6 +782,7 @@ pub fn parse_stake_deactivate_stake(
     let blockhash_query = BlockhashQuery::new_from_matches(matches);
     let nonce_account = pubkey_of(matches, NONCE_ARG.name);
     let memo = matches.value_of(MEMO_ARG.name).map(String::from);
+    let seed = value_t!(matches, "seed", String).ok();
     let (stake_authority, stake_authority_pubkey) =
         signer_of(matches, STAKE_AUTHORITY_ARG.name, wallet_manager)?;
     let (nonce_authority, nonce_authority_pubkey) =
@@ -765,6 +806,7 @@ pub fn parse_stake_deactivate_stake(
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
             memo,
+            seed,
             fee_payer: signer_info.index_of(fee_payer_pubkey).unwrap(),
         },
         signers: signer_info.signers,
@@ -780,12 +822,13 @@ pub fn parse_stake_withdraw_stake(
         pubkey_of_signer(matches, "stake_account_pubkey", wallet_manager)?.unwrap();
     let destination_account_pubkey =
         pubkey_of_signer(matches, "destination_account_pubkey", wallet_manager)?.unwrap();
-    let lamports = lamports_of_sol(matches, "amount").unwrap();
+    let amount = SpendAmount::new_from_matches(matches, "amount");
     let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
     let dump_transaction_message = matches.is_present(DUMP_TRANSACTION_MESSAGE.name);
     let blockhash_query = BlockhashQuery::new_from_matches(matches);
     let nonce_account = pubkey_of(matches, NONCE_ARG.name);
     let memo = matches.value_of(MEMO_ARG.name).map(String::from);
+    let seed = value_t!(matches, "seed", String).ok();
     let (withdraw_authority, withdraw_authority_pubkey) =
         signer_of(matches, WITHDRAW_AUTHORITY_ARG.name, wallet_manager)?;
     let (nonce_authority, nonce_authority_pubkey) =
@@ -807,7 +850,7 @@ pub fn parse_stake_withdraw_stake(
         command: CliCommand::WithdrawStake {
             stake_account_pubkey,
             destination_account_pubkey,
-            lamports,
+            amount,
             withdraw_authority: signer_info.index_of(withdraw_authority_pubkey).unwrap(),
             sign_only,
             dump_transaction_message,
@@ -815,6 +858,7 @@ pub fn parse_stake_withdraw_stake(
             nonce_account,
             nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
             memo,
+            seed,
             fee_payer: signer_info.index_of(fee_payer_pubkey).unwrap(),
             custodian: custodian_pubkey.and_then(|_| signer_info.index_of(custodian_pubkey)),
         },
@@ -896,8 +940,12 @@ pub fn parse_show_stake_account(
 
 pub fn parse_show_stake_history(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
+    let limit_results = value_of(matches, "limit").unwrap();
     Ok(CliCommandInfo {
-        command: CliCommand::ShowStakeHistory { use_lamports_unit },
+        command: CliCommand::ShowStakeHistory {
+            use_lamports_unit,
+            limit_results,
+        },
         signers: vec![],
     })
 }
@@ -1134,13 +1182,21 @@ pub fn process_deactivate_stake_account(
     nonce_account: Option<Pubkey>,
     nonce_authority: SignerIndex,
     memo: Option<&String>,
+    seed: Option<&String>,
     fee_payer: SignerIndex,
 ) -> ProcessResult {
     let (recent_blockhash, fee_calculator) =
         blockhash_query.get_blockhash_and_fee_calculator(rpc_client, config.commitment)?;
     let stake_authority = config.signers[stake_authority];
+
+    let stake_account_address = if let Some(seed) = seed {
+        Pubkey::create_with_seed(&stake_account_pubkey, seed, &solana_stake_program::id())?
+    } else {
+        *stake_account_pubkey
+    };
+
     let ixs = vec![stake_instruction::deactivate_stake(
-        stake_account_pubkey,
+        &stake_account_address,
         &stake_authority.pubkey(),
     )]
     .with_memo(memo);
@@ -1196,7 +1252,7 @@ pub fn process_withdraw_stake(
     config: &CliConfig,
     stake_account_pubkey: &Pubkey,
     destination_account_pubkey: &Pubkey,
-    lamports: u64,
+    amount: SpendAmount,
     withdraw_authority: SignerIndex,
     custodian: Option<SignerIndex>,
     sign_only: bool,
@@ -1205,35 +1261,57 @@ pub fn process_withdraw_stake(
     nonce_account: Option<&Pubkey>,
     nonce_authority: SignerIndex,
     memo: Option<&String>,
+    seed: Option<&String>,
     fee_payer: SignerIndex,
 ) -> ProcessResult {
-    let (recent_blockhash, fee_calculator) =
-        blockhash_query.get_blockhash_and_fee_calculator(rpc_client, config.commitment)?;
     let withdraw_authority = config.signers[withdraw_authority];
     let custodian = custodian.map(|index| config.signers[index]);
 
-    let ixs = vec![stake_instruction::withdraw(
-        stake_account_pubkey,
-        &withdraw_authority.pubkey(),
-        destination_account_pubkey,
-        lamports,
-        custodian.map(|signer| signer.pubkey()).as_ref(),
-    )]
-    .with_memo(memo);
+    let stake_account_address = if let Some(seed) = seed {
+        Pubkey::create_with_seed(&stake_account_pubkey, seed, &solana_stake_program::id())?
+    } else {
+        *stake_account_pubkey
+    };
+
+    let (recent_blockhash, fee_calculator) =
+        blockhash_query.get_blockhash_and_fee_calculator(rpc_client, config.commitment)?;
 
     let fee_payer = config.signers[fee_payer];
     let nonce_authority = config.signers[nonce_authority];
 
-    let message = if let Some(nonce_account) = &nonce_account {
-        Message::new_with_nonce(
-            ixs,
-            Some(&fee_payer.pubkey()),
-            nonce_account,
-            &nonce_authority.pubkey(),
-        )
-    } else {
-        Message::new(&ixs, Some(&fee_payer.pubkey()))
+    let build_message = |lamports| {
+        let ixs = vec![stake_instruction::withdraw(
+            &stake_account_address,
+            &withdraw_authority.pubkey(),
+            destination_account_pubkey,
+            lamports,
+            custodian.map(|signer| signer.pubkey()).as_ref(),
+        )]
+        .with_memo(memo);
+
+        if let Some(nonce_account) = &nonce_account {
+            Message::new_with_nonce(
+                ixs,
+                Some(&fee_payer.pubkey()),
+                nonce_account,
+                &nonce_authority.pubkey(),
+            )
+        } else {
+            Message::new(&ixs, Some(&fee_payer.pubkey()))
+        }
     };
+
+    let (message, _) = resolve_spend_tx_and_check_account_balances(
+        rpc_client,
+        sign_only,
+        amount,
+        &fee_calculator,
+        &stake_account_address,
+        &fee_payer.pubkey(),
+        build_message,
+        config.commitment,
+    )?;
+
     let mut tx = Transaction::new_unsigned(message);
 
     if sign_only {
@@ -1734,7 +1812,7 @@ pub fn make_cli_reward(
         let rate_change = reward.amount as f64 / (reward.post_balance - reward.amount) as f64;
 
         let wallclock_epochs_per_year =
-            (SECONDS_PER_DAY * 356) as f64 / wallclock_epoch_duration as f64;
+            (SECONDS_PER_DAY * 365) as f64 / wallclock_epoch_duration as f64;
         let apr = rate_change * wallclock_epochs_per_year;
 
         Some(CliEpochReward {
@@ -1744,6 +1822,7 @@ pub fn make_cli_reward(
             post_balance: reward.post_balance,
             percent_change: rate_change * 100.0,
             apr: Some(apr * 100.0),
+            commission: reward.commission,
         })
     } else {
         None
@@ -1846,6 +1925,7 @@ pub fn process_show_stake_history(
     rpc_client: &RpcClient,
     config: &CliConfig,
     use_lamports_unit: bool,
+    limit_results: usize,
 ) -> ProcessResult {
     let stake_history_account = rpc_client.get_account(&stake_history::id())?;
     let stake_history =
@@ -1853,8 +1933,18 @@ pub fn process_show_stake_history(
             CliError::RpcRequestError("Failed to deserialize stake history".to_string())
         })?;
 
+    let limit_results = match config.output_format {
+        OutputFormat::Json | OutputFormat::JsonCompact => std::usize::MAX,
+        _ => {
+            if limit_results == 0 {
+                std::usize::MAX
+            } else {
+                limit_results
+            }
+        }
+    };
     let mut entries: Vec<CliStakeHistoryEntry> = vec![];
-    for entry in stake_history.deref() {
+    for entry in stake_history.deref().iter().take(limit_results) {
         entries.push(entry.into());
     }
     let stake_history_output = CliStakeHistory {
@@ -2023,10 +2113,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner {
-            path: default_keypair_file.clone(),
-            arg_name: String::new(),
-        };
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let stake_account_keypair = Keypair::new();
         write_keypair(&stake_account_keypair, tmp_file.as_file_mut()).unwrap();
@@ -3030,7 +3117,7 @@ mod tests {
                 command: CliCommand::WithdrawStake {
                     stake_account_pubkey,
                     destination_account_pubkey: stake_account_pubkey,
-                    lamports: 42_000_000_000,
+                    amount: SpendAmount::Some(42_000_000_000),
                     withdraw_authority: 0,
                     custodian: None,
                     sign_only: false,
@@ -3039,6 +3126,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 0,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
@@ -3062,7 +3150,7 @@ mod tests {
                 command: CliCommand::WithdrawStake {
                     stake_account_pubkey,
                     destination_account_pubkey: stake_account_pubkey,
-                    lamports: 42_000_000_000,
+                    amount: SpendAmount::Some(42_000_000_000),
                     withdraw_authority: 1,
                     custodian: None,
                     sign_only: false,
@@ -3071,6 +3159,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 0,
                 },
                 signers: vec![
@@ -3099,7 +3188,7 @@ mod tests {
                 command: CliCommand::WithdrawStake {
                     stake_account_pubkey,
                     destination_account_pubkey: stake_account_pubkey,
-                    lamports: 42_000_000_000,
+                    amount: SpendAmount::Some(42_000_000_000),
                     withdraw_authority: 0,
                     custodian: Some(1),
                     sign_only: false,
@@ -3108,6 +3197,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 0,
                 },
                 signers: vec![
@@ -3144,7 +3234,7 @@ mod tests {
                 command: CliCommand::WithdrawStake {
                     stake_account_pubkey,
                     destination_account_pubkey: stake_account_pubkey,
-                    lamports: 42_000_000_000,
+                    amount: SpendAmount::Some(42_000_000_000),
                     withdraw_authority: 0,
                     custodian: None,
                     sign_only: false,
@@ -3156,6 +3246,7 @@ mod tests {
                     nonce_account: Some(nonce_account),
                     nonce_authority: 1,
                     memo: None,
+                    seed: None,
                     fee_payer: 1,
                 },
                 signers: vec![
@@ -3185,6 +3276,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 0,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
@@ -3211,6 +3303,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 0,
                 },
                 signers: vec![
@@ -3247,6 +3340,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 0,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
@@ -3273,6 +3367,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 0,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
@@ -3309,6 +3404,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 1,
                 },
                 signers: vec![
@@ -3354,6 +3450,7 @@ mod tests {
                     nonce_account: Some(nonce_account),
                     nonce_authority: 2,
                     memo: None,
+                    seed: None,
                     fee_payer: 1,
                 },
                 signers: vec![
@@ -3384,6 +3481,7 @@ mod tests {
                     nonce_account: None,
                     nonce_authority: 0,
                     memo: None,
+                    seed: None,
                     fee_payer: 1,
                 },
                 signers: vec![

cli/src/vote.rs

@@ -82,7 +82,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .takes_value(true)
                         .help("Seed for address generation; if specified, the resulting account will be at a derived address of the VOTE ACCOUNT pubkey")
                 )
-		.arg(memo_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-authorize-voter")
@@ -109,7 +109,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .required(true),
                         "New authorized vote signer. "),
                 )
-		.arg(memo_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-authorize-withdrawer")
@@ -136,7 +136,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .required(true),
                         "New authorized withdrawer. "),
                 )
-		.arg(memo_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-update-validator")
@@ -166,7 +166,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .validator(is_valid_signer)
                         .help("Authorized withdrawer keypair"),
                 )
-		.arg(memo_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-update-commission")
@@ -196,7 +196,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .validator(is_valid_signer)
                         .help("Authorized withdrawer keypair"),
                 )
-		.arg(memo_arg())
+                .arg(memo_arg())
         )
         .subcommand(
             SubCommand::with_name("vote-account")
@@ -266,7 +266,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .validator(is_valid_signer)
                         .help("Authorized withdrawer [default: cli config keypair]"),
                 )
-		.arg(memo_arg())
+                .arg(memo_arg())
         )
     }
 }
@@ -826,10 +826,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner {
-            path: default_keypair_file.clone(),
-            arg_name: String::new(),
-        };
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
 
         let test_authorize_voter = test_commands.clone().get_matches_from(vec![
             "test",

cli/tests/nonce.rs

@@ -22,44 +22,38 @@ use solana_sdk::{
 #[test]
 fn test_nonce() {
     let mint_keypair = Keypair::new();
-    full_battery_tests(
-        TestValidator::with_no_fees(mint_keypair.pubkey()),
-        mint_keypair,
-        None,
-        false,
-    );
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
+
+    full_battery_tests(test_validator, None, false);
 }
 
 #[test]
 fn test_nonce_with_seed() {
     let mint_keypair = Keypair::new();
-    full_battery_tests(
-        TestValidator::with_no_fees(mint_keypair.pubkey()),
-        mint_keypair,
-        Some(String::from("seed")),
-        false,
-    );
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
+
+    full_battery_tests(test_validator, Some(String::from("seed")), false);
 }
 
 #[test]
 fn test_nonce_with_authority() {
     let mint_keypair = Keypair::new();
-    full_battery_tests(
-        TestValidator::with_no_fees(mint_keypair.pubkey()),
-        mint_keypair,
-        None,
-        true,
-    );
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
+
+    full_battery_tests(test_validator, None, true);
 }
 
 fn full_battery_tests(
     test_validator: TestValidator,
-    mint_keypair: Keypair,
     seed: Option<String>,
     use_nonce_authority: bool,
 ) {
-    let faucet_addr = run_local_faucet(mint_keypair, None);
-
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     let json_rpc_url = test_validator.rpc_url();
@@ -71,7 +65,7 @@ fn full_battery_tests(
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_payer,
         &config_payer.signers[0].pubkey(),
         2000,
     )
@@ -220,9 +214,9 @@ fn full_battery_tests(
 fn test_create_account_with_seed() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
-
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let offline_nonce_authority_signer = keypair_from_seed(&[1u8; 32]).unwrap();
     let online_nonce_creator_signer = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -233,14 +227,14 @@ fn test_create_account_with_seed() {
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &CliConfig::recent_for_tests(),
         &offline_nonce_authority_signer.pubkey(),
         42,
     )
     .unwrap();
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &CliConfig::recent_for_tests(),
         &online_nonce_creator_signer.pubkey(),
         4242,
     )

cli/tests/program.rs

@@ -28,8 +28,9 @@ fn test_cli_program_deploy_non_upgradeable() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -46,8 +47,6 @@ fn test_cli_program_deploy_non_upgradeable() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 4 * minimum_balance_for_rent_exemption, // min balance for rent exemption for three programs + leftover for tx processing
     };
@@ -104,8 +103,6 @@ fn test_cli_program_deploy_non_upgradeable() {
     let custom_address_keypair = Keypair::new();
     config.signers = vec![&custom_address_keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 2 * minimum_balance_for_rent_exemption, // Anything over minimum_balance_for_rent_exemption should trigger err
     };
@@ -147,8 +144,9 @@ fn test_cli_program_deploy_no_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -171,8 +169,6 @@ fn test_cli_program_deploy_no_authority() {
     let keypair = Keypair::new();
     config.json_rpc_url = test_validator.rpc_url();
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_programdata + minimum_balance_for_program,
     };
@@ -231,8 +227,9 @@ fn test_cli_program_deploy_with_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -256,8 +253,6 @@ fn test_cli_program_deploy_with_authority() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_programdata + minimum_balance_for_program,
     };
@@ -560,8 +555,9 @@ fn test_cli_program_write_buffer() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -586,8 +582,6 @@ fn test_cli_program_write_buffer() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -843,8 +837,9 @@ fn test_cli_program_set_buffer_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -864,8 +859,6 @@ fn test_cli_program_set_buffer_authority() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -957,8 +950,9 @@ fn test_cli_program_mismatch_buffer_authority() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -978,8 +972,6 @@ fn test_cli_program_mismatch_buffer_authority() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -1047,8 +1039,9 @@ fn test_cli_program_show() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1071,8 +1064,6 @@ fn test_cli_program_show() {
     // Airdrop
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };
@@ -1228,8 +1219,9 @@ fn test_cli_program_dump() {
     pathbuf.set_extension("so");
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1252,8 +1244,6 @@ fn test_cli_program_dump() {
     // Airdrop
     config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 100 * minimum_balance_for_buffer,
     };

cli/tests/request_airdrop.rs

@@ -10,15 +10,13 @@ use solana_sdk::{
 #[test]
 fn test_cli_request_airdrop() {
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
-
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let mut bob_config = CliConfig::recent_for_tests();
     bob_config.json_rpc_url = test_validator.rpc_url();
     bob_config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 50,
     };

cli/tests/stake.rs

@@ -26,8 +26,9 @@ use solana_stake_program::{
 #[test]
 fn test_stake_delegation_force() {
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -37,13 +38,8 @@ fn test_stake_delegation_force() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     // Create vote account
     let vote_keypair = Keypair::new();
@@ -119,8 +115,9 @@ fn test_seed_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -132,7 +129,7 @@ fn test_seed_stake_delegation_and_deactivation() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_validator,
         &config_validator.signers[0].pubkey(),
         100_000,
     )
@@ -192,6 +189,7 @@ fn test_seed_stake_delegation_and_deactivation() {
         nonce_account: None,
         nonce_authority: 0,
         memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
@@ -202,8 +200,9 @@ fn test_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -217,7 +216,7 @@ fn test_stake_delegation_and_deactivation() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_validator,
         &config_validator.signers[0].pubkey(),
         100_000,
     )
@@ -271,6 +270,7 @@ fn test_stake_delegation_and_deactivation() {
         nonce_account: None,
         nonce_authority: 0,
         memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
@@ -281,8 +281,9 @@ fn test_offline_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -307,7 +308,7 @@ fn test_offline_stake_delegation_and_deactivation() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_validator,
         &config_validator.signers[0].pubkey(),
         100_000,
     )
@@ -316,7 +317,7 @@ fn test_offline_stake_delegation_and_deactivation() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_offline,
         &config_offline.signers[0].pubkey(),
         100_000,
     )
@@ -392,6 +393,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         nonce_account: None,
         nonce_authority: 0,
         memo: None,
+        seed: None,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
@@ -410,6 +412,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         nonce_account: None,
         nonce_authority: 0,
         memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config_payer).unwrap();
@@ -420,8 +423,9 @@ fn test_nonced_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -435,13 +439,8 @@ fn test_nonced_stake_delegation_and_deactivation() {
         .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     // Create stake account
     let stake_keypair = Keypair::new();
@@ -529,6 +528,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();
@@ -539,8 +539,9 @@ fn test_stake_authorize() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -550,13 +551,8 @@ fn test_stake_authorize() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     let offline_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let mut config_offline = CliConfig::recent_for_tests();
@@ -569,7 +565,7 @@ fn test_stake_authorize() {
 
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &config_offline,
         &config_offline.signers[0].pubkey(),
         100_000,
     )
@@ -809,8 +805,9 @@ fn test_stake_authorize_with_fee_payer() {
     const SIG_FEE: u64 = 42;
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), SIG_FEE);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, SIG_FEE, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -836,13 +833,13 @@ fn test_stake_authorize_with_fee_payer() {
     config_offline.command = CliCommand::ClusterVersion;
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &default_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &default_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &payer_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_payer, &payer_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &payer_pubkey);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     check_ready(&rpc_client);
@@ -937,8 +934,9 @@ fn test_stake_split() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -957,16 +955,11 @@ fn test_stake_split() {
     config_offline.command = CliCommand::ClusterVersion;
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        500_000,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 500_000)
+        .unwrap();
     check_recent_balance(500_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
@@ -1084,8 +1077,9 @@ fn test_stake_set_lockup() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1104,16 +1098,11 @@ fn test_stake_set_lockup() {
     config_offline.command = CliCommand::ClusterVersion;
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        500_000,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 500_000)
+        .unwrap();
     check_recent_balance(500_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
@@ -1134,7 +1123,7 @@ fn test_stake_set_lockup() {
         stake_account: 1,
         seed: None,
         staker: Some(offline_pubkey),
-        withdrawer: Some(offline_pubkey),
+        withdrawer: Some(config.signers[0].pubkey()),
         lockup,
         amount: SpendAmount::Some(10 * minimum_stake_balance),
         sign_only: false,
@@ -1347,8 +1336,9 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     solana_logger::setup();
 
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -1366,16 +1356,11 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     // Verify that we cannot reach the cluster
     process_command(&config_offline).unwrap_err();
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        200_000,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 200_000)
+        .unwrap();
     check_recent_balance(200_000, &rpc_client, &config.signers[0].pubkey());
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 100_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
     check_recent_balance(100_000, &rpc_client, &offline_pubkey);
 
     // Create nonce account
@@ -1470,7 +1455,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     config_offline.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
         destination_account_pubkey: recipient_pubkey,
-        lamports: 42,
+        amount: SpendAmount::Some(42),
         withdraw_authority: 0,
         custodian: None,
         sign_only: true,
@@ -1479,6 +1464,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         memo: None,
+        seed: None,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
@@ -1488,7 +1474,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     config.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
         destination_account_pubkey: recipient_pubkey,
-        lamports: 42,
+        amount: SpendAmount::Some(42),
         withdraw_authority: 0,
         custodian: None,
         sign_only: false,
@@ -1500,6 +1486,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         memo: None,
+        seed: None,
         fee_payer: 0,
     };
     process_command(&config).unwrap();

cli/tests/transfer.rs

@@ -22,8 +22,9 @@ use solana_sdk::{
 fn test_transfer() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -38,7 +39,7 @@ fn test_transfer() {
     let sender_pubkey = config.signers[0].pubkey();
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 50_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
     check_recent_balance(50_000, &rpc_client, &sender_pubkey);
     check_recent_balance(0, &rpc_client, &recipient_pubkey);
 
@@ -94,7 +95,7 @@ fn test_transfer() {
     process_command(&offline).unwrap_err();
 
     let offline_pubkey = offline.signers[0].pubkey();
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_pubkey, 50).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &offline, &offline_pubkey, 50).unwrap();
     check_recent_balance(50, &rpc_client, &offline_pubkey);
 
     // Offline transfer
@@ -273,8 +274,9 @@ fn test_transfer() {
 fn test_transfer_multisession_signing() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let to_pubkey = Pubkey::new(&[1u8; 32]);
     let offline_from_signer = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -284,11 +286,16 @@ fn test_transfer_multisession_signing() {
     // Setup accounts
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_from_signer.pubkey(), 43)
-        .unwrap();
     request_and_confirm_airdrop(
         &rpc_client,
-        &faucet_addr,
+        &CliConfig::recent_for_tests(),
+        &offline_from_signer.pubkey(),
+        43,
+    )
+    .unwrap();
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &CliConfig::recent_for_tests(),
         &offline_fee_payer_signer.pubkey(),
         3,
     )
@@ -394,8 +401,9 @@ fn test_transfer_multisession_signing() {
 fn test_transfer_all() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -409,7 +417,7 @@ fn test_transfer_all() {
     let sender_pubkey = config.signers[0].pubkey();
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 50_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
     check_recent_balance(50_000, &rpc_client, &sender_pubkey);
     check_recent_balance(0, &rpc_client, &recipient_pubkey);
 
@@ -441,8 +449,9 @@ fn test_transfer_all() {
 fn test_transfer_unfunded_recipient() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -456,7 +465,7 @@ fn test_transfer_unfunded_recipient() {
     let sender_pubkey = config.signers[0].pubkey();
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 50_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
     check_recent_balance(50_000, &rpc_client, &sender_pubkey);
     check_recent_balance(0, &rpc_client, &recipient_pubkey);
 
@@ -488,8 +497,9 @@ fn test_transfer_unfunded_recipient() {
 fn test_transfer_with_seed() {
     solana_logger::setup();
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_custom_fees(mint_pubkey, 1, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -511,8 +521,8 @@ fn test_transfer_with_seed() {
     )
     .unwrap();
 
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &sender_pubkey, 1).unwrap();
-    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &derived_address, 50_000).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 1).unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &derived_address, 50_000).unwrap();
     check_recent_balance(1, &rpc_client, &sender_pubkey);
     check_recent_balance(50_000, &rpc_client, &derived_address);
     check_recent_balance(0, &rpc_client, &recipient_pubkey);

cli/tests/vote.rs

@@ -19,8 +19,9 @@ use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersion
 #[test]
 fn test_vote_authorize_and_withdraw() {
     let mint_keypair = Keypair::new();
-    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+    let mint_pubkey = mint_keypair.pubkey();
     let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator = TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr));
 
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
@@ -30,13 +31,8 @@ fn test_vote_authorize_and_withdraw() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
-    request_and_confirm_airdrop(
-        &rpc_client,
-        &faucet_addr,
-        &config.signers[0].pubkey(),
-        100_000,
-    )
-    .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 100_000)
+        .unwrap();
 
     // Create vote account
     let vote_account_keypair = Keypair::new();

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.6.5"
+version = "1.6.20"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -24,14 +24,14 @@ semver = "0.11.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.6.5" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-faucet = { path = "../faucet", version = "=1.6.5" }
-solana-net-utils = { path = "../net-utils", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
-solana-vote-program = { path = "../programs/vote", version = "=1.6.5" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.20" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-faucet = { path = "../faucet", version = "=1.6.20" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.20" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tungstenite = "0.10.1"
@@ -40,7 +40,7 @@ url = "2.1.1"
 [dev-dependencies]
 assert_matches = "1.3.0"
 jsonrpc-http-server = "17.0.0"
-solana-logger = { path = "../logger", version = "=1.6.5" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/blockhash_query.rs

@@ -361,7 +361,7 @@ mod tests {
         let nonce_pubkey = Pubkey::new(&[4u8; 32]);
         let rpc_nonce_account = UiAccount::encode(
             &nonce_pubkey,
-            nonce_account,
+            &nonce_account,
             UiAccountEncoding::Base64,
             None,
             None,

client/src/client_error.rs

@@ -1,5 +1,5 @@
 use {
-    crate::rpc_request,
+    crate::{rpc_request, rpc_response},
     solana_faucet::faucet::FaucetError,
     solana_sdk::{
         signature::SignerError, transaction::TransactionError, transport::TransportError,
@@ -30,6 +30,24 @@ pub enum ClientErrorKind {
     Custom(String),
 }
 
+impl ClientErrorKind {
+    pub fn get_transaction_error(&self) -> Option<TransactionError> {
+        match self {
+            Self::RpcError(rpc_request::RpcError::RpcResponseError {
+                data:
+                    rpc_request::RpcResponseErrorData::SendTransactionPreflightFailure(
+                        rpc_response::RpcSimulateTransactionResult {
+                            err: Some(tx_err), ..
+                        },
+                    ),
+                ..
+            }) => Some(tx_err.clone()),
+            Self::TransactionError(tx_err) => Some(tx_err.clone()),
+            _ => None,
+        }
+    }
+}
+
 impl From<TransportError> for ClientErrorKind {
     fn from(err: TransportError) -> Self {
         match err {
@@ -86,6 +104,10 @@ impl ClientError {
     pub fn kind(&self) -> &ClientErrorKind {
         &self.kind
     }
+
+    pub fn get_transaction_error(&self) -> Option<TransactionError> {
+        self.kind.get_transaction_error()
+    }
 }
 
 impl From<ClientErrorKind> for ClientError {

client/src/fees.rs

@@ -0,0 +1,9 @@
+use crate::{fee_calculator::FeeCalculator, hash::Hash};
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Fees {
+    pub blockhash: Hash,
+    pub fee_calculator: FeeCalculator,
+    pub last_valid_block_height: u64,
+}

client/src/http_sender.rs

@@ -7,7 +7,11 @@ use {
         rpc_sender::RpcSender,
     },
     log::*,
-    reqwest::{self, header::CONTENT_TYPE, StatusCode},
+    reqwest::{
+        self,
+        header::{CONTENT_TYPE, RETRY_AFTER},
+        StatusCode,
+    },
     std::{
         sync::{
             atomic::{AtomicU64, Ordering},
@@ -83,14 +87,24 @@ impl RpcSender for HttpSender {
                         if response.status() == StatusCode::TOO_MANY_REQUESTS
                             && too_many_requests_retries > 0
                         {
+                            let mut duration = Duration::from_millis(500);
+                            if let Some(retry_after) = response.headers().get(RETRY_AFTER) {
+                                if let Ok(retry_after) = retry_after.to_str() {
+                                    if let Ok(retry_after) = retry_after.parse::<u64>() {
+                                        if retry_after < 120 {
+                                            duration = Duration::from_secs(retry_after);
+                                        }
+                                    }
+                                }
+                            }
+
                             too_many_requests_retries -= 1;
                             debug!(
-                                "Server responded with {:?}, {} retries left",
-                                response, too_many_requests_retries
+                                "Too many requests: server responded with {:?}, {} retries left, pausing for {:?}",
+                                response, too_many_requests_retries, duration
                             );
 
-                            // Sleep for 500ms to give the server a break
-                            sleep(Duration::from_millis(500));
+                            sleep(duration);
                             continue;
                         }
                         return Err(response.error_for_status().unwrap_err().into());

client/src/lib.rs

@@ -18,3 +18,4 @@ pub mod rpc_request;
 pub mod rpc_response;
 pub mod rpc_sender;
 pub mod thin_client;
+pub mod tpu_client;

client/src/mock_sender.rs

@@ -124,6 +124,8 @@ impl RpcSender for MockSender {
             }
             RpcRequest::GetTransactionCount => Value::Number(Number::from(1234)),
             RpcRequest::GetSlot => Value::Number(Number::from(0)),
+            RpcRequest::GetMaxShredInsertSlot => Value::Number(Number::from(0)),
+            RpcRequest::RequestAirdrop => Value::String(Signature::new(&[8; 64]).to_string()),
             RpcRequest::SendTransaction => {
                 let signature = if self.url == "malicious" {
                     Signature::new(&[8; 64]).to_string()

client/src/pubsub_client.rs

@@ -3,7 +3,9 @@ use {
         rpc_config::{
             RpcSignatureSubscribeConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter,
         },
-        rpc_response::{Response as RpcResponse, RpcLogsResponse, RpcSignatureResult, SlotInfo},
+        rpc_response::{
+            Response as RpcResponse, RpcLogsResponse, RpcSignatureResult, SlotInfo, SlotUpdate,
+        },
     },
     log::*,
     serde::de::DeserializeOwned,
@@ -340,6 +342,54 @@ impl PubsubClient {
 
         Ok((result, receiver))
     }
+
+    pub fn slot_updates_subscribe(
+        url: &str,
+        handler: impl Fn(SlotUpdate) + Send + 'static,
+    ) -> Result<PubsubClientSubscription<SlotUpdate>, PubsubClientError> {
+        let url = Url::parse(url)?;
+        let (socket, _response) = connect(url)?;
+
+        let socket = Arc::new(RwLock::new(socket));
+        let exit = Arc::new(AtomicBool::new(false));
+        let exit_clone = exit.clone();
+        let subscription_id = PubsubClientSubscription::<SlotUpdate>::send_subscribe(
+            &socket,
+            json!({
+                "jsonrpc":"2.0","id":1,"method":"slotsUpdatesSubscribe","params":[]
+            })
+            .to_string(),
+        )?;
+
+        let t_cleanup = {
+            let socket = socket.clone();
+            std::thread::spawn(move || {
+                loop {
+                    if exit_clone.load(Ordering::Relaxed) {
+                        break;
+                    }
+                    match PubsubClientSubscription::read_message(&socket) {
+                        Ok(message) => handler(message),
+                        Err(err) => {
+                            info!("receive error: {:?}", err);
+                            break;
+                        }
+                    }
+                }
+
+                info!("websocket - exited receive loop");
+            })
+        };
+
+        Ok(PubsubClientSubscription {
+            message_type: PhantomData,
+            operation: "slotsUpdates",
+            socket,
+            subscription_id,
+            t_cleanup: Some(t_cleanup),
+            exit,
+        })
+    }
 }
 
 #[cfg(test)]

client/src/rpc_client.rs

@@ -4,12 +4,7 @@ use {
         http_sender::HttpSender,
         mock_sender::{MockSender, Mocks},
         rpc_config::RpcAccountInfoConfig,
-        rpc_config::{
-            RpcConfirmedBlockConfig, RpcConfirmedTransactionConfig, RpcEpochConfig,
-            RpcGetConfirmedSignaturesForAddress2Config, RpcLargestAccountsConfig,
-            RpcProgramAccountsConfig, RpcSendTransactionConfig, RpcSimulateTransactionConfig,
-            RpcTokenAccountsFilter,
-        },
+        rpc_config::*,
         rpc_request::{RpcError, RpcRequest, RpcResponseErrorData, TokenAccountsFilter},
         rpc_response::*,
         rpc_sender::RpcSender,
@@ -49,41 +44,36 @@ use {
     },
 };
 
-pub struct RpcClient {
-    sender: Box<dyn RpcSender + Send + Sync + 'static>,
+#[derive(Default)]
+pub struct RpcClientConfig {
     commitment_config: CommitmentConfig,
-    node_version: RwLock<Option<semver::Version>>,
+    confirm_transaction_initial_timeout: Option<Duration>,
 }
 
-fn serialize_encode_transaction(
-    transaction: &Transaction,
-    encoding: UiTransactionEncoding,
-) -> ClientResult<String> {
-    let serialized = serialize(transaction)
-        .map_err(|e| ClientErrorKind::Custom(format!("transaction serialization failed: {}", e)))?;
-    let encoded = match encoding {
-        UiTransactionEncoding::Base58 => bs58::encode(serialized).into_string(),
-        UiTransactionEncoding::Base64 => base64::encode(serialized),
-        _ => {
-            return Err(ClientErrorKind::Custom(format!(
-                "unsupported transaction encoding: {}. Supported encodings: base58, base64",
-                encoding
-            ))
-            .into())
+impl RpcClientConfig {
+    fn with_commitment(commitment_config: CommitmentConfig) -> Self {
+        RpcClientConfig {
+            commitment_config,
+            ..Self::default()
         }
-    };
-    Ok(encoded)
+    }
+}
+
+pub struct RpcClient {
+    sender: Box<dyn RpcSender + Send + Sync + 'static>,
+    config: RpcClientConfig,
+    node_version: RwLock<Option<semver::Version>>,
 }
 
 impl RpcClient {
     fn new_sender<T: RpcSender + Send + Sync + 'static>(
         sender: T,
-        commitment_config: CommitmentConfig,
+        config: RpcClientConfig,
     ) -> Self {
         Self {
             sender: Box::new(sender),
             node_version: RwLock::new(None),
-            commitment_config,
+            config,
         }
     }
 
@@ -92,13 +82,16 @@ impl RpcClient {
     }
 
     pub fn new_with_commitment(url: String, commitment_config: CommitmentConfig) -> Self {
-        Self::new_sender(HttpSender::new(url), commitment_config)
+        Self::new_sender(
+            HttpSender::new(url),
+            RpcClientConfig::with_commitment(commitment_config),
+        )
     }
 
     pub fn new_with_timeout(url: String, timeout: Duration) -> Self {
         Self::new_sender(
             HttpSender::new_with_timeout(url, timeout),
-            CommitmentConfig::default(),
+            RpcClientConfig::with_commitment(CommitmentConfig::default()),
         )
     }
 
@@ -109,18 +102,36 @@ impl RpcClient {
     ) -> Self {
         Self::new_sender(
             HttpSender::new_with_timeout(url, timeout),
-            commitment_config,
+            RpcClientConfig::with_commitment(commitment_config),
+        )
+    }
+
+    pub fn new_with_timeouts_and_commitment(
+        url: String,
+        timeout: Duration,
+        commitment_config: CommitmentConfig,
+        confirm_transaction_initial_timeout: Duration,
+    ) -> Self {
+        Self::new_sender(
+            HttpSender::new_with_timeout(url, timeout),
+            RpcClientConfig {
+                commitment_config,
+                confirm_transaction_initial_timeout: Some(confirm_transaction_initial_timeout),
+            },
         )
     }
 
     pub fn new_mock(url: String) -> Self {
-        Self::new_sender(MockSender::new(url), CommitmentConfig::default())
+        Self::new_sender(
+            MockSender::new(url),
+            RpcClientConfig::with_commitment(CommitmentConfig::default()),
+        )
     }
 
     pub fn new_mock_with_mocks(url: String, mocks: Mocks) -> Self {
         Self::new_sender(
             MockSender::new_with_mocks(url, mocks),
-            CommitmentConfig::default(),
+            RpcClientConfig::with_commitment(CommitmentConfig::default()),
         )
     }
 
@@ -159,7 +170,7 @@ impl RpcClient {
     }
 
     pub fn commitment(&self) -> CommitmentConfig {
-        self.commitment_config
+        self.config.commitment_config
     }
 
     fn use_deprecated_commitment(&self) -> Result<bool, RpcError> {
@@ -184,7 +195,7 @@ impl RpcClient {
 
     pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
         Ok(self
-            .confirm_transaction_with_commitment(signature, self.commitment_config)?
+            .confirm_transaction_with_commitment(signature, self.commitment())?
             .value)
     }
 
@@ -210,8 +221,7 @@ impl RpcClient {
             transaction,
             RpcSendTransactionConfig {
                 preflight_commitment: Some(
-                    self.maybe_map_commitment(self.commitment_config)?
-                        .commitment,
+                    self.maybe_map_commitment(self.commitment())?.commitment,
                 ),
                 ..RpcSendTransactionConfig::default()
             },
@@ -300,7 +310,7 @@ impl RpcClient {
         self.simulate_transaction_with_config(
             transaction,
             RpcSimulateTransactionConfig {
-                commitment: Some(self.commitment_config),
+                commitment: Some(self.commitment()),
                 ..RpcSimulateTransactionConfig::default()
             },
         )
@@ -338,7 +348,7 @@ impl RpcClient {
         &self,
         signature: &Signature,
     ) -> ClientResult<Option<transaction::Result<()>>> {
-        self.get_signature_status_with_commitment(signature, self.commitment_config)
+        self.get_signature_status_with_commitment(signature, self.commitment())
     }
 
     pub fn get_signature_statuses(
@@ -396,7 +406,7 @@ impl RpcClient {
     }
 
     pub fn get_slot(&self) -> ClientResult<Slot> {
-        self.get_slot_with_commitment(self.commitment_config)
+        self.get_slot_with_commitment(self.commitment())
     }
 
     pub fn get_slot_with_commitment(
@@ -409,6 +419,20 @@ impl RpcClient {
         )
     }
 
+    pub fn get_block_height(&self) -> ClientResult<u64> {
+        self.get_block_height_with_commitment(self.commitment())
+    }
+
+    pub fn get_block_height_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> ClientResult<u64> {
+        self.send(
+            RpcRequest::GetBlockHeight,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )
+    }
+
     pub fn get_slot_leaders(&self, start_slot: Slot, limit: u64) -> ClientResult<Vec<Pubkey>> {
         self.send(RpcRequest::GetSlotLeaders, json!([start_slot, limit]))
             .and_then(|slot_leaders: Vec<String>| {
@@ -427,6 +451,18 @@ impl RpcClient {
             })
     }
 
+    /// Get block production for the current epoch
+    pub fn get_block_production(&self) -> RpcResult<RpcBlockProduction> {
+        self.send(RpcRequest::GetBlockProduction, Value::Null)
+    }
+
+    pub fn get_block_production_with_config(
+        &self,
+        config: RpcBlockProductionConfig,
+    ) -> RpcResult<RpcBlockProduction> {
+        self.send(RpcRequest::GetBlockProduction, json!(config))
+    }
+
     pub fn get_stake_activation(
         &self,
         stake_account: Pubkey,
@@ -438,14 +474,14 @@ impl RpcClient {
                 stake_account.to_string(),
                 RpcEpochConfig {
                     epoch,
-                    commitment: Some(self.commitment_config),
+                    commitment: Some(self.commitment()),
                 }
             ]),
         )
     }
 
     pub fn supply(&self) -> RpcResult<RpcSupply> {
-        self.supply_with_commitment(self.commitment_config)
+        self.supply_with_commitment(self.commitment())
     }
 
     pub fn supply_with_commitment(
@@ -461,7 +497,7 @@ impl RpcClient {
     #[deprecated(since = "1.5.19", note = "Please use RpcClient::supply() instead")]
     #[allow(deprecated)]
     pub fn total_supply(&self) -> ClientResult<u64> {
-        self.total_supply_with_commitment(self.commitment_config)
+        self.total_supply_with_commitment(self.commitment())
     }
 
     #[deprecated(
@@ -493,17 +529,24 @@ impl RpcClient {
     }
 
     pub fn get_vote_accounts(&self) -> ClientResult<RpcVoteAccountStatus> {
-        self.get_vote_accounts_with_commitment(self.commitment_config)
+        self.get_vote_accounts_with_commitment(self.commitment())
     }
 
     pub fn get_vote_accounts_with_commitment(
         &self,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<RpcVoteAccountStatus> {
-        self.send(
-            RpcRequest::GetVoteAccounts,
-            json!([self.maybe_map_commitment(commitment_config)?]),
-        )
+        self.get_vote_accounts_with_config(RpcGetVoteAccountsConfig {
+            commitment: Some(self.maybe_map_commitment(commitment_config)?),
+            ..RpcGetVoteAccountsConfig::default()
+        })
+    }
+
+    pub fn get_vote_accounts_with_config(
+        &self,
+        config: RpcGetVoteAccountsConfig,
+    ) -> ClientResult<RpcVoteAccountStatus> {
+        self.send(RpcRequest::GetVoteAccounts, json!([config]))
     }
 
     pub fn wait_for_max_stake(
@@ -715,7 +758,7 @@ impl RpcClient {
     }
 
     pub fn get_epoch_info(&self) -> ClientResult<EpochInfo> {
-        self.get_epoch_info_with_commitment(self.commitment_config)
+        self.get_epoch_info_with_commitment(self.commitment())
     }
 
     pub fn get_epoch_info_with_commitment(
@@ -732,7 +775,7 @@ impl RpcClient {
         &self,
         slot: Option<Slot>,
     ) -> ClientResult<Option<RpcLeaderSchedule>> {
-        self.get_leader_schedule_with_commitment(slot, self.commitment_config)
+        self.get_leader_schedule_with_commitment(slot, self.commitment())
     }
 
     pub fn get_leader_schedule_with_commitment(
@@ -740,12 +783,23 @@ impl RpcClient {
         slot: Option<Slot>,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<Option<RpcLeaderSchedule>> {
-        self.send(
-            RpcRequest::GetLeaderSchedule,
-            json!([slot, self.maybe_map_commitment(commitment_config)?]),
+        self.get_leader_schedule_with_config(
+            slot,
+            RpcLeaderScheduleConfig {
+                commitment: Some(self.maybe_map_commitment(commitment_config)?),
+                ..RpcLeaderScheduleConfig::default()
+            },
         )
     }
 
+    pub fn get_leader_schedule_with_config(
+        &self,
+        slot: Option<Slot>,
+        config: RpcLeaderScheduleConfig,
+    ) -> ClientResult<Option<RpcLeaderSchedule>> {
+        self.send(RpcRequest::GetLeaderSchedule, json!([slot, config]))
+    }
+
     pub fn get_epoch_schedule(&self) -> ClientResult<EpochSchedule> {
         self.send(RpcRequest::GetEpochSchedule, Value::Null)
     }
@@ -791,7 +845,7 @@ impl RpcClient {
                 addresses,
                 RpcEpochConfig {
                     epoch,
-                    commitment: Some(self.commitment_config),
+                    commitment: Some(self.commitment()),
                 }
             ]),
         )
@@ -857,7 +911,7 @@ impl RpcClient {
     /// Note that `get_account` returns `Err(..)` if the account does not exist whereas
     /// `get_account_with_commitment` returns `Ok(None)` if the account does not exist.
     pub fn get_account(&self, pubkey: &Pubkey) -> ClientResult<Account> {
-        self.get_account_with_commitment(pubkey, self.commitment_config)?
+        self.get_account_with_commitment(pubkey, self.commitment())?
             .value
             .ok_or_else(|| RpcError::ForUser(format!("AccountNotFound: pubkey={}", pubkey)).into())
     }
@@ -913,7 +967,7 @@ impl RpcClient {
 
     pub fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> ClientResult<Vec<Option<Account>>> {
         Ok(self
-            .get_multiple_accounts_with_commitment(pubkeys, self.commitment_config)?
+            .get_multiple_accounts_with_commitment(pubkeys, self.commitment())?
             .value)
     }
 
@@ -967,7 +1021,7 @@ impl RpcClient {
     /// Request the balance of the account `pubkey`.
     pub fn get_balance(&self, pubkey: &Pubkey) -> ClientResult<u64> {
         Ok(self
-            .get_balance_with_commitment(pubkey, self.commitment_config)?
+            .get_balance_with_commitment(pubkey, self.commitment())?
             .value)
     }
 
@@ -989,12 +1043,11 @@ impl RpcClient {
         self.get_program_accounts_with_config(
             pubkey,
             RpcProgramAccountsConfig {
-                filters: None,
                 account_config: RpcAccountInfoConfig {
-                    encoding: Some(UiAccountEncoding::Base64),
-                    commitment: Some(self.commitment_config),
+                    encoding: Some(UiAccountEncoding::Base64Zstd),
                     ..RpcAccountInfoConfig::default()
                 },
+                ..RpcProgramAccountsConfig::default()
             },
         )
     }
@@ -1004,7 +1057,10 @@ impl RpcClient {
         pubkey: &Pubkey,
         config: RpcProgramAccountsConfig,
     ) -> ClientResult<Vec<(Pubkey, Account)>> {
-        let commitment = config.account_config.commitment.unwrap_or_default();
+        let commitment = config
+            .account_config
+            .commitment
+            .unwrap_or_else(|| self.commitment());
         let commitment = self.maybe_map_commitment(commitment)?;
         let account_config = RpcAccountInfoConfig {
             commitment: Some(commitment),
@@ -1023,7 +1079,7 @@ impl RpcClient {
 
     /// Request the transaction count.
     pub fn get_transaction_count(&self) -> ClientResult<u64> {
-        self.get_transaction_count_with_commitment(self.commitment_config)
+        self.get_transaction_count_with_commitment(self.commitment())
     }
 
     pub fn get_transaction_count_with_commitment(
@@ -1036,9 +1092,37 @@ impl RpcClient {
         )
     }
 
+    pub fn get_fees(&self) -> ClientResult<Fees> {
+        Ok(self.get_fees_with_commitment(self.commitment())?.value)
+    }
+
+    pub fn get_fees_with_commitment(&self, commitment_config: CommitmentConfig) -> RpcResult<Fees> {
+        let Response {
+            context,
+            value: fees,
+        } = self.send::<Response<RpcFees>>(
+            RpcRequest::GetFees,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        )?;
+        let blockhash = fees.blockhash.parse().map_err(|_| {
+            ClientError::new_with_request(
+                RpcError::ParseError("Hash".to_string()).into(),
+                RpcRequest::GetFees,
+            )
+        })?;
+        Ok(Response {
+            context,
+            value: Fees {
+                blockhash,
+                fee_calculator: fees.fee_calculator,
+                last_valid_block_height: fees.last_valid_block_height,
+            },
+        })
+    }
+
     pub fn get_recent_blockhash(&self) -> ClientResult<(Hash, FeeCalculator)> {
         let (blockhash, fee_calculator, _last_valid_slot) = self
-            .get_recent_blockhash_with_commitment(self.commitment_config)?
+            .get_recent_blockhash_with_commitment(self.commitment())?
             .value;
         Ok((blockhash, fee_calculator))
     }
@@ -1054,6 +1138,7 @@ impl RpcClient {
                     blockhash,
                     fee_calculator,
                     last_valid_slot,
+                    ..
                 },
         }) = self
             .send::<Response<RpcFees>>(
@@ -1061,6 +1146,19 @@ impl RpcClient {
                 json!([self.maybe_map_commitment(commitment_config)?]),
             ) {
             (context, blockhash, fee_calculator, last_valid_slot)
+        } else if let Ok(Response {
+            context,
+            value:
+                DeprecatedRpcFees {
+                    blockhash,
+                    fee_calculator,
+                    last_valid_slot,
+                },
+        }) = self.send::<Response<DeprecatedRpcFees>>(
+            RpcRequest::GetFees,
+            json!([self.maybe_map_commitment(commitment_config)?]),
+        ) {
+            (context, blockhash, fee_calculator, last_valid_slot)
         } else if let Ok(Response {
             context,
             value:
@@ -1097,7 +1195,7 @@ impl RpcClient {
         blockhash: &Hash,
     ) -> ClientResult<Option<FeeCalculator>> {
         Ok(self
-            .get_fee_calculator_for_blockhash_with_commitment(blockhash, self.commitment_config)?
+            .get_fee_calculator_for_blockhash_with_commitment(blockhash, self.commitment())?
             .value)
     }
 
@@ -1179,7 +1277,7 @@ impl RpcClient {
 
     pub fn get_token_account(&self, pubkey: &Pubkey) -> ClientResult<Option<UiTokenAccount>> {
         Ok(self
-            .get_token_account_with_commitment(pubkey, self.commitment_config)?
+            .get_token_account_with_commitment(pubkey, self.commitment())?
             .value)
     }
 
@@ -1240,7 +1338,7 @@ impl RpcClient {
 
     pub fn get_token_account_balance(&self, pubkey: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
-            .get_token_account_balance_with_commitment(pubkey, self.commitment_config)?
+            .get_token_account_balance_with_commitment(pubkey, self.commitment())?
             .value)
     }
 
@@ -1267,7 +1365,7 @@ impl RpcClient {
             .get_token_accounts_by_delegate_with_commitment(
                 delegate,
                 token_account_filter,
-                self.commitment_config,
+                self.commitment(),
             )?
             .value)
     }
@@ -1306,7 +1404,7 @@ impl RpcClient {
             .get_token_accounts_by_owner_with_commitment(
                 owner,
                 token_account_filter,
-                self.commitment_config,
+                self.commitment(),
             )?
             .value)
     }
@@ -1338,7 +1436,7 @@ impl RpcClient {
 
     pub fn get_token_supply(&self, mint: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
-            .get_token_supply_with_commitment(mint, self.commitment_config)?
+            .get_token_supply_with_commitment(mint, self.commitment())?
             .value)
     }
 
@@ -1356,6 +1454,64 @@ impl RpcClient {
         )
     }
 
+    pub fn request_airdrop(&self, pubkey: &Pubkey, lamports: u64) -> ClientResult<Signature> {
+        self.request_airdrop_with_config(
+            pubkey,
+            lamports,
+            RpcRequestAirdropConfig {
+                commitment: Some(self.commitment()),
+                ..RpcRequestAirdropConfig::default()
+            },
+        )
+    }
+
+    pub fn request_airdrop_with_blockhash(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        recent_blockhash: &Hash,
+    ) -> ClientResult<Signature> {
+        self.request_airdrop_with_config(
+            pubkey,
+            lamports,
+            RpcRequestAirdropConfig {
+                commitment: Some(self.commitment()),
+                recent_blockhash: Some(recent_blockhash.to_string()),
+            },
+        )
+    }
+
+    pub fn request_airdrop_with_config(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        config: RpcRequestAirdropConfig,
+    ) -> ClientResult<Signature> {
+        let commitment = config.commitment.unwrap_or_default();
+        let commitment = self.maybe_map_commitment(commitment)?;
+        let config = RpcRequestAirdropConfig {
+            commitment: Some(commitment),
+            ..config
+        };
+        self.send(
+            RpcRequest::RequestAirdrop,
+            json!([pubkey.to_string(), lamports, config]),
+        )
+        .and_then(|signature: String| {
+            Signature::from_str(&signature).map_err(|err| {
+                ClientErrorKind::Custom(format!("signature deserialization failed: {}", err)).into()
+            })
+        })
+        .map_err(|_| {
+            RpcError::ForUser(
+                "airdrop request failed. \
+                This can happen when the rate limit is reached."
+                    .to_string(),
+            )
+            .into()
+        })
+    }
+
     fn poll_balance_with_timeout_and_commitment(
         &self,
         pubkey: &Pubkey,
@@ -1422,7 +1578,7 @@ impl RpcClient {
 
     /// Poll the server to confirm a transaction.
     pub fn poll_for_signature(&self, signature: &Signature) -> ClientResult<()> {
-        self.poll_for_signature_with_commitment(signature, self.commitment_config)
+        self.poll_for_signature_with_commitment(signature, self.commitment())
     }
 
     /// Poll the server to confirm a transaction.
@@ -1532,7 +1688,7 @@ impl RpcClient {
     ) -> ClientResult<Signature> {
         self.send_and_confirm_transaction_with_spinner_and_commitment(
             transaction,
-            self.commitment_config,
+            self.commitment(),
         )
     }
 
@@ -1557,6 +1713,24 @@ impl RpcClient {
         commitment: CommitmentConfig,
         config: RpcSendTransactionConfig,
     ) -> ClientResult<Signature> {
+        let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
+            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
+                .value
+                .0
+        } else {
+            transaction.message.recent_blockhash
+        };
+        let signature = self.send_transaction_with_config(transaction, config)?;
+        self.confirm_transaction_with_spinner(&signature, &recent_blockhash, commitment)?;
+        Ok(signature)
+    }
+
+    pub fn confirm_transaction_with_spinner(
+        &self,
+        signature: &Signature,
+        recent_blockhash: &Hash,
+        commitment: CommitmentConfig,
+    ) -> ClientResult<()> {
         let desired_confirmations = if commitment.is_finalized() {
             MAX_LOCKOUT_HISTORY + 1
         } else {
@@ -1568,29 +1742,27 @@ impl RpcClient {
 
         progress_bar.set_message(&format!(
             "[{}/{}] Finalizing transaction {}",
-            confirmations, desired_confirmations, transaction.signatures[0],
+            confirmations, desired_confirmations, signature,
         ));
-        let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
-            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
-                .value
-                .0
-        } else {
-            transaction.message.recent_blockhash
-        };
-        let signature = self.send_transaction_with_config(transaction, config)?;
+
+        let now = Instant::now();
+        let confirm_transaction_initial_timeout = self
+            .config
+            .confirm_transaction_initial_timeout
+            .unwrap_or_default();
         let (signature, status) = loop {
             // Get recent commitment in order to count confirmations for successful transactions
             let status = self
                 .get_signature_status_with_commitment(&signature, CommitmentConfig::processed())?;
             if status.is_none() {
-                if self
+                let blockhash_not_found = self
                     .get_fee_calculator_for_blockhash_with_commitment(
                         &recent_blockhash,
                         CommitmentConfig::processed(),
                     )?
                     .value
-                    .is_none()
-                {
+                    .is_none();
+                if blockhash_not_found && now.elapsed() >= confirm_transaction_initial_timeout {
                     break (signature, status);
                 }
             } else {
@@ -1624,7 +1796,7 @@ impl RpcClient {
             {
                 progress_bar.set_message("Transaction confirmed");
                 progress_bar.finish_and_clear();
-                return Ok(signature);
+                return Ok(());
             }
 
             progress_bar.set_message(&format!(
@@ -1661,6 +1833,26 @@ impl RpcClient {
     }
 }
 
+fn serialize_encode_transaction(
+    transaction: &Transaction,
+    encoding: UiTransactionEncoding,
+) -> ClientResult<String> {
+    let serialized = serialize(transaction)
+        .map_err(|e| ClientErrorKind::Custom(format!("transaction serialization failed: {}", e)))?;
+    let encoded = match encoding {
+        UiTransactionEncoding::Base58 => bs58::encode(serialized).into_string(),
+        UiTransactionEncoding::Base64 => base64::encode(serialized),
+        _ => {
+            return Err(ClientErrorKind::Custom(format!(
+                "unsupported transaction encoding: {}. Supported encodings: base58, base64",
+                encoding
+            ))
+            .into())
+        }
+    };
+    Ok(encoded)
+}
+
 #[derive(Debug, Default)]
 pub struct GetConfirmedSignaturesForAddress2Config {
     pub before: Option<Signature>,

client/src/rpc_config.rs

@@ -23,14 +23,82 @@ pub struct RpcSendTransactionConfig {
     pub encoding: Option<UiTransactionEncoding>,
 }
 
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSimulateTransactionAccountsConfig {
+    pub encoding: Option<UiAccountEncoding>,
+    pub addresses: Vec<String>,
+}
+
 #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcSimulateTransactionConfig {
     #[serde(default)]
     pub sig_verify: bool,
+    #[serde(default)]
+    pub replace_recent_blockhash: bool,
     #[serde(flatten)]
     pub commitment: Option<CommitmentConfig>,
     pub encoding: Option<UiTransactionEncoding>,
+    pub accounts: Option<RpcSimulateTransactionAccountsConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcRequestAirdropConfig {
+    pub recent_blockhash: Option<String>, // base-58 encoded blockhash
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcLeaderScheduleConfig {
+    pub identity: Option<String>, // validator identity, as a base-58 encoded string
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProductionConfigRange {
+    pub first_slot: Slot,
+    pub last_slot: Option<Slot>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProductionConfig {
+    pub identity: Option<String>, // validator identity, as a base-58 encoded string
+    pub range: Option<RpcBlockProductionConfigRange>, // current epoch if `None`
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcGetVoteAccountsConfig {
+    pub vote_pubkey: Option<String>, // validator vote address, as a base-58 encoded string
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+    pub keep_unstaked_delinquents: Option<bool>,
+    pub delinquent_slot_distance: Option<u64>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum RpcLeaderScheduleConfigWrapper {
+    SlotOnly(Option<Slot>),
+    ConfigOnly(Option<RpcLeaderScheduleConfig>),
+}
+
+impl RpcLeaderScheduleConfigWrapper {
+    pub fn unzip(&self) -> (Option<Slot>, Option<RpcLeaderScheduleConfig>) {
+        match &self {
+            RpcLeaderScheduleConfigWrapper::SlotOnly(slot) => (*slot, None),
+            RpcLeaderScheduleConfigWrapper::ConfigOnly(config) => (None, config.clone()),
+        }
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -71,6 +139,7 @@ pub struct RpcProgramAccountsConfig {
     pub filters: Option<Vec<RpcFilterType>>,
     #[serde(flatten)]
     pub account_config: RpcAccountInfoConfig,
+    pub with_context: Option<bool>,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

client/src/rpc_custom_error.rs

@@ -15,6 +15,8 @@ pub const JSON_RPC_SERVER_ERROR_TRANSACTION_PRECOMPILE_VERIFICATION_FAILURE: i64
 pub const JSON_RPC_SERVER_ERROR_SLOT_SKIPPED: i64 = -32007;
 pub const JSON_RPC_SERVER_ERROR_NO_SNAPSHOT: i64 = -32008;
 pub const JSON_RPC_SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED: i64 = -32009;
+pub const JSON_RPC_SERVER_ERROR_KEY_EXCLUDED_FROM_SECONDARY_INDEX: i64 = -32010;
+pub const JSON_RPC_SERVER_ERROR_TRANSACTION_HISTORY_NOT_AVAILABLE: i64 = -32011;
 
 pub enum RpcCustomError {
     BlockCleanedUp {
@@ -40,6 +42,10 @@ pub enum RpcCustomError {
     LongTermStorageSlotSkipped {
         slot: Slot,
     },
+    KeyExcludedFromSecondaryIndex {
+        index_key: String,
+    },
+    TransactionHistoryNotAvailable,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -117,6 +123,24 @@ impl From<RpcCustomError> for Error {
                 message: format!("Slot {} was skipped, or missing in long-term storage", slot),
                 data: None,
             },
+            RpcCustomError::KeyExcludedFromSecondaryIndex { index_key } => Self {
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_KEY_EXCLUDED_FROM_SECONDARY_INDEX,
+                ),
+                message: format!(
+                    "{} excluded from account secondary indexes; \
+                    this RPC method unavailable for key",
+                    index_key
+                ),
+                data: None,
+            },
+            RpcCustomError::TransactionHistoryNotAvailable => Self {
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_TRANSACTION_HISTORY_NOT_AVAILABLE,
+                ),
+                message: "Transaction history is not available from this node".to_string(),
+                data: None,
+            },
         }
     }
 }

client/src/rpc_request.rs

@@ -11,6 +11,8 @@ pub enum RpcRequest {
     DeregisterNode,
     GetAccountInfo,
     GetBalance,
+    GetBlockHeight,
+    GetBlockProduction,
     GetBlockTime,
     GetClusterNodes,
     GetConfirmedBlock,
@@ -83,6 +85,8 @@ impl fmt::Display for RpcRequest {
             RpcRequest::DeregisterNode => "deregisterNode",
             RpcRequest::GetAccountInfo => "getAccountInfo",
             RpcRequest::GetBalance => "getBalance",
+            RpcRequest::GetBlockHeight => "getBlockHeight",
+            RpcRequest::GetBlockProduction => "getBlockProduction",
             RpcRequest::GetBlockTime => "getBlockTime",
             RpcRequest::GetClusterNodes => "getClusterNodes",
             RpcRequest::GetConfirmedBlock => "getConfirmedBlock",

client/src/rpc_response.rs

@@ -4,6 +4,7 @@ use {
     solana_sdk::{
         clock::{Epoch, Slot, UnixTimestamp},
         fee_calculator::{FeeCalculator, FeeRateGovernor},
+        hash::Hash,
         inflation::Inflation,
         transaction::{Result, TransactionError},
     },
@@ -46,6 +47,23 @@ pub struct RpcFees {
     pub blockhash: String,
     pub fee_calculator: FeeCalculator,
     pub last_valid_slot: Slot,
+    pub last_valid_block_height: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct DeprecatedRpcFees {
+    pub blockhash: String,
+    pub fee_calculator: FeeCalculator,
+    pub last_valid_slot: Slot,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Fees {
+    pub blockhash: Hash,
+    pub fee_calculator: FeeCalculator,
+    pub last_valid_block_height: u64,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -206,11 +224,28 @@ pub struct RpcContactInfo {
     pub version: Option<String>,
     /// First 4 bytes of the FeatureSet identifier
     pub feature_set: Option<u32>,
+    /// Shred version
+    pub shred_version: Option<u16>,
 }
 
 /// Map of leader base58 identity pubkeys to the slot indices relative to the first epoch slot
 pub type RpcLeaderSchedule = HashMap<String, Vec<usize>>;
 
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProductionRange {
+    pub first_slot: Slot,
+    pub last_slot: Slot,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockProduction {
+    /// Map of leader base58 identity pubkeys to a tuple of `(number of leader slots, number of blocks produced)`
+    pub by_identity: HashMap<String, (usize, usize)>,
+    pub range: RpcBlockProductionRange,
+}
+
 #[derive(Serialize, Deserialize, Clone)]
 #[serde(rename_all = "kebab-case")]
 pub struct RpcVersionInfo {
@@ -254,10 +289,10 @@ pub struct RpcVoteAccountStatus {
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcVoteAccountInfo {
-    /// Vote account pubkey as base-58 encoded string
+    /// Vote account address, as base-58 encoded string
     pub vote_pubkey: String,
 
-    /// The pubkey of the node that votes using this account
+    /// The validator identity, as base-58 encoded string
     pub node_pubkey: String,
 
     /// The current stake, in lamports, delegated to this vote account
@@ -292,6 +327,7 @@ pub struct RpcSignatureConfirmation {
 pub struct RpcSimulateTransactionResult {
     pub err: Option<TransactionError>,
     pub logs: Option<Vec<String>>,
+    pub accounts: Option<Vec<Option<UiAccount>>>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -367,8 +403,9 @@ pub struct RpcPerfSample {
 pub struct RpcInflationReward {
     pub epoch: Epoch,
     pub effective_slot: Slot,
-    pub amount: u64,       // lamports
-    pub post_balance: u64, // lamports
+    pub amount: u64,            // lamports
+    pub post_balance: u64,      // lamports
+    pub commission: Option<u8>, // Vote account commission when the reward was credited
 }
 
 impl From<ConfirmedTransactionStatusWithSignature> for RpcConfirmedTransactionStatusWithSignature {

client/src/thin_client.rs

@@ -169,8 +169,8 @@ impl ThinClient {
         let rpc_clients: Vec<_> = rpc_addrs.into_iter().map(RpcClient::new_socket).collect();
         let optimizer = ClientOptimizer::new(rpc_clients.len());
         Self {
-            tpu_addrs,
             transactions_socket,
+            tpu_addrs,
             rpc_clients,
             optimizer,
         }

client/src/tpu_client.rs

@@ -0,0 +1,393 @@
+use crate::{
+    pubsub_client::{PubsubClient, PubsubClientError, PubsubClientSubscription},
+    rpc_client::RpcClient,
+    rpc_response::SlotUpdate,
+};
+use bincode::serialize;
+use log::*;
+use solana_sdk::{clock::Slot, pubkey::Pubkey, transaction::Transaction};
+use std::{
+    collections::{HashMap, HashSet, VecDeque},
+    net::{SocketAddr, UdpSocket},
+    str::FromStr,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, RwLock,
+    },
+    thread::JoinHandle,
+    time::{Duration, Instant},
+};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum TpuSenderError {
+    #[error("Pubsub error: {0:?}")]
+    PubsubError(#[from] PubsubClientError),
+    #[error("RPC error: {0:?}")]
+    RpcError(#[from] crate::client_error::ClientError),
+    #[error("IO error: {0:?}")]
+    IoError(#[from] std::io::Error),
+}
+
+type Result<T> = std::result::Result<T, TpuSenderError>;
+
+/// Default number of slots used to build TPU socket fanout set
+pub const DEFAULT_FANOUT_SLOTS: u64 = 12;
+
+/// Maximum number of slots used to build TPU socket fanout set
+pub const MAX_FANOUT_SLOTS: u64 = 100;
+
+/// Config params for `TpuClient`
+#[derive(Clone, Debug)]
+pub struct TpuClientConfig {
+    /// The range of upcoming slots to include when determining which
+    /// leaders to send transactions to (min: 1, max: 100)
+    pub fanout_slots: u64,
+}
+
+impl Default for TpuClientConfig {
+    fn default() -> Self {
+        Self {
+            fanout_slots: DEFAULT_FANOUT_SLOTS,
+        }
+    }
+}
+
+/// Client which sends transactions directly to the current leader's TPU port over UDP.
+/// The client uses RPC to determine the current leader and fetch node contact info
+pub struct TpuClient {
+    send_socket: UdpSocket,
+    fanout_slots: u64,
+    leader_tpu_service: LeaderTpuService,
+    exit: Arc<AtomicBool>,
+}
+
+impl TpuClient {
+    /// Serializes and sends a transaction to the current leader's TPU port
+    pub fn send_transaction(&self, transaction: &Transaction) -> bool {
+        let wire_transaction = serialize(transaction).expect("serialization should succeed");
+        self.send_wire_transaction(&wire_transaction)
+    }
+
+    /// Sends a transaction to the current leader's TPU port
+    pub fn send_wire_transaction(&self, wire_transaction: &[u8]) -> bool {
+        let mut sent = false;
+        for tpu_address in self
+            .leader_tpu_service
+            .leader_tpu_sockets(self.fanout_slots)
+        {
+            if self
+                .send_socket
+                .send_to(wire_transaction, tpu_address)
+                .is_ok()
+            {
+                sent = true;
+            }
+        }
+        sent
+    }
+
+    /// Create a new client that disconnects when dropped
+    pub fn new(
+        rpc_client: Arc<RpcClient>,
+        websocket_url: &str,
+        config: TpuClientConfig,
+    ) -> Result<Self> {
+        let exit = Arc::new(AtomicBool::new(false));
+        let leader_tpu_service = LeaderTpuService::new(rpc_client, websocket_url, exit.clone())?;
+
+        Ok(Self {
+            send_socket: UdpSocket::bind("0.0.0.0:0").unwrap(),
+            fanout_slots: config.fanout_slots.min(MAX_FANOUT_SLOTS).max(1),
+            leader_tpu_service,
+            exit,
+        })
+    }
+}
+
+impl Drop for TpuClient {
+    fn drop(&mut self) {
+        self.exit.store(true, Ordering::Relaxed);
+        self.leader_tpu_service.join();
+    }
+}
+
+struct LeaderTpuCache {
+    first_slot: Slot,
+    leaders: Vec<Pubkey>,
+    leader_tpu_map: HashMap<Pubkey, SocketAddr>,
+}
+
+impl LeaderTpuCache {
+    fn new(rpc_client: &RpcClient, first_slot: Slot) -> Self {
+        let leaders = Self::fetch_slot_leaders(rpc_client, first_slot).unwrap_or_default();
+        let leader_tpu_map = Self::fetch_cluster_tpu_sockets(&rpc_client).unwrap_or_default();
+        Self {
+            first_slot,
+            leaders,
+            leader_tpu_map,
+        }
+    }
+
+    // Last slot that has a cached leader pubkey
+    fn last_slot(&self) -> Slot {
+        self.first_slot + self.leaders.len().saturating_sub(1) as u64
+    }
+
+    // Get the TPU sockets for the current leader and upcoming leaders according to fanout size
+    fn get_leader_sockets(&self, current_slot: Slot, fanout_slots: u64) -> Vec<SocketAddr> {
+        let mut leader_set = HashSet::new();
+        let mut leader_sockets = Vec::new();
+        for leader_slot in current_slot..current_slot + fanout_slots {
+            if let Some(leader) = self.get_slot_leader(leader_slot) {
+                if let Some(tpu_socket) = self.leader_tpu_map.get(leader) {
+                    if leader_set.insert(*leader) {
+                        leader_sockets.push(*tpu_socket);
+                    }
+                }
+            }
+        }
+        leader_sockets
+    }
+
+    fn get_slot_leader(&self, slot: Slot) -> Option<&Pubkey> {
+        if slot >= self.first_slot {
+            let index = slot - self.first_slot;
+            self.leaders.get(index as usize)
+        } else {
+            None
+        }
+    }
+
+    fn fetch_cluster_tpu_sockets(rpc_client: &RpcClient) -> Result<HashMap<Pubkey, SocketAddr>> {
+        let cluster_contact_info = rpc_client.get_cluster_nodes()?;
+        Ok(cluster_contact_info
+            .into_iter()
+            .filter_map(|contact_info| {
+                Some((
+                    Pubkey::from_str(&contact_info.pubkey).ok()?,
+                    contact_info.tpu?,
+                ))
+            })
+            .collect())
+    }
+
+    fn fetch_slot_leaders(rpc_client: &RpcClient, start_slot: Slot) -> Result<Vec<Pubkey>> {
+        Ok(rpc_client.get_slot_leaders(start_slot, 2 * MAX_FANOUT_SLOTS)?)
+    }
+}
+
+// 48 chosen because it's unlikely that 12 leaders in a row will miss their slots
+const MAX_SLOT_SKIP_DISTANCE: u64 = 48;
+
+#[derive(Clone, Debug)]
+struct RecentLeaderSlots(Arc<RwLock<VecDeque<Slot>>>);
+impl RecentLeaderSlots {
+    fn new(current_slot: Slot) -> Self {
+        let mut recent_slots = VecDeque::new();
+        recent_slots.push_back(current_slot);
+        Self(Arc::new(RwLock::new(recent_slots)))
+    }
+
+    fn record_slot(&self, current_slot: Slot) {
+        let mut recent_slots = self.0.write().unwrap();
+        recent_slots.push_back(current_slot);
+        // 12 recent slots should be large enough to avoid a misbehaving
+        // validator from affecting the median recent slot
+        while recent_slots.len() > 12 {
+            recent_slots.pop_front();
+        }
+    }
+
+    // Estimate the current slot from recent slot notifications.
+    fn estimated_current_slot(&self) -> Slot {
+        let mut recent_slots: Vec<Slot> = self.0.read().unwrap().iter().cloned().collect();
+        assert!(!recent_slots.is_empty());
+        recent_slots.sort_unstable();
+
+        // Validators can broadcast invalid blocks that are far in the future
+        // so check if the current slot is in line with the recent progression.
+        let max_index = recent_slots.len() - 1;
+        let median_index = max_index / 2;
+        let median_recent_slot = recent_slots[median_index];
+        let expected_current_slot = median_recent_slot + (max_index - median_index) as u64;
+        let max_reasonable_current_slot = expected_current_slot + MAX_SLOT_SKIP_DISTANCE;
+
+        // Return the highest slot that doesn't exceed what we believe is a
+        // reasonable slot.
+        recent_slots
+            .into_iter()
+            .rev()
+            .find(|slot| *slot <= max_reasonable_current_slot)
+            .unwrap()
+    }
+}
+
+#[cfg(test)]
+impl From<Vec<Slot>> for RecentLeaderSlots {
+    fn from(recent_slots: Vec<Slot>) -> Self {
+        assert!(!recent_slots.is_empty());
+        Self(Arc::new(RwLock::new(recent_slots.into_iter().collect())))
+    }
+}
+
+/// Service that tracks upcoming leaders and maintains an up-to-date mapping
+/// of leader id to TPU socket address.
+struct LeaderTpuService {
+    recent_slots: RecentLeaderSlots,
+    leader_tpu_cache: Arc<RwLock<LeaderTpuCache>>,
+    subscription: Option<PubsubClientSubscription<SlotUpdate>>,
+    t_leader_tpu_service: Option<JoinHandle<()>>,
+}
+
+impl LeaderTpuService {
+    fn new(rpc_client: Arc<RpcClient>, websocket_url: &str, exit: Arc<AtomicBool>) -> Result<Self> {
+        let start_slot = rpc_client.get_max_shred_insert_slot()?;
+
+        let recent_slots = RecentLeaderSlots::new(start_slot);
+        let leader_tpu_cache = Arc::new(RwLock::new(LeaderTpuCache::new(&rpc_client, start_slot)));
+
+        let subscription = if !websocket_url.is_empty() {
+            let recent_slots = recent_slots.clone();
+            Some(PubsubClient::slot_updates_subscribe(
+                websocket_url,
+                move |update| {
+                    let current_slot = match update {
+                        // This update indicates that a full slot was received by the connected
+                        // node so we can stop sending transactions to the leader for that slot
+                        SlotUpdate::Completed { slot, .. } => slot.saturating_add(1),
+                        // This update indicates that we have just received the first shred from
+                        // the leader for this slot and they are probably still accepting transactions.
+                        SlotUpdate::FirstShredReceived { slot, .. } => slot,
+                        _ => return,
+                    };
+
+                    recent_slots.record_slot(current_slot);
+                },
+            )?)
+        } else {
+            None
+        };
+
+        let t_leader_tpu_service = Some({
+            let recent_slots = recent_slots.clone();
+            let leader_tpu_cache = leader_tpu_cache.clone();
+            std::thread::Builder::new()
+                .name("ldr-tpu-srv".to_string())
+                .spawn(move || Self::run(rpc_client, recent_slots, leader_tpu_cache, exit))
+                .unwrap()
+        });
+
+        Ok(LeaderTpuService {
+            recent_slots,
+            leader_tpu_cache,
+            subscription,
+            t_leader_tpu_service,
+        })
+    }
+
+    fn join(&mut self) {
+        if let Some(mut subscription) = self.subscription.take() {
+            let _ = subscription.send_unsubscribe();
+            let _ = subscription.shutdown();
+        }
+        if let Some(t_handle) = self.t_leader_tpu_service.take() {
+            t_handle.join().unwrap();
+        }
+    }
+
+    fn leader_tpu_sockets(&self, fanout_slots: u64) -> Vec<SocketAddr> {
+        let current_slot = self.recent_slots.estimated_current_slot();
+        self.leader_tpu_cache
+            .read()
+            .unwrap()
+            .get_leader_sockets(current_slot, fanout_slots)
+    }
+
+    fn run(
+        rpc_client: Arc<RpcClient>,
+        recent_slots: RecentLeaderSlots,
+        leader_tpu_cache: Arc<RwLock<LeaderTpuCache>>,
+        exit: Arc<AtomicBool>,
+    ) {
+        let mut last_cluster_refresh = Instant::now();
+        let mut sleep_ms = 1000;
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            // Refresh cluster TPU ports every 5min in case validators restart with new port configuration
+            // or new validators come online
+            if last_cluster_refresh.elapsed() > Duration::from_secs(5 * 60) {
+                if let Ok(leader_tpu_map) = LeaderTpuCache::fetch_cluster_tpu_sockets(&rpc_client) {
+                    leader_tpu_cache.write().unwrap().leader_tpu_map = leader_tpu_map;
+                    last_cluster_refresh = Instant::now();
+                } else {
+                    sleep_ms = 100;
+                    continue;
+                }
+            }
+
+            // Sleep a few slots before checking if leader cache needs to be refreshed again
+            std::thread::sleep(Duration::from_millis(sleep_ms));
+
+            let current_slot = recent_slots.estimated_current_slot();
+            if current_slot
+                >= leader_tpu_cache
+                    .read()
+                    .unwrap()
+                    .last_slot()
+                    .saturating_sub(MAX_FANOUT_SLOTS)
+            {
+                if let Ok(slot_leaders) =
+                    LeaderTpuCache::fetch_slot_leaders(&rpc_client, current_slot)
+                {
+                    let mut leader_tpu_cache = leader_tpu_cache.write().unwrap();
+                    leader_tpu_cache.first_slot = current_slot;
+                    leader_tpu_cache.leaders = slot_leaders;
+                } else {
+                    sleep_ms = 100;
+                    continue;
+                }
+            }
+
+            sleep_ms = 1000;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_slot(recent_slots: RecentLeaderSlots, expected_slot: Slot) {
+        assert_eq!(recent_slots.estimated_current_slot(), expected_slot);
+    }
+
+    #[test]
+    fn test_recent_leader_slots() {
+        assert_slot(RecentLeaderSlots::new(0), 0);
+
+        let mut recent_slots: Vec<Slot> = (1..=12).collect();
+        assert_slot(RecentLeaderSlots::from(recent_slots.clone()), 12);
+
+        recent_slots.reverse();
+        assert_slot(RecentLeaderSlots::from(recent_slots), 12);
+
+        assert_slot(
+            RecentLeaderSlots::from(vec![0, 1 + MAX_SLOT_SKIP_DISTANCE]),
+            1 + MAX_SLOT_SKIP_DISTANCE,
+        );
+        assert_slot(
+            RecentLeaderSlots::from(vec![0, 2 + MAX_SLOT_SKIP_DISTANCE]),
+            0,
+        );
+
+        assert_slot(RecentLeaderSlots::from(vec![1]), 1);
+        assert_slot(RecentLeaderSlots::from(vec![1, 100]), 1);
+        assert_slot(RecentLeaderSlots::from(vec![1, 2, 100]), 2);
+        assert_slot(RecentLeaderSlots::from(vec![1, 2, 3, 100]), 3);
+        assert_slot(RecentLeaderSlots::from(vec![1, 2, 3, 99, 100]), 3);
+    }
+}

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.6.5"
+version = "1.6.20"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -29,12 +29,12 @@ fs_extra = "1.2.0"
 flate2 = "1.0"
 indexmap = { version = "1.5", features = ["rayon"] }
 itertools = "0.9.0"
-jsonrpc-core = "17.0.0"
-jsonrpc-core-client = { version = "17.0.0", features = ["ipc", "ws"] }
-jsonrpc-derive = "17.0.0"
-jsonrpc-http-server = "17.0.0"
-jsonrpc-pubsub = "17.0.0"
-jsonrpc-ws-server = "17.0.0"
+jsonrpc-core = "17.1.0"
+jsonrpc-core-client = { version = "17.1.0", features = ["ipc", "ws"] }
+jsonrpc-derive = "17.1.0"
+jsonrpc-http-server = "17.1.0"
+jsonrpc-pubsub = "17.1.0"
+jsonrpc-ws-server = "17.1.0"
 libc = "0.2.81"
 log = "0.4.11"
 lru = "0.6.1"
@@ -48,42 +48,41 @@ raptorq = "1.4.2"
 rayon = "1.5.0"
 regex = "1.3.9"
 retain_mut = "0.1.2"
-rustversion = "1.0.4"
 serde = "1.0.122"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.6.5" }
-solana-banks-server = { path = "../banks-server", version = "=1.6.5" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.6.5" }
-solana-client = { path = "../client", version = "=1.6.5" }
-solana-faucet = { path = "../faucet", version = "=1.6.5" }
-solana-ledger = { path = "../ledger", version = "=1.6.5" }
-solana-logger = { path = "../logger", version = "=1.6.5" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.6.5" }
-solana-metrics = { path = "../metrics", version = "=1.6.5" }
-solana-measure = { path = "../measure", version = "=1.6.5" }
-solana-net-utils = { path = "../net-utils", version = "=1.6.5" }
-solana-perf = { path = "../perf", version = "=1.6.5" }
-solana-program-test = { path = "../program-test", version = "=1.6.5" }
-solana-runtime = { path = "../runtime", version = "=1.6.5" }
-solana-sdk = { path = "../sdk", version = "=1.6.5" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.6.5" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.6.5" }
-solana-stake-program = { path = "../programs/stake", version = "=1.6.5" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.6.5" }
-solana-streamer = { path = "../streamer", version = "=1.6.5" }
-solana-sys-tuner = { path = "../sys-tuner", version = "=1.6.5" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.6.5" }
-solana-version = { path = "../version", version = "=1.6.5" }
-solana-vote-program = { path = "../programs/vote", version = "=1.6.5" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.20" }
+solana-banks-server = { path = "../banks-server", version = "=1.6.20" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.20" }
+solana-client = { path = "../client", version = "=1.6.20" }
+solana-faucet = { path = "../faucet", version = "=1.6.20" }
+solana-ledger = { path = "../ledger", version = "=1.6.20" }
+solana-logger = { path = "../logger", version = "=1.6.20" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.6.20" }
+solana-metrics = { path = "../metrics", version = "=1.6.20" }
+solana-measure = { path = "../measure", version = "=1.6.20" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.20" }
+solana-perf = { path = "../perf", version = "=1.6.20" }
+solana-program-test = { path = "../program-test", version = "=1.6.20" }
+solana-runtime = { path = "../runtime", version = "=1.6.20" }
+solana-sdk = { path = "../sdk", version = "=1.6.20" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.6.20" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.6.20" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.20" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.6.20" }
+solana-streamer = { path = "../streamer", version = "=1.6.20" }
+solana-sys-tuner = { path = "../sys-tuner", version = "=1.6.20" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.20" }
+solana-version = { path = "../version", version = "=1.6.20" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.20" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio_02 = { version = "0.2", package = "tokio", features = ["full"] }
 tokio-util = { version = "0.3", features = ["codec"] } # This crate needs to stay in sync with tokio_02, until that dependency can be removed
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.6.5" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.6.20" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/benches/crds_gossip_pull.rs

@@ -29,13 +29,8 @@ fn bench_hash_as_u64(bencher: &mut Bencher) {
 fn bench_build_crds_filters(bencher: &mut Bencher) {
     let thread_pool = ThreadPoolBuilder::new().build().unwrap();
     let mut rng = thread_rng();
-    let mut crds_gossip_pull = CrdsGossipPull::default();
+    let crds_gossip_pull = CrdsGossipPull::default();
     let mut crds = Crds::default();
-    for _ in 0..50_000 {
-        crds_gossip_pull
-            .purged_values
-            .push_back((solana_sdk::hash::new_rand(&mut rng), rng.gen()));
-    }
     let mut num_inserts = 0;
     for _ in 0..90_000 {
         if crds

core/benches/crds_shards.rs

@@ -3,30 +3,34 @@
 extern crate test;
 
 use rand::{thread_rng, Rng};
-use solana_core::contact_info::ContactInfo;
-use solana_core::crds::VersionedCrdsValue;
-use solana_core::crds_shards::CrdsShards;
-use solana_core::crds_value::{CrdsData, CrdsValue};
-use solana_sdk::pubkey;
+use solana_core::{
+    crds::{Crds, VersionedCrdsValue},
+    crds_shards::CrdsShards,
+    crds_value::CrdsValue,
+};
 use solana_sdk::timing::timestamp;
+use std::iter::repeat_with;
 use test::Bencher;
 
 const CRDS_SHARDS_BITS: u32 = 8;
 
-fn new_test_crds_value() -> VersionedCrdsValue {
-    let data = CrdsData::ContactInfo(ContactInfo::new_localhost(&pubkey::new_rand(), timestamp()));
-    VersionedCrdsValue::new(timestamp(), CrdsValue::new_unsigned(data))
+fn new_test_crds_value<R: Rng>(rng: &mut R) -> VersionedCrdsValue {
+    let value = CrdsValue::new_rand(rng, None);
+    let label = value.label();
+    let mut crds = Crds::default();
+    crds.insert(value, timestamp()).unwrap();
+    crds.get(&label).cloned().unwrap()
 }
 
 fn bench_crds_shards_find(bencher: &mut Bencher, num_values: usize, mask_bits: u32) {
-    let values: Vec<VersionedCrdsValue> = std::iter::repeat_with(new_test_crds_value)
+    let mut rng = thread_rng();
+    let values: Vec<_> = repeat_with(|| new_test_crds_value(&mut rng))
         .take(num_values)
         .collect();
     let mut shards = CrdsShards::new(CRDS_SHARDS_BITS);
     for (index, value) in values.iter().enumerate() {
         assert!(shards.insert(index, value));
     }
-    let mut rng = thread_rng();
     bencher.iter(|| {
         let mask = rng.gen();
         let _hits = shards.find(mask, mask_bits).count();

core/benches/retransmit_stage.rs

@@ -81,8 +81,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     let keypair = Arc::new(Keypair::new());
     let slot = 0;
     let parent = 0;
-    let shredder =
-        Shredder::new(slot, parent, 0.0, keypair, 0, 0).expect("Failed to create entry shredder");
+    let shredder = Shredder::new(slot, parent, keypair, 0, 0).unwrap();
     let mut data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
 
     let num_packets = data_shreds.len();

core/benches/shredder.rs

@@ -8,8 +8,8 @@ use raptorq::{Decoder, Encoder};
 use solana_ledger::entry::{create_ticks, Entry};
 use solana_ledger::shred::{
     max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, Shredder,
-    MAX_DATA_SHREDS_PER_FEC_BLOCK, RECOMMENDED_FEC_RATE, SHRED_PAYLOAD_SIZE,
-    SIZE_OF_DATA_SHRED_IGNORED_TAIL, SIZE_OF_DATA_SHRED_PAYLOAD,
+    MAX_DATA_SHREDS_PER_FEC_BLOCK, SHRED_PAYLOAD_SIZE, SIZE_OF_DATA_SHRED_IGNORED_TAIL,
+    SIZE_OF_DATA_SHRED_PAYLOAD,
 };
 use solana_perf::test_tx;
 use solana_sdk::hash::Hash;
@@ -39,8 +39,7 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
         Some(shred_size),
     );
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
-    let shredder =
-        Shredder::new(1, 0, RECOMMENDED_FEC_RATE, Arc::new(Keypair::new()), 0, 0).unwrap();
+    let shredder = Shredder::new(1, 0, Arc::new(Keypair::new()), 0, 0).unwrap();
     let data_shreds = shredder
         .entries_to_data_shreds(
             &entries,
@@ -75,7 +74,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
     let num_ticks = max_ticks_per_n_shreds(1, Some(SIZE_OF_DATA_SHRED_PAYLOAD)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
+        let shredder = Shredder::new(1, 0, kp.clone(), 0, 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -94,7 +93,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     // 1Mb
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
+        let shredder = Shredder::new(1, 0, kp.clone(), 0, 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -107,7 +106,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
     let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
     let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
-    let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0, 0).unwrap();
+    let shredder = Shredder::new(1, 0, kp, 0, 0).unwrap();
     let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
     bencher.iter(|| {
         let raw = &mut Shredder::deshred(&data_shreds).unwrap();
@@ -133,9 +132,8 @@ fn bench_shredder_coding(bencher: &mut Bencher) {
     let data_shreds = make_shreds(symbol_count);
     bencher.iter(|| {
         Shredder::generate_coding_shreds(
-            RECOMMENDED_FEC_RATE,
             &data_shreds[..symbol_count],
-            symbol_count,
+            true, // is_last_in_slot
         )
         .len();
     })
@@ -146,18 +144,16 @@ fn bench_shredder_decoding(bencher: &mut Bencher) {
     let symbol_count = MAX_DATA_SHREDS_PER_FEC_BLOCK as usize;
     let data_shreds = make_shreds(symbol_count);
     let coding_shreds = Shredder::generate_coding_shreds(
-        RECOMMENDED_FEC_RATE,
         &data_shreds[..symbol_count],
-        symbol_count,
+        true, // is_last_in_slot
     );
     bencher.iter(|| {
         Shredder::try_recovery(
             coding_shreds[..].to_vec(),
             symbol_count,
             symbol_count,
-            0,
-            0,
-            1,
+            0, // first index
+            1, // slot
         )
         .unwrap();
     })

core/src/bank_weight_fork_choice.rs

@@ -1,154 +0,0 @@
-use crate::{
-    consensus::{ComputedBankState, Tower},
-    fork_choice::ForkChoice,
-    progress_map::{ForkStats, ProgressMap},
-};
-use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::{clock::Slot, timing};
-use std::time::Instant;
-use std::{
-    collections::{HashMap, HashSet},
-    sync::{Arc, RwLock},
-};
-
-#[derive(Default)]
-pub struct BankWeightForkChoice {}
-
-impl ForkChoice for BankWeightForkChoice {
-    fn compute_bank_stats(
-        &mut self,
-        bank: &Bank,
-        _tower: &Tower,
-        progress: &mut ProgressMap,
-        computed_bank_state: &ComputedBankState,
-    ) {
-        let bank_slot = bank.slot();
-        // Only time progress map should be missing a bank slot
-        // is if this node was the leader for this slot as those banks
-        // are not replayed in replay_active_banks()
-        let parent_weight = bank
-            .parent()
-            .and_then(|b| progress.get(&b.slot()))
-            .map(|x| x.fork_stats.fork_weight)
-            .unwrap_or(0);
-
-        let stats = progress
-            .get_fork_stats_mut(bank_slot)
-            .expect("All frozen banks must exist in the Progress map");
-
-        let ComputedBankState { bank_weight, .. } = computed_bank_state;
-        stats.weight = *bank_weight;
-        stats.fork_weight = stats.weight + parent_weight;
-    }
-
-    // Returns:
-    // 1) The heaviest overall bank
-    // 2) The heaviest bank on the same fork as the last vote (doesn't require a
-    // switching proof to vote for)
-    fn select_forks(
-        &self,
-        frozen_banks: &[Arc<Bank>],
-        tower: &Tower,
-        progress: &ProgressMap,
-        ancestors: &HashMap<u64, HashSet<u64>>,
-        _bank_forks: &RwLock<BankForks>,
-    ) -> (Arc<Bank>, Option<Arc<Bank>>) {
-        let tower_start = Instant::now();
-        assert!(!frozen_banks.is_empty());
-        let num_frozen_banks = frozen_banks.len();
-
-        trace!("frozen_banks {}", frozen_banks.len());
-        let num_old_banks = frozen_banks
-            .iter()
-            .filter(|b| b.slot() < tower.root())
-            .count();
-
-        let last_voted_slot = tower.last_voted_slot();
-        let mut heaviest_bank_on_same_fork = None;
-        let mut heaviest_same_fork_weight = 0;
-        let stats: Vec<&ForkStats> = frozen_banks
-            .iter()
-            .map(|bank| {
-                // Only time progress map should be missing a bank slot
-                // is if this node was the leader for this slot as those banks
-                // are not replayed in replay_active_banks()
-                let stats = progress
-                    .get_fork_stats(bank.slot())
-                    .expect("All frozen banks must exist in the Progress map");
-
-                if let Some(last_voted_slot) = last_voted_slot {
-                    if ancestors
-                        .get(&bank.slot())
-                        .expect("Entry in frozen banks must exist in ancestors")
-                        .contains(&last_voted_slot)
-                    {
-                        // Descendant of last vote cannot be locked out
-                        assert!(!stats.is_locked_out);
-
-                        // ancestors(slot) should not contain the slot itself,
-                        // so we should never get the same bank as the last vote
-                        assert_ne!(bank.slot(), last_voted_slot);
-                        // highest weight, lowest slot first. frozen_banks is sorted
-                        // from least slot to greatest slot, so if two banks have
-                        // the same fork weight, the lower slot will be picked
-                        if stats.fork_weight > heaviest_same_fork_weight {
-                            heaviest_bank_on_same_fork = Some(bank.clone());
-                            heaviest_same_fork_weight = stats.fork_weight;
-                        }
-                    }
-                }
-
-                stats
-            })
-            .collect();
-        let num_not_recent = stats.iter().filter(|s| !s.is_recent).count();
-        let num_has_voted = stats.iter().filter(|s| s.has_voted).count();
-        let num_empty = stats.iter().filter(|s| s.is_empty).count();
-        let num_threshold_failure = stats.iter().filter(|s| !s.vote_threshold).count();
-        let num_votable_threshold_failure = stats
-            .iter()
-            .filter(|s| s.is_recent && !s.has_voted && !s.vote_threshold)
-            .count();
-
-        let mut candidates: Vec<_> = frozen_banks.iter().zip(stats.iter()).collect();
-
-        //highest weight, lowest slot first
-        candidates.sort_by_key(|b| (b.1.fork_weight, 0i64 - b.0.slot() as i64));
-        let rv = candidates
-            .last()
-            .expect("frozen banks was nonempty so candidates must also be nonempty");
-        let ms = timing::duration_as_ms(&tower_start.elapsed());
-        let weights: Vec<(u128, u64, u64)> = candidates
-            .iter()
-            .map(|x| (x.1.weight, x.0.slot(), x.1.block_height))
-            .collect();
-        debug!(
-            "@{:?} tower duration: {:?} len: {}/{} weights: {:?}",
-            timing::timestamp(),
-            ms,
-            candidates.len(),
-            stats.iter().filter(|s| !s.has_voted).count(),
-            weights,
-        );
-        datapoint_debug!(
-            "replay_stage-select_forks",
-            ("frozen_banks", num_frozen_banks as i64, i64),
-            ("not_recent", num_not_recent as i64, i64),
-            ("has_voted", num_has_voted as i64, i64),
-            ("old_banks", num_old_banks as i64, i64),
-            ("empty_banks", num_empty as i64, i64),
-            ("threshold_failure", num_threshold_failure as i64, i64),
-            (
-                "votable_threshold_failure",
-                num_votable_threshold_failure as i64,
-                i64
-            ),
-            ("tower_duration", ms as i64, i64),
-        );
-
-        (rv.0.clone(), heaviest_bank_on_same_fork)
-    }
-
-    fn mark_fork_invalid_candidate(&mut self, _invalid_slot: Slot) {}
-    fn mark_fork_valid_candidate(&mut self, _valid_slots: &[Slot]) {}
-}

core/src/banking_stage.rs

@@ -567,15 +567,8 @@ impl BankingStage {
             }
             return;
         }
-        let next_leader = match poh_recorder
-            .lock()
-            .unwrap()
-            .leader_after_n_slots(FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET)
-        {
-            Some(pubkey) => pubkey,
-            None => return,
-        };
-        let addr = match cluster_info.lookup_contact_info(&next_leader, |ci| ci.tpu_forwards) {
+
+        let addr = match next_leader_tpu_forwards(cluster_info, poh_recorder) {
             Some(addr) => addr,
             None => return,
         };
@@ -823,6 +816,7 @@ impl BankingStage {
                     TransactionTokenBalancesSet::new(pre_token_balances, post_token_balances),
                     inner_instructions,
                     transaction_logs,
+                    tx_results.rent_debits,
                 );
             }
         }
@@ -991,16 +985,15 @@ impl BankingStage {
     fn transactions_from_packets(
         msgs: &Packets,
         transaction_indexes: &[usize],
-        secp256k1_program_enabled: bool,
+        libsecp256k1_0_5_upgrade_enabled: bool,
     ) -> (Vec<HashedTransaction<'static>>, Vec<usize>) {
         transaction_indexes
             .iter()
             .filter_map(|tx_index| {
                 let p = &msgs.packets[*tx_index];
                 let tx: Transaction = limited_deserialize(&p.data[0..p.meta.size]).ok()?;
-                if secp256k1_program_enabled {
-                    tx.verify_precompiles().ok()?;
-                }
+                tx.verify_precompiles(libsecp256k1_0_5_upgrade_enabled)
+                    .ok()?;
                 let message_bytes = Self::packet_message(p)?;
                 let message_hash = Message::hash_raw_message(message_bytes);
                 Some((
@@ -1064,7 +1057,7 @@ impl BankingStage {
         let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets(
             msgs,
             &packet_indexes,
-            bank.secp256k1_program_enabled(),
+            bank.libsecp256k1_0_5_upgrade_enabled(),
         );
         packet_conversion_time.stop();
 
@@ -1135,7 +1128,7 @@ impl BankingStage {
         let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets(
             msgs,
             &transaction_indexes,
-            bank.secp256k1_program_enabled(),
+            bank.libsecp256k1_0_5_upgrade_enabled(),
         );
 
         let tx_count = transaction_to_packet_indexes.len();
@@ -1369,6 +1362,36 @@ impl BankingStage {
     }
 }
 
+pub(crate) fn next_leader_tpu(
+    cluster_info: &ClusterInfo,
+    poh_recorder: &Mutex<PohRecorder>,
+) -> Option<std::net::SocketAddr> {
+    if let Some(leader_pubkey) = poh_recorder
+        .lock()
+        .unwrap()
+        .leader_after_n_slots(FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET)
+    {
+        cluster_info.lookup_contact_info(&leader_pubkey, |leader| leader.tpu)
+    } else {
+        None
+    }
+}
+
+fn next_leader_tpu_forwards(
+    cluster_info: &ClusterInfo,
+    poh_recorder: &Arc<Mutex<PohRecorder>>,
+) -> Option<std::net::SocketAddr> {
+    if let Some(leader_pubkey) = poh_recorder
+        .lock()
+        .unwrap()
+        .leader_after_n_slots(FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET)
+    {
+        cluster_info.lookup_contact_info(&leader_pubkey, |leader| leader.tpu_forwards)
+    } else {
+        None
+    }
+}
+
 pub fn create_test_recorder(
     bank: &Arc<Bank>,
     blockstore: &Arc<Blockstore>,
@@ -1391,6 +1414,7 @@ pub fn create_test_recorder(
         blockstore,
         &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
         &poh_config,
+        exit.clone(),
     );
     poh_recorder.set_bank(&bank);
 
@@ -1436,10 +1460,7 @@ mod tests {
     use std::{
         net::SocketAddr,
         path::Path,
-        sync::{
-            atomic::{AtomicBool, Ordering},
-            mpsc::Receiver,
-        },
+        sync::atomic::{AtomicBool, Ordering},
         thread::sleep,
     };
 
@@ -1796,11 +1817,12 @@ mod tests {
                 &Arc::new(blockstore),
                 &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
                 &Arc::new(PohConfig::default()),
+                Arc::new(AtomicBool::default()),
             );
             let recorder = poh_recorder.recorder();
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
-            let (poh_simulator, exit) = simulate_poh(record_receiver, &poh_recorder);
+            let poh_simulator = simulate_poh(record_receiver, &poh_recorder);
 
             poh_recorder.lock().unwrap().set_working_bank(working_bank);
             let pubkey = solana_sdk::pubkey::new_rand();
@@ -1870,7 +1892,11 @@ mod tests {
             // Should receive nothing from PohRecorder b/c record failed
             assert!(entry_receiver.try_recv().is_err());
 
-            exit.store(true, Ordering::Relaxed);
+            poh_recorder
+                .lock()
+                .unwrap()
+                .is_exited
+                .store(true, Ordering::Relaxed);
             let _ = poh_simulator.join();
         }
         Blockstore::destroy(&ledger_path).unwrap();
@@ -2062,11 +2088,12 @@ mod tests {
                 &Arc::new(blockstore),
                 &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
                 &Arc::new(PohConfig::default()),
+                Arc::new(AtomicBool::default()),
             );
             let recorder = poh_recorder.recorder();
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
-            let (poh_simulator, exit) = simulate_poh(record_receiver, &poh_recorder);
+            let poh_simulator = simulate_poh(record_receiver, &poh_recorder);
 
             poh_recorder.lock().unwrap().set_working_bank(working_bank);
             let (gossip_vote_sender, _gossip_vote_receiver) = unbounded();
@@ -2121,7 +2148,11 @@ mod tests {
                 Err(PohRecorderError::MaxHeightReached)
             );
 
-            exit.store(true, Ordering::Relaxed);
+            poh_recorder
+                .lock()
+                .unwrap()
+                .is_exited
+                .store(true, Ordering::Relaxed);
             let _ = poh_simulator.join();
 
             assert_eq!(bank.get_balance(&pubkey), 1);
@@ -2130,12 +2161,11 @@ mod tests {
     }
 
     fn simulate_poh(
-        record_receiver: Receiver<Record>,
+        record_receiver: CrossbeamReceiver<Record>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
-    ) -> (JoinHandle<()>, Arc<AtomicBool>) {
-        let exit = Arc::new(AtomicBool::new(false));
-        let exit_ = exit.clone();
+    ) -> JoinHandle<()> {
         let poh_recorder = poh_recorder.clone();
+        let is_exited = poh_recorder.lock().unwrap().is_exited.clone();
         let tick_producer = Builder::new()
             .name("solana-simulate_poh".to_string())
             .spawn(move || loop {
@@ -2144,11 +2174,11 @@ mod tests {
                     &record_receiver,
                     Duration::from_millis(10),
                 );
-                if exit_.load(Ordering::Relaxed) {
+                if is_exited.load(Ordering::Relaxed) {
                     break;
                 }
             });
-        (tick_producer.unwrap(), exit)
+        tick_producer.unwrap()
     }
 
     #[test]
@@ -2189,13 +2219,14 @@ mod tests {
                 &Arc::new(blockstore),
                 &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
                 &Arc::new(PohConfig::default()),
+                Arc::new(AtomicBool::default()),
             );
             let recorder = poh_recorder.recorder();
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
             poh_recorder.lock().unwrap().set_working_bank(working_bank);
 
-            let (poh_simulator, exit) = simulate_poh(record_receiver, &poh_recorder);
+            let poh_simulator = simulate_poh(record_receiver, &poh_recorder);
 
             let (gossip_vote_sender, _gossip_vote_receiver) = unbounded();
 
@@ -2208,7 +2239,11 @@ mod tests {
                 &gossip_vote_sender,
             );
 
-            exit.store(true, Ordering::Relaxed);
+            poh_recorder
+                .lock()
+                .unwrap()
+                .is_exited
+                .store(true, Ordering::Relaxed);
             let _ = poh_simulator.join();
 
             assert!(result.is_ok());
@@ -2290,14 +2325,14 @@ mod tests {
                 &Arc::new(blockstore),
                 &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
                 &Arc::new(PohConfig::default()),
+                Arc::new(AtomicBool::default()),
             );
 
             // Poh Recorder has no working bank, so should throw MaxHeightReached error on
             // record
             let recorder = poh_recorder.recorder();
 
-            let (poh_simulator, exit) =
-                simulate_poh(record_receiver, &Arc::new(Mutex::new(poh_recorder)));
+            let poh_simulator = simulate_poh(record_receiver, &Arc::new(Mutex::new(poh_recorder)));
 
             let (gossip_vote_sender, _gossip_vote_receiver) = unbounded();
 
@@ -2317,7 +2352,7 @@ mod tests {
             let expected: Vec<usize> = (0..transactions.len()).collect();
             assert_eq!(retryable_txs, expected);
 
-            exit.store(true, Ordering::Relaxed);
+            recorder.is_exited.store(true, Ordering::Relaxed);
             let _ = poh_simulator.join();
         }
 
@@ -2375,11 +2410,12 @@ mod tests {
                 &blockstore,
                 &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
                 &Arc::new(PohConfig::default()),
+                Arc::new(AtomicBool::default()),
             );
             let recorder = poh_recorder.recorder();
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
-            let (poh_simulator, exit) = simulate_poh(record_receiver, &poh_recorder);
+            let poh_simulator = simulate_poh(record_receiver, &poh_recorder);
 
             poh_recorder.lock().unwrap().set_working_bank(working_bank);
 
@@ -2434,7 +2470,11 @@ mod tests {
                 }
             }
 
-            exit.store(true, Ordering::Relaxed);
+            poh_recorder
+                .lock()
+                .unwrap()
+                .is_exited
+                .store(true, Ordering::Relaxed);
             let _ = poh_simulator.join();
         }
         Blockstore::destroy(&ledger_path).unwrap();
@@ -2449,7 +2489,6 @@ mod tests {
         Arc<Mutex<PohRecorder>>,
         Receiver<WorkingBankEntry>,
         JoinHandle<()>,
-        Arc<AtomicBool>,
     ) {
         Blockstore::destroy(&ledger_path).unwrap();
         let genesis_config_info = create_slow_genesis_config(10_000);
@@ -2461,6 +2500,7 @@ mod tests {
         let blockstore =
             Blockstore::open(&ledger_path).expect("Expected to be able to open database ledger");
         let bank = Arc::new(Bank::new_no_wallclock_throttle(&genesis_config));
+        let exit = Arc::new(AtomicBool::default());
         let (poh_recorder, entry_receiver, record_receiver) = PohRecorder::new(
             bank.tick_height(),
             bank.last_blockhash(),
@@ -2471,6 +2511,7 @@ mod tests {
             &Arc::new(blockstore),
             &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
             &Arc::new(PohConfig::default()),
+            exit,
         );
         let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
@@ -2483,7 +2524,7 @@ mod tests {
             system_transaction::transfer(&mint_keypair, &pubkey1, 1, genesis_config.hash()),
             system_transaction::transfer(&mint_keypair, &pubkey2, 1, genesis_config.hash()),
         ];
-        let (poh_simulator, exit) = simulate_poh(record_receiver, &poh_recorder);
+        let poh_simulator = simulate_poh(record_receiver, &poh_recorder);
 
         (
             transactions,
@@ -2491,7 +2532,6 @@ mod tests {
             poh_recorder,
             entry_receiver,
             poh_simulator,
-            exit,
         )
     }
 
@@ -2499,7 +2539,7 @@ mod tests {
     fn test_consume_buffered_packets() {
         let ledger_path = get_tmp_ledger_path!();
         {
-            let (transactions, bank, poh_recorder, _entry_receiver, poh_simulator, exit) =
+            let (transactions, bank, poh_recorder, _entry_receiver, poh_simulator) =
                 setup_conflicting_transactions(&ledger_path);
             let recorder = poh_recorder.lock().unwrap().recorder();
             let num_conflicting_transactions = transactions.len();
@@ -2553,7 +2593,11 @@ mod tests {
                     assert_eq!(buffered_packets[0].1.len(), num_expected_unprocessed);
                 }
             }
-            exit.store(true, Ordering::Relaxed);
+            poh_recorder
+                .lock()
+                .unwrap()
+                .is_exited
+                .store(true, Ordering::Relaxed);
             let _ = poh_simulator.join();
         }
         Blockstore::destroy(&ledger_path).unwrap();
@@ -2563,7 +2607,7 @@ mod tests {
     fn test_consume_buffered_packets_interrupted() {
         let ledger_path = get_tmp_ledger_path!();
         {
-            let (transactions, bank, poh_recorder, _entry_receiver, poh_simulator, exit) =
+            let (transactions, bank, poh_recorder, _entry_receiver, poh_simulator) =
                 setup_conflicting_transactions(&ledger_path);
             let num_conflicting_transactions = transactions.len();
             let packets_vec = to_packets_chunked(&transactions, 1);
@@ -2613,7 +2657,7 @@ mod tests {
                     // should still be unprocessed
                     assert_eq!(
                         buffered_packets.len(),
-                        packets_vec[interrupted_iteration + 1..].iter().count()
+                        packets_vec[interrupted_iteration + 1..].len()
                     );
                     for ((remaining_unprocessed_packet, _, _forwarded), original_packet) in
                         buffered_packets
@@ -2640,7 +2684,11 @@ mod tests {
             }
 
             t_consume.join().unwrap();
-            exit.store(true, Ordering::Relaxed);
+            poh_recorder
+                .lock()
+                .unwrap()
+                .is_exited
+                .store(true, Ordering::Relaxed);
             let _ = poh_simulator.join();
         }
         Blockstore::destroy(&ledger_path).unwrap();

core/src/bigtable_upload_service.rs

@@ -8,10 +8,10 @@ use std::{
 use tokio::runtime::Runtime;
 
 // Delay uploading the largest confirmed root for this many slots.  This is done in an attempt to
-// ensure that the `CacheBlockTimeService` has had enough time to add the block time for the root
+// ensure that the `CacheBlockMetaService` has had enough time to add the block time for the root
 // before it's uploaded to BigTable.
 //
-// A more direct connection between CacheBlockTimeService and BigTableUploadService would be
+// A more direct connection between CacheBlockMetaService and BigTableUploadService would be
 // preferable...
 const LARGEST_CONFIRMED_ROOT_UPLOAD_DELAY: usize = 100;
 

core/src/broadcast_stage.rs

@@ -23,7 +23,7 @@ use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
 use solana_runtime::bank::Bank;
 use solana_sdk::timing::timestamp;
 use solana_sdk::{clock::Slot, pubkey::Pubkey};
-use solana_streamer::sendmmsg::send_mmsg;
+use solana_streamer::{sendmmsg::send_mmsg, socket::is_global};
 use std::sync::atomic::AtomicU64;
 use std::{
     collections::HashMap,
@@ -387,10 +387,15 @@ pub fn broadcast_shreds(
     let mut shred_select = Measure::start("shred_select");
     let packets: Vec<_> = shreds
         .iter()
-        .map(|shred| {
+        .filter_map(|shred| {
             let broadcast_index = weighted_best(&peers_and_stakes, shred.seed());
+            let node = &peers[broadcast_index];
 
-            (&shred.payload, &peers[broadcast_index].tvu)
+            if is_global(&node.tvu) {
+                Some((&shred.payload, &node.tvu))
+            } else {
+                None
+            }
         })
         .collect();
     shred_select.stop();
@@ -447,7 +452,7 @@ pub mod test {
         entry::create_ticks,
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
-        shred::{max_ticks_per_n_shreds, ProcessShredsStats, Shredder, RECOMMENDED_FEC_RATE},
+        shred::{max_ticks_per_n_shreds, ProcessShredsStats, Shredder},
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::{
@@ -476,7 +481,7 @@ pub mod test {
         let coding_shreds = Shredder::data_shreds_to_coding_shreds(
             &keypair,
             &data_shreds[0..],
-            RECOMMENDED_FEC_RATE,
+            true, // is_last_in_slot
             &mut ProcessShredsStats::default(),
         )
         .unwrap();

core/src/broadcast_stage/broadcast_fake_shreds_run.rs

@@ -1,6 +1,6 @@
 use super::*;
 use solana_ledger::entry::Entry;
-use solana_ledger::shred::{Shredder, RECOMMENDED_FEC_RATE};
+use solana_ledger::shred::Shredder;
 use solana_sdk::hash::Hash;
 use solana_sdk::signature::Keypair;
 
@@ -47,7 +47,6 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         let shredder = Shredder::new(
             bank.slot(),
             bank.parent().unwrap().slot(),
-            RECOMMENDED_FEC_RATE,
             self.keypair.clone(),
             (bank.tick_height() % bank.ticks_per_slot()) as u8,
             self.shred_version,

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -71,7 +71,6 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         let shredder = Shredder::new(
             bank.slot(),
             bank.parent().unwrap().slot(),
-            0.0,
             self.keypair.clone(),
             (bank.tick_height() % bank.ticks_per_slot()) as u8,
             self.shred_version,

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -8,7 +8,7 @@ use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
 use solana_ledger::{
     entry::Entry,
     shred::{
-        ProcessShredsStats, Shred, Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK, RECOMMENDED_FEC_RATE,
+        ProcessShredsStats, Shred, Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK,
         SHRED_TICK_REFERENCE_MASK,
     },
 };
@@ -121,7 +121,6 @@ impl StandardBroadcastRun {
         let (data_shreds, next_shred_index) = Shredder::new(
             slot,
             parent_slot,
-            RECOMMENDED_FEC_RATE,
             self.keypair.clone(),
             reference_tick,
             self.shred_version,
@@ -261,7 +260,6 @@ impl StandardBroadcastRun {
             num_expected_batches,
             slot_start_ts: self
                 .slot_broadcast_start
-                .clone()
                 .expect("Start timestamp must exist for a slot if we're broadcasting the slot"),
         });
         get_leader_schedule_time.stop();
@@ -452,8 +450,7 @@ fn make_coding_shreds(
                 .collect()
         }
     };
-    Shredder::data_shreds_to_coding_shreds(keypair, &data_shreds, RECOMMENDED_FEC_RATE, stats)
-        .unwrap()
+    Shredder::data_shreds_to_coding_shreds(keypair, &data_shreds, is_slot_end, stats).unwrap()
 }
 
 impl BroadcastRun for StandardBroadcastRun {

core/src/cache_block_meta_service.rs

@@ -0,0 +1,74 @@
+pub use solana_ledger::blockstore_processor::CacheBlockMetaSender;
+use {
+    crossbeam_channel::{Receiver, RecvTimeoutError},
+    solana_ledger::blockstore::Blockstore,
+    solana_measure::measure::Measure,
+    solana_runtime::bank::Bank,
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::{self, Builder, JoinHandle},
+        time::Duration,
+    },
+};
+
+pub type CacheBlockMetaReceiver = Receiver<Arc<Bank>>;
+
+pub struct CacheBlockMetaService {
+    thread_hdl: JoinHandle<()>,
+}
+
+const CACHE_BLOCK_TIME_WARNING_MS: u64 = 150;
+
+impl CacheBlockMetaService {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        cache_block_meta_receiver: CacheBlockMetaReceiver,
+        blockstore: Arc<Blockstore>,
+        exit: &Arc<AtomicBool>,
+    ) -> Self {
+        let exit = exit.clone();
+        let thread_hdl = Builder::new()
+            .name("solana-cache-block-time".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                let recv_result = cache_block_meta_receiver.recv_timeout(Duration::from_secs(1));
+                match recv_result {
+                    Err(RecvTimeoutError::Disconnected) => {
+                        break;
+                    }
+                    Ok(bank) => {
+                        let mut cache_block_meta_timer = Measure::start("cache_block_meta_timer");
+                        Self::cache_block_meta(bank, &blockstore);
+                        cache_block_meta_timer.stop();
+                        if cache_block_meta_timer.as_ms() > CACHE_BLOCK_TIME_WARNING_MS {
+                            warn!(
+                                "cache_block_meta operation took: {}ms",
+                                cache_block_meta_timer.as_ms()
+                            );
+                        }
+                    }
+                    _ => {}
+                }
+            })
+            .unwrap();
+        Self { thread_hdl }
+    }
+
+    fn cache_block_meta(bank: Arc<Bank>, blockstore: &Arc<Blockstore>) {
+        if let Err(e) = blockstore.cache_block_time(bank.slot(), bank.clock().unix_timestamp) {
+            error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
+        }
+        if let Err(e) = blockstore.cache_block_height(bank.slot(), bank.block_height()) {
+            error!("cache_block_height failed: slot {:?} {:?}", bank.slot(), e);
+        }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}

core/src/cache_block_time_service.rs

@@ -1,69 +0,0 @@
-use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
-use solana_ledger::blockstore::Blockstore;
-use solana_measure::measure::Measure;
-use solana_runtime::bank::Bank;
-use std::{
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-    thread::{self, Builder, JoinHandle},
-    time::Duration,
-};
-
-pub type CacheBlockTimeReceiver = Receiver<Arc<Bank>>;
-pub type CacheBlockTimeSender = Sender<Arc<Bank>>;
-
-pub struct CacheBlockTimeService {
-    thread_hdl: JoinHandle<()>,
-}
-
-const CACHE_BLOCK_TIME_WARNING_MS: u64 = 150;
-
-impl CacheBlockTimeService {
-    #[allow(clippy::new_ret_no_self)]
-    pub fn new(
-        cache_block_time_receiver: CacheBlockTimeReceiver,
-        blockstore: Arc<Blockstore>,
-        exit: &Arc<AtomicBool>,
-    ) -> Self {
-        let exit = exit.clone();
-        let thread_hdl = Builder::new()
-            .name("solana-cache-block-time".to_string())
-            .spawn(move || loop {
-                if exit.load(Ordering::Relaxed) {
-                    break;
-                }
-                let recv_result = cache_block_time_receiver.recv_timeout(Duration::from_secs(1));
-                match recv_result {
-                    Err(RecvTimeoutError::Disconnected) => {
-                        break;
-                    }
-                    Ok(bank) => {
-                        let mut cache_block_time_timer = Measure::start("cache_block_time_timer");
-                        Self::cache_block_time(bank, &blockstore);
-                        cache_block_time_timer.stop();
-                        if cache_block_time_timer.as_ms() > CACHE_BLOCK_TIME_WARNING_MS {
-                            warn!(
-                                "cache_block_time operation took: {}ms",
-                                cache_block_time_timer.as_ms()
-                            );
-                        }
-                    }
-                    _ => {}
-                }
-            })
-            .unwrap();
-        Self { thread_hdl }
-    }
-
-    fn cache_block_time(bank: Arc<Bank>, blockstore: &Arc<Blockstore>) {
-        if let Err(e) = blockstore.cache_block_time(bank.slot(), bank.clock().unix_timestamp) {
-            error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
-        }
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        self.thread_hdl.join()
-    }
-}

core/src/cluster_info_metrics.rs

@@ -0,0 +1,404 @@
+use crate::crds_gossip::CrdsGossip;
+use solana_measure::measure::Measure;
+use solana_sdk::pubkey::Pubkey;
+use std::{
+    collections::HashMap,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        RwLock,
+    },
+    time::Instant,
+};
+
+#[derive(Default)]
+pub(crate) struct Counter(AtomicU64);
+
+impl Counter {
+    pub(crate) fn add_measure(&self, x: &mut Measure) {
+        x.stop();
+        self.0.fetch_add(x.as_us(), Ordering::Relaxed);
+    }
+    pub(crate) fn add_relaxed(&self, x: u64) {
+        self.0.fetch_add(x, Ordering::Relaxed);
+    }
+    fn clear(&self) -> u64 {
+        self.0.swap(0, Ordering::Relaxed)
+    }
+}
+
+pub(crate) struct ScopedTimer<'a> {
+    clock: Instant,
+    metric: &'a AtomicU64,
+}
+
+impl<'a> From<&'a Counter> for ScopedTimer<'a> {
+    // Output should be assigned to a *named* variable, otherwise it is
+    // immediately dropped.
+    #[must_use]
+    fn from(counter: &'a Counter) -> Self {
+        Self {
+            clock: Instant::now(),
+            metric: &counter.0,
+        }
+    }
+}
+
+impl Drop for ScopedTimer<'_> {
+    fn drop(&mut self) {
+        let micros = self.clock.elapsed().as_micros();
+        self.metric.fetch_add(micros as u64, Ordering::Relaxed);
+    }
+}
+
+#[derive(Default)]
+pub(crate) struct GossipStats {
+    pub(crate) all_tvu_peers: Counter,
+    pub(crate) entrypoint2: Counter,
+    pub(crate) entrypoint: Counter,
+    pub(crate) epoch_slots_lookup: Counter,
+    pub(crate) filter_pull_response: Counter,
+    pub(crate) generate_pull_responses: Counter,
+    pub(crate) get_accounts_hash: Counter,
+    pub(crate) get_votes: Counter,
+    pub(crate) gossip_packets_dropped_count: Counter,
+    pub(crate) handle_batch_ping_messages_time: Counter,
+    pub(crate) handle_batch_pong_messages_time: Counter,
+    pub(crate) handle_batch_prune_messages_time: Counter,
+    pub(crate) handle_batch_pull_requests_time: Counter,
+    pub(crate) handle_batch_pull_responses_time: Counter,
+    pub(crate) handle_batch_push_messages_time: Counter,
+    pub(crate) mark_pull_request: Counter,
+    pub(crate) new_pull_requests: Counter,
+    pub(crate) new_pull_requests_count: Counter,
+    pub(crate) new_pull_requests_pings_count: Counter,
+    pub(crate) new_push_requests2: Counter,
+    pub(crate) new_push_requests: Counter,
+    pub(crate) new_push_requests_num: Counter,
+    pub(crate) packets_received_count: Counter,
+    pub(crate) packets_received_prune_messages_count: Counter,
+    pub(crate) packets_received_pull_requests_count: Counter,
+    pub(crate) packets_received_pull_responses_count: Counter,
+    pub(crate) packets_received_push_messages_count: Counter,
+    pub(crate) packets_received_verified_count: Counter,
+    pub(crate) packets_sent_gossip_requests_count: Counter,
+    pub(crate) packets_sent_prune_messages_count: Counter,
+    pub(crate) packets_sent_pull_requests_count: Counter,
+    pub(crate) packets_sent_pull_responses_count: Counter,
+    pub(crate) packets_sent_push_messages_count: Counter,
+    pub(crate) process_gossip_packets_time: Counter,
+    pub(crate) process_prune: Counter,
+    pub(crate) process_pull_requests: Counter,
+    pub(crate) process_pull_response: Counter,
+    pub(crate) process_pull_response_count: Counter,
+    pub(crate) process_pull_response_fail_insert: Counter,
+    pub(crate) process_pull_response_fail_timeout: Counter,
+    pub(crate) process_pull_response_len: Counter,
+    pub(crate) process_pull_response_success: Counter,
+    pub(crate) process_pull_response_timeout: Counter,
+    pub(crate) process_push_message: Counter,
+    pub(crate) prune_message_count: Counter,
+    pub(crate) prune_message_len: Counter,
+    pub(crate) prune_received_cache: Counter,
+    pub(crate) pull_from_entrypoint_count: Counter,
+    pub(crate) pull_request_ping_pong_check_failed_count: Counter,
+    pub(crate) pull_requests_count: Counter,
+    pub(crate) purge: Counter,
+    pub(crate) push_message_count: Counter,
+    pub(crate) push_message_value_count: Counter,
+    pub(crate) push_response_count: Counter,
+    pub(crate) push_vote_read: Counter,
+    pub(crate) repair_peers: Counter,
+    pub(crate) require_stake_for_gossip_unknown_feature_set: Counter,
+    pub(crate) require_stake_for_gossip_unknown_stakes: Counter,
+    pub(crate) skip_pull_response_shred_version: Counter,
+    pub(crate) skip_pull_shred_version: Counter,
+    pub(crate) skip_push_message_shred_version: Counter,
+    pub(crate) trim_crds_table_failed: Counter,
+    pub(crate) trim_crds_table_purged_values_count: Counter,
+    pub(crate) tvu_peers: Counter,
+}
+
+pub(crate) fn submit_gossip_stats(
+    stats: &GossipStats,
+    gossip: &RwLock<CrdsGossip>,
+    stakes: &HashMap<Pubkey, u64>,
+) {
+    let (table_size, num_nodes, purged_values_size, failed_inserts_size) = {
+        let gossip = gossip.read().unwrap();
+        (
+            gossip.crds.len(),
+            gossip.crds.num_nodes(),
+            gossip.crds.num_purged(),
+            gossip.pull.failed_inserts.len(),
+        )
+    };
+    let num_nodes_staked = stakes.values().filter(|stake| **stake > 0).count();
+    datapoint_info!(
+        "cluster_info_stats",
+        ("entrypoint", stats.entrypoint.clear(), i64),
+        ("entrypoint2", stats.entrypoint2.clear(), i64),
+        ("push_vote_read", stats.push_vote_read.clear(), i64),
+        ("get_votes", stats.get_votes.clear(), i64),
+        ("get_accounts_hash", stats.get_accounts_hash.clear(), i64),
+        ("all_tvu_peers", stats.all_tvu_peers.clear(), i64),
+        ("tvu_peers", stats.tvu_peers.clear(), i64),
+        (
+            "new_push_requests_num",
+            stats.new_push_requests_num.clear(),
+            i64
+        ),
+        ("table_size", table_size as i64, i64),
+        ("purged_values_size", purged_values_size as i64, i64),
+        ("failed_inserts_size", failed_inserts_size as i64, i64),
+        ("num_nodes", num_nodes as i64, i64),
+        ("num_nodes_staked", num_nodes_staked as i64, i64),
+    );
+    datapoint_info!(
+        "cluster_info_stats2",
+        (
+            "gossip_packets_dropped_count",
+            stats.gossip_packets_dropped_count.clear(),
+            i64
+        ),
+        ("repair_peers", stats.repair_peers.clear(), i64),
+        ("new_push_requests", stats.new_push_requests.clear(), i64),
+        ("new_push_requests2", stats.new_push_requests2.clear(), i64),
+        ("purge", stats.purge.clear(), i64),
+        (
+            "process_gossip_packets_time",
+            stats.process_gossip_packets_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_ping_messages_time",
+            stats.handle_batch_ping_messages_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_pong_messages_time",
+            stats.handle_batch_pong_messages_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_prune_messages_time",
+            stats.handle_batch_prune_messages_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_pull_requests_time",
+            stats.handle_batch_pull_requests_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_pull_responses_time",
+            stats.handle_batch_pull_responses_time.clear(),
+            i64
+        ),
+        (
+            "handle_batch_push_messages_time",
+            stats.handle_batch_push_messages_time.clear(),
+            i64
+        ),
+        (
+            "process_pull_resp",
+            stats.process_pull_response.clear(),
+            i64
+        ),
+        ("filter_pull_resp", stats.filter_pull_response.clear(), i64),
+        (
+            "process_pull_resp_count",
+            stats.process_pull_response_count.clear(),
+            i64
+        ),
+        (
+            "pull_response_fail_insert",
+            stats.process_pull_response_fail_insert.clear(),
+            i64
+        ),
+        (
+            "pull_response_fail_timeout",
+            stats.process_pull_response_fail_timeout.clear(),
+            i64
+        ),
+        (
+            "pull_response_success",
+            stats.process_pull_response_success.clear(),
+            i64
+        ),
+        (
+            "process_pull_resp_timeout",
+            stats.process_pull_response_timeout.clear(),
+            i64
+        ),
+        (
+            "push_response_count",
+            stats.push_response_count.clear(),
+            i64
+        ),
+    );
+    datapoint_info!(
+        "cluster_info_stats3",
+        (
+            "process_pull_resp_len",
+            stats.process_pull_response_len.clear(),
+            i64
+        ),
+        (
+            "process_pull_requests",
+            stats.process_pull_requests.clear(),
+            i64
+        ),
+        (
+            "pull_request_ping_pong_check_failed_count",
+            stats.pull_request_ping_pong_check_failed_count.clear(),
+            i64
+        ),
+        (
+            "new_pull_requests_pings_count",
+            stats.new_pull_requests_pings_count.clear(),
+            i64
+        ),
+        (
+            "generate_pull_responses",
+            stats.generate_pull_responses.clear(),
+            i64
+        ),
+        ("process_prune", stats.process_prune.clear(), i64),
+        (
+            "process_push_message",
+            stats.process_push_message.clear(),
+            i64
+        ),
+        (
+            "prune_received_cache",
+            stats.prune_received_cache.clear(),
+            i64
+        ),
+        ("epoch_slots_lookup", stats.epoch_slots_lookup.clear(), i64),
+        ("new_pull_requests", stats.new_pull_requests.clear(), i64),
+        ("mark_pull_request", stats.mark_pull_request.clear(), i64),
+    );
+    datapoint_info!(
+        "cluster_info_stats4",
+        (
+            "skip_push_message_shred_version",
+            stats.skip_push_message_shred_version.clear(),
+            i64
+        ),
+        (
+            "skip_pull_response_shred_version",
+            stats.skip_pull_response_shred_version.clear(),
+            i64
+        ),
+        (
+            "skip_pull_shred_version",
+            stats.skip_pull_shred_version.clear(),
+            i64
+        ),
+        ("push_message_count", stats.push_message_count.clear(), i64),
+        (
+            "push_message_value_count",
+            stats.push_message_value_count.clear(),
+            i64
+        ),
+        (
+            "new_pull_requests_count",
+            stats.new_pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "pull_from_entrypoint_count",
+            stats.pull_from_entrypoint_count.clear(),
+            i64
+        ),
+        (
+            "prune_message_count",
+            stats.prune_message_count.clear(),
+            i64
+        ),
+        ("prune_message_len", stats.prune_message_len.clear(), i64),
+    );
+    datapoint_info!(
+        "cluster_info_stats5",
+        (
+            "pull_requests_count",
+            stats.pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_count",
+            stats.packets_received_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_prune_messages_count",
+            stats.packets_received_prune_messages_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_pull_requests_count",
+            stats.packets_received_pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_pull_responses_count",
+            stats.packets_received_pull_responses_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_push_messages_count",
+            stats.packets_received_push_messages_count.clear(),
+            i64
+        ),
+        (
+            "packets_received_verified_count",
+            stats.packets_received_verified_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_gossip_requests_count",
+            stats.packets_sent_gossip_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_prune_messages_count",
+            stats.packets_sent_prune_messages_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_pull_requests_count",
+            stats.packets_sent_pull_requests_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_pull_responses_count",
+            stats.packets_sent_pull_responses_count.clear(),
+            i64
+        ),
+        (
+            "packets_sent_push_messages_count",
+            stats.packets_sent_push_messages_count.clear(),
+            i64
+        ),
+        (
+            "require_stake_for_gossip_unknown_feature_set",
+            stats.require_stake_for_gossip_unknown_feature_set.clear(),
+            i64
+        ),
+        (
+            "require_stake_for_gossip_unknown_stakes",
+            stats.require_stake_for_gossip_unknown_stakes.clear(),
+            i64
+        ),
+        (
+            "trim_crds_table_failed",
+            stats.trim_crds_table_failed.clear(),
+            i64
+        ),
+        (
+            "trim_crds_table_purged_values_count",
+            stats.trim_crds_table_purged_values_count.clear(),
+            i64
+        ),
+    );
+}

core/src/cluster_info_vote_listener.rs

@@ -1,5 +1,6 @@
 use crate::{
     cluster_info::{ClusterInfo, GOSSIP_SLEEP_MILLIS},
+    crds::Cursor,
     crds_value::CrdsValueLabel,
     optimistic_confirmation_verifier::OptimisticConfirmationVerifier,
     optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender},
@@ -326,23 +327,18 @@ impl ClusterInfoVoteListener {
         verified_vote_label_packets_sender: VerifiedLabelVotePacketsSender,
         verified_vote_transactions_sender: VerifiedVoteTransactionsSender,
     ) -> Result<()> {
-        let mut last_ts = 0;
-        loop {
-            if exit.load(Ordering::Relaxed) {
-                return Ok(());
-            }
-            let (labels, votes, new_ts) = cluster_info.get_votes(last_ts);
+        let mut cursor = Cursor::default();
+        while !exit.load(Ordering::Relaxed) {
+            let (labels, votes) = cluster_info.get_votes(&mut cursor);
             inc_new_counter_debug!("cluster_info_vote_listener-recv_count", votes.len());
-
-            last_ts = new_ts;
             if !votes.is_empty() {
                 let (vote_txs, packets) = Self::verify_votes(votes, labels);
                 verified_vote_transactions_sender.send(vote_txs)?;
                 verified_vote_label_packets_sender.send(packets)?;
             }
-
             sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS));
         }
+        Ok(())
     }
 
     #[allow(clippy::type_complexity)]
@@ -1430,26 +1426,19 @@ mod tests {
             let slot_vote_tracker = vote_tracker.get_slot_vote_tracker(vote_slot).unwrap();
             let r_slot_vote_tracker = &slot_vote_tracker.read().unwrap();
 
+            assert_eq!(
+                r_slot_vote_tracker
+                    .optimistic_votes_tracker(&vote_bank_hash)
+                    .unwrap()
+                    .stake(),
+                100
+            );
             if events == vec![1] {
                 // Check `gossip_only_stake` is not incremented
-                assert_eq!(
-                    r_slot_vote_tracker
-                        .optimistic_votes_tracker(&vote_bank_hash)
-                        .unwrap()
-                        .stake(),
-                    100
-                );
                 assert_eq!(r_slot_vote_tracker.gossip_only_stake, 0);
             } else {
                 // Check that both the `gossip_only_stake` and `total_voted_stake` both
                 // increased
-                assert_eq!(
-                    r_slot_vote_tracker
-                        .optimistic_votes_tracker(&vote_bank_hash)
-                        .unwrap()
-                        .stake(),
-                    100
-                );
                 assert_eq!(r_slot_vote_tracker.gossip_only_stake, 100);
             }
         }

core/src/cluster_slot_state_verifier.rs

@@ -3,9 +3,10 @@ use crate::{
     progress_map::ProgressMap,
 };
 use solana_sdk::{clock::Slot, hash::Hash};
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 
-pub type GossipDuplicateConfirmedSlots = BTreeMap<Slot, Hash>;
+pub(crate) type DuplicateSlotsTracker = BTreeSet<Slot>;
+pub(crate) type GossipDuplicateConfirmedSlots = BTreeMap<Slot, Hash>;
 type SlotStateHandler = fn(Slot, &Hash, Option<&Hash>, bool, bool) -> Vec<ResultingStateChange>;
 
 #[derive(PartialEq, Debug)]
@@ -234,10 +235,12 @@ fn apply_state_changes(
     }
 }
 
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn check_slot_agrees_with_cluster(
     slot: Slot,
     root: Slot,
     frozen_hash: Option<Hash>,
+    duplicate_slots_tracker: &mut DuplicateSlotsTracker,
     gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots,
     ancestors: &HashMap<Slot, HashSet<Slot>>,
     descendants: &HashMap<Slot, HashSet<Slot>>,
@@ -258,6 +261,15 @@ pub(crate) fn check_slot_agrees_with_cluster(
         return;
     }
 
+    // Needs to happen before the frozen_hash.is_none() check below to account for duplicate
+    // signals arriving before the bank is constructed in replay.
+    if matches!(slot_state_update, SlotStateUpdate::Duplicate) {
+        // If this slot has already been processed before, return
+        if !duplicate_slots_tracker.insert(slot) {
+            return;
+        }
+    }
+
     if frozen_hash.is_none() {
         // If the bank doesn't even exist in BankForks yet,
         // then there's nothing to do as replay of the slot
@@ -275,18 +287,7 @@ pub(crate) fn check_slot_agrees_with_cluster(
         &frozen_hash,
         is_local_replay_duplicate_confirmed,
     );
-    let mut is_slot_duplicate =
-        progress.is_unconfirmed_duplicate(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map");
-    if matches!(slot_state_update, SlotStateUpdate::Duplicate) {
-        if is_slot_duplicate {
-            // Already processed duplicate signal for this slot, no need to continue
-            return;
-        } else {
-            // Otherwise, mark the slot as duplicate so the appropriate state changes
-            // will trigger
-            is_slot_duplicate = true;
-        }
-    }
+    let is_slot_duplicate = duplicate_slots_tracker.contains(&slot);
     let is_dead = progress.is_dead(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map");
 
     info!(
@@ -687,6 +688,84 @@ mod test {
             .unwrap());
     }
 
+    fn run_test_state_duplicate_then_bank_frozen(initial_bank_hash: Option<Hash>) {
+        // Common state
+        let InitialState {
+            mut heaviest_subtree_fork_choice,
+            mut progress,
+            ancestors,
+            descendants,
+            bank_forks,
+            ..
+        } = setup();
+
+        // Setup a duplicate slot state transition with the initial bank state of the duplicate slot
+        // determined by `initial_bank_hash`, which can be:
+        // 1) A default hash (unfrozen bank),
+        // 2) None (a slot that hasn't even started replay yet).
+        let root = 0;
+        let mut duplicate_slots_tracker = DuplicateSlotsTracker::default();
+        let gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default();
+        let duplicate_slot = 2;
+        check_slot_agrees_with_cluster(
+            duplicate_slot,
+            root,
+            initial_bank_hash,
+            &mut duplicate_slots_tracker,
+            &gossip_duplicate_confirmed_slots,
+            &ancestors,
+            &descendants,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            SlotStateUpdate::Duplicate,
+        );
+
+        // Now freeze the bank
+        let frozen_duplicate_slot_hash = bank_forks
+            .read()
+            .unwrap()
+            .get(duplicate_slot)
+            .unwrap()
+            .hash();
+        check_slot_agrees_with_cluster(
+            duplicate_slot,
+            root,
+            Some(frozen_duplicate_slot_hash),
+            &mut duplicate_slots_tracker,
+            &gossip_duplicate_confirmed_slots,
+            &ancestors,
+            &descendants,
+            &mut progress,
+            &mut heaviest_subtree_fork_choice,
+            SlotStateUpdate::Frozen,
+        );
+
+        // Progress map should have the correct updates, fork choice should mark duplicate
+        // as unvotable
+        assert!(progress.is_unconfirmed_duplicate(duplicate_slot).unwrap());
+
+        // The ancestor of the duplicate slot should be the best slot now
+        let (duplicate_ancestor, duplicate_parent_hash) = {
+            let r_bank_forks = bank_forks.read().unwrap();
+            let parent_bank = r_bank_forks.get(duplicate_slot).unwrap().parent().unwrap();
+            (parent_bank.slot(), parent_bank.hash())
+        };
+        assert_eq!(
+            heaviest_subtree_fork_choice.best_overall_slot(),
+            (duplicate_ancestor, duplicate_parent_hash)
+        );
+    }
+
+    #[test]
+    fn test_state_unfrozen_bank_duplicate_then_bank_frozen() {
+        run_test_state_duplicate_then_bank_frozen(Some(Hash::default()));
+    }
+
+    #[test]
+    fn test_state_unreplayed_bank_duplicate_then_bank_frozen() {
+        run_test_state_duplicate_then_bank_frozen(None);
+    }
+
     #[test]
     fn test_state_ancestor_confirmed_descendant_duplicate() {
         // Common state
@@ -705,6 +784,7 @@ mod test {
             (3, slot3_hash)
         );
         let root = 0;
+        let mut duplicate_slots_tracker = DuplicateSlotsTracker::default();
         let mut gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default();
 
         // Mark slot 2 as duplicate confirmed
@@ -714,6 +794,7 @@ mod test {
             2,
             root,
             Some(slot2_hash),
+            &mut duplicate_slots_tracker,
             &gossip_duplicate_confirmed_slots,
             &ancestors,
             &descendants,
@@ -727,11 +808,13 @@ mod test {
             (3, slot3_hash)
         );
 
-        // Mark 3 as duplicate, should not remove slot 2 from fork choice
+        // Mark 3 as duplicate, should not remove the duplicate confirmed slot 2 from
+        // fork choice
         check_slot_agrees_with_cluster(
             3,
             root,
             Some(slot3_hash),
+            &mut duplicate_slots_tracker,
             &gossip_duplicate_confirmed_slots,
             &ancestors,
             &descendants,
@@ -764,12 +847,15 @@ mod test {
             (3, slot3_hash)
         );
         let root = 0;
+        let mut duplicate_slots_tracker = DuplicateSlotsTracker::default();
         let mut gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default();
-        // Mark 2 as duplicate confirmed
+
+        // Mark 2 as duplicate
         check_slot_agrees_with_cluster(
             2,
             root,
             Some(bank_forks.read().unwrap().get(2).unwrap().hash()),
+            &mut duplicate_slots_tracker,
             &gossip_duplicate_confirmed_slots,
             &ancestors,
             &descendants,
@@ -790,6 +876,7 @@ mod test {
             3,
             root,
             Some(slot3_hash),
+            &mut duplicate_slots_tracker,
             &gossip_duplicate_confirmed_slots,
             &ancestors,
             &descendants,

core/src/cluster_slots.rs

@@ -1,5 +1,5 @@
 use crate::{
-    cluster_info::ClusterInfo, contact_info::ContactInfo, epoch_slots::EpochSlots,
+    cluster_info::ClusterInfo, contact_info::ContactInfo, crds::Cursor, epoch_slots::EpochSlots,
     serve_repair::RepairType,
 };
 use itertools::Itertools;
@@ -7,10 +7,7 @@ use solana_runtime::{bank_forks::BankForks, epoch_stakes::NodeIdToVoteAccounts};
 use solana_sdk::{clock::Slot, pubkey::Pubkey};
 use std::{
     collections::{BTreeMap, HashMap, HashSet},
-    sync::{
-        atomic::{AtomicU64, Ordering},
-        Arc, RwLock,
-    },
+    sync::{Arc, Mutex, RwLock},
 };
 
 // Limit the size of cluster-slots map in case
@@ -22,22 +19,26 @@ pub type SlotPubkeys = HashMap<Pubkey, u64>;
 #[derive(Default)]
 pub struct ClusterSlots {
     cluster_slots: RwLock<BTreeMap<Slot, Arc<RwLock<SlotPubkeys>>>>,
-    since: AtomicU64,
     validator_stakes: RwLock<Arc<NodeIdToVoteAccounts>>,
     epoch: RwLock<Option<u64>>,
+    cursor: Mutex<Cursor>,
 }
 
 impl ClusterSlots {
     pub fn lookup(&self, slot: Slot) -> Option<Arc<RwLock<SlotPubkeys>>> {
         self.cluster_slots.read().unwrap().get(&slot).cloned()
     }
+
     pub fn update(&self, root: Slot, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
         self.update_peers(bank_forks);
-        let since = self.since.load(Ordering::Relaxed);
-        let (epoch_slots, since) = cluster_info.get_epoch_slots_since(since);
-        self.update_internal(root, epoch_slots, since);
+        let epoch_slots = {
+            let mut cursor = self.cursor.lock().unwrap();
+            cluster_info.get_epoch_slots(&mut cursor)
+        };
+        self.update_internal(root, epoch_slots);
     }
-    fn update_internal(&self, root: Slot, epoch_slots_list: Vec<EpochSlots>, since: Option<u64>) {
+
+    fn update_internal(&self, root: Slot, epoch_slots_list: Vec<EpochSlots>) {
         // Attach validator's total stake.
         let epoch_slots_list: Vec<_> = {
             let validator_stakes = self.validator_stakes.read().unwrap();
@@ -86,9 +87,6 @@ impl ClusterSlots {
                 cluster_slots.split_off(&key);
             }
         }
-        if let Some(since) = since {
-            self.since.store(since + 1, Ordering::Relaxed);
-        }
     }
 
     pub fn collect(&self, id: &Pubkey) -> HashSet<Slot> {
@@ -206,23 +204,20 @@ mod tests {
     fn test_default() {
         let cs = ClusterSlots::default();
         assert!(cs.cluster_slots.read().unwrap().is_empty());
-        assert_eq!(cs.since.load(Ordering::Relaxed), 0);
     }
 
     #[test]
     fn test_update_noop() {
         let cs = ClusterSlots::default();
-        cs.update_internal(0, vec![], None);
+        cs.update_internal(0, vec![]);
         assert!(cs.cluster_slots.read().unwrap().is_empty());
-        assert_eq!(cs.since.load(Ordering::Relaxed), 0);
     }
 
     #[test]
     fn test_update_empty() {
         let cs = ClusterSlots::default();
         let epoch_slot = EpochSlots::default();
-        cs.update_internal(0, vec![epoch_slot], Some(0));
-        assert_eq!(cs.since.load(Ordering::Relaxed), 1);
+        cs.update_internal(0, vec![epoch_slot]);
         assert!(cs.lookup(0).is_none());
     }
 
@@ -232,8 +227,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[0], 0);
-        cs.update_internal(0, vec![epoch_slot], Some(0));
-        assert_eq!(cs.since.load(Ordering::Relaxed), 1);
+        cs.update_internal(0, vec![epoch_slot]);
         assert!(cs.lookup(0).is_none());
     }
 
@@ -242,8 +236,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, vec![epoch_slot], Some(0));
-        assert_eq!(cs.since.load(Ordering::Relaxed), 1);
+        cs.update_internal(0, vec![epoch_slot]);
         assert!(cs.lookup(0).is_none());
         assert!(cs.lookup(1).is_some());
         assert_eq!(
@@ -373,7 +366,7 @@ mod tests {
         );
 
         *cs.validator_stakes.write().unwrap() = map;
-        cs.update_internal(0, vec![epoch_slot], None);
+        cs.update_internal(0, vec![epoch_slot]);
         assert!(cs.lookup(1).is_some());
         assert_eq!(
             cs.lookup(1)
@@ -390,7 +383,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, vec![epoch_slot], None);
+        cs.update_internal(0, vec![epoch_slot]);
         let self_id = solana_sdk::pubkey::new_rand();
         assert_eq!(
             cs.generate_repairs_for_missing_slots(&self_id, 0),
@@ -404,7 +397,7 @@ mod tests {
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
         let self_id = epoch_slot.from;
-        cs.update_internal(0, vec![epoch_slot], None);
+        cs.update_internal(0, vec![epoch_slot]);
         let slots: Vec<Slot> = cs.collect(&self_id).into_iter().collect();
         assert_eq!(slots, vec![1]);
     }
@@ -415,7 +408,7 @@ mod tests {
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
         let self_id = epoch_slot.from;
-        cs.update_internal(0, vec![epoch_slot], None);
+        cs.update_internal(0, vec![epoch_slot]);
         assert!(cs
             .generate_repairs_for_missing_slots(&self_id, 0)
             .is_empty());

core/src/cluster_slots_service.rs

@@ -185,19 +185,21 @@ impl ClusterSlotsService {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::cluster_info::Node;
+    use crate::{cluster_info::Node, crds_value::CrdsValueLabel};
 
     #[test]
     pub fn test_update_lowest_slot() {
-        let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
+        let pubkey = Pubkey::new_unique();
+        let node_info = Node::new_localhost_with_pubkey(&pubkey);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(node_info.info);
-        ClusterSlotsService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
+        ClusterSlotsService::update_lowest_slot(&pubkey, 5, &cluster_info);
         cluster_info.flush_push_queue();
-        let lowest = cluster_info
-            .get_lowest_slot_for_node(&Pubkey::default(), None, |lowest_slot, _| {
-                lowest_slot.clone()
-            })
-            .unwrap();
+        let lowest = {
+            let label = CrdsValueLabel::LowestSlot(pubkey);
+            let gossip = cluster_info.gossip.read().unwrap();
+            let entry = gossip.crds.get(&label).unwrap();
+            entry.value.lowest_slot().unwrap().clone()
+        };
         assert_eq!(lowest.lowest, 5);
     }
 }