Use u64 behind the scenes for solana-tokens (bp #13815) (#13818)

* Use u64 behind the scenes for solana-tokens (#13815)

* Use u64 behind the scenes for Allocations

* Fixup readme

* Clippy and remove errant comments

(cherry picked from commit 5e2d38227f)

# Conflicts:
#	tokens/src/commands.rs

* Fix conflicts

Co-authored-by: Tyera Eulberg <teulberg@gmail.com>
Co-authored-by: Tyera Eulberg <tyera@solana.com>

.buildkite/hooks/pre-command

@@ -31,4 +31,9 @@ export CARGO_TARGET_CACHE=$HOME/cargo-target-cache/"$CHANNEL"-"$BUILDKITE_LABEL"
 
   mkdir -p "$CARGO_TARGET_CACHE"/target
   rsync -a --delete --link-dest="$CARGO_TARGET_CACHE" "$CARGO_TARGET_CACHE"/target .
+
+  # Don't reuse BPF target build artifacts due to incremental build issues with
+  # `std:
+  #    "found possibly newer version of crate `std` which `xyz` depends on
+  rm -rf target/bpfel-unknown-unknown
 )

.cache/fontconfig/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by fontconfig.
+# For information about cache directory tags, see:
+#       http://www.brynosaurus.com/cachedir/

.travis.yml

@@ -34,6 +34,8 @@ jobs:
         - stable
       install:
         - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
       script:
         - source ci/env.sh
         - ci/publish-tarball.sh

Cargo.toml

@@ -16,6 +16,7 @@ members = [
     "dos",
     "download-utils",
     "faucet",
+    "frozen-abi",
     "perf",
     "validator",
     "genesis",
@@ -30,12 +31,14 @@ members = [
     "merkle-tree",
     "stake-o-matic",
     "storage-bigtable",
+    "storage-proto",
     "streamer",
     "measure",
     "metrics",
     "net-shaper",
     "notifier",
     "poh-bench",
+    "program-test",
     "programs/secp256k1",
     "programs/bpf_loader",
     "programs/budget",
@@ -51,6 +54,8 @@ members = [
     "ramp-tps",
     "runtime",
     "sdk",
+    "sdk/cargo-build-bpf",
+    "sdk/cargo-test-bpf",
     "scripts",
     "stake-accounts",
     "stake-monitor",

README.md

@@ -61,8 +61,9 @@ $ cargo test
 ### Starting a local testnet
 Start your own testnet locally, instructions are in the [online docs](https://docs.solana.com/cluster/bench-tps).
 
-### Accessing the remote testnet
+### Accessing the remote development cluster
-* `testnet` - public stable testnet accessible via devnet.solana.com. Runs 24/7
+* `devnet` - stable public cluster for development accessible via
+devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://docs.solana.com/clusters)
 
 # Benchmarking
 

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.4.0"
+version = "1.4.12"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,12 +18,13 @@ lazy_static = "1.4.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.4.0" }
+solana-config-program = { path = "../programs/config", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.12" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.12" }
-spl-token-v2-0 = { package = "spl-token", version = "=2.0.6", features = ["skip-no-mangle"] }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
+zstd = "0.5.1"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -12,9 +12,14 @@ pub mod parse_token;
 pub mod parse_vote;
 pub mod validator_info;
 
-use crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount};
+use {
-use solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey};
+    crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
-use std::str::FromStr;
+    solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey},
+    std::{
+        io::{Read, Write},
+        str::FromStr,
+    },
+};
 
 pub type StringAmount = String;
 
@@ -44,6 +49,8 @@ pub enum UiAccountEncoding {
     Base58,
     Base64,
     JsonParsed,
+    #[serde(rename = "base64+zstd")]
+    Base64Zstd,
 }
 
 impl UiAccount {
@@ -66,6 +73,19 @@ impl UiAccount {
                 base64::encode(slice_data(&account.data, data_slice_config)),
                 encoding,
             ),
+            UiAccountEncoding::Base64Zstd => {
+                let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
+                match encoder
+                    .write_all(slice_data(&account.data, data_slice_config))
+                    .and_then(|()| encoder.finish())
+                {
+                    Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
+                    Err(_) => UiAccountData::Binary(
+                        base64::encode(slice_data(&account.data, data_slice_config)),
+                        UiAccountEncoding::Base64,
+                    ),
+                }
+            }
             UiAccountEncoding::JsonParsed => {
                 if let Ok(parsed_data) =
                     parse_account_data(pubkey, &account.owner, &account.data, additional_data)
@@ -92,6 +112,16 @@ impl UiAccount {
             UiAccountData::Binary(blob, encoding) => match encoding {
                 UiAccountEncoding::Base58 => bs58::decode(blob).into_vec().ok(),
                 UiAccountEncoding::Base64 => base64::decode(blob).ok(),
+                UiAccountEncoding::Base64Zstd => base64::decode(blob)
+                    .ok()
+                    .map(|zstd_data| {
+                        let mut data = vec![];
+                        zstd::stream::read::Decoder::new(zstd_data.as_slice())
+                            .and_then(|mut reader| reader.read_to_end(&mut data))
+                            .map(|_| data)
+                            .ok()
+                    })
+                    .flatten(),
                 UiAccountEncoding::Binary | UiAccountEncoding::JsonParsed => None,
             },
         }?;
@@ -179,4 +209,25 @@ mod test {
         });
         assert_eq!(slice_data(&data, slice_config), &[] as &[u8]);
     }
+
+    #[test]
+    fn test_base64_zstd() {
+        let encoded_account = UiAccount::encode(
+            &Pubkey::default(),
+            Account {
+                data: vec![0; 1024],
+                ..Account::default()
+            },
+            UiAccountEncoding::Base64Zstd,
+            None,
+            None,
+        );
+        assert!(matches!(
+            encoded_account.data,
+            UiAccountData::Binary(_, UiAccountEncoding::Base64Zstd)
+        ));
+
+        let decoded_account = encoded_account.decode().unwrap();
+        assert_eq!(decoded_account.data, vec![0; 1024]);
+    }
 }

account-decoder/src/parse_account_data.rs

@@ -111,8 +111,8 @@ mod test {
 
     #[test]
     fn test_parse_account_data() {
-        let account_pubkey = Pubkey::new_rand();
+        let account_pubkey = solana_sdk::pubkey::new_rand();
-        let other_program = Pubkey::new_rand();
+        let other_program = solana_sdk::pubkey::new_rand();
         let data = vec![0; 4];
         assert!(parse_account_data(&account_pubkey, &other_program, &data, None).is_err());
 

account-decoder/src/parse_config.rs

@@ -117,7 +117,7 @@ mod test {
             }))
             .unwrap(),
         };
-        let info_pubkey = Pubkey::new_rand();
+        let info_pubkey = solana_sdk::pubkey::new_rand();
         let validator_info_config_account = create_config_account(
             vec![(validator_info::id(), false), (info_pubkey, true)],
             &validator_info,

account-decoder/src/parse_stake.rs

@@ -134,7 +134,6 @@ impl From<Delegation> for UiDelegation {
 mod test {
     use super::*;
     use bincode::serialize;
-    use solana_sdk::pubkey::Pubkey;
 
     #[test]
     fn test_parse_stake() {
@@ -145,8 +144,8 @@ mod test {
             StakeAccountType::Uninitialized
         );
 
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
-        let custodian = Pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         let authorized = Authorized::auto(&pubkey);
         let lockup = Lockup {
             unix_timestamp: 0,
@@ -180,7 +179,7 @@ mod test {
             })
         );
 
-        let voter_pubkey = Pubkey::new_rand();
+        let voter_pubkey = solana_sdk::pubkey::new_rand();
         let stake = Stake {
             delegation: Delegation {
                 voter_pubkey,

account-decoder/src/parse_sysvar.rs

@@ -105,6 +105,7 @@ pub enum SysvarAccountType {
 pub struct UiClock {
     pub slot: Slot,
     pub epoch: Epoch,
+    pub epoch_start_timestamp: UnixTimestamp,
     pub leader_schedule_epoch: Epoch,
     pub unix_timestamp: UnixTimestamp,
 }
@@ -114,6 +115,7 @@ impl From<Clock> for UiClock {
         Self {
             slot: clock.slot,
             epoch: clock.epoch,
+            epoch_start_timestamp: clock.epoch_start_timestamp,
             leader_schedule_epoch: clock.leader_schedule_epoch,
             unix_timestamp: clock.unix_timestamp,
         }
@@ -212,15 +214,14 @@ pub struct UiStakeHistoryEntry {
 mod test {
     use super::*;
     use solana_sdk::{
-        fee_calculator::FeeCalculator,
+        account::create_account, fee_calculator::FeeCalculator, hash::Hash,
-        hash::Hash,
+        sysvar::recent_blockhashes::IterItem,
-        sysvar::{recent_blockhashes::IterItem, Sysvar},
     };
     use std::iter::FromIterator;
 
     #[test]
     fn test_parse_sysvars() {
-        let clock_sysvar = Clock::default().create_account(1);
+        let clock_sysvar = create_account(&Clock::default(), 1);
         assert_eq!(
             parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
             SysvarAccountType::Clock(UiClock::default()),
@@ -233,13 +234,13 @@ mod test {
             first_normal_epoch: 1,
             first_normal_slot: 12,
         };
-        let epoch_schedule_sysvar = epoch_schedule.create_account(1);
+        let epoch_schedule_sysvar = create_account(&epoch_schedule, 1);
         assert_eq!(
             parse_sysvar(&epoch_schedule_sysvar.data, &sysvar::epoch_schedule::id()).unwrap(),
             SysvarAccountType::EpochSchedule(epoch_schedule),
         );
 
-        let fees_sysvar = Fees::default().create_account(1);
+        let fees_sysvar = create_account(&Fees::default(), 1);
         assert_eq!(
             parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
             SysvarAccountType::Fees(UiFees::default()),
@@ -251,7 +252,7 @@ mod test {
         };
         let recent_blockhashes =
             RecentBlockhashes::from_iter(vec![IterItem(0, &hash, &fee_calculator)].into_iter());
-        let recent_blockhashes_sysvar = recent_blockhashes.create_account(1);
+        let recent_blockhashes_sysvar = create_account(&recent_blockhashes, 1);
         assert_eq!(
             parse_sysvar(
                 &recent_blockhashes_sysvar.data,
@@ -269,13 +270,13 @@ mod test {
             exemption_threshold: 2.0,
             burn_percent: 5,
         };
-        let rent_sysvar = rent.create_account(1);
+        let rent_sysvar = create_account(&rent, 1);
         assert_eq!(
             parse_sysvar(&rent_sysvar.data, &sysvar::rent::id()).unwrap(),
             SysvarAccountType::Rent(rent.into()),
         );
 
-        let rewards_sysvar = Rewards::default().create_account(1);
+        let rewards_sysvar = create_account(&Rewards::default(), 1);
         assert_eq!(
             parse_sysvar(&rewards_sysvar.data, &sysvar::rewards::id()).unwrap(),
             SysvarAccountType::Rewards(UiRewards::default()),
@@ -283,7 +284,7 @@ mod test {
 
         let mut slot_hashes = SlotHashes::default();
         slot_hashes.add(1, hash);
-        let slot_hashes_sysvar = slot_hashes.create_account(1);
+        let slot_hashes_sysvar = create_account(&slot_hashes, 1);
         assert_eq!(
             parse_sysvar(&slot_hashes_sysvar.data, &sysvar::slot_hashes::id()).unwrap(),
             SysvarAccountType::SlotHashes(vec![UiSlotHashEntry {
@@ -294,7 +295,7 @@ mod test {
 
         let mut slot_history = SlotHistory::default();
         slot_history.add(42);
-        let slot_history_sysvar = slot_history.create_account(1);
+        let slot_history_sysvar = create_account(&slot_history, 1);
         assert_eq!(
             parse_sysvar(&slot_history_sysvar.data, &sysvar::slot_history::id()).unwrap(),
             SysvarAccountType::SlotHistory(UiSlotHistory {
@@ -310,7 +311,7 @@ mod test {
             deactivating: 3,
         };
         stake_history.add(1, stake_history_entry.clone());
-        let stake_history_sysvar = stake_history.create_account(1);
+        let stake_history_sysvar = create_account(&stake_history, 1);
         assert_eq!(
             parse_sysvar(&stake_history_sysvar.data, &sysvar::stake_history::id()).unwrap(),
             SysvarAccountType::StakeHistory(vec![UiStakeHistoryEntry {
@@ -319,7 +320,7 @@ mod test {
             }]),
         );
 
-        let bad_pubkey = Pubkey::new_rand();
+        let bad_pubkey = solana_sdk::pubkey::new_rand();
         assert!(parse_sysvar(&stake_history_sysvar.data, &bad_pubkey).is_err());
 
         let bad_data = vec![0; 4];

account-decoder/src/parse_token.rs

@@ -4,7 +4,9 @@ use crate::{
 };
 use solana_sdk::pubkey::Pubkey;
 use spl_token_v2_0::{
-    solana_sdk::{program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey},
+    solana_program::{
+        program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
+    },
     state::{Account, AccountState, Mint, Multisig},
 };
 use std::str::FromStr;
@@ -21,6 +23,16 @@ pub fn spl_token_v2_0_native_mint() -> Pubkey {
     Pubkey::from_str(&spl_token_v2_0::native_mint::id().to_string()).unwrap()
 }
 
+// A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
+pub fn spl_token_v2_0_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
+    SplTokenPubkey::from_str(&pubkey.to_string()).unwrap()
+}
+
+// A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
+pub fn pubkey_from_spl_token_v2_0(pubkey: &SplTokenPubkey) -> Pubkey {
+    Pubkey::from_str(&pubkey.to_string()).unwrap()
+}
+
 pub fn parse_token(
     data: &[u8],
     mint_decimals: Option<u8>,

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.6"
 rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.4.12" }
-solana-measure = { path = "../measure", version = "1.4.0" }
+solana-measure = { path = "../measure", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ crossbeam-channel = "0.4"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.4.0" }
+solana-core = { path = "../core", version = "1.4.12" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
+solana-streamer = { path = "../streamer", version = "1.4.12" }
-solana-perf = { path = "../perf", version = "1.4.0" }
+solana-perf = { path = "../perf", version = "1.4.12" }
-solana-ledger = { path = "../ledger", version = "1.4.0" }
+solana-ledger = { path = "../ledger", version = "1.4.12" }
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.4.12" }
-solana-measure = { path = "../measure", version = "1.4.0" }
+solana-measure = { path = "../measure", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -20,7 +20,6 @@ use solana_perf::packet::to_packets_chunked;
 use solana_runtime::{bank::Bank, bank_forks::BankForks};
 use solana_sdk::{
     hash::Hash,
-    pubkey::Pubkey,
     signature::Keypair,
     signature::Signature,
     system_transaction,
@@ -69,7 +68,7 @@ fn make_accounts_txs(
     hash: Hash,
     same_payer: bool,
 ) -> Vec<Transaction> {
-    let to_pubkey = Pubkey::new_rand();
+    let to_pubkey = solana_sdk::pubkey::new_rand();
     let payer_key = Keypair::new();
     let dummy = system_transaction::transfer(&payer_key, &to_pubkey, 1, hash);
     (0..total_num_transactions)
@@ -78,9 +77,9 @@ fn make_accounts_txs(
             let mut new = dummy.clone();
             let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
             if !same_payer {
-                new.message.account_keys[0] = Pubkey::new_rand();
+                new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
             }
-            new.message.account_keys[1] = Pubkey::new_rand();
+            new.message.account_keys[1] = solana_sdk::pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
         })
@@ -241,7 +240,7 @@ fn main() {
         let base_tx_count = bank.transaction_count();
         let mut txs_processed = 0;
         let mut root = 1;
-        let collector = Pubkey::new_rand();
+        let collector = solana_sdk::pubkey::new_rand();
         let config = Config {
             packets_per_batch: packets_per_chunk,
             chunk_len,

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.4.0"
+version = "1.4.12"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,15 +12,15 @@ edition = "2018"
 async-trait = "0.1.36"
 bincode = "1.3.1"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.0" }
+solana-banks-interface = { path = "../banks-interface", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-tarpc = { version = "0.22.0", features = ["full"] }
+tarpc = { version = "0.23.0", features = ["full"] }
-tokio = "0.2"
+tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.4.12" }
-solana-banks-server = { path = "../banks-server", version = "1.4.0" }
+solana-banks-server = { path = "../banks-server", version = "1.4.12" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -10,9 +10,17 @@ use futures::future::join_all;
 pub use solana_banks_interface::{BanksClient, TransactionStatus};
 use solana_banks_interface::{BanksRequest, BanksResponse};
 use solana_sdk::{
-    account::Account, clock::Slot, commitment_config::CommitmentLevel,
+    account::{from_account, Account},
-    fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature,
+    clock::Slot,
-    transaction::Transaction, transport,
+    commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    rent::Rent,
+    signature::Signature,
+    sysvar,
+    transaction::Transaction,
+    transport,
 };
 use std::io::{self, Error, ErrorKind};
 use tarpc::{
@@ -40,6 +48,9 @@ pub trait BanksClientExt {
     /// use them to calculate the transaction fee.
     async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)>;
 
+    /// Return the cluster rent
+    async fn get_rent(&mut self) -> io::Result<Rent>;
+
     /// Send a transaction and return after the transaction has been rejected or
     /// reached the given level of commitment.
     async fn process_transaction_with_commitment(
@@ -108,6 +119,17 @@ impl BanksClientExt for BanksClient {
             .await
     }
 
+    async fn get_rent(&mut self) -> io::Result<Rent> {
+        let rent_sysvar = self
+            .get_account(sysvar::rent::id())
+            .await?
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Rent sysvar not present"))?;
+
+        from_account::<Rent>(&rent_sysvar).ok_or_else(|| {
+            io::Error::new(io::ErrorKind::Other, "Failed to deserialize Rent sysvar")
+        })
+    }
+
     async fn get_recent_blockhash(&mut self) -> io::Result<Hash> {
         Ok(self.get_fees().await?.1)
     }
@@ -213,10 +235,10 @@ mod tests {
     use super::*;
     use solana_banks_server::banks_server::start_local_server;
     use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
-    use solana_sdk::{message::Message, pubkey::Pubkey, signature::Signer, system_instruction};
+    use solana_sdk::{message::Message, signature::Signer, system_instruction};
     use std::sync::{Arc, RwLock};
     use tarpc::transport;
-    use tokio::{runtime::Runtime, time::delay_for};
+    use tokio::{runtime::Runtime, time::sleep};
 
     #[test]
     fn test_banks_client_new() {
@@ -235,7 +257,7 @@ mod tests {
             &genesis.genesis_config,
         ))));
 
-        let bob_pubkey = Pubkey::new_rand();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
         let mint_pubkey = genesis.mint_keypair.pubkey();
         let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
         let message = Message::new(&[instruction], Some(&mint_pubkey));
@@ -265,7 +287,7 @@ mod tests {
         ))));
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
-        let bob_pubkey = Pubkey::new_rand();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
         let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
@@ -285,7 +307,7 @@ mod tests {
                 if root_slot > last_valid_slot {
                     break;
                 }
-                delay_for(Duration::from_millis(100)).await;
+                sleep(Duration::from_millis(100)).await;
                 status = banks_client.get_transaction_status(signature).await?;
             }
             assert!(status.unwrap().err.is_none());

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.4.0"
+version = "1.4.12"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,8 +10,8 @@ edition = "2018"
 
 [dependencies]
 serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-tarpc = { version = "0.22.0", features = ["full"] }
+tarpc = { version = "0.23.0", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.4.0"
+version = "1.4.12"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,12 +12,12 @@ edition = "2018"
 bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.8"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.0" }
+solana-banks-interface = { path = "../banks-interface", version = "1.4.12" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
+solana-metrics = { path = "../metrics", version = "1.4.12" }
-tarpc = { version = "0.22.0", features = ["full"] }
+tarpc = { version = "0.23.0", features = ["full"] }
-tokio = "0.2"
+tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [lib]

banks-server/src/banks_server.rs

@@ -5,11 +5,7 @@ use futures::{
     prelude::stream::{self, StreamExt},
 };
 use solana_banks_interface::{Banks, BanksRequest, BanksResponse, TransactionStatus};
-use solana_runtime::{
+use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache};
-    bank::Bank,
-    bank_forks::BankForks,
-    commitment::{BlockCommitmentCache, CommitmentSlots},
-};
 use solana_sdk::{
     account::Account,
     clock::Slot,
@@ -21,7 +17,6 @@ use solana_sdk::{
     transaction::{self, Transaction},
 };
 use std::{
-    collections::HashMap,
     io,
     net::{Ipv4Addr, SocketAddr},
     sync::{
@@ -38,7 +33,7 @@ use tarpc::{
     server::{self, Channel, Handler},
     transport,
 };
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use tokio_serde::formats::Bincode;
 
 #[derive(Clone)]
@@ -84,11 +79,9 @@ impl BanksServer {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
         let slot = bank.slot();
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(
+        let block_commitment_cache = Arc::new(RwLock::new(
-            HashMap::default(),
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
-            0,
+        ));
-            CommitmentSlots::new_from_slot(slot),
-        )));
         Builder::new()
             .name("solana-bank-forks-client".to_string())
             .spawn(move || Self::run(&bank, transaction_receiver))
@@ -109,18 +102,21 @@ impl BanksServer {
 
     async fn poll_signature_status(
         self,
-        signature: Signature,
+        signature: &Signature,
+        blockhash: &Hash,
         last_valid_slot: Slot,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        let mut status = self.bank(commitment).get_signature_status(&signature);
+        let mut status = self
+            .bank(commitment)
+            .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            delay_for(Duration::from_millis(200)).await;
+            sleep(Duration::from_millis(200)).await;
             let bank = self.bank(commitment);
             if bank.slot() > last_valid_slot {
                 break;
             }
-            status = bank.get_signature_status(&signature);
+            status = bank.get_signature_status_with_blockhash(signature, blockhash);
         }
         status
     }
@@ -193,13 +189,13 @@ impl Banks for BanksServer {
             .read()
             .unwrap()
             .root_bank()
-            .get_blockhash_last_valid_slot(&blockhash)
+            .get_blockhash_last_valid_slot(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
         let info =
             TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
         self.transaction_sender.send(info).unwrap();
-        self.poll_signature_status(signature, last_valid_slot, commitment)
+        self.poll_signature_status(&signature, blockhash, last_valid_slot, commitment)
             .await
     }
 

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-core = { path = "../core", version = "1.4.0" }
+solana-core = { path = "../core", version = "1.4.12" }
-solana-genesis = { path = "../genesis", version = "1.4.0" }
+solana-genesis = { path = "../genesis", version = "1.4.12" }
-solana-client = { path = "../client", version = "1.4.0" }
+solana-client = { path = "../client", version = "1.4.12" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
+solana-faucet = { path = "../faucet", version = "1.4.12" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.4.0" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.4.12" }
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
+solana-metrics = { path = "../metrics", version = "1.4.12" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
+solana-net-utils = { path = "../net-utils", version = "1.4.12" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.4.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.4.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/tests/bench_exchange.rs

@@ -39,7 +39,7 @@ fn test_exchange_local_cluster() {
     } = config;
     let accounts_in_groups = batch_size * account_groups;
 
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![100_000; NUM_NODES],
         cluster_lamports: 100_000_000_000_000,
         validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
@@ -86,7 +86,7 @@ fn test_exchange_bank_client() {
     solana_logger::setup();
     let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
     let mut bank = Bank::new(&genesis_config);
-    bank.add_builtin_program("exchange_program", id(), process_instruction);
+    bank.add_builtin("exchange_program", id(), process_instruction);
     let clients = vec![BankClient::new(bank)];
 
     let mut config = Config::default();

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
+solana-streamer = { path = "../streamer", version = "1.4.12" }
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
+solana-net-utils = { path = "../net-utils", version = "1.4.12" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,23 +15,23 @@ log = "0.4.8"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-core = { path = "../core", version = "1.4.0" }
+solana-core = { path = "../core", version = "1.4.12" }
-solana-genesis = { path = "../genesis", version = "1.4.0" }
+solana-genesis = { path = "../genesis", version = "1.4.12" }
-solana-client = { path = "../client", version = "1.4.0" }
+solana-client = { path = "../client", version = "1.4.12" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
+solana-faucet = { path = "../faucet", version = "1.4.12" }
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
+solana-metrics = { path = "../metrics", version = "1.4.12" }
-solana-measure = { path = "../measure", version = "1.4.0" }
+solana-measure = { path = "../measure", version = "1.4.12" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
+solana-net-utils = { path = "../net-utils", version = "1.4.12" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.4.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.4.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/tests/bench_tps.rs

@@ -15,7 +15,7 @@ fn test_bench_tps_local_cluster(config: Config) {
 
     solana_logger::setup();
     const NUM_NODES: usize = 1;
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![999_990; NUM_NODES],
         cluster_lamports: 200_000_000,
         validator_configs: vec![ValidatorConfig::default(); NUM_NODES],

cargo

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# shellcheck source=ci/rust-version.sh
+here=$(dirname "$0")
+
+source "${here}"/ci/rust-version.sh all
+
+toolchain=
+case "$1" in
+  stable)
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    shift
+    ;;
+  nightly)
+    # shellcheck disable=SC2054 # rust_nightly is sourced from rust-version.sh
+    toolchain="$rust_nightly"
+    shift
+    ;;
+  +*)
+    toolchain="${1#+}"
+    shift
+    ;;
+  *)
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    ;;
+esac
+
+set -x
+exec cargo "+${toolchain}" "${@}"

cargo-build-bpf

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-build-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

cargo-test-bpf

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+export CARGO_BUILD_BPF="$here"/cargo-build-bpf
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-test-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

ci/buildkite-pipeline.sh

@@ -175,6 +175,30 @@ EOF
       "Stable-perf skipped as no relevant files were modified"
   fi
 
+  # Downstream backwards compatibility
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+             ^scripts/build-downstream-projects.sh \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "scripts/build-downstream-projects.sh"
+    name: "downstream-projects"
+    timeout_in_minutes: 30
+EOF
+  else
+    annotate --style info \
+      "downstream-projects skipped as no relevant files were modified"
+  fi
   # Benches...
   if affects \
              .rs$ \

ci/nits.sh

@@ -26,8 +26,9 @@ declare print_free_tree=(
   ':runtime/src/**.rs'
   ':sdk/bpf/rust/rust-utils/**.rs'
   ':sdk/**.rs'
-  ':^sdk/src/program_option.rs'
+  ':^sdk/cargo-build-bpf/**.rs'
-  ':^sdk/src/program_stubs.rs'
+  ':^sdk/program/src/program_option.rs'
+  ':^sdk/program/src/program_stubs.rs'
   ':programs/**.rs'
   ':^**bin**.rs'
   ':^**bench**.rs'

ci/order-crates-for-publishing.py

@@ -42,10 +42,10 @@ def get_packages():
         sys.exit(1)
 
     # Order dependencies
-    deleted_dependencies = []
     sorted_dependency_graph = []
     max_iterations = pow(len(dependency_graph),2)
-    while len(deleted_dependencies) < len(dependency_graph):
+    while dependency_graph:
+        deleted_packages = []
         if max_iterations == 0:
             # One day be more helpful and find the actual cycle for the user...
             sys.exit('Error: Circular dependency suspected between these packages: \n {}\n'.format('\n '.join(dependency_graph.keys())))
@@ -53,13 +53,17 @@ def get_packages():
         max_iterations -= 1
 
         for package, dependencies in dependency_graph.items():
+            if package in deleted_packages:
+                continue
             for dependency in dependencies:
                 if dependency in dependency_graph:
                     break
             else:
-                deleted_dependencies.append(package)
+                deleted_packages.append(package)
                 sorted_dependency_graph.append((package, manifest_path[package]))
 
+        dependency_graph = {p: d for p, d in dependency_graph.items() if not p in deleted_packages }
+
 
     return sorted_dependency_graph
 

ci/publish-crate.sh

@@ -4,6 +4,8 @@ cd "$(dirname "$0")/.."
 source ci/semver_bash/semver.sh
 source ci/rust-version.sh stable
 
+cargo="$(readlink -f ./cargo)"
+
 # shellcheck disable=SC2086
 is_crate_version_uploaded() {
   name=$1
@@ -66,11 +68,11 @@ for Cargo_toml in $Cargo_tomls; do
       (
         set -x
         rm -rf crate-test
-        cargo +"$rust_stable" init crate-test
+        "$cargo" stable init crate-test
         cd crate-test/
         echo "${crate_name} = \"${expectedCrateVersion}\"" >> Cargo.toml
         echo "[workspace]" >> Cargo.toml
-        cargo +"$rust_stable" check
+        "$cargo" stable check
       ) && really_uploaded=1
       if ((really_uploaded)); then
         break;

ci/publish-tarball.sh

@@ -91,17 +91,15 @@ echo --- Creating release tarball
   cp "${RELEASE_BASENAME}"/version.yml "${TARBALL_BASENAME}"-$TARGET.yml
 )
 
-# Metrics tarball is platform agnostic, only publish it from Linux
+# Maybe tarballs are platform agnostic, only publish them from the Linux build
 MAYBE_TARBALLS=
 if [[ "$CI_OS_NAME" = linux ]]; then
-  metrics/create-metrics-tarball.sh
   (
     set -x
     sdk/bpf/scripts/package.sh
     [[ -f bpf-sdk.tar.bz2 ]]
-
   )
-  MAYBE_TARBALLS="bpf-sdk.tar.bz2 solana-metrics.tar.bz2"
+  MAYBE_TARBALLS="bpf-sdk.tar.bz2"
 fi
 
 source ci/upload-ci-artifact.sh
@@ -126,7 +124,7 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.
         /usr/bin/s3cmd --acl-public put /solana/"$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
 
       echo Published to:
-      $DRYRUN ci/format-url.sh http://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
+      $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
     )
 
     if [[ -n $TAG ]]; then
@@ -149,4 +147,30 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.
   fi
 done
 
+
+# Create install wrapper for release.solana.com
+if [[ -n $BUILDKITE ]]; then
+  cat > release.solana.com-install <<EOF
+SOLANA_RELEASE=$CHANNEL_OR_TAG
+SOLANA_INSTALL_INIT_ARGS=$CHANNEL_OR_TAG
+SOLANA_DOWNLOAD_ROOT=http://release.solana.com
+EOF
+  cat install/solana-install-init.sh >> release.solana.com-install
+
+  echo --- AWS S3 Store: "install"
+  (
+    set -x
+    $DRYRUN docker run \
+      --rm \
+      --env AWS_ACCESS_KEY_ID \
+      --env AWS_SECRET_ACCESS_KEY \
+      --volume "$PWD:/solana" \
+      eremite/aws-cli:2018.12.18 \
+      /usr/bin/s3cmd --acl-public put /solana/release.solana.com-install s3://release.solana.com/"$CHANNEL_OR_TAG"/install
+
+    echo Published to:
+    $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/install
+  )
+fi
+
 echo --- ok

ci/test-bench.sh

@@ -6,7 +6,8 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-source ci/rust-version.sh all
+
+cargo="$(readlink -f "./cargo")"
 
 set -o pipefail
 export RUST_BACKTRACE=1
@@ -27,35 +28,35 @@ test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 
 # Ensure all dependencies are built
-_ cargo +$rust_nightly build --release
+_ "$cargo" nightly build --release
 
 # Remove "BENCH_FILE", if it exists so that the following commands can append
 rm -f "$BENCH_FILE"
 
 # Run sdk benches
-_ cargo +$rust_nightly bench --manifest-path sdk/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path sdk/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run runtime benches
-_ cargo +$rust_nightly bench --manifest-path runtime/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path runtime/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run core benches
-_ cargo +$rust_nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run bpf benches
-_ cargo +$rust_nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
+_ "$cargo" nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
   -- -Z unstable-options --format=json --nocapture | tee -a "$BENCH_FILE"
 
 # Run banking/accounts bench. Doesn't require nightly, but use since it is already built.
-_ cargo +$rust_nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
+_ "$cargo" nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
-_ cargo +$rust_nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
+_ "$cargo" nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
 
 # `solana-upload-perf` disabled as it can take over 30 minutes to complete for some
 # reason
 exit 0
-_ cargo +$rust_nightly run --release --package solana-upload-perf \
+_ "$cargo" nightly run --release --package solana-upload-perf \
   -- "$BENCH_FILE" "$TARGET_BRANCH" "$UPLOAD_METRICS" | tee "$BENCH_ARTIFACT"
 
 upload-ci-artifact "$BENCH_FILE"

ci/test-checks.sh

@@ -8,6 +8,9 @@ source ci/_
 source ci/rust-version.sh stable
 source ci/rust-version.sh nightly
 eval "$(ci/channel-info.sh)"
+cargo="$(readlink -f "./cargo")"
+
+scripts/increment-cargo-version.sh check
 
 echo --- build environment
 (
@@ -16,14 +19,14 @@ echo --- build environment
   rustup run "$rust_stable" rustc --version --verbose
   rustup run "$rust_nightly" rustc --version --verbose
 
-  cargo +"$rust_stable" --version --verbose
+  "$cargo" stable --version --verbose
-  cargo +"$rust_nightly" --version --verbose
+  "$cargo" nightly --version --verbose
 
-  cargo +"$rust_stable" clippy --version --verbose
+  "$cargo" stable clippy --version --verbose
-  cargo +"$rust_nightly" clippy --version --verbose
+  "$cargo" nightly clippy --version --verbose
 
   # audit is done only with stable
-  cargo +"$rust_stable" audit --version
+  "$cargo" stable audit --version
 )
 
 export RUST_BACKTRACE=1
@@ -49,26 +52,49 @@ else
 fi
 
 _ ci/order-crates-for-publishing.py
-_ cargo +"$rust_stable" fmt --all -- --check
+_ "$cargo" stable fmt --all -- --check
 
 # -Z... is needed because of clippy bug: https://github.com/rust-lang/rust-clippy/issues/4612
 # run nightly clippy for `sdk/` as there's a moderate amount of nightly-only code there
-_ cargo +"$rust_nightly" clippy \
+_ "$cargo" nightly clippy \
   -Zunstable-options --workspace --all-targets \
   -- --deny=warnings --allow=clippy::stable_sort_primitive
 
-_ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
+cargo_audit_ignores=(
+  # failure is officially deprecated/unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `failure` dependency.
+  --ignore RUSTSEC-2020-0036
+
+  # `net2` crate has been deprecated; use `socket2` instead
+  #
+  # Blocked on https://github.com/paritytech/jsonrpc/issues/575
+  --ignore RUSTSEC-2020-0016
+
+  # stdweb is unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `stdweb` dependency.
+  --ignore RUSTSEC-2020-0056
+
+  # Potential segfault in the time crate
+  #
+  # Blocked on multiple crates updating `time` to >= 0.2.23
+  --ignore RUSTSEC-2020-0071
+)
+_ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"
 
 {
   cd programs/bpf
-  _ cargo +"$rust_stable" audit
+  _ "$cargo" stable audit
   for project in rust/*/ ; do
     echo "+++ do_bpf_checks $project"
     (
       cd "$project"
-      _ cargo +"$rust_stable" fmt -- --check
+      _ "$cargo" stable fmt -- --check
-      _ cargo +"$rust_nightly" test
+      _ "$cargo" nightly test
-      _ cargo +"$rust_nightly" clippy -- --deny=warnings --allow=clippy::missing_safety_doc
+      _ "$cargo" nightly clippy -- --deny=warnings \
+        --allow=clippy::missing_safety_doc \
+        --allow=clippy::stable_sort_primitive
     )
   done
 }

ci/test-stable.sh

@@ -2,6 +2,8 @@
 set -e
 cd "$(dirname "$0")/.."
 
+cargo="$(readlink -f "./cargo")"
+
 source ci/_
 
 annotate() {
@@ -37,12 +39,15 @@ NPROC=$((NPROC>14 ? 14 : NPROC))
 echo "Executing $testName"
 case $testName in
 test-stable)
-  _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
   ;;
 test-stable-perf)
+  # BPF solana-sdk legacy compile test
+  ./cargo-build-bpf --manifest-path sdk/Cargo.toml
+
   # BPF program tests
   _ make -C programs/bpf/c tests
-  _ cargo +"$rust_stable" test \
+  _ "$cargo" stable test \
     --manifest-path programs/bpf/Cargo.toml \
     --no-default-features --features=bpf_c,bpf_rust -- --nocapture
 
@@ -62,13 +67,13 @@ test-stable-perf)
     export SOLANA_CUDA=1
   fi
 
-  _ cargo +"$rust_stable" build --bins ${V:+--verbose}
+  _ "$cargo" stable build --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
-  _ cargo +"$rust_stable" run --manifest-path poh-bench/Cargo.toml ${V:+--verbose} -- --hashes-per-tick 10
+  _ "$cargo" stable run --manifest-path poh-bench/Cargo.toml ${V:+--verbose} -- --hashes-per-tick 10
   ;;
 test-local-cluster)
-  _ cargo +"$rust_stable" build --release --bins ${V:+--verbose}
+  _ "$cargo" stable build --release --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
   exit 0
   ;;
 *)

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.4.0"
+version = "1.4.12"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.4.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
 thiserror = "1.0.20"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/input_parsers.rs

@@ -189,6 +189,7 @@ pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentC
         "recent" => CommitmentConfig::recent(),
         "root" => CommitmentConfig::root(),
         "single" => CommitmentConfig::single(),
+        "singleGossip" => CommitmentConfig::single_gossip(),
         _ => CommitmentConfig::default(),
     })
 }
@@ -228,8 +229,8 @@ mod tests {
         assert_eq!(values_of(&matches, "multiple"), Some(vec![50, 39]));
         assert_eq!(values_of::<u64>(&matches, "single"), None);
 
-        let pubkey0 = Pubkey::new_rand();
+        let pubkey0 = solana_sdk::pubkey::new_rand();
-        let pubkey1 = Pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
         let matches = app().clone().get_matches_from(vec![
             "test",
             "--multiple",
@@ -251,7 +252,7 @@ mod tests {
         assert_eq!(value_of(&matches, "single"), Some(50));
         assert_eq!(value_of::<u64>(&matches, "multiple"), None);
 
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let matches = app()
             .clone()
             .get_matches_from(vec!["test", "--single", &pubkey.to_string()]);
@@ -331,8 +332,8 @@ mod tests {
 
     #[test]
     fn test_pubkeys_sigs_of() {
-        let key1 = Pubkey::new_rand();
+        let key1 = solana_sdk::pubkey::new_rand();
-        let key2 = Pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
         let sig1 = Keypair::new().sign_message(&[0u8]);
         let sig2 = Keypair::new().sign_message(&[1u8]);
         let signer1 = format!("{}={}", key1, sig1);

clap-utils/src/keypair.rs

@@ -298,7 +298,24 @@ pub fn keypair_from_seed_phrase(
         keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
     } else {
         let sanitized = sanitize_seed_phrase(seed_phrase);
-        let mnemonic = Mnemonic::from_phrase(&sanitized, Language::English)?;
+        let parse_language_fn = || {
+            for language in &[
+                Language::English,
+                Language::ChineseSimplified,
+                Language::ChineseTraditional,
+                Language::Japanese,
+                Language::Spanish,
+                Language::Korean,
+                Language::French,
+                Language::Italian,
+            ] {
+                if let Ok(mnemonic) = Mnemonic::from_phrase(&sanitized, *language) {
+                    return Ok(mnemonic);
+                }
+            }
+            Err("Can't get mnemonic from seed phrases")
+        };
+        let mnemonic = parse_language_fn()?;
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
         let seed = Seed::new(&mnemonic, &passphrase);
         keypair_from_seed(seed.as_bytes())?

cli-config/Cargo.toml

@@ -3,13 +3,13 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
-dirs = "2.0.2"
+dirs-next = "2.0.0"
 lazy_static = "1.4.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"

cli-config/src/config.rs

@@ -5,7 +5,7 @@ use url::Url;
 
 lazy_static! {
     pub static ref CONFIG_FILE: Option<String> = {
-        dirs::home_dir().map(|mut path| {
+        dirs_next::home_dir().map(|mut path| {
             path.extend(&[".config", "solana", "cli", "config.yml"]);
             path.to_str().unwrap().to_string()
         })
@@ -25,7 +25,7 @@ pub struct Config {
 impl Default for Config {
     fn default() -> Self {
         let keypair_path = {
-            let mut keypair_path = dirs::home_dir().expect("home directory");
+            let mut keypair_path = dirs_next::home_dir().expect("home directory");
             keypair_path.extend(&[".config", "solana", "id.json"]);
             keypair_path.to_str().unwrap().to_string()
         };

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,13 +17,13 @@ indicatif = "0.15.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.12" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-client = { path = "../client", version = "1.4.0" }
+solana-client = { path = "../client", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.12" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.12" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -132,7 +132,7 @@ impl fmt::Display for CliBlockProduction {
             "{}",
             style(format!(
                 "  {:<44}  {:>15}  {:>15}  {:>15}  {:>23}",
-                "Identity Pubkey",
+                "Identity",
                 "Leader Slots",
                 "Blocks Produced",
                 "Skipped Slots",
@@ -301,7 +301,7 @@ pub struct CliValidatorsStakeByVersion {
 pub struct CliValidators {
     pub total_active_stake: u64,
     pub total_current_stake: u64,
-    pub total_deliquent_stake: u64,
+    pub total_delinquent_stake: u64,
     pub current_validators: Vec<CliValidator>,
     pub delinquent_validators: Vec<CliValidator>,
     pub stake_by_version: BTreeMap<String, CliValidatorsStakeByVersion>,
@@ -360,7 +360,7 @@ impl fmt::Display for CliValidators {
             "Active Stake:",
             &build_balance_message(self.total_active_stake, self.use_lamports_unit, true),
         )?;
-        if self.total_deliquent_stake > 0 {
+        if self.total_delinquent_stake > 0 {
             writeln_name_value(
                 f,
                 "Current Stake:",
@@ -376,11 +376,11 @@ impl fmt::Display for CliValidators {
                 &format!(
                     "{} ({:0.2}%)",
                     &build_balance_message(
-                        self.total_deliquent_stake,
+                        self.total_delinquent_stake,
                         self.use_lamports_unit,
                         true
                     ),
-                    100. * self.total_deliquent_stake as f64 / self.total_active_stake as f64
+                    100. * self.total_delinquent_stake as f64 / self.total_active_stake as f64
                 ),
             )?;
         }
@@ -412,8 +412,8 @@ impl fmt::Display for CliValidators {
             "{}",
             style(format!(
                 "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
-                "Identity Pubkey",
+                "Identity",
-                "Vote Account Pubkey",
+                "Vote Account",
                 "Commission",
                 "Last Vote",
                 "Root Block",
@@ -520,7 +520,7 @@ impl fmt::Display for CliNonceAccount {
             )
         )?;
         let nonce = self.nonce.as_deref().unwrap_or("uninitialized");
-        writeln!(f, "Nonce: {}", nonce)?;
+        writeln!(f, "Nonce blockhash: {}", nonce)?;
         if let Some(fees) = self.lamports_per_signature {
             writeln!(f, "Fee: {} lamports per signature", fees)?;
         } else {
@@ -661,13 +661,8 @@ impl fmt::Display for CliStakeState {
                 if lockup.unix_timestamp != UnixTimestamp::default() {
                     writeln!(
                         f,
-                        "Lockup Timestamp: {} (UnixTimestamp: {})",
+                        "Lockup Timestamp: {}",
-                        DateTime::<Utc>::from_utc(
+                        unix_timestamp_to_string(lockup.unix_timestamp)
-                            NaiveDateTime::from_timestamp(lockup.unix_timestamp, 0),
-                            Utc
-                        )
-                        .to_rfc3339_opts(SecondsFormat::Secs, true),
-                        lockup.unix_timestamp
                     )?;
                 }
                 if lockup.epoch != Epoch::default() {
@@ -952,8 +947,8 @@ impl VerboseDisplay for CliValidatorInfo {}
 
 impl fmt::Display for CliValidatorInfo {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln_name_value(f, "Validator Identity Pubkey:", &self.identity_pubkey)?;
+        writeln_name_value(f, "Validator Identity:", &self.identity_pubkey)?;
-        writeln_name_value(f, "  Info Pubkey:", &self.info_pubkey)?;
+        writeln_name_value(f, "  Info Address:", &self.info_pubkey)?;
         for (key, value) in self.info.iter() {
             writeln_name_value(
                 f,
@@ -1008,7 +1003,12 @@ impl fmt::Display for CliVoteAccount {
                 None => "~".to_string(),
             }
         )?;
-        writeln!(f, "Recent Timestamp: {:?}", self.recent_timestamp)?;
+        writeln!(
+            f,
+            "Recent Timestamp: {} from slot {}",
+            unix_timestamp_to_string(self.recent_timestamp.timestamp),
+            self.recent_timestamp.slot
+        )?;
         if !self.votes.is_empty() {
             writeln!(f, "Recent Votes:")?;
             for vote in &self.votes {
@@ -1093,19 +1093,22 @@ pub struct CliBlockTime {
 impl QuietDisplay for CliBlockTime {}
 impl VerboseDisplay for CliBlockTime {}
 
+fn unix_timestamp_to_string(unix_timestamp: UnixTimestamp) -> String {
+    format!(
+        "{} (UnixTimestamp: {})",
+        match NaiveDateTime::from_timestamp_opt(unix_timestamp, 0) {
+            Some(ndt) =>
+                DateTime::<Utc>::from_utc(ndt, Utc).to_rfc3339_opts(SecondsFormat::Secs, true),
+            None => "unknown".to_string(),
+        },
+        unix_timestamp,
+    )
+}
+
 impl fmt::Display for CliBlockTime {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln_name_value(f, "Block:", &self.slot.to_string())?;
-        writeln_name_value(
+        writeln_name_value(f, "Date:", &unix_timestamp_to_string(self.timestamp))
-            f,
-            "Date:",
-            &format!(
-                "{} (UnixTimestamp: {})",
-                DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(self.timestamp, 0), Utc)
-                    .to_rfc3339_opts(SecondsFormat::Secs, true),
-                self.timestamp
-            ),
-        )
     }
 }
 

cli-output/src/display.rs

@@ -197,6 +197,15 @@ pub fn write_transaction<W: io::Write>(
                 )?;
             }
         }
+
+        if let Some(log_messages) = &transaction_status.log_messages {
+            if !log_messages.is_empty() {
+                writeln!(w, "{}Log Messages:", prefix,)?;
+                for log_message in log_messages {
+                    writeln!(w, "{}  {}", prefix, log_message,)?;
+                }
+            }
+        }
     } else {
         writeln!(w, "{}Status: Unavailable", prefix)?;
     }

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.4.12"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -16,39 +16,41 @@ clap = "2.33.1"
 criterion-stats = "0.3.0"
 ctrlc = { version = "3.1.5", features = ["termination"] }
 console = "0.11.3"
-dirs = "2.0.2"
+dirs-next = "2.0.0"
 log = "0.4.8"
 Inflector = "0.11.4"
 indicatif = "0.15.0"
 humantime = "2.0.1"
 num-traits = "0.2"
 pretty-hex = "0.1.1"
-reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.12" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.12" }
-solana-cli-config = { path = "../cli-config", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-cli-output = { path = "../cli-output", version = "1.4.0" }
+solana-cli-config = { path = "../cli-config", version = "1.4.12" }
-solana-client = { path = "../client", version = "1.4.0" }
+solana-cli-output = { path = "../cli-output", version = "1.4.12" }
-solana-config-program = { path = "../programs/config", version = "1.4.0" }
+solana-client = { path = "../client", version = "1.4.12" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
+solana-config-program = { path = "../programs/config", version = "1.4.12" }
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-faucet = { path = "../faucet", version = "1.4.12" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.4.0" }
+solana-net-utils = { path = "../net-utils", version = "1.4.12" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana_rbpf = "=0.1.33"
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.4.12" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.12" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.12" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
-solana-vote-signer = { path = "../vote-signer", version = "1.4.0" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.12" }
+solana-vote-signer = { path = "../vote-signer", version = "1.4.12" }
 thiserror = "1.0.20"
+tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.4.0" }
+solana-core = { path = "../core", version = "1.4.12" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/checks.rs

@@ -54,13 +54,43 @@ pub fn check_account_for_multiple_fees_with_commitment(
     fee_calculator: &FeeCalculator,
     messages: &[&Message],
     commitment: CommitmentConfig,
+) -> Result<(), CliError> {
+    check_account_for_spend_multiple_fees_with_commitment(
+        rpc_client,
+        account_pubkey,
+        0,
+        fee_calculator,
+        messages,
+        commitment,
+    )
+}
+
+pub fn check_account_for_spend_multiple_fees_with_commitment(
+    rpc_client: &RpcClient,
+    account_pubkey: &Pubkey,
+    balance: u64,
+    fee_calculator: &FeeCalculator,
+    messages: &[&Message],
+    commitment: CommitmentConfig,
 ) -> Result<(), CliError> {
     let fee = calculate_fee(fee_calculator, messages);
-    if !check_account_for_balance_with_commitment(rpc_client, account_pubkey, fee, commitment)
+    if !check_account_for_balance_with_commitment(
+        rpc_client,
+        account_pubkey,
+        balance + fee,
+        commitment,
+    )
     .map_err(Into::<ClientError>::into)?
     {
+        if balance > 0 {
+            return Err(CliError::InsufficientFundsForSpendAndFee(
+                lamports_to_sol(balance),
+                lamports_to_sol(fee),
+            ));
+        } else {
             return Err(CliError::InsufficientFundsForFee(lamports_to_sol(fee)));
         }
+    }
     Ok(())
 }
 
@@ -131,7 +161,7 @@ mod tests {
             context: RpcResponseContext { slot: 1 },
             value: json!(account_balance),
         });
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let fee_calculator = FeeCalculator::new(1);
 
         let pubkey0 = Pubkey::new(&[0; 32]);
@@ -191,7 +221,7 @@ mod tests {
             context: RpcResponseContext { slot: 1 },
             value: json!(account_balance),
         });
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
 
         let mut mocks = HashMap::new();
         mocks.insert(RpcRequest::GetBalance, account_balance_response);
@@ -237,9 +267,9 @@ mod tests {
 
     #[test]
     fn test_check_unique_pubkeys() {
-        let pubkey0 = Pubkey::new_rand();
+        let pubkey0 = solana_sdk::pubkey::new_rand();
         let pubkey_clone = pubkey0;
-        let pubkey1 = Pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
 
         check_unique_pubkeys((&pubkey0, "foo".to_string()), (&pubkey1, "bar".to_string()))
             .expect("unexpected result");

cli/src/cli.rs

@@ -1,12 +1,15 @@
 use crate::{
-    checks::*, cluster_query::*, feature::*, inflation::*, nonce::*, spend_utils::*, stake::*,
+    checks::*, cluster_query::*, feature::*, inflation::*, nonce::*, send_tpu::*, spend_utils::*,
-    validator_info::*, vote::*,
+    stake::*, validator_info::*, vote::*,
 };
+use bincode::serialize;
+use bip39::{Language, Mnemonic, MnemonicType, Seed};
 use clap::{value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use log::*;
 use num_traits::FromPrimitive;
 use serde_json::{self, json, Value};
 use solana_account_decoder::{UiAccount, UiAccountEncoding};
+use solana_bpf_loader_program::bpf_verifier;
 use solana_clap_utils::{
     self,
     commitment::commitment_arg_with_default,
@@ -28,26 +31,28 @@ use solana_client::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     nonce_utils,
     rpc_client::RpcClient,
-    rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig},
+    rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig, RpcTransactionLogsFilter},
     rpc_request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS,
-    rpc_response::RpcKeyedAccount,
+    rpc_response::{RpcKeyedAccount, RpcLeaderSchedule},
 };
 #[cfg(not(test))]
 use solana_faucet::faucet::request_airdrop_transaction;
 #[cfg(test)]
 use solana_faucet::faucet_mock::request_airdrop_transaction;
+use solana_rbpf::vm::EbpfVm;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     bpf_loader, bpf_loader_deprecated,
-    clock::{Epoch, Slot, DEFAULT_TICKS_PER_SECOND},
+    clock::{Epoch, Slot},
     commitment_config::CommitmentConfig,
     decode_error::DecodeError,
     hash::Hash,
     instruction::{Instruction, InstructionError},
     loader_instruction,
     message::Message,
+    native_token::Sol,
     pubkey::{Pubkey, MAX_SEED_LEN},
-    signature::{Keypair, Signature, Signer, SignerError},
+    signature::{keypair_from_seed, Keypair, Signature, Signer, SignerError},
     signers::Signers,
     system_instruction::{self, SystemError},
     system_program,
@@ -60,12 +65,13 @@ use solana_stake_program::{
 use solana_transaction_status::{EncodedTransaction, UiTransactionEncoding};
 use solana_vote_program::vote_state::VoteAuthorize;
 use std::{
+    cmp::min,
     collections::HashMap,
     error,
     fmt::Write as FmtWrite,
     fs::File,
     io::{Read, Write},
-    net::{IpAddr, SocketAddr},
+    net::{IpAddr, SocketAddr, UdpSocket},
     str::FromStr,
     sync::Arc,
     thread::sleep,
@@ -98,7 +104,7 @@ pub enum CliCommand {
     Fees,
     FirstAvailableBlock,
     GetBlock {
-        slot: Slot,
+        slot: Option<Slot>,
     },
     GetBlockTime {
         slot: Option<Slot>,
@@ -114,6 +120,9 @@ pub enum CliCommand {
     },
     LeaderSchedule,
     LiveSlots,
+    Logs {
+        filter: RpcTransactionLogsFilter,
+    },
     Ping {
         lamports: u64,
         interval: Duration,
@@ -175,6 +184,7 @@ pub enum CliCommand {
         program_location: String,
         address: Option<SignerIndex>,
         use_deprecated_loader: bool,
+        allow_excessive_balance: bool,
     },
     // Stake Commands
     CreateStakeAccount {
@@ -575,6 +585,7 @@ pub fn parse_command(
             command: CliCommand::LiveSlots,
             signers: vec![],
         }),
+        ("logs", Some(matches)) => parse_logs(matches, wallet_manager),
         ("block-production", Some(matches)) => parse_show_block_production(matches),
         ("gossip", Some(_matches)) => Ok(CliCommandInfo {
             command: CliCommand::ShowGossip,
@@ -606,13 +617,13 @@ pub fn parse_command(
                 signers.push(signer);
                 1
             });
-            let use_deprecated_loader = matches.is_present("use_deprecated_loader");
 
             Ok(CliCommandInfo {
                 command: CliCommand::Deploy {
                     program_location: matches.value_of("program_location").unwrap().to_string(),
                     address,
-                    use_deprecated_loader,
+                    use_deprecated_loader: matches.is_present("use_deprecated_loader"),
+                    allow_excessive_balance: matches.is_present("allow_excessive_balance"),
                 },
                 signers,
             })
@@ -1026,20 +1037,36 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
 ) -> Result<(), Box<dyn error::Error>> {
     let progress_bar = new_spinner_progress_bar();
     let mut send_retries = 5;
+    let mut leader_schedule: Option<RpcLeaderSchedule> = None;
+    let mut leader_schedule_epoch = 0;
+    let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+    let cluster_nodes = rpc_client.get_cluster_nodes().ok();
+
     loop {
         let mut status_retries = 15;
 
+        progress_bar.set_message("Finding leader node...");
+        let epoch_info = rpc_client.get_epoch_info_with_commitment(commitment)?;
+        if epoch_info.epoch > leader_schedule_epoch || leader_schedule.is_none() {
+            leader_schedule = rpc_client
+                .get_leader_schedule_with_commitment(Some(epoch_info.absolute_slot), commitment)?;
+            leader_schedule_epoch = epoch_info.epoch;
+        }
+        let tpu_address = get_leader_tpu(
+            min(epoch_info.slot_index + 1, epoch_info.slots_in_epoch),
+            leader_schedule.as_ref(),
+            cluster_nodes.as_ref(),
+        );
+
         // Send all transactions
         let mut pending_transactions = HashMap::new();
         let num_transactions = transactions.len();
         for transaction in transactions {
-            if cfg!(not(test)) {
+            if let Some(tpu_address) = tpu_address {
-                // Delay ~1 tick between write transactions in an attempt to reduce AccountInUse errors
+                let wire_transaction =
-                // when all the write transactions modify the same program account (eg, deploying a
+                    serialize(&transaction).expect("serialization should succeed");
-                // new program)
+                send_transaction_tpu(&send_socket, &tpu_address, &wire_transaction);
-                sleep(Duration::from_millis(1000 / DEFAULT_TICKS_PER_SECOND));
+            } else {
-            }
-
                 let _result = rpc_client
                     .send_transaction_with_config(
                         &transaction,
@@ -1049,10 +1076,11 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
                         },
                     )
                     .ok();
+            }
             pending_transactions.insert(transaction.signatures[0], transaction);
 
             progress_bar.set_message(&format!(
-                "[{}/{}] Transactions sent",
+                "[{}/{}] Total Transactions sent",
                 pending_transactions.len(),
                 num_transactions
             ));
@@ -1088,6 +1116,11 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
                         let _ = pending_transactions.remove(&signature);
                     }
                 }
+                progress_bar.set_message(&format!(
+                    "[{}/{}] Transactions confirmed",
+                    num_transactions - pending_transactions.len(),
+                    num_transactions
+                ));
             }
 
             if pending_transactions.is_empty() {
@@ -1129,8 +1162,52 @@ fn process_deploy(
     program_location: &str,
     address: Option<SignerIndex>,
     use_deprecated_loader: bool,
+    allow_excessive_balance: bool,
+) -> ProcessResult {
+    const WORDS: usize = 12;
+    // Create ephemeral keypair to use for program address, if not provided
+    let mnemonic = Mnemonic::new(MnemonicType::for_word_count(WORDS)?, Language::English);
+    let seed = Seed::new(&mnemonic, "");
+    let new_keypair = keypair_from_seed(seed.as_bytes())?;
+
+    let result = do_process_deploy(
+        rpc_client,
+        config,
+        program_location,
+        address,
+        use_deprecated_loader,
+        allow_excessive_balance,
+        new_keypair,
+    );
+
+    if result.is_err() && address.is_none() {
+        let phrase: &str = mnemonic.phrase();
+        let divider = String::from_utf8(vec![b'='; phrase.len()]).unwrap();
+        eprintln!(
+            "{}\nTo reuse this address, recover the ephemeral keypair file with",
+            divider
+        );
+        eprintln!(
+            "`solana-keygen recover` and the following {}-word seed phrase,",
+            WORDS
+        );
+        eprintln!(
+            "then pass it as the [PROGRAM_ADDRESS_SIGNER] argument to `solana deploy ...`\n{}\n{}\n{}",
+            divider, phrase, divider
+        );
+    }
+    result
+}
+
+fn do_process_deploy(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    program_location: &str,
+    address: Option<SignerIndex>,
+    use_deprecated_loader: bool,
+    allow_excessive_balance: bool,
+    new_keypair: Keypair,
 ) -> ProcessResult {
-    let new_keypair = Keypair::new(); // Create ephemeral keypair to use for program address, if not provided
     let program_id = if let Some(i) = address {
         config.signers[i]
     } else {
@@ -1144,6 +1221,9 @@ fn process_deploy(
         CliError::DynamicProgramError(format!("Unable to read program file: {}", err))
     })?;
 
+    EbpfVm::create_executable_from_elf(&program_data, Some(|x| bpf_verifier::check(x, false)))
+        .map_err(|err| CliError::DynamicProgramError(format!("ELF error: {}", err)))?;
+
     let loader_id = if use_deprecated_loader {
         bpf_loader_deprecated::id()
     } else {
@@ -1154,11 +1234,12 @@ fn process_deploy(
     let signers = [config.signers[0], program_id];
 
     // Check program account to see if partial initialization has occurred
-    let initial_instructions = if let Some(account) = rpc_client
+    let (initial_instructions, balance_needed) = if let Some(account) = rpc_client
         .get_account_with_commitment(&program_id.pubkey(), config.commitment)?
         .value
     {
         let mut instructions: Vec<Instruction> = vec![];
+        let mut balance_needed = 0;
         if account.executable {
             return Err(CliError::DynamicProgramError(
                 "Program account is already executable".to_string(),
@@ -1182,21 +1263,35 @@ fn process_deploy(
             }
         }
         if account.lamports < minimum_balance {
+            let balance = minimum_balance - account.lamports;
             instructions.push(system_instruction::transfer(
                 &config.signers[0].pubkey(),
                 &program_id.pubkey(),
-                minimum_balance - account.lamports,
+                balance,
             ));
+            balance_needed = balance;
+        } else if account.lamports > minimum_balance
+            && system_program::check_id(&account.owner)
+            && !allow_excessive_balance
+        {
+            return Err(CliError::DynamicProgramError(format!(
+                "Program account has a balance: {:?}; it may already be in use",
+                Sol(account.lamports)
+            ))
+            .into());
         }
-        instructions
+        (instructions, balance_needed)
     } else {
+        (
             vec![system_instruction::create_account(
                 &config.signers[0].pubkey(),
                 &program_id.pubkey(),
                 minimum_balance,
                 program_data.len() as u64,
                 &loader_id,
-        )]
+            )],
+            minimum_balance,
+        )
     };
     let initial_message = if !initial_instructions.is_empty() {
         Some(Message::new(
@@ -1239,9 +1334,10 @@ fn process_deploy(
         .get_recent_blockhash_with_commitment(config.commitment)?
         .value;
 
-    check_account_for_multiple_fees_with_commitment(
+    check_account_for_spend_multiple_fees_with_commitment(
         rpc_client,
         &config.signers[0].pubkey(),
+        balance_needed,
         &fee_calculator,
         &messages,
         config.commitment,
@@ -1266,8 +1362,8 @@ fn process_deploy(
             config.commitment,
             config.send_transaction_config,
         );
-        log_instruction_custom_error::<SystemError>(result, &config).map_err(|_| {
+        log_instruction_custom_error::<SystemError>(result, &config).map_err(|err| {
-            CliError::DynamicProgramError("Program account allocation failed".to_string())
+            CliError::DynamicProgramError(format!("Program account allocation failed: {}", err))
         })?;
     }
 
@@ -1290,8 +1386,8 @@ fn process_deploy(
         config.commitment,
         last_valid_slot,
     )
-    .map_err(|_| {
+    .map_err(|err| {
-        CliError::DynamicProgramError("Data writes to program account failed".to_string())
+        CliError::DynamicProgramError(format!("Data writes to program account failed: {}", err))
     })?;
 
     let (blockhash, _, _) = rpc_client
@@ -1453,7 +1549,8 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             process_inflation_subcommand(&rpc_client, config, inflation_subcommand)
         }
         CliCommand::LeaderSchedule => process_leader_schedule(&rpc_client),
-        CliCommand::LiveSlots => process_live_slots(&config.websocket_url),
+        CliCommand::LiveSlots => process_live_slots(&config),
+        CliCommand::Logs { filter } => process_logs(&config, filter),
         CliCommand::Ping {
             lamports,
             interval,
@@ -1565,12 +1662,14 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             program_location,
             address,
             use_deprecated_loader,
+            allow_excessive_balance,
         } => process_deploy(
             &rpc_client,
             config,
             program_location,
             *address,
             *use_deprecated_loader,
+            *allow_excessive_balance,
         ),
 
         // Stake Commands
@@ -2187,7 +2286,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                 .arg(
                     Arg::with_name("address_signer")
                         .index(2)
-                        .value_name("SIGNER_KEYPAIR")
+                        .value_name("PROGRAM_ADDRESS_SIGNER")
                         .takes_value(true)
                         .validator(is_valid_signer)
                         .help("The signer for the desired address of the program [default: new random address]")
@@ -2199,6 +2298,12 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .hidden(true) // Don't document this argument to discourage its use
                         .help("Use the deprecated BPF loader")
                 )
+                .arg(
+                    Arg::with_name("allow_excessive_balance")
+                        .long("allow-excessive-deploy-account-balance")
+                        .takes_value(false)
+                        .help("Use the designated program id, even if the account already holds a large balance of SOL")
+                )
                 .arg(commitment_arg_with_default("max")),
         )
         .subcommand(
@@ -2355,7 +2460,10 @@ mod tests {
             .unwrap();
         assert_eq!(signer_info.signers.len(), 1);
         assert_eq!(signer_info.index_of(None), Some(0));
-        assert_eq!(signer_info.index_of(Some(Pubkey::new_rand())), None);
+        assert_eq!(
+            signer_info.index_of(Some(solana_sdk::pubkey::new_rand())),
+            None
+        );
 
         let keypair0 = keypair_from_seed(&[1u8; 32]).unwrap();
         let keypair0_pubkey = keypair0.pubkey();
@@ -2411,7 +2519,7 @@ mod tests {
     fn test_cli_parse_command() {
         let test_commands = app("test", "desc", "version");
 
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let pubkey_string = format!("{}", pubkey);
 
         let default_keypair = Keypair::new();
@@ -2507,7 +2615,7 @@ mod tests {
         assert!(parse_command(&test_bad_signature, &default_signer, &mut None).is_err());
 
         // Test CreateAddressWithSeed
-        let from_pubkey = Some(Pubkey::new_rand());
+        let from_pubkey = Some(solana_sdk::pubkey::new_rand());
         let from_str = from_pubkey.unwrap().to_string();
         for (name, program_id) in &[
             ("STAKE", solana_stake_program::id()),
@@ -2564,6 +2672,7 @@ mod tests {
                     program_location: "/Users/test/program.o".to_string(),
                     address: None,
                     use_deprecated_loader: false,
+                    allow_excessive_balance: false,
                 },
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2585,6 +2694,7 @@ mod tests {
                     program_location: "/Users/test/program.o".to_string(),
                     address: Some(1),
                     use_deprecated_loader: false,
+                    allow_excessive_balance: false,
                 },
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2664,7 +2774,7 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let new_authorized_pubkey = Pubkey::new_rand();
+        let new_authorized_pubkey = solana_sdk::pubkey::new_rand();
         config.signers = vec![&bob_keypair];
         config.command = CliCommand::VoteAuthorize {
             vote_account_pubkey: bob_pubkey,
@@ -2686,7 +2796,7 @@ mod tests {
 
         let bob_keypair = Keypair::new();
         let bob_pubkey = bob_keypair.pubkey();
-        let custodian = Pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         config.command = CliCommand::CreateStakeAccount {
             stake_account: 1,
             seed: None,
@@ -2709,8 +2819,8 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
-        let to_pubkey = Pubkey::new_rand();
+        let to_pubkey = solana_sdk::pubkey::new_rand();
         config.command = CliCommand::WithdrawStake {
             stake_account_pubkey,
             destination_account_pubkey: to_pubkey,
@@ -2727,7 +2837,7 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
         config.command = CliCommand::DeactivateStake {
             stake_account_pubkey,
             stake_authority: 0,
@@ -2740,7 +2850,7 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
         let split_stake_account = Keypair::new();
         config.command = CliCommand::SplitStake {
             stake_account_pubkey,
@@ -2758,8 +2868,8 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
-        let source_stake_account_pubkey = Pubkey::new_rand();
+        let source_stake_account_pubkey = solana_sdk::pubkey::new_rand();
         let merge_stake_account = Keypair::new();
         config.command = CliCommand::MergeStake {
             stake_account_pubkey,
@@ -2782,7 +2892,7 @@ mod tests {
         assert_eq!(process_command(&config).unwrap(), "1234");
 
         // CreateAddressWithSeed
-        let from_pubkey = Pubkey::new_rand();
+        let from_pubkey = solana_sdk::pubkey::new_rand();
         config.signers = vec![];
         config.command = CliCommand::CreateAddressWithSeed {
             from_pubkey: Some(from_pubkey),
@@ -2795,7 +2905,7 @@ mod tests {
         assert_eq!(address.unwrap(), expected_address.to_string());
 
         // Need airdrop cases
-        let to = Pubkey::new_rand();
+        let to = solana_sdk::pubkey::new_rand();
         config.signers = vec![&keypair];
         config.command = CliCommand::Airdrop {
             faucet_host: None,
@@ -2898,6 +3008,7 @@ mod tests {
             program_location: pathbuf.to_str().unwrap().to_string(),
             address: None,
             use_deprecated_loader: false,
+            allow_excessive_balance: false,
         };
         let result = process_command(&config);
         let json: Value = serde_json::from_str(&result.unwrap()).unwrap();
@@ -2916,6 +3027,7 @@ mod tests {
             program_location: "bad/file/location.so".to_string(),
             address: None,
             use_deprecated_loader: false,
+            allow_excessive_balance: false,
         };
         assert!(process_command(&config).is_err());
     }

cli/src/cluster_query.rs

@@ -1,12 +1,16 @@
 use crate::{
     cli::{CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult},
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
+    stake::is_stake_program_v2_enabled,
 };
 use chrono::{Local, TimeZone};
 use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use solana_clap_utils::{
-    commitment::commitment_arg, input_parsers::*, input_validators::*, keypair::DefaultSigner,
+    commitment::{commitment_arg, commitment_arg_with_default},
+    input_parsers::*,
+    input_validators::*,
+    keypair::DefaultSigner,
 };
 use solana_cli_output::{
     display::{
@@ -20,13 +24,14 @@ use solana_client::{
     rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClient},
     rpc_config::{
         RpcAccountInfoConfig, RpcLargestAccountsConfig, RpcLargestAccountsFilter,
-        RpcProgramAccountsConfig,
+        RpcProgramAccountsConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter,
     },
     rpc_filter,
     rpc_response::SlotInfo,
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
+    account::from_account,
     account_utils::StateMut,
     clock::{self, Clock, Slot},
     commitment_config::CommitmentConfig,
@@ -38,8 +43,7 @@ use solana_sdk::{
     system_instruction, system_program,
     sysvar::{
         self,
-        stake_history::{self, StakeHistory},
+        stake_history::{self},
-        Sysvar,
     },
     transaction::Transaction,
 };
@@ -73,8 +77,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .validator(is_slot)
                         .value_name("SLOT")
                         .takes_value(true)
-                        .index(1)
+                        .index(1),
-                        .required(true),
                 ),
         )
         .subcommand(
@@ -233,6 +236,26 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("live-slots")
                 .about("Show information about the current slot progression"),
         )
+        .subcommand(
+            SubCommand::with_name("logs")
+                .about("Stream transaction logs")
+                .arg(
+                    pubkey!(Arg::with_name("address")
+                        .index(1)
+                        .value_name("ADDRESS"),
+                        "Account address to monitor \
+                         [default: monitor all transactions except for votes] \
+                        ")
+                )
+                .arg(
+                    Arg::with_name("include_votes")
+                        .long("include-votes")
+                        .takes_value(false)
+                        .conflicts_with("address")
+                        .help("Include vote transactions when monitoring all transactions")
+                )
+                .arg(commitment_arg_with_default("singleGossip")),
+        )
         .subcommand(
             SubCommand::with_name("block-production")
                 .about("Show information about block production")
@@ -363,7 +386,7 @@ pub fn parse_cluster_ping(
 }
 
 pub fn parse_get_block(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let slot = value_t_or_exit!(matches, "slot", Slot);
+    let slot = value_of(matches, "slot");
     Ok(CliCommandInfo {
         command: CliCommand::GetBlock { slot },
         signers: vec![],
@@ -625,7 +648,7 @@ pub fn process_cluster_date(rpc_client: &RpcClient, config: &CliConfig) -> Proce
     let result = rpc_client
         .get_account_with_commitment(&sysvar::clock::id(), CommitmentConfig::default())?;
     if let Some(clock_account) = result.value {
-        let clock: Clock = Sysvar::from_account(&clock_account).ok_or_else(|| {
+        let clock: Clock = from_account(&clock_account).ok_or_else(|| {
             CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
         })?;
         let block_time = CliBlockTime {
@@ -700,7 +723,17 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
     Ok("".to_string())
 }
 
-pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot) -> ProcessResult {
+pub fn process_get_block(
+    rpc_client: &RpcClient,
+    _config: &CliConfig,
+    slot: Option<Slot>,
+) -> ProcessResult {
+    let slot = if let Some(slot) = slot {
+        slot
+    } else {
+        rpc_client.get_slot()?
+    };
+
     let mut block =
         rpc_client.get_confirmed_block_with_encoding(slot, UiTransactionEncoding::Base64)?;
 
@@ -716,18 +749,23 @@ pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot
         let mut total_rewards = 0;
         println!("Rewards:",);
         println!(
-            "  {:<44}  {:<15}  {:<13}  {:>14}",
+            "  {:<44}  {:^15}  {:<15}  {:<20}  {:>14}",
-            "Address", "Amount", "New Balance", "Percent Change"
+            "Address", "Type", "Amount", "New Balance", "Percent Change"
         );
         for reward in block.rewards {
             let sign = if reward.lamports < 0 { "-" } else { "" };
 
             total_rewards += reward.lamports;
             println!(
-                "  {:<44}  {:>15}  {}",
+                "  {:<44}  {:^15}  {:>15}  {}",
                 reward.pubkey,
+                if let Some(reward_type) = reward.reward_type {
+                    format!("{}", reward_type)
+                } else {
+                    "-".to_string()
+                },
                 format!(
-                    "{}◎{:<14.4}",
+                    "{}◎{:<14.9}",
                     sign,
                     lamports_to_sol(reward.lamports.abs() as u64)
                 ),
@@ -735,7 +773,7 @@ pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot
                     "          -                 -".to_string()
                 } else {
                     format!(
-                        "◎{:<12.4}  {:>13.4}%",
+                        "◎{:<19.9}  {:>13.9}%",
                         lamports_to_sol(reward.post_balance),
                         reward.lamports.abs() as f64
                             / (reward.post_balance as f64 - reward.lamports as f64)
@@ -746,7 +784,7 @@ pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot
 
         let sign = if total_rewards < 0 { "-" } else { "" };
         println!(
-            "Total Rewards: {}◎{:12.9}",
+            "Total Rewards: {}◎{:<12.9}",
             sign,
             lamports_to_sol(total_rewards.abs() as u64)
         );
@@ -1146,24 +1184,83 @@ pub fn process_ping(
     Ok("".to_string())
 }
 
-pub fn process_live_slots(url: &str) -> ProcessResult {
+pub fn parse_logs(
-    let exit = Arc::new(AtomicBool::new(false));
+    matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let address = pubkey_of_signer(matches, "address", wallet_manager)?;
+    let include_votes = matches.is_present("include_votes");
 
-    // Disable Ctrl+C handler as sometimes the PubsubClient shutdown can stall.  Also it doesn't
+    let filter = match address {
-    // really matter that the shutdown is clean because the process is terminating.
+        None => {
-    /*
+            if include_votes {
-    let exit_clone = exit.clone();
+                RpcTransactionLogsFilter::AllWithVotes
-    ctrlc::set_handler(move || {
+            } else {
-        exit_clone.store(true, Ordering::Relaxed);
+                RpcTransactionLogsFilter::All
-    })?;
+            }
-    */
+        }
+        Some(address) => RpcTransactionLogsFilter::Mentions(vec![address.to_string()]),
+    };
+
+    Ok(CliCommandInfo {
+        command: CliCommand::Logs { filter },
+        signers: vec![],
+    })
+}
+
+pub fn process_logs(config: &CliConfig, filter: &RpcTransactionLogsFilter) -> ProcessResult {
+    println!(
+        "Streaming transaction logs{}. {:?} commitment",
+        match filter {
+            RpcTransactionLogsFilter::All => "".into(),
+            RpcTransactionLogsFilter::AllWithVotes => " (including votes)".into(),
+            RpcTransactionLogsFilter::Mentions(addresses) =>
+                format!(" mentioning {}", addresses.join(",")),
+        },
+        config.commitment.commitment
+    );
+
+    let (_client, receiver) = PubsubClient::logs_subscribe(
+        &config.websocket_url,
+        filter.clone(),
+        RpcTransactionLogsConfig {
+            commitment: Some(config.commitment),
+        },
+    )?;
+
+    loop {
+        match receiver.recv() {
+            Ok(logs) => {
+                println!("Transaction executed in slot {}:", logs.context.slot);
+                println!("  Signature: {}", logs.value.signature);
+                println!(
+                    "  Status: {}",
+                    logs.value
+                        .err
+                        .map(|err| err.to_string())
+                        .unwrap_or_else(|| "Ok".to_string())
+                );
+                println!("  Log Messages:");
+                for log in logs.value.logs {
+                    println!("    {}", log);
+                }
+            }
+            Err(err) => {
+                return Ok(format!("Disconnected: {}", err));
+            }
+        }
+    }
+}
+
+pub fn process_live_slots(config: &CliConfig) -> ProcessResult {
+    let exit = Arc::new(AtomicBool::new(false));
 
     let mut current: Option<SlotInfo> = None;
     let mut message = "".to_string();
 
     let slot_progress = new_spinner_progress_bar();
     slot_progress.set_message("Connecting...");
-    let (mut client, receiver) = PubsubClient::slot_subscribe(url)?;
+    let (mut client, receiver) = PubsubClient::slot_subscribe(&config.websocket_url)?;
     slot_progress.set_message("Connected.");
 
     let spacer = "|";
@@ -1253,14 +1350,16 @@ pub fn process_show_gossip(rpc_client: &RpcClient, config: &CliConfig) -> Proces
         .into_iter()
         .map(|node| {
             format!(
-                "{:15} | {:44} | {:6} | {:5} | {:5} | {}",
+                "{:15} | {:44} | {:6} | {:5} | {:21} | {}",
                 node.gossip
                     .map(|addr| addr.ip().to_string())
                     .unwrap_or_else(|| "none".to_string()),
                 format_labeled_address(&node.pubkey, &config.address_labels),
                 format_port(node.gossip),
                 format_port(node.tpu),
-                format_port(node.rpc),
+                node.rpc
+                    .map(|addr| addr.to_string())
+                    .unwrap_or_else(|| "none".to_string()),
                 node.version.unwrap_or_else(|| "unknown".to_string()),
             )
         })
@@ -1268,9 +1367,9 @@ pub fn process_show_gossip(rpc_client: &RpcClient, config: &CliConfig) -> Proces
 
     Ok(format!(
         "IP Address      | Node identifier                              \
-         | Gossip | TPU   | RPC   | Version\n\
+         | Gossip | TPU   | RPC Address           | Version\n\
          ----------------+----------------------------------------------+\
-         --------+-------+-------+----------------\n\
+         --------+-------+-----------------------+----------------\n\
          {}\n\
          Nodes: {}",
         s.join("\n"),
@@ -1325,14 +1424,16 @@ pub fn process_show_stakes(
         .get_program_accounts_with_config(&solana_stake_program::id(), program_accounts_config)?;
     let stake_history_account = rpc_client.get_account(&stake_history::id())?;
     let clock_account = rpc_client.get_account(&sysvar::clock::id())?;
-    let clock: Clock = Sysvar::from_account(&clock_account).ok_or_else(|| {
+    let clock: Clock = from_account(&clock_account).ok_or_else(|| {
         CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
     })?;
     progress_bar.finish_and_clear();
 
-    let stake_history = StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+    let stake_history = from_account(&stake_history_account).ok_or_else(|| {
         CliError::RpcRequestError("Failed to deserialize stake history".to_string())
     })?;
+    // At v1.6, this check can be removed and simply passed as `true`
+    let stake_program_v2_enabled = is_stake_program_v2_enabled(rpc_client);
 
     let mut stake_accounts: Vec<CliKeyedStakeState> = vec![];
     for (stake_pubkey, stake_account) in all_stake_accounts {
@@ -1348,6 +1449,7 @@ pub fn process_show_stakes(
                                 use_lamports_unit,
                                 &stake_history,
                                 &clock,
+                                stake_program_v2_enabled,
                             ),
                         });
                     }
@@ -1366,6 +1468,7 @@ pub fn process_show_stakes(
                                 use_lamports_unit,
                                 &stake_history,
                                 &clock,
+                                stake_program_v2_enabled,
                             ),
                         });
                     }
@@ -1405,12 +1508,12 @@ pub fn process_show_validators(
         .map(|vote_account| vote_account.activated_stake)
         .sum();
 
-    let total_deliquent_stake = vote_accounts
+    let total_delinquent_stake = vote_accounts
         .delinquent
         .iter()
         .map(|vote_account| vote_account.activated_stake)
         .sum();
-    let total_current_stake = total_active_stake - total_deliquent_stake;
+    let total_current_stake = total_active_stake - total_delinquent_stake;
 
     let mut current = vote_accounts.current;
     current.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
@@ -1464,7 +1567,7 @@ pub fn process_show_validators(
     let cli_validators = CliValidators {
         total_active_stake,
         total_current_stake,
-        total_deliquent_stake,
+        total_delinquent_stake,
         current_validators,
         delinquent_validators,
         stake_by_version,

cli/src/feature.rs

@@ -9,12 +9,13 @@ use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::*};
 use solana_cli_output::{QuietDisplay, VerboseDisplay};
 use solana_client::{client_error::ClientError, rpc_client::RpcClient};
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
-use solana_runtime::{
+use solana_sdk::{
+    clock::Slot,
     feature::{self, Feature},
     feature_set::FEATURE_NAMES,
-};
+    message::Message,
-use solana_sdk::{
+    pubkey::Pubkey,
-    clock::Slot, message::Message, pubkey::Pubkey, system_instruction, transaction::Transaction,
+    transaction::Transaction,
 };
 use std::{collections::HashMap, fmt, sync::Arc};
 
@@ -230,7 +231,7 @@ fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, u6
 }
 
 // Feature activation is only allowed when 95% of the active stake is on the current feature set
-fn feature_activation_allowed(rpc_client: &RpcClient) -> Result<bool, ClientError> {
+fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<bool, ClientError> {
     let my_feature_set = solana_version::Version::default().feature_set;
 
     let active_stake_by_feature_set = active_stake_by_feature_set(rpc_client)?;
@@ -240,8 +241,23 @@ fn feature_activation_allowed(rpc_client: &RpcClient) -> Result<bool, ClientErro
         .map(|percentage| *percentage >= 95)
         .unwrap_or(false);
 
-    if !feature_activation_allowed {
+    if !feature_activation_allowed && !quiet {
-        println!("\n{}", style("Stake By Feature Set:").bold());
+        if active_stake_by_feature_set.get(&my_feature_set).is_none() {
+            println!(
+                "{}",
+                style("To activate features the tool and cluster feature sets must match, select a tool version that matches the cluster")
+                    .bold());
+        } else {
+            println!(
+                "{}",
+                style("To activate features the stake must be >= 95%").bold()
+            );
+        }
+        println!(
+            "{}",
+            style(format!("Tool Feture Set: {}", my_feature_set)).bold()
+        );
+        println!("{}", style("Cluster Feature Sets and Stakes:").bold());
         for (feature_set, percentage) in active_stake_by_feature_set.iter() {
             if *feature_set == 0 {
                 println!("unknown - {}%", percentage);
@@ -258,6 +274,7 @@ fn feature_activation_allowed(rpc_client: &RpcClient) -> Result<bool, ClientErro
                 );
             }
         }
+        println!();
     }
 
     Ok(feature_activation_allowed)
@@ -278,7 +295,7 @@ fn process_status(
         let feature_id = &feature_ids[i];
         let feature_name = FEATURE_NAMES.get(feature_id).unwrap();
         if let Some(account) = account {
-            if let Some(feature) = Feature::from_account(&account) {
+            if let Some(feature) = feature::from_account(&account) {
                 let feature_status = match feature.activated_at {
                     None => CliFeatureStatus::Pending,
                     Some(activation_slot) => CliFeatureStatus::Active(activation_slot),
@@ -299,9 +316,10 @@ fn process_status(
         });
     }
 
+    let feature_activation_allowed = feature_activation_allowed(rpc_client, features.len() <= 1)?;
     let feature_set = CliFeatures {
         features,
-        feature_activation_allowed: feature_activation_allowed(rpc_client)?,
+        feature_activation_allowed,
         inactive,
     };
     Ok(config.output_format.formatted_string(&feature_set))
@@ -318,12 +336,12 @@ fn process_activate(
         .next()
         .unwrap();
     if let Some(account) = account {
-        if Feature::from_account(&account).is_some() {
+        if feature::from_account(&account).is_some() {
             return Err(format!("{} has already been activated", feature_id).into());
         }
     }
 
-    if !feature_activation_allowed(rpc_client)? {
+    if !feature_activation_allowed(rpc_client, false)? {
         return Err("Feature activation is not allowed at this time".into());
     }
 
@@ -338,15 +356,11 @@ fn process_activate(
         &config.signers[0].pubkey(),
         |lamports| {
             Message::new(
-                &[
+                &feature::activate_with_lamports(
-                    system_instruction::transfer(
-                        &config.signers[0].pubkey(),
                     &feature_id,
+                    &config.signers[0].pubkey(),
                     lamports,
                 ),
-                    system_instruction::allocate(&feature_id, Feature::size_of() as u64),
-                    system_instruction::assign(&feature_id, &feature::id()),
-                ],
                 Some(&config.signers[0].pubkey()),
             )
         },

cli/src/lib.rs

@@ -26,6 +26,7 @@ pub mod cluster_query;
 pub mod feature;
 pub mod inflation;
 pub mod nonce;
+pub mod send_tpu;
 pub mod spend_utils;
 pub mod stake;
 pub mod test_utils;

cli/src/nonce.rs

@@ -580,6 +580,7 @@ mod tests {
         fee_calculator::FeeCalculator,
         hash::hash,
         nonce::{self, state::Versions, State},
+        nonce_account,
         signature::{read_keypair_file, write_keypair, Keypair, Signer},
         system_program,
     };
@@ -833,7 +834,7 @@ mod tests {
     #[test]
     fn test_check_nonce_account() {
         let blockhash = Hash::default();
-        let nonce_pubkey = Pubkey::new_rand();
+        let nonce_pubkey = solana_sdk::pubkey::new_rand();
         let data = Versions::new_current(State::Initialized(nonce::state::Data {
             authority: nonce_pubkey,
             blockhash,
@@ -869,7 +870,7 @@ mod tests {
         }
 
         let data = Versions::new_current(State::Initialized(nonce::state::Data {
-            authority: Pubkey::new_rand(),
+            authority: solana_sdk::pubkey::new_rand(),
             blockhash,
             fee_calculator: FeeCalculator::default(),
         }));
@@ -891,7 +892,7 @@ mod tests {
 
     #[test]
     fn test_account_identity_ok() {
-        let nonce_account = nonce::create_account(1).into_inner();
+        let nonce_account = nonce_account::create_account(1).into_inner();
         assert_eq!(account_identity_ok(&nonce_account), Ok(()));
 
         let system_account = Account::new(1, 0, &system_program::id());
@@ -910,7 +911,7 @@ mod tests {
 
     #[test]
     fn test_state_from_account() {
-        let mut nonce_account = nonce::create_account(1).into_inner();
+        let mut nonce_account = nonce_account::create_account(1).into_inner();
         assert_eq!(state_from_account(&nonce_account), Ok(State::Uninitialized));
 
         let data = nonce::state::Data {
@@ -935,7 +936,7 @@ mod tests {
 
     #[test]
     fn test_data_from_helpers() {
-        let mut nonce_account = nonce::create_account(1).into_inner();
+        let mut nonce_account = nonce_account::create_account(1).into_inner();
         let state = state_from_account(&nonce_account).unwrap();
         assert_eq!(
             data_from_state(&state),

cli/src/send_tpu.rs

@@ -0,0 +1,29 @@
+use log::*;
+use solana_client::rpc_response::{RpcContactInfo, RpcLeaderSchedule};
+use std::net::{SocketAddr, UdpSocket};
+
+pub fn get_leader_tpu(
+    slot_index: u64,
+    leader_schedule: Option<&RpcLeaderSchedule>,
+    cluster_nodes: Option<&Vec<RpcContactInfo>>,
+) -> Option<SocketAddr> {
+    leader_schedule?
+        .iter()
+        .find(|(_pubkey, slots)| slots.iter().any(|slot| *slot as u64 == slot_index))
+        .and_then(|(pubkey, _)| {
+            cluster_nodes?
+                .iter()
+                .find(|contact_info| contact_info.pubkey == *pubkey)
+                .and_then(|contact_info| contact_info.tpu)
+        })
+}
+
+pub fn send_transaction_tpu(
+    send_socket: &UdpSocket,
+    tpu_address: &SocketAddr,
+    wire_transaction: &[u8],
+) {
+    if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
+        warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
+    }
+}

cli/src/stake.rs

@@ -23,20 +23,25 @@ use solana_cli_output::{
     CliStakeType,
 };
 use solana_client::{
-    blockhash_query::BlockhashQuery, nonce_utils, rpc_client::RpcClient,
+    blockhash_query::BlockhashQuery,
-    rpc_request::DELINQUENT_VALIDATOR_SLOT_DISTANCE,
+    client_error::{ClientError, ClientErrorKind},
+    nonce_utils,
+    rpc_client::RpcClient,
+    rpc_custom_error,
+    rpc_request::{self, DELINQUENT_VALIDATOR_SLOT_DISTANCE},
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
+    account::from_account,
     account_utils::StateMut,
-    clock::{Clock, Epoch, Slot, UnixTimestamp},
+    clock::{Clock, Epoch, Slot, UnixTimestamp, SECONDS_PER_DAY},
+    feature, feature_set,
     message::Message,
     pubkey::Pubkey,
     system_instruction::SystemError,
     sysvar::{
         clock,
         stake_history::{self, StakeHistory},
-        Sysvar,
     },
     transaction::Transaction,
 };
@@ -1497,6 +1502,7 @@ pub fn build_stake_state(
     use_lamports_unit: bool,
     stake_history: &StakeHistory,
     clock: &Clock,
+    stake_program_v2_enabled: bool,
 ) -> CliStakeState {
     match stake_state {
         StakeState::Stake(
@@ -1508,9 +1514,12 @@ pub fn build_stake_state(
             stake,
         ) => {
             let current_epoch = clock.epoch;
-            let (active_stake, activating_stake, deactivating_stake) = stake
+            let (active_stake, activating_stake, deactivating_stake) =
-                .delegation
+                stake.delegation.stake_activating_and_deactivating(
-                .stake_activating_and_deactivating(current_epoch, Some(stake_history));
+                    current_epoch,
+                    Some(stake_history),
+                    stake_program_v2_enabled,
+                );
             let lockup = if lockup.is_in_force(clock, None) {
                 Some(lockup.into())
             } else {
@@ -1605,10 +1614,26 @@ pub(crate) fn fetch_epoch_rewards(
             .get(0)
             .ok_or_else(|| format!("Unable to fetch first confirmed block for epoch {}", epoch))?;
 
-        let first_confirmed_block = rpc_client.get_confirmed_block_with_encoding(
+        let first_confirmed_block = match rpc_client.get_confirmed_block_with_encoding(
             first_confirmed_block_in_epoch,
             solana_transaction_status::UiTransactionEncoding::Base64,
-        )?;
+        ) {
+            Ok(first_confirmed_block) => first_confirmed_block,
+            Err(ClientError {
+                kind:
+                    ClientErrorKind::RpcError(rpc_request::RpcError::RpcResponseError {
+                        code: rpc_custom_error::JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE,
+                        ..
+                    }),
+                ..
+            }) => {
+                // RPC node doesn't have this block
+                break;
+            }
+            Err(err) => {
+                return Err(err.into());
+            }
+        };
 
         let epoch_start_time = if let Some(block_time) = first_confirmed_block.block_time {
             block_time
@@ -1620,13 +1645,13 @@ pub(crate) fn fetch_epoch_rewards(
         let previous_epoch_rewards = first_confirmed_block.rewards;
 
         if let Some((effective_slot, epoch_end_time, epoch_rewards)) = epoch_info {
-            let wall_clock_epoch_duration =
+            let wallclock_epoch_duration =
                 { Local.timestamp(epoch_end_time, 0) - Local.timestamp(epoch_start_time, 0) }
                     .to_std()?
                     .as_secs_f64();
 
-            const SECONDS_PER_YEAR: f64 = (24 * 60 * 60 * 356) as f64;
+            let wallclock_epochs_per_year =
-            let percent_of_year = SECONDS_PER_YEAR / wall_clock_epoch_duration;
+                (SECONDS_PER_DAY * 356) as f64 / wallclock_epoch_duration;
 
             if let Some(reward) = epoch_rewards
                 .into_iter()
@@ -1642,7 +1667,7 @@ pub(crate) fn fetch_epoch_rewards(
                         amount: reward.lamports.abs() as u64,
                         post_balance: reward.post_balance,
                         percent_change: balance_increase_percent,
-                        apr: balance_increase_percent * percent_of_year,
+                        apr: balance_increase_percent * wallclock_epochs_per_year,
                     });
                 }
             }
@@ -1676,12 +1701,11 @@ pub fn process_show_stake_account(
     match stake_account.state() {
         Ok(stake_state) => {
             let stake_history_account = rpc_client.get_account(&stake_history::id())?;
-            let stake_history =
+            let stake_history = from_account(&stake_history_account).ok_or_else(|| {
-                StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
                 CliError::RpcRequestError("Failed to deserialize stake history".to_string())
             })?;
             let clock_account = rpc_client.get_account(&clock::id())?;
-            let clock: Clock = Sysvar::from_account(&clock_account).ok_or_else(|| {
+            let clock: Clock = from_account(&clock_account).ok_or_else(|| {
                 CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
             })?;
 
@@ -1691,6 +1715,7 @@ pub fn process_show_stake_account(
                 use_lamports_unit,
                 &stake_history,
                 &clock,
+                is_stake_program_v2_enabled(rpc_client), // At v1.6, this check can be removed and simply passed as `true`
             );
 
             if state.stake_type == CliStakeType::Stake {
@@ -1718,7 +1743,7 @@ pub fn process_show_stake_history(
     use_lamports_unit: bool,
 ) -> ProcessResult {
     let stake_history_account = rpc_client.get_account(&stake_history::id())?;
-    let stake_history = StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+    let stake_history = from_account::<StakeHistory>(&stake_history_account).ok_or_else(|| {
         CliError::RpcRequestError("Failed to deserialize stake history".to_string())
     })?;
 
@@ -1861,6 +1886,15 @@ pub fn process_delegate_stake(
     }
 }
 
+pub fn is_stake_program_v2_enabled(rpc_client: &RpcClient) -> bool {
+    rpc_client
+        .get_account(&feature_set::stake_program_v2::id())
+        .ok()
+        .and_then(|account| feature::from_account(&account))
+        .and_then(|feature| feature.activated_at)
+        .is_some()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -2406,9 +2440,9 @@ mod tests {
         );
 
         // Test CreateStakeAccount SubCommand
-        let custodian = Pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         let custodian_string = format!("{}", custodian);
-        let authorized = Pubkey::new_rand();
+        let authorized = solana_sdk::pubkey::new_rand();
         let authorized_string = format!("{}", authorized);
         let test_create_stake_account = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2546,7 +2580,7 @@ mod tests {
         );
 
         // Test DelegateStake Subcommand
-        let vote_account_pubkey = Pubkey::new_rand();
+        let vote_account_pubkey = solana_sdk::pubkey::new_rand();
         let vote_account_string = vote_account_pubkey.to_string();
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2573,7 +2607,7 @@ mod tests {
         );
 
         // Test DelegateStake Subcommand w/ authority
-        let vote_account_pubkey = Pubkey::new_rand();
+        let vote_account_pubkey = solana_sdk::pubkey::new_rand();
         let vote_account_string = vote_account_pubkey.to_string();
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2692,7 +2726,7 @@ mod tests {
         );
 
         // Test Delegate Subcommand w/ absent fee payer
-        let key1 = Pubkey::new_rand();
+        let key1 = solana_sdk::pubkey::new_rand();
         let sig1 = Keypair::new().sign_message(&[0u8]);
         let signer1 = format!("{}={}", key1, sig1);
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
@@ -2732,7 +2766,7 @@ mod tests {
         );
 
         // Test Delegate Subcommand w/ absent fee payer and absent nonce authority
-        let key2 = Pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
         let sig2 = Keypair::new().sign_message(&[0u8]);
         let signer2 = format!("{}={}", key2, sig2);
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
@@ -3060,7 +3094,7 @@ mod tests {
         );
 
         // Test Deactivate Subcommand w/ absent fee payer
-        let key1 = Pubkey::new_rand();
+        let key1 = solana_sdk::pubkey::new_rand();
         let sig1 = Keypair::new().sign_message(&[0u8]);
         let signer1 = format!("{}={}", key1, sig1);
         let test_deactivate_stake = test_commands.clone().get_matches_from(vec![
@@ -3097,7 +3131,7 @@ mod tests {
         );
 
         // Test Deactivate Subcommand w/ absent fee payer and nonce authority
-        let key2 = Pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
         let sig2 = Keypair::new().sign_message(&[0u8]);
         let signer2 = format!("{}={}", key2, sig2);
         let test_deactivate_stake = test_commands.clone().get_matches_from(vec![
@@ -3276,7 +3310,7 @@ mod tests {
         let stake_account_keypair = Keypair::new();
         write_keypair(&stake_account_keypair, tmp_file.as_file_mut()).unwrap();
 
-        let source_stake_account_pubkey = Pubkey::new_rand();
+        let source_stake_account_pubkey = solana_sdk::pubkey::new_rand();
         let test_merge_stake_account = test_commands.clone().get_matches_from(vec![
             "test",
             "merge-stake",

cli/src/validator_info.rs

@@ -486,7 +486,7 @@ mod tests {
 
     #[test]
     fn test_parse_validator_info() {
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let keys = vec![(validator_info::id(), false), (pubkey, true)];
         let config = ConfigKeys { keys };
 

cli/src/vote.rs

@@ -915,7 +915,7 @@ mod tests {
         );
 
         // test init with an authed voter
-        let authed = Pubkey::new_rand();
+        let authed = solana_sdk::pubkey::new_rand();
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let keypair = Keypair::new();
         write_keypair(&keypair, tmp_file.as_file_mut()).unwrap();

cli/tests/deploy.rs

@@ -55,7 +55,7 @@ fn test_cli_deploy_program() {
         faucet_host: None,
         faucet_port: faucet_addr.port(),
         pubkey: None,
-        lamports: 3 * minimum_balance_for_rent_exemption, // min balance for rent exemption for two programs + leftover for tx processing
+        lamports: 4 * minimum_balance_for_rent_exemption, // min balance for rent exemption for three programs + leftover for tx processing
     };
     config.signers = vec![&keypair];
     process_command(&config).unwrap();
@@ -64,6 +64,7 @@ fn test_cli_deploy_program() {
         program_location: pathbuf.to_str().unwrap().to_string(),
         address: None,
         use_deprecated_loader: false,
+        allow_excessive_balance: false,
     };
 
     let response = process_command(&config);
@@ -98,6 +99,7 @@ fn test_cli_deploy_program() {
         program_location: pathbuf.to_str().unwrap().to_string(),
         address: Some(1),
         use_deprecated_loader: false,
+        allow_excessive_balance: false,
     };
     process_command(&config).unwrap();
     let account1 = rpc_client
@@ -113,6 +115,44 @@ fn test_cli_deploy_program() {
     // Attempt to redeploy to the same address
     process_command(&config).unwrap_err();
 
+    // Attempt to deploy to account with excess balance
+    let custom_address_keypair = Keypair::new();
+    config.command = CliCommand::Airdrop {
+        faucet_host: None,
+        faucet_port: faucet_addr.port(),
+        pubkey: None,
+        lamports: 2 * minimum_balance_for_rent_exemption, // Anything over minimum_balance_for_rent_exemption should trigger err
+    };
+    config.signers = vec![&custom_address_keypair];
+    process_command(&config).unwrap();
+
+    config.signers = vec![&keypair, &custom_address_keypair];
+    config.command = CliCommand::Deploy {
+        program_location: pathbuf.to_str().unwrap().to_string(),
+        address: Some(1),
+        use_deprecated_loader: false,
+        allow_excessive_balance: false,
+    };
+    process_command(&config).unwrap_err();
+
+    // Use forcing parameter to deploy to account with excess balance
+    config.command = CliCommand::Deploy {
+        program_location: pathbuf.to_str().unwrap().to_string(),
+        address: Some(1),
+        use_deprecated_loader: false,
+        allow_excessive_balance: true,
+    };
+    process_command(&config).unwrap();
+    let account2 = rpc_client
+        .get_account_with_commitment(&custom_address_keypair.pubkey(), CommitmentConfig::recent())
+        .unwrap()
+        .value
+        .unwrap();
+    assert_eq!(account2.lamports, 2 * minimum_balance_for_rent_exemption);
+    assert_eq!(account2.owner, bpf_loader::id());
+    assert_eq!(account2.executable, true);
+    assert_eq!(account0.data, account2.data);
+
     server.close().unwrap();
     remove_dir_all(ledger_path).unwrap();
 }

cli/tests/nonce.rs

@@ -10,7 +10,7 @@ use solana_client::{
     rpc_client::RpcClient,
 };
 use solana_core::contact_info::ContactInfo;
-use solana_core::test_validator::{TestValidator, TestValidatorOptions};
+use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     commitment_config::CommitmentConfig,
@@ -172,7 +172,7 @@ fn full_battery_tests(
     assert_ne!(first_nonce, third_nonce);
 
     // Withdraw from nonce account
-    let payee_pubkey = Pubkey::new_rand();
+    let payee_pubkey = solana_sdk::pubkey::new_rand();
     config_payer.signers = authorized_signers;
     config_payer.command = CliCommand::WithdrawFromNonceAccount {
         nonce_account,
@@ -231,17 +231,14 @@ fn full_battery_tests(
 
 #[test]
 fn test_create_account_with_seed() {
+    solana_logger::setup();
     let TestValidator {
         server,
         leader_data,
         alice: mint_keypair,
         ledger_path,
         ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
+    } = TestValidator::run_with_fees(1);
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);

cli/tests/stake.rs

@@ -9,7 +9,7 @@ use solana_client::{
     nonce_utils,
     rpc_client::RpcClient,
 };
-use solana_core::test_validator::{TestValidator, TestValidatorOptions};
+use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     account_utils::StateMut,
@@ -848,11 +848,7 @@ fn test_stake_authorize_with_fee_payer() {
         alice,
         ledger_path,
         ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
+    } = TestValidator::run_with_fees(SIG_FEE);
-        fees: SIG_FEE,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
     let (sender, receiver) = channel();
     run_local_faucet(alice, sender, None);
     let faucet_addr = receiver.recv().unwrap();
@@ -985,11 +981,7 @@ fn test_stake_split() {
         alice,
         ledger_path,
         ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
+    } = TestValidator::run_with_fees(1);
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
     let (sender, receiver) = channel();
     run_local_faucet(alice, sender, None);
     let faucet_addr = receiver.recv().unwrap();
@@ -1140,11 +1132,7 @@ fn test_stake_set_lockup() {
         alice,
         ledger_path,
         ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
+    } = TestValidator::run_with_fees(1);
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
     let (sender, receiver) = channel();
     run_local_faucet(alice, sender, None);
     let faucet_addr = receiver.recv().unwrap();

cli/tests/transfer.rs

@@ -9,7 +9,7 @@ use solana_client::{
     nonce_utils,
     rpc_client::RpcClient,
 };
-use solana_core::test_validator::{TestValidator, TestValidatorOptions};
+use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     commitment_config::CommitmentConfig,
@@ -21,17 +21,14 @@ use std::{fs::remove_dir_all, sync::mpsc::channel};
 
 #[test]
 fn test_transfer() {
+    solana_logger::setup();
     let TestValidator {
         server,
         leader_data,
         alice: mint_keypair,
         ledger_path,
         ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
+    } = TestValidator::run_with_fees(1);
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);
@@ -252,17 +249,14 @@ fn test_transfer() {
 
 #[test]
 fn test_transfer_multisession_signing() {
+    solana_logger::setup();
     let TestValidator {
         server,
         leader_data,
         alice: mint_keypair,
         ledger_path,
         ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
+    } = TestValidator::run_with_fees(1);
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);
@@ -382,17 +376,14 @@ fn test_transfer_multisession_signing() {
 
 #[test]
 fn test_transfer_all() {
+    solana_logger::setup();
     let TestValidator {
         server,
         leader_data,
         alice: mint_keypair,
         ledger_path,
         ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
+    } = TestValidator::run_with_fees(1);
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);

cli/tests/vote.rs

@@ -12,7 +12,6 @@ use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     account_utils::StateMut,
     commitment_config::CommitmentConfig,
-    pubkey::Pubkey,
     signature::{Keypair, Signer},
 };
 use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersions};
@@ -110,7 +109,7 @@ fn test_vote_authorize_and_withdraw() {
     assert_eq!(authorized_withdrawer, withdraw_authority.pubkey());
 
     // Withdraw from vote account
-    let destination_account = Pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
+    let destination_account = solana_sdk::pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
     config.signers = vec![&default_signer, &withdraw_authority];
     config.command = CliCommand::WithdrawFromVoteAccount {
         vote_account_pubkey,

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.4.0"
+version = "1.4.12"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -17,16 +17,18 @@ indicatif = "0.15.0"
 jsonrpc-core = "15.0.0"
 log = "0.4.8"
 rayon = "1.4.0"
-reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+semver = "0.11.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.12" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
+solana-net-utils = { path = "../net-utils", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.12" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.12" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -35,7 +37,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "15.0.0"
 jsonrpc-http-server = "15.0.0"
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/client_error.rs

@@ -50,10 +50,10 @@ impl Into<TransportError> for ClientErrorKind {
 #[derive(Error, Debug)]
 #[error("{kind}")]
 pub struct ClientError {
-    request: Option<rpc_request::RpcRequest>,
+    pub request: Option<rpc_request::RpcRequest>,
 
     #[source]
-    kind: ClientErrorKind,
+    pub kind: ClientErrorKind,
 }
 
 impl ClientError {

client/src/http_sender.rs

@@ -1,6 +1,8 @@
 use crate::{
     client_error::Result,
-    rpc_request::{RpcError, RpcRequest},
+    rpc_custom_error,
+    rpc_request::{RpcError, RpcRequest, RpcResponseErrorData},
+    rpc_response::RpcSimulateTransactionResult,
     rpc_sender::RpcSender,
 };
 use log::*;
@@ -27,6 +29,13 @@ impl HttpSender {
     }
 }
 
+#[derive(Deserialize, Debug)]
+struct RpcErrorObject {
+    code: i64,
+    message: String,
+    data: serde_json::Value,
+}
+
 impl RpcSender for HttpSender {
     fn send(&self, request: RpcRequest, params: serde_json::Value) -> Result<serde_json::Value> {
         // Concurrent requests are not supported so reuse the same request id for all requests
@@ -63,11 +72,36 @@ impl RpcSender for HttpSender {
 
                     let json: serde_json::Value = serde_json::from_str(&response.text()?)?;
                     if json["error"].is_object() {
-                        return Err(RpcError::RpcRequestError(format!(
+                        return match serde_json::from_value::<RpcErrorObject>(json["error"].clone())
-                            "RPC Error response: {}",
+                        {
-                            serde_json::to_string(&json["error"]).unwrap()
+                            Ok(rpc_error_object) => {
+                                let data = match rpc_error_object.code {
+                                    rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE => {
+                                        match serde_json::from_value::<RpcSimulateTransactionResult>(json["error"]["data"].clone()) {
+                                            Ok(data) => RpcResponseErrorData::SendTransactionPreflightFailure(data),
+                                            Err(err) => {
+                                                debug!("Failed to deserialize RpcSimulateTransactionResult: {:?}", err);
+                                                RpcResponseErrorData::Empty
+                                            }
+                                        }
+                                    },
+                                    _ => RpcResponseErrorData::Empty
+                                };
+
+                                Err(RpcError::RpcResponseError {
+                                    code: rpc_error_object.code,
+                                    message: rpc_error_object.message,
+                                    data,
+                                }
+                                .into())
+                            }
+                            Err(err) => Err(RpcError::RpcRequestError(format!(
+                                "Failed to deserialize RPC error response: {} [{}]",
+                                serde_json::to_string(&json["error"]).unwrap(),
+                                err
                             ))
-                        .into());
+                            .into()),
+                        };
                     }
                     return Ok(json["result"].clone());
                 }

client/src/lib.rs

@@ -10,6 +10,7 @@ pub mod perf_utils;
 pub mod pubsub_client;
 pub mod rpc_client;
 pub mod rpc_config;
+pub mod rpc_custom_error;
 pub mod rpc_filter;
 pub mod rpc_request;
 pub mod rpc_response;

client/src/mock_sender.rs

@@ -1,17 +1,19 @@
 use crate::{
     client_error::Result,
     rpc_request::RpcRequest,
-    rpc_response::{Response, RpcResponseContext},
+    rpc_response::{Response, RpcResponseContext, RpcVersionInfo},
     rpc_sender::RpcSender,
 };
-use serde_json::{Number, Value};
+use serde_json::{json, Number, Value};
 use solana_sdk::{
+    epoch_info::EpochInfo,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
     instruction::InstructionError,
     signature::Signature,
     transaction::{self, Transaction, TransactionError},
 };
 use solana_transaction_status::TransactionStatus;
+use solana_version::Version;
 use std::{collections::HashMap, sync::RwLock};
 
 pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
@@ -57,6 +59,13 @@ impl RpcSender for MockSender {
                     serde_json::to_value(FeeCalculator::default()).unwrap(),
                 ),
             })?,
+            RpcRequest::GetEpochInfo => serde_json::to_value(EpochInfo {
+                epoch: 1,
+                slot_index: 2,
+                slots_in_epoch: 32,
+                absolute_slot: 34,
+                block_height: 34,
+            })?,
             RpcRequest::GetFeeCalculatorForBlockhash => {
                 let value = if self.url == "blockhash_expired" {
                     Value::Null
@@ -112,13 +121,20 @@ impl RpcSender for MockSender {
                     Signature::new(&[8; 64]).to_string()
                 } else {
                     let tx_str = params.as_array().unwrap()[0].as_str().unwrap().to_string();
-                    let data = bs58::decode(tx_str).into_vec().unwrap();
+                    let data = base64::decode(tx_str).unwrap();
                     let tx: Transaction = bincode::deserialize(&data).unwrap();
                     tx.signatures[0].to_string()
                 };
                 Value::String(signature)
             }
             RpcRequest::GetMinimumBalanceForRentExemption => Value::Number(Number::from(20)),
+            RpcRequest::GetVersion => {
+                let version = Version::default();
+                json!(RpcVersionInfo {
+                    solana_core: version.to_string(),
+                    feature_set: Some(version.feature_set),
+                })
+            }
             _ => Value::Null,
         };
         Ok(val)

client/src/pubsub_client.rs

@@ -1,4 +1,7 @@
-use crate::rpc_response::{Response as RpcResponse, RpcSignatureResult, SlotInfo};
+use crate::{
+    rpc_config::{RpcSignatureSubscribeConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter},
+    rpc_response::{Response as RpcResponse, RpcLogsResponse, RpcSignatureResult, SlotInfo},
+};
 use log::*;
 use serde::de::DeserializeOwned;
 use serde_json::{
@@ -20,8 +23,6 @@ use thiserror::Error;
 use tungstenite::{client::AutoStream, connect, Message, WebSocket};
 use url::{ParseError, Url};
 
-type PubsubSignatureResponse = PubsubClientSubscription<RpcResponse<RpcSignatureResult>>;
-
 #[derive(Debug, Error)]
 pub enum PubsubClientError {
     #[error("url parse error")]
@@ -33,8 +34,8 @@ pub enum PubsubClientError {
     #[error("json parse error")]
     JsonParseError(#[from] serde_json::error::Error),
 
-    #[error("unexpected message format")]
+    #[error("unexpected message format: {0}")]
-    UnexpectedMessageError,
+    UnexpectedMessageError(String),
 }
 
 pub struct PubsubClientSubscription<T>
@@ -89,8 +90,11 @@ where
                 return Ok(x);
             }
         }
-
+        // TODO: Add proper JSON RPC response/error handling...
-        Err(PubsubClientError::UnexpectedMessageError)
+        Err(PubsubClientError::UnexpectedMessageError(format!(
+            "{:?}",
+            json_msg
+        )))
     }
 
     pub fn send_unsubscribe(&self) -> Result<(), PubsubClientError> {
@@ -114,14 +118,18 @@ where
         let message_text = &message.into_text().unwrap();
         let json_msg: Map<String, Value> = serde_json::from_str(message_text)?;
 
-        if let Some(Object(value_1)) = json_msg.get("params") {
+        if let Some(Object(params)) = json_msg.get("params") {
-            if let Some(value_2) = value_1.get("result") {
+            if let Some(result) = params.get("result") {
-                let x: T = serde_json::from_value::<T>(value_2.clone()).unwrap();
+                let x: T = serde_json::from_value::<T>(result.clone()).unwrap();
                 return Ok(x);
             }
         }
 
-        Err(PubsubClientError::UnexpectedMessageError)
+        // TODO: Add proper JSON RPC response/error handling...
+        Err(PubsubClientError::UnexpectedMessageError(format!(
+            "{:?}",
+            json_msg
+        )))
     }
 
     pub fn shutdown(&mut self) -> std::thread::Result<()> {
@@ -138,15 +146,79 @@ where
     }
 }
 
-const SLOT_OPERATION: &str = "slot";
+pub type LogsSubscription = (
-const SIGNATURE_OPERATION: &str = "signature";
+    PubsubClientSubscription<RpcResponse<RpcLogsResponse>>,
+    Receiver<RpcResponse<RpcLogsResponse>>,
+);
+pub type SlotsSubscription = (PubsubClientSubscription<SlotInfo>, Receiver<SlotInfo>);
+pub type SignatureSubscription = (
+    PubsubClientSubscription<RpcResponse<RpcSignatureResult>>,
+    Receiver<RpcResponse<RpcSignatureResult>>,
+);
 
 pub struct PubsubClient {}
 
 impl PubsubClient {
-    pub fn slot_subscribe(
+    pub fn logs_subscribe(
         url: &str,
-    ) -> Result<(PubsubClientSubscription<SlotInfo>, Receiver<SlotInfo>), PubsubClientError> {
+        filter: RpcTransactionLogsFilter,
+        config: RpcTransactionLogsConfig,
+    ) -> Result<LogsSubscription, PubsubClientError> {
+        let url = Url::parse(url)?;
+        let (socket, _response) = connect(url)?;
+        let (sender, receiver) = channel();
+
+        let socket = Arc::new(RwLock::new(socket));
+        let socket_clone = socket.clone();
+        let exit = Arc::new(AtomicBool::new(false));
+        let exit_clone = exit.clone();
+
+        let subscription_id =
+            PubsubClientSubscription::<RpcResponse<RpcLogsResponse>>::send_subscribe(
+                &socket_clone,
+                json!({
+                    "jsonrpc":"2.0","id":1,"method":"logsSubscribe","params":[filter, config]
+                })
+                .to_string(),
+            )?;
+
+        let t_cleanup = std::thread::spawn(move || {
+            loop {
+                if exit_clone.load(Ordering::Relaxed) {
+                    break;
+                }
+
+                match PubsubClientSubscription::read_message(&socket_clone) {
+                    Ok(message) => match sender.send(message) {
+                        Ok(_) => (),
+                        Err(err) => {
+                            info!("receive error: {:?}", err);
+                            break;
+                        }
+                    },
+                    Err(err) => {
+                        info!("receive error: {:?}", err);
+                        break;
+                    }
+                }
+            }
+
+            info!("websocket - exited receive loop");
+        });
+
+        let result = PubsubClientSubscription {
+            message_type: PhantomData,
+            operation: "logs",
+            socket,
+            subscription_id,
+            t_cleanup: Some(t_cleanup),
+            exit,
+        };
+
+        Ok((result, receiver))
+    }
+
+    pub fn slot_subscribe(url: &str) -> Result<SlotsSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let (socket, _response) = connect(url)?;
         let (sender, receiver) = channel::<SlotInfo>();
@@ -158,41 +230,37 @@ impl PubsubClient {
         let subscription_id = PubsubClientSubscription::<SlotInfo>::send_subscribe(
             &socket_clone,
             json!({
-                "jsonrpc":"2.0","id":1,"method":format!("{}Subscribe", SLOT_OPERATION),"params":[]
+                "jsonrpc":"2.0","id":1,"method":"slotSubscribe","params":[]
             })
             .to_string(),
-        )
+        )?;
-        .unwrap();
 
         let t_cleanup = std::thread::spawn(move || {
             loop {
                 if exit_clone.load(Ordering::Relaxed) {
                     break;
                 }
-
+                match PubsubClientSubscription::read_message(&socket_clone) {
-                let message: Result<SlotInfo, PubsubClientError> =
+                    Ok(message) => match sender.send(message) {
-                    PubsubClientSubscription::read_message(&socket_clone);
-
-                if let Ok(msg) = message {
-                    match sender.send(msg) {
                         Ok(_) => (),
                         Err(err) => {
                             info!("receive error: {:?}", err);
                             break;
                         }
-                    }
+                    },
-                } else {
+                    Err(err) => {
-                    info!("receive error: {:?}", message);
+                        info!("receive error: {:?}", err);
                         break;
                     }
                 }
+            }
 
             info!("websocket - exited receive loop");
         });
 
-        let result: PubsubClientSubscription<SlotInfo> = PubsubClientSubscription {
+        let result = PubsubClientSubscription {
             message_type: PhantomData,
-            operation: SLOT_OPERATION,
+            operation: "slot",
             socket,
             subscription_id,
             t_cleanup: Some(t_cleanup),
@@ -205,16 +273,11 @@ impl PubsubClient {
     pub fn signature_subscribe(
         url: &str,
         signature: &Signature,
-    ) -> Result<
+        config: Option<RpcSignatureSubscribeConfig>,
-        (
+    ) -> Result<SignatureSubscription, PubsubClientError> {
-            PubsubSignatureResponse,
-            Receiver<RpcResponse<RpcSignatureResult>>,
-        ),
-        PubsubClientError,
-    > {
         let url = Url::parse(url)?;
         let (socket, _response) = connect(url)?;
-        let (sender, receiver) = channel::<RpcResponse<RpcSignatureResult>>();
+        let (sender, receiver) = channel();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();
@@ -223,10 +286,10 @@ impl PubsubClient {
         let body = json!({
             "jsonrpc":"2.0",
             "id":1,
-            "method":format!("{}Subscribe", SIGNATURE_OPERATION),
+            "method":"signatureSubscribe",
             "params":[
                 signature.to_string(),
-                {"enableReceivedNotification": true }
+                config
             ]
         })
         .to_string();
@@ -234,8 +297,7 @@ impl PubsubClient {
             PubsubClientSubscription::<RpcResponse<RpcSignatureResult>>::send_subscribe(
                 &socket_clone,
                 body,
-            )
+            )?;
-            .unwrap();
 
         let t_cleanup = std::thread::spawn(move || {
             loop {
@@ -263,10 +325,9 @@ impl PubsubClient {
             info!("websocket - exited receive loop");
         });
 
-        let result: PubsubClientSubscription<RpcResponse<RpcSignatureResult>> =
+        let result = PubsubClientSubscription {
-            PubsubClientSubscription {
             message_type: PhantomData,
-                operation: SIGNATURE_OPERATION,
+            operation: "signature",
             socket,
             subscription_id,
             t_cleanup: Some(t_cleanup),

client/src/rpc_client.rs

@@ -8,7 +8,7 @@ use crate::{
         RpcProgramAccountsConfig, RpcSendTransactionConfig, RpcSimulateTransactionConfig,
         RpcTokenAccountsFilter,
     },
-    rpc_request::{RpcError, RpcRequest, TokenAccountsFilter},
+    rpc_request::{RpcError, RpcRequest, RpcResponseErrorData, TokenAccountsFilter},
     rpc_response::*,
     rpc_sender::RpcSender,
 };
@@ -41,12 +41,15 @@ use solana_transaction_status::{
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     net::SocketAddr,
+    sync::RwLock,
     thread::sleep,
     time::{Duration, Instant},
 };
 
 pub struct RpcClient {
     sender: Box<dyn RpcSender + Send + Sync + 'static>,
+    commitment_config: CommitmentConfig,
+    default_cluster_transaction_encoding: RwLock<Option<UiTransactionEncoding>>,
 }
 
 fn serialize_encode_transaction(
@@ -70,26 +73,41 @@ fn serialize_encode_transaction(
 }
 
 impl RpcClient {
-    pub fn new_sender<T: RpcSender + Send + Sync + 'static>(sender: T) -> Self {
+    fn new_sender<T: RpcSender + Send + Sync + 'static>(
+        sender: T,
+        commitment_config: CommitmentConfig,
+    ) -> Self {
         Self {
             sender: Box::new(sender),
+            default_cluster_transaction_encoding: RwLock::new(None),
+            commitment_config,
         }
     }
 
     pub fn new(url: String) -> Self {
-        Self::new_sender(HttpSender::new(url))
+        Self::new_with_commitment(url, CommitmentConfig::default())
+    }
+
+    pub fn new_with_commitment(url: String, commitment_config: CommitmentConfig) -> Self {
+        Self::new_sender(HttpSender::new(url), commitment_config)
     }
 
     pub fn new_with_timeout(url: String, timeout: Duration) -> Self {
-        Self::new_sender(HttpSender::new_with_timeout(url, timeout))
+        Self::new_sender(
+            HttpSender::new_with_timeout(url, timeout),
+            CommitmentConfig::default(),
+        )
     }
 
     pub fn new_mock(url: String) -> Self {
-        Self::new_sender(MockSender::new(url))
+        Self::new_sender(MockSender::new(url), CommitmentConfig::default())
     }
 
     pub fn new_mock_with_mocks(url: String, mocks: Mocks) -> Self {
-        Self::new_sender(MockSender::new_with_mocks(url, mocks))
+        Self::new_sender(
+            MockSender::new_with_mocks(url, mocks),
+            CommitmentConfig::default(),
+        )
     }
 
     pub fn new_socket(addr: SocketAddr) -> Self {
@@ -103,10 +121,14 @@ impl RpcClient {
 
     pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
         Ok(self
-            .confirm_transaction_with_commitment(signature, CommitmentConfig::default())?
+            .confirm_transaction_with_commitment(signature, self.commitment_config)?
             .value)
     }
 
+    pub fn commitment(&self) -> CommitmentConfig {
+        self.commitment_config
+    }
+
     pub fn confirm_transaction_with_commitment(
         &self,
         signature: &Signature,
@@ -125,7 +147,38 @@ impl RpcClient {
     }
 
     pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<Signature> {
-        self.send_transaction_with_config(transaction, RpcSendTransactionConfig::default())
+        self.send_transaction_with_config(
+            transaction,
+            RpcSendTransactionConfig {
+                preflight_commitment: Some(self.commitment_config.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
+        )
+    }
+
+    fn default_cluster_transaction_encoding(&self) -> Result<UiTransactionEncoding, RpcError> {
+        let default_cluster_transaction_encoding =
+            self.default_cluster_transaction_encoding.read().unwrap();
+        if let Some(encoding) = *default_cluster_transaction_encoding {
+            Ok(encoding)
+        } else {
+            drop(default_cluster_transaction_encoding);
+            let cluster_version = self.get_version().map_err(|e| {
+                RpcError::RpcRequestError(format!("cluster version query failed: {}", e))
+            })?;
+            let cluster_version =
+                semver::Version::parse(&cluster_version.solana_core).map_err(|e| {
+                    RpcError::RpcRequestError(format!("failed to parse cluster version: {}", e))
+                })?;
+            // Prefer base64 since 1.3.16
+            let encoding = if cluster_version < semver::Version::new(1, 3, 16) {
+                UiTransactionEncoding::Base58
+            } else {
+                UiTransactionEncoding::Base64
+            };
+            *self.default_cluster_transaction_encoding.write().unwrap() = Some(encoding);
+            Ok(encoding)
+        }
     }
 
     pub fn send_transaction_with_config(
@@ -133,12 +186,43 @@ impl RpcClient {
         transaction: &Transaction,
         config: RpcSendTransactionConfig,
     ) -> ClientResult<Signature> {
-        let encoding = config.encoding.unwrap_or(UiTransactionEncoding::Base58);
+        let encoding = if let Some(encoding) = config.encoding {
+            encoding
+        } else {
+            self.default_cluster_transaction_encoding()?
+        };
+        let config = RpcSendTransactionConfig {
+            encoding: Some(encoding),
+            ..config
+        };
         let serialized_encoded = serialize_encode_transaction(transaction, encoding)?;
-        let signature_base58_str: String = self.send(
+        let signature_base58_str: String = match self.send(
             RpcRequest::SendTransaction,
             json!([serialized_encoded, config]),
-        )?;
+        ) {
+            Ok(signature_base58_str) => signature_base58_str,
+            Err(err) => {
+                if let ClientErrorKind::RpcError(RpcError::RpcResponseError {
+                    code,
+                    message,
+                    data,
+                }) = &err.kind
+                {
+                    debug!("{} {}", code, message);
+                    if let RpcResponseErrorData::SendTransactionPreflightFailure(
+                        RpcSimulateTransactionResult {
+                            logs: Some(logs), ..
+                        },
+                    ) = data
+                    {
+                        for (i, log) in logs.iter().enumerate() {
+                            debug!("{:>3}: {}", i + 1, log);
+                        }
+                    }
+                }
+                return Err(err);
+            }
+        };
 
         let signature = signature_base58_str
             .parse::<Signature>()
@@ -161,26 +245,34 @@ impl RpcClient {
     pub fn simulate_transaction(
         &self,
         transaction: &Transaction,
-        sig_verify: bool,
     ) -> RpcResult<RpcSimulateTransactionResult> {
         self.simulate_transaction_with_config(
             transaction,
-            sig_verify,
+            RpcSimulateTransactionConfig {
-            RpcSimulateTransactionConfig::default(),
+                commitment: Some(self.commitment_config),
+                ..RpcSimulateTransactionConfig::default()
+            },
         )
     }
 
     pub fn simulate_transaction_with_config(
         &self,
         transaction: &Transaction,
-        sig_verify: bool,
         config: RpcSimulateTransactionConfig,
     ) -> RpcResult<RpcSimulateTransactionResult> {
-        let encoding = config.encoding.unwrap_or(UiTransactionEncoding::Base58);
+        let encoding = if let Some(encoding) = config.encoding {
+            encoding
+        } else {
+            self.default_cluster_transaction_encoding()?
+        };
+        let config = RpcSimulateTransactionConfig {
+            encoding: Some(encoding),
+            ..config
+        };
         let serialized_encoded = serialize_encode_transaction(transaction, encoding)?;
         self.send(
             RpcRequest::SimulateTransaction,
-            json!([serialized_encoded, { "sigVerify": sig_verify }]),
+            json!([serialized_encoded, config]),
         )
     }
 
@@ -188,7 +280,7 @@ impl RpcClient {
         &self,
         signature: &Signature,
     ) -> ClientResult<Option<transaction::Result<()>>> {
-        self.get_signature_status_with_commitment(signature, CommitmentConfig::default())
+        self.get_signature_status_with_commitment(signature, self.commitment_config)
     }
 
     pub fn get_signature_statuses(
@@ -246,7 +338,7 @@ impl RpcClient {
     }
 
     pub fn get_slot(&self) -> ClientResult<Slot> {
-        self.get_slot_with_commitment(CommitmentConfig::default())
+        self.get_slot_with_commitment(self.commitment_config)
     }
 
     pub fn get_slot_with_commitment(
@@ -264,7 +356,7 @@ impl RpcClient {
     }
 
     pub fn total_supply(&self) -> ClientResult<u64> {
-        self.total_supply_with_commitment(CommitmentConfig::default())
+        self.total_supply_with_commitment(self.commitment_config)
     }
 
     pub fn total_supply_with_commitment(
@@ -282,7 +374,7 @@ impl RpcClient {
     }
 
     pub fn get_vote_accounts(&self) -> ClientResult<RpcVoteAccountStatus> {
-        self.get_vote_accounts_with_commitment(CommitmentConfig::default())
+        self.get_vote_accounts_with_commitment(self.commitment_config)
     }
 
     pub fn get_vote_accounts_with_commitment(
@@ -410,7 +502,7 @@ impl RpcClient {
     }
 
     pub fn get_epoch_info(&self) -> ClientResult<EpochInfo> {
-        self.get_epoch_info_with_commitment(CommitmentConfig::default())
+        self.get_epoch_info_with_commitment(self.commitment_config)
     }
 
     pub fn get_epoch_info_with_commitment(
@@ -424,7 +516,7 @@ impl RpcClient {
         &self,
         slot: Option<Slot>,
     ) -> ClientResult<Option<RpcLeaderSchedule>> {
-        self.get_leader_schedule_with_commitment(slot, CommitmentConfig::default())
+        self.get_leader_schedule_with_commitment(slot, self.commitment_config)
     }
 
     pub fn get_leader_schedule_with_commitment(
@@ -518,8 +610,10 @@ impl RpcClient {
         }
     }
 
+    /// Note that `get_account` returns `Err(..)` if the account does not exist whereas
+    /// `get_account_with_commitment` returns `Ok(None)` if the account does not exist.
     pub fn get_account(&self, pubkey: &Pubkey) -> ClientResult<Account> {
-        self.get_account_with_commitment(pubkey, CommitmentConfig::default())?
+        self.get_account_with_commitment(pubkey, self.commitment_config)?
             .value
             .ok_or_else(|| RpcError::ForUser(format!("AccountNotFound: pubkey={}", pubkey)).into())
     }
@@ -567,7 +661,7 @@ impl RpcClient {
 
     pub fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> ClientResult<Vec<Option<Account>>> {
         Ok(self
-            .get_multiple_accounts_with_commitment(pubkeys, CommitmentConfig::default())?
+            .get_multiple_accounts_with_commitment(pubkeys, self.commitment_config)?
             .value)
     }
 
@@ -621,7 +715,7 @@ impl RpcClient {
     /// Request the balance of the account `pubkey`.
     pub fn get_balance(&self, pubkey: &Pubkey) -> ClientResult<u64> {
         Ok(self
-            .get_balance_with_commitment(pubkey, CommitmentConfig::default())?
+            .get_balance_with_commitment(pubkey, self.commitment_config)?
             .value)
     }
 
@@ -663,7 +757,7 @@ impl RpcClient {
 
     /// Request the transaction count.
     pub fn get_transaction_count(&self) -> ClientResult<u64> {
-        self.get_transaction_count_with_commitment(CommitmentConfig::default())
+        self.get_transaction_count_with_commitment(self.commitment_config)
     }
 
     pub fn get_transaction_count_with_commitment(
@@ -675,7 +769,7 @@ impl RpcClient {
 
     pub fn get_recent_blockhash(&self) -> ClientResult<(Hash, FeeCalculator)> {
         let (blockhash, fee_calculator, _last_valid_slot) = self
-            .get_recent_blockhash_with_commitment(CommitmentConfig::default())?
+            .get_recent_blockhash_with_commitment(self.commitment_config)?
             .value;
         Ok((blockhash, fee_calculator))
     }
@@ -732,10 +826,7 @@ impl RpcClient {
         blockhash: &Hash,
     ) -> ClientResult<Option<FeeCalculator>> {
         Ok(self
-            .get_fee_calculator_for_blockhash_with_commitment(
+            .get_fee_calculator_for_blockhash_with_commitment(blockhash, self.commitment_config)?
-                blockhash,
-                CommitmentConfig::default(),
-            )?
             .value)
     }
 
@@ -811,7 +902,7 @@ impl RpcClient {
 
     pub fn get_token_account(&self, pubkey: &Pubkey) -> ClientResult<Option<UiTokenAccount>> {
         Ok(self
-            .get_token_account_with_commitment(pubkey, CommitmentConfig::default())?
+            .get_token_account_with_commitment(pubkey, self.commitment_config)?
             .value)
     }
 
@@ -872,7 +963,7 @@ impl RpcClient {
 
     pub fn get_token_account_balance(&self, pubkey: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
-            .get_token_account_balance_with_commitment(pubkey, CommitmentConfig::default())?
+            .get_token_account_balance_with_commitment(pubkey, self.commitment_config)?
             .value)
     }
 
@@ -896,7 +987,7 @@ impl RpcClient {
             .get_token_accounts_by_delegate_with_commitment(
                 delegate,
                 token_account_filter,
-                CommitmentConfig::default(),
+                self.commitment_config,
             )?
             .value)
     }
@@ -935,7 +1026,7 @@ impl RpcClient {
             .get_token_accounts_by_owner_with_commitment(
                 owner,
                 token_account_filter,
-                CommitmentConfig::default(),
+                self.commitment_config,
             )?
             .value)
     }
@@ -967,7 +1058,7 @@ impl RpcClient {
 
     pub fn get_token_supply(&self, mint: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
-            .get_token_supply_with_commitment(mint, CommitmentConfig::default())?
+            .get_token_supply_with_commitment(mint, self.commitment_config)?
             .value)
     }
 
@@ -1048,7 +1139,7 @@ impl RpcClient {
 
     /// Poll the server to confirm a transaction.
     pub fn poll_for_signature(&self, signature: &Signature) -> ClientResult<()> {
-        self.poll_for_signature_with_commitment(signature, CommitmentConfig::default())
+        self.poll_for_signature_with_commitment(signature, self.commitment_config)
     }
 
     /// Poll the server to confirm a transaction.
@@ -1156,10 +1247,9 @@ impl RpcClient {
         &self,
         transaction: &Transaction,
     ) -> ClientResult<Signature> {
-        self.send_and_confirm_transaction_with_spinner_and_config(
+        self.send_and_confirm_transaction_with_spinner_and_commitment(
             transaction,
-            CommitmentConfig::default(),
+            self.commitment_config,
-            RpcSendTransactionConfig::default(),
         )
     }
 
@@ -1171,7 +1261,10 @@ impl RpcClient {
         self.send_and_confirm_transaction_with_spinner_and_config(
             transaction,
             commitment,
-            RpcSendTransactionConfig::default(),
+            RpcSendTransactionConfig {
+                preflight_commitment: Some(commitment.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
         )
     }
 
@@ -1418,7 +1511,7 @@ mod tests {
         let rpc_client = RpcClient::new_mock("succeeds".to_string());
 
         let key = Keypair::new();
-        let to = Pubkey::new_rand();
+        let to = solana_sdk::pubkey::new_rand();
         let blockhash = Hash::default();
         let tx = system_transaction::transfer(&key, &to, 50, blockhash);
 
@@ -1471,7 +1564,7 @@ mod tests {
         let rpc_client = RpcClient::new_mock("succeeds".to_string());
 
         let key = Keypair::new();
-        let to = Pubkey::new_rand();
+        let to = solana_sdk::pubkey::new_rand();
         let blockhash = Hash::default();
         let tx = system_transaction::transfer(&key, &to, 50, blockhash);
         let result = rpc_client.send_and_confirm_transaction(&tx);

client/src/rpc_config.rs

@@ -71,6 +71,20 @@ pub struct RpcProgramAccountsConfig {
     pub account_config: RpcAccountInfoConfig,
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum RpcTransactionLogsFilter {
+    All,
+    AllWithVotes,
+    Mentions(Vec<String>), // base58-encoded list of addresses
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcTransactionLogsConfig {
+    pub commitment: Option<CommitmentConfig>,
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum RpcTokenAccountsFilter {

client/src/rpc_custom_error.rs

@@ -1,13 +1,15 @@
+//! Implementation defined RPC server errors
+
+use crate::rpc_response::RpcSimulateTransactionResult;
 use jsonrpc_core::{Error, ErrorCode};
-use solana_client::rpc_response::RpcSimulateTransactionResult;
 use solana_sdk::clock::Slot;
 
-const JSON_RPC_SERVER_ERROR_1: i64 = -32001;
+pub const JSON_RPC_SERVER_ERROR_BLOCK_CLEANED_UP: i64 = -32001;
-const JSON_RPC_SERVER_ERROR_2: i64 = -32002;
+pub const JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE: i64 = -32002;
-const JSON_RPC_SERVER_ERROR_3: i64 = -32003;
+pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE: i64 = -32003;
-const JSON_RPC_SERVER_ERROR_4: i64 = -32004;
+pub const JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE: i64 = -32004;
-const JSON_RPC_SERVER_ERROR_5: i64 = -32005;
+pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY: i64 = -32005;
-const JSON_RPC_SERVER_ERROR_6: i64 = -32006;
+pub const JSON_RPC_SERVER_ERROR_TRANSACTION_PRECOMPILE_VERIFICATION_FAILURE: i64 = -32006;
 
 pub enum RpcCustomError {
     BlockCleanedUp {
@@ -33,7 +35,7 @@ impl From<RpcCustomError> for Error {
                 slot,
                 first_available_block,
             } => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_1),
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_BLOCK_CLEANED_UP),
                 message: format!(
                     "Block {} cleaned up, does not exist on node. First available block: {}",
                     slot, first_available_block,
@@ -41,27 +43,33 @@ impl From<RpcCustomError> for Error {
                 data: None,
             },
             RpcCustomError::SendTransactionPreflightFailure { message, result } => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_2),
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE,
+                ),
                 message,
                 data: Some(serde_json::json!(result)),
             },
             RpcCustomError::TransactionSignatureVerificationFailure => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_3),
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE,
+                ),
                 message: "Transaction signature verification failure".to_string(),
                 data: None,
             },
             RpcCustomError::BlockNotAvailable { slot } => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_4),
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE),
                 message: format!("Block not available for slot {}", slot),
                 data: None,
             },
             RpcCustomError::RpcNodeUnhealthy => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_5),
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY),
                 message: "RPC node is unhealthy".to_string(),
                 data: None,
             },
             RpcCustomError::TransactionPrecompileVerificationFailure(e) => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_6),
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE,
+                ),
                 message: format!("Transaction precompile verification failure {:?}", e),
                 data: None,
             },

client/src/rpc_request.rs

@@ -1,3 +1,4 @@
+use crate::rpc_response::RpcSimulateTransactionResult;
 use serde_json::{json, Value};
 use solana_sdk::pubkey::Pubkey;
 use std::fmt;
@@ -138,10 +139,42 @@ impl RpcRequest {
     }
 }
 
+#[derive(Debug)]
+pub enum RpcResponseErrorData {
+    Empty,
+    SendTransactionPreflightFailure(RpcSimulateTransactionResult),
+}
+
+impl fmt::Display for RpcResponseErrorData {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            RpcResponseErrorData::SendTransactionPreflightFailure(
+                RpcSimulateTransactionResult {
+                    logs: Some(logs), ..
+                },
+            ) => {
+                if logs.is_empty() {
+                    Ok(())
+                } else {
+                    // Give the user a hint that there is more useful logging information available...
+                    write!(f, "[{} log messages]", logs.len())
+                }
+            }
+            _ => Ok(()),
+        }
+    }
+}
+
 #[derive(Debug, Error)]
 pub enum RpcError {
-    #[error("rpc request error: {0}")]
+    #[error("RPC request error: {0}")]
     RpcRequestError(String),
+    #[error("RPC response error {code}: {message} {data}")]
+    RpcResponseError {
+        code: i64,
+        message: String,
+        data: RpcResponseErrorData,
+    },
     #[error("parse error: expected {0}")]
     ParseError(String), /* "expected" */
     // Anything in a `ForUser` needs to die.  The caller should be
@@ -226,7 +259,7 @@ mod tests {
 
         // Test request with CommitmentConfig and params
         let test_request = RpcRequest::GetTokenAccountsByOwner;
-        let mint = Pubkey::new_rand();
+        let mint = solana_sdk::pubkey::new_rand();
         let token_account_filter = RpcTokenAccountsFilter::Mint(mint.to_string());
         let request = test_request
             .build_request_json(1, json!([addr, token_account_filter, commitment_config]));

client/src/rpc_response.rs

@@ -108,6 +108,14 @@ pub enum RpcSignatureResult {
     ReceivedSignature(ReceivedSignatureResult),
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcLogsResponse {
+    pub signature: String, // Signature as base58 string
+    pub err: Option<TransactionError>,
+    pub logs: Vec<String>,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct ProcessedSignatureResult {

client/src/thin_client.rs

@@ -3,7 +3,7 @@
 //! messages to the network directly. The binary encoding of its messages are
 //! unstable and may change in future releases.
 
-use crate::{rpc_client::RpcClient, rpc_response::Response};
+use crate::{rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response};
 use bincode::{serialize_into, serialized_size};
 use log::*;
 use solana_sdk::{
@@ -276,6 +276,16 @@ impl ThinClient {
         )
     }
 
+    pub fn get_program_accounts_with_config(
+        &self,
+        pubkey: &Pubkey,
+        config: RpcProgramAccountsConfig,
+    ) -> TransportResult<Vec<(Pubkey, Account)>> {
+        self.rpc_client()
+            .get_program_accounts_with_config(pubkey, config)
+            .map_err(|e| e.into())
+    }
+
     pub fn wait_for_balance_with_commitment(
         &self,
         pubkey: &Pubkey,

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.4.12"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -34,53 +34,54 @@ jsonrpc-http-server = "15.0.0"
 jsonrpc-pubsub = "15.0.0"
 jsonrpc-ws-server = "15.0.0"
 log = "0.4.8"
+lru = "0.6.0"
 num_cpus = "1.13.0"
 num-traits = "0.2"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 raptorq = "1.4.2"
-rayon = "1.4.0"
+rayon = "1.4.1"
 regex = "1.3.9"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.12" }
-solana-banks-server = { path = "../banks-server", version = "1.4.0" }
+solana-banks-server = { path = "../banks-server", version = "1.4.12" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.12" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
+solana-client = { path = "../client", version = "1.4.12" }
-solana-client = { path = "../client", version = "1.4.0" }
+solana-faucet = { path = "../faucet", version = "1.4.12" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
+solana-frozen-abi = { path = "../frozen-abi", version = "1.4.12" }
-solana-ledger = { path = "../ledger", version = "1.4.0" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.4.12" }
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-ledger = { path = "../ledger", version = "1.4.12" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.4.12" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.4.12" }
-solana-measure = { path = "../measure", version = "1.4.0" }
+solana-metrics = { path = "../metrics", version = "1.4.12" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
+solana-measure = { path = "../measure", version = "1.4.12" }
-solana-perf = { path = "../perf", version = "1.4.0" }
+solana-net-utils = { path = "../net-utils", version = "1.4.12" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
+solana-perf = { path = "../perf", version = "1.4.12" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.4.12" }
-solana-sdk-macro-frozen-abi = { path = "../sdk/macro-frozen-abi", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.12" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.12" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.0" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.12" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
+solana-streamer = { path = "../streamer", version = "1.4.12" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.4.0" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.4.12" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.12" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-version = { path = "../version", version = "1.4.12" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.12" }
-solana-vote-signer = { path = "../vote-signer", version = "1.4.0" }
+solana-vote-signer = { path = "../vote-signer", version = "1.4.12" }
-spl-token-v2-0 = { package = "spl-token", version = "=2.0.6", features = ["skip-no-mangle"] }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.2", features = ["full"] }
 tokio_01 = { version = "0.1", package = "tokio" }
 tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
 tokio_io_01 = { version = "0.1", package = "tokio-io" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.12" }
 trees = "0.2.1"
 
 [dev-dependencies]
 matches = "0.1.6"
-reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
 systemstat = "0.1.5"
@@ -94,6 +95,9 @@ name = "banking_stage"
 [[bench]]
 name = "blockstore"
 
+[[bench]]
+name = "crds"
+
 [[bench]]
 name = "crds_gossip_pull"
 

core/benches/banking_stage.rs

@@ -20,7 +20,7 @@ use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;
 use solana_sdk::hash::Hash;
 use solana_sdk::message::Message;
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::pubkey;
 use solana_sdk::signature::Keypair;
 use solana_sdk::signature::Signature;
 use solana_sdk::signature::Signer;
@@ -56,7 +56,7 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
     let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
     let bank = Arc::new(Bank::new(&genesis_config));
     let ledger_path = get_tmp_ledger_path!();
-    let my_pubkey = Pubkey::new_rand();
+    let my_pubkey = pubkey::new_rand();
     {
         let blockstore = Arc::new(
             Blockstore::open(&ledger_path).expect("Expected to be able to open database ledger"),
@@ -94,15 +94,15 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
 }
 
 fn make_accounts_txs(txes: usize, mint_keypair: &Keypair, hash: Hash) -> Vec<Transaction> {
-    let to_pubkey = Pubkey::new_rand();
+    let to_pubkey = pubkey::new_rand();
     let dummy = system_transaction::transfer(mint_keypair, &to_pubkey, 1, hash);
     (0..txes)
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
             let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
-            new.message.account_keys[0] = Pubkey::new_rand();
+            new.message.account_keys[0] = pubkey::new_rand();
-            new.message.account_keys[1] = Pubkey::new_rand();
+            new.message.account_keys[1] = pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
         })
@@ -117,7 +117,7 @@ fn make_programs_txs(txes: usize, hash: Hash) -> Vec<Transaction> {
             let mut instructions = vec![];
             let from_key = Keypair::new();
             for _ in 1..progs {
-                let to_key = Pubkey::new_rand();
+                let to_key = pubkey::new_rand();
                 instructions.push(system_instruction::transfer(&from_key.pubkey(), &to_key, 1));
             }
             let message = Message::new(&instructions, Some(&from_key.pubkey()));

core/benches/cluster_info.rs

@@ -8,7 +8,7 @@ use solana_core::broadcast_stage::{broadcast_shreds, get_broadcast_peers};
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
 use solana_ledger::shred::Shred;
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::pubkey;
 use solana_sdk::timing::timestamp;
 use std::{
     collections::HashMap,
@@ -20,7 +20,7 @@ use test::Bencher;
 #[bench]
 fn broadcast_shreds_bench(bencher: &mut Bencher) {
     solana_logger::setup();
-    let leader_pubkey = Pubkey::new_rand();
+    let leader_pubkey = pubkey::new_rand();
     let leader_info = Node::new_localhost_with_pubkey(&leader_pubkey);
     let cluster_info = ClusterInfo::new_with_invalid_keypair(leader_info.info);
     let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
@@ -30,7 +30,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
     let mut stakes = HashMap::new();
     const NUM_PEERS: usize = 200;
     for _ in 0..NUM_PEERS {
-        let id = Pubkey::new_rand();
+        let id = pubkey::new_rand();
         let contact_info = ContactInfo::new_localhost(&id, timestamp());
         cluster_info.insert_info(contact_info);
         stakes.insert(id, thread_rng().gen_range(1, NUM_PEERS) as u64);

core/benches/crds.rs

@@ -0,0 +1,31 @@
+#![feature(test)]
+
+extern crate test;
+
+use rand::{thread_rng, Rng};
+use rayon::ThreadPoolBuilder;
+use solana_core::crds::Crds;
+use solana_core::crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
+use solana_core::crds_value::CrdsValue;
+use solana_sdk::pubkey::Pubkey;
+use std::collections::HashMap;
+use test::Bencher;
+
+#[bench]
+fn bench_find_old_labels(bencher: &mut Bencher) {
+    let thread_pool = ThreadPoolBuilder::new().build().unwrap();
+    let mut rng = thread_rng();
+    let mut crds = Crds::default();
+    let now = CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS + CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS / 1000;
+    std::iter::repeat_with(|| (CrdsValue::new_rand(&mut rng, None), rng.gen_range(0, now)))
+        .take(50_000)
+        .for_each(|(v, ts)| assert!(crds.insert(v, ts).is_ok()));
+    let mut timeouts = HashMap::new();
+    timeouts.insert(Pubkey::default(), CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS);
+    bencher.iter(|| {
+        let out = crds.find_old_labels(&thread_pool, now, &timeouts);
+        assert!(out.len() > 10);
+        assert!(out.len() < 250);
+        out
+    });
+}

core/benches/crds_gossip_pull.rs

@@ -8,13 +8,13 @@ use solana_core::cluster_info::MAX_BLOOM_SIZE;
 use solana_core::crds::Crds;
 use solana_core::crds_gossip_pull::{CrdsFilter, CrdsGossipPull};
 use solana_core::crds_value::CrdsValue;
-use solana_sdk::hash::Hash;
+use solana_sdk::hash;
 use test::Bencher;
 
 #[bench]
 fn bench_hash_as_u64(bencher: &mut Bencher) {
     let mut rng = thread_rng();
-    let hashes: Vec<_> = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
+    let hashes: Vec<_> = std::iter::repeat_with(|| hash::new_rand(&mut rng))
         .take(1000)
         .collect();
     bencher.iter(|| {
@@ -34,12 +34,12 @@ fn bench_build_crds_filters(bencher: &mut Bencher) {
     for _ in 0..50_000 {
         crds_gossip_pull
             .purged_values
-            .push_back((Hash::new_rand(&mut rng), rng.gen()));
+            .push_back((solana_sdk::hash::new_rand(&mut rng), rng.gen()));
     }
     let mut num_inserts = 0;
     for _ in 0..90_000 {
         if crds
-            .insert(CrdsValue::new_rand(&mut rng), rng.gen())
+            .insert(CrdsValue::new_rand(&mut rng, None), rng.gen())
             .is_ok()
         {
             num_inserts += 1;

core/benches/crds_shards.rs

@@ -7,14 +7,14 @@ use solana_core::contact_info::ContactInfo;
 use solana_core::crds::VersionedCrdsValue;
 use solana_core::crds_shards::CrdsShards;
 use solana_core::crds_value::{CrdsData, CrdsValue};
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::pubkey;
 use solana_sdk::timing::timestamp;
 use test::Bencher;
 
 const CRDS_SHARDS_BITS: u32 = 8;
 
 fn new_test_crds_value() -> VersionedCrdsValue {
-    let data = CrdsData::ContactInfo(ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp()));
+    let data = CrdsData::ContactInfo(ContactInfo::new_localhost(&pubkey::new_rand(), timestamp()));
     VersionedCrdsValue::new(timestamp(), CrdsValue::new_unsigned(data))
 }
 

core/benches/retransmit_stage.rs

@@ -14,7 +14,7 @@ use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_runtime::bank_forks::BankForks;
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::pubkey;
 use solana_sdk::timing::timestamp;
 use std::net::UdpSocket;
 use std::sync::atomic::{AtomicUsize, Ordering};
@@ -34,7 +34,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     const NUM_PEERS: usize = 4;
     let mut peer_sockets = Vec::new();
     for _ in 0..NUM_PEERS {
-        let id = Pubkey::new_rand();
+        let id = pubkey::new_rand();
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut contact_info = ContactInfo::new_localhost(&id, timestamp());
         contact_info.tvu = socket.local_addr().unwrap();

core/build.rs

@@ -1 +1 @@
-../sdk/build.rs
+../frozen-abi/build.rs

core/src/accounts_hash_verifier.rs

@@ -204,6 +204,7 @@ mod tests {
         {
             let message = make_accounts_hashes_message(&validator1, vec![(0, hash1)]).unwrap();
             cluster_info.push_message(message);
+            cluster_info.flush_push_queue();
         }
         slot_to_hash.insert(0, hash2);
         trusted_validators.insert(validator1.pubkey());
@@ -254,6 +255,7 @@ mod tests {
                 100,
             );
         }
+        cluster_info.flush_push_queue();
         let cluster_hashes = cluster_info
             .get_accounts_hash_for_node(&keypair.pubkey(), |c| c.clone())
             .unwrap();

core/src/bank_weight_fork_choice.rs

@@ -60,7 +60,7 @@ impl ForkChoice for BankWeightForkChoice {
         trace!("frozen_banks {}", frozen_banks.len());
         let num_old_banks = frozen_banks
             .iter()
-            .filter(|b| b.slot() < tower.root().unwrap_or(0))
+            .filter(|b| b.slot() < tower.root())
             .count();
 
         let last_voted_slot = tower.last_voted_slot();

core/src/banking_stage.rs

@@ -534,13 +534,14 @@ impl BankingStage {
             mut loaded_accounts,
             results,
             inner_instructions,
+            transaction_logs,
             mut retryable_txs,
             tx_count,
             signature_count,
         ) = bank.load_and_execute_transactions(
             batch,
             MAX_PROCESSING_AGE,
-            None,
+            transaction_status_sender.is_some(),
             transaction_status_sender.is_some(),
         );
         load_execute_time.stop();
@@ -580,6 +581,7 @@ impl BankingStage {
                     tx_results.processing_results,
                     TransactionBalancesSet::new(pre_balances, post_balances),
                     inner_instructions,
+                    transaction_logs,
                     sender,
                 );
             }
@@ -1244,16 +1246,16 @@ mod tests {
             bank.process_transaction(&fund_tx).unwrap();
 
             // good tx
-            let to = Pubkey::new_rand();
+            let to = solana_sdk::pubkey::new_rand();
             let tx = system_transaction::transfer(&mint_keypair, &to, 1, start_hash);
 
             // good tx, but no verify
-            let to2 = Pubkey::new_rand();
+            let to2 = solana_sdk::pubkey::new_rand();
             let tx_no_ver = system_transaction::transfer(&keypair, &to2, 2, start_hash);
 
             // bad tx, AccountNotFound
             let keypair = Keypair::new();
-            let to3 = Pubkey::new_rand();
+            let to3 = solana_sdk::pubkey::new_rand();
             let tx_anf = system_transaction::transfer(&keypair, &to3, 1, start_hash);
 
             // send 'em over
@@ -1446,9 +1448,9 @@ mod tests {
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
             poh_recorder.lock().unwrap().set_working_bank(working_bank);
-            let pubkey = Pubkey::new_rand();
+            let pubkey = solana_sdk::pubkey::new_rand();
             let keypair2 = Keypair::new();
-            let pubkey2 = Pubkey::new_rand();
+            let pubkey2 = solana_sdk::pubkey::new_rand();
 
             let transactions = vec![
                 system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()),
@@ -1526,7 +1528,7 @@ mod tests {
             mint_keypair,
             ..
         } = create_genesis_config(10_000);
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
 
         let transactions = vec![
             None,
@@ -1607,7 +1609,7 @@ mod tests {
             mint_keypair,
             ..
         } = create_genesis_config(10_000);
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
 
         let transactions = vec![
             system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()),
@@ -1678,8 +1680,8 @@ mod tests {
 
     #[test]
     fn test_should_process_or_forward_packets() {
-        let my_pubkey = Pubkey::new_rand();
+        let my_pubkey = solana_sdk::pubkey::new_rand();
-        let my_pubkey1 = Pubkey::new_rand();
+        let my_pubkey1 = solana_sdk::pubkey::new_rand();
 
         assert_eq!(
             BankingStage::consume_or_forward_packets(&my_pubkey, None, true, false,),
@@ -1725,7 +1727,7 @@ mod tests {
             ..
         } = create_genesis_config(10_000);
         let bank = Arc::new(Bank::new(&genesis_config));
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
 
         let transactions = vec![system_transaction::transfer(
             &mint_keypair,
@@ -1822,8 +1824,8 @@ mod tests {
             ..
         } = create_genesis_config(10_000);
         let bank = Arc::new(Bank::new(&genesis_config));
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
-        let pubkey1 = Pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
 
         let transactions = vec![
             system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()),
@@ -1918,7 +1920,7 @@ mod tests {
         } = create_genesis_config(10_000);
         let bank = Arc::new(Bank::new(&genesis_config));
 
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
 
         let transactions =
             vec![
@@ -1936,7 +1938,7 @@ mod tests {
                 bank.slot(),
                 Some((4, 4)),
                 bank.ticks_per_slot(),
-                &Pubkey::new_rand(),
+                &solana_sdk::pubkey::new_rand(),
                 &Arc::new(blockstore),
                 &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
                 &Arc::new(PohConfig::default()),
@@ -1976,8 +1978,8 @@ mod tests {
             ..
         } = create_genesis_config(10_000);
         let bank = Arc::new(Bank::new(&genesis_config));
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
-        let pubkey1 = Pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
         let keypair1 = Keypair::new();
 
         let success_tx =

core/src/broadcast_stage/broadcast_fake_shreds_run.rs

@@ -140,13 +140,12 @@ impl BroadcastRun for BroadcastFakeShredsRun {
 mod tests {
     use super::*;
     use crate::contact_info::ContactInfo;
-    use solana_sdk::pubkey::Pubkey;
     use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
     #[test]
     fn test_tvu_peers_ordering() {
         let cluster = ClusterInfo::new_with_invalid_keypair(ContactInfo::new_localhost(
-            &Pubkey::new_rand(),
+            &solana_sdk::pubkey::new_rand(),
             0,
         ));
         cluster.insert_info(ContactInfo::new_with_socketaddr(&SocketAddr::new(

core/src/broadcast_stage/broadcast_utils.rs

@@ -92,7 +92,7 @@ mod tests {
         let bank0 = Arc::new(Bank::new(&genesis_config));
         let tx = system_transaction::transfer(
             &mint_keypair,
-            &Pubkey::new_rand(),
+            &solana_sdk::pubkey::new_rand(),
             1,
             genesis_config.hash(),
         );

core/src/cache_block_time_service.rs

@@ -2,7 +2,7 @@ use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
 use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
 use solana_runtime::bank::Bank;
-use solana_sdk::timing::slot_duration_from_slots_per_year;
+use solana_sdk::{feature_set, timing::slot_duration_from_slots_per_year};
 use std::{
     collections::HashMap,
     sync::{
@@ -60,15 +60,26 @@ impl CacheBlockTimeService {
     }
 
     fn cache_block_time(bank: Arc<Bank>, blockstore: &Arc<Blockstore>) {
+        if bank
+            .feature_set
+            .is_active(&feature_set::timestamp_correction::id())
+        {
+            if let Err(e) = blockstore.cache_block_time(bank.slot(), bank.clock().unix_timestamp) {
+                error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
+            }
+        } else {
             let slot_duration = slot_duration_from_slots_per_year(bank.slots_per_year());
             let epoch = bank.epoch_schedule().get_epoch(bank.slot());
             let stakes = HashMap::new();
             let stakes = bank.epoch_vote_accounts(epoch).unwrap_or(&stakes);
 
-        if let Err(e) = blockstore.cache_block_time(bank.slot(), slot_duration, stakes) {
+            if let Err(e) =
+                blockstore.cache_block_time_from_slot_entries(bank.slot(), slot_duration, stakes)
+            {
                 error!("cache_block_time failed: slot {:?} {:?}", bank.slot(), e);
             }
         }
+    }
 
     pub fn join(self) -> thread::Result<()> {
         self.thread_hdl.join()

core/src/cluster_info_vote_listener.rs

@@ -27,7 +27,7 @@ use solana_runtime::{
     vote_sender_types::{ReplayVoteReceiver, ReplayedVote},
 };
 use solana_sdk::{
-    clock::{Epoch, Slot},
+    clock::{Epoch, Slot, DEFAULT_MS_PER_SLOT},
     epoch_schedule::EpochSchedule,
     hash::Hash,
     pubkey::Pubkey,
@@ -98,7 +98,7 @@ impl VoteTracker {
             epoch_schedule: *root_bank.epoch_schedule(),
             ..VoteTracker::default()
         };
-        vote_tracker.process_new_root_bank(&root_bank);
+        vote_tracker.progress_with_new_root_bank(&root_bank);
         assert_eq!(
             *vote_tracker.leader_schedule_epoch.read().unwrap(),
             root_bank.get_leader_schedule_epoch(root_bank.slot())
@@ -174,7 +174,7 @@ impl VoteTracker {
         self.keys.get_or_insert(&pubkey);
     }
 
-    fn update_leader_schedule_epoch(&self, root_bank: &Bank) {
+    fn progress_leader_schedule_epoch(&self, root_bank: &Bank) {
         // Update with any newly calculated epoch state about future epochs
         let start_leader_schedule_epoch = *self.leader_schedule_epoch.read().unwrap();
         let mut greatest_leader_schedule_epoch = start_leader_schedule_epoch;
@@ -205,7 +205,7 @@ impl VoteTracker {
         }
     }
 
-    fn update_new_root(&self, root_bank: &Bank) {
+    fn purge_stale_state(&self, root_bank: &Bank) {
         // Purge any outdated slot data
         let new_root = root_bank.slot();
         let root_epoch = root_bank.epoch();
@@ -220,15 +220,15 @@ impl VoteTracker {
             self.epoch_authorized_voters
                 .write()
                 .unwrap()
-                .retain(|epoch, _| epoch >= &root_epoch);
+                .retain(|epoch, _| *epoch >= root_epoch);
             self.keys.purge();
             *self.current_epoch.write().unwrap() = root_epoch;
         }
     }
 
-    fn process_new_root_bank(&self, root_bank: &Bank) {
+    fn progress_with_new_root_bank(&self, root_bank: &Bank) {
-        self.update_leader_schedule_epoch(root_bank);
+        self.progress_leader_schedule_epoch(root_bank);
-        self.update_new_root(root_bank);
+        self.purge_stale_state(root_bank);
     }
 }
 
@@ -425,7 +425,7 @@ impl ClusterInfoVoteListener {
         blockstore: Arc<Blockstore>,
         bank_notification_sender: Option<BankNotificationSender>,
     ) -> Result<()> {
-        let mut optimistic_confirmation_verifier =
+        let mut confirmation_verifier =
             OptimisticConfirmationVerifier::new(bank_forks.read().unwrap().root());
         let mut last_process_root = Instant::now();
         loop {
@@ -434,21 +434,21 @@ impl ClusterInfoVoteListener {
             }
 
             let root_bank = bank_forks.read().unwrap().root_bank().clone();
-            if last_process_root.elapsed().as_millis() > 400 {
+            if last_process_root.elapsed().as_millis() > DEFAULT_MS_PER_SLOT as u128 {
-                let unrooted_optimistic_slots = optimistic_confirmation_verifier
+                let unrooted_optimistic_slots = confirmation_verifier
-                    .get_unrooted_optimistic_slots(&root_bank, &blockstore);
+                    .verify_for_unrooted_optimistic_slots(&root_bank, &blockstore);
                 // SlotVoteTracker's for all `slots` in `unrooted_optimistic_slots`
                 // should still be available because we haven't purged in
-                // `process_new_root_bank()` yet, which is called below
+                // `progress_with_new_root_bank()` yet, which is called below
                 OptimisticConfirmationVerifier::log_unrooted_optimistic_slots(
                     &root_bank,
                     &vote_tracker,
                     &unrooted_optimistic_slots,
                 );
-                vote_tracker.process_new_root_bank(&root_bank);
+                vote_tracker.progress_with_new_root_bank(&root_bank);
                 last_process_root = Instant::now();
             }
-            let optimistic_confirmed_slots = Self::get_and_process_votes(
+            let confirmed_slots = Self::listen_and_confirm_votes(
                 &gossip_vote_txs_receiver,
                 &vote_tracker,
                 &root_bank,
@@ -457,19 +457,17 @@ impl ClusterInfoVoteListener {
                 &replay_votes_receiver,
                 &bank_notification_sender,
             );
-
+            match confirmed_slots {
-            if let Err(e) = optimistic_confirmed_slots {
+                Ok(confirmed_slots) => {
-                match e {
+                    confirmation_verifier.add_new_optimistic_confirmed_slots(confirmed_slots);
+                }
+                Err(e) => match e {
                     Error::CrossbeamRecvTimeoutError(RecvTimeoutError::Timeout)
                     | Error::ReadyTimeoutError => (),
                     _ => {
                         error!("thread {:?} error {:?}", thread::current().name(), e);
                     }
-                }
+                },
-            } else {
-                let optimistic_confirmed_slots = optimistic_confirmed_slots.unwrap();
-                optimistic_confirmation_verifier
-                    .add_new_optimistic_confirmed_slots(optimistic_confirmed_slots);
             }
         }
     }
@@ -483,7 +481,7 @@ impl ClusterInfoVoteListener {
         verified_vote_sender: &VerifiedVoteSender,
         replay_votes_receiver: &ReplayVoteReceiver,
     ) -> Result<Vec<(Slot, Hash)>> {
-        Self::get_and_process_votes(
+        Self::listen_and_confirm_votes(
             gossip_vote_txs_receiver,
             vote_tracker,
             root_bank,
@@ -494,7 +492,7 @@ impl ClusterInfoVoteListener {
         )
     }
 
-    fn get_and_process_votes(
+    fn listen_and_confirm_votes(
         gossip_vote_txs_receiver: &VerifiedVoteTransactionsReceiver,
         vote_tracker: &VoteTracker,
         root_bank: &Bank,
@@ -523,7 +521,7 @@ impl ClusterInfoVoteListener {
             let gossip_vote_txs: Vec<_> = gossip_vote_txs_receiver.try_iter().flatten().collect();
             let replay_votes: Vec<_> = replay_votes_receiver.try_iter().collect();
             if !gossip_vote_txs.is_empty() || !replay_votes.is_empty() {
-                return Ok(Self::process_votes(
+                return Ok(Self::filter_and_confirm_with_new_votes(
                     vote_tracker,
                     gossip_vote_txs,
                     replay_votes,
@@ -541,7 +539,7 @@ impl ClusterInfoVoteListener {
     }
 
     #[allow(clippy::too_many_arguments)]
-    fn update_new_votes(
+    fn track_new_votes_and_notify_confirmations(
         vote: Vote,
         vote_pubkey: &Pubkey,
         vote_tracker: &VoteTracker,
@@ -557,56 +555,52 @@ impl ClusterInfoVoteListener {
             return;
         }
 
-        let last_vote_slot = vote.slots.last().unwrap();
+        let last_vote_slot = *vote.slots.last().unwrap();
+        let last_vote_hash = vote.hash;
 
         let root = root_bank.slot();
-        let last_vote_hash = vote.hash;
         let mut is_new_vote = false;
-        for slot in vote.slots.iter().rev() {
+        // If slot is before the root, ignore it
-            // If slot is before the root, or so far ahead we don't have
+        for slot in vote.slots.iter().filter(|slot| **slot > root).rev() {
-            // stake information, then ignore it
+            let slot = *slot;
-            let epoch = root_bank.epoch_schedule().get_epoch(*slot);
+
+            // if we don't have stake information, ignore it
+            let epoch = root_bank.epoch_schedule().get_epoch(slot);
             let epoch_stakes = root_bank.epoch_stakes(epoch);
-            if *slot <= root || epoch_stakes.is_none() {
+            if epoch_stakes.is_none() {
                 continue;
             }
             let epoch_stakes = epoch_stakes.unwrap();
-            let epoch_vote_accounts = Stakes::vote_accounts(epoch_stakes.stakes());
-            let total_epoch_stake = epoch_stakes.total_stake();
             let unduplicated_pubkey = vote_tracker.keys.get_or_insert(&vote_pubkey);
 
             // The last vote slot, which is the greatest slot in the stack
             // of votes in a vote transaction, qualifies for optimistic confirmation.
-            let update_optimistic_confirmation_info = if slot == last_vote_slot {
+            if slot == last_vote_slot {
-                let stake = epoch_vote_accounts
+                let vote_accounts = Stakes::vote_accounts(epoch_stakes.stakes());
+                let stake = vote_accounts
                     .get(&vote_pubkey)
                     .map(|(stake, _)| *stake)
-                    .unwrap_or(0);
+                    .unwrap_or_default();
-                Some((stake, last_vote_hash))
+                let total_stake = epoch_stakes.total_stake();
-            } else {
-                None
-            };
 
-            // If this vote for this slot qualifies for optimistic confirmation
-            if let Some((stake, hash)) = update_optimistic_confirmation_info {
                 // Fast track processing of the last slot in a vote transactions
                 // so that notifications for optimistic confirmation can be sent
                 // as soon as possible.
-                let (is_confirmed, is_new) = Self::add_optimistic_confirmation_vote(
+                let (is_confirmed, is_new) = Self::track_optimistic_confirmation_vote(
                     vote_tracker,
-                    *slot,
+                    last_vote_slot,
-                    hash,
+                    last_vote_hash,
                     unduplicated_pubkey.clone(),
                     stake,
-                    total_epoch_stake,
+                    total_stake,
                 );
 
                 if is_confirmed {
-                    new_optimistic_confirmed_slots.push((*slot, last_vote_hash));
+                    new_optimistic_confirmed_slots.push((last_vote_slot, last_vote_hash));
                     // Notify subscribers about new optimistic confirmation
                     if let Some(sender) = bank_notification_sender {
                         sender
-                            .send(BankNotification::OptimisticallyConfirmed(*slot))
+                            .send(BankNotification::OptimisticallyConfirmed(last_vote_slot))
                             .unwrap_or_else(|err| {
                                 warn!("bank_notification_sender failed: {:?}", err)
                             });
@@ -617,7 +611,7 @@ impl ClusterInfoVoteListener {
                     // By now:
                     // 1) The vote must have come from ReplayStage,
                     // 2) We've seen this vote from replay for this hash before
-                    // (`add_optimistic_confirmation_vote()` will not set `is_new == true`
+                    // (`track_optimistic_confirmation_vote()` will not set `is_new == true`
                     // for same slot different hash), so short circuit because this vote
                     // has no new information
 
@@ -629,7 +623,7 @@ impl ClusterInfoVoteListener {
                 is_new_vote = is_new;
             }
 
-            diff.entry(*slot)
+            diff.entry(slot)
                 .or_default()
                 .entry(unduplicated_pubkey)
                 .and_modify(|seen_in_gossip_previously| {
@@ -644,24 +638,12 @@ impl ClusterInfoVoteListener {
         }
     }
 
-    fn process_votes(
+    fn filter_gossip_votes(
         vote_tracker: &VoteTracker,
-        gossip_vote_txs: Vec<Transaction>,
+        vote_pubkey: &Pubkey,
-        replayed_votes: Vec<ReplayedVote>,
+        vote: &Vote,
-        root_bank: &Bank,
+        gossip_tx: &Transaction,
-        subscriptions: &RpcSubscriptions,
+    ) -> bool {
-        verified_vote_sender: &VerifiedVoteSender,
-        bank_notification_sender: &Option<BankNotificationSender>,
-    ) -> Vec<(Slot, Hash)> {
-        let mut diff: HashMap<Slot, HashMap<Arc<Pubkey>, bool>> = HashMap::new();
-        let mut new_optimistic_confirmed_slots = vec![];
-
-        // Process votes from gossip and ReplayStage
-        for (is_gossip, (vote_pubkey, vote, _)) in gossip_vote_txs
-            .iter()
-            .filter_map(|gossip_tx| {
-                vote_transaction::parse_vote_transaction(gossip_tx)
-                    .filter(|(vote_pubkey, vote, _)| {
         if vote.slots.is_empty() {
             return false;
         }
@@ -687,12 +669,33 @@ impl ClusterInfoVoteListener {
         }
 
         true
+    }
+
+    fn filter_and_confirm_with_new_votes(
+        vote_tracker: &VoteTracker,
+        gossip_vote_txs: Vec<Transaction>,
+        replayed_votes: Vec<ReplayedVote>,
+        root_bank: &Bank,
+        subscriptions: &RpcSubscriptions,
+        verified_vote_sender: &VerifiedVoteSender,
+        bank_notification_sender: &Option<BankNotificationSender>,
+    ) -> Vec<(Slot, Hash)> {
+        let mut diff: HashMap<Slot, HashMap<Arc<Pubkey>, bool>> = HashMap::new();
+        let mut new_optimistic_confirmed_slots = vec![];
+
+        // Process votes from gossip and ReplayStage
+        for (is_gossip, (vote_pubkey, vote, _)) in gossip_vote_txs
+            .iter()
+            .filter_map(|gossip_tx| {
+                vote_transaction::parse_vote_transaction(gossip_tx)
+                    .filter(|(vote_pubkey, vote, _)| {
+                        Self::filter_gossip_votes(vote_tracker, vote_pubkey, vote, gossip_tx)
                     })
                     .map(|v| (true, v))
             })
             .chain(replayed_votes.into_iter().map(|v| (false, v)))
         {
-            Self::update_new_votes(
+            Self::track_new_votes_and_notify_confirmations(
                 vote,
                 &vote_pubkey,
                 &vote_tracker,
@@ -757,7 +760,7 @@ impl ClusterInfoVoteListener {
 
     // Returns if the slot was optimistically confirmed, and whether
     // the slot was new
-    fn add_optimistic_confirmation_vote(
+    fn track_optimistic_confirmation_vote(
         vote_tracker: &VoteTracker,
         slot: Slot,
         hash: Hash,
@@ -898,7 +901,7 @@ mod tests {
         let (vote_tracker, bank, _, _) = setup();
 
         // Check outdated slots are purged with new root
-        let new_voter = Arc::new(Pubkey::new_rand());
+        let new_voter = Arc::new(solana_sdk::pubkey::new_rand());
         // Make separate copy so the original doesn't count toward
         // the ref count, which would prevent cleanup
         let new_voter_ = Arc::new(*new_voter);
@@ -909,7 +912,7 @@ mod tests {
             .unwrap()
             .contains_key(&bank.slot()));
         let bank1 = Bank::new_from_parent(&bank, &Pubkey::default(), bank.slot() + 1);
-        vote_tracker.process_new_root_bank(&bank1);
+        vote_tracker.progress_with_new_root_bank(&bank1);
         assert!(!vote_tracker
             .slot_vote_trackers
             .read()
@@ -926,7 +929,7 @@ mod tests {
             bank.epoch_schedule()
                 .get_first_slot_in_epoch(current_epoch + 1),
         );
-        vote_tracker.process_new_root_bank(&new_epoch_bank);
+        vote_tracker.progress_with_new_root_bank(&new_epoch_bank);
         assert!(!vote_tracker.keys.0.read().unwrap().contains(&new_voter));
         assert_eq!(
             *vote_tracker.current_epoch.read().unwrap(),
@@ -956,7 +959,7 @@ mod tests {
         );
         let next_leader_schedule_bank =
             Bank::new_from_parent(&bank, &Pubkey::default(), next_leader_schedule_computed);
-        vote_tracker.update_leader_schedule_epoch(&next_leader_schedule_bank);
+        vote_tracker.progress_leader_schedule_epoch(&next_leader_schedule_bank);
         assert_eq!(
             *vote_tracker.leader_schedule_epoch.read().unwrap(),
             next_leader_schedule_epoch
@@ -1007,7 +1010,7 @@ mod tests {
             &votes_sender,
             &replay_votes_sender,
         );
-        ClusterInfoVoteListener::get_and_process_votes(
+        ClusterInfoVoteListener::listen_and_confirm_votes(
             &votes_receiver,
             &vote_tracker,
             &bank3,
@@ -1036,7 +1039,7 @@ mod tests {
             &votes_sender,
             &replay_votes_sender,
         );
-        ClusterInfoVoteListener::get_and_process_votes(
+        ClusterInfoVoteListener::listen_and_confirm_votes(
             &votes_receiver,
             &vote_tracker,
             &bank3,
@@ -1114,7 +1117,7 @@ mod tests {
         );
 
         // Check that all the votes were registered for each validator correctly
-        ClusterInfoVoteListener::get_and_process_votes(
+        ClusterInfoVoteListener::listen_and_confirm_votes(
             &votes_txs_receiver,
             &vote_tracker,
             &bank0,
@@ -1233,7 +1236,7 @@ mod tests {
         }
 
         // Read and process votes from channel `votes_receiver`
-        ClusterInfoVoteListener::get_and_process_votes(
+        ClusterInfoVoteListener::listen_and_confirm_votes(
             &votes_txs_receiver,
             &vote_tracker,
             &bank0,
@@ -1328,7 +1331,7 @@ mod tests {
                         ))
                         .unwrap();
                 }
-                let _ = ClusterInfoVoteListener::get_and_process_votes(
+                let _ = ClusterInfoVoteListener::listen_and_confirm_votes(
                     &votes_receiver,
                     &vote_tracker,
                     &bank,
@@ -1474,7 +1477,7 @@ mod tests {
         )];
 
         let (verified_vote_sender, _verified_vote_receiver) = unbounded();
-        ClusterInfoVoteListener::process_votes(
+        ClusterInfoVoteListener::filter_and_confirm_with_new_votes(
             &vote_tracker,
             vote_tx,
             // Add gossip vote for same slot, should not affect outcome
@@ -1545,7 +1548,7 @@ mod tests {
 
         let new_root_bank =
             Bank::new_from_parent(&bank, &Pubkey::default(), first_slot_in_new_epoch - 2);
-        ClusterInfoVoteListener::process_votes(
+        ClusterInfoVoteListener::filter_and_confirm_with_new_votes(
             &vote_tracker,
             vote_txs,
             vec![(
@@ -1681,7 +1684,7 @@ mod tests {
     fn run_test_verify_votes_1_pass(hash: Option<Hash>) {
         let vote_tx = test_vote_tx(hash);
         let votes = vec![vote_tx];
-        let labels = vec![CrdsValueLabel::Vote(0, Pubkey::new_rand())];
+        let labels = vec![CrdsValueLabel::Vote(0, solana_sdk::pubkey::new_rand())];
         let (vote_txs, packets) = ClusterInfoVoteListener::verify_votes(votes, labels);
         assert_eq!(vote_txs.len(), 1);
         verify_packets_len(&packets, 1);
@@ -1698,7 +1701,7 @@ mod tests {
         let mut bad_vote = vote_tx.clone();
         bad_vote.signatures[0] = Signature::default();
         let votes = vec![vote_tx.clone(), bad_vote, vote_tx];
-        let label = CrdsValueLabel::Vote(0, Pubkey::new_rand());
+        let label = CrdsValueLabel::Vote(0, solana_sdk::pubkey::new_rand());
         let labels: Vec<_> = (0..votes.len()).map(|_| label.clone()).collect();
         let (vote_txs, packets) = ClusterInfoVoteListener::verify_votes(votes, labels);
         assert_eq!(vote_txs.len(), 2);

core/src/cluster_slots.rs

@@ -237,8 +237,8 @@ mod tests {
         let mut c1 = ContactInfo::default();
         let mut c2 = ContactInfo::default();
         let mut map = HashMap::new();
-        let k1 = Pubkey::new_rand();
+        let k1 = solana_sdk::pubkey::new_rand();
-        let k2 = Pubkey::new_rand();
+        let k2 = solana_sdk::pubkey::new_rand();
         map.insert(Arc::new(k1), std::u64::MAX / 2);
         map.insert(Arc::new(k2), 0);
         cs.cluster_slots
@@ -259,8 +259,8 @@ mod tests {
         let mut c1 = ContactInfo::default();
         let mut c2 = ContactInfo::default();
         let mut map = HashMap::new();
-        let k1 = Pubkey::new_rand();
+        let k1 = solana_sdk::pubkey::new_rand();
-        let k2 = Pubkey::new_rand();
+        let k2 = solana_sdk::pubkey::new_rand();
         map.insert(Arc::new(k2), 0);
         cs.cluster_slots
             .write()
@@ -290,7 +290,7 @@ mod tests {
         let cs = ClusterSlots::default();
         let mut contact_infos = vec![ContactInfo::default(); 2];
         for ci in contact_infos.iter_mut() {
-            ci.id = Pubkey::new_rand();
+            ci.id = solana_sdk::pubkey::new_rand();
         }
         let slot = 9;
 
@@ -359,7 +359,7 @@ mod tests {
         let mut epoch_slot = EpochSlots::default();
         epoch_slot.fill(&[1], 0);
         cs.update_internal(0, (vec![epoch_slot], None));
-        let self_id = Pubkey::new_rand();
+        let self_id = solana_sdk::pubkey::new_rand();
         assert_eq!(
             cs.generate_repairs_for_missing_slots(&self_id, 0),
             vec![RepairType::HighestShred(1, 0)]

core/src/cluster_slots_service.rs

@@ -181,6 +181,7 @@ mod test {
         let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
         let cluster_info = ClusterInfo::new_with_invalid_keypair(node_info.info);
         ClusterSlotsService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
+        cluster_info.flush_push_queue();
         let lowest = cluster_info
             .get_lowest_slot_for_node(&Pubkey::default(), None, |lowest_slot, _| {
                 lowest_slot.clone()

core/src/commitment_service.rs

@@ -375,19 +375,22 @@ mod tests {
 
         let rooted_stake_amount = 40;
 
-        let sk1 = Pubkey::new_rand();
+        let sk1 = solana_sdk::pubkey::new_rand();
-        let pk1 = Pubkey::new_rand();
+        let pk1 = solana_sdk::pubkey::new_rand();
-        let mut vote_account1 = vote_state::create_account(&pk1, &Pubkey::new_rand(), 0, 100);
+        let mut vote_account1 =
+            vote_state::create_account(&pk1, &solana_sdk::pubkey::new_rand(), 0, 100);
         let stake_account1 =
             stake_state::create_account(&sk1, &pk1, &vote_account1, &genesis_config.rent, 100);
-        let sk2 = Pubkey::new_rand();
+        let sk2 = solana_sdk::pubkey::new_rand();
-        let pk2 = Pubkey::new_rand();
+        let pk2 = solana_sdk::pubkey::new_rand();
-        let mut vote_account2 = vote_state::create_account(&pk2, &Pubkey::new_rand(), 0, 50);
+        let mut vote_account2 =
+            vote_state::create_account(&pk2, &solana_sdk::pubkey::new_rand(), 0, 50);
         let stake_account2 =
             stake_state::create_account(&sk2, &pk2, &vote_account2, &genesis_config.rent, 50);
-        let sk3 = Pubkey::new_rand();
+        let sk3 = solana_sdk::pubkey::new_rand();
-        let pk3 = Pubkey::new_rand();
+        let pk3 = solana_sdk::pubkey::new_rand();
-        let mut vote_account3 = vote_state::create_account(&pk3, &Pubkey::new_rand(), 0, 1);
+        let mut vote_account3 =
+            vote_state::create_account(&pk3, &solana_sdk::pubkey::new_rand(), 0, 1);
         let stake_account3 = stake_state::create_account(
             &sk3,
             &pk3,
@@ -395,9 +398,10 @@ mod tests {
             &genesis_config.rent,
             rooted_stake_amount,
         );
-        let sk4 = Pubkey::new_rand();
+        let sk4 = solana_sdk::pubkey::new_rand();
-        let pk4 = Pubkey::new_rand();
+        let pk4 = solana_sdk::pubkey::new_rand();
-        let mut vote_account4 = vote_state::create_account(&pk4, &Pubkey::new_rand(), 0, 1);
+        let mut vote_account4 =
+            vote_state::create_account(&pk4, &solana_sdk::pubkey::new_rand(), 0, 1);
         let stake_account4 = stake_state::create_account(
             &sk4,
             &pk4,

core/src/completed_data_sets_service.rs

@@ -1,6 +1,7 @@
 use crate::rpc_subscriptions::RpcSubscriptions;
 use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
 use solana_ledger::blockstore::{Blockstore, CompletedDataSetInfo};
+use solana_ledger::entry::Entry;
 use solana_sdk::signature::Signature;
 use std::{
     sync::{
@@ -61,10 +62,7 @@ impl CompletedDataSetsService {
             } = completed_set_info;
             match blockstore.get_entries_in_data_block(slot, start_index, end_index, None) {
                 Ok(entries) => {
-                    let transactions = entries
+                    let transactions = Self::get_transaction_signatures(entries);
-                        .into_iter()
-                        .flat_map(|e| e.transactions.into_iter().map(|t| t.signatures[0]))
-                        .collect::<Vec<Signature>>();
                     if !transactions.is_empty() {
                         rpc_subscriptions.notify_signatures_received((slot, transactions));
                     }
@@ -76,7 +74,51 @@ impl CompletedDataSetsService {
         Ok(())
     }
 
+    fn get_transaction_signatures(entries: Vec<Entry>) -> Vec<Signature> {
+        entries
+            .into_iter()
+            .flat_map(|e| {
+                e.transactions
+                    .into_iter()
+                    .filter_map(|mut t| t.signatures.drain(..).next())
+            })
+            .collect::<Vec<Signature>>()
+    }
+
     pub fn join(self) -> thread::Result<()> {
         self.thread_hdl.join()
     }
 }
+
+#[cfg(test)]
+pub mod test {
+    use super::*;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, Signer};
+    use solana_sdk::transaction::Transaction;
+
+    #[test]
+    fn test_zero_signatures() {
+        let tx = Transaction::new_with_payer(&[], None);
+        let entries = vec![Entry::new(&Hash::default(), 1, vec![tx])];
+        let signatures = CompletedDataSetsService::get_transaction_signatures(entries);
+        assert!(signatures.is_empty());
+    }
+
+    #[test]
+    fn test_multi_signatures() {
+        let kp = Keypair::new();
+        let tx =
+            Transaction::new_signed_with_payer(&[], Some(&kp.pubkey()), &[&kp], Hash::default());
+        let entries = vec![Entry::new(&Hash::default(), 1, vec![tx.clone()])];
+        let signatures = CompletedDataSetsService::get_transaction_signatures(entries);
+        assert_eq!(signatures.len(), 1);
+
+        let entries = vec![
+            Entry::new(&Hash::default(), 1, vec![tx.clone(), tx.clone()]),
+            Entry::new(&Hash::default(), 1, vec![tx]),
+        ];
+        let signatures = CompletedDataSetsService::get_transaction_signatures(entries);
+        assert_eq!(signatures.len(), 3);
+    }
+}

core/src/consensus.rs

@@ -30,11 +30,11 @@ use std::{
 };
 use thiserror::Error;
 
-#[derive(PartialEq, Clone, Debug)]
+#[derive(PartialEq, Clone, Debug, AbiExample)]
 pub enum SwitchForkDecision {
     SwitchProof(Hash),
-    NoSwitch,
+    SameFork,
-    FailedSwitchThreshold,
+    FailedSwitchThreshold(u64, u64),
 }
 
 impl SwitchForkDecision {
@@ -45,8 +45,11 @@ impl SwitchForkDecision {
         authorized_voter_pubkey: &Pubkey,
     ) -> Option<Instruction> {
         match self {
-            SwitchForkDecision::FailedSwitchThreshold => None,
+            SwitchForkDecision::FailedSwitchThreshold(_, total_stake) => {
-            SwitchForkDecision::NoSwitch => Some(vote_instruction::vote(
+                assert_ne!(*total_stake, 0);
+                None
+            }
+            SwitchForkDecision::SameFork => Some(vote_instruction::vote(
                 vote_account_pubkey,
                 authorized_voter_pubkey,
                 vote,
@@ -61,6 +64,10 @@ impl SwitchForkDecision {
             }
         }
     }
+
+    pub fn can_vote(&self) -> bool {
+        !matches!(self, SwitchForkDecision::FailedSwitchThreshold(_, _))
+    }
 }
 
 pub const VOTE_THRESHOLD_DEPTH: usize = 8;
@@ -82,7 +89,7 @@ pub(crate) struct ComputedBankState {
     pub pubkey_votes: Arc<PubkeyVotes>,
 }
 
-#[frozen_abi(digest = "2ZUeCLMVQxmHYbeqMH7M97ifVSKoVErGvRHzyxcQRjgU")]
+#[frozen_abi(digest = "Eay84NBbJqiMBfE7HHH2o6e51wcvoU79g8zCi5sw6uj3")]
 #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, AbiExample)]
 pub struct Tower {
     node_pubkey: Pubkey,
@@ -100,7 +107,11 @@ pub struct Tower {
     // (This is a special field for slashing-free validator restart with edge cases).
     // This could be emptied after some time; but left intact indefinitely for easier
     // implementation
+    // Further, stray slot can be stale or not. `Stale` here means whether given
+    // bank_forks (=~ ledger) lacks the slot or not.
     stray_restored_slot: Option<Slot>,
+    #[serde(skip)]
+    pub last_switch_threshold_check: Option<(Slot, SwitchForkDecision)>,
 }
 
 impl Default for Tower {
@@ -115,6 +126,7 @@ impl Default for Tower {
             path: PathBuf::default(),
             tmp_path: PathBuf::default(),
             stray_restored_slot: Option::default(),
+            last_switch_threshold_check: Option::default(),
         };
         // VoteState::root_slot is ensured to be Some in Tower
         tower.lockouts.root_slot = Some(Slot::default());
@@ -377,17 +389,14 @@ impl Tower {
     pub fn record_bank_vote(&mut self, vote: Vote) -> Option<Slot> {
         let slot = vote.last_voted_slot().unwrap_or(0);
         trace!("{} record_vote for {}", self.node_pubkey, slot);
-        let root_slot = self.lockouts.root_slot;
+        let old_root = self.root();
         self.lockouts.process_vote_unchecked(&vote);
         self.last_vote = vote;
+        let new_root = self.root();
 
-        datapoint_info!(
+        datapoint_info!("tower-vote", ("latest", slot, i64), ("root", new_root, i64));
-            "tower-vote",
+        if old_root != new_root {
-            ("latest", slot, i64),
+            Some(new_root)
-            ("root", self.lockouts.root_slot.unwrap_or(0), i64)
-        );
-        if root_slot != self.lockouts.root_slot {
-            Some(self.lockouts.root_slot.unwrap())
         } else {
             None
         }
@@ -431,13 +440,13 @@ impl Tower {
 
     // root may be forcibly set by arbitrary replay root slot, for example from a root
     // after replaying a snapshot.
-    // Also, tower.root() couldn't be None; do_initialize_lockouts() ensures that.
+    // Also, tower.root() couldn't be None; initialize_lockouts() ensures that.
     // Conceptually, every tower must have been constructed from a concrete starting point,
     // which establishes the origin of trust (i.e. root) whether booting from genesis (slot 0) or
     // snapshot (slot N). In other words, there should be no possibility a Tower doesn't have
     // root, unlike young vote accounts.
-    pub fn root(&self) -> Option<Slot> {
+    pub fn root(&self) -> Slot {
-        self.lockouts.root_slot
+        self.lockouts.root_slot.unwrap()
     }
 
     // a slot is recent if it's newer than the last vote we have
@@ -493,7 +502,7 @@ impl Tower {
         false
     }
 
-    pub(crate) fn check_switch_threshold(
+    fn make_check_switch_threshold_decision(
         &self,
         switch_slot: u64,
         ancestors: &HashMap<Slot, HashSet<u64>>,
@@ -504,13 +513,66 @@ impl Tower {
     ) -> SwitchForkDecision {
         self.last_voted_slot()
             .map(|last_voted_slot| {
-                let root = self.lockouts.root_slot.unwrap_or(0);
+                let root = self.root();
                 let empty_ancestors = HashSet::default();
+                let empty_ancestors_due_to_minor_unsynced_ledger = || {
+                    // This condition (stale stray last vote) shouldn't occur under normal validator
+                    // operation, indicating something unusual happened.
+                    // This condition could be introduced by manual ledger mishandling,
+                    // validator SEGV, OS/HW crash, or plain No Free Space FS error.
+
+                    // However, returning empty ancestors as a fallback here shouldn't result in
+                    // slashing by itself (Note that we couldn't fully preclude any kind of slashing if
+                    // the failure was OS or HW level).
+
+                    // Firstly, lockout is ensured elsewhere.
+
+                    // Also, there is no risk of optimistic conf. violation. Although empty ancestors
+                    // could result in incorrect (= more than actual) locked_out_stake and
+                    // false-positive SwitchProof later in this function, there should be no such a
+                    // heavier fork candidate, first of all, if the last vote (or any of its
+                    // unavailable ancestors) were already optimistically confirmed.
+                    // The only exception is that other validator is already violating it...
+                    if self.is_first_switch_check() && switch_slot < last_voted_slot {
+                        // `switch < last` is needed not to warn! this message just because of using
+                        // newer snapshots on validator restart
+                        let message = format!(
+                          "bank_forks doesn't have corresponding data for the stray restored \
+                           last vote({}), meaning some inconsistency between saved tower and ledger.",
+                          last_voted_slot
+                        );
+                        warn!("{}", message);
+                        datapoint_warn!("tower_warn", ("warn", message, String));
+                    }
+                    &empty_ancestors
+                };
+
+                let suspended_decision_due_to_major_unsynced_ledger = || {
+                    // This peculiar corner handling is needed mainly for a tower which is newer than
+                    // blockstore. (Yeah, we tolerate it for ease of maintaining validator by operators)
+                    // This condition could be introduced by manual ledger mishandling,
+                    // validator SEGV, OS/HW crash, or plain No Free Space FS error.
+
+                    // When we're in this clause, it basically means validator is badly running
+                    // with a future tower while replaying past slots, especially problematic is
+                    // last_voted_slot.
+                    // So, don't re-vote on it by returning pseudo FailedSwitchThreshold, otherwise
+                    // there would be slashing because of double vote on one of last_vote_ancestors.
+                    // (Well, needless to say, re-creating the duplicate block must be handled properly
+                    // at the banking stage: https://github.com/solana-labs/solana/issues/8232)
+                    //
+                    // To be specific, the replay stage is tricked into a false perception where
+                    // last_vote_ancestors is AVAILABLE for descendant-of-`switch_slot`,  stale, and
+                    // stray slots (which should always be empty_ancestors).
+                    //
+                    // This is covered by test_future_tower_* in local_cluster
+                    SwitchForkDecision::FailedSwitchThreshold(0, total_stake)
+                };
 
                 let last_vote_ancestors =
                     ancestors.get(&last_voted_slot).unwrap_or_else(|| {
                         if !self.is_stray_last_vote() {
-                            // Unless last vote is stray, ancestors.get(last_voted_slot) must
+                            // Unless last vote is stray and stale, ancestors.get(last_voted_slot) must
                             // return Some(_), justifying to panic! here.
                             // Also, adjust_lockouts_after_replay() correctly makes last_voted_slot None,
                             // if all saved votes are ancestors of replayed_root_slot. So this code shouldn't be
@@ -520,10 +582,7 @@ impl Tower {
                             // all of them.
                             panic!("no ancestors found with slot: {}", last_voted_slot);
                         } else {
-                            // bank_forks doesn't have corresponding data for the stray restored last vote,
+                            empty_ancestors_due_to_minor_unsynced_ledger()
-                            // meaning some inconsistency between saved tower and ledger.
-                            // (newer snapshot, or only a saved tower is moved over to new setup?)
-                            &empty_ancestors
                         }
                     });
 
@@ -532,12 +591,21 @@ impl Tower {
                 if switch_slot == last_voted_slot || switch_slot_ancestors.contains(&last_voted_slot) {
                     // If the `switch_slot is a descendant of the last vote,
                     // no switching proof is necessary
-                    return SwitchForkDecision::NoSwitch;
+                    return SwitchForkDecision::SameFork;
                 }
 
-                // Should never consider switching to an ancestor
+                if last_vote_ancestors.contains(&switch_slot) {
-                // of your last vote
+                    if !self.is_stray_last_vote() {
-                assert!(!last_vote_ancestors.contains(&switch_slot));
+                        panic!(
+                            "Should never consider switching to slot ({}), which is ancestors({:?}) of last vote: {}",
+                            switch_slot,
+                            last_vote_ancestors,
+                            last_voted_slot
+                        );
+                    } else {
+                        return suspended_decision_due_to_major_unsynced_ledger();
+                    }
+                }
 
                 // By this point, we know the `switch_slot` is on a different fork
                 // (is neither an ancestor nor descendant of `last_vote`), so a
@@ -598,7 +666,7 @@ impl Tower {
                             }
 
                             // Only count lockouts on slots that are:
-                            // 1) Not ancestors of `last_vote`
+                            // 1) Not ancestors of `last_vote`, meaning being on different fork
                             // 2) Not from before the current root as we can't determine if
                             // anything before the root was an ancestor of `last_vote` or not
                             if !last_vote_ancestors.contains(lockout_interval_start)
@@ -622,10 +690,43 @@ impl Tower {
                 if (locked_out_stake as f64 / total_stake as f64) > SWITCH_FORK_THRESHOLD {
                     SwitchForkDecision::SwitchProof(switch_proof)
                 } else {
-                    SwitchForkDecision::FailedSwitchThreshold
+                    SwitchForkDecision::FailedSwitchThreshold(locked_out_stake, total_stake)
                 }
             })
-            .unwrap_or(SwitchForkDecision::NoSwitch)
+            .unwrap_or(SwitchForkDecision::SameFork)
+    }
+
+    pub(crate) fn check_switch_threshold(
+        &mut self,
+        switch_slot: u64,
+        ancestors: &HashMap<Slot, HashSet<u64>>,
+        descendants: &HashMap<Slot, HashSet<u64>>,
+        progress: &ProgressMap,
+        total_stake: u64,
+        epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
+    ) -> SwitchForkDecision {
+        let decision = self.make_check_switch_threshold_decision(
+            switch_slot,
+            ancestors,
+            descendants,
+            progress,
+            total_stake,
+            epoch_vote_accounts,
+        );
+        let new_check = Some((switch_slot, decision.clone()));
+        if new_check != self.last_switch_threshold_check {
+            trace!(
+                "new switch threshold check: slot {}: {:?}",
+                switch_slot,
+                decision,
+            );
+            self.last_switch_threshold_check = new_check;
+        }
+        decision
+    }
+
+    fn is_first_switch_check(&self) -> bool {
+        self.last_switch_threshold_check.is_none()
     }
 
     pub fn check_vote_stake_threshold(
@@ -760,28 +861,21 @@ impl Tower {
     // The tower root can be older/newer if the validator booted from a newer/older snapshot, so
     // tower lockouts may need adjustment
     pub fn adjust_lockouts_after_replay(
-        self,
+        mut self,
         replayed_root: Slot,
         slot_history: &SlotHistory,
     ) -> Result<Self> {
-        info!(
-            "adjusting lockouts (after replay up to {}): {:?}",
-            replayed_root,
-            self.voted_slots()
-        );
-
         // sanity assertions for roots
-        assert_eq!(slot_history.check(replayed_root), Check::Found);
+        let tower_root = self.root();
-        assert!(self.root().is_some());
+        info!(
-        let tower_root = self.root().unwrap();
+            "adjusting lockouts (after replay up to {}): {:?} tower root: {} replayed root: {}",
-        // reconcile_blockstore_roots_with_tower() should already have aligned these.
+            replayed_root,
-        assert!(
+            self.voted_slots(),
-            tower_root <= replayed_root,
+            tower_root,
-            format!(
+            replayed_root,
-                "tower root: {:?} >= replayed root slot: {}",
-                tower_root, replayed_root
-            )
         );
+        assert_eq!(slot_history.check(replayed_root), Check::Found);
+
         assert!(
             self.last_vote == Vote::default() && self.lockouts.votes.is_empty()
                 || self.last_vote != Vote::default() && !self.lockouts.votes.is_empty(),
@@ -791,12 +885,10 @@ impl Tower {
             )
         );
 
-        // return immediately if votes are empty...
+        if let Some(last_voted_slot) = self.last_voted_slot() {
-        if self.lockouts.votes.is_empty() {
+            if tower_root <= replayed_root {
-            return Ok(self);
+                // Normally, we goes into this clause with possible help of
-        }
+                // reconcile_blockstore_roots_with_tower()
-
-        let last_voted_slot = self.last_voted_slot().unwrap();
                 if slot_history.check(last_voted_slot) == Check::TooOld {
                     // We could try hard to anchor with other older votes, but opt to simplify the
                     // following logic
@@ -806,15 +898,54 @@ impl Tower {
                     ));
                 }
 
-        self.do_adjust_lockouts_after_replay(tower_root, replayed_root, slot_history)
+                self.adjust_lockouts_with_slot_history(slot_history)?;
+                self.initialize_root(replayed_root);
+            } else {
+                // This should never occur under normal operation.
+                // While this validator's voting is suspended this way,
+                // suspended_decision_due_to_major_unsynced_ledger() will be also touched.
+                let message = format!(
+                    "For some reason, we're REPROCESSING slots which has already been \
+                     voted and ROOTED by us; \
+                     VOTING will be SUSPENDED UNTIL {}!",
+                    last_voted_slot,
+                );
+                error!("{}", message);
+                datapoint_error!("tower_error", ("error", message, String));
+
+                // Let's pass-through adjust_lockouts_with_slot_history just for sanitization,
+                // using a synthesized SlotHistory.
+
+                let mut warped_slot_history = (*slot_history).clone();
+                // Blockstore doesn't have the tower_root slot because of
+                // (replayed_root < tower_root) in this else clause, meaning the tower is from
+                // the future from the view of blockstore.
+                // Pretend the blockstore has the future tower_root to anchor exactly with that
+                // slot by adding tower_root to a slot history. The added slot will be newer
+                // than all slots in the slot history (remember tower_root > replayed_root),
+                // satisfying the slot history invariant.
+                // Thus, the whole process will be safe as well because tower_root exists
+                // within both tower and slot history, guaranteeing the success of adjustment
+                // and retaining all of future votes correctly while sanitizing.
+                warped_slot_history.add(tower_root);
+
+                self.adjust_lockouts_with_slot_history(&warped_slot_history)?;
+                // don't update root; future tower's root should be kept across validator
+                // restarts to continue to show the scary messages at restarts until the next
+                // voting.
+            }
+        } else {
+            // This else clause is for newly created tower.
+            // initialize_lockouts_from_bank() should ensure the following invariant,
+            // otherwise we're screwing something up.
+            assert_eq!(tower_root, replayed_root);
         }
 
-    fn do_adjust_lockouts_after_replay(
+        Ok(self)
-        mut self,
+    }
-        tower_root: Slot,
+
-        replayed_root: Slot,
+    fn adjust_lockouts_with_slot_history(&mut self, slot_history: &SlotHistory) -> Result<()> {
-        slot_history: &SlotHistory,
+        let tower_root = self.root();
-    ) -> Result<Self> {
         // retained slots will be consisted only from divergent slots
         let mut retain_flags_for_each_vote_in_reverse: Vec<_> =
             Vec::with_capacity(self.lockouts.votes.len());
@@ -855,14 +986,20 @@ impl Tower {
             }
 
             if let Some(checked_slot) = checked_slot {
-                // This is really special, only if tower is initialized (root = slot 0) for genesis and contains
+                // This is really special, only if tower is initialized and contains
-                // a vote (= slot 0) for the genesis, the slot 0 can repeat only once
+                // a vote for the root, the root slot can repeat only once
-                let voting_from_genesis = *slot_in_tower == checked_slot && *slot_in_tower == 0;
+                let voting_for_root =
+                    *slot_in_tower == checked_slot && *slot_in_tower == tower_root;
 
-                if !voting_from_genesis {
+                if !voting_for_root {
                     // Unless we're voting since genesis, slots_in_tower must always be older than last checked_slot
                     // including all vote slot and the root slot.
-                    assert!(*slot_in_tower < checked_slot)
+                    assert!(
+                        *slot_in_tower < checked_slot,
+                        "slot_in_tower({}) < checked_slot({})",
+                        *slot_in_tower,
+                        checked_slot
+                    );
                 }
             }
 
@@ -890,15 +1027,10 @@ impl Tower {
             retain_flags_for_each_vote_in_reverse.into_iter().rev();
 
         let original_votes_len = self.lockouts.votes.len();
-        self.do_initialize_lockouts(replayed_root, move |_| {
+        self.initialize_lockouts(move |_| retain_flags_for_each_vote.next().unwrap());
-            retain_flags_for_each_vote.next().unwrap()
-        });
 
         if self.lockouts.votes.is_empty() {
-            info!(
+            info!("All restored votes were behind; resetting root_slot and last_vote in tower!");
-                "All restored votes were behind replayed_root({}); resetting root_slot and last_vote in tower!",
-                replayed_root
-            );
             // we might not have banks for those votes so just reset.
             // That's because the votes may well past replayed_root
             self.last_vote = Vote::default();
@@ -917,7 +1049,7 @@ impl Tower {
             self.stray_restored_slot = Some(self.last_vote.last_voted_slot().unwrap());
         }
 
-        Ok(self)
+        Ok(())
     }
 
     fn initialize_lockouts_from_bank(
@@ -930,18 +1062,19 @@ impl Tower {
             let vote_state = VoteState::deserialize(&vote_account.data)
                 .expect("vote_account isn't a VoteState?");
             self.lockouts = vote_state;
-            self.do_initialize_lockouts(root, |v| v.slot > root);
+            self.initialize_root(root);
+            self.initialize_lockouts(|v| v.slot > root);
             trace!(
-                "{} lockouts initialized to {:?}",
+                "Lockouts in tower for {} is initialized using bank {}",
                 self.node_pubkey,
-                self.lockouts
+                bank.slot(),
             );
             assert_eq!(
                 self.lockouts.node_pubkey, self.node_pubkey,
                 "vote account's node_pubkey doesn't match",
             );
         } else {
-            self.do_initialize_lockouts(root, |_| true);
+            self.initialize_root(root);
             info!(
                 "vote account({}) not found in bank (slot={})",
                 vote_account_pubkey,
@@ -950,11 +1083,14 @@ impl Tower {
         }
     }
 
-    fn do_initialize_lockouts<F: FnMut(&Lockout) -> bool>(&mut self, root: Slot, should_retain: F) {
+    fn initialize_lockouts<F: FnMut(&Lockout) -> bool>(&mut self, should_retain: F) {
+        self.lockouts.votes.retain(should_retain);
+    }
+
     // Updating root is needed to correctly restore from newly-saved tower for the next
     // boot
+    fn initialize_root(&mut self, root: Slot) {
         self.lockouts.root_slot = Some(root);
-        self.lockouts.votes.retain(should_retain);
     }
 
     pub fn get_filename(path: &Path, node_pubkey: &Pubkey) -> PathBuf {
@@ -986,6 +1122,7 @@ impl Tower {
             bincode::serialize_into(&mut file, &saved_tower)?;
             // file.sync_all() hurts performance; pipeline sync-ing and submitting votes to the cluster!
         }
+        trace!("persisted votes: {:?}", self.voted_slots());
         fs::rename(&new_filename, &filename)?;
         // self.path.parent().sync_all() hurts performance same as the above sync
 
@@ -1045,6 +1182,19 @@ pub enum TowerError {
 
     #[error("The tower is fatally inconsistent with blockstore: {0}")]
     FatallyInconsistent(&'static str),
+
+    #[error("The tower is useless because of new hard fork: {0}")]
+    HardFork(Slot),
+}
+
+impl TowerError {
+    pub fn is_file_missing(&self) -> bool {
+        if let TowerError::IOError(io_err) = &self {
+            io_err.kind() == std::io::ErrorKind::NotFound
+        } else {
+            false
+        }
+    }
 }
 
 #[frozen_abi(digest = "Gaxfwvx5MArn52mKZQgzHmDCyn5YfCuTHvp5Et3rFfpp")]
@@ -1070,13 +1220,16 @@ impl SavedTower {
     }
 }
 
-// Given an untimely crash, tower may have roots that are not reflected in blockstore because
+// Given an untimely crash, tower may have roots that are not reflected in blockstore,
-// `ReplayState::handle_votable_bank()` saves tower before setting blockstore roots
+// or the reverse of this.
+// That's because we don't impose any ordering guarantee or any kind of write barriers
+// between tower (plain old POSIX fs calls) and blockstore (through RocksDB), when
+// `ReplayState::handle_votable_bank()` saves tower before setting blockstore roots.
 pub fn reconcile_blockstore_roots_with_tower(
     tower: &Tower,
     blockstore: &Blockstore,
 ) -> blockstore_db::Result<()> {
-    if let Some(tower_root) = tower.root() {
+    let tower_root = tower.root();
     let last_blockstore_root = blockstore.last_root();
     if last_blockstore_root < tower_root {
         // Ensure tower_root itself to exist and be marked as rooted in the blockstore
@@ -1091,12 +1244,22 @@ pub fn reconcile_blockstore_roots_with_tower(
                 ),
             })
             .collect();
-            assert!(
+        if !new_roots.is_empty() {
-                !new_roots.is_empty(),
+            info!(
-                "at least 1 parent slot must be found"
+                "Reconciling slots as root based on tower root: {:?} ({}..{}) ",
+                new_roots, tower_root, last_blockstore_root
+            );
+            blockstore.set_roots(&new_roots)?;
+        } else {
+            // This indicates we're in bad state; but still don't panic here.
+            // That's because we might have a chance of recovering properly with
+            // newer snapshot.
+            warn!(
+                "Couldn't find any ancestor slots from tower root ({}) \
+                 towards blockstore root ({}); blockstore pruned or only \
+                 tower moved into new ledger?",
+                tower_root, last_blockstore_root,
             );
-
-            blockstore.set_roots(&new_roots)?
         }
     }
     Ok(())
@@ -1267,7 +1430,7 @@ pub mod test {
                 &ancestors,
                 &descendants,
                 &self.progress,
-                &tower,
+                tower,
             );
 
             // Make sure this slot isn't locked out or failing threshold
@@ -1456,7 +1619,7 @@ pub mod test {
                 &mut account.data,
             )
             .expect("serialize state");
-            stakes.push((Pubkey::new_rand(), (*lamports, account)));
+            stakes.push((solana_sdk::pubkey::new_rand(), (*lamports, account)));
         }
         stakes
     }
@@ -1464,11 +1627,11 @@ pub mod test {
     #[test]
     fn test_to_vote_instruction() {
         let vote = Vote::default();
-        let mut decision = SwitchForkDecision::FailedSwitchThreshold;
+        let mut decision = SwitchForkDecision::FailedSwitchThreshold(0, 1);
         assert!(decision
             .to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default())
             .is_none());
-        decision = SwitchForkDecision::NoSwitch;
+        decision = SwitchForkDecision::SameFork;
         assert_eq!(
             decision.to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default()),
             Some(vote_instruction::vote(
@@ -1571,7 +1734,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::NoSwitch
+            SwitchForkDecision::SameFork
         );
 
         // Trying to switch to another fork at 110 should fail
@@ -1584,7 +1747,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         // Adding another validator lockout on a descendant of last vote should
@@ -1599,7 +1762,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         // Adding another validator lockout on an ancestor of last vote should
@@ -1614,7 +1777,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         // Adding another validator lockout on a different fork, but the lockout
@@ -1629,7 +1792,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         // Adding another validator lockout on a different fork, and the lockout
@@ -1646,7 +1809,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         // Adding another validator lockout on a different fork, and the lockout
@@ -1697,7 +1860,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
     }
 
@@ -2365,7 +2528,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::NoSwitch
+            SwitchForkDecision::SameFork
         );
 
         // Trying to switch to another fork at 110 should fail
@@ -2378,7 +2541,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         vote_simulator.simulate_lockout_interval(111, (10, 49), &other_vote_account);
@@ -2456,7 +2619,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         // Add lockout_interval which should be excluded
@@ -2470,7 +2633,7 @@ pub mod test {
                 total_stake,
                 bank0.epoch_vote_accounts(0).unwrap(),
             ),
-            SwitchForkDecision::FailedSwitchThreshold
+            SwitchForkDecision::FailedSwitchThreshold(0, 20000)
         );
 
         // Add lockout_interval which should not be excluded
@@ -2619,8 +2782,7 @@ pub mod test {
     }
 
     #[test]
-    #[should_panic(expected = "at least 1 parent slot must be found")]
+    fn test_reconcile_blockstore_roots_with_tower_nop_no_parent() {
-    fn test_reconcile_blockstore_roots_with_tower_panic_no_parent() {
         solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
         {
@@ -2636,7 +2798,9 @@ pub mod test {
 
             let mut tower = Tower::new_with_key(&Pubkey::default());
             tower.lockouts.root_slot = Some(4);
+            assert_eq!(blockstore.last_root(), 0);
             reconcile_blockstore_roots_with_tower(&tower, &blockstore).unwrap();
+            assert_eq!(blockstore.last_root(), 0);
         }
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
@@ -2660,13 +2824,13 @@ pub mod test {
             .unwrap();
 
         assert_eq!(tower.voted_slots(), vec![2, 3]);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
 
         tower = tower
             .adjust_lockouts_after_replay(replayed_root_slot, &slot_history)
             .unwrap();
         assert_eq!(tower.voted_slots(), vec![2, 3]);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
     }
 
     #[test]
@@ -2688,7 +2852,7 @@ pub mod test {
             .unwrap();
 
         assert_eq!(tower.voted_slots(), vec![2, 3]);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
     }
 
     #[test]
@@ -2712,12 +2876,12 @@ pub mod test {
             .unwrap();
 
         assert_eq!(tower.voted_slots(), vec![] as Vec<Slot>);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
         assert_eq!(tower.stray_restored_slot, None);
     }
 
     #[test]
-    fn test_adjust_lockouts_after_relay_all_rooted_with_too_old() {
+    fn test_adjust_lockouts_after_replay_all_rooted_with_too_old() {
         use solana_sdk::slot_history::MAX_ENTRIES;
 
         let mut tower = Tower::new_for_tests(10, 0.9);
@@ -2735,7 +2899,7 @@ pub mod test {
             .adjust_lockouts_after_replay(MAX_ENTRIES, &slot_history)
             .unwrap();
         assert_eq!(tower.voted_slots(), vec![] as Vec<Slot>);
-        assert_eq!(tower.root(), Some(MAX_ENTRIES));
+        assert_eq!(tower.root(), MAX_ENTRIES);
     }
 
     #[test]
@@ -2758,7 +2922,7 @@ pub mod test {
             .unwrap();
 
         assert_eq!(tower.voted_slots(), vec![3, 4]);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
     }
 
     #[test]
@@ -2779,7 +2943,7 @@ pub mod test {
             .unwrap();
 
         assert_eq!(tower.voted_slots(), vec![5, 6]);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
     }
 
     #[test]
@@ -2823,7 +2987,7 @@ pub mod test {
             .unwrap();
 
         assert_eq!(tower.voted_slots(), vec![3, 4, 5]);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
     }
 
     #[test]
@@ -2839,7 +3003,7 @@ pub mod test {
             .unwrap();
 
         assert_eq!(tower.voted_slots(), vec![] as Vec<Slot>);
-        assert_eq!(tower.root(), Some(replayed_root_slot));
+        assert_eq!(tower.root(), replayed_root_slot);
     }
 
     #[test]
@@ -2920,4 +3084,92 @@ pub mod test {
             "The tower is fatally inconsistent with blockstore: not too old once after got too old?"
         );
     }
+
+    #[test]
+    #[should_panic(expected = "slot_in_tower(2) < checked_slot(1)")]
+    fn test_adjust_lockouts_after_replay_reversed_votes() {
+        let mut tower = Tower::new_for_tests(10, 0.9);
+        tower.lockouts.votes.push_back(Lockout::new(2));
+        tower.lockouts.votes.push_back(Lockout::new(1));
+        let vote = Vote::new(vec![1], Hash::default());
+        tower.last_vote = vote;
+
+        let mut slot_history = SlotHistory::default();
+        slot_history.add(0);
+        slot_history.add(2);
+
+        tower
+            .adjust_lockouts_after_replay(2, &slot_history)
+            .unwrap();
+    }
+
+    #[test]
+    #[should_panic(expected = "slot_in_tower(3) < checked_slot(3)")]
+    fn test_adjust_lockouts_after_replay_repeated_non_root_votes() {
+        let mut tower = Tower::new_for_tests(10, 0.9);
+        tower.lockouts.votes.push_back(Lockout::new(2));
+        tower.lockouts.votes.push_back(Lockout::new(3));
+        tower.lockouts.votes.push_back(Lockout::new(3));
+        let vote = Vote::new(vec![3], Hash::default());
+        tower.last_vote = vote;
+
+        let mut slot_history = SlotHistory::default();
+        slot_history.add(0);
+        slot_history.add(2);
+
+        tower
+            .adjust_lockouts_after_replay(2, &slot_history)
+            .unwrap();
+    }
+
+    #[test]
+    fn test_adjust_lockouts_after_replay_vote_on_root() {
+        let mut tower = Tower::new_for_tests(10, 0.9);
+        tower.lockouts.root_slot = Some(42);
+        tower.lockouts.votes.push_back(Lockout::new(42));
+        tower.lockouts.votes.push_back(Lockout::new(43));
+        tower.lockouts.votes.push_back(Lockout::new(44));
+        let vote = Vote::new(vec![44], Hash::default());
+        tower.last_vote = vote;
+
+        let mut slot_history = SlotHistory::default();
+        slot_history.add(42);
+
+        let tower = tower.adjust_lockouts_after_replay(42, &slot_history);
+        assert_eq!(tower.unwrap().voted_slots(), [43, 44]);
+    }
+
+    #[test]
+    fn test_adjust_lockouts_after_replay_vote_on_genesis() {
+        let mut tower = Tower::new_for_tests(10, 0.9);
+        tower.lockouts.votes.push_back(Lockout::new(0));
+        let vote = Vote::new(vec![0], Hash::default());
+        tower.last_vote = vote;
+
+        let mut slot_history = SlotHistory::default();
+        slot_history.add(0);
+
+        assert!(tower.adjust_lockouts_after_replay(0, &slot_history).is_ok());
+    }
+
+    #[test]
+    fn test_adjust_lockouts_after_replay_future_tower() {
+        let mut tower = Tower::new_for_tests(10, 0.9);
+        tower.lockouts.votes.push_back(Lockout::new(13));
+        tower.lockouts.votes.push_back(Lockout::new(14));
+        let vote = Vote::new(vec![14], Hash::default());
+        tower.last_vote = vote;
+        tower.initialize_root(12);
+
+        let mut slot_history = SlotHistory::default();
+        slot_history.add(0);
+        slot_history.add(2);
+
+        let tower = tower
+            .adjust_lockouts_after_replay(2, &slot_history)
+            .unwrap();
+        assert_eq!(tower.root(), 12);
+        assert_eq!(tower.voted_slots(), vec![13, 14]);
+        assert_eq!(tower.stray_restored_slot, Some(14));
+    }
 }

core/src/contact_info.rs

@@ -124,13 +124,21 @@ impl ContactInfo {
         }
     }
 
+    /// New random ContactInfo for tests and simulations.
+    pub(crate) fn new_rand<R: rand::Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
+        let delay = 10 * 60 * 1000; // 10 minutes
+        let now = timestamp() - delay + rng.gen_range(0, 2 * delay);
+        let pubkey = pubkey.unwrap_or_else(solana_sdk::pubkey::new_rand);
+        ContactInfo::new_localhost(&pubkey, now)
+    }
+
     #[cfg(test)]
     /// ContactInfo with multicast addresses for adversarial testing.
     pub fn new_multicast() -> Self {
         let addr = socketaddr!("224.0.1.255:1000");
         assert!(addr.ip().is_multicast());
         Self {
-            id: Pubkey::new_rand(),
+            id: solana_sdk::pubkey::new_rand(),
             gossip: addr,
             tvu: addr,
             tvu_forwards: addr,

core/src/crds.rs

@@ -24,12 +24,17 @@
 //! A value is updated to a new version if the labels match, and the value
 //! wallclock is later, or the value hash is greater.
 
+use crate::contact_info::ContactInfo;
 use crate::crds_shards::CrdsShards;
-use crate::crds_value::{CrdsValue, CrdsValueLabel};
+use crate::crds_value::{CrdsData, CrdsValue, CrdsValueLabel};
 use bincode::serialize;
-use indexmap::map::{Entry, IndexMap};
+use indexmap::map::{rayon::ParValues, Entry, IndexMap, Iter, Values};
+use indexmap::set::IndexSet;
+use rayon::{prelude::*, ThreadPool};
 use solana_sdk::hash::{hash, Hash};
 use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::Keypair;
+use solana_sdk::timing::timestamp;
 use std::cmp;
 use std::collections::HashMap;
 use std::ops::Index;
@@ -39,9 +44,11 @@ const CRDS_SHARDS_BITS: u32 = 8;
 #[derive(Clone)]
 pub struct Crds {
     /// Stores the map of labels and values
-    pub table: IndexMap<CrdsValueLabel, VersionedCrdsValue>,
+    table: IndexMap<CrdsValueLabel, VersionedCrdsValue>,
-    pub num_inserts: usize,
+    pub num_inserts: usize, // Only used in tests.
-    pub shards: CrdsShards,
+    shards: CrdsShards,
+    // Indices of all crds values which are node ContactInfo.
+    nodes: IndexSet<usize>,
 }
 
 #[derive(PartialEq, Debug)]
@@ -84,14 +91,22 @@ impl VersionedCrdsValue {
             value_hash,
         }
     }
+
+    /// New random VersionedCrdsValue for tests and simulations.
+    pub fn new_rand<R: rand::Rng>(rng: &mut R, keypair: Option<&Keypair>) -> Self {
+        let delay = 10 * 60 * 1000; // 10 minutes
+        let now = timestamp() - delay + rng.gen_range(0, 2 * delay);
+        Self::new(now, CrdsValue::new_rand(rng, keypair))
+    }
 }
 
 impl Default for Crds {
     fn default() -> Self {
         Crds {
-            table: IndexMap::new(),
+            table: IndexMap::default(),
             num_inserts: 0,
             shards: CrdsShards::new(CRDS_SHARDS_BITS),
+            nodes: IndexSet::default(),
         }
     }
 }
@@ -121,15 +136,19 @@ impl Crds {
         let label = new_value.value.label();
         match self.table.entry(label) {
             Entry::Vacant(entry) => {
-                assert!(self.shards.insert(entry.index(), &new_value));
+                let entry_index = entry.index();
+                self.shards.insert(entry_index, &new_value);
+                if let CrdsData::ContactInfo(_) = new_value.value.data {
+                    self.nodes.insert(entry_index);
+                }
                 entry.insert(new_value);
                 self.num_inserts += 1;
                 Ok(None)
             }
             Entry::Occupied(mut entry) if *entry.get() < new_value => {
                 let index = entry.index();
-                assert!(self.shards.remove(index, entry.get()));
+                self.shards.remove(index, entry.get());
-                assert!(self.shards.insert(index, &new_value));
+                self.shards.insert(index, &new_value);
                 self.num_inserts += 1;
                 Ok(Some(entry.insert(new_value)))
             }
@@ -159,6 +178,60 @@ impl Crds {
         self.table.get(label)
     }
 
+    pub fn get(&self, label: &CrdsValueLabel) -> Option<&VersionedCrdsValue> {
+        self.table.get(label)
+    }
+
+    pub fn get_contact_info(&self, pubkey: &Pubkey) -> Option<&ContactInfo> {
+        let label = CrdsValueLabel::ContactInfo(*pubkey);
+        self.table.get(&label)?.value.contact_info()
+    }
+
+    /// Returns all entries which are ContactInfo.
+    pub fn get_nodes(&self) -> impl Iterator<Item = &VersionedCrdsValue> {
+        self.nodes.iter().map(move |i| self.table.index(*i))
+    }
+
+    /// Returns ContactInfo of all known nodes.
+    pub fn get_nodes_contact_info(&self) -> impl Iterator<Item = &ContactInfo> {
+        self.get_nodes().map(|v| match &v.value.data {
+            CrdsData::ContactInfo(info) => info,
+            _ => panic!("this should not happen!"),
+        })
+    }
+
+    pub fn len(&self) -> usize {
+        self.table.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.table.is_empty()
+    }
+
+    pub fn iter(&self) -> Iter<'_, CrdsValueLabel, VersionedCrdsValue> {
+        self.table.iter()
+    }
+
+    pub fn values(&self) -> Values<'_, CrdsValueLabel, VersionedCrdsValue> {
+        self.table.values()
+    }
+
+    pub fn par_values(&self) -> ParValues<'_, CrdsValueLabel, VersionedCrdsValue> {
+        self.table.par_values()
+    }
+
+    /// Returns all crds values which the first 'mask_bits'
+    /// of their hash value is equal to 'mask'.
+    pub fn filter_bitmask(
+        &self,
+        mask: u64,
+        mask_bits: u32,
+    ) -> impl Iterator<Item = &VersionedCrdsValue> {
+        self.shards
+            .find(mask, mask_bits)
+            .map(move |i| self.table.index(i))
+    }
+
     fn update_label_timestamp(&mut self, id: &CrdsValueLabel, now: u64) {
         if let Some(e) = self.table.get_mut(id) {
             e.local_timestamp = cmp::max(e.local_timestamp, now);
@@ -167,8 +240,8 @@ impl Crds {
 
     /// Update the timestamp's of all the labels that are associated with Pubkey
     pub fn update_record_timestamp(&mut self, pubkey: &Pubkey, now: u64) {
-        for label in &CrdsValue::record_labels(pubkey) {
+        for label in CrdsValue::record_labels(*pubkey) {
-            self.update_label_timestamp(label, now);
+            self.update_label_timestamp(&label, now);
         }
     }
 
@@ -176,37 +249,51 @@ impl Crds {
     /// * timeouts - Pubkey specific timeouts with Pubkey::default() as the default timeout.
     pub fn find_old_labels(
         &self,
+        thread_pool: &ThreadPool,
         now: u64,
         timeouts: &HashMap<Pubkey, u64>,
     ) -> Vec<CrdsValueLabel> {
         let default_timeout = *timeouts
             .get(&Pubkey::default())
             .expect("must have default timeout");
+        thread_pool.install(|| {
             self.table
-            .iter()
+                .par_iter()
+                .with_min_len(1024)
                 .filter_map(|(k, v)| {
                     let timeout = timeouts.get(&k.pubkey()).unwrap_or(&default_timeout);
                     if v.local_timestamp.saturating_add(*timeout) <= now {
-                    Some(k)
+                        Some(k.clone())
                     } else {
                         None
                     }
                 })
-            .cloned()
                 .collect()
+        })
     }
 
-    pub fn remove(&mut self, key: &CrdsValueLabel) {
+    pub fn remove(&mut self, key: &CrdsValueLabel) -> Option<VersionedCrdsValue> {
-        if let Some((index, _, value)) = self.table.swap_remove_full(key) {
+        let (index, _, value) = self.table.swap_remove_full(key)?;
-            assert!(self.shards.remove(index, &value));
+        self.shards.remove(index, &value);
-            // The previously last element in the table is now moved to the
+        if let CrdsData::ContactInfo(_) = value.value.data {
-            // 'index' position. Shards need to be updated accordingly.
+            self.nodes.swap_remove(&index);
-            if index < self.table.len() {
+        }
+        // If index == self.table.len(), then the removed entry was the last
+        // entry in the table, in which case no other keys were modified.
+        // Otherwise, the previously last element in the table is now moved to
+        // the 'index' position; and so shards and nodes need to be updated
+        // accordingly.
+        let size = self.table.len();
+        if index < size {
             let value = self.table.index(index);
-                assert!(self.shards.remove(self.table.len(), value));
+            self.shards.remove(size, value);
-                assert!(self.shards.insert(index, value));
+            self.shards.insert(index, value);
+            if let CrdsData::ContactInfo(_) = value.value.data {
+                self.nodes.swap_remove(&size);
+                self.nodes.insert(index);
             }
         }
+        Some(value)
     }
 }
 
@@ -214,8 +301,8 @@ impl Crds {
 mod test {
     use super::*;
     use crate::contact_info::ContactInfo;
-    use crate::crds_value::CrdsData;
     use rand::{thread_rng, Rng};
+    use rayon::ThreadPoolBuilder;
 
     #[test]
     fn test_insert() {
@@ -288,48 +375,67 @@ mod test {
     }
     #[test]
     fn test_find_old_records_default() {
+        let thread_pool = ThreadPoolBuilder::new().build().unwrap();
         let mut crds = Crds::default();
         let val = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::default()));
         assert_eq!(crds.insert(val.clone(), 1), Ok(None));
         let mut set = HashMap::new();
         set.insert(Pubkey::default(), 0);
-        assert!(crds.find_old_labels(0, &set).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 0, &set).is_empty());
         set.insert(Pubkey::default(), 1);
-        assert_eq!(crds.find_old_labels(2, &set), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 2, &set),
+            vec![val.label()]
+        );
         set.insert(Pubkey::default(), 2);
-        assert_eq!(crds.find_old_labels(4, &set), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 4, &set),
+            vec![val.label()]
+        );
     }
     #[test]
     fn test_find_old_records_with_override() {
+        let thread_pool = ThreadPoolBuilder::new().build().unwrap();
         let mut rng = thread_rng();
         let mut crds = Crds::default();
         let mut timeouts = HashMap::new();
-        let val = CrdsValue::new_rand(&mut rng);
+        let val = CrdsValue::new_rand(&mut rng, None);
         timeouts.insert(Pubkey::default(), 3);
         assert_eq!(crds.insert(val.clone(), 0), Ok(None));
-        assert!(crds.find_old_labels(2, &timeouts).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 2, &timeouts).is_empty());
         timeouts.insert(val.pubkey(), 1);
-        assert_eq!(crds.find_old_labels(2, &timeouts), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 2, &timeouts),
+            vec![val.label()]
+        );
         timeouts.insert(val.pubkey(), u64::MAX);
-        assert!(crds.find_old_labels(2, &timeouts).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 2, &timeouts).is_empty());
         timeouts.insert(Pubkey::default(), 1);
-        assert!(crds.find_old_labels(2, &timeouts).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 2, &timeouts).is_empty());
         timeouts.remove(&val.pubkey());
-        assert_eq!(crds.find_old_labels(2, &timeouts), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 2, &timeouts),
+            vec![val.label()]
+        );
     }
     #[test]
     fn test_remove_default() {
+        let thread_pool = ThreadPoolBuilder::new().build().unwrap();
         let mut crds = Crds::default();
         let val = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::default()));
         assert_matches!(crds.insert(val.clone(), 1), Ok(_));
         let mut set = HashMap::new();
         set.insert(Pubkey::default(), 1);
-        assert_eq!(crds.find_old_labels(2, &set), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 2, &set),
+            vec![val.label()]
+        );
         crds.remove(&val.label());
-        assert!(crds.find_old_labels(2, &set).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 2, &set).is_empty());
     }
     #[test]
     fn test_find_old_records_staked() {
+        let thread_pool = ThreadPoolBuilder::new().build().unwrap();
         let mut crds = Crds::default();
         let val = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::default()));
         assert_eq!(crds.insert(val.clone(), 1), Ok(None));
@@ -337,20 +443,26 @@ mod test {
         //now < timestamp
         set.insert(Pubkey::default(), 0);
         set.insert(val.pubkey(), 0);
-        assert!(crds.find_old_labels(0, &set).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 0, &set).is_empty());
 
         //pubkey shouldn't expire since its timeout is MAX
         set.insert(val.pubkey(), std::u64::MAX);
-        assert!(crds.find_old_labels(2, &set).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 2, &set).is_empty());
 
         //default has max timeout, but pubkey should still expire
         set.insert(Pubkey::default(), std::u64::MAX);
         set.insert(val.pubkey(), 1);
-        assert_eq!(crds.find_old_labels(2, &set), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 2, &set),
+            vec![val.label()]
+        );
 
         set.insert(val.pubkey(), 2);
-        assert!(crds.find_old_labels(2, &set).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 2, &set).is_empty());
-        assert_eq!(crds.find_old_labels(3, &set), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 3, &set),
+            vec![val.label()]
+        );
     }
 
     #[test]
@@ -361,25 +473,29 @@ mod test {
         }
 
         let mut crds = Crds::default();
-        let pubkeys: Vec<_> = std::iter::repeat_with(Pubkey::new_rand).take(256).collect();
+        let keypairs: Vec<_> = std::iter::repeat_with(Keypair::new).take(256).collect();
         let mut rng = thread_rng();
         let mut num_inserts = 0;
+        let mut num_overrides = 0;
         for _ in 0..4096 {
-            let pubkey = pubkeys[rng.gen_range(0, pubkeys.len())];
+            let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
-            let value = VersionedCrdsValue::new(
+            let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
-                rng.gen(), // local_timestamp
+            match crds.insert_versioned(value) {
-                CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+                Ok(None) => {
-                    &pubkey,
-                    rng.gen(), // now
-                ))),
-            );
-            if crds.insert_versioned(value).is_ok() {
-                check_crds_shards(&crds);
                     num_inserts += 1;
+                    check_crds_shards(&crds);
+                }
+                Ok(Some(_)) => {
+                    num_inserts += 1;
+                    num_overrides += 1;
+                    check_crds_shards(&crds);
+                }
+                Err(_) => (),
             }
         }
         assert_eq!(num_inserts, crds.num_inserts);
         assert!(num_inserts > 700);
+        assert!(num_overrides > 500);
         assert!(crds.table.len() > 200);
         assert!(num_inserts > crds.table.len());
         check_crds_shards(&crds);
@@ -392,8 +508,58 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_crds_nodes() {
+        fn check_crds_nodes(crds: &Crds) -> usize {
+            let num_nodes = crds
+                .table
+                .values()
+                .filter(|value| matches!(value.value.data, CrdsData::ContactInfo(_)))
+                .count();
+            assert_eq!(num_nodes, crds.get_nodes_contact_info().count());
+            num_nodes
+        }
+        let mut rng = thread_rng();
+        let keypairs: Vec<_> = std::iter::repeat_with(Keypair::new).take(256).collect();
+        let mut crds = Crds::default();
+        let mut num_inserts = 0;
+        let mut num_overrides = 0;
+        for _ in 0..4096 {
+            let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
+            let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
+            match crds.insert_versioned(value) {
+                Ok(None) => {
+                    num_inserts += 1;
+                    check_crds_nodes(&crds);
+                }
+                Ok(Some(_)) => {
+                    num_inserts += 1;
+                    num_overrides += 1;
+                    check_crds_nodes(&crds);
+                }
+                Err(_) => (),
+            }
+        }
+        assert_eq!(num_inserts, crds.num_inserts);
+        assert!(num_inserts > 700);
+        assert!(num_overrides > 500);
+        assert!(crds.table.len() > 200);
+        assert!(num_inserts > crds.table.len());
+        let num_nodes = check_crds_nodes(&crds);
+        assert!(num_nodes * 3 < crds.table.len());
+        assert!(num_nodes > 150);
+        // Remove values one by one and assert that nodes indices stay valid.
+        while !crds.table.is_empty() {
+            let index = rng.gen_range(0, crds.table.len());
+            let key = crds.table.get_index(index).unwrap().0.clone();
+            crds.remove(&key);
+            check_crds_nodes(&crds);
+        }
+    }
+
     #[test]
     fn test_remove_staked() {
+        let thread_pool = ThreadPoolBuilder::new().build().unwrap();
         let mut crds = Crds::default();
         let val = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::default()));
         assert_matches!(crds.insert(val.clone(), 1), Ok(_));
@@ -402,9 +568,12 @@ mod test {
         //default has max timeout, but pubkey should still expire
         set.insert(Pubkey::default(), std::u64::MAX);
         set.insert(val.pubkey(), 1);
-        assert_eq!(crds.find_old_labels(2, &set), vec![val.label()]);
+        assert_eq!(
+            crds.find_old_labels(&thread_pool, 2, &set),
+            vec![val.label()]
+        );
         crds.remove(&val.label());
-        assert!(crds.find_old_labels(2, &set).is_empty());
+        assert!(crds.find_old_labels(&thread_pool, 2, &set).is_empty());
     }
 
     #[test]
@@ -484,14 +653,14 @@ mod test {
         let v1 = VersionedCrdsValue::new(
             1,
             CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-                &Pubkey::new_rand(),
+                &solana_sdk::pubkey::new_rand(),
                 0,
             ))),
         );
         let v2 = VersionedCrdsValue::new(
             1,
             CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
-                &Pubkey::new_rand(),
+                &solana_sdk::pubkey::new_rand(),
                 0,
             ))),
         );