patches bug in turbine's neighbors computation (#14565) (#14569)

Removing local node's index early from the set here:
https://github.com/solana-labs/solana/blob/e1b59ded4/core/src/retransmit_stage.rs#L346
distorts the order of nodes depending on which node is computing the
turbine fan-out tree, and results in incorrect neighbors computation.

(cherry picked from commit cfcca1cd3c)

Co-authored-by: behzad nouri <behzadnouri@gmail.com>

.buildkite/hooks/pre-command

@@ -31,4 +31,9 @@ export CARGO_TARGET_CACHE=$HOME/cargo-target-cache/"$CHANNEL"-"$BUILDKITE_LABEL"
 
   mkdir -p "$CARGO_TARGET_CACHE"/target
   rsync -a --delete --link-dest="$CARGO_TARGET_CACHE" "$CARGO_TARGET_CACHE"/target .
+
+  # Don't reuse BPF target build artifacts due to incremental build issues with
+  # `std:
+  #    "found possibly newer version of crate `std` which `xyz` depends on
+  rm -rf target/bpfel-unknown-unknown
 )

.gitignore

@@ -14,6 +14,8 @@
 
 /config/
 
+.cache
+
 # log files
 *.log
 log-*.txt

.mergify.yml

@@ -50,14 +50,6 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v1.2 backport
-    conditions:
-      - label=v1.2
-    actions:
-      backport:
-        ignore_conflicts: true
-        branches:
-          - v1.2
   - name: v1.3 backport
     conditions:
       - label=v1.3
@@ -74,3 +66,19 @@ pull_request_rules:
         ignore_conflicts: true
         branches:
           - v1.4
+  - name: v1.5 backport
+    conditions:
+      - label=v1.5
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.5
+  - name: v1.6 backport
+    conditions:
+      - label=v1.6
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.6

.travis.yml

@@ -34,6 +34,8 @@ jobs:
         - stable
       install:
         - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
       script:
         - source ci/env.sh
         - ci/publish-tarball.sh

Cargo.toml

@@ -16,6 +16,7 @@ members = [
     "dos",
     "download-utils",
     "faucet",
+    "frozen-abi",
     "perf",
     "validator",
     "genesis",
@@ -30,12 +31,14 @@ members = [
     "merkle-tree",
     "stake-o-matic",
     "storage-bigtable",
+    "storage-proto",
     "streamer",
     "measure",
     "metrics",
     "net-shaper",
     "notifier",
     "poh-bench",
+    "program-test",
     "programs/secp256k1",
     "programs/bpf_loader",
     "programs/budget",
@@ -50,7 +53,10 @@ members = [
     "remote-wallet",
     "ramp-tps",
     "runtime",
+    "runtime/store-tool",
     "sdk",
+    "sdk/cargo-build-bpf",
+    "sdk/cargo-test-bpf",
     "scripts",
     "stake-accounts",
     "stake-monitor",
@@ -61,7 +67,6 @@ members = [
     "upload-perf",
     "net-utils",
     "version",
-    "vote-signer",
     "cli",
     "rayon-threadlimit",
     "watchtower",

README.md

@@ -29,7 +29,7 @@ On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc
 
 ```bash
 $ sudo apt-get update
-$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang
+$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang make
 ```
 
 ## **2. Download the source code.**
@@ -61,8 +61,9 @@ $ cargo test
 ### Starting a local testnet
 Start your own testnet locally, instructions are in the [online docs](https://docs.solana.com/cluster/bench-tps).
 
-### Accessing the remote testnet
-* `testnet` - public stable testnet accessible via devnet.solana.com. Runs 24/7
+### Accessing the remote development cluster
+* `devnet` - stable public cluster for development accessible via
+devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://docs.solana.com/clusters)
 
 # Benchmarking
 
@@ -107,3 +108,5 @@ send us that patch!
 # Disclaimer
 
 All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
+
+Any content produced by Solana, or developer resources that Solana provides, are for educational and inspiration purposes only.  Solana does not encourage, induce or sanction the deployment of any such applications in violation of applicable laws or regulations.

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.4.0"
+version = "1.5.3"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,12 +18,13 @@ lazy_static = "1.4.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=2.0.6", features = ["skip-no-mangle"] }
+solana-config-program = { path = "../programs/config", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.3" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.3" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
+zstd = "0.5.1"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -12,9 +12,14 @@ pub mod parse_token;
 pub mod parse_vote;
 pub mod validator_info;
 
-use crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount};
-use solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey};
-use std::str::FromStr;
+use {
+    crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
+    solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey},
+    std::{
+        io::{Read, Write},
+        str::FromStr,
+    },
+};
 
 pub type StringAmount = String;
 
@@ -44,6 +49,8 @@ pub enum UiAccountEncoding {
     Base58,
     Base64,
     JsonParsed,
+    #[serde(rename = "base64+zstd")]
+    Base64Zstd,
 }
 
 impl UiAccount {
@@ -66,6 +73,19 @@ impl UiAccount {
                 base64::encode(slice_data(&account.data, data_slice_config)),
                 encoding,
             ),
+            UiAccountEncoding::Base64Zstd => {
+                let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
+                match encoder
+                    .write_all(slice_data(&account.data, data_slice_config))
+                    .and_then(|()| encoder.finish())
+                {
+                    Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
+                    Err(_) => UiAccountData::Binary(
+                        base64::encode(slice_data(&account.data, data_slice_config)),
+                        UiAccountEncoding::Base64,
+                    ),
+                }
+            }
             UiAccountEncoding::JsonParsed => {
                 if let Ok(parsed_data) =
                     parse_account_data(pubkey, &account.owner, &account.data, additional_data)
@@ -92,6 +112,16 @@ impl UiAccount {
             UiAccountData::Binary(blob, encoding) => match encoding {
                 UiAccountEncoding::Base58 => bs58::decode(blob).into_vec().ok(),
                 UiAccountEncoding::Base64 => base64::decode(blob).ok(),
+                UiAccountEncoding::Base64Zstd => base64::decode(blob)
+                    .ok()
+                    .map(|zstd_data| {
+                        let mut data = vec![];
+                        zstd::stream::read::Decoder::new(zstd_data.as_slice())
+                            .and_then(|mut reader| reader.read_to_end(&mut data))
+                            .map(|_| data)
+                            .ok()
+                    })
+                    .flatten(),
                 UiAccountEncoding::Binary | UiAccountEncoding::JsonParsed => None,
             },
         }?;
@@ -179,4 +209,25 @@ mod test {
         });
         assert_eq!(slice_data(&data, slice_config), &[] as &[u8]);
     }
+
+    #[test]
+    fn test_base64_zstd() {
+        let encoded_account = UiAccount::encode(
+            &Pubkey::default(),
+            Account {
+                data: vec![0; 1024],
+                ..Account::default()
+            },
+            UiAccountEncoding::Base64Zstd,
+            None,
+            None,
+        );
+        assert!(matches!(
+            encoded_account.data,
+            UiAccountData::Binary(_, UiAccountEncoding::Base64Zstd)
+        ));
+
+        let decoded_account = encoded_account.decode().unwrap();
+        assert_eq!(decoded_account.data, vec![0; 1024]);
+    }
 }

account-decoder/src/parse_account_data.rs

@@ -81,7 +81,7 @@ pub fn parse_account_data(
 ) -> Result<ParsedAccount, ParseAccountError> {
     let program_name = PARSABLE_PROGRAM_IDS
         .get(program_id)
-        .ok_or_else(|| ParseAccountError::ProgramNotParsable)?;
+        .ok_or(ParseAccountError::ProgramNotParsable)?;
     let additional_data = additional_data.unwrap_or_default();
     let parsed_json = match program_name {
         ParsableAccount::Config => serde_json::to_value(parse_config(data, pubkey)?)?,
@@ -111,14 +111,14 @@ mod test {
 
     #[test]
     fn test_parse_account_data() {
-        let account_pubkey = Pubkey::new_rand();
-        let other_program = Pubkey::new_rand();
+        let account_pubkey = solana_sdk::pubkey::new_rand();
+        let other_program = solana_sdk::pubkey::new_rand();
         let data = vec![0; 4];
         assert!(parse_account_data(&account_pubkey, &other_program, &data, None).is_err());
 
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
         let parsed = parse_account_data(
             &account_pubkey,

account-decoder/src/parse_config.rs

@@ -117,7 +117,7 @@ mod test {
             }))
             .unwrap(),
         };
-        let info_pubkey = Pubkey::new_rand();
+        let info_pubkey = solana_sdk::pubkey::new_rand();
         let validator_info_config_account = create_config_account(
             vec![(validator_info::id(), false), (info_pubkey, true)],
             &validator_info,

account-decoder/src/parse_stake.rs

@@ -134,7 +134,6 @@ impl From<Delegation> for UiDelegation {
 mod test {
     use super::*;
     use bincode::serialize;
-    use solana_sdk::pubkey::Pubkey;
 
     #[test]
     fn test_parse_stake() {
@@ -145,8 +144,8 @@ mod test {
             StakeAccountType::Uninitialized
         );
 
-        let pubkey = Pubkey::new_rand();
-        let custodian = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         let authorized = Authorized::auto(&pubkey);
         let lockup = Lockup {
             unix_timestamp: 0,
@@ -180,7 +179,7 @@ mod test {
             })
         );
 
-        let voter_pubkey = Pubkey::new_rand();
+        let voter_pubkey = solana_sdk::pubkey::new_rand();
         let stake = Stake {
             delegation: Delegation {
                 voter_pubkey,

account-decoder/src/parse_sysvar.rs

@@ -105,6 +105,7 @@ pub enum SysvarAccountType {
 pub struct UiClock {
     pub slot: Slot,
     pub epoch: Epoch,
+    pub epoch_start_timestamp: UnixTimestamp,
     pub leader_schedule_epoch: Epoch,
     pub unix_timestamp: UnixTimestamp,
 }
@@ -114,6 +115,7 @@ impl From<Clock> for UiClock {
         Self {
             slot: clock.slot,
             epoch: clock.epoch,
+            epoch_start_timestamp: clock.epoch_start_timestamp,
             leader_schedule_epoch: clock.leader_schedule_epoch,
             unix_timestamp: clock.unix_timestamp,
         }
@@ -212,15 +214,13 @@ pub struct UiStakeHistoryEntry {
 mod test {
     use super::*;
     use solana_sdk::{
-        fee_calculator::FeeCalculator,
-        hash::Hash,
-        sysvar::{recent_blockhashes::IterItem, Sysvar},
+        account::create_account, fee_calculator::FeeCalculator, hash::Hash,
+        sysvar::recent_blockhashes::IterItem,
     };
-    use std::iter::FromIterator;
 
     #[test]
     fn test_parse_sysvars() {
-        let clock_sysvar = Clock::default().create_account(1);
+        let clock_sysvar = create_account(&Clock::default(), 1);
         assert_eq!(
             parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
             SysvarAccountType::Clock(UiClock::default()),
@@ -233,13 +233,13 @@ mod test {
             first_normal_epoch: 1,
             first_normal_slot: 12,
         };
-        let epoch_schedule_sysvar = epoch_schedule.create_account(1);
+        let epoch_schedule_sysvar = create_account(&epoch_schedule, 1);
         assert_eq!(
             parse_sysvar(&epoch_schedule_sysvar.data, &sysvar::epoch_schedule::id()).unwrap(),
             SysvarAccountType::EpochSchedule(epoch_schedule),
         );
 
-        let fees_sysvar = Fees::default().create_account(1);
+        let fees_sysvar = create_account(&Fees::default(), 1);
         assert_eq!(
             parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
             SysvarAccountType::Fees(UiFees::default()),
@@ -249,9 +249,10 @@ mod test {
         let fee_calculator = FeeCalculator {
             lamports_per_signature: 10,
         };
-        let recent_blockhashes =
-            RecentBlockhashes::from_iter(vec![IterItem(0, &hash, &fee_calculator)].into_iter());
-        let recent_blockhashes_sysvar = recent_blockhashes.create_account(1);
+        let recent_blockhashes: RecentBlockhashes = vec![IterItem(0, &hash, &fee_calculator)]
+            .into_iter()
+            .collect();
+        let recent_blockhashes_sysvar = create_account(&recent_blockhashes, 1);
         assert_eq!(
             parse_sysvar(
                 &recent_blockhashes_sysvar.data,
@@ -269,13 +270,13 @@ mod test {
             exemption_threshold: 2.0,
             burn_percent: 5,
         };
-        let rent_sysvar = rent.create_account(1);
+        let rent_sysvar = create_account(&rent, 1);
         assert_eq!(
             parse_sysvar(&rent_sysvar.data, &sysvar::rent::id()).unwrap(),
             SysvarAccountType::Rent(rent.into()),
         );
 
-        let rewards_sysvar = Rewards::default().create_account(1);
+        let rewards_sysvar = create_account(&Rewards::default(), 1);
         assert_eq!(
             parse_sysvar(&rewards_sysvar.data, &sysvar::rewards::id()).unwrap(),
             SysvarAccountType::Rewards(UiRewards::default()),
@@ -283,7 +284,7 @@ mod test {
 
         let mut slot_hashes = SlotHashes::default();
         slot_hashes.add(1, hash);
-        let slot_hashes_sysvar = slot_hashes.create_account(1);
+        let slot_hashes_sysvar = create_account(&slot_hashes, 1);
         assert_eq!(
             parse_sysvar(&slot_hashes_sysvar.data, &sysvar::slot_hashes::id()).unwrap(),
             SysvarAccountType::SlotHashes(vec![UiSlotHashEntry {
@@ -294,7 +295,7 @@ mod test {
 
         let mut slot_history = SlotHistory::default();
         slot_history.add(42);
-        let slot_history_sysvar = slot_history.create_account(1);
+        let slot_history_sysvar = create_account(&slot_history, 1);
         assert_eq!(
             parse_sysvar(&slot_history_sysvar.data, &sysvar::slot_history::id()).unwrap(),
             SysvarAccountType::SlotHistory(UiSlotHistory {
@@ -310,7 +311,7 @@ mod test {
             deactivating: 3,
         };
         stake_history.add(1, stake_history_entry.clone());
-        let stake_history_sysvar = stake_history.create_account(1);
+        let stake_history_sysvar = create_account(&stake_history, 1);
         assert_eq!(
             parse_sysvar(&stake_history_sysvar.data, &sysvar::stake_history::id()).unwrap(),
             SysvarAccountType::StakeHistory(vec![UiStakeHistoryEntry {
@@ -319,7 +320,7 @@ mod test {
             }]),
         );
 
-        let bad_pubkey = Pubkey::new_rand();
+        let bad_pubkey = solana_sdk::pubkey::new_rand();
         assert!(parse_sysvar(&stake_history_sysvar.data, &bad_pubkey).is_err());
 
         let bad_data = vec![0; 4];

account-decoder/src/parse_token.rs

@@ -4,7 +4,9 @@ use crate::{
 };
 use solana_sdk::pubkey::Pubkey;
 use spl_token_v2_0::{
-    solana_sdk::{program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey},
+    solana_program::{
+        program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
+    },
     state::{Account, AccountState, Mint, Multisig},
 };
 use std::str::FromStr;
@@ -21,6 +23,16 @@ pub fn spl_token_v2_0_native_mint() -> Pubkey {
     Pubkey::from_str(&spl_token_v2_0::native_mint::id().to_string()).unwrap()
 }
 
+// A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
+pub fn spl_token_v2_0_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
+    SplTokenPubkey::from_str(&pubkey.to_string()).unwrap()
+}
+
+// A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
+pub fn pubkey_from_spl_token_v2_0(pubkey: &SplTokenPubkey) -> Pubkey {
+    Pubkey::from_str(&pubkey.to_string()).unwrap()
+}
+
 pub fn parse_token(
     data: &[u8],
     mint_decimals: Option<u8>,

account-decoder/src/parse_vote.rs

@@ -128,11 +128,13 @@ mod test {
     fn test_parse_vote() {
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
-        let mut expected_vote_state = UiVoteState::default();
-        expected_vote_state.node_pubkey = Pubkey::default().to_string();
-        expected_vote_state.authorized_withdrawer = Pubkey::default().to_string();
+        let expected_vote_state = UiVoteState {
+            node_pubkey: Pubkey::default().to_string(),
+            authorized_withdrawer: Pubkey::default().to_string(),
+            ..UiVoteState::default()
+        };
         assert_eq!(
             parse_vote(&vote_account_data).unwrap(),
             VoteAccountType::Vote(expected_vote_state)

accounts-bench/Cargo.toml

@@ -2,20 +2,20 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-log = "0.4.6"
+log = "0.4.11"
 rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-measure = { path = "../measure", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.5.3" }
+solana-runtime = { path = "../runtime", version = "1.5.3" }
+solana-measure = { path = "../measure", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -2,13 +2,11 @@ use clap::{crate_description, crate_name, value_t, App, Arg};
 use rayon::prelude::*;
 use solana_measure::measure::Measure;
 use solana_runtime::{
-    accounts::{create_test_accounts, update_accounts, Accounts},
+    accounts::{create_test_accounts, update_accounts_bench, Accounts},
     accounts_index::Ancestors,
 };
 use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
-use std::env;
-use std::fs;
-use std::path::PathBuf;
+use std::{collections::HashSet, env, fs, path::PathBuf};
 
 fn main() {
     solana_logger::setup();
@@ -56,7 +54,8 @@ fn main() {
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
-    let accounts = Accounts::new(vec![path], &ClusterType::Testnet);
+    let accounts =
+        Accounts::new_with_config(vec![path], &ClusterType::Testnet, HashSet::new(), false);
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");
     let pubkeys: Vec<_> = (0..num_slots)
@@ -92,13 +91,16 @@ fn main() {
             time.stop();
             println!("{}", time);
             for slot in 0..num_slots {
-                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
+                update_accounts_bench(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
                 accounts.add_root((x * num_slots + slot) as u64);
             }
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors).0;
+            let hash = accounts
+                .accounts_db
+                .update_accounts_hash(0, &ancestors, true)
+                .0;
             time.stop();
             println!("hash: {} {}", hash, time);
             create_test_accounts(&accounts, &mut pubkeys, 1, 0);

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,19 +11,19 @@ publish = false
 [dependencies]
 clap = "2.33.1"
 crossbeam-channel = "0.4"
-log = "0.4.6"
+log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.4.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
-solana-perf = { path = "../perf", version = "1.4.0" }
-solana-ledger = { path = "../ledger", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-measure = { path = "../measure", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-core = { path = "../core", version = "1.5.3" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-streamer = { path = "../streamer", version = "1.5.3" }
+solana-perf = { path = "../perf", version = "1.5.3" }
+solana-ledger = { path = "../ledger", version = "1.5.3" }
+solana-logger = { path = "../logger", version = "1.5.3" }
+solana-runtime = { path = "../runtime", version = "1.5.3" }
+solana-measure = { path = "../measure", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -17,10 +17,11 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_perf::packet::to_packets_chunked;
-use solana_runtime::{bank::Bank, bank_forks::BankForks};
+use solana_runtime::{
+    accounts_background_service::ABSRequestSender, bank::Bank, bank_forks::BankForks,
+};
 use solana_sdk::{
     hash::Hash,
-    pubkey::Pubkey,
     signature::Keypair,
     signature::Signature,
     system_transaction,
@@ -69,7 +70,7 @@ fn make_accounts_txs(
     hash: Hash,
     same_payer: bool,
 ) -> Vec<Transaction> {
-    let to_pubkey = Pubkey::new_rand();
+    let to_pubkey = solana_sdk::pubkey::new_rand();
     let payer_key = Keypair::new();
     let dummy = system_transaction::transfer(&payer_key, &to_pubkey, 1, hash);
     (0..total_num_transactions)
@@ -78,9 +79,9 @@ fn make_accounts_txs(
             let mut new = dummy.clone();
             let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
             if !same_payer {
-                new.message.account_keys[0] = Pubkey::new_rand();
+                new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
             }
-            new.message.account_keys[1] = Pubkey::new_rand();
+            new.message.account_keys[1] = solana_sdk::pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
         })
@@ -241,7 +242,7 @@ fn main() {
         let base_tx_count = bank.transaction_count();
         let mut txs_processed = 0;
         let mut root = 1;
-        let collector = Pubkey::new_rand();
+        let collector = solana_sdk::pubkey::new_rand();
         let config = Config {
             packets_per_batch: packets_per_chunk,
             chunk_len,
@@ -324,7 +325,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None, None);
+                    bank_forks.set_root(root, &ABSRequestSender::default(), None);
                     root += 1;
                 }
                 debug!(

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.4.0"
+version = "1.5.3"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,18 +9,18 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-async-trait = "0.1.36"
 bincode = "1.3.1"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-tarpc = { version = "0.22.0", features = ["full"] }
-tokio = "0.2"
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+tarpc = { version = "0.23.0", features = ["full"] }
+tokio = { version = "0.3.5", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-banks-server = { path = "../banks-server", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "1.5.3" }
+solana-banks-server = { path = "../banks-server", version = "1.5.3" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -5,178 +5,250 @@
 //! but they are undocumented, may change over time, and are generally more
 //! cumbersome to use.
 
-use async_trait::async_trait;
-use futures::future::join_all;
-pub use solana_banks_interface::{BanksClient, TransactionStatus};
+use futures::{future::join_all, Future, FutureExt};
+pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
 use solana_banks_interface::{BanksRequest, BanksResponse};
 use solana_sdk::{
-    account::Account, clock::Slot, commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature,
-    transaction::Transaction, transport,
+    account::{from_account, Account},
+    clock::Slot,
+    commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    rent::Rent,
+    signature::Signature,
+    sysvar,
+    transaction::{self, Transaction},
+    transport,
 };
 use std::io::{self, Error, ErrorKind};
 use tarpc::{
-    client, context,
-    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
+    client::{self, channel::RequestDispatch, NewClient},
+    context::{self, Context},
+    rpc::{ClientMessage, Response},
     serde_transport::tcp,
+    Transport,
 };
 use tokio::{net::ToSocketAddrs, time::Duration};
 use tokio_serde::formats::Bincode;
 
-#[async_trait]
-pub trait BanksClientExt {
+// This exists only for backward compatibility
+pub trait BanksClientExt {}
+
+#[derive(Clone)]
+pub struct BanksClient {
+    inner: TarpcClient,
+}
+
+impl BanksClient {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new<C>(
+        config: client::Config,
+        transport: C,
+    ) -> NewClient<TarpcClient, RequestDispatch<BanksRequest, BanksResponse, C>>
+    where
+        C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>>,
+    {
+        TarpcClient::new(config, transport)
+    }
+
+    pub fn send_transaction_with_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+    ) -> impl Future<Output = io::Result<()>> + '_ {
+        self.inner.send_transaction_with_context(ctx, transaction)
+    }
+
+    pub fn get_fees_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
+        self.inner
+            .get_fees_with_commitment_and_context(ctx, commitment)
+    }
+
+    pub fn get_transaction_status_with_context(
+        &mut self,
+        ctx: Context,
+        signature: Signature,
+    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+        self.inner
+            .get_transaction_status_with_context(ctx, signature)
+    }
+
+    pub fn get_slot_with_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.inner.get_slot_with_context(ctx, commitment)
+    }
+
+    pub fn process_transaction_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<transaction::Result<()>>>> + '_ {
+        self.inner
+            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+    }
+
+    pub fn get_account_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.inner
+            .get_account_with_commitment_and_context(ctx, address, commitment)
+    }
+
     /// Send a transaction and return immediately. The server will resend the
     /// transaction until either it is accepted by the cluster or the transaction's
     /// blockhash expires.
-    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()>;
-
-    /// Return a recent, rooted blockhash from the server. The cluster will only accept
-    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
-    /// method to get both a blockhash and the blockhash's last valid slot.
-    async fn get_recent_blockhash(&mut self) -> io::Result<Hash>;
+    pub fn send_transaction(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = io::Result<()>> + '_ {
+        self.send_transaction_with_context(context::current(), transaction)
+    }
 
     /// Return the fee parameters associated with a recent, rooted blockhash. The cluster
     /// will use the transaction's blockhash to look up these same fee parameters and
     /// use them to calculate the transaction fee.
-    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)>;
+    pub fn get_fees(
+        &mut self,
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
+        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::Root)
+    }
+
+    /// Return the cluster rent
+    pub fn get_rent(&mut self) -> impl Future<Output = io::Result<Rent>> + '_ {
+        self.get_account(sysvar::rent::id()).map(|result| {
+            let rent_sysvar = result?
+                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Rent sysvar not present"))?;
+            from_account::<Rent>(&rent_sysvar).ok_or_else(|| {
+                io::Error::new(io::ErrorKind::Other, "Failed to deserialize Rent sysvar")
+            })
+        })
+    }
+
+    /// Return a recent, rooted blockhash from the server. The cluster will only accept
+    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
+    /// method to get both a blockhash and the blockhash's last valid slot.
+    pub fn get_recent_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
+        self.get_fees().map(|result| Ok(result?.1))
+    }
 
     /// Send a transaction and return after the transaction has been rejected or
     /// reached the given level of commitment.
-    async fn process_transaction_with_commitment(
+    pub fn process_transaction_with_commitment(
         &mut self,
         transaction: Transaction,
         commitment: CommitmentLevel,
-    ) -> transport::Result<()>;
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        self.process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .map(|result| match result? {
+                None => {
+                    Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into())
+                }
+                Some(transaction_result) => Ok(transaction_result?),
+            })
+    }
 
-    /// Send a transaction and return after the transaction has been finalized or rejected.
-    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()>;
+    /// Send a transaction and return until the transaction has been finalized or rejected.
+    pub fn process_transaction(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
+    }
+
+    pub async fn process_transactions_with_commitment(
+        &mut self,
+        transactions: Vec<Transaction>,
+        commitment: CommitmentLevel,
+    ) -> transport::Result<()> {
+        let mut clients: Vec<_> = transactions.iter().map(|_| self.clone()).collect();
+        let futures = clients
+            .iter_mut()
+            .zip(transactions)
+            .map(|(client, transaction)| {
+                client.process_transaction_with_commitment(transaction, commitment)
+            });
+        let statuses = join_all(futures).await;
+        statuses.into_iter().collect() // Convert Vec<Result<_, _>> to Result<Vec<_>>
+    }
+
+    /// Send transactions and return until the transaction has been finalized or rejected.
+    pub fn process_transactions(
+        &mut self,
+        transactions: Vec<Transaction>,
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        self.process_transactions_with_commitment(transactions, CommitmentLevel::default())
+    }
+
+    /// Return the most recent rooted slot height. All transactions at or below this height
+    /// are said to be finalized. The cluster will not fork to a higher slot height.
+    pub fn get_root_slot(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.get_slot_with_context(context::current(), CommitmentLevel::Root)
+    }
+
+    /// Return the account at the given address at the slot corresponding to the given
+    /// commitment level. If the account is not found, None is returned.
+    pub fn get_account_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.get_account_with_commitment_and_context(context::current(), address, commitment)
+    }
+
+    /// Return the account at the given address at the time of the most recent root slot.
+    /// If the account is not found, None is returned.
+    pub fn get_account(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.get_account_with_commitment(address, CommitmentLevel::default())
+    }
+
+    /// Return the balance in lamports of an account at the given address at the slot
+    /// corresponding to the given commitment level.
+    pub fn get_balance_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<u64>> + '_ {
+        self.get_account_with_commitment_and_context(context::current(), address, commitment)
+            .map(|result| Ok(result?.map(|x| x.lamports).unwrap_or(0)))
+    }
+
+    /// Return the balance in lamports of an account at the given address at the time
+    /// of the most recent root slot.
+    pub fn get_balance(&mut self, address: Pubkey) -> impl Future<Output = io::Result<u64>> + '_ {
+        self.get_balance_with_commitment(address, CommitmentLevel::default())
+    }
 
     /// Return the status of a transaction with a signature matching the transaction's first
     /// signature. Return None if the transaction is not found, which may be because the
     /// blockhash was expired or the fee-paying account had insufficient funds to pay the
     /// transaction fee. Note that servers rarely store the full transaction history. This
     /// method may return None if the transaction status has been discarded.
-    async fn get_transaction_status(
+    pub fn get_transaction_status(
         &mut self,
         signature: Signature,
-    ) -> io::Result<Option<TransactionStatus>>;
+    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+        self.get_transaction_status_with_context(context::current(), signature)
+    }
 
     /// Same as get_transaction_status, but for multiple transactions.
-    async fn get_transaction_statuses(
-        &mut self,
-        signatures: Vec<Signature>,
-    ) -> io::Result<Vec<Option<TransactionStatus>>>;
-
-    /// Return the most recent rooted slot height. All transactions at or below this height
-    /// are said to be finalized. The cluster will not fork to a higher slot height.
-    async fn get_root_slot(&mut self) -> io::Result<Slot>;
-
-    /// Return the account at the given address at the slot corresponding to the given
-    /// commitment level. If the account is not found, None is returned.
-    async fn get_account_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<Option<Account>>;
-
-    /// Return the account at the given address at the time of the most recent root slot.
-    /// If the account is not found, None is returned.
-    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>>;
-
-    /// Return the balance in lamports of an account at the given address at the slot
-    /// corresponding to the given commitment level.
-    async fn get_balance_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<u64>;
-
-    /// Return the balance in lamports of an account at the given address at the time
-    /// of the most recent root slot.
-    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64>;
-}
-
-#[async_trait]
-impl BanksClientExt for BanksClient {
-    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()> {
-        self.send_transaction_with_context(context::current(), transaction)
-            .await
-    }
-
-    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)> {
-        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::Root)
-            .await
-    }
-
-    async fn get_recent_blockhash(&mut self) -> io::Result<Hash> {
-        Ok(self.get_fees().await?.1)
-    }
-
-    async fn process_transaction_with_commitment(
-        &mut self,
-        transaction: Transaction,
-        commitment: CommitmentLevel,
-    ) -> transport::Result<()> {
-        let mut ctx = context::current();
-        ctx.deadline += Duration::from_secs(50);
-        let result = self
-            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
-            .await?;
-        match result {
-            None => Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into()),
-            Some(transaction_result) => Ok(transaction_result?),
-        }
-    }
-
-    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()> {
-        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_root_slot(&mut self) -> io::Result<Slot> {
-        self.get_slot_with_context(context::current(), CommitmentLevel::Root)
-            .await
-    }
-
-    async fn get_account_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<Option<Account>> {
-        self.get_account_with_commitment_and_context(context::current(), address, commitment)
-            .await
-    }
-
-    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>> {
-        self.get_account_with_commitment(address, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_balance_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<u64> {
-        let account = self
-            .get_account_with_commitment_and_context(context::current(), address, commitment)
-            .await?;
-        Ok(account.map(|x| x.lamports).unwrap_or(0))
-    }
-
-    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64> {
-        self.get_balance_with_commitment(address, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_transaction_status(
-        &mut self,
-        signature: Signature,
-    ) -> io::Result<Option<TransactionStatus>> {
-        self.get_transaction_status_with_context(context::current(), signature)
-            .await
-    }
-
-    async fn get_transaction_statuses(
+    pub async fn get_transaction_statuses(
         &mut self,
         signatures: Vec<Signature>,
     ) -> io::Result<Vec<Option<TransactionStatus>>> {
@@ -197,15 +269,20 @@ impl BanksClientExt for BanksClient {
     }
 }
 
-pub async fn start_client(
-    transport: UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>>,
-) -> io::Result<BanksClient> {
-    BanksClient::new(client::Config::default(), transport).spawn()
+pub async fn start_client<C>(transport: C) -> io::Result<BanksClient>
+where
+    C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>> + Send + 'static,
+{
+    Ok(BanksClient {
+        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
+    })
 }
 
 pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
     let transport = tcp::connect(addr, Bincode::default).await?;
-    BanksClient::new(client::Config::default(), transport).spawn()
+    Ok(BanksClient {
+        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
+    })
 }
 
 #[cfg(test)]
@@ -213,10 +290,10 @@ mod tests {
     use super::*;
     use solana_banks_server::banks_server::start_local_server;
     use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
-    use solana_sdk::{message::Message, pubkey::Pubkey, signature::Signer, system_instruction};
+    use solana_sdk::{message::Message, signature::Signer, system_instruction};
     use std::sync::{Arc, RwLock};
     use tarpc::transport;
-    use tokio::{runtime::Runtime, time::delay_for};
+    use tokio::{runtime::Runtime, time::sleep};
 
     #[test]
     fn test_banks_client_new() {
@@ -235,15 +312,14 @@ mod tests {
             &genesis.genesis_config,
         ))));
 
-        let bob_pubkey = Pubkey::new_rand();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
         let mint_pubkey = genesis.mint_keypair.pubkey();
         let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
             let client_transport = start_local_server(&bank_forks).await;
-            let mut banks_client =
-                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+            let mut banks_client = start_client(client_transport).await?;
 
             let recent_blockhash = banks_client.get_recent_blockhash().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
@@ -265,14 +341,13 @@ mod tests {
         ))));
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
-        let bob_pubkey = Pubkey::new_rand();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
         let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
             let client_transport = start_local_server(&bank_forks).await;
-            let mut banks_client =
-                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+            let mut banks_client = start_client(client_transport).await?;
             let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             let signature = transaction.signatures[0];
@@ -285,7 +360,7 @@ mod tests {
                 if root_slot > last_valid_slot {
                     break;
                 }
-                delay_for(Duration::from_millis(100)).await;
+                sleep(Duration::from_millis(100)).await;
                 status = banks_client.get_transaction_status(signature).await?;
             }
             assert!(status.unwrap().err.is_none());

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.4.0"
+version = "1.5.3"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,9 +9,13 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
+mio = "0.7.6"
 serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-tarpc = { version = "0.22.0", features = ["full"] }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+tarpc = { version = "0.23.0", features = ["full"] }
+
+[dev-dependencies]
+tokio = { version = "0.3.5", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.4.0"
+version = "1.5.3"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,13 +11,14 @@ edition = "2018"
 [dependencies]
 bincode = "1.3.1"
 futures = "0.3"
-log = "0.4.8"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
-tarpc = { version = "0.22.0", features = ["full"] }
-tokio = "0.2"
+log = "0.4.11"
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "1.5.3" }
+solana-runtime = { path = "../runtime", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-metrics = { path = "../metrics", version = "1.5.3" }
+tarpc = { version = "0.23.0", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [lib]

banks-server/src/banks_server.rs

@@ -5,11 +5,7 @@ use futures::{
     prelude::stream::{self, StreamExt},
 };
 use solana_banks_interface::{Banks, BanksRequest, BanksResponse, TransactionStatus};
-use solana_runtime::{
-    bank::Bank,
-    bank_forks::BankForks,
-    commitment::{BlockCommitmentCache, CommitmentSlots},
-};
+use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache};
 use solana_sdk::{
     account::Account,
     clock::Slot,
@@ -21,7 +17,6 @@ use solana_sdk::{
     transaction::{self, Transaction},
 };
 use std::{
-    collections::HashMap,
     io,
     net::{Ipv4Addr, SocketAddr},
     sync::{
@@ -38,7 +33,7 @@ use tarpc::{
     server::{self, Channel, Handler},
     transport,
 };
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use tokio_serde::formats::Bincode;
 
 #[derive(Clone)]
@@ -84,11 +79,9 @@ impl BanksServer {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
         let slot = bank.slot();
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(
-            HashMap::default(),
-            0,
-            CommitmentSlots::new_from_slot(slot),
-        )));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
         Builder::new()
             .name("solana-bank-forks-client".to_string())
             .spawn(move || Self::run(&bank, transaction_receiver))
@@ -109,23 +102,36 @@ impl BanksServer {
 
     async fn poll_signature_status(
         self,
-        signature: Signature,
+        signature: &Signature,
+        blockhash: &Hash,
         last_valid_slot: Slot,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        let mut status = self.bank(commitment).get_signature_status(&signature);
+        let mut status = self
+            .bank(commitment)
+            .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            delay_for(Duration::from_millis(200)).await;
+            sleep(Duration::from_millis(200)).await;
             let bank = self.bank(commitment);
             if bank.slot() > last_valid_slot {
                 break;
             }
-            status = bank.get_signature_status(&signature);
+            status = bank.get_signature_status_with_blockhash(signature, blockhash);
         }
         status
     }
 }
 
+fn verify_transaction(transaction: &Transaction) -> transaction::Result<()> {
+    if let Err(err) = transaction.verify() {
+        Err(err)
+    } else if let Err(err) = transaction.verify_precompiles() {
+        Err(err)
+    } else {
+        Ok(())
+    }
+}
+
 #[tarpc::server]
 impl Banks for BanksServer {
     async fn send_transaction_with_context(self, _: Context, transaction: Transaction) {
@@ -187,19 +193,23 @@ impl Banks for BanksServer {
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
+        if let Err(err) = verify_transaction(&transaction) {
+            return Some(Err(err));
+        }
+
         let blockhash = &transaction.message.recent_blockhash;
         let last_valid_slot = self
             .bank_forks
             .read()
             .unwrap()
             .root_bank()
-            .get_blockhash_last_valid_slot(&blockhash)
+            .get_blockhash_last_valid_slot(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
         let info =
             TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
         self.transaction_sender.send(info).unwrap();
-        self.poll_signature_status(signature, last_valid_slot, commitment)
+        self.poll_signature_status(&signature, blockhash, last_valid_slot, commitment)
             .await
     }
 

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,28 +11,28 @@ publish = false
 [dependencies]
 clap = "2.33.1"
 itertools = "0.9.0"
-log = "0.4.8"
+log = "0.4.11"
 num-derive = "0.3"
 num-traits = "0.2"
 rand = "0.7.0"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-core = { path = "../core", version = "1.4.0" }
-solana-genesis = { path = "../genesis", version = "1.4.0" }
-solana-client = { path = "../client", version = "1.4.0" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-core = { path = "../core", version = "1.5.3" }
+solana-genesis = { path = "../genesis", version = "1.5.3" }
+solana-client = { path = "../client", version = "1.5.3" }
+solana-faucet = { path = "../faucet", version = "1.5.3" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.5.3" }
+solana-logger = { path = "../logger", version = "1.5.3" }
+solana-metrics = { path = "../metrics", version = "1.5.3" }
+solana-net-utils = { path = "../net-utils", version = "1.5.3" }
+solana-runtime = { path = "../runtime", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.4.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.3" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/src/cli.rs

@@ -163,7 +163,8 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
         )
 }
 
-pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
+#[allow(clippy::field_reassign_with_default)]
+pub fn extract_args(matches: &ArgMatches) -> Config {
     let mut args = Config::default();
 
     args.entrypoint_addr = solana_net_utils::parse_host_port(

bench-exchange/tests/bench_exchange.rs

@@ -22,15 +22,17 @@ fn test_exchange_local_cluster() {
 
     const NUM_NODES: usize = 1;
 
-    let mut config = Config::default();
-    config.identity = Keypair::new();
-    config.duration = Duration::from_secs(1);
-    config.fund_amount = 100_000;
-    config.threads = 1;
-    config.transfer_delay = 20; // 15
-    config.batch_size = 100; // 1000;
-    config.chunk_size = 10; // 200;
-    config.account_groups = 1; // 10;
+    let config = Config {
+        identity: Keypair::new(),
+        duration: Duration::from_secs(1),
+        fund_amount: 100_000,
+        threads: 1,
+        transfer_delay: 20, // 15
+        batch_size: 100,    // 1000
+        chunk_size: 10,     // 200
+        account_groups: 1,  // 10
+        ..Config::default()
+    };
     let Config {
         fund_amount,
         batch_size,
@@ -39,7 +41,7 @@ fn test_exchange_local_cluster() {
     } = config;
     let accounts_in_groups = batch_size * account_groups;
 
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![100_000; NUM_NODES],
         cluster_lamports: 100_000_000_000_000,
         validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
@@ -86,18 +88,21 @@ fn test_exchange_bank_client() {
     solana_logger::setup();
     let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
     let mut bank = Bank::new(&genesis_config);
-    bank.add_builtin_program("exchange_program", id(), process_instruction);
+    bank.add_builtin("exchange_program", id(), process_instruction);
     let clients = vec![BankClient::new(bank)];
 
-    let mut config = Config::default();
-    config.identity = identity;
-    config.duration = Duration::from_secs(1);
-    config.fund_amount = 100_000;
-    config.threads = 1;
-    config.transfer_delay = 20; // 0;
-    config.batch_size = 100; // 1500;
-    config.chunk_size = 10; // 1500;
-    config.account_groups = 1; // 50;
-
-    do_bench_exchange(clients, config);
+    do_bench_exchange(
+        clients,
+        Config {
+            identity,
+            duration: Duration::from_secs(1),
+            fund_amount: 100_000,
+            threads: 1,
+            transfer_delay: 20, // 0;
+            batch_size: 100,    // 1500;
+            chunk_size: 10,     // 1500;
+            account_groups: 1,  // 50;
+            ..Config::default()
+        },
+    );
 }

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-streamer = { path = "../streamer", version = "1.5.3" }
+solana-logger = { path = "../logger", version = "1.5.3" }
+solana-net-utils = { path = "../net-utils", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,27 +11,27 @@ publish = false
 [dependencies]
 bincode = "1.3.1"
 clap = "2.33.1"
-log = "0.4.8"
+log = "0.4.11"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-core = { path = "../core", version = "1.4.0" }
-solana-genesis = { path = "../genesis", version = "1.4.0" }
-solana-client = { path = "../client", version = "1.4.0" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
-solana-measure = { path = "../measure", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-core = { path = "../core", version = "1.5.3" }
+solana-genesis = { path = "../genesis", version = "1.5.3" }
+solana-client = { path = "../client", version = "1.5.3" }
+solana-faucet = { path = "../faucet", version = "1.5.3" }
+solana-logger = { path = "../logger", version = "1.5.3" }
+solana-metrics = { path = "../metrics", version = "1.5.3" }
+solana-measure = { path = "../measure", version = "1.5.3" }
+solana-net-utils = { path = "../net-utils", version = "1.5.3" }
+solana-runtime = { path = "../runtime", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.4.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.3" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -938,10 +938,12 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
 
-        let mut config = Config::default();
-        config.id = id;
-        config.tx_count = 10;
-        config.duration = Duration::from_secs(5);
+        let config = Config {
+            id,
+            tx_count: 10,
+            duration: Duration::from_secs(5),
+            ..Config::default()
+        };
 
         let keypair_count = config.tx_count * config.keypair_multiplier;
         let keypairs =

bench-tps/src/cli.rs

@@ -196,7 +196,7 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
 /// * `matches` - command line arguments parsed by clap
 /// # Panics
 /// Panics if there is trouble parsing any of the arguments
-pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
+pub fn extract_args(matches: &ArgMatches) -> Config {
     let mut args = Config::default();
 
     if let Some(addr) = matches.value_of("entrypoint") {

bench-tps/tests/bench_tps.rs

@@ -15,7 +15,7 @@ fn test_bench_tps_local_cluster(config: Config) {
 
     solana_logger::setup();
     const NUM_NODES: usize = 1;
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![999_990; NUM_NODES],
         cluster_lamports: 200_000_000,
         validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
@@ -60,9 +60,9 @@ fn test_bench_tps_local_cluster(config: Config) {
 #[test]
 #[serial]
 fn test_bench_tps_local_cluster_solana() {
-    let mut config = Config::default();
-    config.tx_count = 100;
-    config.duration = Duration::from_secs(10);
-
-    test_bench_tps_local_cluster(config);
+    test_bench_tps_local_cluster(Config {
+        tx_count: 100,
+        duration: Duration::from_secs(10),
+        ..Config::default()
+    });
 }

cargo

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# shellcheck source=ci/rust-version.sh
+here=$(dirname "$0")
+
+source "${here}"/ci/rust-version.sh all
+
+toolchain=
+case "$1" in
+  stable)
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    shift
+    ;;
+  nightly)
+    # shellcheck disable=SC2054 # rust_nightly is sourced from rust-version.sh
+    toolchain="$rust_nightly"
+    shift
+    ;;
+  +*)
+    toolchain="${1#+}"
+    shift
+    ;;
+  *)
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    ;;
+esac
+
+set -x
+exec cargo "+${toolchain}" "${@}"

cargo-build-bpf

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-build-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

cargo-test-bpf

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+export CARGO_BUILD_BPF="$here"/cargo-build-bpf
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-test-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

ci/README.md

@@ -47,6 +47,8 @@ sudo ./setup-new-buildkite-agent/setup-buildkite.sh
 ```
 - Copy the pubkey contents from `~buildkite-agent/.ssh/id_ecdsa.pub` and
 add the pubkey as an authorized SSH key on github.
+  - In net/scripts/solana-user-authorized_keys.sh
+  - Bug mvines to add it to the "solana-grimes" github user
 - Edit `/etc/buildkite-agent/buildkite-agent.cfg` and/or `/etc/systemd/system/buildkite-agent@*` to the desired configuration of the agent(s)
 - Copy `ejson` keys from another CI node at `/opt/ejson/keys/`
 to the same location on the new node.

ci/buildkite-pipeline.sh

@@ -175,6 +175,30 @@ EOF
       "Stable-perf skipped as no relevant files were modified"
   fi
 
+  # Downstream backwards compatibility
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+             ^scripts/build-downstream-projects.sh \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "scripts/build-downstream-projects.sh"
+    name: "downstream-projects"
+    timeout_in_minutes: 30
+EOF
+  else
+    annotate --style info \
+      "downstream-projects skipped as no relevant files were modified"
+  fi
   # Benches...
   if affects \
              .rs$ \
@@ -239,7 +263,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
   annotate --style info --context pr-backlink \
     "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
 
-  if [[ $GITHUB_USER = "dependabot-preview[bot]" ]]; then
+  if [[ $GITHUB_USER = "dependabot[bot]" ]]; then
     command_step dependabot "ci/dependabot-pr.sh" 5
     wait_step
   fi

ci/dependabot-updater.sh

@@ -6,13 +6,13 @@ source ci/_
 
 commit_range="$(git merge-base HEAD origin/master)..HEAD"
 parsed_update_args="$(
-  git log "$commit_range" --author "dependabot-preview" --oneline -n1 |
+  git log "$commit_range" --author "dependabot\[bot\]" --oneline -n1 |
     grep -o '[Bb]ump.*$' |
     sed -r 's/[Bb]ump ([^ ]+) from ([^ ]+) to ([^ ]+)/-p \1:\2 --precise \3/'
 )"
 # relaxed_parsed_update_args is temporal measure...
 relaxed_parsed_update_args="$(
-  git log "$commit_range" --author "dependabot-preview" --oneline -n1 |
+  git log "$commit_range" --author "dependabot\[bot\]" --oneline -n1 |
     grep -o '[Bb]ump.*$' |
     sed -r 's/[Bb]ump ([^ ]+) from [^ ]+ to ([^ ]+)/-p \1 --precise \2/'
 )"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.46.0
+FROM solanalabs/rust:1.48.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.46.0
+FROM rust:1.48.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/nits.sh

@@ -26,8 +26,9 @@ declare print_free_tree=(
   ':runtime/src/**.rs'
   ':sdk/bpf/rust/rust-utils/**.rs'
   ':sdk/**.rs'
-  ':^sdk/src/program_option.rs'
-  ':^sdk/src/program_stubs.rs'
+  ':^sdk/cargo-build-bpf/**.rs'
+  ':^sdk/program/src/program_option.rs'
+  ':^sdk/program/src/program_stubs.rs'
   ':programs/**.rs'
   ':^**bin**.rs'
   ':^**bench**.rs'

ci/order-crates-for-publishing.py

@@ -42,10 +42,10 @@ def get_packages():
         sys.exit(1)
 
     # Order dependencies
-    deleted_dependencies = []
     sorted_dependency_graph = []
     max_iterations = pow(len(dependency_graph),2)
-    while len(deleted_dependencies) < len(dependency_graph):
+    while dependency_graph:
+        deleted_packages = []
         if max_iterations == 0:
             # One day be more helpful and find the actual cycle for the user...
             sys.exit('Error: Circular dependency suspected between these packages: \n {}\n'.format('\n '.join(dependency_graph.keys())))
@@ -53,13 +53,17 @@ def get_packages():
         max_iterations -= 1
 
         for package, dependencies in dependency_graph.items():
+            if package in deleted_packages:
+                continue
             for dependency in dependencies:
                 if dependency in dependency_graph:
                     break
             else:
-                deleted_dependencies.append(package)
+                deleted_packages.append(package)
                 sorted_dependency_graph.append((package, manifest_path[package]))
 
+        dependency_graph = {p: d for p, d in dependency_graph.items() if not p in deleted_packages }
+
 
     return sorted_dependency_graph
 

ci/publish-bpf-sdk.sh

@@ -16,21 +16,12 @@ fi
   [[ -f bpf-sdk.tar.bz2 ]]
 )
 
+source ci/upload-ci-artifact.sh
 echo --- AWS S3 Store
 if [[ -z $CHANNEL_OR_TAG ]]; then
   echo Skipped
 else
-  (
-    set -x
-    docker run \
-      --rm \
-      --env AWS_ACCESS_KEY_ID \
-      --env AWS_SECRET_ACCESS_KEY \
-      --volume "$PWD:/solana" \
-      eremite/aws-cli:2018.12.18 \
-      /usr/bin/s3cmd --acl-public put /solana/bpf-sdk.tar.bz2 \
-      s3://solana-sdk/"$CHANNEL_OR_TAG"/bpf-sdk.tar.bz2
-  )
+  upload-s3-artifact "/solana/bpf-sdk.tar.bz2" "s3://solana-sdk/$CHANNEL_OR_TAG/bpf-sdk.tar.bz2"
 fi
 
 exit 0

ci/publish-crate.sh

@@ -4,6 +4,8 @@ cd "$(dirname "$0")/.."
 source ci/semver_bash/semver.sh
 source ci/rust-version.sh stable
 
+cargo="$(readlink -f ./cargo)"
+
 # shellcheck disable=SC2086
 is_crate_version_uploaded() {
   name=$1
@@ -66,11 +68,11 @@ for Cargo_toml in $Cargo_tomls; do
       (
         set -x
         rm -rf crate-test
-        cargo +"$rust_stable" init crate-test
+        "$cargo" stable init crate-test
         cd crate-test/
         echo "${crate_name} = \"${expectedCrateVersion}\"" >> Cargo.toml
         echo "[workspace]" >> Cargo.toml
-        cargo +"$rust_stable" check
+        "$cargo" stable check
       ) && really_uploaded=1
       if ((really_uploaded)); then
         break;

ci/publish-tarball.sh

@@ -91,17 +91,15 @@ echo --- Creating release tarball
   cp "${RELEASE_BASENAME}"/version.yml "${TARBALL_BASENAME}"-$TARGET.yml
 )
 
-# Metrics tarball is platform agnostic, only publish it from Linux
+# Maybe tarballs are platform agnostic, only publish them from the Linux build
 MAYBE_TARBALLS=
 if [[ "$CI_OS_NAME" = linux ]]; then
-  metrics/create-metrics-tarball.sh
   (
     set -x
     sdk/bpf/scripts/package.sh
     [[ -f bpf-sdk.tar.bz2 ]]
-
   )
-  MAYBE_TARBALLS="bpf-sdk.tar.bz2 solana-metrics.tar.bz2"
+  MAYBE_TARBALLS="bpf-sdk.tar.bz2"
 fi
 
 source ci/upload-ci-artifact.sh
@@ -115,19 +113,10 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.
 
   if [[ -n $BUILDKITE ]]; then
     echo --- AWS S3 Store: "$file"
-    (
-      set -x
-      $DRYRUN docker run \
-        --rm \
-        --env AWS_ACCESS_KEY_ID \
-        --env AWS_SECRET_ACCESS_KEY \
-        --volume "$PWD:/solana" \
-        eremite/aws-cli:2018.12.18 \
-        /usr/bin/s3cmd --acl-public put /solana/"$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
+    upload-s3-artifact "/solana/$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
 
-      echo Published to:
-      $DRYRUN ci/format-url.sh http://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
-    )
+    echo Published to:
+    $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
 
     if [[ -n $TAG ]]; then
       ci/upload-github-release-asset.sh "$file"
@@ -149,4 +138,22 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.
   fi
 done
 
+
+# Create install wrapper for release.solana.com
+if [[ -n $DO_NOT_PUBLISH_TAR ]]; then
+  echo "Skipping publishing install wrapper"
+elif [[ -n $BUILDKITE ]]; then
+  cat > release.solana.com-install <<EOF
+SOLANA_RELEASE=$CHANNEL_OR_TAG
+SOLANA_INSTALL_INIT_ARGS=$CHANNEL_OR_TAG
+SOLANA_DOWNLOAD_ROOT=http://release.solana.com
+EOF
+  cat install/solana-install-init.sh >> release.solana.com-install
+
+  echo --- AWS S3 Store: "install"
+  $DRYRUN upload-s3-artifact "/solana/release.solana.com-install" "s3://release.solana.com/$CHANNEL_OR_TAG/install"
+  echo Published to:
+  $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/install
+fi
+
 echo --- ok

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.46.0
+  stable_version=1.48.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-08-17
+  nightly_version=2020-12-13
 fi
 
 

ci/setup-new-buildkite-agent/setup-new-machine.sh

@@ -7,6 +7,7 @@ SOLANA_ROOT="$HERE"/../..
 source "$HERE"/utils.sh
 
 ensure_env || exit 1
+check_ssh_authorized_keys || exit 1
 
 set -ex
 

ci/setup-new-buildkite-agent/setup-ssh.sh

@@ -6,6 +6,11 @@ HERE="$(dirname "$0")"
 source "$HERE"/utils.sh
 
 ensure_env || exit 1
+# This is a last ditch effort to prevent the caller from locking themselves
+# out of the machine. Exiting here will likely leave the system in some
+# half-configured state. To prevent this, duplicate the next line at the top
+# of the entrypoint script.
+check_ssh_authorized_keys || exit 1
 
 set -xe
 # Setup sshd

ci/setup-new-buildkite-agent/utils.sh

@@ -14,3 +14,33 @@ ensure_env() {
   $RC
 }
 
+# Some scripts disable SSH password logins. If no one hash setup authorized_keys
+# this will result in the machine being remotely inaccessible. Check that the
+# user running this script has setup their keys
+check_ssh_authorized_keys() {
+  declare rc=false
+  declare user_home=
+  if [[ -n "$SUDO_USER" ]]; then
+    declare user uid gid home
+    declare passwd_entry
+    passwd_entry="$(grep "$SUDO_USER:[^:]*:$SUDO_UID:$SUDO_GID" /etc/passwd)"
+    IFS=: read -r user _ uid gid _ home _ <<<"$passwd_entry"
+    if [[ "$user" == "$SUDO_USER" && "$uid" == "$SUDO_UID" && "$gid" == "$SUDO_GID" ]]; then
+      user_home="$home"
+    fi
+  else
+    user_home="$HOME"
+  fi
+  declare authorized_keys="${user_home}/.ssh/authorized_keys"
+  if [[ -n "$user_home" ]]; then
+    [[ -s "$authorized_keys" ]] && rc=true
+  fi
+  if ! $rc; then
+    echo "ERROR! This script will disable SSH password logins and you don't"
+    echo "appear to have set up any authorized keys.  Please add you SSH"
+    echo "public key to ${authorized_keys} before continuing!"
+  fi
+  $rc
+}
+
+check_ssh_authorized_keys

ci/test-bench.sh

@@ -6,7 +6,8 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-source ci/rust-version.sh all
+
+cargo="$(readlink -f "./cargo")"
 
 set -o pipefail
 export RUST_BACKTRACE=1
@@ -27,35 +28,35 @@ test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 
 # Ensure all dependencies are built
-_ cargo +$rust_nightly build --release
+_ "$cargo" nightly build --release
 
 # Remove "BENCH_FILE", if it exists so that the following commands can append
 rm -f "$BENCH_FILE"
 
 # Run sdk benches
-_ cargo +$rust_nightly bench --manifest-path sdk/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path sdk/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run runtime benches
-_ cargo +$rust_nightly bench --manifest-path runtime/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path runtime/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run core benches
-_ cargo +$rust_nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run bpf benches
-_ cargo +$rust_nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
+_ "$cargo" nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
   -- -Z unstable-options --format=json --nocapture | tee -a "$BENCH_FILE"
 
 # Run banking/accounts bench. Doesn't require nightly, but use since it is already built.
-_ cargo +$rust_nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
-_ cargo +$rust_nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
+_ "$cargo" nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
+_ "$cargo" nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
 
 # `solana-upload-perf` disabled as it can take over 30 minutes to complete for some
 # reason
 exit 0
-_ cargo +$rust_nightly run --release --package solana-upload-perf \
+_ "$cargo" nightly run --release --package solana-upload-perf \
   -- "$BENCH_FILE" "$TARGET_BRANCH" "$UPLOAD_METRICS" | tee "$BENCH_ARTIFACT"
 
 upload-ci-artifact "$BENCH_FILE"

ci/test-checks.sh

@@ -8,6 +8,9 @@ source ci/_
 source ci/rust-version.sh stable
 source ci/rust-version.sh nightly
 eval "$(ci/channel-info.sh)"
+cargo="$(readlink -f "./cargo")"
+
+scripts/increment-cargo-version.sh check
 
 echo --- build environment
 (
@@ -16,14 +19,14 @@ echo --- build environment
   rustup run "$rust_stable" rustc --version --verbose
   rustup run "$rust_nightly" rustc --version --verbose
 
-  cargo +"$rust_stable" --version --verbose
-  cargo +"$rust_nightly" --version --verbose
+  "$cargo" stable --version --verbose
+  "$cargo" nightly --version --verbose
 
-  cargo +"$rust_stable" clippy --version --verbose
-  cargo +"$rust_nightly" clippy --version --verbose
+  "$cargo" stable clippy --version --verbose
+  "$cargo" nightly clippy --version --verbose
 
   # audit is done only with stable
-  cargo +"$rust_stable" audit --version
+  "$cargo" stable audit --version
 )
 
 export RUST_BACKTRACE=1
@@ -41,7 +44,7 @@ if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
     echo "$0:   [tree (for outdated Cargo.lock sync)|check (for compilation error)|update -p foo --precise x.y.z (for your Cargo.toml update)] ..." >&2
     exit "$check_status"
   fi
-    
+
   # Ensure nightly and --benches
   _ scripts/cargo-for-all-lock-files.sh +"$rust_nightly" check --locked --all-targets
 else
@@ -49,26 +52,51 @@ else
 fi
 
 _ ci/order-crates-for-publishing.py
-_ cargo +"$rust_stable" fmt --all -- --check
+_ "$cargo" stable fmt --all -- --check
 
 # -Z... is needed because of clippy bug: https://github.com/rust-lang/rust-clippy/issues/4612
 # run nightly clippy for `sdk/` as there's a moderate amount of nightly-only code there
-_ cargo +"$rust_nightly" clippy \
-  -Zunstable-options --workspace --all-targets \
-  -- --deny=warnings --allow=clippy::stable_sort_primitive
+_ "$cargo" nightly clippy -Zunstable-options --workspace --all-targets -- --deny=warnings
 
-_ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
+cargo_audit_ignores=(
+  # failure is officially deprecated/unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `failure` dependency.
+  --ignore RUSTSEC-2020-0036
+
+  # `net2` crate has been deprecated; use `socket2` instead
+  #
+  # Blocked on https://github.com/paritytech/jsonrpc/issues/575
+  --ignore RUSTSEC-2020-0016
+
+  # stdweb is unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `stdweb` dependency.
+  --ignore RUSTSEC-2020-0056
+
+  # Potential segfault in the time crate
+  #
+  # Blocked on multiple crates updating `time` to >= 0.2.23
+  --ignore RUSTSEC-2020-0071
+
+  # difference is unmaintained
+  #
+  # Blocked on predicates v1.0.6 removing its dependency on `difference`
+  --ignore RUSTSEC-2020-0095
+
+)
+_ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"
 
 {
   cd programs/bpf
-  _ cargo +"$rust_stable" audit
+  _ "$cargo" stable audit
   for project in rust/*/ ; do
     echo "+++ do_bpf_checks $project"
     (
       cd "$project"
-      _ cargo +"$rust_stable" fmt -- --check
-      _ cargo +"$rust_nightly" test
-      _ cargo +"$rust_nightly" clippy -- --deny=warnings --allow=clippy::missing_safety_doc
+      _ "$cargo" stable fmt -- --check
+      _ "$cargo" nightly test
+      _ "$cargo" nightly clippy -- --deny=warnings --allow=clippy::missing_safety_doc
     )
   done
 }

ci/test-sanity.sh

@@ -8,10 +8,16 @@ source ci/_
 (
   echo --- git diff --check
   set -x
+
+  if [[ -n $CI_BASE_BRANCH ]]
+  then branch="$CI_BASE_BRANCH"
+  else branch="master"
+  fi
+
   # Look for failed mergify.io backports by searching leftover conflict markers
   # Also check for any trailing whitespaces!
-  git fetch origin "$CI_BASE_BRANCH"
-  git diff "$(git merge-base HEAD "origin/$CI_BASE_BRANCH")..HEAD" --check --oneline
+  git fetch origin "$branch"
+  git diff "$(git merge-base HEAD "origin/$branch")" --check --oneline
 )
 
 echo

ci/test-stable.sh

@@ -2,6 +2,8 @@
 set -e
 cd "$(dirname "$0")/.."
 
+cargo="$(readlink -f "./cargo")"
+
 source ci/_
 
 annotate() {
@@ -19,9 +21,6 @@ export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 source scripts/ulimit-n.sh
 
-# Clear cached json keypair files
-rm -rf "$HOME/.config/solana"
-
 # Clear the C dependency files, if dependency moves these files are not regenerated
 test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
@@ -37,12 +36,15 @@ NPROC=$((NPROC>14 ? 14 : NPROC))
 echo "Executing $testName"
 case $testName in
 test-stable)
-  _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
   ;;
 test-stable-perf)
+  # BPF solana-sdk legacy compile test
+  ./cargo-build-bpf --manifest-path sdk/Cargo.toml
+
   # BPF program tests
   _ make -C programs/bpf/c tests
-  _ cargo +"$rust_stable" test \
+  _ "$cargo" stable test \
     --manifest-path programs/bpf/Cargo.toml \
     --no-default-features --features=bpf_c,bpf_rust -- --nocapture
 
@@ -62,13 +64,13 @@ test-stable-perf)
     export SOLANA_CUDA=1
   fi
 
-  _ cargo +"$rust_stable" build --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
-  _ cargo +"$rust_stable" run --manifest-path poh-bench/Cargo.toml ${V:+--verbose} -- --hashes-per-tick 10
+  _ "$cargo" stable build --bins ${V:+--verbose}
+  _ "$cargo" stable test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable run --manifest-path poh-bench/Cargo.toml ${V:+--verbose} -- --hashes-per-tick 10
   ;;
 test-local-cluster)
-  _ cargo +"$rust_stable" build --release --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable build --release --bins ${V:+--verbose}
+  _ "$cargo" stable test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
   exit 0
   ;;
 *)

ci/upload-ci-artifact.sh

@@ -16,3 +16,16 @@ upload-ci-artifact() {
   fi
 }
 
+upload-s3-artifact() {
+  echo "--- artifact: $1 to $2"
+  (
+    set -x
+    docker run \
+      --rm \
+      --env AWS_ACCESS_KEY_ID \
+      --env AWS_SECRET_ACCESS_KEY \
+      --volume "$PWD:/solana" \
+      eremite/aws-cli:2018.12.18 \
+      /usr/bin/s3cmd --acl-public put "$1" "$2"
+  )
+}

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.4.0"
+version = "1.5.3"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,9 +11,9 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-thiserror = "1.0.20"
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"
 chrono = "0.4"

clap-utils/src/input_parsers.rs

@@ -189,6 +189,7 @@ pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentC
         "recent" => CommitmentConfig::recent(),
         "root" => CommitmentConfig::root(),
         "single" => CommitmentConfig::single(),
+        "singleGossip" => CommitmentConfig::single_gossip(),
         _ => CommitmentConfig::default(),
     })
 }
@@ -228,8 +229,8 @@ mod tests {
         assert_eq!(values_of(&matches, "multiple"), Some(vec![50, 39]));
         assert_eq!(values_of::<u64>(&matches, "single"), None);
 
-        let pubkey0 = Pubkey::new_rand();
-        let pubkey1 = Pubkey::new_rand();
+        let pubkey0 = solana_sdk::pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
         let matches = app().clone().get_matches_from(vec![
             "test",
             "--multiple",
@@ -251,7 +252,7 @@ mod tests {
         assert_eq!(value_of(&matches, "single"), Some(50));
         assert_eq!(value_of::<u64>(&matches, "multiple"), None);
 
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let matches = app()
             .clone()
             .get_matches_from(vec!["test", "--single", &pubkey.to_string()]);
@@ -331,8 +332,8 @@ mod tests {
 
     #[test]
     fn test_pubkeys_sigs_of() {
-        let key1 = Pubkey::new_rand();
-        let key2 = Pubkey::new_rand();
+        let key1 = solana_sdk::pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
         let sig1 = Keypair::new().sign_message(&[0u8]);
         let sig2 = Keypair::new().sign_message(&[1u8]);
         let signer1 = format!("{}={}", key1, sig1);

clap-utils/src/input_validators.rs

@@ -1,7 +1,7 @@
 use crate::keypair::{parse_keypair_path, KeypairUrl, ASK_KEYWORD};
 use chrono::DateTime;
 use solana_sdk::{
-    clock::Slot,
+    clock::{Epoch, Slot},
     hash::Hash,
     pubkey::Pubkey,
     signature::{read_keypair_file, Signature},
@@ -148,6 +148,40 @@ where
     }
 }
 
+pub fn is_url_or_moniker<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    match url::Url::parse(&normalize_to_url_if_moniker(string.as_ref())) {
+        Ok(url) => {
+            if url.has_host() {
+                Ok(())
+            } else {
+                Err("no host provided".to_string())
+            }
+        }
+        Err(err) => Err(format!("{}", err)),
+    }
+}
+
+pub fn normalize_to_url_if_moniker(url_or_moniker: &str) -> String {
+    match url_or_moniker {
+        "m" | "mainnet-beta" => "https://api.mainnet-beta.solana.com",
+        "t" | "testnet" => "https://testnet.solana.com",
+        "d" | "devnet" => "https://devnet.solana.com",
+        "l" | "localhost" => "http://localhost:8899",
+        url => url,
+    }
+    .to_string()
+}
+
+pub fn is_epoch<T>(epoch: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<Epoch, _>(epoch)
+}
+
 pub fn is_slot<T>(slot: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,

clap-utils/src/keypair.rs

@@ -58,6 +58,15 @@ impl CliSignerInfo {
             Some(0)
         }
     }
+    pub fn index_of_or_none(&self, pubkey: Option<Pubkey>) -> Option<usize> {
+        if let Some(pubkey) = pubkey {
+            self.signers
+                .iter()
+                .position(|signer| signer.pubkey() == pubkey)
+        } else {
+            None
+        }
+    }
 }
 
 pub struct DefaultSigner {
@@ -298,7 +307,24 @@ pub fn keypair_from_seed_phrase(
         keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
     } else {
         let sanitized = sanitize_seed_phrase(seed_phrase);
-        let mnemonic = Mnemonic::from_phrase(&sanitized, Language::English)?;
+        let parse_language_fn = || {
+            for language in &[
+                Language::English,
+                Language::ChineseSimplified,
+                Language::ChineseTraditional,
+                Language::Japanese,
+                Language::Spanish,
+                Language::Korean,
+                Language::French,
+                Language::Italian,
+            ] {
+                if let Ok(mnemonic) = Mnemonic::from_phrase(&sanitized, *language) {
+                    return Ok(mnemonic);
+                }
+            }
+            Err("Can't get mnemonic from seed phrases")
+        };
+        let mnemonic = parse_language_fn()?;
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
         let seed = Seed::new(&mnemonic, &passphrase);
         keypair_from_seed(seed.as_bytes())?

cli-config/Cargo.toml

@@ -3,13 +3,13 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
-dirs = "2.0.2"
+dirs-next = "2.0.0"
 lazy_static = "1.4.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"

cli-config/src/config.rs

@@ -5,7 +5,7 @@ use url::Url;
 
 lazy_static! {
     pub static ref CONFIG_FILE: Option<String> = {
-        dirs::home_dir().map(|mut path| {
+        dirs_next::home_dir().map(|mut path| {
             path.extend(&[".config", "solana", "cli", "config.yml"]);
             path.to_str().unwrap().to_string()
         })
@@ -25,7 +25,7 @@ pub struct Config {
 impl Default for Config {
     fn default() -> Self {
         let keypair_path = {
-            let mut keypair_path = dirs::home_dir().expect("home directory");
+            let mut keypair_path = dirs_next::home_dir().expect("home directory");
             keypair_path.extend(&[".config", "solana", "id.json"]);
             keypair_path.to_str().unwrap().to_string()
         };
@@ -76,17 +76,6 @@ impl Config {
         ws_url.to_string()
     }
 
-    pub fn compute_rpc_banks_url(json_rpc_url: &str) -> String {
-        let json_rpc_url: Option<Url> = json_rpc_url.parse().ok();
-        if json_rpc_url.is_none() {
-            return "".to_string();
-        }
-        let mut url = json_rpc_url.unwrap();
-        let port = url.port().unwrap_or(8899);
-        url.set_port(Some(port + 3)).expect("unable to set port");
-        url.to_string()
-    }
-
     pub fn import_address_labels<P>(&mut self, filename: P) -> Result<(), io::Error>
     where
         P: AsRef<Path>,
@@ -133,28 +122,4 @@ mod test {
 
         assert_eq!(Config::compute_websocket_url(&"garbage"), String::new());
     }
-
-    #[test]
-    fn compute_rpc_banks_url() {
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"http://devnet.solana.com"),
-            "http://devnet.solana.com:8902/".to_string()
-        );
-
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"https://devnet.solana.com"),
-            "https://devnet.solana.com:8902/".to_string()
-        );
-
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"http://example.com:8899"),
-            "http://example.com:8902/".to_string()
-        );
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"https://example.com:1234"),
-            "https://example.com:1237/".to_string()
-        );
-
-        assert_eq!(Config::compute_rpc_banks_url(&"garbage"), String::new());
-    }
 }

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,13 +17,13 @@ indicatif = "0.15.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-client = { path = "../client", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.3" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-client = { path = "../client", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.3" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.3" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.3" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -10,7 +10,8 @@ use serde_json::{Map, Value};
 use solana_account_decoder::parse_token::UiTokenAccount;
 use solana_clap_utils::keypair::SignOnly;
 use solana_client::rpc_response::{
-    RpcAccountBalance, RpcKeyedAccount, RpcSupply, RpcVoteAccountInfo,
+    RpcAccountBalance, RpcInflationGovernor, RpcInflationRate, RpcKeyedAccount, RpcSupply,
+    RpcVoteAccountInfo,
 };
 use solana_sdk::{
     clock::{self, Epoch, Slot, UnixTimestamp},
@@ -132,7 +133,7 @@ impl fmt::Display for CliBlockProduction {
             "{}",
             style(format!(
                 "  {:<44}  {:>15}  {:>15}  {:>15}  {:>23}",
-                "Identity Pubkey",
+                "Identity",
                 "Leader Slots",
                 "Blocks Produced",
                 "Skipped Slots",
@@ -241,6 +242,9 @@ impl fmt::Display for CliEpochInfo {
         )?;
         writeln_name_value(f, "Slot:", &self.epoch_info.absolute_slot.to_string())?;
         writeln_name_value(f, "Epoch:", &self.epoch_info.epoch.to_string())?;
+        if let Some(transaction_count) = &self.epoch_info.transaction_count {
+            writeln_name_value(f, "Transaction Count:", &transaction_count.to_string())?;
+        }
         let start_slot = self.epoch_info.absolute_slot - self.epoch_info.slot_index;
         let end_slot = start_slot + self.epoch_info.slots_in_epoch;
         writeln_name_value(
@@ -301,7 +305,7 @@ pub struct CliValidatorsStakeByVersion {
 pub struct CliValidators {
     pub total_active_stake: u64,
     pub total_current_stake: u64,
-    pub total_deliquent_stake: u64,
+    pub total_delinquent_stake: u64,
     pub current_validators: Vec<CliValidator>,
     pub delinquent_validators: Vec<CliValidator>,
     pub stake_by_version: BTreeMap<String, CliValidatorsStakeByVersion>,
@@ -360,7 +364,7 @@ impl fmt::Display for CliValidators {
             "Active Stake:",
             &build_balance_message(self.total_active_stake, self.use_lamports_unit, true),
         )?;
-        if self.total_deliquent_stake > 0 {
+        if self.total_delinquent_stake > 0 {
             writeln_name_value(
                 f,
                 "Current Stake:",
@@ -376,11 +380,11 @@ impl fmt::Display for CliValidators {
                 &format!(
                     "{} ({:0.2}%)",
                     &build_balance_message(
-                        self.total_deliquent_stake,
+                        self.total_delinquent_stake,
                         self.use_lamports_unit,
                         true
                     ),
-                    100. * self.total_deliquent_stake as f64 / self.total_active_stake as f64
+                    100. * self.total_delinquent_stake as f64 / self.total_active_stake as f64
                 ),
             )?;
         }
@@ -412,8 +416,8 @@ impl fmt::Display for CliValidators {
             "{}",
             style(format!(
                 "  {:<44}  {:<38}  {}  {}  {}  {:>10}  {:^8}  {}",
-                "Identity Pubkey",
-                "Vote Account Pubkey",
+                "Identity",
+                "Vote Account",
                 "Commission",
                 "Last Vote",
                 "Root Block",
@@ -520,7 +524,7 @@ impl fmt::Display for CliNonceAccount {
             )
         )?;
         let nonce = self.nonce.as_deref().unwrap_or("uninitialized");
-        writeln!(f, "Nonce: {}", nonce)?;
+        writeln!(f, "Nonce blockhash: {}", nonce)?;
         if let Some(fees) = self.lamports_per_signature {
             writeln!(f, "Fee: {} lamports per signature", fees)?;
         } else {
@@ -541,7 +545,15 @@ impl CliStakeVec {
 }
 
 impl QuietDisplay for CliStakeVec {}
-impl VerboseDisplay for CliStakeVec {}
+impl VerboseDisplay for CliStakeVec {
+    fn write_str(&self, w: &mut dyn std::fmt::Write) -> std::fmt::Result {
+        for state in &self.0 {
+            writeln!(w)?;
+            VerboseDisplay::write_str(state, w)?;
+        }
+        Ok(())
+    }
+}
 
 impl fmt::Display for CliStakeVec {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -562,7 +574,12 @@ pub struct CliKeyedStakeState {
 }
 
 impl QuietDisplay for CliKeyedStakeState {}
-impl VerboseDisplay for CliKeyedStakeState {}
+impl VerboseDisplay for CliKeyedStakeState {
+    fn write_str(&self, w: &mut dyn std::fmt::Write) -> std::fmt::Result {
+        writeln!(w, "Stake Pubkey: {}", self.stake_pubkey)?;
+        VerboseDisplay::write_str(&self.stake_state, w)
+    }
+}
 
 impl fmt::Display for CliKeyedStakeState {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -579,7 +596,46 @@ pub struct CliEpochReward {
     pub amount: u64,       // lamports
     pub post_balance: u64, // lamports
     pub percent_change: f64,
-    pub apr: f64,
+    pub apr: Option<f64>,
+}
+
+fn show_votes_and_credits(
+    f: &mut fmt::Formatter,
+    votes: &[CliLockout],
+    epoch_voting_history: &[CliEpochVotingHistory],
+) -> fmt::Result {
+    if votes.is_empty() {
+        return Ok(());
+    }
+
+    writeln!(f, "Recent Votes:")?;
+    for vote in votes {
+        writeln!(f, "- slot: {}", vote.slot)?;
+        writeln!(f, "  confirmation count: {}", vote.confirmation_count)?;
+    }
+    writeln!(f, "Epoch Voting History:")?;
+    writeln!(
+        f,
+        "* missed credits include slots unavailable to vote on due to delinquent leaders",
+    )?;
+    for entry in epoch_voting_history {
+        writeln!(
+            f, // tame fmt so that this will be folded like following
+            "- epoch: {}",
+            entry.epoch
+        )?;
+        writeln!(
+            f,
+            "  credits range: [{}..{})",
+            entry.prev_credits, entry.credits
+        )?;
+        writeln!(
+            f,
+            "  credits/slots: {}/{}",
+            entry.credits_earned, entry.slots_in_epoch
+        )?;
+    }
+    Ok(())
 }
 
 fn show_epoch_rewards(
@@ -594,19 +650,22 @@ fn show_epoch_rewards(
         writeln!(f, "Epoch Rewards:")?;
         writeln!(
             f,
-            "  {:<8}  {:<11}  {:<15}  {:<15}  {:>14}  {:>14}",
+            "  {:<6}  {:<11}  {:<16}  {:<16}  {:>14}  {:>14}",
             "Epoch", "Reward Slot", "Amount", "New Balance", "Percent Change", "APR"
         )?;
         for reward in epoch_rewards {
             writeln!(
                 f,
-                "  {:<8}  {:<11}  ◎{:<14.9}  ◎{:<14.9}  {:>13.9}%  {:>13.9}%",
+                "  {:<6}  {:<11}  ◎{:<16.9}  ◎{:<14.9}  {:>13.2}%  {}",
                 reward.epoch,
                 reward.effective_slot,
                 lamports_to_sol(reward.amount),
                 lamports_to_sol(reward.post_balance),
                 reward.percent_change,
-                reward.apr,
+                reward
+                    .apr
+                    .map(|apr| format!("{:>13.2}%", apr))
+                    .unwrap_or_default(),
             )?;
         }
     }
@@ -619,6 +678,8 @@ pub struct CliStakeState {
     pub stake_type: CliStakeType,
     pub account_balance: u64,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub credits_observed: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub delegated_stake: Option<u64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub delegated_vote_account_address: Option<String>,
@@ -647,7 +708,15 @@ pub struct CliStakeState {
 }
 
 impl QuietDisplay for CliStakeState {}
-impl VerboseDisplay for CliStakeState {}
+impl VerboseDisplay for CliStakeState {
+    fn write_str(&self, w: &mut dyn std::fmt::Write) -> std::fmt::Result {
+        write!(w, "{}", self)?;
+        if let Some(credits) = self.credits_observed {
+            writeln!(w, "Credits Observed: {}", credits)?;
+        }
+        Ok(())
+    }
+}
 
 impl fmt::Display for CliStakeState {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -661,13 +730,8 @@ impl fmt::Display for CliStakeState {
                 if lockup.unix_timestamp != UnixTimestamp::default() {
                     writeln!(
                         f,
-                        "Lockup Timestamp: {} (UnixTimestamp: {})",
-                        DateTime::<Utc>::from_utc(
-                            NaiveDateTime::from_timestamp(lockup.unix_timestamp, 0),
-                            Utc
-                        )
-                        .to_rfc3339_opts(SecondsFormat::Secs, true),
-                        lockup.unix_timestamp
+                        "Lockup Timestamp: {}",
+                        unix_timestamp_to_string(lockup.unix_timestamp)
                     )?;
                 }
                 if lockup.epoch != Epoch::default() {
@@ -952,8 +1016,8 @@ impl VerboseDisplay for CliValidatorInfo {}
 
 impl fmt::Display for CliValidatorInfo {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln_name_value(f, "Validator Identity Pubkey:", &self.identity_pubkey)?;
-        writeln_name_value(f, "  Info Pubkey:", &self.info_pubkey)?;
+        writeln_name_value(f, "Validator Identity:", &self.identity_pubkey)?;
+        writeln_name_value(f, "  Info Address:", &self.info_pubkey)?;
         for (key, value) in self.info.iter() {
             writeln_name_value(
                 f,
@@ -1008,25 +1072,13 @@ impl fmt::Display for CliVoteAccount {
                 None => "~".to_string(),
             }
         )?;
-        writeln!(f, "Recent Timestamp: {:?}", self.recent_timestamp)?;
-        if !self.votes.is_empty() {
-            writeln!(f, "Recent Votes:")?;
-            for vote in &self.votes {
-                writeln!(
-                    f,
-                    "- slot: {}\n  confirmation count: {}",
-                    vote.slot, vote.confirmation_count
-                )?;
-            }
-            writeln!(f, "Epoch Voting History:")?;
-            for epoch_info in &self.epoch_voting_history {
-                writeln!(
-                    f,
-                    "- epoch: {}\n  slots in epoch: {}\n  credits earned: {}",
-                    epoch_info.epoch, epoch_info.slots_in_epoch, epoch_info.credits_earned,
-                )?;
-            }
-        }
+        writeln!(
+            f,
+            "Recent Timestamp: {} from slot {}",
+            unix_timestamp_to_string(self.recent_timestamp.timestamp),
+            self.recent_timestamp.slot
+        )?;
+        show_votes_and_credits(f, &self.votes, &self.epoch_voting_history)?;
         show_epoch_rewards(f, &self.epoch_rewards)?;
         Ok(())
     }
@@ -1065,6 +1117,8 @@ pub struct CliEpochVotingHistory {
     pub epoch: Epoch,
     pub slots_in_epoch: u64,
     pub credits_earned: u64,
+    pub credits: u64,
+    pub prev_credits: u64,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -1093,18 +1147,119 @@ pub struct CliBlockTime {
 impl QuietDisplay for CliBlockTime {}
 impl VerboseDisplay for CliBlockTime {}
 
+fn unix_timestamp_to_string(unix_timestamp: UnixTimestamp) -> String {
+    format!(
+        "{} (UnixTimestamp: {})",
+        match NaiveDateTime::from_timestamp_opt(unix_timestamp, 0) {
+            Some(ndt) =>
+                DateTime::<Utc>::from_utc(ndt, Utc).to_rfc3339_opts(SecondsFormat::Secs, true),
+            None => "unknown".to_string(),
+        },
+        unix_timestamp,
+    )
+}
+
 impl fmt::Display for CliBlockTime {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln_name_value(f, "Block:", &self.slot.to_string())?;
-        writeln_name_value(
+        writeln_name_value(f, "Date:", &unix_timestamp_to_string(self.timestamp))
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliLeaderSchedule {
+    pub epoch: Epoch,
+    pub leader_schedule_entries: Vec<CliLeaderScheduleEntry>,
+}
+
+impl QuietDisplay for CliLeaderSchedule {}
+impl VerboseDisplay for CliLeaderSchedule {}
+
+impl fmt::Display for CliLeaderSchedule {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        for entry in &self.leader_schedule_entries {
+            writeln!(f, "  {:<15} {:<44}", entry.slot, entry.leader)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliLeaderScheduleEntry {
+    pub slot: Slot,
+    pub leader: String,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliInflation {
+    pub governor: RpcInflationGovernor,
+    pub current_rate: RpcInflationRate,
+}
+
+impl QuietDisplay for CliInflation {}
+impl VerboseDisplay for CliInflation {}
+
+impl fmt::Display for CliInflation {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "{}", style("Inflation Governor:").bold())?;
+        if (self.governor.initial - self.governor.terminal).abs() < f64::EPSILON {
+            writeln!(
+                f,
+                "Fixed APR:               {:>5.2}%",
+                self.governor.terminal * 100.
+            )?;
+        } else {
+            writeln!(
+                f,
+                "Initial APR:             {:>5.2}%",
+                self.governor.initial * 100.
+            )?;
+            writeln!(
+                f,
+                "Terminal APR:            {:>5.2}%",
+                self.governor.terminal * 100.
+            )?;
+            writeln!(
+                f,
+                "Rate reduction per year: {:>5.2}%",
+                self.governor.taper * 100.
+            )?;
+        }
+        if self.governor.foundation_term > 0. {
+            writeln!(
+                f,
+                "Foundation percentage:   {:>5.2}%",
+                self.governor.foundation
+            )?;
+            writeln!(
+                f,
+                "Foundation term:         {:.1} years",
+                self.governor.foundation_term
+            )?;
+        }
+
+        writeln!(
             f,
-            "Date:",
-            &format!(
-                "{} (UnixTimestamp: {})",
-                DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(self.timestamp, 0), Utc)
-                    .to_rfc3339_opts(SecondsFormat::Secs, true),
-                self.timestamp
-            ),
+            "\n{}",
+            style(format!("Inflation for Epoch {}:", self.current_rate.epoch)).bold()
+        )?;
+        writeln!(
+            f,
+            "Total APR:               {:>5.2}%",
+            self.current_rate.total * 100.
+        )?;
+        writeln!(
+            f,
+            "Staking APR:             {:>5.2}%",
+            self.current_rate.validator * 100.
+        )?;
+        writeln!(
+            f,
+            "Foundation APR:          {:>5.2}%",
+            self.current_rate.foundation * 100.
         )
     }
 }

cli-output/src/display.rs

@@ -27,7 +27,7 @@ pub fn build_balance_message(lamports: u64, use_lamports_unit: bool, show_unit:
 
 // Pretty print a "name value"
 pub fn println_name_value(name: &str, value: &str) {
-    let styled_value = if value == "" {
+    let styled_value = if value.is_empty() {
         style("(not set)").italic()
     } else {
         style(value)
@@ -36,7 +36,7 @@ pub fn println_name_value(name: &str, value: &str) {
 }
 
 pub fn writeln_name_value(f: &mut fmt::Formatter, name: &str, value: &str) -> fmt::Result {
-    let styled_value = if value == "" {
+    let styled_value = if value.is_empty() {
         style("(not set)").italic()
     } else {
         style(value)
@@ -197,6 +197,15 @@ pub fn write_transaction<W: io::Write>(
                 )?;
             }
         }
+
+        if let Some(log_messages) = &transaction_status.log_messages {
+            if !log_messages.is_empty() {
+                writeln!(w, "{}Log Messages:", prefix,)?;
+                for log_message in log_messages {
+                    writeln!(w, "{}  {}", prefix, log_message,)?;
+                }
+            }
+        }
     } else {
         writeln!(w, "{}Status: Unavailable", prefix)?;
     }

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.5.3"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -16,39 +16,40 @@ clap = "2.33.1"
 criterion-stats = "0.3.0"
 ctrlc = { version = "3.1.5", features = ["termination"] }
 console = "0.11.3"
-dirs = "2.0.2"
-log = "0.4.8"
+dirs-next = "2.0.0"
+log = "0.4.11"
 Inflector = "0.11.4"
 indicatif = "0.15.0"
 humantime = "2.0.1"
 num-traits = "0.2"
-pretty-hex = "0.1.1"
-reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+pretty-hex = "0.2.1"
+reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-cli-config = { path = "../cli-config", version = "1.4.0" }
-solana-cli-output = { path = "../cli-output", version = "1.4.0" }
-solana-client = { path = "../client", version = "1.4.0" }
-solana-config-program = { path = "../programs/config", version = "1.4.0" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
-solana-vote-signer = { path = "../vote-signer", version = "1.4.0" }
-thiserror = "1.0.20"
+solana-account-decoder = { path = "../account-decoder", version = "1.5.3" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.3" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-cli-config = { path = "../cli-config", version = "1.5.3" }
+solana-cli-output = { path = "../cli-output", version = "1.5.3" }
+solana-client = { path = "../client", version = "1.5.3" }
+solana-config-program = { path = "../programs/config", version = "1.5.3" }
+solana-faucet = { path = "../faucet", version = "1.5.3" }
+solana-logger = { path = "../logger", version = "1.5.3" }
+solana-net-utils = { path = "../net-utils", version = "1.5.3" }
+solana_rbpf = "=0.2.3"
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.3" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.3" }
+thiserror = "1.0.21"
+tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.4.0" }
+solana-core = { path = "../core", version = "1.5.3" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/checks.rs

@@ -54,12 +54,42 @@ pub fn check_account_for_multiple_fees_with_commitment(
     fee_calculator: &FeeCalculator,
     messages: &[&Message],
     commitment: CommitmentConfig,
+) -> Result<(), CliError> {
+    check_account_for_spend_multiple_fees_with_commitment(
+        rpc_client,
+        account_pubkey,
+        0,
+        fee_calculator,
+        messages,
+        commitment,
+    )
+}
+
+pub fn check_account_for_spend_multiple_fees_with_commitment(
+    rpc_client: &RpcClient,
+    account_pubkey: &Pubkey,
+    balance: u64,
+    fee_calculator: &FeeCalculator,
+    messages: &[&Message],
+    commitment: CommitmentConfig,
 ) -> Result<(), CliError> {
     let fee = calculate_fee(fee_calculator, messages);
-    if !check_account_for_balance_with_commitment(rpc_client, account_pubkey, fee, commitment)
-        .map_err(Into::<ClientError>::into)?
+    if !check_account_for_balance_with_commitment(
+        rpc_client,
+        account_pubkey,
+        balance + fee,
+        commitment,
+    )
+    .map_err(Into::<ClientError>::into)?
     {
-        return Err(CliError::InsufficientFundsForFee(lamports_to_sol(fee)));
+        if balance > 0 {
+            return Err(CliError::InsufficientFundsForSpendAndFee(
+                lamports_to_sol(balance),
+                lamports_to_sol(fee),
+            ));
+        } else {
+            return Err(CliError::InsufficientFundsForFee(lamports_to_sol(fee)));
+        }
     }
     Ok(())
 }
@@ -131,7 +161,7 @@ mod tests {
             context: RpcResponseContext { slot: 1 },
             value: json!(account_balance),
         });
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let fee_calculator = FeeCalculator::new(1);
 
         let pubkey0 = Pubkey::new(&[0; 32]);
@@ -191,7 +221,7 @@ mod tests {
             context: RpcResponseContext { slot: 1 },
             value: json!(account_balance),
         });
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
 
         let mut mocks = HashMap::new();
         mocks.insert(RpcRequest::GetBalance, account_balance_response);
@@ -237,9 +267,9 @@ mod tests {
 
     #[test]
     fn test_check_unique_pubkeys() {
-        let pubkey0 = Pubkey::new_rand();
+        let pubkey0 = solana_sdk::pubkey::new_rand();
         let pubkey_clone = pubkey0;
-        let pubkey1 = Pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
 
         check_unique_pubkeys((&pubkey0, "foo".to_string()), (&pubkey1, "bar".to_string()))
             .expect("unexpected result");

cli/src/cli.rs

@@ -1,11 +1,11 @@
 use crate::{
-    checks::*, cluster_query::*, feature::*, inflation::*, nonce::*, spend_utils::*, stake::*,
+    cluster_query::*, feature::*, inflation::*, nonce::*, program::*, spend_utils::*, stake::*,
     validator_info::*, vote::*,
 };
 use clap::{value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use log::*;
 use num_traits::FromPrimitive;
-use serde_json::{self, json, Value};
+use serde_json::{self, Value};
 use solana_account_decoder::{UiAccount, UiAccountEncoding};
 use solana_clap_utils::{
     self,
@@ -18,9 +18,7 @@ use solana_clap_utils::{
     offline::*,
 };
 use solana_cli_output::{
-    display::{
-        build_balance_message, new_spinner_progress_bar, println_name_value, println_transaction,
-    },
+    display::{build_balance_message, println_name_value, println_transaction},
     return_signers, CliAccount, CliSignature, OutputFormat,
 };
 use solana_client::{
@@ -28,8 +26,7 @@ use solana_client::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     nonce_utils,
     rpc_client::RpcClient,
-    rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig},
-    rpc_request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS,
+    rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig, RpcTransactionLogsFilter},
     rpc_response::RpcKeyedAccount,
 };
 #[cfg(not(test))]
@@ -38,17 +35,14 @@ use solana_faucet::faucet::request_airdrop_transaction;
 use solana_faucet::faucet_mock::request_airdrop_transaction;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    bpf_loader, bpf_loader_deprecated,
-    clock::{Epoch, Slot, DEFAULT_TICKS_PER_SECOND},
+    clock::{Epoch, Slot},
     commitment_config::CommitmentConfig,
     decode_error::DecodeError,
     hash::Hash,
-    instruction::{Instruction, InstructionError},
-    loader_instruction,
+    instruction::InstructionError,
     message::Message,
     pubkey::{Pubkey, MAX_SEED_LEN},
-    signature::{Keypair, Signature, Signer, SignerError},
-    signers::Signers,
+    signature::{Signature, Signer, SignerError},
     system_instruction::{self, SystemError},
     system_program,
     transaction::{Transaction, TransactionError},
@@ -64,7 +58,7 @@ use std::{
     error,
     fmt::Write as FmtWrite,
     fs::File,
-    io::{Read, Write},
+    io::Write,
     net::{IpAddr, SocketAddr},
     str::FromStr,
     sync::Arc,
@@ -74,7 +68,6 @@ use std::{
 use thiserror::Error;
 use url::Url;
 
-const DATA_CHUNK_SIZE: usize = 229; // Keep program chunks under PACKET_DATA_SIZE
 pub const DEFAULT_RPC_TIMEOUT_SECONDS: &str = "30";
 
 #[derive(Debug, PartialEq)]
@@ -82,9 +75,11 @@ pub const DEFAULT_RPC_TIMEOUT_SECONDS: &str = "30";
 pub enum CliCommand {
     // Cluster Query Commands
     Catchup {
-        node_pubkey: Pubkey,
+        node_pubkey: Option<Pubkey>,
         node_json_rpc_url: Option<String>,
         follow: bool,
+        our_localhost_port: Option<u16>,
+        log: bool,
     },
     ClusterDate,
     ClusterVersion,
@@ -98,7 +93,7 @@ pub enum CliCommand {
     Fees,
     FirstAvailableBlock,
     GetBlock {
-        slot: Slot,
+        slot: Option<Slot>,
     },
     GetBlockTime {
         slot: Option<Slot>,
@@ -112,13 +107,20 @@ pub enum CliCommand {
     LargestAccounts {
         filter: Option<RpcLargestAccountsFilter>,
     },
-    LeaderSchedule,
+    LeaderSchedule {
+        epoch: Option<Epoch>,
+    },
     LiveSlots,
+    Logs {
+        filter: RpcTransactionLogsFilter,
+    },
     Ping {
         lamports: u64,
         interval: Duration,
         count: Option<u64>,
         timeout: Duration,
+        blockhash: Option<Hash>,
+        print_timestamp: bool,
     },
     ShowBlockProduction {
         epoch: Option<Epoch>,
@@ -143,6 +145,9 @@ pub enum CliCommand {
         limit: usize,
         show_transactions: bool,
     },
+    WaitForMaxStake {
+        max_stake_percent: f32,
+    },
     // Nonce commands
     AuthorizeNonceAccount {
         nonce_account: Pubkey,
@@ -175,7 +180,9 @@ pub enum CliCommand {
         program_location: String,
         address: Option<SignerIndex>,
         use_deprecated_loader: bool,
+        allow_excessive_balance: bool,
     },
+    Program(ProgramCliCommand),
     // Stake Commands
     CreateStakeAccount {
         stake_account: SignerIndex,
@@ -429,7 +436,7 @@ impl CliConfig<'_> {
     ) -> (SettingType, String) {
         settings
             .into_iter()
-            .find(|(_, value)| value != "")
+            .find(|(_, value)| !value.is_empty())
             .expect("no nonempty setting")
     }
 
@@ -458,11 +465,12 @@ impl CliConfig<'_> {
         json_rpc_cmd_url: &str,
         json_rpc_cfg_url: &str,
     ) -> (SettingType, String) {
-        Self::first_nonempty_setting(vec![
+        let (setting_type, url_or_moniker) = Self::first_nonempty_setting(vec![
             (SettingType::Explicit, json_rpc_cmd_url.to_string()),
             (SettingType::Explicit, json_rpc_cfg_url.to_string()),
             (SettingType::SystemDefault, Self::default_json_rpc_url()),
-        ])
+        ]);
+        (setting_type, normalize_to_url_if_moniker(&url_or_moniker))
     }
 
     pub fn compute_keypair_path_setting(
@@ -487,13 +495,15 @@ impl CliConfig<'_> {
     }
 
     pub fn recent_for_tests() -> Self {
-        let mut config = Self::default();
-        config.commitment = CommitmentConfig::recent();
-        config.send_transaction_config = RpcSendTransactionConfig {
-            skip_preflight: true,
-            ..RpcSendTransactionConfig::default()
-        };
-        config
+        Self {
+            commitment: CommitmentConfig::recent(),
+            send_transaction_config: RpcSendTransactionConfig {
+                skip_preflight: true,
+                preflight_commitment: Some(CommitmentConfig::recent().commitment),
+                ..RpcSendTransactionConfig::default()
+            },
+            ..Self::default()
+        }
     }
 }
 
@@ -566,15 +576,13 @@ pub fn parse_command(
         ("supply", Some(matches)) => parse_supply(matches),
         ("total-supply", Some(matches)) => parse_total_supply(matches),
         ("transaction-count", Some(matches)) => parse_get_transaction_count(matches),
-        ("leader-schedule", Some(_matches)) => Ok(CliCommandInfo {
-            command: CliCommand::LeaderSchedule,
-            signers: vec![],
-        }),
+        ("leader-schedule", Some(matches)) => parse_leader_schedule(matches),
         ("ping", Some(matches)) => parse_cluster_ping(matches, default_signer, wallet_manager),
         ("live-slots", Some(_matches)) => Ok(CliCommandInfo {
             command: CliCommand::LiveSlots,
             signers: vec![],
         }),
+        ("logs", Some(matches)) => parse_logs(matches, wallet_manager),
         ("block-production", Some(matches)) => parse_show_block_production(matches),
         ("gossip", Some(_matches)) => Ok(CliCommandInfo {
             command: CliCommand::ShowGossip,
@@ -606,17 +614,27 @@ pub fn parse_command(
                 signers.push(signer);
                 1
             });
-            let use_deprecated_loader = matches.is_present("use_deprecated_loader");
 
             Ok(CliCommandInfo {
                 command: CliCommand::Deploy {
                     program_location: matches.value_of("program_location").unwrap().to_string(),
                     address,
-                    use_deprecated_loader,
+                    use_deprecated_loader: matches.is_present("use_deprecated_loader"),
+                    allow_excessive_balance: matches.is_present("allow_excessive_balance"),
                 },
                 signers,
             })
         }
+        ("program", Some(matches)) => {
+            parse_program_subcommand(matches, default_signer, wallet_manager)
+        }
+        ("wait-for-max-stake", Some(matches)) => {
+            let max_stake_percent = value_t_or_exit!(matches, "max_percent", f32);
+            Ok(CliCommandInfo {
+                command: CliCommand::WaitForMaxStake { max_stake_percent },
+                signers: vec![],
+            })
+        }
         // Stake Commands
         ("create-stake-account", Some(matches)) => {
             parse_stake_create_account(matches, default_signer, wallet_manager)
@@ -972,6 +990,7 @@ fn process_confirm(
     }
 }
 
+#[allow(clippy::unnecessary_wraps)]
 fn process_decode_transaction(transaction: &Transaction) -> ProcessResult {
     println_transaction(transaction, &None, "");
     Ok("".to_string())
@@ -1017,309 +1036,6 @@ fn process_show_account(
     Ok(account_string)
 }
 
-fn send_and_confirm_transactions_with_spinner<T: Signers>(
-    rpc_client: &RpcClient,
-    mut transactions: Vec<Transaction>,
-    signer_keys: &T,
-    commitment: CommitmentConfig,
-    mut last_valid_slot: Slot,
-) -> Result<(), Box<dyn error::Error>> {
-    let progress_bar = new_spinner_progress_bar();
-    let mut send_retries = 5;
-    loop {
-        let mut status_retries = 15;
-
-        // Send all transactions
-        let mut pending_transactions = HashMap::new();
-        let num_transactions = transactions.len();
-        for transaction in transactions {
-            if cfg!(not(test)) {
-                // Delay ~1 tick between write transactions in an attempt to reduce AccountInUse errors
-                // when all the write transactions modify the same program account (eg, deploying a
-                // new program)
-                sleep(Duration::from_millis(1000 / DEFAULT_TICKS_PER_SECOND));
-            }
-
-            let _result = rpc_client
-                .send_transaction_with_config(
-                    &transaction,
-                    RpcSendTransactionConfig {
-                        preflight_commitment: Some(commitment.commitment),
-                        ..RpcSendTransactionConfig::default()
-                    },
-                )
-                .ok();
-            pending_transactions.insert(transaction.signatures[0], transaction);
-
-            progress_bar.set_message(&format!(
-                "[{}/{}] Transactions sent",
-                pending_transactions.len(),
-                num_transactions
-            ));
-        }
-
-        // Collect statuses for all the transactions, drop those that are confirmed
-        while status_retries > 0 {
-            status_retries -= 1;
-
-            progress_bar.set_message(&format!(
-                "[{}/{}] Transactions confirmed",
-                num_transactions - pending_transactions.len(),
-                num_transactions
-            ));
-
-            let mut statuses = vec![];
-            let pending_signatures = pending_transactions.keys().cloned().collect::<Vec<_>>();
-            for pending_signatures_chunk in
-                pending_signatures.chunks(MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS - 1)
-            {
-                statuses.extend(
-                    rpc_client
-                        .get_signature_statuses_with_history(pending_signatures_chunk)?
-                        .value
-                        .into_iter(),
-                );
-            }
-            assert_eq!(statuses.len(), pending_signatures.len());
-
-            for (signature, status) in pending_signatures.into_iter().zip(statuses.into_iter()) {
-                if let Some(status) = status {
-                    if status.confirmations.is_none() || status.confirmations.unwrap() > 1 {
-                        let _ = pending_transactions.remove(&signature);
-                    }
-                }
-            }
-
-            if pending_transactions.is_empty() {
-                return Ok(());
-            }
-
-            let slot = rpc_client.get_slot_with_commitment(commitment)?;
-            if slot > last_valid_slot {
-                break;
-            }
-
-            if cfg!(not(test)) {
-                // Retry twice a second
-                sleep(Duration::from_millis(500));
-            }
-        }
-
-        if send_retries == 0 {
-            return Err("Transactions failed".into());
-        }
-        send_retries -= 1;
-
-        // Re-sign any failed transactions with a new blockhash and retry
-        let (blockhash, _fee_calculator, new_last_valid_slot) = rpc_client
-            .get_recent_blockhash_with_commitment(commitment)?
-            .value;
-        last_valid_slot = new_last_valid_slot;
-        transactions = vec![];
-        for (_, mut transaction) in pending_transactions.into_iter() {
-            transaction.try_sign(signer_keys, blockhash)?;
-            transactions.push(transaction);
-        }
-    }
-}
-
-fn process_deploy(
-    rpc_client: &RpcClient,
-    config: &CliConfig,
-    program_location: &str,
-    address: Option<SignerIndex>,
-    use_deprecated_loader: bool,
-) -> ProcessResult {
-    let new_keypair = Keypair::new(); // Create ephemeral keypair to use for program address, if not provided
-    let program_id = if let Some(i) = address {
-        config.signers[i]
-    } else {
-        &new_keypair
-    };
-    let mut file = File::open(program_location).map_err(|err| {
-        CliError::DynamicProgramError(format!("Unable to open program file: {}", err))
-    })?;
-    let mut program_data = Vec::new();
-    file.read_to_end(&mut program_data).map_err(|err| {
-        CliError::DynamicProgramError(format!("Unable to read program file: {}", err))
-    })?;
-
-    let loader_id = if use_deprecated_loader {
-        bpf_loader_deprecated::id()
-    } else {
-        bpf_loader::id()
-    };
-
-    let minimum_balance = rpc_client.get_minimum_balance_for_rent_exemption(program_data.len())?;
-    let signers = [config.signers[0], program_id];
-
-    // Check program account to see if partial initialization has occurred
-    let initial_instructions = if let Some(account) = rpc_client
-        .get_account_with_commitment(&program_id.pubkey(), config.commitment)?
-        .value
-    {
-        let mut instructions: Vec<Instruction> = vec![];
-        if account.executable {
-            return Err(CliError::DynamicProgramError(
-                "Program account is already executable".to_string(),
-            )
-            .into());
-        }
-        if account.owner != loader_id && !system_program::check_id(&account.owner) {
-            return Err(CliError::DynamicProgramError(
-                "Program account is already owned by another account".to_string(),
-            )
-            .into());
-        }
-
-        if account.data.is_empty() && system_program::check_id(&account.owner) {
-            instructions.push(system_instruction::allocate(
-                &program_id.pubkey(),
-                program_data.len() as u64,
-            ));
-            if account.owner != loader_id {
-                instructions.push(system_instruction::assign(&program_id.pubkey(), &loader_id));
-            }
-        }
-        if account.lamports < minimum_balance {
-            instructions.push(system_instruction::transfer(
-                &config.signers[0].pubkey(),
-                &program_id.pubkey(),
-                minimum_balance - account.lamports,
-            ));
-        }
-        instructions
-    } else {
-        vec![system_instruction::create_account(
-            &config.signers[0].pubkey(),
-            &program_id.pubkey(),
-            minimum_balance,
-            program_data.len() as u64,
-            &loader_id,
-        )]
-    };
-    let initial_message = if !initial_instructions.is_empty() {
-        Some(Message::new(
-            &initial_instructions,
-            Some(&config.signers[0].pubkey()),
-        ))
-    } else {
-        None
-    };
-
-    // Build transactions to calculate fees
-    let mut messages: Vec<&Message> = Vec::new();
-
-    if let Some(message) = &initial_message {
-        messages.push(message);
-    }
-
-    let mut write_messages = vec![];
-    for (chunk, i) in program_data.chunks(DATA_CHUNK_SIZE).zip(0..) {
-        let instruction = loader_instruction::write(
-            &program_id.pubkey(),
-            &loader_id,
-            (i * DATA_CHUNK_SIZE) as u32,
-            chunk.to_vec(),
-        );
-        let message = Message::new(&[instruction], Some(&signers[0].pubkey()));
-        write_messages.push(message);
-    }
-    let mut write_message_refs = vec![];
-    for message in write_messages.iter() {
-        write_message_refs.push(message);
-    }
-    messages.append(&mut write_message_refs);
-
-    let instruction = loader_instruction::finalize(&program_id.pubkey(), &loader_id);
-    let finalize_message = Message::new(&[instruction], Some(&signers[0].pubkey()));
-    messages.push(&finalize_message);
-
-    let (blockhash, fee_calculator, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
-
-    check_account_for_multiple_fees_with_commitment(
-        rpc_client,
-        &config.signers[0].pubkey(),
-        &fee_calculator,
-        &messages,
-        config.commitment,
-    )?;
-
-    if let Some(message) = initial_message {
-        trace!("Creating or modifying program account");
-        let num_required_signatures = message.header.num_required_signatures;
-
-        let mut initial_transaction = Transaction::new_unsigned(message);
-        // Most of the initial_transaction combinations require both the fee-payer and new program
-        // account to sign the transaction. One (transfer) only requires the fee-payer signature.
-        // This check is to ensure signing does not fail on a KeypairPubkeyMismatch error from an
-        // extraneous signature.
-        if num_required_signatures == 2 {
-            initial_transaction.try_sign(&signers, blockhash)?;
-        } else {
-            initial_transaction.try_sign(&[signers[0]], blockhash)?;
-        }
-        let result = rpc_client.send_and_confirm_transaction_with_spinner_and_config(
-            &initial_transaction,
-            config.commitment,
-            config.send_transaction_config,
-        );
-        log_instruction_custom_error::<SystemError>(result, &config).map_err(|_| {
-            CliError::DynamicProgramError("Program account allocation failed".to_string())
-        })?;
-    }
-
-    let (blockhash, _, last_valid_slot) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
-
-    let mut write_transactions = vec![];
-    for message in write_messages.into_iter() {
-        let mut tx = Transaction::new_unsigned(message);
-        tx.try_sign(&signers, blockhash)?;
-        write_transactions.push(tx);
-    }
-
-    trace!("Writing program data");
-    send_and_confirm_transactions_with_spinner(
-        &rpc_client,
-        write_transactions,
-        &signers,
-        config.commitment,
-        last_valid_slot,
-    )
-    .map_err(|_| {
-        CliError::DynamicProgramError("Data writes to program account failed".to_string())
-    })?;
-
-    let (blockhash, _, _) = rpc_client
-        .get_recent_blockhash_with_commitment(config.commitment)?
-        .value;
-    let mut finalize_tx = Transaction::new_unsigned(finalize_message);
-    finalize_tx.try_sign(&signers, blockhash)?;
-
-    trace!("Finalizing program account");
-    rpc_client
-        .send_and_confirm_transaction_with_spinner_and_config(
-            &finalize_tx,
-            config.commitment,
-            RpcSendTransactionConfig {
-                skip_preflight: true,
-                ..RpcSendTransactionConfig::default()
-            },
-        )
-        .map_err(|e| {
-            CliError::DynamicProgramError(format!("Finalizing program account failed: {}", e))
-        })?;
-
-    Ok(json!({
-        "programId": format!("{}", program_id.pubkey()),
-    })
-    .to_string())
-}
-
 #[allow(clippy::too_many_arguments)]
 fn process_transfer(
     rpc_client: &RpcClient,
@@ -1425,7 +1141,17 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             node_pubkey,
             node_json_rpc_url,
             follow,
-        } => process_catchup(&rpc_client, config, node_pubkey, node_json_rpc_url, *follow),
+            our_localhost_port,
+            log,
+        } => process_catchup(
+            &rpc_client,
+            config,
+            *node_pubkey,
+            node_json_rpc_url.clone(),
+            *follow,
+            *our_localhost_port,
+            *log,
+        ),
         CliCommand::ClusterDate => process_cluster_date(&rpc_client, config),
         CliCommand::ClusterVersion => process_cluster_version(&rpc_client, config),
         CliCommand::CreateAddressWithSeed {
@@ -1452,14 +1178,28 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::Inflation(inflation_subcommand) => {
             process_inflation_subcommand(&rpc_client, config, inflation_subcommand)
         }
-        CliCommand::LeaderSchedule => process_leader_schedule(&rpc_client),
-        CliCommand::LiveSlots => process_live_slots(&config.websocket_url),
+        CliCommand::LeaderSchedule { epoch } => {
+            process_leader_schedule(&rpc_client, config, *epoch)
+        }
+        CliCommand::LiveSlots => process_live_slots(&config),
+        CliCommand::Logs { filter } => process_logs(&config, filter),
         CliCommand::Ping {
             lamports,
             interval,
             count,
             timeout,
-        } => process_ping(&rpc_client, config, *lamports, interval, count, timeout),
+            blockhash,
+            print_timestamp,
+        } => process_ping(
+            &rpc_client,
+            config,
+            *lamports,
+            interval,
+            count,
+            timeout,
+            blockhash,
+            *print_timestamp,
+        ),
         CliCommand::ShowBlockProduction { epoch, slot_limit } => {
             process_show_block_production(&rpc_client, config, *epoch, *slot_limit)
         }
@@ -1473,6 +1213,9 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *use_lamports_unit,
             vote_account_pubkeys.as_deref(),
         ),
+        CliCommand::WaitForMaxStake { max_stake_percent } => {
+            process_wait_for_max_stake(&rpc_client, config, *max_stake_percent)
+        }
         CliCommand::ShowValidators { use_lamports_unit } => {
             process_show_validators(&rpc_client, config, *use_lamports_unit)
         }
@@ -1565,13 +1308,18 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             program_location,
             address,
             use_deprecated_loader,
+            allow_excessive_balance,
         } => process_deploy(
             &rpc_client,
             config,
             program_location,
             *address,
             *use_deprecated_loader,
+            *allow_excessive_balance,
         ),
+        CliCommand::Program(program_subcommand) => {
+            process_program_subcommand(&rpc_client, config, program_subcommand)
+        }
 
         // Stake Commands
 
@@ -2057,6 +1805,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
         .feature_subcommands()
         .inflation_subcommands()
         .nonce_subcommands()
+        .program_subcommands()
         .stake_subcommands()
         .subcommand(
             SubCommand::with_name("airdrop")
@@ -2187,7 +1936,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                 .arg(
                     Arg::with_name("address_signer")
                         .index(2)
-                        .value_name("SIGNER_KEYPAIR")
+                        .value_name("PROGRAM_ADDRESS_SIGNER")
                         .takes_value(true)
                         .validator(is_valid_signer)
                         .help("The signer for the desired address of the program [default: new random address]")
@@ -2199,7 +1948,13 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .hidden(true) // Don't document this argument to discourage its use
                         .help("Use the deprecated BPF loader")
                 )
-                .arg(commitment_arg_with_default("max")),
+                .arg(
+                    Arg::with_name("allow_excessive_balance")
+                        .long("allow-excessive-deploy-account-balance")
+                        .takes_value(false)
+                        .help("Use the designated program id, even if the account already holds a large balance of SOL")
+                )
+                .arg(commitment_arg_with_default("singleGossip")),
         )
         .subcommand(
             SubCommand::with_name("pay")
@@ -2304,7 +2059,7 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
 #[cfg(test)]
 mod tests {
     use super::*;
-    use serde_json::Value;
+    use serde_json::{json, Value};
     use solana_client::{
         blockhash_query,
         mock_sender::SIGNATURE,
@@ -2313,7 +2068,7 @@ mod tests {
     };
     use solana_sdk::{
         pubkey::Pubkey,
-        signature::{keypair_from_seed, read_keypair_file, write_keypair_file, Presigner},
+        signature::{keypair_from_seed, read_keypair_file, write_keypair_file, Keypair, Presigner},
         transaction::TransactionError,
     };
     use std::path::PathBuf;
@@ -2355,7 +2110,10 @@ mod tests {
             .unwrap();
         assert_eq!(signer_info.signers.len(), 1);
         assert_eq!(signer_info.index_of(None), Some(0));
-        assert_eq!(signer_info.index_of(Some(Pubkey::new_rand())), None);
+        assert_eq!(
+            signer_info.index_of(Some(solana_sdk::pubkey::new_rand())),
+            None
+        );
 
         let keypair0 = keypair_from_seed(&[1u8; 32]).unwrap();
         let keypair0_pubkey = keypair0.pubkey();
@@ -2411,7 +2169,7 @@ mod tests {
     fn test_cli_parse_command() {
         let test_commands = app("test", "desc", "version");
 
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let pubkey_string = format!("{}", pubkey);
 
         let default_keypair = Keypair::new();
@@ -2507,7 +2265,7 @@ mod tests {
         assert!(parse_command(&test_bad_signature, &default_signer, &mut None).is_err());
 
         // Test CreateAddressWithSeed
-        let from_pubkey = Some(Pubkey::new_rand());
+        let from_pubkey = Some(solana_sdk::pubkey::new_rand());
         let from_str = from_pubkey.unwrap().to_string();
         for (name, program_id) in &[
             ("STAKE", solana_stake_program::id()),
@@ -2564,6 +2322,7 @@ mod tests {
                     program_location: "/Users/test/program.o".to_string(),
                     address: None,
                     use_deprecated_loader: false,
+                    allow_excessive_balance: false,
                 },
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2585,6 +2344,7 @@ mod tests {
                     program_location: "/Users/test/program.o".to_string(),
                     address: Some(1),
                     use_deprecated_loader: false,
+                    allow_excessive_balance: false,
                 },
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2623,9 +2383,11 @@ mod tests {
     #[allow(clippy::cognitive_complexity)]
     fn test_cli_process_command() {
         // Success cases
-        let mut config = CliConfig::default();
-        config.rpc_client = Some(RpcClient::new_mock("succeeds".to_string()));
-        config.json_rpc_url = "http://127.0.0.1:8899".to_string();
+        let mut config = CliConfig {
+            rpc_client: Some(RpcClient::new_mock("succeeds".to_string())),
+            json_rpc_url: "http://127.0.0.1:8899".to_string(),
+            ..CliConfig::default()
+        };
 
         let keypair = Keypair::new();
         let pubkey = keypair.pubkey().to_string();
@@ -2664,7 +2426,7 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let new_authorized_pubkey = Pubkey::new_rand();
+        let new_authorized_pubkey = solana_sdk::pubkey::new_rand();
         config.signers = vec![&bob_keypair];
         config.command = CliCommand::VoteAuthorize {
             vote_account_pubkey: bob_pubkey,
@@ -2686,7 +2448,7 @@ mod tests {
 
         let bob_keypair = Keypair::new();
         let bob_pubkey = bob_keypair.pubkey();
-        let custodian = Pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         config.command = CliCommand::CreateStakeAccount {
             stake_account: 1,
             seed: None,
@@ -2709,8 +2471,8 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
-        let to_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
+        let to_pubkey = solana_sdk::pubkey::new_rand();
         config.command = CliCommand::WithdrawStake {
             stake_account_pubkey,
             destination_account_pubkey: to_pubkey,
@@ -2727,7 +2489,7 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
         config.command = CliCommand::DeactivateStake {
             stake_account_pubkey,
             stake_authority: 0,
@@ -2740,7 +2502,7 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
         let split_stake_account = Keypair::new();
         config.command = CliCommand::SplitStake {
             stake_account_pubkey,
@@ -2758,8 +2520,8 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_account_pubkey = Pubkey::new_rand();
-        let source_stake_account_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = solana_sdk::pubkey::new_rand();
+        let source_stake_account_pubkey = solana_sdk::pubkey::new_rand();
         let merge_stake_account = Keypair::new();
         config.command = CliCommand::MergeStake {
             stake_account_pubkey,
@@ -2782,7 +2544,7 @@ mod tests {
         assert_eq!(process_command(&config).unwrap(), "1234");
 
         // CreateAddressWithSeed
-        let from_pubkey = Pubkey::new_rand();
+        let from_pubkey = solana_sdk::pubkey::new_rand();
         config.signers = vec![];
         config.command = CliCommand::CreateAddressWithSeed {
             from_pubkey: Some(from_pubkey),
@@ -2795,7 +2557,7 @@ mod tests {
         assert_eq!(address.unwrap(), expected_address.to_string());
 
         // Need airdrop cases
-        let to = Pubkey::new_rand();
+        let to = solana_sdk::pubkey::new_rand();
         config.signers = vec![&keypair];
         config.command = CliCommand::Airdrop {
             faucet_host: None,
@@ -2898,13 +2660,14 @@ mod tests {
             program_location: pathbuf.to_str().unwrap().to_string(),
             address: None,
             use_deprecated_loader: false,
+            allow_excessive_balance: false,
         };
         let result = process_command(&config);
         let json: Value = serde_json::from_str(&result.unwrap()).unwrap();
         let program_id = json
             .as_object()
             .unwrap()
-            .get("programId")
+            .get("ProgramId")
             .unwrap()
             .as_str()
             .unwrap();
@@ -2916,6 +2679,7 @@ mod tests {
             program_location: "bad/file/location.so".to_string(),
             address: None,
             use_deprecated_loader: false,
+            allow_excessive_balance: false,
         };
         assert!(process_command(&config).is_err());
     }

cli/src/cluster_query.rs

@@ -1,12 +1,17 @@
 use crate::{
     cli::{CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult},
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
+    stake::is_stake_program_v2_enabled,
 };
 use chrono::{Local, TimeZone};
 use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use solana_clap_utils::{
-    commitment::commitment_arg, input_parsers::*, input_validators::*, keypair::DefaultSigner,
+    commitment::{commitment_arg, commitment_arg_with_default},
+    input_parsers::*,
+    input_validators::*,
+    keypair::DefaultSigner,
+    offline::{blockhash_arg, BLOCKHASH_ARG},
 };
 use solana_cli_output::{
     display::{
@@ -20,26 +25,28 @@ use solana_client::{
     rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClient},
     rpc_config::{
         RpcAccountInfoConfig, RpcLargestAccountsConfig, RpcLargestAccountsFilter,
-        RpcProgramAccountsConfig,
+        RpcProgramAccountsConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter,
     },
     rpc_filter,
     rpc_response::SlotInfo,
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
+    account::from_account,
     account_utils::StateMut,
     clock::{self, Clock, Slot},
     commitment_config::CommitmentConfig,
     epoch_schedule::Epoch,
+    hash::Hash,
     message::Message,
     native_token::lamports_to_sol,
     pubkey::{self, Pubkey},
+    rpc_port::DEFAULT_RPC_PORT_STR,
     signature::Signature,
     system_instruction, system_program,
     sysvar::{
         self,
-        stake_history::{self, StakeHistory},
-        Sysvar,
+        stake_history::{self},
     },
     transaction::Transaction,
 };
@@ -52,7 +59,7 @@ use std::{
         Arc,
     },
     thread::sleep,
-    time::{Duration, Instant},
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 
 static CHECK_MARK: Emoji = Emoji("✅ ", "");
@@ -73,8 +80,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .validator(is_slot)
                         .value_name("SLOT")
                         .takes_value(true)
-                        .index(1)
-                        .required(true),
+                        .index(1),
                 ),
         )
         .subcommand(
@@ -83,14 +89,14 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 .arg(
                     pubkey!(Arg::with_name("node_pubkey")
                         .index(1)
-                        .value_name("VALIDATOR_PUBKEY")
-                        .required(true),
+                        .value_name("OUR_VALIDATOR_PUBKEY")
+                        .required(false),
                         "Identity pubkey of the validator"),
                 )
                 .arg(
                     Arg::with_name("node_json_rpc_url")
                         .index(2)
-                        .value_name("URL")
+                        .value_name("OUR_URL")
                         .takes_value(true)
                         .validator(is_url)
                         .help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
@@ -101,6 +107,21 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .takes_value(false)
                         .help("Continue reporting progress even after the validator has caught up"),
                 )
+                .arg(
+                    Arg::with_name("our_localhost")
+                        .long("our-localhost")
+                        .takes_value(false)
+                        .value_name("PORT")
+                        .default_value(&DEFAULT_RPC_PORT_STR)
+                        .validator(is_port)
+                        .help("Guess Identity pubkey and validator rpc node assuming local (possibly private) validator"),
+                )
+                .arg(
+                    Arg::with_name("log")
+                        .long("log")
+                        .takes_value(false)
+                        .help("Don't update the progress inplace; instead show updates with its own new lines"),
+                )
                 .arg(commitment_arg()),
         )
         .subcommand(
@@ -127,7 +148,17 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .help("Slot number of the block to query")
             )
         )
-        .subcommand(SubCommand::with_name("leader-schedule").about("Display leader schedule"))
+        .subcommand(SubCommand::with_name("leader-schedule")
+            .about("Display leader schedule")
+            .arg(
+                Arg::with_name("epoch")
+                    .long("epoch")
+                    .takes_value(true)
+                    .value_name("EPOCH")
+                    .validator(is_epoch)
+                    .help("Epoch to show leader schedule for. (default: current)")
+            )
+        )
         .subcommand(
             SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
@@ -209,6 +240,13 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .takes_value(true)
                         .help("Stop after submitting count transactions"),
                 )
+                .arg(
+                    Arg::with_name("print_timestamp")
+                        .short("D")
+                        .long("print-timestamp")
+                        .takes_value(false)
+                        .help("Print timestamp (unix time + microseconds as in gettimeofday) before each line"),
+                )
                 .arg(
                     Arg::with_name("lamports")
                         .long("lamports")
@@ -227,12 +265,33 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .default_value("15")
                         .help("Wait up to timeout seconds for transaction confirmation"),
                 )
+                .arg(blockhash_arg())
                 .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("live-slots")
                 .about("Show information about the current slot progression"),
         )
+        .subcommand(
+            SubCommand::with_name("logs")
+                .about("Stream transaction logs")
+                .arg(
+                    pubkey!(Arg::with_name("address")
+                        .index(1)
+                        .value_name("ADDRESS"),
+                        "Account address to monitor \
+                         [default: monitor all transactions except for votes] \
+                        ")
+                )
+                .arg(
+                    Arg::with_name("include_votes")
+                        .long("include-votes")
+                        .takes_value(false)
+                        .conflicts_with("address")
+                        .help("Include vote transactions when monitoring all transactions")
+                )
+                .arg(commitment_arg_with_default("singleGossip")),
+        )
         .subcommand(
             SubCommand::with_name("block-production")
                 .about("Show information about block production")
@@ -318,6 +377,17 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .help("Display the full transactions"),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("wait-for-max-stake")
+                .about("Wait for the max stake of any one node to drop below a percentage of total.")
+                .arg(
+                    Arg::with_name("max_percent")
+                        .long("max-percent")
+                        .value_name("PERCENT")
+                        .takes_value(true)
+                        .index(1),
+                ),
+        )
     }
 }
 
@@ -325,14 +395,31 @@ pub fn parse_catchup(
     matches: &ArgMatches<'_>,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
+    let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?;
+    let mut our_localhost_port = value_t!(matches, "our_localhost", u16).ok();
+    // if there is no explicitly specified --our-localhost,
+    // disable the guess mode (= our_localhost_port)
+    if matches.occurrences_of("our_localhost") == 0 {
+        our_localhost_port = None
+    }
     let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
+    // requirement of node_pubkey is relaxed only if our_localhost_port
+    if our_localhost_port.is_none() && node_pubkey.is_none() {
+        return Err(CliError::BadParameter(
+            "OUR_VALIDATOR_PUBKEY (and possibly OUR_URL) must be specified \
+             unless --our-localhost is given"
+                .into(),
+        ));
+    }
     let follow = matches.is_present("follow");
+    let log = matches.is_present("log");
     Ok(CliCommandInfo {
         command: CliCommand::Catchup {
             node_pubkey,
             node_json_rpc_url,
             follow,
+            our_localhost_port,
+            log,
         },
         signers: vec![],
     })
@@ -351,19 +438,23 @@ pub fn parse_cluster_ping(
         None
     };
     let timeout = Duration::from_secs(value_t_or_exit!(matches, "timeout", u64));
+    let blockhash = value_of(matches, BLOCKHASH_ARG.name);
+    let print_timestamp = matches.is_present("print_timestamp");
     Ok(CliCommandInfo {
         command: CliCommand::Ping {
             lamports,
             interval,
             count,
             timeout,
+            blockhash,
+            print_timestamp,
         },
         signers: vec![default_signer.signer_from_path(matches, wallet_manager)?],
     })
 }
 
 pub fn parse_get_block(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let slot = value_t_or_exit!(matches, "slot", Slot);
+    let slot = value_of(matches, "slot");
     Ok(CliCommandInfo {
         command: CliCommand::GetBlock { slot },
         signers: vec![],
@@ -508,38 +599,76 @@ pub fn parse_transaction_history(
 pub fn process_catchup(
     rpc_client: &RpcClient,
     config: &CliConfig,
-    node_pubkey: &Pubkey,
-    node_json_rpc_url: &Option<String>,
+    node_pubkey: Option<Pubkey>,
+    mut node_json_rpc_url: Option<String>,
     follow: bool,
+    our_localhost_port: Option<u16>,
+    log: bool,
 ) -> ProcessResult {
     let sleep_interval = 5;
 
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message("Connecting...");
 
-    let node_client = if let Some(node_json_rpc_url) = node_json_rpc_url {
-        RpcClient::new(node_json_rpc_url.to_string())
-    } else {
-        let rpc_addr = loop {
-            let cluster_nodes = rpc_client.get_cluster_nodes()?;
-            if let Some(contact_info) = cluster_nodes
-                .iter()
-                .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
-            {
-                if let Some(rpc_addr) = contact_info.rpc {
-                    break rpc_addr;
-                }
-                progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
-            } else {
-                progress_bar.set_message(&format!(
-                    "Contact information not found for {}",
-                    node_pubkey
-                ));
-            }
-            sleep(Duration::from_secs(sleep_interval as u64));
-        };
+    if let Some(our_localhost_port) = our_localhost_port {
+        let gussed_default = Some(format!("http://localhost:{}", our_localhost_port));
+        if node_json_rpc_url.is_some() && node_json_rpc_url != gussed_default {
+            // go to new line to leave this message on console
+            println!(
+                "Prefering explicitly given rpc ({}) as us, \
+                 although --our-localhost is given\n",
+                node_json_rpc_url.as_ref().unwrap()
+            );
+        } else {
+            node_json_rpc_url = gussed_default;
+        }
+    }
 
-        RpcClient::new_socket(rpc_addr)
+    let (node_client, node_pubkey) = if our_localhost_port.is_some() {
+        let client = RpcClient::new(node_json_rpc_url.unwrap());
+        let guessed_default = Some(client.get_identity()?);
+        (
+            client,
+            (if node_pubkey.is_some() && node_pubkey != guessed_default {
+                // go to new line to leave this message on console
+                println!(
+                    "Prefering explicitly given node pubkey ({}) as us, \
+                     although --our-localhost is given\n",
+                    node_pubkey.unwrap()
+                );
+                node_pubkey
+            } else {
+                guessed_default
+            })
+            .unwrap(),
+        )
+    } else if let Some(node_pubkey) = node_pubkey {
+        if let Some(node_json_rpc_url) = node_json_rpc_url {
+            (RpcClient::new(node_json_rpc_url), node_pubkey)
+        } else {
+            let rpc_addr = loop {
+                let cluster_nodes = rpc_client.get_cluster_nodes()?;
+                if let Some(contact_info) = cluster_nodes
+                    .iter()
+                    .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
+                {
+                    if let Some(rpc_addr) = contact_info.rpc {
+                        break rpc_addr;
+                    }
+                    progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
+                } else {
+                    progress_bar.set_message(&format!(
+                        "Contact information not found for {}",
+                        node_pubkey
+                    ));
+                }
+                sleep(Duration::from_secs(sleep_interval as u64));
+            };
+
+            (RpcClient::new_socket(rpc_addr), node_pubkey)
+        }
+    } else {
+        unreachable!()
     };
 
     let reported_node_pubkey = loop {
@@ -556,7 +685,7 @@ pub fn process_catchup(
         }
     };
 
-    if reported_node_pubkey != *node_pubkey {
+    if reported_node_pubkey != node_pubkey {
         return Err(format!(
             "The identity reported by node RPC URL does not match.  Expected: {:?}.  Reported: {:?}",
             node_pubkey, reported_node_pubkey
@@ -564,15 +693,41 @@ pub fn process_catchup(
         .into());
     }
 
-    if rpc_client.get_identity()? == *node_pubkey {
+    if rpc_client.get_identity()? == node_pubkey {
         return Err("Both RPC URLs reference the same node, unable to monitor for catchup.  Try a different --url".into());
     }
 
     let mut previous_rpc_slot = std::u64::MAX;
     let mut previous_slot_distance = 0;
+    let mut retry_count = 0;
+    let max_retry_count = 5;
+    let mut get_slot_while_retrying = |client: &RpcClient| {
+        loop {
+            match client.get_slot_with_commitment(config.commitment) {
+                Ok(r) => {
+                    retry_count = 0;
+                    return Ok(r);
+                }
+                Err(e) => {
+                    if retry_count >= max_retry_count {
+                        return Err(e);
+                    }
+                    retry_count += 1;
+                    if log {
+                        // go to new line to leave this message on console
+                        println!("Retrying({}/{}): {}\n", retry_count, max_retry_count, e);
+                    }
+                    sleep(Duration::from_secs(1));
+                }
+            };
+        }
+    };
+
     loop {
-        let rpc_slot = rpc_client.get_slot_with_commitment(config.commitment)?;
-        let node_slot = node_client.get_slot_with_commitment(config.commitment)?;
+        // humbly retry; the reference node (rpc_client) could be spotty,
+        // especially if pointing to api.meinnet-beta.solana.com at times
+        let rpc_slot = get_slot_while_retrying(rpc_client)?;
+        let node_slot = get_slot_while_retrying(&node_client)?;
         if !follow && node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
             progress_bar.finish_and_clear();
             return Ok(format!(
@@ -595,15 +750,21 @@ pub fn process_catchup(
         };
 
         progress_bar.set_message(&format!(
-            "{} slots behind (us:{} them:{}){}",
-            slot_distance,
+            "{} slot(s) {} (us:{} them:{}){}",
+            slot_distance.abs(),
+            if slot_distance >= 0 {
+                "behind"
+            } else {
+                "ahead"
+            },
             node_slot,
             rpc_slot,
             if slot_distance == 0 || previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
                 format!(
-                    ", {} at {:.1} slots/second{}",
+                    ", {} node is {} at {:.1} slots/second{}",
+                    if slot_distance >= 0 { "our" } else { "their" },
                     if slots_per_second < 0.0 {
                         "falling behind"
                     } else {
@@ -612,8 +773,11 @@ pub fn process_catchup(
                     slots_per_second,
                     time_remaining
                 )
-            }
+            },
         ));
+        if log {
+            println!();
+        }
 
         sleep(Duration::from_secs(sleep_interval as u64));
         previous_rpc_slot = rpc_slot;
@@ -625,7 +789,7 @@ pub fn process_cluster_date(rpc_client: &RpcClient, config: &CliConfig) -> Proce
     let result = rpc_client
         .get_account_with_commitment(&sysvar::clock::id(), CommitmentConfig::default())?;
     if let Some(clock_account) = result.value {
-        let clock: Clock = Sysvar::from_account(&clock_account).ok_or_else(|| {
+        let clock: Clock = from_account(&clock_account).ok_or_else(|| {
             CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
         })?;
         let block_time = CliBlockTime {
@@ -665,9 +829,27 @@ pub fn process_first_available_block(rpc_client: &RpcClient) -> ProcessResult {
     Ok(format!("{}", first_available_block))
 }
 
-pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
+pub fn parse_leader_schedule(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let epoch = value_of(matches, "epoch");
+    Ok(CliCommandInfo {
+        command: CliCommand::LeaderSchedule { epoch },
+        signers: vec![],
+    })
+}
+
+pub fn process_leader_schedule(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    epoch: Option<Epoch>,
+) -> ProcessResult {
     let epoch_info = rpc_client.get_epoch_info()?;
-    let first_slot_in_epoch = epoch_info.absolute_slot - epoch_info.slot_index;
+    let epoch = epoch.unwrap_or(epoch_info.epoch);
+    if epoch > epoch_info.epoch {
+        return Err(format!("Epoch {} is in the future", epoch).into());
+    }
+
+    let epoch_schedule = rpc_client.get_epoch_schedule()?;
+    let first_slot_in_epoch = epoch_schedule.get_first_slot_in_epoch(epoch);
 
     let leader_schedule = rpc_client.get_leader_schedule(Some(first_slot_in_epoch))?;
     if leader_schedule.is_none() {
@@ -689,18 +871,31 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
         }
     }
 
+    let mut leader_schedule_entries = vec![];
     for (slot_index, leader) in leader_per_slot_index.iter().enumerate() {
-        println!(
-            "  {:<15} {:<44}",
-            first_slot_in_epoch + slot_index as u64,
-            leader
-        );
+        leader_schedule_entries.push(CliLeaderScheduleEntry {
+            slot: first_slot_in_epoch + slot_index as u64,
+            leader: leader.to_string(),
+        });
     }
 
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&CliLeaderSchedule {
+        epoch,
+        leader_schedule_entries,
+    }))
 }
 
-pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot) -> ProcessResult {
+pub fn process_get_block(
+    rpc_client: &RpcClient,
+    _config: &CliConfig,
+    slot: Option<Slot>,
+) -> ProcessResult {
+    let slot = if let Some(slot) = slot {
+        slot
+    } else {
+        rpc_client.get_slot()?
+    };
+
     let mut block =
         rpc_client.get_confirmed_block_with_encoding(slot, UiTransactionEncoding::Base64)?;
 
@@ -716,18 +911,23 @@ pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot
         let mut total_rewards = 0;
         println!("Rewards:",);
         println!(
-            "  {:<44}  {:<15}  {:<13}  {:>14}",
-            "Address", "Amount", "New Balance", "Percent Change"
+            "  {:<44}  {:^15}  {:<15}  {:<20}  {:>14}",
+            "Address", "Type", "Amount", "New Balance", "Percent Change"
         );
         for reward in block.rewards {
             let sign = if reward.lamports < 0 { "-" } else { "" };
 
             total_rewards += reward.lamports;
             println!(
-                "  {:<44}  {:>15}  {}",
+                "  {:<44}  {:^15}  {:>15}  {}",
                 reward.pubkey,
+                if let Some(reward_type) = reward.reward_type {
+                    format!("{}", reward_type)
+                } else {
+                    "-".to_string()
+                },
                 format!(
-                    "{}◎{:<14.4}",
+                    "{}◎{:<14.9}",
                     sign,
                     lamports_to_sol(reward.lamports.abs() as u64)
                 ),
@@ -735,10 +935,11 @@ pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot
                     "          -                 -".to_string()
                 } else {
                     format!(
-                        "◎{:<12.4}  {:>13.4}%",
+                        "◎{:<19.9}  {:>13.9}%",
                         lamports_to_sol(reward.post_balance),
-                        reward.lamports.abs() as f64
-                            / (reward.post_balance as f64 - reward.lamports as f64)
+                        (reward.lamports.abs() as f64
+                            / (reward.post_balance as f64 - reward.lamports as f64))
+                            * 100.0
                     )
                 }
             );
@@ -746,7 +947,7 @@ pub fn process_get_block(rpc_client: &RpcClient, _config: &CliConfig, slot: Slot
 
         let sign = if total_rewards < 0 { "-" } else { "" };
         println!(
-            "Total Rewards: {}◎{:12.9}",
+            "Total Rewards: {}◎{:<12.9}",
             sign,
             lamports_to_sol(total_rewards.abs() as u64)
         );
@@ -823,7 +1024,7 @@ pub fn process_show_block_production(
     slot_limit: Option<u64>,
 ) -> ProcessResult {
     let epoch_schedule = rpc_client.get_epoch_schedule()?;
-    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::root())?;
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::max())?;
 
     let epoch = epoch.unwrap_or(epoch_info.epoch);
     if epoch > epoch_info.epoch {
@@ -1011,6 +1212,8 @@ pub fn process_ping(
     interval: &Duration,
     count: &Option<u64>,
     timeout: &Duration,
+    fixed_blockhash: &Option<Hash>,
+    print_timestamp: bool,
 ) -> ProcessResult {
     println_name_value("Source Account:", &config.signers[0].pubkey().to_string());
     println!();
@@ -1028,9 +1231,21 @@ pub fn process_ping(
     let (mut blockhash, mut fee_calculator) = rpc_client.get_recent_blockhash()?;
     let mut blockhash_transaction_count = 0;
     let mut blockhash_acquired = Instant::now();
+    if let Some(fixed_blockhash) = fixed_blockhash {
+        let blockhash_origin = if *fixed_blockhash != Hash::default() {
+            blockhash = *fixed_blockhash;
+            "supplied from cli arguments"
+        } else {
+            "fetched from cluster"
+        };
+        println!(
+            "Fixed blockhash is used: {} ({})",
+            blockhash, blockhash_origin
+        );
+    }
     'mainloop: for seq in 0..count.unwrap_or(std::u64::MAX) {
         let now = Instant::now();
-        if now.duration_since(blockhash_acquired).as_secs() > 60 {
+        if fixed_blockhash.is_none() && now.duration_since(blockhash_acquired).as_secs() > 60 {
             // Fetch a new blockhash every minute
             let (new_blockhash, new_fee_calculator) = rpc_client.get_new_blockhash(&blockhash)?;
             blockhash = new_blockhash;
@@ -1061,6 +1276,18 @@ pub fn process_ping(
         let mut tx = Transaction::new_unsigned(message);
         tx.try_sign(&config.signers, blockhash)?;
 
+        let timestamp = || {
+            let micros = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_micros();
+            if print_timestamp {
+                format!("[{}.{:06}] ", micros / 1_000_000, micros % 1_000_000)
+            } else {
+                format!("")
+            }
+        };
+
         match rpc_client.send_transaction(&tx) {
             Ok(signature) => {
                 let transaction_sent = Instant::now();
@@ -1074,15 +1301,20 @@ pub fn process_ping(
                                 let elapsed_time_millis = elapsed_time.as_millis() as u64;
                                 confirmation_time.push_back(elapsed_time_millis);
                                 println!(
-                                    "{}{} lamport(s) transferred: seq={:<3} time={:>4}ms signature={}",
+                                    "{}{}{} lamport(s) transferred: seq={:<3} time={:>4}ms signature={}",
+                                    timestamp(),
                                     CHECK_MARK, lamports, seq, elapsed_time_millis, signature
                                 );
                                 confirmed_count += 1;
                             }
                             Err(err) => {
                                 println!(
-                                    "{}Transaction failed:    seq={:<3} error={:?} signature={}",
-                                    CROSS_MARK, seq, err, signature
+                                    "{}{}Transaction failed:    seq={:<3} error={:?} signature={}",
+                                    timestamp(),
+                                    CROSS_MARK,
+                                    seq,
+                                    err,
+                                    signature
                                 );
                             }
                         }
@@ -1091,8 +1323,11 @@ pub fn process_ping(
 
                     if elapsed_time >= *timeout {
                         println!(
-                            "{}Confirmation timeout:  seq={:<3}             signature={}",
-                            CROSS_MARK, seq, signature
+                            "{}{}Confirmation timeout:  seq={:<3}             signature={}",
+                            timestamp(),
+                            CROSS_MARK,
+                            seq,
+                            signature
                         );
                         break;
                     }
@@ -1110,8 +1345,11 @@ pub fn process_ping(
             }
             Err(err) => {
                 println!(
-                    "{}Submit failed:         seq={:<3} error={:?}",
-                    CROSS_MARK, seq, err
+                    "{}{}Submit failed:         seq={:<3} error={:?}",
+                    timestamp(),
+                    CROSS_MARK,
+                    seq,
+                    err
                 );
             }
         }
@@ -1146,24 +1384,83 @@ pub fn process_ping(
     Ok("".to_string())
 }
 
-pub fn process_live_slots(url: &str) -> ProcessResult {
-    let exit = Arc::new(AtomicBool::new(false));
+pub fn parse_logs(
+    matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let address = pubkey_of_signer(matches, "address", wallet_manager)?;
+    let include_votes = matches.is_present("include_votes");
 
-    // Disable Ctrl+C handler as sometimes the PubsubClient shutdown can stall.  Also it doesn't
-    // really matter that the shutdown is clean because the process is terminating.
-    /*
-    let exit_clone = exit.clone();
-    ctrlc::set_handler(move || {
-        exit_clone.store(true, Ordering::Relaxed);
-    })?;
-    */
+    let filter = match address {
+        None => {
+            if include_votes {
+                RpcTransactionLogsFilter::AllWithVotes
+            } else {
+                RpcTransactionLogsFilter::All
+            }
+        }
+        Some(address) => RpcTransactionLogsFilter::Mentions(vec![address.to_string()]),
+    };
+
+    Ok(CliCommandInfo {
+        command: CliCommand::Logs { filter },
+        signers: vec![],
+    })
+}
+
+pub fn process_logs(config: &CliConfig, filter: &RpcTransactionLogsFilter) -> ProcessResult {
+    println!(
+        "Streaming transaction logs{}. {:?} commitment",
+        match filter {
+            RpcTransactionLogsFilter::All => "".into(),
+            RpcTransactionLogsFilter::AllWithVotes => " (including votes)".into(),
+            RpcTransactionLogsFilter::Mentions(addresses) =>
+                format!(" mentioning {}", addresses.join(",")),
+        },
+        config.commitment.commitment
+    );
+
+    let (_client, receiver) = PubsubClient::logs_subscribe(
+        &config.websocket_url,
+        filter.clone(),
+        RpcTransactionLogsConfig {
+            commitment: Some(config.commitment),
+        },
+    )?;
+
+    loop {
+        match receiver.recv() {
+            Ok(logs) => {
+                println!("Transaction executed in slot {}:", logs.context.slot);
+                println!("  Signature: {}", logs.value.signature);
+                println!(
+                    "  Status: {}",
+                    logs.value
+                        .err
+                        .map(|err| err.to_string())
+                        .unwrap_or_else(|| "Ok".to_string())
+                );
+                println!("  Log Messages:");
+                for log in logs.value.logs {
+                    println!("    {}", log);
+                }
+            }
+            Err(err) => {
+                return Ok(format!("Disconnected: {}", err));
+            }
+        }
+    }
+}
+
+pub fn process_live_slots(config: &CliConfig) -> ProcessResult {
+    let exit = Arc::new(AtomicBool::new(false));
 
     let mut current: Option<SlotInfo> = None;
     let mut message = "".to_string();
 
     let slot_progress = new_spinner_progress_bar();
     slot_progress.set_message("Connecting...");
-    let (mut client, receiver) = PubsubClient::slot_subscribe(url)?;
+    let (mut client, receiver) = PubsubClient::slot_subscribe(&config.websocket_url)?;
     slot_progress.set_message("Connected.");
 
     let spacer = "|";
@@ -1253,14 +1550,16 @@ pub fn process_show_gossip(rpc_client: &RpcClient, config: &CliConfig) -> Proces
         .into_iter()
         .map(|node| {
             format!(
-                "{:15} | {:44} | {:6} | {:5} | {:5} | {}",
+                "{:15} | {:44} | {:6} | {:5} | {:21} | {}",
                 node.gossip
                     .map(|addr| addr.ip().to_string())
                     .unwrap_or_else(|| "none".to_string()),
                 format_labeled_address(&node.pubkey, &config.address_labels),
                 format_port(node.gossip),
                 format_port(node.tpu),
-                format_port(node.rpc),
+                node.rpc
+                    .map(|addr| addr.to_string())
+                    .unwrap_or_else(|| "none".to_string()),
                 node.version.unwrap_or_else(|| "unknown".to_string()),
             )
         })
@@ -1268,9 +1567,9 @@ pub fn process_show_gossip(rpc_client: &RpcClient, config: &CliConfig) -> Proces
 
     Ok(format!(
         "IP Address      | Node identifier                              \
-         | Gossip | TPU   | RPC   | Version\n\
+         | Gossip | TPU   | RPC Address           | Version\n\
          ----------------+----------------------------------------------+\
-         --------+-------+-------+----------------\n\
+         --------+-------+-----------------------+----------------\n\
          {}\n\
          Nodes: {}",
         s.join("\n"),
@@ -1325,14 +1624,16 @@ pub fn process_show_stakes(
         .get_program_accounts_with_config(&solana_stake_program::id(), program_accounts_config)?;
     let stake_history_account = rpc_client.get_account(&stake_history::id())?;
     let clock_account = rpc_client.get_account(&sysvar::clock::id())?;
-    let clock: Clock = Sysvar::from_account(&clock_account).ok_or_else(|| {
+    let clock: Clock = from_account(&clock_account).ok_or_else(|| {
         CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
     })?;
     progress_bar.finish_and_clear();
 
-    let stake_history = StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+    let stake_history = from_account(&stake_history_account).ok_or_else(|| {
         CliError::RpcRequestError("Failed to deserialize stake history".to_string())
     })?;
+    // At v1.6, this check can be removed and simply passed as `true`
+    let stake_program_v2_enabled = is_stake_program_v2_enabled(rpc_client);
 
     let mut stake_accounts: Vec<CliKeyedStakeState> = vec![];
     for (stake_pubkey, stake_account) in all_stake_accounts {
@@ -1348,6 +1649,7 @@ pub fn process_show_stakes(
                                 use_lamports_unit,
                                 &stake_history,
                                 &clock,
+                                stake_program_v2_enabled,
                             ),
                         });
                     }
@@ -1366,6 +1668,7 @@ pub fn process_show_stakes(
                                 use_lamports_unit,
                                 &stake_history,
                                 &clock,
+                                stake_program_v2_enabled,
                             ),
                         });
                     }
@@ -1379,6 +1682,16 @@ pub fn process_show_stakes(
         .formatted_string(&CliStakeVec::new(stake_accounts)))
 }
 
+pub fn process_wait_for_max_stake(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    max_stake_percent: f32,
+) -> ProcessResult {
+    let now = std::time::Instant::now();
+    rpc_client.wait_for_max_stake(config.commitment, max_stake_percent)?;
+    Ok(format!("Done waiting, took: {}s", now.elapsed().as_secs()))
+}
+
 pub fn process_show_validators(
     rpc_client: &RpcClient,
     config: &CliConfig,
@@ -1405,12 +1718,12 @@ pub fn process_show_validators(
         .map(|vote_account| vote_account.activated_stake)
         .sum();
 
-    let total_deliquent_stake = vote_accounts
+    let total_delinquent_stake = vote_accounts
         .delinquent
         .iter()
         .map(|vote_account| vote_account.activated_stake)
         .sum();
-    let total_current_stake = total_active_stake - total_deliquent_stake;
+    let total_current_stake = total_active_stake - total_delinquent_stake;
 
     let mut current = vote_accounts.current;
     current.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
@@ -1464,7 +1777,7 @@ pub fn process_show_validators(
     let cli_validators = CliValidators {
         total_active_stake,
         total_current_stake,
-        total_deliquent_stake,
+        total_delinquent_stake,
         current_validators,
         delinquent_validators,
         stake_by_version,
@@ -1539,6 +1852,7 @@ mod tests {
     use super::*;
     use crate::cli::{app, parse_command};
     use solana_sdk::signature::{write_keypair, Keypair};
+    use std::str::FromStr;
     use tempfile::NamedTempFile;
 
     fn make_tmp_file() -> (String, NamedTempFile) {
@@ -1674,8 +1988,11 @@ mod tests {
             "2",
             "-t",
             "3",
+            "-D",
             "--commitment",
             "max",
+            "--blockhash",
+            "4CCNp28j6AhGq7PkjPDP4wbQWBS8LLbQin2xV5n8frKX",
         ]);
         assert_eq!(
             parse_command(&test_ping, &default_signer, &mut None).unwrap(),
@@ -1685,6 +2002,10 @@ mod tests {
                     interval: Duration::from_secs(1),
                     count: Some(2),
                     timeout: Duration::from_secs(3),
+                    blockhash: Some(
+                        Hash::from_str("4CCNp28j6AhGq7PkjPDP4wbQWBS8LLbQin2xV5n8frKX").unwrap()
+                    ),
+                    print_timestamp: true,
                 },
                 signers: vec![default_keypair.into()],
             }

cli/src/feature.rs

@@ -9,12 +9,13 @@ use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::*};
 use solana_cli_output::{QuietDisplay, VerboseDisplay};
 use solana_client::{client_error::ClientError, rpc_client::RpcClient};
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
-use solana_runtime::{
+use solana_sdk::{
+    clock::Slot,
     feature::{self, Feature},
     feature_set::FEATURE_NAMES,
-};
-use solana_sdk::{
-    clock::Slot, message::Message, pubkey::Pubkey, system_instruction, transaction::Transaction,
+    message::Message,
+    pubkey::Pubkey,
+    transaction::Transaction,
 };
 use std::{collections::HashMap, fmt, sync::Arc};
 
@@ -230,7 +231,7 @@ fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, u6
 }
 
 // Feature activation is only allowed when 95% of the active stake is on the current feature set
-fn feature_activation_allowed(rpc_client: &RpcClient) -> Result<bool, ClientError> {
+fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<bool, ClientError> {
     let my_feature_set = solana_version::Version::default().feature_set;
 
     let active_stake_by_feature_set = active_stake_by_feature_set(rpc_client)?;
@@ -240,8 +241,23 @@ fn feature_activation_allowed(rpc_client: &RpcClient) -> Result<bool, ClientErro
         .map(|percentage| *percentage >= 95)
         .unwrap_or(false);
 
-    if !feature_activation_allowed {
-        println!("\n{}", style("Stake By Feature Set:").bold());
+    if !feature_activation_allowed && !quiet {
+        if active_stake_by_feature_set.get(&my_feature_set).is_none() {
+            println!(
+                "{}",
+                style("To activate features the tool and cluster feature sets must match, select a tool version that matches the cluster")
+                    .bold());
+        } else {
+            println!(
+                "{}",
+                style("To activate features the stake must be >= 95%").bold()
+            );
+        }
+        println!(
+            "{}",
+            style(format!("Tool Feture Set: {}", my_feature_set)).bold()
+        );
+        println!("{}", style("Cluster Feature Sets and Stakes:").bold());
         for (feature_set, percentage) in active_stake_by_feature_set.iter() {
             if *feature_set == 0 {
                 println!("unknown - {}%", percentage);
@@ -258,6 +274,7 @@ fn feature_activation_allowed(rpc_client: &RpcClient) -> Result<bool, ClientErro
                 );
             }
         }
+        println!();
     }
 
     Ok(feature_activation_allowed)
@@ -278,7 +295,7 @@ fn process_status(
         let feature_id = &feature_ids[i];
         let feature_name = FEATURE_NAMES.get(feature_id).unwrap();
         if let Some(account) = account {
-            if let Some(feature) = Feature::from_account(&account) {
+            if let Some(feature) = feature::from_account(&account) {
                 let feature_status = match feature.activated_at {
                     None => CliFeatureStatus::Pending,
                     Some(activation_slot) => CliFeatureStatus::Active(activation_slot),
@@ -299,9 +316,10 @@ fn process_status(
         });
     }
 
+    let feature_activation_allowed = feature_activation_allowed(rpc_client, features.len() <= 1)?;
     let feature_set = CliFeatures {
         features,
-        feature_activation_allowed: feature_activation_allowed(rpc_client)?,
+        feature_activation_allowed,
         inactive,
     };
     Ok(config.output_format.formatted_string(&feature_set))
@@ -318,12 +336,12 @@ fn process_activate(
         .next()
         .unwrap();
     if let Some(account) = account {
-        if Feature::from_account(&account).is_some() {
+        if feature::from_account(&account).is_some() {
             return Err(format!("{} has already been activated", feature_id).into());
         }
     }
 
-    if !feature_activation_allowed(rpc_client)? {
+    if !feature_activation_allowed(rpc_client, false)? {
         return Err("Feature activation is not allowed at this time".into());
     }
 
@@ -338,15 +356,11 @@ fn process_activate(
         &config.signers[0].pubkey(),
         |lamports| {
             Message::new(
-                &[
-                    system_instruction::transfer(
-                        &config.signers[0].pubkey(),
-                        &feature_id,
-                        lamports,
-                    ),
-                    system_instruction::allocate(&feature_id, Feature::size_of() as u64),
-                    system_instruction::assign(&feature_id, &feature::id()),
-                ],
+                &feature::activate_with_lamports(
+                    &feature_id,
+                    &config.signers[0].pubkey(),
+                    lamports,
+                ),
                 Some(&config.signers[0].pubkey()),
             )
         },

cli/src/inflation.rs

@@ -1,7 +1,7 @@
 use crate::cli::{CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult};
 use clap::{App, ArgMatches, SubCommand};
-use console::style;
 use solana_clap_utils::keypair::*;
+use solana_cli_output::CliInflation;
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use std::sync::Arc;
@@ -34,56 +34,18 @@ pub fn parse_inflation_subcommand(
 
 pub fn process_inflation_subcommand(
     rpc_client: &RpcClient,
-    _config: &CliConfig,
+    config: &CliConfig,
     inflation_subcommand: &InflationCliCommand,
 ) -> ProcessResult {
     assert_eq!(*inflation_subcommand, InflationCliCommand::Show);
 
     let governor = rpc_client.get_inflation_governor()?;
-    let current_inflation_rate = rpc_client.get_inflation_rate()?;
+    let current_rate = rpc_client.get_inflation_rate()?;
 
-    println!("{}", style("Inflation Governor:").bold());
-    if (governor.initial - governor.terminal).abs() < f64::EPSILON {
-        println!(
-            "Fixed APR:               {:>5.2}%",
-            governor.terminal * 100.
-        );
-    } else {
-        println!("Initial APR:             {:>5.2}%", governor.initial * 100.);
-        println!(
-            "Terminal APR:            {:>5.2}%",
-            governor.terminal * 100.
-        );
-        println!("Rate reduction per year: {:>5.2}%", governor.taper * 100.);
-    }
-    if governor.foundation_term > 0. {
-        println!("Foundation percentage:   {:>5.2}%", governor.foundation);
-        println!(
-            "Foundation term:         {:.1} years",
-            governor.foundation_term
-        );
-    }
+    let inflation = CliInflation {
+        governor,
+        current_rate,
+    };
 
-    println!(
-        "\n{}",
-        style(format!(
-            "Inflation for Epoch {}:",
-            current_inflation_rate.epoch
-        ))
-        .bold()
-    );
-    println!(
-        "Total APR:               {:>5.2}%",
-        current_inflation_rate.total * 100.
-    );
-    println!(
-        "Staking APR:             {:>5.2}%",
-        current_inflation_rate.validator * 100.
-    );
-    println!(
-        "Foundation APR:          {:>5.2}%",
-        current_inflation_rate.foundation * 100.
-    );
-
-    Ok("".to_string())
+    Ok(config.output_format.formatted_string(&inflation))
 }

cli/src/lib.rs

@@ -26,6 +26,8 @@ pub mod cluster_query;
 pub mod feature;
 pub mod inflation;
 pub mod nonce;
+pub mod program;
+pub mod send_tpu;
 pub mod spend_utils;
 pub mod stake;
 pub mod test_utils;

cli/src/main.rs

@@ -7,7 +7,7 @@ use console::style;
 use solana_clap_utils::{
     commitment::COMMITMENT_ARG,
     input_parsers::commitment_of,
-    input_validators::is_url,
+    input_validators::{is_url, is_url_or_moniker},
     keypair::{CliSigners, DefaultSigner, SKIP_SEED_PHRASE_VALIDATION_ARG},
     DisplayError,
 };
@@ -168,6 +168,7 @@ pub fn parse_args<'a>(
     let CliCommandInfo { command, signers } =
         parse_command(&matches, &default_signer, &mut wallet_manager)?;
 
+    let verbose = matches.is_present("verbose");
     let output_format = matches
         .value_of("output_format")
         .map(|value| match value {
@@ -175,13 +176,22 @@ pub fn parse_args<'a>(
             "json-compact" => OutputFormat::JsonCompact,
             _ => unreachable!(),
         })
-        .unwrap_or(OutputFormat::Display);
+        .unwrap_or(if verbose {
+            OutputFormat::DisplayVerbose
+        } else {
+            OutputFormat::Display
+        });
 
-    let commitment = matches
-        .subcommand_name()
-        .and_then(|name| matches.subcommand_matches(name))
-        .and_then(|sub_matches| commitment_of(sub_matches, COMMITMENT_ARG.long))
-        .unwrap_or_default();
+    let commitment = {
+        let mut sub_matches = matches;
+        while let Some(subcommand_name) = sub_matches.subcommand_name() {
+            sub_matches = sub_matches
+                .subcommand_matches(subcommand_name)
+                .expect("subcommand_matches");
+        }
+        commitment_of(sub_matches, COMMITMENT_ARG.long)
+    }
+    .unwrap_or_default();
 
     let address_labels = if matches.is_present("no_address_labels") {
         HashMap::new()
@@ -198,10 +208,13 @@ pub fn parse_args<'a>(
             keypair_path: default_signer_path,
             rpc_client: None,
             rpc_timeout,
-            verbose: matches.is_present("verbose"),
+            verbose,
             output_format,
             commitment,
-            send_transaction_config: RpcSendTransactionConfig::default(),
+            send_transaction_config: RpcSendTransactionConfig {
+                preflight_commitment: Some(commitment.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
             address_labels,
         },
         signers,
@@ -233,11 +246,14 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         Arg::with_name("json_rpc_url")
             .short("u")
             .long("url")
-            .value_name("URL")
+            .value_name("URL_OR_MONIKER")
             .takes_value(true)
             .global(true)
-            .validator(is_url)
-            .help("JSON RPC URL for the solana cluster"),
+            .validator(is_url_or_moniker)
+            .help(
+                "URL for Solana's JSON RPC or moniker (or their first letter): \
+                   [mainnet-beta, testnet, devnet, localhost]",
+            ),
     )
     .arg(
         Arg::with_name("websocket_url")

cli/src/nonce.rs

@@ -580,6 +580,7 @@ mod tests {
         fee_calculator::FeeCalculator,
         hash::hash,
         nonce::{self, state::Versions, State},
+        nonce_account,
         signature::{read_keypair_file, write_keypair, Keypair, Signer},
         system_program,
     };
@@ -833,7 +834,7 @@ mod tests {
     #[test]
     fn test_check_nonce_account() {
         let blockhash = Hash::default();
-        let nonce_pubkey = Pubkey::new_rand();
+        let nonce_pubkey = solana_sdk::pubkey::new_rand();
         let data = Versions::new_current(State::Initialized(nonce::state::Data {
             authority: nonce_pubkey,
             blockhash,
@@ -869,7 +870,7 @@ mod tests {
         }
 
         let data = Versions::new_current(State::Initialized(nonce::state::Data {
-            authority: Pubkey::new_rand(),
+            authority: solana_sdk::pubkey::new_rand(),
             blockhash,
             fee_calculator: FeeCalculator::default(),
         }));
@@ -891,7 +892,7 @@ mod tests {
 
     #[test]
     fn test_account_identity_ok() {
-        let nonce_account = nonce::create_account(1).into_inner();
+        let nonce_account = nonce_account::create_account(1).into_inner();
         assert_eq!(account_identity_ok(&nonce_account), Ok(()));
 
         let system_account = Account::new(1, 0, &system_program::id());
@@ -910,7 +911,7 @@ mod tests {
 
     #[test]
     fn test_state_from_account() {
-        let mut nonce_account = nonce::create_account(1).into_inner();
+        let mut nonce_account = nonce_account::create_account(1).into_inner();
         assert_eq!(state_from_account(&nonce_account), Ok(State::Uninitialized));
 
         let data = nonce::state::Data {
@@ -935,7 +936,7 @@ mod tests {
 
     #[test]
     fn test_data_from_helpers() {
-        let mut nonce_account = nonce::create_account(1).into_inner();
+        let mut nonce_account = nonce_account::create_account(1).into_inner();
         let state = state_from_account(&nonce_account).unwrap();
         assert_eq!(
             data_from_state(&state),

cli/src/send_tpu.rs

@@ -0,0 +1,29 @@
+use log::*;
+use solana_client::rpc_response::{RpcContactInfo, RpcLeaderSchedule};
+use std::net::{SocketAddr, UdpSocket};
+
+pub fn get_leader_tpu(
+    slot_index: u64,
+    leader_schedule: Option<&RpcLeaderSchedule>,
+    cluster_nodes: Option<&Vec<RpcContactInfo>>,
+) -> Option<SocketAddr> {
+    leader_schedule?
+        .iter()
+        .find(|(_pubkey, slots)| slots.iter().any(|slot| *slot as u64 == slot_index))
+        .and_then(|(pubkey, _)| {
+            cluster_nodes?
+                .iter()
+                .find(|contact_info| contact_info.pubkey == *pubkey)
+                .and_then(|contact_info| contact_info.tpu)
+        })
+}
+
+pub fn send_transaction_tpu(
+    send_socket: &UdpSocket,
+    tpu_address: &SocketAddr,
+    wire_transaction: &[u8],
+) {
+    if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
+        warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
+    }
+}

cli/src/stake.rs

@@ -23,20 +23,25 @@ use solana_cli_output::{
     CliStakeType,
 };
 use solana_client::{
-    blockhash_query::BlockhashQuery, nonce_utils, rpc_client::RpcClient,
-    rpc_request::DELINQUENT_VALIDATOR_SLOT_DISTANCE,
+    blockhash_query::BlockhashQuery,
+    client_error::{ClientError, ClientErrorKind},
+    nonce_utils,
+    rpc_client::RpcClient,
+    rpc_custom_error,
+    rpc_request::{self, DELINQUENT_VALIDATOR_SLOT_DISTANCE},
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
+    account::from_account,
     account_utils::StateMut,
-    clock::{Clock, Epoch, Slot, UnixTimestamp},
+    clock::{Clock, Epoch, Slot, UnixTimestamp, SECONDS_PER_DAY},
+    feature, feature_set,
     message::Message,
     pubkey::Pubkey,
     system_instruction::SystemError,
     sysvar::{
         clock,
         stake_history::{self, StakeHistory},
-        Sysvar,
     },
     transaction::Transaction,
 };
@@ -1497,6 +1502,7 @@ pub fn build_stake_state(
     use_lamports_unit: bool,
     stake_history: &StakeHistory,
     clock: &Clock,
+    stake_program_v2_enabled: bool,
 ) -> CliStakeState {
     match stake_state {
         StakeState::Stake(
@@ -1508,9 +1514,12 @@ pub fn build_stake_state(
             stake,
         ) => {
             let current_epoch = clock.epoch;
-            let (active_stake, activating_stake, deactivating_stake) = stake
-                .delegation
-                .stake_activating_and_deactivating(current_epoch, Some(stake_history));
+            let (active_stake, activating_stake, deactivating_stake) =
+                stake.delegation.stake_activating_and_deactivating(
+                    current_epoch,
+                    Some(stake_history),
+                    stake_program_v2_enabled,
+                );
             let lockup = if lockup.is_in_force(clock, None) {
                 Some(lockup.into())
             } else {
@@ -1519,6 +1528,7 @@ pub fn build_stake_state(
             CliStakeState {
                 stake_type: CliStakeType::Stake,
                 account_balance,
+                credits_observed: Some(stake.credits_observed),
                 delegated_stake: Some(stake.delegation.stake),
                 delegated_vote_account_address: if stake.delegation.voter_pubkey
                     != Pubkey::default()
@@ -1570,6 +1580,7 @@ pub fn build_stake_state(
             CliStakeState {
                 stake_type: CliStakeType::Initialized,
                 account_balance,
+                credits_observed: Some(0),
                 authorized: Some(authorized.into()),
                 lockup,
                 use_lamports_unit,
@@ -1605,10 +1616,26 @@ pub(crate) fn fetch_epoch_rewards(
             .get(0)
             .ok_or_else(|| format!("Unable to fetch first confirmed block for epoch {}", epoch))?;
 
-        let first_confirmed_block = rpc_client.get_confirmed_block_with_encoding(
+        let first_confirmed_block = match rpc_client.get_confirmed_block_with_encoding(
             first_confirmed_block_in_epoch,
             solana_transaction_status::UiTransactionEncoding::Base64,
-        )?;
+        ) {
+            Ok(first_confirmed_block) => first_confirmed_block,
+            Err(ClientError {
+                kind:
+                    ClientErrorKind::RpcError(rpc_request::RpcError::RpcResponseError {
+                        code: rpc_custom_error::JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE,
+                        ..
+                    }),
+                ..
+            }) => {
+                // RPC node doesn't have this block
+                break;
+            }
+            Err(err) => {
+                return Err(err.into());
+            }
+        };
 
         let epoch_start_time = if let Some(block_time) = first_confirmed_block.block_time {
             block_time
@@ -1620,29 +1647,37 @@ pub(crate) fn fetch_epoch_rewards(
         let previous_epoch_rewards = first_confirmed_block.rewards;
 
         if let Some((effective_slot, epoch_end_time, epoch_rewards)) = epoch_info {
-            let wall_clock_epoch_duration =
-                { Local.timestamp(epoch_end_time, 0) - Local.timestamp(epoch_start_time, 0) }
-                    .to_std()?
-                    .as_secs_f64();
-
-            const SECONDS_PER_YEAR: f64 = (24 * 60 * 60 * 356) as f64;
-            let percent_of_year = SECONDS_PER_YEAR / wall_clock_epoch_duration;
+            let wallclock_epoch_duration = if epoch_end_time > epoch_start_time {
+                Some(
+                    { Local.timestamp(epoch_end_time, 0) - Local.timestamp(epoch_start_time, 0) }
+                        .to_std()?
+                        .as_secs_f64(),
+                )
+            } else {
+                None
+            };
 
             if let Some(reward) = epoch_rewards
                 .into_iter()
                 .find(|reward| reward.pubkey == address.to_string())
             {
                 if reward.post_balance > reward.lamports.try_into().unwrap_or(0) {
-                    let balance_increase_percent = reward.lamports.abs() as f64
+                    let rate_change = reward.lamports.abs() as f64
                         / (reward.post_balance as f64 - reward.lamports as f64);
 
+                    let apr = wallclock_epoch_duration.map(|wallclock_epoch_duration| {
+                        let wallclock_epochs_per_year =
+                            (SECONDS_PER_DAY * 356) as f64 / wallclock_epoch_duration;
+                        rate_change * wallclock_epochs_per_year
+                    });
+
                     all_epoch_rewards.push(CliEpochReward {
                         epoch,
                         effective_slot,
                         amount: reward.lamports.abs() as u64,
                         post_balance: reward.post_balance,
-                        percent_change: balance_increase_percent,
-                        apr: balance_increase_percent * percent_of_year,
+                        percent_change: rate_change * 100.0,
+                        apr: apr.map(|r| r * 100.0),
                     });
                 }
             }
@@ -1676,12 +1711,11 @@ pub fn process_show_stake_account(
     match stake_account.state() {
         Ok(stake_state) => {
             let stake_history_account = rpc_client.get_account(&stake_history::id())?;
-            let stake_history =
-                StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
-                    CliError::RpcRequestError("Failed to deserialize stake history".to_string())
-                })?;
+            let stake_history = from_account(&stake_history_account).ok_or_else(|| {
+                CliError::RpcRequestError("Failed to deserialize stake history".to_string())
+            })?;
             let clock_account = rpc_client.get_account(&clock::id())?;
-            let clock: Clock = Sysvar::from_account(&clock_account).ok_or_else(|| {
+            let clock: Clock = from_account(&clock_account).ok_or_else(|| {
                 CliError::RpcRequestError("Failed to deserialize clock sysvar".to_string())
             })?;
 
@@ -1691,15 +1725,17 @@ pub fn process_show_stake_account(
                 use_lamports_unit,
                 &stake_history,
                 &clock,
+                is_stake_program_v2_enabled(rpc_client), // At v1.6, this check can be removed and simply passed as `true`
             );
 
             if state.stake_type == CliStakeType::Stake {
                 if let Some(activation_epoch) = state.activation_epoch {
-                    state.epoch_rewards = Some(fetch_epoch_rewards(
-                        rpc_client,
-                        stake_account_address,
-                        activation_epoch,
-                    )?);
+                    let rewards =
+                        fetch_epoch_rewards(rpc_client, stake_account_address, activation_epoch);
+                    match rewards {
+                        Ok(rewards) => state.epoch_rewards = Some(rewards),
+                        Err(error) => eprintln!("Failed to fetch epoch rewards: {:?}", error),
+                    };
                 }
             }
             Ok(config.output_format.formatted_string(&state))
@@ -1718,7 +1754,7 @@ pub fn process_show_stake_history(
     use_lamports_unit: bool,
 ) -> ProcessResult {
     let stake_history_account = rpc_client.get_account(&stake_history::id())?;
-    let stake_history = StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+    let stake_history = from_account::<StakeHistory>(&stake_history_account).ok_or_else(|| {
         CliError::RpcRequestError("Failed to deserialize stake history".to_string())
     })?;
 
@@ -1861,6 +1897,15 @@ pub fn process_delegate_stake(
     }
 }
 
+pub fn is_stake_program_v2_enabled(rpc_client: &RpcClient) -> bool {
+    rpc_client
+        .get_account(&feature_set::stake_program_v2::id())
+        .ok()
+        .and_then(|account| feature::from_account(&account))
+        .and_then(|feature| feature.activated_at)
+        .is_some()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -2406,9 +2451,9 @@ mod tests {
         );
 
         // Test CreateStakeAccount SubCommand
-        let custodian = Pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         let custodian_string = format!("{}", custodian);
-        let authorized = Pubkey::new_rand();
+        let authorized = solana_sdk::pubkey::new_rand();
         let authorized_string = format!("{}", authorized);
         let test_create_stake_account = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2546,7 +2591,7 @@ mod tests {
         );
 
         // Test DelegateStake Subcommand
-        let vote_account_pubkey = Pubkey::new_rand();
+        let vote_account_pubkey = solana_sdk::pubkey::new_rand();
         let vote_account_string = vote_account_pubkey.to_string();
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2573,7 +2618,7 @@ mod tests {
         );
 
         // Test DelegateStake Subcommand w/ authority
-        let vote_account_pubkey = Pubkey::new_rand();
+        let vote_account_pubkey = solana_sdk::pubkey::new_rand();
         let vote_account_string = vote_account_pubkey.to_string();
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2692,7 +2737,7 @@ mod tests {
         );
 
         // Test Delegate Subcommand w/ absent fee payer
-        let key1 = Pubkey::new_rand();
+        let key1 = solana_sdk::pubkey::new_rand();
         let sig1 = Keypair::new().sign_message(&[0u8]);
         let signer1 = format!("{}={}", key1, sig1);
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
@@ -2732,7 +2777,7 @@ mod tests {
         );
 
         // Test Delegate Subcommand w/ absent fee payer and absent nonce authority
-        let key2 = Pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
         let sig2 = Keypair::new().sign_message(&[0u8]);
         let signer2 = format!("{}={}", key2, sig2);
         let test_delegate_stake = test_commands.clone().get_matches_from(vec![
@@ -3060,7 +3105,7 @@ mod tests {
         );
 
         // Test Deactivate Subcommand w/ absent fee payer
-        let key1 = Pubkey::new_rand();
+        let key1 = solana_sdk::pubkey::new_rand();
         let sig1 = Keypair::new().sign_message(&[0u8]);
         let signer1 = format!("{}={}", key1, sig1);
         let test_deactivate_stake = test_commands.clone().get_matches_from(vec![
@@ -3097,7 +3142,7 @@ mod tests {
         );
 
         // Test Deactivate Subcommand w/ absent fee payer and nonce authority
-        let key2 = Pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
         let sig2 = Keypair::new().sign_message(&[0u8]);
         let signer2 = format!("{}={}", key2, sig2);
         let test_deactivate_stake = test_commands.clone().get_matches_from(vec![
@@ -3276,7 +3321,7 @@ mod tests {
         let stake_account_keypair = Keypair::new();
         write_keypair(&stake_account_keypair, tmp_file.as_file_mut()).unwrap();
 
-        let source_stake_account_pubkey = Pubkey::new_rand();
+        let source_stake_account_pubkey = solana_sdk::pubkey::new_rand();
         let test_merge_stake_account = test_commands.clone().get_matches_from(vec![
             "test",
             "merge-stake",

cli/src/validator_info.rs

@@ -486,7 +486,7 @@ mod tests {
 
     #[test]
     fn test_parse_validator_info() {
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let keys = vec![(validator_info::id(), false), (pubkey, true)];
         let config = ConfigKeys { keys };
 

cli/src/vote.rs

@@ -685,22 +685,27 @@ pub fn process_show_vote_account(
         for vote in &vote_state.votes {
             votes.push(vote.into());
         }
-        for (epoch, credits, prev_credits) in vote_state.epoch_credits() {
+        for (epoch, credits, prev_credits) in vote_state.epoch_credits().iter().copied() {
             let credits_earned = credits - prev_credits;
-            let slots_in_epoch = epoch_schedule.get_slots_in_epoch(*epoch);
+            let slots_in_epoch = epoch_schedule.get_slots_in_epoch(epoch);
             epoch_voting_history.push(CliEpochVotingHistory {
-                epoch: *epoch,
+                epoch,
                 slots_in_epoch,
                 credits_earned,
+                credits,
+                prev_credits,
             });
         }
     }
 
-    let epoch_rewards = Some(crate::stake::fetch_epoch_rewards(
-        rpc_client,
-        vote_account_address,
-        1,
-    )?);
+    let epoch_rewards = match crate::stake::fetch_epoch_rewards(rpc_client, vote_account_address, 1)
+    {
+        Ok(rewards) => Some(rewards),
+        Err(error) => {
+            eprintln!("Failed to fetch epoch rewards: {:?}", error);
+            None
+        }
+    };
 
     let vote_account_data = CliVoteAccount {
         account_balance: vote_account.lamports,
@@ -915,7 +920,7 @@ mod tests {
         );
 
         // test init with an authed voter
-        let authed = Pubkey::new_rand();
+        let authed = solana_sdk::pubkey::new_rand();
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let keypair = Keypair::new();
         write_keypair(&keypair, tmp_file.as_file_mut()).unwrap();

cli/tests/deploy.rs

@@ -1,118 +0,0 @@
-use serde_json::Value;
-use solana_cli::cli::{process_command, CliCommand, CliConfig};
-use solana_client::rpc_client::RpcClient;
-use solana_core::test_validator::TestValidator;
-use solana_faucet::faucet::run_local_faucet;
-use solana_sdk::{
-    bpf_loader,
-    commitment_config::CommitmentConfig,
-    pubkey::Pubkey,
-    signature::{Keypair, Signer},
-};
-use std::{
-    fs::{remove_dir_all, File},
-    io::Read,
-    path::PathBuf,
-    str::FromStr,
-    sync::mpsc::channel,
-};
-
-#[test]
-fn test_cli_deploy_program() {
-    solana_logger::setup();
-
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
-
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
-
-    let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
-    let faucet_addr = receiver.recv().unwrap();
-
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
-
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
-    let mut program_data = Vec::new();
-    file.read_to_end(&mut program_data).unwrap();
-    let minimum_balance_for_rent_exemption = rpc_client
-        .get_minimum_balance_for_rent_exemption(program_data.len())
-        .unwrap();
-
-    let mut config = CliConfig::recent_for_tests();
-    let keypair = Keypair::new();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
-    config.command = CliCommand::Airdrop {
-        faucet_host: None,
-        faucet_port: faucet_addr.port(),
-        pubkey: None,
-        lamports: 3 * minimum_balance_for_rent_exemption, // min balance for rent exemption for two programs + leftover for tx processing
-    };
-    config.signers = vec![&keypair];
-    process_command(&config).unwrap();
-
-    config.command = CliCommand::Deploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        address: None,
-        use_deprecated_loader: false,
-    };
-
-    let response = process_command(&config);
-    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
-    let program_id_str = json
-        .as_object()
-        .unwrap()
-        .get("programId")
-        .unwrap()
-        .as_str()
-        .unwrap();
-    let program_id = Pubkey::from_str(&program_id_str).unwrap();
-    let account0 = rpc_client
-        .get_account_with_commitment(&program_id, CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(account0.lamports, minimum_balance_for_rent_exemption);
-    assert_eq!(account0.owner, bpf_loader::id());
-    assert_eq!(account0.executable, true);
-
-    let mut file = File::open(pathbuf.to_str().unwrap().to_string()).unwrap();
-    let mut elf = Vec::new();
-    file.read_to_end(&mut elf).unwrap();
-
-    assert_eq!(account0.data, elf);
-
-    // Test custom address
-    let custom_address_keypair = Keypair::new();
-    config.signers = vec![&keypair, &custom_address_keypair];
-    config.command = CliCommand::Deploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
-        address: Some(1),
-        use_deprecated_loader: false,
-    };
-    process_command(&config).unwrap();
-    let account1 = rpc_client
-        .get_account_with_commitment(&custom_address_keypair.pubkey(), CommitmentConfig::recent())
-        .unwrap()
-        .value
-        .unwrap();
-    assert_eq!(account1.lamports, minimum_balance_for_rent_exemption);
-    assert_eq!(account1.owner, bpf_loader::id());
-    assert_eq!(account1.executable, true);
-    assert_eq!(account0.data, account1.data);
-
-    // Attempt to redeploy to the same address
-    process_command(&config).unwrap_err();
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
-}

cli/tests/nonce.rs

@@ -9,8 +9,7 @@ use solana_client::{
     nonce_utils,
     rpc_client::RpcClient,
 };
-use solana_core::contact_info::ContactInfo;
-use solana_core::test_validator::{TestValidator, TestValidatorOptions};
+use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     commitment_config::CommitmentConfig,
@@ -19,69 +18,53 @@ use solana_sdk::{
     signature::{keypair_from_seed, Keypair, Signer},
     system_program,
 };
-use std::{fs::remove_dir_all, sync::mpsc::channel};
+use std::sync::mpsc::channel;
 
 #[test]
 fn test_nonce() {
-    solana_logger::setup();
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
-
-    full_battery_tests(leader_data, alice, None, false);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
+    let mint_keypair = Keypair::new();
+    full_battery_tests(
+        TestValidator::with_no_fees(mint_keypair.pubkey()),
+        mint_keypair,
+        None,
+        false,
+    );
 }
 
 #[test]
 fn test_nonce_with_seed() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
-
-    full_battery_tests(leader_data, alice, Some(String::from("seed")), false);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
+    let mint_keypair = Keypair::new();
+    full_battery_tests(
+        TestValidator::with_no_fees(mint_keypair.pubkey()),
+        mint_keypair,
+        Some(String::from("seed")),
+        false,
+    );
 }
 
 #[test]
 fn test_nonce_with_authority() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
-
-    full_battery_tests(leader_data, alice, None, true);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
+    let mint_keypair = Keypair::new();
+    full_battery_tests(
+        TestValidator::with_no_fees(mint_keypair.pubkey()),
+        mint_keypair,
+        None,
+        true,
+    );
 }
 
 fn full_battery_tests(
-    leader_data: ContactInfo,
-    alice: Keypair,
+    test_validator: TestValidator,
+    mint_keypair: Keypair,
     seed: Option<String>,
     use_nonce_authority: bool,
 ) {
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
-    let json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
+    let json_rpc_url = test_validator.rpc_url();
 
     let mut config_payer = CliConfig::recent_for_tests();
     config_payer.json_rpc_url = json_rpc_url.clone();
@@ -172,7 +155,7 @@ fn full_battery_tests(
     assert_ne!(first_nonce, third_nonce);
 
     // Withdraw from nonce account
-    let payee_pubkey = Pubkey::new_rand();
+    let payee_pubkey = solana_sdk::pubkey::new_rand();
     config_payer.signers = authorized_signers;
     config_payer.command = CliCommand::WithdrawFromNonceAccount {
         nonce_account,
@@ -231,17 +214,9 @@ fn full_battery_tests(
 
 #[test]
 fn test_create_account_with_seed() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice: mint_keypair,
-        ledger_path,
-        ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
+    solana_logger::setup();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);
@@ -253,7 +228,7 @@ fn test_create_account_with_seed() {
     let config = CliConfig::recent_for_tests();
 
     // Setup accounts
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     request_and_confirm_airdrop(
         &rpc_client,
         &faucet_addr,
@@ -285,8 +260,7 @@ fn test_create_account_with_seed() {
     check_recent_balance(0, &rpc_client, &nonce_address);
 
     let mut creator_config = CliConfig::recent_for_tests();
-    creator_config.json_rpc_url =
-        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    creator_config.json_rpc_url = test_validator.rpc_url();
     creator_config.signers = vec![&online_nonce_creator_signer];
     creator_config.command = CliCommand::CreateNonceAccount {
         nonce_account: 0,
@@ -336,8 +310,7 @@ fn test_create_account_with_seed() {
 
     // And submit it
     let mut submit_config = CliConfig::recent_for_tests();
-    submit_config.json_rpc_url =
-        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    submit_config.json_rpc_url = test_validator.rpc_url();
     submit_config.signers = vec![&authority_presigner];
     submit_config.command = CliCommand::Transfer {
         amount: SpendAmount::Some(10),
@@ -358,7 +331,4 @@ fn test_create_account_with_seed() {
     check_recent_balance(31, &rpc_client, &offline_nonce_authority_signer.pubkey());
     check_recent_balance(4000, &rpc_client, &online_nonce_creator_signer.pubkey());
     check_recent_balance(10, &rpc_client, &to_address);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }

cli/tests/request_airdrop.rs

@@ -2,24 +2,23 @@ use solana_cli::cli::{process_command, CliCommand, CliConfig};
 use solana_client::rpc_client::RpcClient;
 use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
-use solana_sdk::{commitment_config::CommitmentConfig, signature::Keypair};
-use std::{fs::remove_dir_all, sync::mpsc::channel};
+use solana_sdk::{
+    commitment_config::CommitmentConfig,
+    signature::{Keypair, Signer},
+};
+use std::sync::mpsc::channel;
 
 #[test]
 fn test_cli_request_airdrop() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
     let mut bob_config = CliConfig::recent_for_tests();
-    bob_config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    bob_config.json_rpc_url = test_validator.rpc_url();
     bob_config.command = CliCommand::Airdrop {
         faucet_host: None,
         faucet_port: faucet_addr.port(),
@@ -32,14 +31,11 @@ fn test_cli_request_airdrop() {
     let sig_response = process_command(&bob_config);
     sig_response.unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
 
     let balance = rpc_client
         .get_balance_with_commitment(&bob_config.signers[0].pubkey(), CommitmentConfig::recent())
         .unwrap()
         .value;
     assert_eq!(balance, 50);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }

cli/tests/stake.rs

@@ -9,7 +9,7 @@ use solana_client::{
     nonce_utils,
     rpc_client::RpcClient,
 };
-use solana_core::test_validator::{TestValidator, TestValidatorOptions};
+use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     account_utils::StateMut,
@@ -22,26 +22,21 @@ use solana_stake_program::{
     stake_instruction::LockupArgs,
     stake_state::{Lockup, StakeAuthorize, StakeState},
 };
-use std::{fs::remove_dir_all, sync::mpsc::channel};
+use std::sync::mpsc::channel;
 
 #[test]
 fn test_stake_delegation_force() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let default_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     request_and_confirm_airdrop(
@@ -113,33 +108,23 @@ fn test_stake_delegation_force() {
         fee_payer: 0,
     };
     process_command(&config).unwrap();
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_seed_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        vote_pubkey,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
 
     let validator_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let mut config_validator = CliConfig::recent_for_tests();
-    config_validator.json_rpc_url =
-        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config_validator.json_rpc_url = test_validator.rpc_url();
     config_validator.signers = vec![&validator_keypair];
 
     request_and_confirm_airdrop(
@@ -180,7 +165,7 @@ fn test_seed_stake_delegation_and_deactivation() {
     // Delegate stake
     config_validator.command = CliCommand::DelegateStake {
         stake_account_pubkey: stake_address,
-        vote_account_pubkey: vote_pubkey,
+        vote_account_pubkey: test_validator.vote_account_address(),
         stake_authority: 0,
         force: true,
         sign_only: false,
@@ -202,33 +187,23 @@ fn test_seed_stake_delegation_and_deactivation() {
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        vote_pubkey,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let validator_keypair = Keypair::new();
 
     let mut config_validator = CliConfig::recent_for_tests();
-    config_validator.json_rpc_url =
-        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config_validator.json_rpc_url = test_validator.rpc_url();
     config_validator.signers = vec![&validator_keypair];
 
     let stake_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
@@ -265,7 +240,7 @@ fn test_stake_delegation_and_deactivation() {
     config_validator.signers.pop();
     config_validator.command = CliCommand::DelegateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
-        vote_account_pubkey: vote_pubkey,
+        vote_account_pubkey: test_validator.vote_account_address(),
         stake_authority: 0,
         force: true,
         sign_only: false,
@@ -287,38 +262,27 @@ fn test_stake_delegation_and_deactivation() {
         fee_payer: 0,
     };
     process_command(&config_validator).unwrap();
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_offline_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        vote_pubkey,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
 
     let mut config_validator = CliConfig::recent_for_tests();
-    config_validator.json_rpc_url =
-        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config_validator.json_rpc_url = test_validator.rpc_url();
     let validator_keypair = Keypair::new();
     config_validator.signers = vec![&validator_keypair];
 
     let mut config_payer = CliConfig::recent_for_tests();
-    config_payer.json_rpc_url =
-        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config_payer.json_rpc_url = test_validator.rpc_url();
 
     let stake_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
 
@@ -372,7 +336,7 @@ fn test_offline_stake_delegation_and_deactivation() {
     let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     config_offline.command = CliCommand::DelegateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
-        vote_account_pubkey: vote_pubkey,
+        vote_account_pubkey: test_validator.vote_account_address(),
         stake_authority: 0,
         force: true,
         sign_only: true,
@@ -391,7 +355,7 @@ fn test_offline_stake_delegation_and_deactivation() {
     config_payer.signers = vec![&offline_presigner];
     config_payer.command = CliCommand::DelegateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
-        vote_account_pubkey: vote_pubkey,
+        vote_account_pubkey: test_validator.vote_account_address(),
         stake_authority: 0,
         force: true,
         sign_only: false,
@@ -430,33 +394,24 @@ fn test_offline_stake_delegation_and_deactivation() {
         fee_payer: 0,
     };
     process_command(&config_payer).unwrap();
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_nonced_stake_delegation_and_deactivation() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        vote_pubkey,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
 
     let config_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let mut config = CliConfig::recent_for_tests();
     config.signers = vec![&config_keypair];
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
 
     let minimum_nonce_balance = rpc_client
         .get_minimum_balance_for_rent_exemption(NonceState::size())
@@ -515,7 +470,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
     config.signers = vec![&config_keypair];
     config.command = CliCommand::DelegateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
-        vote_account_pubkey: vote_pubkey,
+        vote_account_pubkey: test_validator.vote_account_address(),
         stake_authority: 0,
         force: true,
         sign_only: false,
@@ -553,31 +508,23 @@ fn test_nonced_stake_delegation_and_deactivation() {
         fee_payer: 0,
     };
     process_command(&config).unwrap();
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_stake_authorize() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let default_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     request_and_confirm_airdrop(
@@ -832,9 +779,6 @@ fn test_stake_authorize() {
     .unwrap()
     .blockhash;
     assert_ne!(nonce_hash, new_nonce_hash);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
@@ -842,34 +786,24 @@ fn test_stake_authorize_with_fee_payer() {
     solana_logger::setup();
     const SIG_FEE: u64 = 42;
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
-        fees: SIG_FEE,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), SIG_FEE);
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let default_signer = Keypair::new();
     let default_pubkey = default_signer.pubkey();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     let payer_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let mut config_payer = CliConfig::recent_for_tests();
     config_payer.signers = vec![&payer_keypair];
-    config_payer.json_rpc_url =
-        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config_payer.json_rpc_url = test_validator.rpc_url();
     let payer_pubkey = config_payer.signers[0].pubkey();
 
     let mut config_offline = CliConfig::recent_for_tests();
@@ -970,36 +904,24 @@ fn test_stake_authorize_with_fee_payer() {
     // `config_offline` however has paid 1 sig due to being both authority
     // and fee payer
     check_recent_balance(100_000 - SIG_FEE, &rpc_client, &offline_pubkey);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_stake_split() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let default_signer = Keypair::new();
     let offline_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     let mut config_offline = CliConfig::recent_for_tests();
@@ -1125,36 +1047,24 @@ fn test_stake_split() {
         &rpc_client,
         &split_account.pubkey(),
     );
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_stake_set_lockup() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let default_signer = Keypair::new();
     let offline_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     let mut config_offline = CliConfig::recent_for_tests();
@@ -1187,8 +1097,10 @@ fn test_stake_set_lockup() {
     let stake_keypair = keypair_from_seed(&[0u8; 32]).unwrap();
     let stake_account_pubkey = stake_keypair.pubkey();
 
-    let mut lockup = Lockup::default();
-    lockup.custodian = config.signers[0].pubkey();
+    let lockup = Lockup {
+        custodian: config.signers[0].pubkey(),
+        ..Lockup::default()
+    };
 
     config.signers.push(&stake_keypair);
     config.command = CliCommand::CreateStakeAccount {
@@ -1398,32 +1310,23 @@ fn test_stake_set_lockup() {
     );
     assert_eq!(current_lockup.epoch, lockup.epoch.unwrap());
     assert_eq!(current_lockup.custodian, offline_pubkey);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_offline_nonced_create_stake_account_and_withdraw() {
     solana_logger::setup();
 
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
-
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let mut config = CliConfig::recent_for_tests();
     let default_signer = keypair_from_seed(&[1u8; 32]).unwrap();
     config.signers = vec![&default_signer];
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
 
     let mut config_offline = CliConfig::recent_for_tests();
     let offline_signer = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -1619,7 +1522,4 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     let seed_address =
         Pubkey::create_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
     check_recent_balance(50_000, &rpc_client, &seed_address);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }

cli/tests/transfer.rs

@@ -9,7 +9,7 @@ use solana_client::{
     nonce_utils,
     rpc_client::RpcClient,
 };
-use solana_core::test_validator::{TestValidator, TestValidatorOptions};
+use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     commitment_config::CommitmentConfig,
@@ -17,33 +17,25 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, NullSigner, Signer},
 };
-use std::{fs::remove_dir_all, sync::mpsc::channel};
+use std::sync::mpsc::channel;
 
 #[test]
 fn test_transfer() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice: mint_keypair,
-        ledger_path,
-        ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
+    solana_logger::setup();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
 
     let default_signer = Keypair::new();
     let default_offline_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     let sender_pubkey = config.signers[0].pubkey();
@@ -245,24 +237,13 @@ fn test_transfer() {
     process_command(&config).unwrap();
     check_recent_balance(28, &rpc_client, &offline_pubkey);
     check_recent_balance(40, &rpc_client, &recipient_pubkey);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_transfer_multisession_signing() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice: mint_keypair,
-        ledger_path,
-        ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
+    solana_logger::setup();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);
@@ -275,7 +256,7 @@ fn test_transfer_multisession_signing() {
     let config = CliConfig::recent_for_tests();
 
     // Setup accounts
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     request_and_confirm_airdrop(
         &rpc_client,
         &faucet_addr,
@@ -357,7 +338,7 @@ fn test_transfer_multisession_signing() {
 
     // Finally submit to the cluster
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&fee_payer_presigner, &from_presigner];
     config.command = CliCommand::Transfer {
         amount: SpendAmount::Some(42),
@@ -375,35 +356,24 @@ fn test_transfer_multisession_signing() {
     check_recent_balance(1, &rpc_client, &offline_from_signer.pubkey());
     check_recent_balance(1, &rpc_client, &offline_fee_payer_signer.pubkey());
     check_recent_balance(42, &rpc_client, &to_pubkey);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }
 
 #[test]
 fn test_transfer_all() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice: mint_keypair,
-        ledger_path,
-        ..
-    } = TestValidator::run_with_options(TestValidatorOptions {
-        fees: 1,
-        bootstrap_validator_lamports: 42_000,
-        ..TestValidatorOptions::default()
-    });
+    solana_logger::setup();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_custom_fees(mint_keypair.pubkey(), 1);
 
     let (sender, receiver) = channel();
     run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
 
     let default_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     let sender_pubkey = config.signers[0].pubkey();
@@ -431,7 +401,4 @@ fn test_transfer_all() {
     process_command(&config).unwrap();
     check_recent_balance(0, &rpc_client, &sender_pubkey);
     check_recent_balance(49_999, &rpc_client, &recipient_pubkey);
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }

cli/tests/vote.rs

@@ -12,30 +12,24 @@ use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     account_utils::StateMut,
     commitment_config::CommitmentConfig,
-    pubkey::Pubkey,
     signature::{Keypair, Signer},
 };
 use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersions};
-use std::{fs::remove_dir_all, sync::mpsc::channel};
+use std::sync::mpsc::channel;
 
 #[test]
 fn test_vote_authorize_and_withdraw() {
-    let TestValidator {
-        server,
-        leader_data,
-        alice,
-        ledger_path,
-        ..
-    } = TestValidator::run();
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
     let (sender, receiver) = channel();
-    run_local_faucet(alice, sender, None);
+    run_local_faucet(mint_keypair, sender, None);
     let faucet_addr = receiver.recv().unwrap();
 
-    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
     let default_signer = Keypair::new();
 
     let mut config = CliConfig::recent_for_tests();
-    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&default_signer];
 
     request_and_confirm_airdrop(
@@ -110,7 +104,7 @@ fn test_vote_authorize_and_withdraw() {
     assert_eq!(authorized_withdrawer, withdraw_authority.pubkey());
 
     // Withdraw from vote account
-    let destination_account = Pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
+    let destination_account = solana_sdk::pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
     config.signers = vec![&default_signer, &withdraw_authority];
     config.command = CliCommand::WithdrawFromVoteAccount {
         vote_account_pubkey,
@@ -131,7 +125,4 @@ fn test_vote_authorize_and_withdraw() {
         withdraw_authority: 1,
     };
     process_command(&config).unwrap();
-
-    server.close().unwrap();
-    remove_dir_all(ledger_path).unwrap();
 }

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.4.0"
+version = "1.5.3"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,18 +15,21 @@ bs58 = "0.3.1"
 clap = "2.33.0"
 indicatif = "0.15.0"
 jsonrpc-core = "15.0.0"
-log = "0.4.8"
+log = "0.4.11"
+net2 = "0.2.37"
 rayon = "1.4.0"
-reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+semver = "0.11.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.3" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-net-utils = { path = "../net-utils", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.3" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -35,7 +38,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "15.0.0"
 jsonrpc-http-server = "15.0.0"
-solana-logger = { path = "../logger", version = "1.4.0" }
+solana-logger = { path = "../logger", version = "1.5.3" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/client_error.rs

@@ -5,6 +5,8 @@ use solana_sdk::{
 use std::io;
 use thiserror::Error;
 
+pub use reqwest; // export `reqwest` for clients
+
 #[derive(Error, Debug)]
 pub enum ClientErrorKind {
     #[error(transparent)]
@@ -50,10 +52,10 @@ impl Into<TransportError> for ClientErrorKind {
 #[derive(Error, Debug)]
 #[error("{kind}")]
 pub struct ClientError {
-    request: Option<rpc_request::RpcRequest>,
+    pub request: Option<rpc_request::RpcRequest>,
 
     #[source]
-    kind: ClientErrorKind,
+    pub kind: ClientErrorKind,
 }
 
 impl ClientError {

client/src/http_sender.rs

@@ -1,6 +1,8 @@
 use crate::{
     client_error::Result,
-    rpc_request::{RpcError, RpcRequest},
+    rpc_custom_error,
+    rpc_request::{RpcError, RpcRequest, RpcResponseErrorData},
+    rpc_response::RpcSimulateTransactionResult,
     rpc_sender::RpcSender,
 };
 use log::*;
@@ -27,6 +29,13 @@ impl HttpSender {
     }
 }
 
+#[derive(Deserialize, Debug)]
+struct RpcErrorObject {
+    code: i64,
+    message: String,
+    data: serde_json::Value,
+}
+
 impl RpcSender for HttpSender {
     fn send(&self, request: RpcRequest, params: serde_json::Value) -> Result<serde_json::Value> {
         // Concurrent requests are not supported so reuse the same request id for all requests
@@ -63,11 +72,36 @@ impl RpcSender for HttpSender {
 
                     let json: serde_json::Value = serde_json::from_str(&response.text()?)?;
                     if json["error"].is_object() {
-                        return Err(RpcError::RpcRequestError(format!(
-                            "RPC Error response: {}",
-                            serde_json::to_string(&json["error"]).unwrap()
-                        ))
-                        .into());
+                        return match serde_json::from_value::<RpcErrorObject>(json["error"].clone())
+                        {
+                            Ok(rpc_error_object) => {
+                                let data = match rpc_error_object.code {
+                                    rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE => {
+                                        match serde_json::from_value::<RpcSimulateTransactionResult>(json["error"]["data"].clone()) {
+                                            Ok(data) => RpcResponseErrorData::SendTransactionPreflightFailure(data),
+                                            Err(err) => {
+                                                debug!("Failed to deserialize RpcSimulateTransactionResult: {:?}", err);
+                                                RpcResponseErrorData::Empty
+                                            }
+                                        }
+                                    },
+                                    _ => RpcResponseErrorData::Empty
+                                };
+
+                                Err(RpcError::RpcResponseError {
+                                    code: rpc_error_object.code,
+                                    message: rpc_error_object.message,
+                                    data,
+                                }
+                                .into())
+                            }
+                            Err(err) => Err(RpcError::RpcRequestError(format!(
+                                "Failed to deserialize RPC error response: {} [{}]",
+                                serde_json::to_string(&json["error"]).unwrap(),
+                                err
+                            ))
+                            .into()),
+                        };
                     }
                     return Ok(json["result"].clone());
                 }

client/src/lib.rs

@@ -10,6 +10,7 @@ pub mod perf_utils;
 pub mod pubsub_client;
 pub mod rpc_client;
 pub mod rpc_config;
+pub mod rpc_custom_error;
 pub mod rpc_filter;
 pub mod rpc_request;
 pub mod rpc_response;

client/src/mock_sender.rs

@@ -1,17 +1,19 @@
 use crate::{
     client_error::Result,
     rpc_request::RpcRequest,
-    rpc_response::{Response, RpcResponseContext},
+    rpc_response::{Response, RpcResponseContext, RpcVersionInfo},
     rpc_sender::RpcSender,
 };
-use serde_json::{Number, Value};
+use serde_json::{json, Number, Value};
 use solana_sdk::{
+    epoch_info::EpochInfo,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
     instruction::InstructionError,
     signature::Signature,
     transaction::{self, Transaction, TransactionError},
 };
 use solana_transaction_status::TransactionStatus;
+use solana_version::Version;
 use std::{collections::HashMap, sync::RwLock};
 
 pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
@@ -46,6 +48,10 @@ impl RpcSender for MockSender {
             return Ok(Value::Null);
         }
         let val = match request {
+            RpcRequest::GetAccountInfo => serde_json::to_value(Response {
+                context: RpcResponseContext { slot: 1 },
+                value: Value::Null,
+            })?,
             RpcRequest::GetBalance => serde_json::to_value(Response {
                 context: RpcResponseContext { slot: 1 },
                 value: Value::Number(Number::from(50)),
@@ -57,6 +63,14 @@ impl RpcSender for MockSender {
                     serde_json::to_value(FeeCalculator::default()).unwrap(),
                 ),
             })?,
+            RpcRequest::GetEpochInfo => serde_json::to_value(EpochInfo {
+                epoch: 1,
+                slot_index: 2,
+                slots_in_epoch: 32,
+                absolute_slot: 34,
+                block_height: 34,
+                transaction_count: Some(123),
+            })?,
             RpcRequest::GetFeeCalculatorForBlockhash => {
                 let value = if self.url == "blockhash_expired" {
                     Value::Null
@@ -112,13 +126,20 @@ impl RpcSender for MockSender {
                     Signature::new(&[8; 64]).to_string()
                 } else {
                     let tx_str = params.as_array().unwrap()[0].as_str().unwrap().to_string();
-                    let data = bs58::decode(tx_str).into_vec().unwrap();
+                    let data = base64::decode(tx_str).unwrap();
                     let tx: Transaction = bincode::deserialize(&data).unwrap();
                     tx.signatures[0].to_string()
                 };
                 Value::String(signature)
             }
             RpcRequest::GetMinimumBalanceForRentExemption => Value::Number(Number::from(20)),
+            RpcRequest::GetVersion => {
+                let version = Version::default();
+                json!(RpcVersionInfo {
+                    solana_core: version.to_string(),
+                    feature_set: Some(version.feature_set),
+                })
+            }
             _ => Value::Null,
         };
         Ok(val)

client/src/pubsub_client.rs

@@ -1,4 +1,7 @@
-use crate::rpc_response::{Response as RpcResponse, RpcSignatureResult, SlotInfo};
+use crate::{
+    rpc_config::{RpcSignatureSubscribeConfig, RpcTransactionLogsConfig, RpcTransactionLogsFilter},
+    rpc_response::{Response as RpcResponse, RpcLogsResponse, RpcSignatureResult, SlotInfo},
+};
 use log::*;
 use serde::de::DeserializeOwned;
 use serde_json::{
@@ -20,8 +23,6 @@ use thiserror::Error;
 use tungstenite::{client::AutoStream, connect, Message, WebSocket};
 use url::{ParseError, Url};
 
-type PubsubSignatureResponse = PubsubClientSubscription<RpcResponse<RpcSignatureResult>>;
-
 #[derive(Debug, Error)]
 pub enum PubsubClientError {
     #[error("url parse error")]
@@ -33,8 +34,8 @@ pub enum PubsubClientError {
     #[error("json parse error")]
     JsonParseError(#[from] serde_json::error::Error),
 
-    #[error("unexpected message format")]
-    UnexpectedMessageError,
+    #[error("unexpected message format: {0}")]
+    UnexpectedMessageError(String),
 }
 
 pub struct PubsubClientSubscription<T>
@@ -89,8 +90,11 @@ where
                 return Ok(x);
             }
         }
-
-        Err(PubsubClientError::UnexpectedMessageError)
+        // TODO: Add proper JSON RPC response/error handling...
+        Err(PubsubClientError::UnexpectedMessageError(format!(
+            "{:?}",
+            json_msg
+        )))
     }
 
     pub fn send_unsubscribe(&self) -> Result<(), PubsubClientError> {
@@ -114,14 +118,18 @@ where
         let message_text = &message.into_text().unwrap();
         let json_msg: Map<String, Value> = serde_json::from_str(message_text)?;
 
-        if let Some(Object(value_1)) = json_msg.get("params") {
-            if let Some(value_2) = value_1.get("result") {
-                let x: T = serde_json::from_value::<T>(value_2.clone()).unwrap();
+        if let Some(Object(params)) = json_msg.get("params") {
+            if let Some(result) = params.get("result") {
+                let x: T = serde_json::from_value::<T>(result.clone()).unwrap();
                 return Ok(x);
             }
         }
 
-        Err(PubsubClientError::UnexpectedMessageError)
+        // TODO: Add proper JSON RPC response/error handling...
+        Err(PubsubClientError::UnexpectedMessageError(format!(
+            "{:?}",
+            json_msg
+        )))
     }
 
     pub fn shutdown(&mut self) -> std::thread::Result<()> {
@@ -138,15 +146,79 @@ where
     }
 }
 
-const SLOT_OPERATION: &str = "slot";
-const SIGNATURE_OPERATION: &str = "signature";
+pub type LogsSubscription = (
+    PubsubClientSubscription<RpcResponse<RpcLogsResponse>>,
+    Receiver<RpcResponse<RpcLogsResponse>>,
+);
+pub type SlotsSubscription = (PubsubClientSubscription<SlotInfo>, Receiver<SlotInfo>);
+pub type SignatureSubscription = (
+    PubsubClientSubscription<RpcResponse<RpcSignatureResult>>,
+    Receiver<RpcResponse<RpcSignatureResult>>,
+);
 
 pub struct PubsubClient {}
 
 impl PubsubClient {
-    pub fn slot_subscribe(
+    pub fn logs_subscribe(
         url: &str,
-    ) -> Result<(PubsubClientSubscription<SlotInfo>, Receiver<SlotInfo>), PubsubClientError> {
+        filter: RpcTransactionLogsFilter,
+        config: RpcTransactionLogsConfig,
+    ) -> Result<LogsSubscription, PubsubClientError> {
+        let url = Url::parse(url)?;
+        let (socket, _response) = connect(url)?;
+        let (sender, receiver) = channel();
+
+        let socket = Arc::new(RwLock::new(socket));
+        let socket_clone = socket.clone();
+        let exit = Arc::new(AtomicBool::new(false));
+        let exit_clone = exit.clone();
+
+        let subscription_id =
+            PubsubClientSubscription::<RpcResponse<RpcLogsResponse>>::send_subscribe(
+                &socket_clone,
+                json!({
+                    "jsonrpc":"2.0","id":1,"method":"logsSubscribe","params":[filter, config]
+                })
+                .to_string(),
+            )?;
+
+        let t_cleanup = std::thread::spawn(move || {
+            loop {
+                if exit_clone.load(Ordering::Relaxed) {
+                    break;
+                }
+
+                match PubsubClientSubscription::read_message(&socket_clone) {
+                    Ok(message) => match sender.send(message) {
+                        Ok(_) => (),
+                        Err(err) => {
+                            info!("receive error: {:?}", err);
+                            break;
+                        }
+                    },
+                    Err(err) => {
+                        info!("receive error: {:?}", err);
+                        break;
+                    }
+                }
+            }
+
+            info!("websocket - exited receive loop");
+        });
+
+        let result = PubsubClientSubscription {
+            message_type: PhantomData,
+            operation: "logs",
+            socket,
+            subscription_id,
+            t_cleanup: Some(t_cleanup),
+            exit,
+        };
+
+        Ok((result, receiver))
+    }
+
+    pub fn slot_subscribe(url: &str) -> Result<SlotsSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let (socket, _response) = connect(url)?;
         let (sender, receiver) = channel::<SlotInfo>();
@@ -158,41 +230,37 @@ impl PubsubClient {
         let subscription_id = PubsubClientSubscription::<SlotInfo>::send_subscribe(
             &socket_clone,
             json!({
-                "jsonrpc":"2.0","id":1,"method":format!("{}Subscribe", SLOT_OPERATION),"params":[]
+                "jsonrpc":"2.0","id":1,"method":"slotSubscribe","params":[]
             })
             .to_string(),
-        )
-        .unwrap();
+        )?;
 
         let t_cleanup = std::thread::spawn(move || {
             loop {
                 if exit_clone.load(Ordering::Relaxed) {
                     break;
                 }
-
-                let message: Result<SlotInfo, PubsubClientError> =
-                    PubsubClientSubscription::read_message(&socket_clone);
-
-                if let Ok(msg) = message {
-                    match sender.send(msg) {
+                match PubsubClientSubscription::read_message(&socket_clone) {
+                    Ok(message) => match sender.send(message) {
                         Ok(_) => (),
                         Err(err) => {
                             info!("receive error: {:?}", err);
                             break;
                         }
+                    },
+                    Err(err) => {
+                        info!("receive error: {:?}", err);
+                        break;
                     }
-                } else {
-                    info!("receive error: {:?}", message);
-                    break;
                 }
             }
 
             info!("websocket - exited receive loop");
         });
 
-        let result: PubsubClientSubscription<SlotInfo> = PubsubClientSubscription {
+        let result = PubsubClientSubscription {
             message_type: PhantomData,
-            operation: SLOT_OPERATION,
+            operation: "slot",
             socket,
             subscription_id,
             t_cleanup: Some(t_cleanup),
@@ -205,16 +273,11 @@ impl PubsubClient {
     pub fn signature_subscribe(
         url: &str,
         signature: &Signature,
-    ) -> Result<
-        (
-            PubsubSignatureResponse,
-            Receiver<RpcResponse<RpcSignatureResult>>,
-        ),
-        PubsubClientError,
-    > {
+        config: Option<RpcSignatureSubscribeConfig>,
+    ) -> Result<SignatureSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let (socket, _response) = connect(url)?;
-        let (sender, receiver) = channel::<RpcResponse<RpcSignatureResult>>();
+        let (sender, receiver) = channel();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();
@@ -223,10 +286,10 @@ impl PubsubClient {
         let body = json!({
             "jsonrpc":"2.0",
             "id":1,
-            "method":format!("{}Subscribe", SIGNATURE_OPERATION),
+            "method":"signatureSubscribe",
             "params":[
                 signature.to_string(),
-                {"enableReceivedNotification": true }
+                config
             ]
         })
         .to_string();
@@ -234,8 +297,7 @@ impl PubsubClient {
             PubsubClientSubscription::<RpcResponse<RpcSignatureResult>>::send_subscribe(
                 &socket_clone,
                 body,
-            )
-            .unwrap();
+            )?;
 
         let t_cleanup = std::thread::spawn(move || {
             loop {
@@ -263,15 +325,14 @@ impl PubsubClient {
             info!("websocket - exited receive loop");
         });
 
-        let result: PubsubClientSubscription<RpcResponse<RpcSignatureResult>> =
-            PubsubClientSubscription {
-                message_type: PhantomData,
-                operation: SIGNATURE_OPERATION,
-                socket,
-                subscription_id,
-                t_cleanup: Some(t_cleanup),
-                exit,
-            };
+        let result = PubsubClientSubscription {
+            message_type: PhantomData,
+            operation: "signature",
+            socket,
+            subscription_id,
+            t_cleanup: Some(t_cleanup),
+            exit,
+        };
 
         Ok((result, receiver))
     }

client/src/rpc_client.rs

@@ -8,7 +8,7 @@ use crate::{
         RpcProgramAccountsConfig, RpcSendTransactionConfig, RpcSimulateTransactionConfig,
         RpcTokenAccountsFilter,
     },
-    rpc_request::{RpcError, RpcRequest, TokenAccountsFilter},
+    rpc_request::{RpcError, RpcRequest, RpcResponseErrorData, TokenAccountsFilter},
     rpc_response::*,
     rpc_sender::RpcSender,
 };
@@ -41,12 +41,15 @@ use solana_transaction_status::{
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     net::SocketAddr,
+    sync::RwLock,
     thread::sleep,
     time::{Duration, Instant},
 };
 
 pub struct RpcClient {
     sender: Box<dyn RpcSender + Send + Sync + 'static>,
+    commitment_config: CommitmentConfig,
+    default_cluster_transaction_encoding: RwLock<Option<UiTransactionEncoding>>,
 }
 
 fn serialize_encode_transaction(
@@ -70,26 +73,41 @@ fn serialize_encode_transaction(
 }
 
 impl RpcClient {
-    pub fn new_sender<T: RpcSender + Send + Sync + 'static>(sender: T) -> Self {
+    fn new_sender<T: RpcSender + Send + Sync + 'static>(
+        sender: T,
+        commitment_config: CommitmentConfig,
+    ) -> Self {
         Self {
             sender: Box::new(sender),
+            default_cluster_transaction_encoding: RwLock::new(None),
+            commitment_config,
         }
     }
 
     pub fn new(url: String) -> Self {
-        Self::new_sender(HttpSender::new(url))
+        Self::new_with_commitment(url, CommitmentConfig::default())
+    }
+
+    pub fn new_with_commitment(url: String, commitment_config: CommitmentConfig) -> Self {
+        Self::new_sender(HttpSender::new(url), commitment_config)
     }
 
     pub fn new_with_timeout(url: String, timeout: Duration) -> Self {
-        Self::new_sender(HttpSender::new_with_timeout(url, timeout))
+        Self::new_sender(
+            HttpSender::new_with_timeout(url, timeout),
+            CommitmentConfig::default(),
+        )
     }
 
     pub fn new_mock(url: String) -> Self {
-        Self::new_sender(MockSender::new(url))
+        Self::new_sender(MockSender::new(url), CommitmentConfig::default())
     }
 
     pub fn new_mock_with_mocks(url: String, mocks: Mocks) -> Self {
-        Self::new_sender(MockSender::new_with_mocks(url, mocks))
+        Self::new_sender(
+            MockSender::new_with_mocks(url, mocks),
+            CommitmentConfig::default(),
+        )
     }
 
     pub fn new_socket(addr: SocketAddr) -> Self {
@@ -103,10 +121,14 @@ impl RpcClient {
 
     pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
         Ok(self
-            .confirm_transaction_with_commitment(signature, CommitmentConfig::default())?
+            .confirm_transaction_with_commitment(signature, self.commitment_config)?
             .value)
     }
 
+    pub fn commitment(&self) -> CommitmentConfig {
+        self.commitment_config
+    }
+
     pub fn confirm_transaction_with_commitment(
         &self,
         signature: &Signature,
@@ -125,7 +147,38 @@ impl RpcClient {
     }
 
     pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<Signature> {
-        self.send_transaction_with_config(transaction, RpcSendTransactionConfig::default())
+        self.send_transaction_with_config(
+            transaction,
+            RpcSendTransactionConfig {
+                preflight_commitment: Some(self.commitment_config.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
+        )
+    }
+
+    fn default_cluster_transaction_encoding(&self) -> Result<UiTransactionEncoding, RpcError> {
+        let default_cluster_transaction_encoding =
+            self.default_cluster_transaction_encoding.read().unwrap();
+        if let Some(encoding) = *default_cluster_transaction_encoding {
+            Ok(encoding)
+        } else {
+            drop(default_cluster_transaction_encoding);
+            let cluster_version = self.get_version().map_err(|e| {
+                RpcError::RpcRequestError(format!("cluster version query failed: {}", e))
+            })?;
+            let cluster_version =
+                semver::Version::parse(&cluster_version.solana_core).map_err(|e| {
+                    RpcError::RpcRequestError(format!("failed to parse cluster version: {}", e))
+                })?;
+            // Prefer base64 since 1.3.16
+            let encoding = if cluster_version < semver::Version::new(1, 3, 16) {
+                UiTransactionEncoding::Base58
+            } else {
+                UiTransactionEncoding::Base64
+            };
+            *self.default_cluster_transaction_encoding.write().unwrap() = Some(encoding);
+            Ok(encoding)
+        }
     }
 
     pub fn send_transaction_with_config(
@@ -133,12 +186,43 @@ impl RpcClient {
         transaction: &Transaction,
         config: RpcSendTransactionConfig,
     ) -> ClientResult<Signature> {
-        let encoding = config.encoding.unwrap_or(UiTransactionEncoding::Base58);
+        let encoding = if let Some(encoding) = config.encoding {
+            encoding
+        } else {
+            self.default_cluster_transaction_encoding()?
+        };
+        let config = RpcSendTransactionConfig {
+            encoding: Some(encoding),
+            ..config
+        };
         let serialized_encoded = serialize_encode_transaction(transaction, encoding)?;
-        let signature_base58_str: String = self.send(
+        let signature_base58_str: String = match self.send(
             RpcRequest::SendTransaction,
             json!([serialized_encoded, config]),
-        )?;
+        ) {
+            Ok(signature_base58_str) => signature_base58_str,
+            Err(err) => {
+                if let ClientErrorKind::RpcError(RpcError::RpcResponseError {
+                    code,
+                    message,
+                    data,
+                }) = &err.kind
+                {
+                    debug!("{} {}", code, message);
+                    if let RpcResponseErrorData::SendTransactionPreflightFailure(
+                        RpcSimulateTransactionResult {
+                            logs: Some(logs), ..
+                        },
+                    ) = data
+                    {
+                        for (i, log) in logs.iter().enumerate() {
+                            debug!("{:>3}: {}", i + 1, log);
+                        }
+                    }
+                }
+                return Err(err);
+            }
+        };
 
         let signature = signature_base58_str
             .parse::<Signature>()
@@ -161,26 +245,34 @@ impl RpcClient {
     pub fn simulate_transaction(
         &self,
         transaction: &Transaction,
-        sig_verify: bool,
     ) -> RpcResult<RpcSimulateTransactionResult> {
         self.simulate_transaction_with_config(
             transaction,
-            sig_verify,
-            RpcSimulateTransactionConfig::default(),
+            RpcSimulateTransactionConfig {
+                commitment: Some(self.commitment_config),
+                ..RpcSimulateTransactionConfig::default()
+            },
         )
     }
 
     pub fn simulate_transaction_with_config(
         &self,
         transaction: &Transaction,
-        sig_verify: bool,
         config: RpcSimulateTransactionConfig,
     ) -> RpcResult<RpcSimulateTransactionResult> {
-        let encoding = config.encoding.unwrap_or(UiTransactionEncoding::Base58);
+        let encoding = if let Some(encoding) = config.encoding {
+            encoding
+        } else {
+            self.default_cluster_transaction_encoding()?
+        };
+        let config = RpcSimulateTransactionConfig {
+            encoding: Some(encoding),
+            ..config
+        };
         let serialized_encoded = serialize_encode_transaction(transaction, encoding)?;
         self.send(
             RpcRequest::SimulateTransaction,
-            json!([serialized_encoded, { "sigVerify": sig_verify }]),
+            json!([serialized_encoded, config]),
         )
     }
 
@@ -188,7 +280,7 @@ impl RpcClient {
         &self,
         signature: &Signature,
     ) -> ClientResult<Option<transaction::Result<()>>> {
-        self.get_signature_status_with_commitment(signature, CommitmentConfig::default())
+        self.get_signature_status_with_commitment(signature, self.commitment_config)
     }
 
     pub fn get_signature_statuses(
@@ -246,7 +338,7 @@ impl RpcClient {
     }
 
     pub fn get_slot(&self) -> ClientResult<Slot> {
-        self.get_slot_with_commitment(CommitmentConfig::default())
+        self.get_slot_with_commitment(self.commitment_config)
     }
 
     pub fn get_slot_with_commitment(
@@ -264,7 +356,7 @@ impl RpcClient {
     }
 
     pub fn total_supply(&self) -> ClientResult<u64> {
-        self.total_supply_with_commitment(CommitmentConfig::default())
+        self.total_supply_with_commitment(self.commitment_config)
     }
 
     pub fn total_supply_with_commitment(
@@ -282,7 +374,7 @@ impl RpcClient {
     }
 
     pub fn get_vote_accounts(&self) -> ClientResult<RpcVoteAccountStatus> {
-        self.get_vote_accounts_with_commitment(CommitmentConfig::default())
+        self.get_vote_accounts_with_commitment(self.commitment_config)
     }
 
     pub fn get_vote_accounts_with_commitment(
@@ -292,6 +384,38 @@ impl RpcClient {
         self.send(RpcRequest::GetVoteAccounts, json!([commitment_config]))
     }
 
+    pub fn wait_for_max_stake(
+        &self,
+        commitment: CommitmentConfig,
+        max_stake_percent: f32,
+    ) -> ClientResult<()> {
+        let mut current_percent;
+        loop {
+            let vote_accounts = self.get_vote_accounts_with_commitment(commitment)?;
+
+            let mut max = 0;
+            let total_active_stake = vote_accounts
+                .current
+                .iter()
+                .chain(vote_accounts.delinquent.iter())
+                .map(|vote_account| {
+                    max = std::cmp::max(max, vote_account.activated_stake);
+                    vote_account.activated_stake
+                })
+                .sum::<u64>();
+            current_percent = 100f32 * max as f32 / total_active_stake as f32;
+            if current_percent < max_stake_percent {
+                break;
+            }
+            info!(
+                "Waiting for stake to drop below {} current: {:.1}",
+                max_stake_percent, current_percent
+            );
+            sleep(Duration::from_secs(10));
+        }
+        Ok(())
+    }
+
     pub fn get_cluster_nodes(&self) -> ClientResult<Vec<RpcContactInfo>> {
         self.send(RpcRequest::GetClusterNodes, Value::Null)
     }
@@ -410,7 +534,7 @@ impl RpcClient {
     }
 
     pub fn get_epoch_info(&self) -> ClientResult<EpochInfo> {
-        self.get_epoch_info_with_commitment(CommitmentConfig::default())
+        self.get_epoch_info_with_commitment(self.commitment_config)
     }
 
     pub fn get_epoch_info_with_commitment(
@@ -424,7 +548,7 @@ impl RpcClient {
         &self,
         slot: Option<Slot>,
     ) -> ClientResult<Option<RpcLeaderSchedule>> {
-        self.get_leader_schedule_with_commitment(slot, CommitmentConfig::default())
+        self.get_leader_schedule_with_commitment(slot, self.commitment_config)
     }
 
     pub fn get_leader_schedule_with_commitment(
@@ -518,8 +642,10 @@ impl RpcClient {
         }
     }
 
+    /// Note that `get_account` returns `Err(..)` if the account does not exist whereas
+    /// `get_account_with_commitment` returns `Ok(None)` if the account does not exist.
     pub fn get_account(&self, pubkey: &Pubkey) -> ClientResult<Account> {
-        self.get_account_with_commitment(pubkey, CommitmentConfig::default())?
+        self.get_account_with_commitment(pubkey, self.commitment_config)?
             .value
             .ok_or_else(|| RpcError::ForUser(format!("AccountNotFound: pubkey={}", pubkey)).into())
     }
@@ -567,7 +693,7 @@ impl RpcClient {
 
     pub fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> ClientResult<Vec<Option<Account>>> {
         Ok(self
-            .get_multiple_accounts_with_commitment(pubkeys, CommitmentConfig::default())?
+            .get_multiple_accounts_with_commitment(pubkeys, self.commitment_config)?
             .value)
     }
 
@@ -621,7 +747,7 @@ impl RpcClient {
     /// Request the balance of the account `pubkey`.
     pub fn get_balance(&self, pubkey: &Pubkey) -> ClientResult<u64> {
         Ok(self
-            .get_balance_with_commitment(pubkey, CommitmentConfig::default())?
+            .get_balance_with_commitment(pubkey, self.commitment_config)?
             .value)
     }
 
@@ -663,7 +789,7 @@ impl RpcClient {
 
     /// Request the transaction count.
     pub fn get_transaction_count(&self) -> ClientResult<u64> {
-        self.get_transaction_count_with_commitment(CommitmentConfig::default())
+        self.get_transaction_count_with_commitment(self.commitment_config)
     }
 
     pub fn get_transaction_count_with_commitment(
@@ -675,7 +801,7 @@ impl RpcClient {
 
     pub fn get_recent_blockhash(&self) -> ClientResult<(Hash, FeeCalculator)> {
         let (blockhash, fee_calculator, _last_valid_slot) = self
-            .get_recent_blockhash_with_commitment(CommitmentConfig::default())?
+            .get_recent_blockhash_with_commitment(self.commitment_config)?
             .value;
         Ok((blockhash, fee_calculator))
     }
@@ -732,10 +858,7 @@ impl RpcClient {
         blockhash: &Hash,
     ) -> ClientResult<Option<FeeCalculator>> {
         Ok(self
-            .get_fee_calculator_for_blockhash_with_commitment(
-                blockhash,
-                CommitmentConfig::default(),
-            )?
+            .get_fee_calculator_for_blockhash_with_commitment(blockhash, self.commitment_config)?
             .value)
     }
 
@@ -811,7 +934,7 @@ impl RpcClient {
 
     pub fn get_token_account(&self, pubkey: &Pubkey) -> ClientResult<Option<UiTokenAccount>> {
         Ok(self
-            .get_token_account_with_commitment(pubkey, CommitmentConfig::default())?
+            .get_token_account_with_commitment(pubkey, self.commitment_config)?
             .value)
     }
 
@@ -872,7 +995,7 @@ impl RpcClient {
 
     pub fn get_token_account_balance(&self, pubkey: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
-            .get_token_account_balance_with_commitment(pubkey, CommitmentConfig::default())?
+            .get_token_account_balance_with_commitment(pubkey, self.commitment_config)?
             .value)
     }
 
@@ -896,7 +1019,7 @@ impl RpcClient {
             .get_token_accounts_by_delegate_with_commitment(
                 delegate,
                 token_account_filter,
-                CommitmentConfig::default(),
+                self.commitment_config,
             )?
             .value)
     }
@@ -935,7 +1058,7 @@ impl RpcClient {
             .get_token_accounts_by_owner_with_commitment(
                 owner,
                 token_account_filter,
-                CommitmentConfig::default(),
+                self.commitment_config,
             )?
             .value)
     }
@@ -967,7 +1090,7 @@ impl RpcClient {
 
     pub fn get_token_supply(&self, mint: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
-            .get_token_supply_with_commitment(mint, CommitmentConfig::default())?
+            .get_token_supply_with_commitment(mint, self.commitment_config)?
             .value)
     }
 
@@ -1048,7 +1171,7 @@ impl RpcClient {
 
     /// Poll the server to confirm a transaction.
     pub fn poll_for_signature(&self, signature: &Signature) -> ClientResult<()> {
-        self.poll_for_signature_with_commitment(signature, CommitmentConfig::default())
+        self.poll_for_signature_with_commitment(signature, self.commitment_config)
     }
 
     /// Poll the server to confirm a transaction.
@@ -1156,10 +1279,9 @@ impl RpcClient {
         &self,
         transaction: &Transaction,
     ) -> ClientResult<Signature> {
-        self.send_and_confirm_transaction_with_spinner_and_config(
+        self.send_and_confirm_transaction_with_spinner_and_commitment(
             transaction,
-            CommitmentConfig::default(),
-            RpcSendTransactionConfig::default(),
+            self.commitment_config,
         )
     }
 
@@ -1171,7 +1293,10 @@ impl RpcClient {
         self.send_and_confirm_transaction_with_spinner_and_config(
             transaction,
             commitment,
-            RpcSendTransactionConfig::default(),
+            RpcSendTransactionConfig {
+                preflight_commitment: Some(commitment.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
         )
     }
 
@@ -1312,7 +1437,7 @@ fn new_spinner_progress_bar() -> ProgressBar {
     progress_bar
 }
 
-pub fn get_rpc_request_str(rpc_addr: SocketAddr, tls: bool) -> String {
+fn get_rpc_request_str(rpc_addr: SocketAddr, tls: bool) -> String {
     if tls {
         format!("https://{}", rpc_addr)
     } else {
@@ -1418,7 +1543,7 @@ mod tests {
         let rpc_client = RpcClient::new_mock("succeeds".to_string());
 
         let key = Keypair::new();
-        let to = Pubkey::new_rand();
+        let to = solana_sdk::pubkey::new_rand();
         let blockhash = Hash::default();
         let tx = system_transaction::transfer(&key, &to, 50, blockhash);
 
@@ -1471,7 +1596,7 @@ mod tests {
         let rpc_client = RpcClient::new_mock("succeeds".to_string());
 
         let key = Keypair::new();
-        let to = Pubkey::new_rand();
+        let to = solana_sdk::pubkey::new_rand();
         let blockhash = Hash::default();
         let tx = system_transaction::transfer(&key, &to, 50, blockhash);
         let result = rpc_client.send_and_confirm_transaction(&tx);

client/src/rpc_config.rs

@@ -71,6 +71,21 @@ pub struct RpcProgramAccountsConfig {
     pub account_config: RpcAccountInfoConfig,
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum RpcTransactionLogsFilter {
+    All,
+    AllWithVotes,
+    Mentions(Vec<String>), // base58-encoded list of addresses
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcTransactionLogsConfig {
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum RpcTokenAccountsFilter {

client/src/rpc_custom_error.rs

@@ -1,13 +1,16 @@
+//! Implementation defined RPC server errors
+
+use crate::rpc_response::RpcSimulateTransactionResult;
 use jsonrpc_core::{Error, ErrorCode};
-use solana_client::rpc_response::RpcSimulateTransactionResult;
 use solana_sdk::clock::Slot;
 
-const JSON_RPC_SERVER_ERROR_1: i64 = -32001;
-const JSON_RPC_SERVER_ERROR_2: i64 = -32002;
-const JSON_RPC_SERVER_ERROR_3: i64 = -32003;
-const JSON_RPC_SERVER_ERROR_4: i64 = -32004;
-const JSON_RPC_SERVER_ERROR_5: i64 = -32005;
-const JSON_RPC_SERVER_ERROR_6: i64 = -32006;
+pub const JSON_RPC_SERVER_ERROR_BLOCK_CLEANED_UP: i64 = -32001;
+pub const JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE: i64 = -32002;
+pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE: i64 = -32003;
+pub const JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE: i64 = -32004;
+pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY: i64 = -32005;
+pub const JSON_RPC_SERVER_ERROR_TRANSACTION_PRECOMPILE_VERIFICATION_FAILURE: i64 = -32006;
+pub const JSON_RPC_SERVER_ERROR_SLOT_SKIPPED: i64 = -32007;
 
 pub enum RpcCustomError {
     BlockCleanedUp {
@@ -24,6 +27,9 @@ pub enum RpcCustomError {
     },
     RpcNodeUnhealthy,
     TransactionPrecompileVerificationFailure(solana_sdk::transaction::TransactionError),
+    SlotSkipped {
+        slot: Slot,
+    },
 }
 
 impl From<RpcCustomError> for Error {
@@ -33,7 +39,7 @@ impl From<RpcCustomError> for Error {
                 slot,
                 first_available_block,
             } => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_1),
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_BLOCK_CLEANED_UP),
                 message: format!(
                     "Block {} cleaned up, does not exist on node. First available block: {}",
                     slot, first_available_block,
@@ -41,30 +47,44 @@ impl From<RpcCustomError> for Error {
                 data: None,
             },
             RpcCustomError::SendTransactionPreflightFailure { message, result } => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_2),
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE,
+                ),
                 message,
                 data: Some(serde_json::json!(result)),
             },
             RpcCustomError::TransactionSignatureVerificationFailure => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_3),
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE,
+                ),
                 message: "Transaction signature verification failure".to_string(),
                 data: None,
             },
             RpcCustomError::BlockNotAvailable { slot } => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_4),
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE),
                 message: format!("Block not available for slot {}", slot),
                 data: None,
             },
             RpcCustomError::RpcNodeUnhealthy => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_5),
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY),
                 message: "RPC node is unhealthy".to_string(),
                 data: None,
             },
             RpcCustomError::TransactionPrecompileVerificationFailure(e) => Self {
-                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_6),
+                code: ErrorCode::ServerError(
+                    JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE,
+                ),
                 message: format!("Transaction precompile verification failure {:?}", e),
                 data: None,
             },
+            RpcCustomError::SlotSkipped { slot } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_SLOT_SKIPPED),
+                message: format!(
+                    "Slot {} was skipped, or missing due to ledger jump to recent snapshot",
+                    slot
+                ),
+                data: None,
+            },
         }
     }
 }

client/src/rpc_request.rs

@@ -1,3 +1,4 @@
+use crate::rpc_response::RpcSimulateTransactionResult;
 use serde_json::{json, Value};
 use solana_sdk::pubkey::Pubkey;
 use std::fmt;
@@ -138,10 +139,42 @@ impl RpcRequest {
     }
 }
 
+#[derive(Debug)]
+pub enum RpcResponseErrorData {
+    Empty,
+    SendTransactionPreflightFailure(RpcSimulateTransactionResult),
+}
+
+impl fmt::Display for RpcResponseErrorData {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            RpcResponseErrorData::SendTransactionPreflightFailure(
+                RpcSimulateTransactionResult {
+                    logs: Some(logs), ..
+                },
+            ) => {
+                if logs.is_empty() {
+                    Ok(())
+                } else {
+                    // Give the user a hint that there is more useful logging information available...
+                    write!(f, "[{} log messages]", logs.len())
+                }
+            }
+            _ => Ok(()),
+        }
+    }
+}
+
 #[derive(Debug, Error)]
 pub enum RpcError {
-    #[error("rpc request error: {0}")]
+    #[error("RPC request error: {0}")]
     RpcRequestError(String),
+    #[error("RPC response error {code}: {message} {data}")]
+    RpcResponseError {
+        code: i64,
+        message: String,
+        data: RpcResponseErrorData,
+    },
     #[error("parse error: expected {0}")]
     ParseError(String), /* "expected" */
     // Anything in a `ForUser` needs to die.  The caller should be
@@ -226,7 +259,7 @@ mod tests {
 
         // Test request with CommitmentConfig and params
         let test_request = RpcRequest::GetTokenAccountsByOwner;
-        let mint = Pubkey::new_rand();
+        let mint = solana_sdk::pubkey::new_rand();
         let token_account_filter = RpcTokenAccountsFilter::Mint(mint.to_string());
         let request = test_request
             .build_request_json(1, json!([addr, token_account_filter, commitment_config]));

client/src/rpc_response.rs

@@ -108,6 +108,14 @@ pub enum RpcSignatureResult {
     ReceivedSignature(ReceivedSignatureResult),
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcLogsResponse {
+    pub signature: String, // Signature as base58 string
+    pub err: Option<TransactionError>,
+    pub logs: Vec<String>,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct ProcessedSignatureResult {

client/src/thin_client.rs

@@ -3,7 +3,7 @@
 //! messages to the network directly. The binary encoding of its messages are
 //! unstable and may change in future releases.
 
-use crate::{rpc_client::RpcClient, rpc_response::Response};
+use crate::{rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response};
 use bincode::{serialize_into, serialized_size};
 use log::*;
 use solana_sdk::{
@@ -276,6 +276,16 @@ impl ThinClient {
         )
     }
 
+    pub fn get_program_accounts_with_config(
+        &self,
+        pubkey: &Pubkey,
+        config: RpcProgramAccountsConfig,
+    ) -> TransportResult<Vec<(Pubkey, Account)>> {
+        self.rpc_client()
+            .get_program_accounts_with_config(pubkey, config)
+            .map_err(|e| e.into())
+    }
+
     pub fn wait_for_balance_with_commitment(
         &self,
         pubkey: &Pubkey,
@@ -389,6 +399,12 @@ impl SyncClient for ThinClient {
             .map(|r| r.value)
     }
 
+    fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> TransportResult<u64> {
+        self.rpc_client()
+            .get_minimum_balance_for_rent_exemption(data_len)
+            .map_err(|e| e.into())
+    }
+
     fn get_recent_blockhash(&self) -> TransportResult<(Hash, FeeCalculator)> {
         let (blockhash, fee_calculator, _last_valid_slot) =
             self.get_recent_blockhash_with_commitment(CommitmentConfig::default())?;

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.0"
+version = "1.5.3"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -14,6 +14,7 @@ edition = "2018"
 codecov = { repository = "solana-labs/solana", branch = "master", service = "github" }
 
 [dependencies]
+ahash = "0.6.1"
 base64 = "0.12.3"
 bincode = "1.3.1"
 bv = { version = "0.11.1", features = ["serde"] }
@@ -33,54 +34,61 @@ jsonrpc-derive = "15.0.0"
 jsonrpc-http-server = "15.0.0"
 jsonrpc-pubsub = "15.0.0"
 jsonrpc-ws-server = "15.0.0"
-log = "0.4.8"
-num_cpus = "1.13.0"
+log = "0.4.11"
+lru = "0.6.1"
+miow = "0.2.2"
+net2 = "0.2.37"
 num-traits = "0.2"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 raptorq = "1.4.2"
-rayon = "1.4.0"
+rayon = "1.4.1"
 regex = "1.3.9"
+rustversion = "1.0.4"
 serde = "1.0.112"
+serde_bytes = "0.11"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.0" }
-solana-banks-server = { path = "../banks-server", version = "1.4.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-client = { path = "../client", version = "1.4.0" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
-solana-ledger = { path = "../ledger", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.4.0" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
-solana-measure = { path = "../measure", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-perf = { path = "../perf", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-sdk-macro-frozen-abi = { path = "../sdk/macro-frozen-abi", version = "1.4.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.0" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.4.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
-solana-vote-signer = { path = "../vote-signer", version = "1.4.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=2.0.6", features = ["skip-no-mangle"] }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.3" }
+solana-banks-server = { path = "../banks-server", version = "1.5.3" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.3" }
+solana-client = { path = "../client", version = "1.5.3" }
+solana-faucet = { path = "../faucet", version = "1.5.3" }
+solana-ledger = { path = "../ledger", version = "1.5.3" }
+solana-logger = { path = "../logger", version = "1.5.3" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.5.3" }
+solana-metrics = { path = "../metrics", version = "1.5.3" }
+solana-measure = { path = "../measure", version = "1.5.3" }
+solana-net-utils = { path = "../net-utils", version = "1.5.3" }
+solana-perf = { path = "../perf", version = "1.5.3" }
+solana-program-test = { path = "../program-test", version = "1.5.3" }
+solana-runtime = { path = "../runtime", version = "1.5.3" }
+solana-sdk = { path = "../sdk", version = "1.5.3" }
+solana-frozen-abi = { path = "../frozen-abi", version = "1.5.3" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.3" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.3" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.3" }
+solana-streamer = { path = "../streamer", version = "1.5.3" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.5.3" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.3" }
+solana-version = { path = "../version", version = "1.5.3" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.3" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.2", features = ["full"] }
 tokio_01 = { version = "0.1", package = "tokio" }
+tokio_01_bytes = { version = "0.4.7", package = "bytes" }
 tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
 tokio_io_01 = { version = "0.1", package = "tokio-io" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.0" }
+tokio_codec_01 = { version = "0.1", package = "tokio-codec" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.3" }
 trees = "0.2.1"
 
 [dev-dependencies]
 matches = "0.1.6"
-reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+num_cpus = "1.13.0"
+reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
 systemstat = "0.1.5"
@@ -94,6 +102,9 @@ name = "banking_stage"
 [[bench]]
 name = "blockstore"
 
+[[bench]]
+name = "crds"
+
 [[bench]]
 name = "crds_gossip_pull"
 

core/benches/banking_stage.rs

@@ -20,7 +20,7 @@ use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;
 use solana_sdk::hash::Hash;
 use solana_sdk::message::Message;
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::pubkey;
 use solana_sdk::signature::Keypair;
 use solana_sdk::signature::Signature;
 use solana_sdk::signature::Signer;
@@ -56,7 +56,7 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
     let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
     let bank = Arc::new(Bank::new(&genesis_config));
     let ledger_path = get_tmp_ledger_path!();
-    let my_pubkey = Pubkey::new_rand();
+    let my_pubkey = pubkey::new_rand();
     {
         let blockstore = Arc::new(
             Blockstore::open(&ledger_path).expect("Expected to be able to open database ledger"),
@@ -94,15 +94,15 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
 }
 
 fn make_accounts_txs(txes: usize, mint_keypair: &Keypair, hash: Hash) -> Vec<Transaction> {
-    let to_pubkey = Pubkey::new_rand();
+    let to_pubkey = pubkey::new_rand();
     let dummy = system_transaction::transfer(mint_keypair, &to_pubkey, 1, hash);
     (0..txes)
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
             let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
-            new.message.account_keys[0] = Pubkey::new_rand();
-            new.message.account_keys[1] = Pubkey::new_rand();
+            new.message.account_keys[0] = pubkey::new_rand();
+            new.message.account_keys[1] = pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
         })
@@ -117,7 +117,7 @@ fn make_programs_txs(txes: usize, hash: Hash) -> Vec<Transaction> {
             let mut instructions = vec![];
             let from_key = Keypair::new();
             for _ in 1..progs {
-                let to_key = Pubkey::new_rand();
+                let to_key = pubkey::new_rand();
                 instructions.push(system_instruction::transfer(&from_key.pubkey(), &to_key, 1));
             }
             let message = Message::new(&instructions, Some(&from_key.pubkey()));

core/benches/cluster_info.rs

@@ -8,7 +8,7 @@ use solana_core::broadcast_stage::{broadcast_shreds, get_broadcast_peers};
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
 use solana_ledger::shred::Shred;
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::pubkey;
 use solana_sdk::timing::timestamp;
 use std::{
     collections::HashMap,
@@ -20,7 +20,7 @@ use test::Bencher;
 #[bench]
 fn broadcast_shreds_bench(bencher: &mut Bencher) {
     solana_logger::setup();
-    let leader_pubkey = Pubkey::new_rand();
+    let leader_pubkey = pubkey::new_rand();
     let leader_info = Node::new_localhost_with_pubkey(&leader_pubkey);
     let cluster_info = ClusterInfo::new_with_invalid_keypair(leader_info.info);
     let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
@@ -30,7 +30,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
     let mut stakes = HashMap::new();
     const NUM_PEERS: usize = 200;
     for _ in 0..NUM_PEERS {
-        let id = Pubkey::new_rand();
+        let id = pubkey::new_rand();
         let contact_info = ContactInfo::new_localhost(&id, timestamp());
         cluster_info.insert_info(contact_info);
         stakes.insert(id, thread_rng().gen_range(1, NUM_PEERS) as u64);

core/benches/crds.rs

@@ -0,0 +1,31 @@
+#![feature(test)]
+
+extern crate test;
+
+use rand::{thread_rng, Rng};
+use rayon::ThreadPoolBuilder;
+use solana_core::crds::Crds;
+use solana_core::crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
+use solana_core::crds_value::CrdsValue;
+use solana_sdk::pubkey::Pubkey;
+use std::collections::HashMap;
+use test::Bencher;
+
+#[bench]
+fn bench_find_old_labels(bencher: &mut Bencher) {
+    let thread_pool = ThreadPoolBuilder::new().build().unwrap();
+    let mut rng = thread_rng();
+    let mut crds = Crds::default();
+    let now = CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS + CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS / 1000;
+    std::iter::repeat_with(|| (CrdsValue::new_rand(&mut rng, None), rng.gen_range(0, now)))
+        .take(50_000)
+        .for_each(|(v, ts)| assert!(crds.insert(v, ts).is_ok()));
+    let mut timeouts = HashMap::new();
+    timeouts.insert(Pubkey::default(), CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS);
+    bencher.iter(|| {
+        let out = crds.find_old_labels(&thread_pool, now, &timeouts);
+        assert!(out.len() > 10);
+        assert!(out.len() < 250);
+        out
+    });
+}