More guard rails for restart with ledger procedure (#10854)

* Add expected_bank_hash required for supermajority
* Print snapshot hash in ledger-tool create-snapshot.

.gitbook.yaml

@@ -15,4 +15,4 @@ redirects:
     wallet/cli-wallets/remote-wallet: ./hardware-wallets/README.md
     wallet/cli-wallets/remote-wallet/ledger: ./hardware-wallets/ledger.md
     wallet/cli-wallets/file-system-wallet: ./file-system-wallet/README.md
-    wallet/support: ./wallet-guide/support.md
+    wallet/support: ./wallet-guide/support.md

.mergify.yml

@@ -1,9 +1,40 @@
 # Validate your changes with:
 #
-#   $ curl -F 'data=@.mergify.yml' https://gh.mergify.io/validate
+#   $ curl -F 'data=@.mergify.yml' https://gh.mergify.io/validate/
 #
 # https://doc.mergify.io/
 pull_request_rules:
+  - name: automatic merge (squash) on CI success
+    conditions:
+      - status-success=buildkite/solana
+      #- status-success=Travis CI - Pull Request
+      - status-success=ci-gate
+      - label=automerge
+      - author≠@dont-squash-my-commits
+    actions:
+      merge:
+        method: squash
+  # Join the dont-squash-my-commits group if you won't like your commits squashed
+  - name: automatic merge (rebase) on CI success
+    conditions:
+      - status-success=buildkite/solana
+      #- status-success=Travis CI - Pull Request
+      - status-success=ci-gate
+      - label=automerge
+      - author=@dont-squash-my-commits
+    actions:
+      merge:
+        method: rebase
+  - name: remove automerge label on CI failure
+    conditions:
+      - label=automerge
+      - "#status-failure!=0"
+    actions:
+      label:
+        remove:
+          - automerge
+      comment:
+        message: automerge label removed due to a CI failure
   - name: remove outdated reviews
     conditions:
       - base=master

.travis.yml

@@ -13,11 +13,14 @@ script:
   - source ci/env.sh
   - ci/publish-tarball.sh
 
+
 branches:
   only:
     - master
     - /^v\d+\.\d+/
 
+if: type IN (api, cron) OR tag IS present
+
 notifications:
   slack:
     on_success: change

RELEASE.md

@@ -116,7 +116,8 @@ There are three release channels that map to branches as follows:
 
 1. After the new release has been tagged, update the Cargo.toml files on **release branch** to the next semantic version (e.g. 0.9.0 -> 0.9.1) with:
      ```
-     scripts/increment-cargo-version.sh patch
+     $ scripts/increment-cargo-version.sh patch
+     $ ./scripts/cargo-for-all-lock-files.sh tree
      ```
 1. Rebuild to get an updated version of `Cargo.lock`:
     ```

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.2.0"
+version = "1.2.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,10 +10,10 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.3.0"
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
+solana-logger = { path = "../logger", version = "1.2.8" }
+solana-runtime = { path = "../runtime", version = "1.2.8" }
+solana-measure = { path = "../measure", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.2.0"
+version = "1.2.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,16 +13,16 @@ crossbeam-channel = "0.4"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.3.0"
-solana-core = { path = "../core", version = "1.2.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-streamer = { path = "../streamer", version = "1.2.0" }
-solana-perf = { path = "../perf", version = "1.2.0" }
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-core = { path = "../core", version = "1.2.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.8" }
+solana-streamer = { path = "../streamer", version = "1.2.8" }
+solana-perf = { path = "../perf", version = "1.2.8" }
+solana-ledger = { path = "../ledger", version = "1.2.8" }
+solana-logger = { path = "../logger", version = "1.2.8" }
+solana-runtime = { path = "../runtime", version = "1.2.8" }
+solana-measure = { path = "../measure", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
+solana-version = { path = "../version", version = "1.2.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -169,7 +169,7 @@ fn main() {
     let (verified_sender, verified_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
     let bank0 = Bank::new(&genesis_config);
-    let mut bank_forks = BankForks::new(0, bank0);
+    let mut bank_forks = BankForks::new(bank0);
     let mut bank = bank_forks.working_bank();
 
     info!("threads: {} txs: {}", num_threads, total_num_transactions);

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.2.0"
+version = "1.2.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.3.0"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-core = { path = "../core", version = "1.2.0" }
-solana-genesis = { path = "../genesis", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.8" }
+solana-core = { path = "../core", version = "1.2.8" }
+solana-genesis = { path = "../genesis", version = "1.2.8" }
+solana-client = { path = "../client", version = "1.2.8" }
+solana-faucet = { path = "../faucet", version = "1.2.8" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.2.8" }
+solana-logger = { path = "../logger", version = "1.2.8" }
+solana-metrics = { path = "../metrics", version = "1.2.8" }
+solana-net-utils = { path = "../net-utils", version = "1.2.8" }
+solana-runtime = { path = "../runtime", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
+solana-version = { path = "../version", version = "1.2.8" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.2.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.2.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/src/bench.rs

@@ -14,6 +14,7 @@ use solana_metrics::datapoint_info;
 use solana_sdk::{
     client::{Client, SyncClient},
     commitment_config::CommitmentConfig,
+    message::Message,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     timing::{duration_as_ms, duration_as_s},
@@ -449,7 +450,7 @@ fn swapper<T>(
             }
             account_group = (account_group + 1) % account_groups as usize;
 
-            let (blockhash, _fee_calculator) = client
+            let (blockhash, _fee_calculator, _last_valid_slot) = client
                 .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                 .expect("Failed to get blockhash");
             let to_swap_txs: Vec<_> = to_swap
@@ -457,16 +458,14 @@ fn swapper<T>(
                 .map(|(signer, swap, profit)| {
                     let s: &Keypair = &signer;
                     let owner = &signer.pubkey();
-                    Transaction::new_signed_instructions(
-                        &[s],
-                        &[exchange_instruction::swap_request(
-                            owner,
-                            &swap.0.pubkey,
-                            &swap.1.pubkey,
-                            &profit,
-                        )],
-                        blockhash,
-                    )
+                    let instruction = exchange_instruction::swap_request(
+                        owner,
+                        &swap.0.pubkey,
+                        &swap.1.pubkey,
+                        &profit,
+                    );
+                    let message = Message::new(&[instruction], Some(&s.pubkey()));
+                    Transaction::new(&[s], message, blockhash)
                 })
                 .collect();
 
@@ -577,7 +576,7 @@ fn trader<T>(
         }
         account_group = (account_group + 1) % account_groups as usize;
 
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .expect("Failed to get blockhash");
 
@@ -588,28 +587,26 @@ fn trader<T>(
                     let owner_pubkey = &owner.pubkey();
                     let trade_pubkey = &trade.pubkey();
                     let space = mem::size_of::<ExchangeState>() as u64;
-                    Transaction::new_signed_instructions(
-                        &[owner.as_ref(), trade],
-                        &[
-                            system_instruction::create_account(
-                                owner_pubkey,
-                                trade_pubkey,
-                                1,
-                                space,
-                                &id(),
-                            ),
-                            exchange_instruction::trade_request(
-                                owner_pubkey,
-                                trade_pubkey,
-                                *side,
-                                pair,
-                                tokens,
-                                price,
-                                src,
-                            ),
-                        ],
-                        blockhash,
-                    )
+                    let instructions = [
+                        system_instruction::create_account(
+                            owner_pubkey,
+                            trade_pubkey,
+                            1,
+                            space,
+                            &id(),
+                        ),
+                        exchange_instruction::trade_request(
+                            owner_pubkey,
+                            trade_pubkey,
+                            *side,
+                            pair,
+                            tokens,
+                            price,
+                            src,
+                        ),
+                    ];
+                    let message = Message::new(&instructions, Some(&owner_pubkey));
+                    Transaction::new(&[owner.as_ref(), trade], message, blockhash)
                 })
                 .collect();
 
@@ -747,13 +744,9 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
             let mut to_fund_txs: Vec<_> = chunk
                 .par_iter()
                 .map(|(k, m)| {
-                    (
-                        k.clone(),
-                        Transaction::new_unsigned_instructions(&system_instruction::transfer_many(
-                            &k.pubkey(),
-                            &m,
-                        )),
-                    )
+                    let instructions = system_instruction::transfer_many(&k.pubkey(), &m);
+                    let message = Message::new(&instructions, Some(&k.pubkey()));
+                    (k.clone(), Transaction::new_unsigned(message))
                 })
                 .collect();
 
@@ -776,7 +769,7 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
                     to_fund_txs.len(),
                 );
 
-                let (blockhash, _fee_calculator) = client
+                let (blockhash, _fee_calculator, _last_valid_slot) = client
                     .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                     .expect("blockhash");
                 to_fund_txs.par_iter_mut().for_each(|(k, tx)| {
@@ -848,9 +841,10 @@ pub fn create_token_accounts<T: Client>(
                     );
                     let request_ix =
                         exchange_instruction::account_request(owner_pubkey, &new_keypair.pubkey());
+                    let message = Message::new(&[create_ix, request_ix], Some(&owner_pubkey));
                     (
                         (from_keypair, new_keypair),
-                        Transaction::new_unsigned_instructions(&[create_ix, request_ix]),
+                        Transaction::new_unsigned(message),
                     )
                 })
                 .collect();
@@ -868,7 +862,7 @@ pub fn create_token_accounts<T: Client>(
 
             let mut retries = 0;
             while !to_create_txs.is_empty() {
-                let (blockhash, _fee_calculator) = client
+                let (blockhash, _fee_calculator, _last_valid_slot) = client
                     .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                     .expect("Failed to get blockhash");
                 to_create_txs
@@ -997,7 +991,7 @@ pub fn airdrop_lamports<T: Client>(
 
     let mut tries = 0;
     loop {
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .expect("Failed to get blockhash");
         match request_airdrop_transaction(&faucet_addr, &id.pubkey(), amount_to_drop, blockhash) {

bench-streamer/Cargo.toml

@@ -2,18 +2,18 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.2.0"
+version = "1.2.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-streamer = { path = "../streamer", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.8" }
+solana-streamer = { path = "../streamer", version = "1.2.8" }
+solana-logger = { path = "../logger", version = "1.2.8" }
+solana-net-utils = { path = "../net-utils", version = "1.2.8" }
+solana-version = { path = "../version", version = "1.2.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.2.0"
+version = "1.2.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,25 +14,25 @@ log = "0.4.8"
 rayon = "1.3.0"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-core = { path = "../core", version = "1.2.0" }
-solana-genesis = { path = "../genesis", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-solana-librapay = { path = "../programs/librapay", version = "1.2.0", optional = true }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.2.0", optional = true }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.8" }
+solana-core = { path = "../core", version = "1.2.8" }
+solana-genesis = { path = "../genesis", version = "1.2.8" }
+solana-client = { path = "../client", version = "1.2.8" }
+solana-faucet = { path = "../faucet", version = "1.2.8" }
+solana-librapay = { path = "../programs/librapay", version = "1.2.8", optional = true }
+solana-logger = { path = "../logger", version = "1.2.8" }
+solana-metrics = { path = "../metrics", version = "1.2.8" }
+solana-measure = { path = "../measure", version = "1.2.8" }
+solana-net-utils = { path = "../net-utils", version = "1.2.8" }
+solana-runtime = { path = "../runtime", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
+solana-move-loader-program = { path = "../programs/move_loader", version = "1.2.8", optional = true }
+solana-version = { path = "../version", version = "1.2.8" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.2.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.2.8" }
 
 [features]
 move = ["solana-librapay", "solana-move-loader-program"]

bench-tps/src/bench.rs

@@ -14,6 +14,7 @@ use solana_sdk::{
     commitment_config::CommitmentConfig,
     fee_calculator::FeeCalculator,
     hash::Hash,
+    message::Message,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     system_instruction, system_transaction,
@@ -55,7 +56,9 @@ type LibraKeys = (Keypair, Pubkey, Pubkey, Vec<Keypair>);
 fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
     loop {
         match client.get_recent_blockhash_with_commitment(CommitmentConfig::recent()) {
-            Ok((blockhash, fee_calculator)) => return (blockhash, fee_calculator),
+            Ok((blockhash, fee_calculator, _last_valid_slot)) => {
+                return (blockhash, fee_calculator)
+            }
             Err(err) => {
                 info!("Couldn't get recent blockhash: {:?}", err);
                 sleep(Duration::from_secs(1));
@@ -650,10 +653,9 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
         let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund
             .par_iter()
             .map(|(k, t)| {
-                let tx = Transaction::new_unsigned_instructions(
-                    &system_instruction::transfer_many(&k.pubkey(), &t),
-                );
-                (*k, tx)
+                let instructions = system_instruction::transfer_many(&k.pubkey(), &t);
+                let message = Message::new(&instructions, Some(&k.pubkey()));
+                (*k, Transaction::new_unsigned(message))
             })
             .collect();
         make_txs.stop();
@@ -968,7 +970,7 @@ fn fund_move_keys<T: Client>(
     let libra_funding_key = Keypair::new();
     let tx = librapay_transaction::create_account(funding_key, &libra_funding_key, 1, blockhash);
     client
-        .send_message(&[funding_key, &libra_funding_key], tx.message)
+        .send_and_confirm_message(&[funding_key, &libra_funding_key], tx.message)
         .unwrap();
 
     info!("minting to funding keypair");
@@ -981,7 +983,7 @@ fn fund_move_keys<T: Client>(
         blockhash,
     );
     client
-        .send_message(&[funding_key, libra_genesis_key], tx.message)
+        .send_and_confirm_message(&[funding_key, libra_genesis_key], tx.message)
         .unwrap();
 
     info!("creating {} move accounts...", keypairs.len());
@@ -1003,7 +1005,7 @@ fn fund_move_keys<T: Client>(
         let ser_size = bincode::serialized_size(&tx).unwrap();
         let mut keys = vec![funding_key];
         keys.extend(&keypairs);
-        client.send_message(&keys, tx.message).unwrap();
+        client.send_and_confirm_message(&keys, tx.message).unwrap();
 
         if i % 10 == 0 {
             info!(
@@ -1021,12 +1023,12 @@ fn fund_move_keys<T: Client>(
         .iter()
         .map(|key| (key.pubkey(), total / NUM_FUNDING_KEYS as u64))
         .collect();
-    let tx = Transaction::new_signed_instructions(
-        &[funding_key],
-        &system_instruction::transfer_many(&funding_key.pubkey(), &pubkey_amounts),
-        blockhash,
-    );
-    client.send_message(&[funding_key], tx.message).unwrap();
+    let instructions = system_instruction::transfer_many(&funding_key.pubkey(), &pubkey_amounts);
+    let message = Message::new(&instructions, Some(&funding_key.pubkey()));
+    let tx = Transaction::new(&[funding_key], message, blockhash);
+    client
+        .send_and_confirm_message(&[funding_key], tx.message)
+        .unwrap();
     let mut balance = 0;
     for _ in 0..20 {
         if let Ok(balance_) = client
@@ -1049,7 +1051,7 @@ fn fund_move_keys<T: Client>(
     for (i, key) in libra_funding_keys.iter().enumerate() {
         let tx = librapay_transaction::create_account(&funding_keys[i], &key, 1, blockhash);
         client
-            .send_message(&[&funding_keys[i], &key], tx.message)
+            .send_and_confirm_message(&[&funding_keys[i], &key], tx.message)
             .unwrap();
 
         let tx = librapay_transaction::transfer(
@@ -1062,7 +1064,7 @@ fn fund_move_keys<T: Client>(
             blockhash,
         );
         client
-            .send_message(&[&funding_keys[i], &libra_funding_key], tx.message)
+            .send_and_confirm_message(&[&funding_keys[i], &libra_funding_key], tx.message)
             .unwrap();
 
         info!("funded libra funding key {}", i);

ci/buildkite-pipeline.sh

@@ -0,0 +1,259 @@
+#!/usr/bin/env bash
+#
+# Builds a buildkite pipeline based on the environment variables
+#
+
+set -e
+cd "$(dirname "$0")"/..
+
+output_file=${1:-/dev/stderr}
+
+if [[ -n $CI_PULL_REQUEST ]]; then
+  IFS=':' read -ra affected_files <<< "$(buildkite-agent meta-data get affected_files)"
+  if [[ ${#affected_files[*]} -eq 0 ]]; then
+    echo "Unable to determine the files affected by this PR"
+    exit 1
+  fi
+else
+  affected_files=()
+fi
+
+annotate() {
+  if [[ -n $BUILDKITE ]]; then
+    buildkite-agent annotate "$@"
+  fi
+}
+
+# Checks if a CI pull request affects one or more path patterns.  Each
+# pattern argument is checked in series. If one of them found to be affected,
+# return immediately as such.
+#
+# Bash regular expressions are permitted in the pattern:
+#     affects .rs$    -- any file or directory ending in .rs
+#     affects .rs     -- also matches foo.rs.bar
+#     affects ^snap/  -- anything under the snap/ subdirectory
+#     affects snap/   -- also matches foo/snap/
+# Any pattern starting with the ! character will be negated:
+#     affects !^docs/  -- anything *not* under the docs/ subdirectory
+#
+affects() {
+  if [[ -z $CI_PULL_REQUEST ]]; then
+    # affected_files metadata is not currently available for non-PR builds so assume
+    # the worse (affected)
+    return 0
+  fi
+  # Assume everyting needs to be tested when any Dockerfile changes
+  for pattern in ^ci/docker-rust/Dockerfile ^ci/docker-rust-nightly/Dockerfile "$@"; do
+    if [[ ${pattern:0:1} = "!" ]]; then
+      for file in "${affected_files[@]}"; do
+        if [[ ! $file =~ ${pattern:1} ]]; then
+          return 0 # affected
+        fi
+      done
+    else
+      for file in "${affected_files[@]}"; do
+        if [[ $file =~ $pattern ]]; then
+          return 0 # affected
+        fi
+      done
+    fi
+  done
+
+  return 1 # not affected
+}
+
+
+# Checks if a CI pull request affects anything other than the provided path patterns
+#
+# Syntax is the same as `affects()` except that the negation prefix is not
+# supported
+#
+affects_other_than() {
+  if [[ -z $CI_PULL_REQUEST ]]; then
+    # affected_files metadata is not currently available for non-PR builds so assume
+    # the worse (affected)
+    return 0
+  fi
+
+  for file in "${affected_files[@]}"; do
+    declare matched=false
+    for pattern in "$@"; do
+        if [[ $file =~ $pattern ]]; then
+          matched=true
+        fi
+    done
+    if ! $matched; then
+      return 0 # affected
+    fi
+  done
+
+  return 1 # not affected
+}
+
+
+start_pipeline() {
+  echo "# $*" > "$output_file"
+  echo "steps:" >> "$output_file"
+}
+
+command_step() {
+  cat >> "$output_file" <<EOF
+  - name: "$1"
+    command: "$2"
+    timeout_in_minutes: $3
+    artifact_paths: "log-*.txt"
+EOF
+}
+
+
+trigger_secondary_step() {
+  cat  >> "$output_file" <<"EOF"
+  - trigger: "solana-secondary"
+    branches: "!pull/*"
+    async: true
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"
+EOF
+}
+
+wait_step() {
+  echo "  - wait" >> "$output_file"
+}
+
+all_test_steps() {
+  command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
+  wait_step
+
+  # Coverage...
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-coverage.sh \
+             ^scripts/coverage.sh \
+      ; then
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 30
+    wait_step
+  else
+    annotate --style info --context test-coverage \
+      "Coverage skipped as no .rs files were modified"
+  fi
+
+  # Full test suite
+  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
+  wait_step
+
+  # Perf test suite
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf"
+    timeout_in_minutes: 40
+    artifact_paths: "log-*.txt"
+    agents:
+      - "queue=cuda"
+EOF
+  else
+    annotate --style info \
+      "Stable-perf skipped as no relevant files were modified"
+  fi
+
+  # Benches...
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-coverage.sh \
+             ^ci/test-bench.sh \
+      ; then
+    command_step bench "ci/test-bench.sh" 30
+  else
+    annotate --style info --context test-bench \
+      "Bench skipped as no .rs files were modified"
+  fi
+
+  command_step "local-cluster" \
+    ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster.sh" \
+    45
+}
+
+pull_or_push_steps() {
+  command_step sanity "ci/test-sanity.sh" 5
+  wait_step
+
+  # Check for any .sh file changes
+  if affects .sh$; then
+    command_step shellcheck "ci/shellcheck.sh" 5
+    wait_step
+  fi
+
+  # Run the full test suite by default, skipping only if modifications are local
+  # to some particular areas of the tree
+  if affects_other_than ^.buildkite ^.travis .md$ ^docs/ ^web3.js/ ^explorer/ ^.gitbook; then
+    all_test_steps
+  fi
+
+  # doc/ changes:
+  if affects ^docs/; then
+    command_step docs ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image docs/build.sh" 5
+  fi
+
+  # web3.js and explorer changes run on Travis...
+}
+
+
+if [[ -n $BUILDKITE_TAG ]]; then
+  start_pipeline "Tag pipeline for $BUILDKITE_TAG"
+
+  annotate --style info --context release-tag \
+    "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
+
+  # Jump directly to the secondary build to publish release artifacts quickly
+  trigger_secondary_step
+  exit 0
+fi
+
+
+if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
+  echo "+++ Affected files in this PR"
+  for file in "${affected_files[@]}"; do
+    echo "- $file"
+  done
+
+  start_pipeline "Pull request pipeline for $BUILDKITE_BRANCH"
+
+  # Add helpful link back to the corresponding Github Pull Request
+  annotate --style info --context pr-backlink \
+    "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
+
+  if [[ $GITHUB_USER = "dependabot-preview[bot]" ]]; then
+    command_step dependabot "ci/dependabot-pr.sh" 5
+    wait_step
+  fi
+  pull_or_push_steps
+  exit 0
+fi
+
+start_pipeline "Push pipeline for ${BUILDKITE_BRANCH:-?unknown branch?}"
+pull_or_push_steps
+wait_step
+trigger_secondary_step
+exit 0

ci/buildkite.yml

@@ -5,6 +5,9 @@
 # Release tags use buildkite-release.yml instead
 
 steps:
+  - command: "ci/test-sanity.sh"
+    name: "sanity"
+    timeout_in_minutes: 5
   - command: "ci/dependabot-pr.sh"
     name: "dependabot"
     timeout_in_minutes: 5

ci/docker-run.sh

@@ -67,6 +67,7 @@ ARGS+=(
   --env BUILDKITE_JOB_ID
   --env CI
   --env CI_BRANCH
+  --env CI_BASE_BRANCH
   --env CI_TAG
   --env CI_BUILD_ID
   --env CI_COMMIT

ci/env.sh

@@ -8,6 +8,7 @@ if [[ -n $CI ]]; then
   export CI=1
   if [[ -n $TRAVIS ]]; then
     export CI_BRANCH=$TRAVIS_BRANCH
+    export CI_BASE_BRANCH=$TRAVIS_BRANCH
     export CI_BUILD_ID=$TRAVIS_BUILD_ID
     export CI_COMMIT=$TRAVIS_COMMIT
     export CI_JOB_ID=$TRAVIS_JOB_ID
@@ -28,8 +29,10 @@ if [[ -n $CI ]]; then
     # to how solana-ci-gate is used to trigger PR builds rather than using the
     # standard Buildkite PR trigger.
     if [[ $CI_BRANCH =~ pull/* ]]; then
+      export CI_BASE_BRANCH=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
       export CI_PULL_REQUEST=true
     else
+      export CI_BASE_BRANCH=$BUILDKITE_BRANCH
       export CI_PULL_REQUEST=
     fi
     export CI_OS_NAME=linux

ci/run-sanity.sh

@@ -19,10 +19,16 @@ while [[ ! -f config/run/init-completed ]]; do
   fi
 done
 
+snapshot_slot=1
+
+# wait a bit longer than snapshot_slot
+while [[ $($solana_cli --url http://localhost:8899 slot --commitment recent) -le $((snapshot_slot + 1)) ]]; do
+  sleep 1
+done
 curl -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1, "method":"validatorExit"}' http://localhost:8899
 
 wait $pid
 
-$solana_ledger_tool create-snapshot --ledger config/ledger 1 config/snapshot-ledger
+$solana_ledger_tool create-snapshot --ledger config/ledger "$snapshot_slot" config/snapshot-ledger
 cp config/ledger/genesis.tar.bz2 config/snapshot-ledger
 $solana_ledger_tool verify --ledger config/snapshot-ledger

ci/semver_bash/README.md

@@ -27,5 +27,5 @@ Alternatively, you can source it from within a script:
     local PATCH=0  
     local SPECIAL=""
     
-    semverParseInto "1.2.3" MAJOR MINOR PATCH SPECIAL  
-    semverParseInto "3.2.1" MAJOR MINOR PATCH SPECIAL  
+    semverParseInto "1.2.8" MAJOR MINOR PATCH SPECIAL
+    semverParseInto "3.2.1" MAJOR MINOR PATCH SPECIAL

ci/test-checks.sh

@@ -6,20 +6,24 @@ cd "$(dirname "$0")/.."
 source ci/_
 source ci/rust-version.sh stable
 source ci/rust-version.sh nightly
+eval "$(ci/channel-info.sh)"
 
 export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 
-# Look for failed mergify.io backports
-_ git show HEAD --check --oneline
 
-if _ scripts/cargo-for-all-lock-files.sh +"$rust_nightly" check --locked --all-targets; then
-  true
+# Only force up-to-date lock files on edge
+if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
+  if _ scripts/cargo-for-all-lock-files.sh +"$rust_nightly" check --locked --all-targets; then
+    true
+  else
+    check_status=$?
+    echo "Some Cargo.lock is outdated; please update them as well"
+    echo "protip: you can use ./scripts/cargo-for-all-lock-files.sh update ..."
+    exit "$check_status"
+  fi
 else
-  check_status=$?
-  echo "Some Cargo.lock is outdated; please update them as well"
-  echo "protip: you can use ./scripts/cargo-for-all-lock-files.sh update ..."
-  exit "$check_status"
+  echo "Note: cargo-for-all-lock-files.sh skipped because $CI_BASE_BRANCH != $EDGE_CHANNEL"
 fi
 
 _ cargo +"$rust_stable" fmt --all -- --check
@@ -29,10 +33,8 @@ _ cargo +"$rust_stable" clippy --workspace -- --deny=warnings
 
 _ cargo +"$rust_stable" audit --version
 _ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
-_ ci/nits.sh
 _ ci/order-crates-for-publishing.py
 _ docs/build.sh
-_ ci/check-ssh-keys.sh
 
 {
   cd programs/bpf

ci/test-sanity.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+source ci/_
+
+(
+  echo --- git diff --check
+  set -x
+  # Look for failed mergify.io backports by searching leftover conflict markers
+  # Also check for any trailing whitespaces!
+  git fetch origin "$CI_BASE_BRANCH"
+  git diff "$(git merge-base HEAD "origin/$CI_BASE_BRANCH")..HEAD" --check --oneline
+)
+
+echo
+
+_ ci/nits.sh
+_ ci/check-ssh-keys.sh
+
+echo --- ok

ci/test-stable.sh

@@ -39,9 +39,9 @@ test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 rm -rf target/xargo # Issue #3105
 
 # Limit compiler jobs to reduce memory usage
-# on machines with 1gb/thread of memory
+# on machines with 2gb/thread of memory
 NPROC=$(nproc)
-NPROC=$((NPROC>16 ? 16 : NPROC))
+NPROC=$((NPROC>14 ? 14 : NPROC))
 
 echo "Executing $testName"
 case $testName in

ci/upload-github-release-asset.sh

@@ -23,10 +23,14 @@ if [[ -z $CI_TAG ]]; then
   exit 1
 fi
 
-if [[ -z $CI_REPO_SLUG ]]; then
-  echo Error: CI_REPO_SLUG not defined
-  exit 1
-fi
+# Force CI_REPO_SLUG since sometimes
+# BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG is not set correctly, causing the
+# artifact upload to fail
+CI_REPO_SLUG=solana-labs/solana
+#if [[ -z $CI_REPO_SLUG ]]; then
+#  echo Error: CI_REPO_SLUG not defined
+#  exit 1
+#fi
 
 releaseId=$( \
   curl -s "https://api.github.com/repos/$CI_REPO_SLUG/releases/tags/$CI_TAG" \
@@ -38,6 +42,7 @@ echo "Github release id for $CI_TAG is $releaseId"
 for file in "$@"; do
   echo "--- Uploading $file to tag $CI_TAG of $CI_REPO_SLUG"
   curl \
+    --verbose \
     --data-binary @"$file" \
     -H "Authorization: token $GITHUB_TOKEN" \
     -H "Content-Type: application/octet-stream" \

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.2.0"
+version = "1.2.8"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/input_validators.rs

@@ -6,50 +6,86 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::{read_keypair_file, Signature},
 };
+use std::fmt::Display;
 use std::str::FromStr;
 
+fn is_parsable_generic<U, T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+    U: FromStr,
+    U::Err: Display,
+{
+    string
+        .as_ref()
+        .parse::<U>()
+        .map(|_| ())
+        .map_err(|err| format!("error parsing '{}': {}", string, err))
+}
+
+// Return an error if string cannot be parsed as type T.
+// Takes a String to avoid second type parameter when used as a clap validator
+pub fn is_parsable<T>(string: String) -> Result<(), String>
+where
+    T: FromStr,
+    T::Err: Display,
+{
+    is_parsable_generic::<T, String>(string)
+}
+
 // Return an error if a pubkey cannot be parsed.
-pub fn is_pubkey(string: String) -> Result<(), String> {
-    match string.parse::<Pubkey>() {
-        Ok(_) => Ok(()),
-        Err(err) => Err(format!("{}", err)),
-    }
+pub fn is_pubkey<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<Pubkey, _>(string)
 }
 
 // Return an error if a hash cannot be parsed.
-pub fn is_hash(string: String) -> Result<(), String> {
-    match string.parse::<Hash>() {
-        Ok(_) => Ok(()),
-        Err(err) => Err(format!("{}", err)),
-    }
+pub fn is_hash<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<Hash, _>(string)
 }
 
 // Return an error if a keypair file cannot be parsed.
-pub fn is_keypair(string: String) -> Result<(), String> {
-    read_keypair_file(&string)
+pub fn is_keypair<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    read_keypair_file(string.as_ref())
         .map(|_| ())
         .map_err(|err| format!("{}", err))
 }
 
 // Return an error if a keypair file cannot be parsed
-pub fn is_keypair_or_ask_keyword(string: String) -> Result<(), String> {
-    if string.as_str() == ASK_KEYWORD {
+pub fn is_keypair_or_ask_keyword<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    if string.as_ref() == ASK_KEYWORD {
         return Ok(());
     }
-    read_keypair_file(&string)
+    read_keypair_file(string.as_ref())
         .map(|_| ())
         .map_err(|err| format!("{}", err))
 }
 
 // Return an error if string cannot be parsed as pubkey string or keypair file location
-pub fn is_pubkey_or_keypair(string: String) -> Result<(), String> {
-    is_pubkey(string.clone()).or_else(|_| is_keypair(string))
+pub fn is_pubkey_or_keypair<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_pubkey(string.as_ref()).or_else(|_| is_keypair(string))
 }
 
 // Return an error if string cannot be parsed as a pubkey string, or a valid Signer that can
 // produce a pubkey()
-pub fn is_valid_pubkey(string: String) -> Result<(), String> {
-    match parse_keypair_path(&string) {
+pub fn is_valid_pubkey<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    match parse_keypair_path(string.as_ref()) {
         KeypairUrl::Filepath(path) => is_keypair(path),
         _ => Ok(()),
     }
@@ -63,13 +99,19 @@ pub fn is_valid_pubkey(string: String) -> Result<(), String> {
 // when paired with an offline `--signer` argument to provide a Presigner (pubkey + signature).
 // Clap validators can't check multiple fields at once, so the verification that a `--signer` is
 // also provided and correct happens in parsing, not in validation.
-pub fn is_valid_signer(string: String) -> Result<(), String> {
+pub fn is_valid_signer<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
     is_valid_pubkey(string)
 }
 
 // Return an error if string cannot be parsed as pubkey=signature string
-pub fn is_pubkey_sig(string: String) -> Result<(), String> {
-    let mut signer = string.split('=');
+pub fn is_pubkey_sig<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    let mut signer = string.as_ref().split('=');
     match Pubkey::from_str(
         signer
             .next()
@@ -90,8 +132,11 @@ pub fn is_pubkey_sig(string: String) -> Result<(), String> {
 }
 
 // Return an error if a url cannot be parsed.
-pub fn is_url(string: String) -> Result<(), String> {
-    match url::Url::parse(&string) {
+pub fn is_url<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    match url::Url::parse(string.as_ref()) {
         Ok(url) => {
             if url.has_host() {
                 Ok(())
@@ -103,20 +148,26 @@ pub fn is_url(string: String) -> Result<(), String> {
     }
 }
 
-pub fn is_slot(slot: String) -> Result<(), String> {
-    slot.parse::<Slot>()
-        .map(|_| ())
-        .map_err(|e| format!("{}", e))
+pub fn is_slot<T>(slot: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<Slot, _>(slot)
 }
 
-pub fn is_port(port: String) -> Result<(), String> {
-    port.parse::<u16>()
-        .map(|_| ())
-        .map_err(|e| format!("{}", e))
+pub fn is_port<T>(port: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<u16, _>(port)
 }
 
-pub fn is_valid_percentage(percentage: String) -> Result<(), String> {
+pub fn is_valid_percentage<T>(percentage: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
     percentage
+        .as_ref()
         .parse::<u8>()
         .map_err(|e| {
             format!(
@@ -136,8 +187,11 @@ pub fn is_valid_percentage(percentage: String) -> Result<(), String> {
         })
 }
 
-pub fn is_amount(amount: String) -> Result<(), String> {
-    if amount.parse::<u64>().is_ok() || amount.parse::<f64>().is_ok() {
+pub fn is_amount<T>(amount: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    if amount.as_ref().parse::<u64>().is_ok() || amount.as_ref().parse::<f64>().is_ok() {
         Ok(())
     } else {
         Err(format!(
@@ -147,8 +201,14 @@ pub fn is_amount(amount: String) -> Result<(), String> {
     }
 }
 
-pub fn is_amount_or_all(amount: String) -> Result<(), String> {
-    if amount.parse::<u64>().is_ok() || amount.parse::<f64>().is_ok() || amount == "ALL" {
+pub fn is_amount_or_all<T>(amount: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    if amount.as_ref().parse::<u64>().is_ok()
+        || amount.as_ref().parse::<f64>().is_ok()
+        || amount.as_ref() == "ALL"
+    {
         Ok(())
     } else {
         Err(format!(
@@ -158,14 +218,20 @@ pub fn is_amount_or_all(amount: String) -> Result<(), String> {
     }
 }
 
-pub fn is_rfc3339_datetime(value: String) -> Result<(), String> {
-    DateTime::parse_from_rfc3339(&value)
+pub fn is_rfc3339_datetime<T>(value: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    DateTime::parse_from_rfc3339(value.as_ref())
         .map(|_| ())
         .map_err(|e| format!("{}", e))
 }
 
-pub fn is_derivation(value: String) -> Result<(), String> {
-    let value = value.replace("'", "");
+pub fn is_derivation<T>(value: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    let value = value.as_ref().replace("'", "");
     let mut parts = value.split('/');
     let account = parts.next().unwrap();
     account
@@ -197,14 +263,14 @@ mod tests {
 
     #[test]
     fn test_is_derivation() {
-        assert_eq!(is_derivation("2".to_string()), Ok(()));
-        assert_eq!(is_derivation("0".to_string()), Ok(()));
-        assert_eq!(is_derivation("65537".to_string()), Ok(()));
-        assert_eq!(is_derivation("0/2".to_string()), Ok(()));
-        assert_eq!(is_derivation("0'/2'".to_string()), Ok(()));
-        assert!(is_derivation("a".to_string()).is_err());
-        assert!(is_derivation("4294967296".to_string()).is_err());
-        assert!(is_derivation("a/b".to_string()).is_err());
-        assert!(is_derivation("0/4294967296".to_string()).is_err());
+        assert_eq!(is_derivation("2"), Ok(()));
+        assert_eq!(is_derivation("0"), Ok(()));
+        assert_eq!(is_derivation("65537"), Ok(()));
+        assert_eq!(is_derivation("0/2"), Ok(()));
+        assert_eq!(is_derivation("0'/2'"), Ok(()));
+        assert!(is_derivation("a").is_err());
+        assert!(is_derivation("4294967296").is_err());
+        assert!(is_derivation("a/b").is_err());
+        assert!(is_derivation("0/4294967296").is_err());
     }
 }

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-config/src/config.rs

@@ -1,6 +1,6 @@
 // Wallet settings that can be configured for long-term use
 use serde_derive::{Deserialize, Serialize};
-use std::io;
+use std::{collections::HashMap, io};
 use url::Url;
 
 lazy_static! {
@@ -17,6 +17,8 @@ pub struct Config {
     pub json_rpc_url: String,
     pub websocket_url: String,
     pub keypair_path: String,
+    #[serde(default)]
+    pub address_labels: HashMap<String, String>,
 }
 
 impl Default for Config {
@@ -36,6 +38,7 @@ impl Default for Config {
             json_rpc_url,
             websocket_url,
             keypair_path,
+            address_labels: HashMap::new(),
         }
     }
 }

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.8"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,28 +27,28 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-cli-config = { path = "../cli-config", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-config-program = { path = "../programs/config", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
-solana-vote-signer = { path = "../vote-signer", version = "1.2.0" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.8" }
+solana-cli-config = { path = "../cli-config", version = "1.2.8" }
+solana-client = { path = "../client", version = "1.2.8" }
+solana-config-program = { path = "../programs/config", version = "1.2.8" }
+solana-faucet = { path = "../faucet", version = "1.2.8" }
+solana-logger = { path = "../logger", version = "1.2.8" }
+solana-net-utils = { path = "../net-utils", version = "1.2.8" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.8" }
+solana-runtime = { path = "../runtime", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.8" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.8" }
+solana-version = { path = "../version", version = "1.2.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.8" }
+solana-vote-signer = { path = "../vote-signer", version = "1.2.8" }
 thiserror = "1.0.19"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.2.0" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
+solana-core = { path = "../core", version = "1.2.8" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.8" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/checks.rs

@@ -88,11 +88,11 @@ mod tests {
         let pubkey0 = Pubkey::new(&[0; 32]);
         let pubkey1 = Pubkey::new(&[1; 32]);
         let ix0 = system_instruction::transfer(&pubkey0, &pubkey1, 1);
-        let message0 = Message::new(&[ix0]);
+        let message0 = Message::new(&[ix0], Some(&pubkey0));
 
         let ix0 = system_instruction::transfer(&pubkey0, &pubkey1, 1);
         let ix1 = system_instruction::transfer(&pubkey1, &pubkey0, 1);
-        let message1 = Message::new(&[ix0, ix1]);
+        let message1 = Message::new(&[ix0, ix1], Some(&pubkey0));
 
         let mut mocks = HashMap::new();
         mocks.insert(RpcRequest::GetBalance, account_balance_response.clone());
@@ -176,13 +176,13 @@ mod tests {
         let pubkey0 = Pubkey::new(&[0; 32]);
         let pubkey1 = Pubkey::new(&[1; 32]);
         let ix0 = system_instruction::transfer(&pubkey0, &pubkey1, 1);
-        let message0 = Message::new(&[ix0]);
+        let message0 = Message::new(&[ix0], Some(&pubkey0));
         assert_eq!(calculate_fee(&fee_calculator, &[&message0]), 1);
 
         // Two messages, additive fees.
         let ix0 = system_instruction::transfer(&pubkey0, &pubkey1, 1);
         let ix1 = system_instruction::transfer(&pubkey1, &pubkey0, 1);
-        let message1 = Message::new(&[ix0, ix1]);
+        let message1 = Message::new(&[ix0, ix1], Some(&pubkey0));
         assert_eq!(calculate_fee(&fee_calculator, &[&message0, &message1]), 3);
     }
 

cli/src/cli.rs

@@ -2,7 +2,7 @@ use crate::{
     checks::*,
     cli_output::{CliAccount, CliSignOnlyData, CliSignature, OutputFormat},
     cluster_query::*,
-    display::println_name_value,
+    display::{new_spinner_progress_bar, println_name_value, println_transaction},
     nonce::{self, *},
     offline::{blockhash_query::BlockhashQuery, *},
     spend_utils::*,
@@ -27,7 +27,7 @@ use solana_clap_utils::{
 use solana_client::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     rpc_client::RpcClient,
-    rpc_config::RpcLargestAccountsFilter,
+    rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig},
     rpc_response::{RpcAccount, RpcKeyedAccount},
 };
 #[cfg(not(test))]
@@ -37,17 +37,18 @@ use solana_faucet::faucet_mock::request_airdrop_transaction;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     bpf_loader,
-    clock::{Epoch, Slot},
+    clock::{Epoch, Slot, DEFAULT_TICKS_PER_SECOND},
     commitment_config::CommitmentConfig,
+    decode_error::DecodeError,
     fee_calculator::FeeCalculator,
     hash::Hash,
     instruction::InstructionError,
     loader_instruction,
     message::Message,
     native_token::lamports_to_sol,
-    program_utils::DecodeError,
     pubkey::{Pubkey, MAX_SEED_LEN},
     signature::{Keypair, Signature, Signer, SignerError},
+    signers::Signers,
     system_instruction::{self, SystemError},
     system_program,
     transaction::{Transaction, TransactionError},
@@ -323,6 +324,16 @@ pub enum CliCommand {
         lamports: u64,
         fee_payer: SignerIndex,
     },
+    MergeStake {
+        stake_account_pubkey: Pubkey,
+        source_stake_account_pubkey: Pubkey,
+        stake_authority: SignerIndex,
+        sign_only: bool,
+        blockhash_query: BlockhashQuery,
+        nonce_account: Option<Pubkey>,
+        nonce_authority: SignerIndex,
+        fee_payer: SignerIndex,
+    },
     ShowStakeHistory {
         use_lamports_unit: bool,
     },
@@ -385,7 +396,7 @@ pub enum CliCommand {
         vote_account_pubkey: Pubkey,
         destination_account_pubkey: Pubkey,
         withdraw_authority: SignerIndex,
-        lamports: u64,
+        withdraw_amount: SpendAmount,
     },
     VoteAuthorize {
         vote_account_pubkey: Pubkey,
@@ -396,6 +407,10 @@ pub enum CliCommand {
         vote_account_pubkey: Pubkey,
         new_identity_account: SignerIndex,
     },
+    VoteUpdateCommission {
+        vote_account_pubkey: Pubkey,
+        commission: u8,
+    },
     // Wallet Commands
     Address,
     Airdrop {
@@ -686,6 +701,9 @@ pub fn parse_command(
         ("split-stake", Some(matches)) => {
             parse_split_stake(matches, default_signer_path, wallet_manager)
         }
+        ("merge-stake", Some(matches)) => {
+            parse_merge_stake(matches, default_signer_path, wallet_manager)
+        }
         ("stake-authorize", Some(matches)) => {
             parse_stake_authorize(matches, default_signer_path, wallet_manager)
         }
@@ -709,6 +727,9 @@ pub fn parse_command(
         ("vote-update-validator", Some(matches)) => {
             parse_vote_update_validator(matches, default_signer_path, wallet_manager)
         }
+        ("vote-update-commission", Some(matches)) => {
+            parse_vote_update_commission(matches, default_signer_path, wallet_manager)
+        }
         ("vote-authorize-voter", Some(matches)) => parse_vote_authorize(
             matches,
             default_signer_path,
@@ -1159,7 +1180,7 @@ fn process_confirm(
                                 "\nTransaction executed in slot {}:",
                                 confirmed_transaction.slot
                             );
-                            crate::display::println_transaction(
+                            println_transaction(
                                 &confirmed_transaction
                                     .transaction
                                     .transaction
@@ -1189,7 +1210,7 @@ fn process_confirm(
 }
 
 fn process_decode_transaction(transaction: &Transaction) -> ProcessResult {
-    crate::display::println_transaction(transaction, &None, "");
+    println_transaction(transaction, &None, "");
     Ok("".to_string())
 }
 
@@ -1227,6 +1248,103 @@ fn process_show_account(
     Ok(account_string)
 }
 
+fn send_and_confirm_transactions_with_spinner<T: Signers>(
+    rpc_client: &RpcClient,
+    mut transactions: Vec<Transaction>,
+    signer_keys: &T,
+) -> Result<(), Box<dyn error::Error>> {
+    let progress_bar = new_spinner_progress_bar();
+    let mut send_retries = 5;
+    loop {
+        let mut status_retries = 15;
+
+        // Send all transactions
+        let mut transactions_signatures = vec![];
+        let num_transactions = transactions.len();
+        for transaction in transactions {
+            if cfg!(not(test)) {
+                // Delay ~1 tick between write transactions in an attempt to reduce AccountInUse errors
+                // when all the write transactions modify the same program account (eg, deploying a
+                // new program)
+                sleep(Duration::from_millis(1000 / DEFAULT_TICKS_PER_SECOND));
+            }
+
+            let signature = rpc_client
+                .send_transaction_with_config(
+                    &transaction,
+                    RpcSendTransactionConfig {
+                        skip_preflight: true,
+                    },
+                )
+                .ok();
+            transactions_signatures.push((transaction, signature));
+
+            progress_bar.set_message(&format!(
+                "[{}/{}] Transactions sent",
+                transactions_signatures.len(),
+                num_transactions
+            ));
+        }
+
+        // Collect statuses for all the transactions, drop those that are confirmed
+        while status_retries > 0 {
+            status_retries -= 1;
+
+            progress_bar.set_message(&format!(
+                "[{}/{}] Transactions confirmed",
+                num_transactions - transactions_signatures.len(),
+                num_transactions
+            ));
+
+            if cfg!(not(test)) {
+                // Retry twice a second
+                sleep(Duration::from_millis(500));
+            }
+
+            transactions_signatures = transactions_signatures
+                .into_iter()
+                .filter(|(_transaction, signature)| {
+                    if let Some(signature) = signature {
+                        if let Ok(status) = rpc_client.get_signature_status(&signature) {
+                            if rpc_client
+                                .get_num_blocks_since_signature_confirmation(&signature)
+                                .unwrap_or(0)
+                                > 1
+                            {
+                                return false;
+                            } else {
+                                return match status {
+                                    None => true,
+                                    Some(result) => result.is_err(),
+                                };
+                            }
+                        }
+                    }
+                    true
+                })
+                .collect();
+
+            if transactions_signatures.is_empty() {
+                return Ok(());
+            }
+        }
+
+        if send_retries == 0 {
+            return Err("Transactions failed".into());
+        }
+        send_retries -= 1;
+
+        // Re-sign any failed transactions with a new blockhash and retry
+        let (blockhash, _fee_calculator) = rpc_client
+            .get_new_blockhash(&transactions_signatures[0].0.message().recent_blockhash)?;
+        transactions = vec![];
+        for (mut transaction, _) in transactions_signatures.into_iter() {
+            transaction.try_sign(signer_keys, blockhash)?;
+            transactions.push(transaction);
+        }
+    }
+}
+
 fn process_deploy(
     rpc_client: &RpcClient,
     config: &CliConfig,
@@ -1252,7 +1370,7 @@ fn process_deploy(
         program_data.len() as u64,
         &bpf_loader::id(),
     );
-    let message = Message::new(&[ix]);
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut create_account_tx = Transaction::new_unsigned(message);
     create_account_tx.try_sign(&[config.signers[0], &program_id], blockhash)?;
     messages.push(&create_account_tx.message);
@@ -1265,7 +1383,7 @@ fn process_deploy(
             (i * DATA_CHUNK_SIZE) as u32,
             chunk.to_vec(),
         );
-        let message = Message::new_with_payer(&[instruction], Some(&signers[0].pubkey()));
+        let message = Message::new(&[instruction], Some(&signers[0].pubkey()));
         let mut tx = Transaction::new_unsigned(message);
         tx.try_sign(&signers, blockhash)?;
         write_transactions.push(tx);
@@ -1275,7 +1393,7 @@ fn process_deploy(
     }
 
     let instruction = loader_instruction::finalize(&program_id.pubkey(), &bpf_loader::id());
-    let message = Message::new_with_payer(&[instruction], Some(&signers[0].pubkey()));
+    let message = Message::new(&[instruction], Some(&signers[0].pubkey()));
     let mut finalize_tx = Transaction::new_unsigned(message);
     finalize_tx.try_sign(&signers, blockhash)?;
     messages.push(&finalize_tx.message);
@@ -1294,15 +1412,18 @@ fn process_deploy(
     })?;
 
     trace!("Writing program data");
-    rpc_client
-        .send_and_confirm_transactions_with_spinner(write_transactions, &signers)
-        .map_err(|_| {
-            CliError::DynamicProgramError("Data writes to program account failed".to_string())
-        })?;
+    send_and_confirm_transactions_with_spinner(&rpc_client, write_transactions, &signers).map_err(
+        |_| CliError::DynamicProgramError("Data writes to program account failed".to_string()),
+    )?;
 
     trace!("Finalizing program account");
     rpc_client
-        .send_and_confirm_transaction_with_spinner(&finalize_tx)
+        .send_and_confirm_transaction_with_spinner_and_config(
+            &finalize_tx,
+            RpcSendTransactionConfig {
+                skip_preflight: true,
+            },
+        )
         .map_err(|e| {
             CliError::DynamicProgramError(format!("Finalizing program account failed: {}", e))
         })?;
@@ -1349,7 +1470,7 @@ fn process_pay(
             if let Some(nonce_account) = &nonce_account {
                 Message::new_with_nonce(vec![ix], None, nonce_account, &nonce_authority.pubkey())
             } else {
-                Message::new(&[ix])
+                Message::new(&[ix], Some(&config.signers[0].pubkey()))
             }
         };
 
@@ -1396,7 +1517,7 @@ fn process_pay(
                 cancelable,
                 lamports,
             );
-            Message::new(&ixs)
+            Message::new(&ixs, Some(&config.signers[0].pubkey()))
         };
         let (message, _) = resolve_spend_tx_and_check_account_balance(
             rpc_client,
@@ -1442,7 +1563,7 @@ fn process_pay(
                 cancelable,
                 lamports,
             );
-            Message::new(&ixs)
+            Message::new(&ixs, Some(&config.signers[0].pubkey()))
         };
         let (message, _) = resolve_spend_tx_and_check_account_balance(
             rpc_client,
@@ -1478,7 +1599,7 @@ fn process_cancel(rpc_client: &RpcClient, config: &CliConfig, pubkey: &Pubkey) -
         pubkey,
         &config.signers[0].pubkey(),
     );
-    let message = Message::new(&[ix]);
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, blockhash)?;
     check_account_for_fee(
@@ -1501,7 +1622,7 @@ fn process_time_elapsed(
     let (blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let ix = budget_instruction::apply_timestamp(&config.signers[0].pubkey(), pubkey, to, dt);
-    let message = Message::new(&[ix]);
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, blockhash)?;
     check_account_for_fee(
@@ -1530,11 +1651,6 @@ fn process_transfer(
 ) -> ProcessResult {
     let from = config.signers[from];
 
-    check_unique_pubkeys(
-        (&from.pubkey(), "cli keypair".to_string()),
-        (to, "to".to_string()),
-    )?;
-
     let (recent_blockhash, fee_calculator) =
         blockhash_query.get_blockhash_and_fee_calculator(rpc_client)?;
 
@@ -1552,7 +1668,7 @@ fn process_transfer(
                 &nonce_authority.pubkey(),
             )
         } else {
-            Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+            Message::new(&ixs, Some(&fee_payer.pubkey()))
         }
     };
 
@@ -1599,7 +1715,7 @@ fn process_witness(
     let (blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
     let ix = budget_instruction::apply_signature(&config.signers[0].pubkey(), pubkey, to);
-    let message = Message::new(&[ix]);
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, blockhash)?;
     check_account_for_fee(
@@ -1655,7 +1771,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             seed,
             program_id,
         } => process_create_address_with_seed(config, from_pubkey.as_ref(), &seed, &program_id),
-        CliCommand::Fees => process_fees(&rpc_client),
+        CliCommand::Fees => process_fees(&rpc_client, config),
         CliCommand::GetBlockTime { slot } => process_get_block_time(&rpc_client, config, *slot),
         CliCommand::GetGenesisHash => process_get_genesis_hash(&rpc_client),
         CliCommand::GetEpochInfo { commitment_config } => {
@@ -1889,6 +2005,27 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *lamports,
             *fee_payer,
         ),
+        CliCommand::MergeStake {
+            stake_account_pubkey,
+            source_stake_account_pubkey,
+            stake_authority,
+            sign_only,
+            blockhash_query,
+            nonce_account,
+            nonce_authority,
+            fee_payer,
+        } => process_merge_stake(
+            &rpc_client,
+            config,
+            &stake_account_pubkey,
+            &source_stake_account_pubkey,
+            *stake_authority,
+            *sign_only,
+            blockhash_query,
+            *nonce_account,
+            *nonce_authority,
+            *fee_payer,
+        ),
         CliCommand::ShowStakeAccount {
             pubkey: stake_account_pubkey,
             use_lamports_unit,
@@ -2018,14 +2155,14 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::WithdrawFromVoteAccount {
             vote_account_pubkey,
             withdraw_authority,
-            lamports,
+            withdraw_amount,
             destination_account_pubkey,
         } => process_withdraw_from_vote_account(
             &rpc_client,
             config,
             vote_account_pubkey,
             *withdraw_authority,
-            *lamports,
+            *withdraw_amount,
             destination_account_pubkey,
         ),
         CliCommand::VoteAuthorize {
@@ -2048,6 +2185,10 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             &vote_account_pubkey,
             *new_identity_account,
         ),
+        CliCommand::VoteUpdateCommission {
+            vote_account_pubkey,
+            commission,
+        } => process_vote_update_commission(&rpc_client, config, &vote_account_pubkey, *commission),
 
         // Wallet Commands
 
@@ -3241,6 +3382,7 @@ mod tests {
         // Success cases
         let mut config = CliConfig::default();
         config.rpc_client = Some(RpcClient::new_mock("succeeds".to_string()));
+        config.json_rpc_url = "http://127.0.0.1:8899".to_string();
 
         let keypair = Keypair::new();
         let pubkey = keypair.pubkey().to_string();
@@ -3328,10 +3470,10 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = Pubkey::new_rand();
         let to_pubkey = Pubkey::new_rand();
         config.command = CliCommand::WithdrawStake {
-            stake_account_pubkey: stake_pubkey,
+            stake_account_pubkey,
             destination_account_pubkey: to_pubkey,
             lamports: 100,
             withdraw_authority: 0,
@@ -3346,9 +3488,9 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = Pubkey::new_rand();
         config.command = CliCommand::DeactivateStake {
-            stake_account_pubkey: stake_pubkey,
+            stake_account_pubkey,
             stake_authority: 0,
             sign_only: false,
             blockhash_query: BlockhashQuery::default(),
@@ -3359,10 +3501,10 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
-        let stake_pubkey = Pubkey::new_rand();
+        let stake_account_pubkey = Pubkey::new_rand();
         let split_stake_account = Keypair::new();
         config.command = CliCommand::SplitStake {
-            stake_account_pubkey: stake_pubkey,
+            stake_account_pubkey,
             stake_authority: 0,
             sign_only: false,
             blockhash_query: BlockhashQuery::default(),
@@ -3377,6 +3519,23 @@ mod tests {
         let result = process_command(&config);
         assert!(result.is_ok());
 
+        let stake_account_pubkey = Pubkey::new_rand();
+        let source_stake_account_pubkey = Pubkey::new_rand();
+        let merge_stake_account = Keypair::new();
+        config.command = CliCommand::MergeStake {
+            stake_account_pubkey,
+            source_stake_account_pubkey,
+            stake_authority: 1,
+            sign_only: false,
+            blockhash_query: BlockhashQuery::default(),
+            nonce_account: None,
+            nonce_authority: 0,
+            fee_payer: 0,
+        };
+        config.signers = vec![&keypair, &merge_stake_account];
+        let result = process_command(&config);
+        assert!(dbg!(result).is_ok());
+
         config.command = CliCommand::GetSlot {
             commitment_config: CommitmentConfig::default(),
         };

cli/src/cli_output.rs

@@ -482,7 +482,7 @@ impl fmt::Display for CliKeyedStakeState {
 #[serde(rename_all = "camelCase")]
 pub struct CliStakeState {
     pub stake_type: CliStakeType,
-    pub total_stake: u64,
+    pub account_balance: u64,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub delegated_stake: Option<u64>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -497,6 +497,16 @@ pub struct CliStakeState {
     pub lockup: Option<CliLockup>,
     #[serde(skip_serializing)]
     pub use_lamports_unit: bool,
+    #[serde(skip_serializing)]
+    pub current_epoch: Epoch,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rent_exempt_reserve: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub active_stake: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub activating_stake: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub deactivating_stake: Option<u64>,
 }
 
 impl fmt::Display for CliStakeState {
@@ -522,52 +532,122 @@ impl fmt::Display for CliStakeState {
             Ok(())
         }
 
+        writeln!(
+            f,
+            "Balance: {}",
+            build_balance_message(self.account_balance, self.use_lamports_unit, true)
+        )?;
+
+        if let Some(rent_exempt_reserve) = self.rent_exempt_reserve {
+            writeln!(
+                f,
+                "Rent Exempt Reserve: {}",
+                build_balance_message(rent_exempt_reserve, self.use_lamports_unit, true)
+            )?;
+        }
+
         match self.stake_type {
             CliStakeType::RewardsPool => writeln!(f, "Stake account is a rewards pool")?,
             CliStakeType::Uninitialized => writeln!(f, "Stake account is uninitialized")?,
             CliStakeType::Initialized => {
-                writeln!(
-                    f,
-                    "Total Stake: {}",
-                    build_balance_message(self.total_stake, self.use_lamports_unit, true)
-                )?;
                 writeln!(f, "Stake account is undelegated")?;
                 show_authorized(f, self.authorized.as_ref().unwrap())?;
                 show_lockup(f, self.lockup.as_ref().unwrap())?;
             }
             CliStakeType::Stake => {
-                writeln!(
-                    f,
-                    "Total Stake: {}",
-                    build_balance_message(self.total_stake, self.use_lamports_unit, true)
-                )?;
-                writeln!(
-                    f,
-                    "Delegated Stake: {}",
-                    build_balance_message(
-                        self.delegated_stake.unwrap(),
-                        self.use_lamports_unit,
-                        true
-                    )
-                )?;
-                if let Some(delegated_vote_account_address) = &self.delegated_vote_account_address {
+                let show_delegation = {
+                    self.active_stake.is_some()
+                        || self.activating_stake.is_some()
+                        || self.deactivating_stake.is_some()
+                        || self
+                            .deactivation_epoch
+                            .map(|de| de > self.current_epoch)
+                            .unwrap_or(true)
+                };
+                if show_delegation {
+                    let delegated_stake = self.delegated_stake.unwrap();
                     writeln!(
                         f,
-                        "Delegated Vote Account Address: {}",
-                        delegated_vote_account_address
-                    )?;
-                }
-                writeln!(
-                    f,
-                    "Stake activates starting from epoch: {}",
-                    self.activation_epoch.unwrap()
-                )?;
-                if let Some(deactivation_epoch) = self.deactivation_epoch {
-                    writeln!(
-                        f,
-                        "Stake deactivates starting from epoch: {}",
-                        deactivation_epoch
+                        "Delegated Stake: {}",
+                        build_balance_message(delegated_stake, self.use_lamports_unit, true)
                     )?;
+                    if self
+                        .deactivation_epoch
+                        .map(|d| self.current_epoch <= d)
+                        .unwrap_or(true)
+                    {
+                        let active_stake = self.active_stake.unwrap_or(0);
+                        writeln!(
+                            f,
+                            "Active Stake: {}",
+                            build_balance_message(active_stake, self.use_lamports_unit, true),
+                        )?;
+                        let activating_stake = self.activating_stake.or_else(|| {
+                            if self.active_stake.is_none() {
+                                Some(delegated_stake)
+                            } else {
+                                None
+                            }
+                        });
+                        if let Some(activating_stake) = activating_stake {
+                            writeln!(
+                                f,
+                                "Activating Stake: {}",
+                                build_balance_message(
+                                    activating_stake,
+                                    self.use_lamports_unit,
+                                    true
+                                ),
+                            )?;
+                            writeln!(
+                                f,
+                                "Stake activates starting from epoch: {}",
+                                self.activation_epoch.unwrap()
+                            )?;
+                        }
+                    }
+
+                    if let Some(deactivation_epoch) = self.deactivation_epoch {
+                        if self.current_epoch > deactivation_epoch {
+                            let deactivating_stake = self.deactivating_stake.or(self.active_stake);
+                            if let Some(deactivating_stake) = deactivating_stake {
+                                writeln!(
+                                    f,
+                                    "Inactive Stake: {}",
+                                    build_balance_message(
+                                        delegated_stake - deactivating_stake,
+                                        self.use_lamports_unit,
+                                        true
+                                    ),
+                                )?;
+                                writeln!(
+                                    f,
+                                    "Deactivating Stake: {}",
+                                    build_balance_message(
+                                        deactivating_stake,
+                                        self.use_lamports_unit,
+                                        true
+                                    ),
+                                )?;
+                            }
+                        }
+                        writeln!(
+                            f,
+                            "Stake deactivates starting from epoch: {}",
+                            deactivation_epoch
+                        )?;
+                    }
+                    if let Some(delegated_vote_account_address) =
+                        &self.delegated_vote_account_address
+                    {
+                        writeln!(
+                            f,
+                            "Delegated Vote Account Address: {}",
+                            delegated_vote_account_address
+                        )?;
+                    }
+                } else {
+                    writeln!(f, "Stake account is undelegated")?;
                 }
                 show_authorized(f, self.authorized.as_ref().unwrap())?;
                 show_lockup(f, self.lockup.as_ref().unwrap())?;
@@ -900,6 +980,7 @@ impl fmt::Display for CliSignOnlyData {
 }
 
 #[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct CliSignature {
     pub signature: String,
 }
@@ -913,6 +994,7 @@ impl fmt::Display for CliSignature {
 }
 
 #[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct CliAccountBalances {
     pub accounts: Vec<RpcAccountBalance>,
 }
@@ -937,6 +1019,7 @@ impl fmt::Display for CliAccountBalances {
 }
 
 #[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct CliSupply {
     pub total: u64,
     pub circulating: u64,
@@ -981,3 +1064,25 @@ impl fmt::Display for CliSupply {
         Ok(())
     }
 }
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliFees {
+    pub slot: Slot,
+    pub blockhash: String,
+    pub lamports_per_signature: u64,
+    pub last_valid_slot: Slot,
+}
+
+impl fmt::Display for CliFees {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Blockhash:", &self.blockhash)?;
+        writeln_name_value(
+            f,
+            "Lamports per signature:",
+            &self.lamports_per_signature.to_string(),
+        )?;
+        writeln_name_value(f, "Last valid slot:", &self.last_valid_slot.to_string())?;
+        Ok(())
+    }
+}

cli/src/cluster_query.rs

@@ -1,12 +1,11 @@
 use crate::{
     cli::{CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult},
     cli_output::*,
-    display::println_name_value,
+    display::{new_spinner_progress_bar, println_name_value},
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
 };
 use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
-use indicatif::{ProgressBar, ProgressStyle};
 use solana_clap_utils::{
     commitment::{commitment_arg, COMMITMENT_ARG},
     input_parsers::*,
@@ -29,7 +28,11 @@ use solana_sdk::{
     native_token::lamports_to_sol,
     pubkey::{self, Pubkey},
     system_instruction, system_program,
-    sysvar::{self, Sysvar},
+    sysvar::{
+        self,
+        stake_history::{self, StakeHistory},
+        Sysvar,
+    },
     transaction::Transaction,
 };
 use std::{
@@ -463,15 +466,6 @@ pub fn parse_transaction_history(
     })
 }
 
-/// Creates a new process bar for processing that will take an unknown amount of time
-fn new_spinner_progress_bar() -> ProgressBar {
-    let progress_bar = ProgressBar::new(42);
-    progress_bar
-        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
-    progress_bar.enable_steady_tick(100);
-    progress_bar
-}
-
 pub fn process_catchup(
     rpc_client: &RpcClient,
     node_pubkey: &Pubkey,
@@ -597,13 +591,16 @@ pub fn process_cluster_version(rpc_client: &RpcClient) -> ProcessResult {
     Ok(remote_version.solana_core)
 }
 
-pub fn process_fees(rpc_client: &RpcClient) -> ProcessResult {
-    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-
-    Ok(format!(
-        "blockhash: {}\nlamports per signature: {}",
-        recent_blockhash, fee_calculator.lamports_per_signature
-    ))
+pub fn process_fees(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
+    let result = rpc_client.get_recent_blockhash_with_commitment(CommitmentConfig::default())?;
+    let (recent_blockhash, fee_calculator, last_valid_slot) = result.value;
+    let fees = CliFees {
+        slot: result.context.slot,
+        blockhash: recent_blockhash.to_string(),
+        lamports_per_signature: fee_calculator.lamports_per_signature,
+        last_valid_slot,
+    };
+    Ok(config.output_format.formatted_string(&fees))
 }
 
 pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
@@ -938,7 +935,7 @@ pub fn process_ping(
 
         let build_message = |lamports| {
             let ix = system_instruction::transfer(&config.signers[0].pubkey(), &to, lamports);
-            Message::new(&[ix])
+            Message::new(&[ix], Some(&config.signers[0].pubkey()))
         };
         let (message, _) = resolve_spend_tx_and_check_account_balance(
             rpc_client,
@@ -1181,8 +1178,13 @@ pub fn process_show_stakes(
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message("Fetching stake accounts...");
     let all_stake_accounts = rpc_client.get_program_accounts(&solana_stake_program::id())?;
+    let stake_history_account = rpc_client.get_account(&stake_history::id())?;
     progress_bar.finish_and_clear();
 
+    let stake_history = StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+        CliError::RpcRequestError("Failed to deserialize stake history".to_string())
+    })?;
+
     let mut stake_accounts: Vec<CliKeyedStakeState> = vec![];
     for (stake_pubkey, stake_account) in all_stake_accounts {
         if let Ok(stake_state) = stake_account.state() {
@@ -1195,6 +1197,7 @@ pub fn process_show_stakes(
                                 stake_account.lamports,
                                 &stake_state,
                                 use_lamports_unit,
+                                &stake_history,
                             ),
                         });
                     }
@@ -1211,6 +1214,7 @@ pub fn process_show_stakes(
                                 stake_account.lamports,
                                 &stake_state,
                                 use_lamports_unit,
+                                &stake_history,
                             ),
                         });
                     }

cli/src/display.rs

@@ -1,5 +1,6 @@
 use crate::cli::SettingType;
 use console::style;
+use indicatif::{ProgressBar, ProgressStyle};
 use solana_sdk::{
     hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
     transaction::Transaction,
@@ -200,3 +201,12 @@ pub fn println_transaction(
         }
     }
 }
+
+/// Creates a new process bar for processing that will take an unknown amount of time
+pub fn new_spinner_progress_bar() -> ProgressBar {
+    let progress_bar = ProgressBar::new(42);
+    progress_bar
+        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
+    progress_bar.enable_steady_tick(100);
+    progress_bar
+}

cli/src/nonce.rs

@@ -437,7 +437,7 @@ pub fn process_authorize_nonce_account(
 
     let nonce_authority = config.signers[nonce_authority];
     let ix = authorize_nonce_account(nonce_account, &nonce_authority.pubkey(), new_authority);
-    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
 
@@ -491,7 +491,7 @@ pub fn process_create_nonce_account(
                 lamports,
             )
         };
-        Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()))
+        Message::new(&ixs, Some(&config.signers[0].pubkey()))
     };
 
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
@@ -560,7 +560,7 @@ pub fn process_new_nonce(
     let nonce_authority = config.signers[nonce_authority];
     let ix = advance_nonce_account(&nonce_account, &nonce_authority.pubkey());
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -619,7 +619,7 @@ pub fn process_withdraw_from_nonce_account(
         destination_account_pubkey,
         lamports,
     );
-    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(

cli/src/stake.rs

@@ -12,7 +12,7 @@ use crate::{
 };
 use clap::{App, Arg, ArgGroup, ArgMatches, SubCommand};
 use solana_clap_utils::{input_parsers::*, input_validators::*, offline::*, ArgConstant};
-use solana_client::rpc_client::RpcClient;
+use solana_client::{rpc_client::RpcClient, rpc_request::DELINQUENT_VALIDATOR_SLOT_DISTANCE};
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     account_utils::StateMut,
@@ -264,6 +264,29 @@ impl StakeSubCommands for App<'_, '_> {
                 .arg(nonce_authority_arg())
                 .arg(fee_payer_arg())
         )
+        .subcommand(
+            SubCommand::with_name("merge-stake")
+                .about("Merges one stake account into another")
+                .arg(
+                    pubkey!(Arg::with_name("stake_account_pubkey")
+                        .index(1)
+                        .value_name("STAKE_ACCOUNT_ADDRESS")
+                        .required(true),
+                        "Stake account to merge into")
+                )
+                .arg(
+                    pubkey!(Arg::with_name("source_stake_account_pubkey")
+                        .index(2)
+                        .value_name("SOURCE_STAKE_ACCOUNT_ADDRESS")
+                        .required(true),
+                        "Source stake account for the merge.  If successful, this stake account will no longer exist after the merge")
+                )
+                .arg(stake_authority_arg())
+                .offline_args()
+                .arg(nonce_arg())
+                .arg(nonce_authority_arg())
+                .arg(fee_payer_arg())
+        )
         .subcommand(
             SubCommand::with_name("withdraw-stake")
                 .about("Withdraw the unstaked SOL from the stake account")
@@ -606,6 +629,47 @@ pub fn parse_split_stake(
     })
 }
 
+pub fn parse_merge_stake(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let stake_account_pubkey =
+        pubkey_of_signer(matches, "stake_account_pubkey", wallet_manager)?.unwrap();
+
+    let source_stake_account_pubkey = pubkey_of(matches, "source_stake_account_pubkey").unwrap();
+
+    let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
+    let blockhash_query = BlockhashQuery::new_from_matches(matches);
+    let nonce_account = pubkey_of(matches, NONCE_ARG.name);
+    let (stake_authority, stake_authority_pubkey) =
+        signer_of(matches, STAKE_AUTHORITY_ARG.name, wallet_manager)?;
+    let (nonce_authority, nonce_authority_pubkey) =
+        signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
+    let (fee_payer, fee_payer_pubkey) = signer_of(matches, FEE_PAYER_ARG.name, wallet_manager)?;
+
+    let mut bulk_signers = vec![stake_authority, fee_payer];
+    if nonce_account.is_some() {
+        bulk_signers.push(nonce_authority);
+    }
+    let signer_info =
+        generate_unique_signers(bulk_signers, matches, default_signer_path, wallet_manager)?;
+
+    Ok(CliCommandInfo {
+        command: CliCommand::MergeStake {
+            stake_account_pubkey,
+            source_stake_account_pubkey,
+            stake_authority: signer_info.index_of(stake_authority_pubkey).unwrap(),
+            sign_only,
+            blockhash_query,
+            nonce_account,
+            nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
+            fee_payer: signer_info.index_of(fee_payer_pubkey).unwrap(),
+        },
+        signers: signer_info.signers,
+    })
+}
+
 pub fn parse_stake_deactivate_stake(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
@@ -825,7 +889,7 @@ pub fn process_create_stake_account(
                 &nonce_authority.pubkey(),
             )
         } else {
-            Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+            Message::new(&ixs, Some(&fee_payer.pubkey()))
         }
     };
 
@@ -924,7 +988,7 @@ pub fn process_stake_authorize(
             &nonce_authority.pubkey(),
         )
     } else {
-        Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+        Message::new(&ixs, Some(&fee_payer.pubkey()))
     };
     let mut tx = Transaction::new_unsigned(message);
 
@@ -978,7 +1042,7 @@ pub fn process_deactivate_stake_account(
             &nonce_authority.pubkey(),
         )
     } else {
-        Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+        Message::new(&ixs, Some(&fee_payer.pubkey()))
     };
     let mut tx = Transaction::new_unsigned(message);
 
@@ -1041,7 +1105,7 @@ pub fn process_withdraw_stake(
             &nonce_authority.pubkey(),
         )
     } else {
-        Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+        Message::new(&ixs, Some(&fee_payer.pubkey()))
     };
     let mut tx = Transaction::new_unsigned(message);
 
@@ -1175,7 +1239,100 @@ pub fn process_split_stake(
             &nonce_authority.pubkey(),
         )
     } else {
-        Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+        Message::new(&ixs, Some(&fee_payer.pubkey()))
+    };
+    let mut tx = Transaction::new_unsigned(message);
+
+    if sign_only {
+        tx.try_partial_sign(&config.signers, recent_blockhash)?;
+        return_signers(&tx, &config)
+    } else {
+        tx.try_sign(&config.signers, recent_blockhash)?;
+        if let Some(nonce_account) = &nonce_account {
+            let nonce_account = rpc_client.get_account(nonce_account)?;
+            check_nonce_account(&nonce_account, &nonce_authority.pubkey(), &recent_blockhash)?;
+        }
+        check_account_for_fee(
+            rpc_client,
+            &tx.message.account_keys[0],
+            &fee_calculator,
+            &tx.message,
+        )?;
+        let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+        log_instruction_custom_error::<StakeError>(result, &config)
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+pub fn process_merge_stake(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    stake_account_pubkey: &Pubkey,
+    source_stake_account_pubkey: &Pubkey,
+    stake_authority: SignerIndex,
+    sign_only: bool,
+    blockhash_query: &BlockhashQuery,
+    nonce_account: Option<Pubkey>,
+    nonce_authority: SignerIndex,
+    fee_payer: SignerIndex,
+) -> ProcessResult {
+    let fee_payer = config.signers[fee_payer];
+
+    check_unique_pubkeys(
+        (&fee_payer.pubkey(), "fee-payer keypair".to_string()),
+        (&stake_account_pubkey, "stake_account".to_string()),
+    )?;
+    check_unique_pubkeys(
+        (&fee_payer.pubkey(), "fee-payer keypair".to_string()),
+        (
+            &source_stake_account_pubkey,
+            "source_stake_account".to_string(),
+        ),
+    )?;
+    check_unique_pubkeys(
+        (&stake_account_pubkey, "stake_account".to_string()),
+        (
+            &source_stake_account_pubkey,
+            "source_stake_account".to_string(),
+        ),
+    )?;
+
+    let stake_authority = config.signers[stake_authority];
+
+    if !sign_only {
+        for stake_account_address in &[stake_account_pubkey, source_stake_account_pubkey] {
+            if let Ok(stake_account) = rpc_client.get_account(stake_account_address) {
+                if stake_account.owner != solana_stake_program::id() {
+                    return Err(CliError::BadParameter(format!(
+                        "Account {} is not a stake account",
+                        stake_account_address
+                    ))
+                    .into());
+                }
+            }
+        }
+    }
+
+    let (recent_blockhash, fee_calculator) =
+        blockhash_query.get_blockhash_and_fee_calculator(rpc_client)?;
+
+    let ixs = stake_instruction::merge(
+        &stake_account_pubkey,
+        &source_stake_account_pubkey,
+        &stake_authority.pubkey(),
+    );
+
+    let nonce_authority = config.signers[nonce_authority];
+
+    let message = if let Some(nonce_account) = &nonce_account {
+        Message::new_with_nonce(
+            ixs,
+            Some(&fee_payer.pubkey()),
+            nonce_account,
+            &nonce_authority.pubkey(),
+        )
+    } else {
+        Message::new(&ixs, Some(&fee_payer.pubkey()))
     };
     let mut tx = Transaction::new_unsigned(message);
 
@@ -1232,7 +1389,7 @@ pub fn process_stake_set_lockup(
             &nonce_authority.pubkey(),
         )
     } else {
-        Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+        Message::new(&ixs, Some(&fee_payer.pubkey()))
     };
     let mut tx = Transaction::new_unsigned(message);
 
@@ -1256,53 +1413,85 @@ pub fn process_stake_set_lockup(
     }
 }
 
+fn u64_some_if_not_zero(n: u64) -> Option<u64> {
+    if n > 0 {
+        Some(n)
+    } else {
+        None
+    }
+}
+
 pub fn build_stake_state(
-    stake_lamports: u64,
+    account_balance: u64,
     stake_state: &StakeState,
     use_lamports_unit: bool,
+    stake_history: &StakeHistory,
 ) -> CliStakeState {
     match stake_state {
         StakeState::Stake(
             Meta {
-                authorized, lockup, ..
+                rent_exempt_reserve,
+                authorized,
+                lockup,
             },
             stake,
-        ) => CliStakeState {
-            stake_type: CliStakeType::Stake,
-            total_stake: stake_lamports,
-            delegated_stake: Some(stake.delegation.stake),
-            delegated_vote_account_address: if stake.delegation.voter_pubkey != Pubkey::default() {
-                Some(stake.delegation.voter_pubkey.to_string())
-            } else {
-                None
-            },
-            activation_epoch: Some(if stake.delegation.activation_epoch < std::u64::MAX {
-                stake.delegation.activation_epoch
-            } else {
-                0
-            }),
-            deactivation_epoch: if stake.delegation.deactivation_epoch < std::u64::MAX {
-                Some(stake.delegation.deactivation_epoch)
-            } else {
-                None
-            },
-            authorized: Some(authorized.into()),
-            lockup: Some(lockup.into()),
-            use_lamports_unit,
-        },
+        ) => {
+            // The first entry in stake history is the previous epoch, so +1 for current
+            let current_epoch = stake_history.iter().next().unwrap().0 + 1;
+            let (active_stake, activating_stake, deactivating_stake) = stake
+                .delegation
+                .stake_activating_and_deactivating(current_epoch, Some(stake_history));
+            CliStakeState {
+                stake_type: CliStakeType::Stake,
+                account_balance,
+                delegated_stake: Some(stake.delegation.stake),
+                delegated_vote_account_address: if stake.delegation.voter_pubkey
+                    != Pubkey::default()
+                {
+                    Some(stake.delegation.voter_pubkey.to_string())
+                } else {
+                    None
+                },
+                activation_epoch: Some(if stake.delegation.activation_epoch < std::u64::MAX {
+                    stake.delegation.activation_epoch
+                } else {
+                    0
+                }),
+                deactivation_epoch: if stake.delegation.deactivation_epoch < std::u64::MAX {
+                    Some(stake.delegation.deactivation_epoch)
+                } else {
+                    None
+                },
+                authorized: Some(authorized.into()),
+                lockup: Some(lockup.into()),
+                use_lamports_unit,
+                current_epoch,
+                rent_exempt_reserve: Some(*rent_exempt_reserve),
+                active_stake: u64_some_if_not_zero(active_stake),
+                activating_stake: u64_some_if_not_zero(activating_stake),
+                deactivating_stake: u64_some_if_not_zero(deactivating_stake),
+            }
+        }
         StakeState::RewardsPool => CliStakeState {
             stake_type: CliStakeType::RewardsPool,
+            account_balance,
+            ..CliStakeState::default()
+        },
+        StakeState::Uninitialized => CliStakeState {
+            account_balance,
             ..CliStakeState::default()
         },
-        StakeState::Uninitialized => CliStakeState::default(),
         StakeState::Initialized(Meta {
-            authorized, lockup, ..
+            rent_exempt_reserve,
+            authorized,
+            lockup,
         }) => CliStakeState {
             stake_type: CliStakeType::Initialized,
-            total_stake: stake_lamports,
+            account_balance,
             authorized: Some(authorized.into()),
             lockup: Some(lockup.into()),
             use_lamports_unit,
+            rent_exempt_reserve: Some(*rent_exempt_reserve),
             ..CliStakeState::default()
         },
     }
@@ -1324,7 +1513,18 @@ pub fn process_show_stake_account(
     }
     match stake_account.state() {
         Ok(stake_state) => {
-            let state = build_stake_state(stake_account.lamports, &stake_state, use_lamports_unit);
+            let stake_history_account = rpc_client.get_account(&stake_history::id())?;
+            let stake_history =
+                StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+                    CliError::RpcRequestError("Failed to deserialize stake history".to_string())
+                })?;
+
+            let state = build_stake_state(
+                stake_account.lamports,
+                &stake_state,
+                use_lamports_unit,
+                &stake_history,
+            );
             Ok(config.output_format.formatted_string(&state))
         }
         Err(err) => Err(CliError::RpcRequestError(format!(
@@ -1399,13 +1599,15 @@ pub fn process_delegate_stake(
                 "Unable to delegate. Vote account has no root slot".to_string(),
             )),
             Some(root_slot) => {
-                let slot = rpc_client.get_slot()?;
-                if root_slot + solana_sdk::clock::DEFAULT_SLOTS_PER_TURN < slot {
-                    Err(CliError::BadParameter(
-                        format!(
-                        "Unable to delegate. Vote account root slot ({}) is too old, the current slot is {}", root_slot, slot
-                        )
-                    ))
+                let min_root_slot = rpc_client
+                    .get_slot()?
+                    .saturating_sub(DELINQUENT_VALIDATOR_SLOT_DISTANCE);
+                if root_slot < min_root_slot {
+                    Err(CliError::DynamicProgramError(format!(
+                        "Unable to delegate.  Vote account appears delinquent \
+                                 because its current root slot, {}, is less than {}",
+                        root_slot, min_root_slot
+                    )))
                 } else {
                     Ok(())
                 }
@@ -1440,7 +1642,7 @@ pub fn process_delegate_stake(
             &nonce_authority.pubkey(),
         )
     } else {
-        Message::new_with_payer(&ixs, Some(&fee_payer.pubkey()))
+        Message::new(&ixs, Some(&fee_payer.pubkey()))
     };
     let mut tx = Transaction::new_unsigned(message);
 
@@ -2878,5 +3080,34 @@ mod tests {
                 ],
             }
         );
+
+        // Test MergeStake SubCommand
+        let (keypair_file, mut tmp_file) = make_tmp_file();
+        let stake_account_keypair = Keypair::new();
+        write_keypair(&stake_account_keypair, tmp_file.as_file_mut()).unwrap();
+
+        let source_stake_account_pubkey = Pubkey::new_rand();
+        let test_merge_stake_account = test_commands.clone().get_matches_from(vec![
+            "test",
+            "merge-stake",
+            &keypair_file,
+            &source_stake_account_pubkey.to_string(),
+        ]);
+        assert_eq!(
+            parse_command(&test_merge_stake_account, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::MergeStake {
+                    stake_account_pubkey: stake_account_keypair.pubkey(),
+                    source_stake_account_pubkey,
+                    stake_authority: 0,
+                    sign_only: false,
+                    blockhash_query: BlockhashQuery::default(),
+                    nonce_account: None,
+                    nonce_authority: 0,
+                    fee_payer: 0,
+                },
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into(),],
+            }
+        );
     }
 }

cli/src/validator_info.rs

@@ -346,7 +346,7 @@ pub fn process_set_validator_info(
                 keys,
                 &validator_info,
             )]);
-            Message::new(&instructions)
+            Message::new(&instructions, Some(&config.signers[0].pubkey()))
         } else {
             println!(
                 "Updating Validator {:?} info at: {:?}",
@@ -359,7 +359,7 @@ pub fn process_set_validator_info(
                 keys,
                 &validator_info,
             )];
-            Message::new_with_payer(&instructions, Some(&config.signers[0].pubkey()))
+            Message::new(&instructions, Some(&config.signers[0].pubkey()))
         }
     };
 

cli/src/vote.rs

@@ -16,8 +16,9 @@ use solana_clap_utils::{
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    account::Account, commitment_config::CommitmentConfig, message::Message, pubkey::Pubkey,
-    system_instruction::SystemError, transaction::Transaction,
+    account::Account, commitment_config::CommitmentConfig, message::Message,
+    native_token::lamports_to_sol, pubkey::Pubkey, system_instruction::SystemError,
+    transaction::Transaction,
 };
 use solana_vote_program::{
     vote_instruction::{self, withdraw, VoteError},
@@ -161,6 +162,35 @@ impl VoteSubCommands for App<'_, '_> {
                         .help("Authorized withdrawer keypair"),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("vote-update-commission")
+                .about("Update the vote account's commission")
+                .arg(
+                    pubkey!(Arg::with_name("vote_account_pubkey")
+                        .index(1)
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
+                        .required(true),
+                        "Vote account to update. "),
+                )
+                .arg(
+                    Arg::with_name("commission")
+                        .index(2)
+                        .value_name("PERCENTAGE")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_percentage)
+                        .help("The new commission")
+                )
+                .arg(
+                    Arg::with_name("authorized_withdrawer")
+                        .index(3)
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Authorized withdrawer keypair"),
+                )
+        )
         .subcommand(
             SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
@@ -203,8 +233,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("AMOUNT")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_amount)
-                        .help("The amount to withdraw, in SOL"),
+                        .validator(is_amount_or_all)
+                        .help("The amount to withdraw, in SOL; accepts keyword ALL"),
                 )
                 .arg(
                     Arg::with_name("authorized_withdrawer")
@@ -309,6 +339,33 @@ pub fn parse_vote_update_validator(
     })
 }
 
+pub fn parse_vote_update_commission(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let (authorized_withdrawer, _) = signer_of(matches, "authorized_withdrawer", wallet_manager)?;
+    let commission = value_t_or_exit!(matches, "commission", u8);
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, authorized_withdrawer],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
+    Ok(CliCommandInfo {
+        command: CliCommand::VoteUpdateCommission {
+            vote_account_pubkey,
+            commission,
+        },
+        signers: signer_info.signers,
+    })
+}
+
 pub fn parse_vote_get_account_command(
     matches: &ArgMatches<'_>,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
@@ -336,7 +393,8 @@ pub fn parse_withdraw_from_vote_account(
         pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
     let destination_account_pubkey =
         pubkey_of_signer(matches, "destination_account_pubkey", wallet_manager)?.unwrap();
-    let lamports = lamports_of_sol(matches, "amount").unwrap();
+    let withdraw_amount = SpendAmount::new_from_matches(matches, "amount");
+
     let (withdraw_authority, withdraw_authority_pubkey) =
         signer_of(matches, "authorized_withdrawer", wallet_manager)?;
 
@@ -353,7 +411,7 @@ pub fn parse_withdraw_from_vote_account(
             vote_account_pubkey,
             destination_account_pubkey,
             withdraw_authority: signer_info.index_of(withdraw_authority_pubkey).unwrap(),
-            lamports,
+            withdraw_amount,
         },
         signers: signer_info.signers,
     })
@@ -417,7 +475,7 @@ pub fn process_create_vote_account(
                 lamports,
             )
         };
-        Message::new(&ixs)
+        Message::new(&ixs, Some(&config.signers[0].pubkey()))
     };
 
     if let Ok(vote_account) = rpc_client.get_account(&vote_account_address) {
@@ -475,7 +533,7 @@ pub fn process_vote_authorize(
         vote_authorize,        // vote or withdraw
     )];
 
-    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
+    let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -508,7 +566,34 @@ pub fn process_vote_update_validator(
         &new_identity_pubkey,
     )];
 
-    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
+    let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&config.signers, recent_blockhash)?;
+    check_account_for_fee(
+        rpc_client,
+        &config.signers[0].pubkey(),
+        &fee_calculator,
+        &tx.message,
+    )?;
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<VoteError>(result, &config)
+}
+
+pub fn process_vote_update_commission(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    vote_account_pubkey: &Pubkey,
+    commission: u8,
+) -> ProcessResult {
+    let authorized_withdrawer = config.signers[1];
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let ixs = vec![vote_instruction::update_commission(
+        vote_account_pubkey,
+        &authorized_withdrawer.pubkey(),
+        commission,
+    )];
+
+    let message = Message::new(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -600,12 +685,28 @@ pub fn process_withdraw_from_vote_account(
     config: &CliConfig,
     vote_account_pubkey: &Pubkey,
     withdraw_authority: SignerIndex,
-    lamports: u64,
+    withdraw_amount: SpendAmount,
     destination_account_pubkey: &Pubkey,
 ) -> ProcessResult {
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let withdraw_authority = config.signers[withdraw_authority];
 
+    let current_balance = rpc_client.get_balance(&vote_account_pubkey)?;
+    let minimum_balance = rpc_client.get_minimum_balance_for_rent_exemption(VoteState::size_of())?;
+
+    let lamports = match withdraw_amount {
+        SpendAmount::All => current_balance.saturating_sub(minimum_balance),
+        SpendAmount::Some(withdraw_amount) => {
+            if current_balance.saturating_sub(withdraw_amount) < minimum_balance {
+                return Err(CliError::BadParameter(format!(
+                    "Withdraw amount too large. The vote account balance must be at least {} SOL to remain rent exempt", lamports_to_sol(minimum_balance)
+                ))
+                .into());
+            }
+            withdraw_amount
+        }
+    };
+
     let ix = withdraw(
         vote_account_pubkey,
         &withdraw_authority.pubkey(),
@@ -613,7 +714,7 @@ pub fn process_withdraw_from_vote_account(
         destination_account_pubkey,
     );
 
-    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut transaction = Transaction::new_unsigned(message);
     transaction.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -842,6 +943,27 @@ mod tests {
             }
         );
 
+        let test_update_commission = test_commands.clone().get_matches_from(vec![
+            "test",
+            "vote-update-commission",
+            &pubkey_string,
+            "42",
+            &keypair_file,
+        ]);
+        assert_eq!(
+            parse_command(&test_update_commission, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::VoteUpdateCommission {
+                    vote_account_pubkey: pubkey,
+                    commission: 42,
+                },
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    Box::new(read_keypair_file(&keypair_file).unwrap()),
+                ],
+            }
+        );
+
         // Test WithdrawFromVoteAccount subcommand
         let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
             "test",
@@ -862,7 +984,33 @@ mod tests {
                     vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
                     destination_account_pubkey: pubkey,
                     withdraw_authority: 0,
-                    lamports: 42_000_000_000
+                    withdraw_amount: SpendAmount::Some(42_000_000_000),
+                },
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
+            }
+        );
+
+        // Test WithdrawFromVoteAccount subcommand
+        let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-from-vote-account",
+            &keypair_file,
+            &pubkey_string,
+            "ALL",
+        ]);
+        assert_eq!(
+            parse_command(
+                &test_withdraw_from_vote_account,
+                &default_keypair_file,
+                &mut None
+            )
+            .unwrap(),
+            CliCommandInfo {
+                command: CliCommand::WithdrawFromVoteAccount {
+                    vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
+                    destination_account_pubkey: pubkey,
+                    withdraw_authority: 0,
+                    withdraw_amount: SpendAmount::All,
                 },
                 signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
@@ -893,7 +1041,7 @@ mod tests {
                     vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
                     destination_account_pubkey: pubkey,
                     withdraw_authority: 1,
-                    lamports: 42_000_000_000
+                    withdraw_amount: SpendAmount::Some(42_000_000_000),
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),

cli/tests/vote.rs

@@ -1,5 +1,9 @@
-use solana_cli::cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig};
-use solana_cli::test_utils::check_balance;
+use solana_cli::{
+    cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
+    offline::{blockhash_query::BlockhashQuery, *},
+    spend_utils::SpendAmount,
+    test_utils::check_balance,
+};
 use solana_client::rpc_client::RpcClient;
 use solana_core::validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
@@ -64,6 +68,23 @@ fn test_vote_authorize_and_withdraw() {
         .max(1);
     check_balance(expected_balance, &rpc_client, &vote_account_pubkey);
 
+    // Transfer in some more SOL
+    config.signers = vec![&default_signer];
+    config.command = CliCommand::Transfer {
+        amount: SpendAmount::Some(1_000),
+        to: vote_account_pubkey,
+        from: 0,
+        sign_only: false,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    process_command(&config).unwrap();
+    let expected_balance = expected_balance + 1_000;
+    check_balance(expected_balance, &rpc_client, &vote_account_pubkey);
+
     // Authorize vote account withdrawal to another signer
     let withdraw_authority = Keypair::new();
     config.signers = vec![&default_signer];
@@ -86,7 +107,7 @@ fn test_vote_authorize_and_withdraw() {
     config.command = CliCommand::WithdrawFromVoteAccount {
         vote_account_pubkey,
         withdraw_authority: 1,
-        lamports: 100,
+        withdraw_amount: SpendAmount::Some(100),
         destination_account_pubkey: destination_account,
     };
     process_command(&config).unwrap();

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.2.0"
+version = "1.2.8"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,10 +19,10 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.8" }
+solana-net-utils = { path = "../net-utils", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.8" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -31,7 +31,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.1.0"
 jsonrpc-http-server = "14.1.0"
-solana-logger = { path = "../logger", version = "1.2.0" }
+solana-logger = { path = "../logger", version = "1.2.8" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/rpc_client.rs

@@ -2,7 +2,7 @@ use crate::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     http_sender::HttpSender,
     mock_sender::{MockSender, Mocks},
-    rpc_config::RpcLargestAccountsConfig,
+    rpc_config::{RpcLargestAccountsConfig, RpcSendTransactionConfig},
     rpc_request::{RpcError, RpcRequest},
     rpc_response::*,
     rpc_sender::RpcSender,
@@ -22,7 +22,6 @@ use solana_sdk::{
     epoch_schedule::EpochSchedule,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
     hash::Hash,
-    inflation::Inflation,
     pubkey::Pubkey,
     signature::Signature,
     signers::Signers,
@@ -33,7 +32,6 @@ use solana_transaction_status::{
 };
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
-    error,
     net::SocketAddr,
     thread::sleep,
     time::{Duration, Instant},
@@ -95,10 +93,20 @@ impl RpcClient {
     }
 
     pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<Signature> {
+        self.send_transaction_with_config(transaction, RpcSendTransactionConfig::default())
+    }
+
+    pub fn send_transaction_with_config(
+        &self,
+        transaction: &Transaction,
+        config: RpcSendTransactionConfig,
+    ) -> ClientResult<Signature> {
         let serialized_encoded = bs58::encode(serialize(transaction).unwrap()).into_string();
 
-        let signature_base58_str: String =
-            self.send(RpcRequest::SendTransaction, json!([serialized_encoded]))?;
+        let signature_base58_str: String = self.send(
+            RpcRequest::SendTransaction,
+            json!([serialized_encoded, config]),
+        )?;
 
         let signature = signature_base58_str
             .parse::<Signature>()
@@ -122,7 +130,7 @@ impl RpcClient {
         &self,
         transaction: &Transaction,
         sig_verify: bool,
-    ) -> RpcResult<TransactionStatus> {
+    ) -> RpcResult<RpcSimulateTransactionResult> {
         let serialized_encoded = bs58::encode(serialize(transaction).unwrap()).into_string();
         self.send(
             RpcRequest::SimulateTransaction,
@@ -346,8 +354,12 @@ impl RpcClient {
         })
     }
 
-    pub fn get_inflation(&self) -> ClientResult<Inflation> {
-        self.send(RpcRequest::GetInflation, Value::Null)
+    pub fn get_inflation_governor(&self) -> ClientResult<RpcInflationGovernor> {
+        self.send(RpcRequest::GetInflationGovernor, Value::Null)
+    }
+
+    pub fn get_inflation_rate(&self) -> ClientResult<RpcInflationRate> {
+        self.send(RpcRequest::GetInflationRate, Value::Null)
     }
 
     pub fn get_version(&self) -> ClientResult<RpcVersionInfo> {
@@ -403,96 +415,6 @@ impl RpcClient {
         }
     }
 
-    pub fn send_and_confirm_transactions_with_spinner<T: Signers>(
-        &self,
-        mut transactions: Vec<Transaction>,
-        signer_keys: &T,
-    ) -> Result<(), Box<dyn error::Error>> {
-        let progress_bar = new_spinner_progress_bar();
-        let mut send_retries = 5;
-        loop {
-            let mut status_retries = 15;
-
-            // Send all transactions
-            let mut transactions_signatures = vec![];
-            let num_transactions = transactions.len();
-            for transaction in transactions {
-                if cfg!(not(test)) {
-                    // Delay ~1 tick between write transactions in an attempt to reduce AccountInUse errors
-                    // when all the write transactions modify the same program account (eg, deploying a
-                    // new program)
-                    sleep(Duration::from_millis(1000 / DEFAULT_TICKS_PER_SECOND));
-                }
-
-                let signature = self.send_transaction(&transaction).ok();
-                transactions_signatures.push((transaction, signature));
-
-                progress_bar.set_message(&format!(
-                    "[{}/{}] Transactions sent",
-                    transactions_signatures.len(),
-                    num_transactions
-                ));
-            }
-
-            // Collect statuses for all the transactions, drop those that are confirmed
-            while status_retries > 0 {
-                status_retries -= 1;
-
-                progress_bar.set_message(&format!(
-                    "[{}/{}] Transactions confirmed",
-                    num_transactions - transactions_signatures.len(),
-                    num_transactions
-                ));
-
-                if cfg!(not(test)) {
-                    // Retry twice a second
-                    sleep(Duration::from_millis(500));
-                }
-
-                transactions_signatures = transactions_signatures
-                    .into_iter()
-                    .filter(|(_transaction, signature)| {
-                        if let Some(signature) = signature {
-                            if let Ok(status) = self.get_signature_status(&signature) {
-                                if self
-                                    .get_num_blocks_since_signature_confirmation(&signature)
-                                    .unwrap_or(0)
-                                    > 1
-                                {
-                                    return false;
-                                } else {
-                                    return match status {
-                                        None => true,
-                                        Some(result) => result.is_err(),
-                                    };
-                                }
-                            }
-                        }
-                        true
-                    })
-                    .collect();
-
-                if transactions_signatures.is_empty() {
-                    return Ok(());
-                }
-            }
-
-            if send_retries == 0 {
-                return Err(RpcError::ForUser("Transactions failed".to_string()).into());
-            }
-            send_retries -= 1;
-
-            // Re-sign any failed transactions with a new blockhash and retry
-            let (blockhash, _fee_calculator) =
-                self.get_new_blockhash(&transactions_signatures[0].0.message().recent_blockhash)?;
-            transactions = vec![];
-            for (mut transaction, _) in transactions_signatures.into_iter() {
-                transaction.try_sign(signer_keys, blockhash)?;
-                transactions.push(transaction);
-            }
-        }
-    }
-
     pub fn resign_transaction<T: Signers>(
         &self,
         tx: &mut Transaction,
@@ -614,26 +536,46 @@ impl RpcClient {
     }
 
     pub fn get_recent_blockhash(&self) -> ClientResult<(Hash, FeeCalculator)> {
-        Ok(self
+        let (blockhash, fee_calculator, _last_valid_slot) = self
             .get_recent_blockhash_with_commitment(CommitmentConfig::default())?
-            .value)
+            .value;
+        Ok((blockhash, fee_calculator))
     }
 
     pub fn get_recent_blockhash_with_commitment(
         &self,
         commitment_config: CommitmentConfig,
-    ) -> RpcResult<(Hash, FeeCalculator)> {
-        let Response {
+    ) -> RpcResult<(Hash, FeeCalculator, Slot)> {
+        let (context, blockhash, fee_calculator, last_valid_slot) = if let Ok(Response {
+            context,
+            value:
+                RpcFees {
+                    blockhash,
+                    fee_calculator,
+                    last_valid_slot,
+                },
+        }) =
+            self.send::<Response<RpcFees>>(RpcRequest::GetFees, json!([commitment_config]))
+        {
+            (context, blockhash, fee_calculator, last_valid_slot)
+        } else if let Ok(Response {
             context,
             value:
                 RpcBlockhashFeeCalculator {
                     blockhash,
                     fee_calculator,
                 },
-        } = self.send::<Response<RpcBlockhashFeeCalculator>>(
+        }) = self.send::<Response<RpcBlockhashFeeCalculator>>(
             RpcRequest::GetRecentBlockhash,
             json!([commitment_config]),
-        )?;
+        ) {
+            (context, blockhash, fee_calculator, 0)
+        } else {
+            return Err(ClientError::new_with_request(
+                RpcError::ParseError("RpcBlockhashFeeCalculator or RpcFees".to_string()).into(),
+                RpcRequest::GetRecentBlockhash,
+            ));
+        };
 
         let blockhash = blockhash.parse().map_err(|_| {
             ClientError::new_with_request(
@@ -643,7 +585,7 @@ impl RpcClient {
         })?;
         Ok(Response {
             context,
-            value: (blockhash, fee_calculator),
+            value: (blockhash, fee_calculator, last_valid_slot),
         })
     }
 
@@ -651,12 +593,28 @@ impl RpcClient {
         &self,
         blockhash: &Hash,
     ) -> ClientResult<Option<FeeCalculator>> {
-        let Response { value, .. } = self.send::<Response<Option<RpcFeeCalculator>>>(
+        Ok(self
+            .get_fee_calculator_for_blockhash_with_commitment(
+                blockhash,
+                CommitmentConfig::default(),
+            )?
+            .value)
+    }
+
+    pub fn get_fee_calculator_for_blockhash_with_commitment(
+        &self,
+        blockhash: &Hash,
+        commitment_config: CommitmentConfig,
+    ) -> RpcResult<Option<FeeCalculator>> {
+        let Response { context, value } = self.send::<Response<Option<RpcFeeCalculator>>>(
             RpcRequest::GetFeeCalculatorForBlockhash,
-            json!([blockhash.to_string()]),
+            json!([blockhash.to_string(), commitment_config]),
         )?;
 
-        Ok(value.map(|rf| rf.fee_calculator))
+        Ok(Response {
+            context,
+            value: value.map(|rf| rf.fee_calculator),
+        })
     }
 
     pub fn get_fee_rate_governor(&self) -> RpcResult<FeeRateGovernor> {
@@ -913,6 +871,17 @@ impl RpcClient {
     pub fn send_and_confirm_transaction_with_spinner(
         &self,
         transaction: &Transaction,
+    ) -> ClientResult<Signature> {
+        self.send_and_confirm_transaction_with_spinner_and_config(
+            transaction,
+            RpcSendTransactionConfig::default(),
+        )
+    }
+
+    pub fn send_and_confirm_transaction_with_spinner_and_config(
+        &self,
+        transaction: &Transaction,
+        config: RpcSendTransactionConfig,
     ) -> ClientResult<Signature> {
         let mut confirmations = 0;
 
@@ -928,7 +897,7 @@ impl RpcClient {
             ));
             let mut status_retries = 15;
             let (signature, status) = loop {
-                let signature = self.send_transaction(transaction)?;
+                let signature = self.send_transaction_with_config(transaction, config.clone())?;
 
                 // Get recent commitment in order to count confirmations for successful transactions
                 let status = self

client/src/rpc_config.rs

@@ -1,4 +1,4 @@
-use solana_sdk::commitment_config::CommitmentConfig;
+use solana_sdk::{clock::Epoch, commitment_config::CommitmentConfig};
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -6,7 +6,13 @@ pub struct RpcSignatureStatusConfig {
     pub search_transaction_history: bool,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSendTransactionConfig {
+    pub skip_preflight: bool,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcSimulateTransactionConfig {
     pub sig_verify: bool,
@@ -26,3 +32,11 @@ pub struct RpcLargestAccountsConfig {
     pub commitment: Option<CommitmentConfig>,
     pub filter: Option<RpcLargestAccountsFilter>,
 }
+
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationConfig {
+    pub epoch: Option<Epoch>,
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}

client/src/rpc_request.rs

@@ -16,15 +16,18 @@ pub enum RpcRequest {
     GetConfirmedTransaction,
     GetEpochInfo,
     GetEpochSchedule,
-    GetGenesisHash,
-    GetIdentity,
-    GetInflation,
-    GetLargestAccounts,
-    GetLeaderSchedule,
-    GetProgramAccounts,
-    GetRecentBlockhash,
     GetFeeCalculatorForBlockhash,
     GetFeeRateGovernor,
+    GetFees,
+    GetGenesisHash,
+    GetIdentity,
+    GetInflationGovernor,
+    GetInflationRate,
+    GetLargestAccounts,
+    GetLeaderSchedule,
+    GetMinimumBalanceForRentExemption,
+    GetProgramAccounts,
+    GetRecentBlockhash,
     GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
@@ -37,13 +40,12 @@ pub enum RpcRequest {
     GetTransactionCount,
     GetVersion,
     GetVoteAccounts,
+    MinimumLedgerSlot,
     RegisterNode,
     RequestAirdrop,
     SendTransaction,
     SimulateTransaction,
     SignVote,
-    GetMinimumBalanceForRentExemption,
-    MinimumLedgerSlot,
 }
 
 impl fmt::Display for RpcRequest {
@@ -61,15 +63,18 @@ impl fmt::Display for RpcRequest {
             RpcRequest::GetConfirmedTransaction => "getConfirmedTransaction",
             RpcRequest::GetEpochInfo => "getEpochInfo",
             RpcRequest::GetEpochSchedule => "getEpochSchedule",
-            RpcRequest::GetGenesisHash => "getGenesisHash",
-            RpcRequest::GetIdentity => "getIdentity",
-            RpcRequest::GetInflation => "getInflation",
-            RpcRequest::GetLargestAccounts => "getLargestAccounts",
-            RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
-            RpcRequest::GetProgramAccounts => "getProgramAccounts",
-            RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetFeeCalculatorForBlockhash => "getFeeCalculatorForBlockhash",
             RpcRequest::GetFeeRateGovernor => "getFeeRateGovernor",
+            RpcRequest::GetFees => "getFees",
+            RpcRequest::GetGenesisHash => "getGenesisHash",
+            RpcRequest::GetIdentity => "getIdentity",
+            RpcRequest::GetInflationGovernor => "getInflationGovernor",
+            RpcRequest::GetInflationRate => "getInflationRate",
+            RpcRequest::GetLargestAccounts => "getLargestAccounts",
+            RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
+            RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
+            RpcRequest::GetProgramAccounts => "getProgramAccounts",
+            RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
@@ -82,13 +87,12 @@ impl fmt::Display for RpcRequest {
             RpcRequest::GetTransactionCount => "getTransactionCount",
             RpcRequest::GetVersion => "getVersion",
             RpcRequest::GetVoteAccounts => "getVoteAccounts",
+            RpcRequest::MinimumLedgerSlot => "minimumLedgerSlot",
             RpcRequest::RegisterNode => "registerNode",
             RpcRequest::RequestAirdrop => "requestAirdrop",
             RpcRequest::SendTransaction => "sendTransaction",
             RpcRequest::SimulateTransaction => "simulateTransaction",
             RpcRequest::SignVote => "signVote",
-            RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
-            RpcRequest::MinimumLedgerSlot => "minimumLedgerSlot",
         };
 
         write!(f, "{}", method)
@@ -99,6 +103,9 @@ pub const NUM_LARGEST_ACCOUNTS: usize = 20;
 pub const MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS: usize = 256;
 pub const MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE: u64 = 10_000;
 
+// Validators that are this number of slots behind are considered delinquent
+pub const DELINQUENT_VALIDATOR_SLOT_DISTANCE: u64 = 128;
+
 impl RpcRequest {
     pub(crate) fn build_request_json(self, id: u64, params: Value) -> Value {
         let jsonrpc = "2.0";
@@ -144,10 +151,6 @@ mod tests {
         let request = test_request.build_request_json(1, Value::Null);
         assert_eq!(request["method"], "getEpochInfo");
 
-        let test_request = RpcRequest::GetInflation;
-        let request = test_request.build_request_json(1, Value::Null);
-        assert_eq!(request["method"], "getInflation");
-
         let test_request = RpcRequest::GetRecentBlockhash;
         let request = test_request.build_request_json(1, Value::Null);
         assert_eq!(request["method"], "getRecentBlockhash");

client/src/rpc_response.rs

@@ -3,6 +3,7 @@ use solana_sdk::{
     account::Account,
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
+    inflation::Inflation,
     pubkey::Pubkey,
     transaction::{Result, TransactionError},
 };
@@ -35,6 +36,14 @@ pub struct RpcBlockhashFeeCalculator {
     pub fee_calculator: FeeCalculator,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcFees {
+    pub blockhash: String,
+    pub fee_calculator: FeeCalculator,
+    pub last_valid_slot: Slot,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcFeeCalculator {
@@ -47,6 +56,37 @@ pub struct RpcFeeRateGovernor {
     pub fee_rate_governor: FeeRateGovernor,
 }
 
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationGovernor {
+    pub initial: f64,
+    pub terminal: f64,
+    pub taper: f64,
+    pub foundation: f64,
+    pub foundation_term: f64,
+}
+
+impl From<Inflation> for RpcInflationGovernor {
+    fn from(inflation: Inflation) -> Self {
+        Self {
+            initial: inflation.initial,
+            terminal: inflation.terminal,
+            taper: inflation.taper,
+            foundation: inflation.foundation,
+            foundation_term: inflation.foundation_term,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationRate {
+    pub total: f64,
+    pub validator: f64,
+    pub foundation: f64,
+    pub epoch: Epoch,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcKeyedAccount {
@@ -171,6 +211,13 @@ pub struct RpcSignatureConfirmation {
     pub status: Result<()>,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSimulateTransactionResult {
+    pub err: Option<TransactionError>,
+    pub logs: Option<Vec<String>>,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcStorageTurn {

client/src/thin_client.rs

@@ -9,7 +9,7 @@ use log::*;
 use solana_sdk::{
     account::Account,
     client::{AsyncClient, Client, SyncClient},
-    clock::MAX_PROCESSING_AGE,
+    clock::{Slot, MAX_PROCESSING_AGE},
     commitment_config::CommitmentConfig,
     epoch_info::EpochInfo,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
@@ -357,7 +357,7 @@ impl Client for ThinClient {
 }
 
 impl SyncClient for ThinClient {
-    fn send_message<T: Signers>(
+    fn send_and_confirm_message<T: Signers>(
         &self,
         keypairs: &T,
         message: Message,
@@ -368,16 +368,16 @@ impl SyncClient for ThinClient {
         Ok(signature)
     }
 
-    fn send_instruction(
+    fn send_and_confirm_instruction(
         &self,
         keypair: &Keypair,
         instruction: Instruction,
     ) -> TransportResult<Signature> {
-        let message = Message::new(&[instruction]);
-        self.send_message(&[keypair], message)
+        let message = Message::new(&[instruction], Some(&keypair.pubkey()));
+        self.send_and_confirm_message(&[keypair], message)
     }
 
-    fn transfer(
+    fn transfer_and_confirm(
         &self,
         lamports: u64,
         keypair: &Keypair,
@@ -385,7 +385,7 @@ impl SyncClient for ThinClient {
     ) -> TransportResult<Signature> {
         let transfer_instruction =
             system_instruction::transfer(&keypair.pubkey(), pubkey, lamports);
-        self.send_instruction(keypair, transfer_instruction)
+        self.send_and_confirm_instruction(keypair, transfer_instruction)
     }
 
     fn get_account_data(&self, pubkey: &Pubkey) -> TransportResult<Option<Vec<u8>>> {
@@ -427,13 +427,15 @@ impl SyncClient for ThinClient {
     }
 
     fn get_recent_blockhash(&self) -> TransportResult<(Hash, FeeCalculator)> {
-        self.get_recent_blockhash_with_commitment(CommitmentConfig::default())
+        let (blockhash, fee_calculator, _last_valid_slot) =
+            self.get_recent_blockhash_with_commitment(CommitmentConfig::default())?;
+        Ok((blockhash, fee_calculator))
     }
 
     fn get_recent_blockhash_with_commitment(
         &self,
         commitment_config: CommitmentConfig,
-    ) -> TransportResult<(Hash, FeeCalculator)> {
+    ) -> TransportResult<(Hash, FeeCalculator, Slot)> {
         let index = self.optimizer.experiment();
         let now = Instant::now();
         let recent_blockhash =
@@ -441,7 +443,7 @@ impl SyncClient for ThinClient {
         match recent_blockhash {
             Ok(Response { value, .. }) => {
                 self.optimizer.report(index, duration_as_ms(&now.elapsed()));
-                Ok(value)
+                Ok((value.0, value.1, value.2))
             }
             Err(e) => {
                 self.optimizer.report(index, std::u64::MAX);
@@ -609,7 +611,7 @@ impl AsyncClient for ThinClient {
         instruction: Instruction,
         recent_blockhash: Hash,
     ) -> TransportResult<Signature> {
-        let message = Message::new(&[instruction]);
+        let message = Message::new(&[instruction], Some(&keypair.pubkey()));
         self.async_send_message(&[keypair], message, recent_blockhash)
     }
     fn async_transfer(

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.8"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -21,6 +21,7 @@ byteorder = "1.3.4"
 chrono = { version = "0.4.11", features = ["serde"] }
 core_affinity = "0.5.10"
 crossbeam-channel = "0.4"
+ed25519-dalek = "=1.0.0-pre.3"
 fs_extra = "1.1.0"
 flate2 = "1.0"
 indexmap = "1.3"
@@ -41,35 +42,35 @@ regex = "1.3.7"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.0" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-ed25519-dalek = "=1.0.0-pre.3"
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-perf = { path = "../perf", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-streamer = { path = "../streamer", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
-solana-vote-signer = { path = "../vote-signer", version = "1.2.0" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.2.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.8" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.8" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.8" }
+solana-client = { path = "../client", version = "1.2.8" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.8" }
+solana-faucet = { path = "../faucet", version = "1.2.8" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.8" }
+solana-ledger = { path = "../ledger", version = "1.2.8" }
+solana-logger = { path = "../logger", version = "1.2.8" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.2.8" }
+solana-metrics = { path = "../metrics", version = "1.2.8" }
+solana-measure = { path = "../measure", version = "1.2.8" }
+solana-net-utils = { path = "../net-utils", version = "1.2.8" }
+solana-perf = { path = "../perf", version = "1.2.8" }
+solana-runtime = { path = "../runtime", version = "1.2.8" }
+solana-sdk = { path = "../sdk", version = "1.2.8" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.8" }
+solana-streamer = { path = "../streamer", version = "1.2.8" }
+solana-version = { path = "../version", version = "1.2.8" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.8" }
+solana-vote-signer = { path = "../vote-signer", version = "1.2.8" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.2.8" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.8" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/benches/banking_stage.rs

@@ -19,6 +19,7 @@ use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;
 use solana_sdk::hash::Hash;
+use solana_sdk::message::Message;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::Keypair;
 use solana_sdk::signature::Signature;
@@ -117,9 +118,8 @@ fn make_programs_txs(txes: usize, hash: Hash) -> Vec<Transaction> {
                 let to_key = Pubkey::new_rand();
                 instructions.push(system_instruction::transfer(&from_key.pubkey(), &to_key, 1));
             }
-            let mut new = Transaction::new_unsigned_instructions(&instructions);
-            new.sign(&[&from_key], hash);
-            new
+            let message = Message::new(&instructions, Some(&from_key.pubkey()));
+            Transaction::new(&[&from_key], message, hash)
         })
         .collect()
 }

core/benches/cluster_info.rs

@@ -3,6 +3,7 @@
 extern crate test;
 
 use rand::{thread_rng, Rng};
+use solana_core::broadcast_stage::broadcast_metrics::TransmitShredsStats;
 use solana_core::broadcast_stage::{broadcast_shreds, get_broadcast_peers};
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
@@ -47,7 +48,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
             &peers_and_stakes,
             &peers,
             &last_datapoint,
-            &mut 0,
+            &mut TransmitShredsStats::default(),
         )
         .unwrap();
     });

core/benches/poh_verify.rs

@@ -12,9 +12,10 @@ const NUM_ENTRIES: usize = 800;
 
 #[bench]
 fn bench_poh_verify_ticks(bencher: &mut Bencher) {
+    solana_logger::setup();
     let zero = Hash::default();
-    let mut cur_hash = hash(&zero.as_ref());
-    let start = *&cur_hash;
+    let start_hash = hash(&zero.as_ref());
+    let mut cur_hash = start_hash;
 
     let mut ticks: Vec<Entry> = Vec::with_capacity(NUM_ENTRIES);
     for _ in 0..NUM_ENTRIES {
@@ -22,15 +23,15 @@ fn bench_poh_verify_ticks(bencher: &mut Bencher) {
     }
 
     bencher.iter(|| {
-        ticks.verify(&start);
+        assert!(ticks.verify(&start_hash));
     })
 }
 
 #[bench]
 fn bench_poh_verify_transaction_entries(bencher: &mut Bencher) {
     let zero = Hash::default();
-    let mut cur_hash = hash(&zero.as_ref());
-    let start = *&cur_hash;
+    let start_hash = hash(&zero.as_ref());
+    let mut cur_hash = start_hash;
 
     let keypair1 = Keypair::new();
     let pubkey1 = keypair1.pubkey();
@@ -42,6 +43,6 @@ fn bench_poh_verify_transaction_entries(bencher: &mut Bencher) {
     }
 
     bencher.iter(|| {
-        ticks.verify(&start);
+        assert!(ticks.verify(&start_hash));
     })
 }

core/benches/retransmit_stage.rs

@@ -49,7 +49,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
 
     let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
     let bank0 = Bank::new(&genesis_config);
-    let bank_forks = BankForks::new(0, bank0);
+    let bank_forks = BankForks::new(bank0);
     let bank = bank_forks.working_bank();
     let bank_forks = Arc::new(RwLock::new(bank_forks));
     let (packet_sender, packet_receiver) = channel();

core/src/accounts_background_service.rs

@@ -15,11 +15,15 @@ pub struct AccountsBackgroundService {
 }
 
 const INTERVAL_MS: u64 = 100;
+const SHRUNKEN_ACCOUNT_PER_SEC: usize = 250;
+const SHRUNKEN_ACCOUNT_PER_INTERVAL: usize =
+    SHRUNKEN_ACCOUNT_PER_SEC / (1000 / INTERVAL_MS as usize);
 
 impl AccountsBackgroundService {
     pub fn new(bank_forks: Arc<RwLock<BankForks>>, exit: &Arc<AtomicBool>) -> Self {
         info!("AccountsBackgroundService active");
         let exit = exit.clone();
+        let mut consumed_budget = 0;
         let t_background = Builder::new()
             .name("solana-accounts-background".to_string())
             .spawn(move || loop {
@@ -30,8 +34,8 @@ impl AccountsBackgroundService {
 
                 bank.process_dead_slots();
 
-                // Currently, given INTERVAL_MS, we process 1 slot/100 ms
-                bank.process_stale_slot();
+                consumed_budget = bank
+                    .process_stale_slot_with_budget(consumed_budget, SHRUNKEN_ACCOUNT_PER_INTERVAL);
 
                 sleep(Duration::from_millis(INTERVAL_MS));
             })

core/src/accounts_hash_verifier.rs

@@ -177,6 +177,7 @@ mod tests {
     use crate::cluster_info::make_accounts_hashes_message;
     use crate::contact_info::ContactInfo;
     use solana_ledger::bank_forks::CompressionType;
+    use solana_ledger::snapshot_utils::SnapshotVersion;
     use solana_sdk::{
         hash::hash,
         signature::{Keypair, Signer},
@@ -239,6 +240,7 @@ mod tests {
                 tar_output_file: PathBuf::from("."),
                 storages: vec![],
                 compression: CompressionType::Bzip2,
+                snapshot_version: SnapshotVersion::default(),
             };
 
             AccountsHashVerifier::process_accounts_package(

core/src/bank_weight_fork_choice.rs

@@ -0,0 +1,152 @@
+use crate::{
+    consensus::{ComputedBankState, Tower},
+    fork_choice::ForkChoice,
+    progress_map::{ForkStats, ProgressMap},
+};
+use solana_ledger::bank_forks::BankForks;
+use solana_runtime::bank::Bank;
+use solana_sdk::timing;
+use std::time::Instant;
+use std::{
+    collections::{HashMap, HashSet},
+    sync::{Arc, RwLock},
+};
+
+#[derive(Default)]
+pub struct BankWeightForkChoice {}
+
+impl ForkChoice for BankWeightForkChoice {
+    fn compute_bank_stats(
+        &mut self,
+        bank: &Bank,
+        _tower: &Tower,
+        progress: &mut ProgressMap,
+        computed_bank_state: &ComputedBankState,
+    ) {
+        let bank_slot = bank.slot();
+        // Only time progress map should be missing a bank slot
+        // is if this node was the leader for this slot as those banks
+        // are not replayed in replay_active_banks()
+        let parent_weight = bank
+            .parent()
+            .and_then(|b| progress.get(&b.slot()))
+            .map(|x| x.fork_stats.fork_weight)
+            .unwrap_or(0);
+
+        let stats = progress
+            .get_fork_stats_mut(bank_slot)
+            .expect("All frozen banks must exist in the Progress map");
+
+        let ComputedBankState { bank_weight, .. } = computed_bank_state;
+        stats.weight = *bank_weight;
+        stats.fork_weight = stats.weight + parent_weight;
+    }
+
+    // Returns:
+    // 1) The heaviest overall bank
+    // 2) The heaviest bank on the same fork as the last vote (doesn't require a
+    // switching proof to vote for)
+    fn select_forks(
+        &self,
+        frozen_banks: &[Arc<Bank>],
+        tower: &Tower,
+        progress: &ProgressMap,
+        ancestors: &HashMap<u64, HashSet<u64>>,
+        _bank_forks: &RwLock<BankForks>,
+    ) -> (Arc<Bank>, Option<Arc<Bank>>) {
+        let tower_start = Instant::now();
+        assert!(!frozen_banks.is_empty());
+        let num_frozen_banks = frozen_banks.len();
+
+        trace!("frozen_banks {}", frozen_banks.len());
+        let num_old_banks = frozen_banks
+            .iter()
+            .filter(|b| b.slot() < tower.root().unwrap_or(0))
+            .count();
+
+        let last_vote = tower.last_vote().slots.last().cloned();
+        let mut heaviest_bank_on_same_fork = None;
+        let mut heaviest_same_fork_weight = 0;
+        let stats: Vec<&ForkStats> = frozen_banks
+            .iter()
+            .map(|bank| {
+                // Only time progress map should be missing a bank slot
+                // is if this node was the leader for this slot as those banks
+                // are not replayed in replay_active_banks()
+                let stats = progress
+                    .get_fork_stats(bank.slot())
+                    .expect("All frozen banks must exist in the Progress map");
+
+                if let Some(last_vote) = last_vote {
+                    if ancestors
+                        .get(&bank.slot())
+                        .expect("Entry in frozen banks must exist in ancestors")
+                        .contains(&last_vote)
+                    {
+                        // Descendant of last vote cannot be locked out
+                        assert!(!stats.is_locked_out);
+
+                        // ancestors(slot) should not contain the slot itself,
+                        // so we should never get the same bank as the last vote
+                        assert_ne!(bank.slot(), last_vote);
+                        // highest weight, lowest slot first. frozen_banks is sorted
+                        // from least slot to greatest slot, so if two banks have
+                        // the same fork weight, the lower slot will be picked
+                        if stats.fork_weight > heaviest_same_fork_weight {
+                            heaviest_bank_on_same_fork = Some(bank.clone());
+                            heaviest_same_fork_weight = stats.fork_weight;
+                        }
+                    }
+                }
+
+                stats
+            })
+            .collect();
+        let num_not_recent = stats.iter().filter(|s| !s.is_recent).count();
+        let num_has_voted = stats.iter().filter(|s| s.has_voted).count();
+        let num_empty = stats.iter().filter(|s| s.is_empty).count();
+        let num_threshold_failure = stats.iter().filter(|s| !s.vote_threshold).count();
+        let num_votable_threshold_failure = stats
+            .iter()
+            .filter(|s| s.is_recent && !s.has_voted && !s.vote_threshold)
+            .count();
+
+        let mut candidates: Vec<_> = frozen_banks.iter().zip(stats.iter()).collect();
+
+        //highest weight, lowest slot first
+        candidates.sort_by_key(|b| (b.1.fork_weight, 0i64 - b.0.slot() as i64));
+        let rv = candidates
+            .last()
+            .expect("frozen banks was nonempty so candidates must also be nonempty");
+        let ms = timing::duration_as_ms(&tower_start.elapsed());
+        let weights: Vec<(u128, u64, u64)> = candidates
+            .iter()
+            .map(|x| (x.1.weight, x.0.slot(), x.1.block_height))
+            .collect();
+        debug!(
+            "@{:?} tower duration: {:?} len: {}/{} weights: {:?}",
+            timing::timestamp(),
+            ms,
+            candidates.len(),
+            stats.iter().filter(|s| !s.has_voted).count(),
+            weights,
+        );
+        datapoint_debug!(
+            "replay_stage-select_forks",
+            ("frozen_banks", num_frozen_banks as i64, i64),
+            ("not_recent", num_not_recent as i64, i64),
+            ("has_voted", num_has_voted as i64, i64),
+            ("old_banks", num_old_banks as i64, i64),
+            ("empty_banks", num_empty as i64, i64),
+            ("threshold_failure", num_threshold_failure as i64, i64),
+            (
+                "votable_threshold_failure",
+                num_votable_threshold_failure as i64,
+                i64
+            ),
+            ("tower_duration", ms as i64, i64),
+        );
+
+        (rv.0.clone(), heaviest_bank_on_same_fork)
+    }
+}

core/src/banking_stage.rs

@@ -51,7 +51,7 @@ type PacketsAndOffsets = (Packets, Vec<usize>);
 pub type UnprocessedPackets = Vec<PacketsAndOffsets>;
 
 /// Transaction forwarding
-pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 4;
+pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 1;
 
 // Fixed thread size seems to be fastest on GCP setup
 pub const NUM_THREADS: u32 = 4;
@@ -514,7 +514,7 @@ impl BankingStage {
             vec![]
         };
         let (mut loaded_accounts, results, mut retryable_txs, tx_count, signature_count) =
-            bank.load_and_execute_transactions(batch, MAX_PROCESSING_AGE);
+            bank.load_and_execute_transactions(batch, MAX_PROCESSING_AGE, None);
         load_execute_time.stop();
 
         let freeze_lock = bank.freeze_lock();

core/src/broadcast_stage.rs

@@ -35,7 +35,7 @@ use std::{
 };
 
 mod broadcast_fake_shreds_run;
-pub(crate) mod broadcast_metrics;
+pub mod broadcast_metrics;
 pub(crate) mod broadcast_utils;
 mod fail_entry_verification_broadcast_run;
 mod standard_broadcast_run;
@@ -374,13 +374,14 @@ pub fn broadcast_shreds(
     peers_and_stakes: &[(u64, usize)],
     peers: &[ContactInfo],
     last_datapoint_submit: &Arc<AtomicU64>,
-    send_mmsg_total: &mut u64,
+    transmit_stats: &mut TransmitShredsStats,
 ) -> Result<()> {
     let broadcast_len = peers_and_stakes.len();
     if broadcast_len == 0 {
         update_peer_stats(1, 1, last_datapoint_submit);
         return Ok(());
     }
+    let mut shred_select = Measure::start("shred_select");
     let packets: Vec<_> = shreds
         .iter()
         .map(|shred| {
@@ -389,6 +390,8 @@ pub fn broadcast_shreds(
             (&shred.payload, &peers[broadcast_index].tvu)
         })
         .collect();
+    shred_select.stop();
+    transmit_stats.shred_select += shred_select.as_us();
 
     let mut sent = 0;
     let mut send_mmsg_time = Measure::start("send_mmsg");
@@ -401,7 +404,7 @@ pub fn broadcast_shreds(
         }
     }
     send_mmsg_time.stop();
-    *send_mmsg_total += send_mmsg_time.as_us();
+    transmit_stats.send_mmsg_elapsed += send_mmsg_time.as_us();
 
     let num_live_peers = num_live_peers(&peers);
     update_peer_stats(

core/src/broadcast_stage/broadcast_metrics.rs

@@ -29,11 +29,12 @@ impl ProcessShredsStats {
 }
 
 #[derive(Default, Clone)]
-pub(crate) struct TransmitShredsStats {
-    pub(crate) transmit_elapsed: u64,
-    pub(crate) send_mmsg_elapsed: u64,
-    pub(crate) get_peers_elapsed: u64,
-    pub(crate) num_shreds: usize,
+pub struct TransmitShredsStats {
+    pub transmit_elapsed: u64,
+    pub send_mmsg_elapsed: u64,
+    pub get_peers_elapsed: u64,
+    pub shred_select: u64,
+    pub num_shreds: usize,
 }
 
 impl BroadcastStats for TransmitShredsStats {
@@ -42,6 +43,7 @@ impl BroadcastStats for TransmitShredsStats {
         self.send_mmsg_elapsed += new_stats.send_mmsg_elapsed;
         self.get_peers_elapsed += new_stats.get_peers_elapsed;
         self.num_shreds += new_stats.num_shreds;
+        self.shred_select += new_stats.shred_select;
     }
     fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
         datapoint_info!(
@@ -58,6 +60,7 @@ impl BroadcastStats for TransmitShredsStats {
             ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
             ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
             ("num_shreds", self.num_shreds as i64, i64),
+            ("shred_select", self.shred_select as i64, i64),
         );
     }
 }
@@ -176,15 +179,16 @@ mod test {
     }
 
     #[test]
-    fn test_update() {
+    fn test_update_broadcast() {
         let start = Instant::now();
         let mut slot_broadcast_stats = SlotBroadcastStats::default();
         slot_broadcast_stats.update(
             &TransmitShredsStats {
                 transmit_elapsed: 1,
-                get_peers_elapsed: 1,
-                send_mmsg_elapsed: 1,
-                num_shreds: 1,
+                get_peers_elapsed: 2,
+                send_mmsg_elapsed: 3,
+                shred_select: 4,
+                num_shreds: 5,
             },
             &Some(BroadcastShredBatchInfo {
                 slot: 0,
@@ -198,16 +202,18 @@ mod test {
         assert_eq!(slot_0_stats.num_batches, 1);
         assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
         assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
 
         slot_broadcast_stats.update(
             &TransmitShredsStats {
-                transmit_elapsed: 1,
-                get_peers_elapsed: 1,
-                send_mmsg_elapsed: 1,
-                num_shreds: 1,
+                transmit_elapsed: 7,
+                get_peers_elapsed: 8,
+                send_mmsg_elapsed: 9,
+                shred_select: 10,
+                num_shreds: 11,
             },
             &None,
         );
@@ -217,9 +223,10 @@ mod test {
         assert_eq!(slot_0_stats.num_batches, 1);
         assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
         assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
 
         // If another batch is given, then total number of batches == num_expected_batches == 2,
         // so the batch should be purged from the HashMap
@@ -228,6 +235,7 @@ mod test {
                 transmit_elapsed: 1,
                 get_peers_elapsed: 1,
                 send_mmsg_elapsed: 1,
+                shred_select: 1,
                 num_shreds: 1,
             },
             &Some(BroadcastShredBatchInfo {

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -137,14 +137,13 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         // Broadcast data
         let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
 
-        let mut send_mmsg_total = 0;
         broadcast_shreds(
             sock,
             &shreds,
             &peers_and_stakes,
             &peers,
             &Arc::new(AtomicU64::new(0)),
-            &mut send_mmsg_total,
+            &mut TransmitShredsStats::default(),
         )?;
 
         Ok(())

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -9,6 +9,7 @@ use solana_ledger::{
 };
 use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
 use std::collections::HashMap;
+use std::sync::RwLock;
 use std::time::Duration;
 
 #[derive(Clone)]
@@ -23,6 +24,14 @@ pub struct StandardBroadcastRun {
     shred_version: u16,
     last_datapoint_submit: Arc<AtomicU64>,
     num_batches: usize,
+    broadcast_peer_cache: Arc<RwLock<BroadcastPeerCache>>,
+    last_peer_update: Arc<AtomicU64>,
+}
+
+#[derive(Default)]
+struct BroadcastPeerCache {
+    peers: Vec<ContactInfo>,
+    peers_and_stakes: Vec<(u64, usize)>,
 }
 
 impl StandardBroadcastRun {
@@ -38,6 +47,8 @@ impl StandardBroadcastRun {
             shred_version,
             last_datapoint_submit: Arc::new(AtomicU64::new(0)),
             num_batches: 0,
+            broadcast_peer_cache: Arc::new(RwLock::new(BroadcastPeerCache::default())),
+            last_peer_update: Arc::new(AtomicU64::new(0)),
         }
     }
 
@@ -293,33 +304,46 @@ impl StandardBroadcastRun {
         shreds: Arc<Vec<Shred>>,
         broadcast_shred_batch_info: Option<BroadcastShredBatchInfo>,
     ) -> Result<()> {
+        const BROADCAST_PEER_UPDATE_INTERVAL_MS: u64 = 1000;
         trace!("Broadcasting {:?} shreds", shreds.len());
         // Get the list of peers to broadcast to
-        let get_peers_start = Instant::now();
-        let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
-        let get_peers_elapsed = get_peers_start.elapsed();
+        let mut get_peers_time = Measure::start("broadcast::get_peers");
+        let now = timestamp();
+        let last = self.last_peer_update.load(Ordering::Relaxed);
+        if now - last > BROADCAST_PEER_UPDATE_INTERVAL_MS
+            && self
+                .last_peer_update
+                .compare_and_swap(now, last, Ordering::Relaxed)
+                == last
+        {
+            let mut w_broadcast_peer_cache = self.broadcast_peer_cache.write().unwrap();
+            let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
+            w_broadcast_peer_cache.peers = peers;
+            w_broadcast_peer_cache.peers_and_stakes = peers_and_stakes;
+        }
+        get_peers_time.stop();
+        let r_broadcast_peer_cache = self.broadcast_peer_cache.read().unwrap();
 
+        let mut transmit_stats = TransmitShredsStats::default();
         // Broadcast the shreds
-        let transmit_start = Instant::now();
-        let mut send_mmsg_total = 0;
+        let mut transmit_time = Measure::start("broadcast_shreds");
         broadcast_shreds(
             sock,
             &shreds,
-            &peers_and_stakes,
-            &peers,
+            &r_broadcast_peer_cache.peers_and_stakes,
+            &r_broadcast_peer_cache.peers,
             &self.last_datapoint_submit,
-            &mut send_mmsg_total,
+            &mut transmit_stats,
         )?;
-        let transmit_elapsed = transmit_start.elapsed();
-        let new_transmit_shreds_stats = TransmitShredsStats {
-            transmit_elapsed: duration_as_us(&transmit_elapsed),
-            get_peers_elapsed: duration_as_us(&get_peers_elapsed),
-            send_mmsg_elapsed: send_mmsg_total,
-            num_shreds: shreds.len(),
-        };
+        drop(r_broadcast_peer_cache);
+        transmit_time.stop();
+
+        transmit_stats.transmit_elapsed = transmit_time.as_us();
+        transmit_stats.get_peers_elapsed = get_peers_time.as_us();
+        transmit_stats.num_shreds = shreds.len();
 
         // Process metrics
-        self.update_transmit_metrics(&new_transmit_shreds_stats, &broadcast_shred_batch_info);
+        self.update_transmit_metrics(&transmit_stats, &broadcast_shred_batch_info);
         Ok(())
     }
 

core/src/cluster_info_vote_listener.rs

@@ -1,6 +1,6 @@
 use crate::{
     cluster_info::{ClusterInfo, GOSSIP_SLEEP_MILLIS},
-    consensus::VOTE_THRESHOLD_SIZE,
+    commitment::VOTE_THRESHOLD_SIZE,
     crds_value::CrdsValueLabel,
     poh_recorder::PohRecorder,
     pubkey_references::LockedPubkeyReferences,
@@ -49,7 +49,7 @@ pub type VerifiedVoteTransactionsReceiver = CrossbeamReceiver<Vec<Transaction>>;
 pub struct SlotVoteTracker {
     voted: HashSet<Arc<Pubkey>>,
     updates: Option<Vec<Arc<Pubkey>>>,
-    total_stake: u64,
+    pub total_stake: u64,
 }
 
 impl SlotVoteTracker {
@@ -62,7 +62,7 @@ impl SlotVoteTracker {
 #[derive(Default)]
 pub struct VoteTracker {
     // Map from a slot to a set of validators who have voted for that slot
-    slot_vote_trackers: RwLock<HashMap<Slot, Arc<RwLock<SlotVoteTracker>>>>,
+    pub slot_vote_trackers: RwLock<HashMap<Slot, Arc<RwLock<SlotVoteTracker>>>>,
     // Don't track votes from people who are not staked, acts as a spam filter
     epoch_authorized_voters: RwLock<HashMap<Epoch, Arc<EpochAuthorizedVoters>>>,
     leader_schedule_epoch: RwLock<Epoch>,
@@ -948,7 +948,7 @@ mod tests {
             );
         let bank = Bank::new(&genesis_config);
         let exit = Arc::new(AtomicBool::new(false));
-        let bank_forks = BankForks::new(0, bank);
+        let bank_forks = BankForks::new(bank);
         let bank = bank_forks.get(0).unwrap().clone();
         let vote_tracker = VoteTracker::new(&bank);
         let ledger_path = get_tmp_ledger_path!();
@@ -1064,7 +1064,7 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let vote_tracker = VoteTracker::new(&bank);
         let exit = Arc::new(AtomicBool::new(false));
-        let bank_forks = BankForks::new(0, bank);
+        let bank_forks = BankForks::new(bank);
         let bank = bank_forks.get(0).unwrap().clone();
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());

core/src/commitment.rs

@@ -1,26 +1,10 @@
-use crate::{consensus::VOTE_THRESHOLD_SIZE, rpc_subscriptions::RpcSubscriptions};
 use solana_ledger::blockstore::Blockstore;
-use solana_measure::measure::Measure;
-use solana_metrics::datapoint_info;
 use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
-use solana_vote_program::{vote_state::VoteState, vote_state::MAX_LOCKOUT_HISTORY};
-use std::{
-    collections::HashMap,
-    sync::atomic::{AtomicBool, Ordering},
-    sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender},
-    sync::{Arc, RwLock},
-    thread::{self, Builder, JoinHandle},
-    time::Duration,
-};
+use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
+use std::{collections::HashMap, sync::Arc};
 
-#[derive(Default)]
-pub struct CacheSlotInfo {
-    pub current_slot: Slot,
-    pub node_root: Slot,
-    pub largest_confirmed_root: Slot,
-    pub highest_confirmed_slot: Slot,
-}
+pub const VOTE_THRESHOLD_SIZE: f64 = 2f64 / 3f64;
 
 pub type BlockCommitmentArray = [u64; MAX_LOCKOUT_HISTORY + 1];
 
@@ -59,9 +43,9 @@ pub struct BlockCommitmentCache {
     largest_confirmed_root: Slot,
     total_stake: u64,
     bank: Arc<Bank>,
-    blockstore: Arc<Blockstore>,
+    pub blockstore: Arc<Blockstore>,
     root: Slot,
-    highest_confirmed_slot: Slot,
+    pub highest_confirmed_slot: Slot,
 }
 
 impl std::fmt::Debug for BlockCommitmentCache {
@@ -151,7 +135,7 @@ impl BlockCommitmentCache {
         self.root
     }
 
-    fn calculate_highest_confirmed_slot(&self) -> Slot {
+    pub fn calculate_highest_confirmed_slot(&self) -> Slot {
         self.highest_slot_with_confirmation_count(1)
     }
 
@@ -219,222 +203,11 @@ impl BlockCommitmentCache {
     }
 }
 
-pub struct CommitmentAggregationData {
-    bank: Arc<Bank>,
-    root: Slot,
-    total_staked: u64,
-}
-
-impl CommitmentAggregationData {
-    pub fn new(bank: Arc<Bank>, root: Slot, total_staked: u64) -> Self {
-        Self {
-            bank,
-            root,
-            total_staked,
-        }
-    }
-}
-
-fn get_largest_confirmed_root(mut rooted_stake: Vec<(Slot, u64)>, total_stake: u64) -> Slot {
-    rooted_stake.sort_by(|a, b| a.0.cmp(&b.0).reverse());
-    let mut stake_sum = 0;
-    for (root, stake) in rooted_stake {
-        stake_sum += stake;
-        if (stake_sum as f64 / total_stake as f64) > VOTE_THRESHOLD_SIZE {
-            return root;
-        }
-    }
-    0
-}
-
-pub struct AggregateCommitmentService {
-    t_commitment: JoinHandle<()>,
-}
-
-impl AggregateCommitmentService {
-    pub fn new(
-        exit: &Arc<AtomicBool>,
-        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
-        subscriptions: Arc<RpcSubscriptions>,
-    ) -> (Sender<CommitmentAggregationData>, Self) {
-        let (sender, receiver): (
-            Sender<CommitmentAggregationData>,
-            Receiver<CommitmentAggregationData>,
-        ) = channel();
-        let exit_ = exit.clone();
-        (
-            sender,
-            Self {
-                t_commitment: Builder::new()
-                    .name("solana-aggregate-stake-lockouts".to_string())
-                    .spawn(move || loop {
-                        if exit_.load(Ordering::Relaxed) {
-                            break;
-                        }
-
-                        if let Err(RecvTimeoutError::Disconnected) =
-                            Self::run(&receiver, &block_commitment_cache, &subscriptions, &exit_)
-                        {
-                            break;
-                        }
-                    })
-                    .unwrap(),
-            },
-        )
-    }
-
-    fn run(
-        receiver: &Receiver<CommitmentAggregationData>,
-        block_commitment_cache: &RwLock<BlockCommitmentCache>,
-        subscriptions: &Arc<RpcSubscriptions>,
-        exit: &Arc<AtomicBool>,
-    ) -> Result<(), RecvTimeoutError> {
-        loop {
-            if exit.load(Ordering::Relaxed) {
-                return Ok(());
-            }
-
-            let mut aggregation_data = receiver.recv_timeout(Duration::from_secs(1))?;
-
-            while let Ok(new_data) = receiver.try_recv() {
-                aggregation_data = new_data;
-            }
-
-            let ancestors = aggregation_data.bank.status_cache_ancestors();
-            if ancestors.is_empty() {
-                continue;
-            }
-
-            let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
-            let (block_commitment, rooted_stake) =
-                Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
-
-            let largest_confirmed_root =
-                get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
-
-            let mut new_block_commitment = BlockCommitmentCache::new(
-                block_commitment,
-                largest_confirmed_root,
-                aggregation_data.total_staked,
-                aggregation_data.bank,
-                block_commitment_cache.read().unwrap().blockstore.clone(),
-                aggregation_data.root,
-                aggregation_data.root,
-            );
-            new_block_commitment.highest_confirmed_slot =
-                new_block_commitment.calculate_highest_confirmed_slot();
-
-            let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
-
-            std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
-            aggregate_commitment_time.stop();
-            datapoint_info!(
-                "block-commitment-cache",
-                (
-                    "aggregate-commitment-ms",
-                    aggregate_commitment_time.as_ms() as i64,
-                    i64
-                )
-            );
-
-            subscriptions.notify_subscribers(CacheSlotInfo {
-                current_slot: w_block_commitment_cache.slot(),
-                node_root: w_block_commitment_cache.root(),
-                largest_confirmed_root: w_block_commitment_cache.largest_confirmed_root(),
-                highest_confirmed_slot: w_block_commitment_cache.highest_confirmed_slot(),
-            });
-        }
-    }
-
-    pub fn aggregate_commitment(
-        ancestors: &[Slot],
-        bank: &Bank,
-    ) -> (HashMap<Slot, BlockCommitment>, Vec<(Slot, u64)>) {
-        assert!(!ancestors.is_empty());
-
-        // Check ancestors is sorted
-        for a in ancestors.windows(2) {
-            assert!(a[0] < a[1]);
-        }
-
-        let mut commitment = HashMap::new();
-        let mut rooted_stake: Vec<(Slot, u64)> = Vec::new();
-        for (_, (lamports, account)) in bank.vote_accounts().into_iter() {
-            if lamports == 0 {
-                continue;
-            }
-            let vote_state = VoteState::from(&account);
-            if vote_state.is_none() {
-                continue;
-            }
-
-            let vote_state = vote_state.unwrap();
-            Self::aggregate_commitment_for_vote_account(
-                &mut commitment,
-                &mut rooted_stake,
-                &vote_state,
-                ancestors,
-                lamports,
-            );
-        }
-
-        (commitment, rooted_stake)
-    }
-
-    fn aggregate_commitment_for_vote_account(
-        commitment: &mut HashMap<Slot, BlockCommitment>,
-        rooted_stake: &mut Vec<(Slot, u64)>,
-        vote_state: &VoteState,
-        ancestors: &[Slot],
-        lamports: u64,
-    ) {
-        assert!(!ancestors.is_empty());
-        let mut ancestors_index = 0;
-        if let Some(root) = vote_state.root_slot {
-            for (i, a) in ancestors.iter().enumerate() {
-                if *a <= root {
-                    commitment
-                        .entry(*a)
-                        .or_insert_with(BlockCommitment::default)
-                        .increase_rooted_stake(lamports);
-                } else {
-                    ancestors_index = i;
-                    break;
-                }
-            }
-            rooted_stake.push((root, lamports));
-        }
-
-        for vote in &vote_state.votes {
-            while ancestors[ancestors_index] <= vote.slot {
-                commitment
-                    .entry(ancestors[ancestors_index])
-                    .or_insert_with(BlockCommitment::default)
-                    .increase_confirmation_stake(vote.confirmation_count as usize, lamports);
-                ancestors_index += 1;
-
-                if ancestors_index == ancestors.len() {
-                    return;
-                }
-            }
-        }
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        self.t_commitment.join()
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
-    use solana_ledger::{
-        genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        get_tmp_ledger_path,
-    };
+    use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::{genesis_config::GenesisConfig, pubkey::Pubkey};
-    use solana_stake_program::stake_state;
-    use solana_vote_program::vote_state::{self, VoteStateVersions};
 
     #[test]
     fn test_block_commitment() {
@@ -512,21 +285,6 @@ mod tests {
         assert!(!block_commitment_cache.is_confirmed_rooted(3));
     }
 
-    #[test]
-    fn test_get_largest_confirmed_root() {
-        assert_eq!(get_largest_confirmed_root(vec![], 10), 0);
-        let mut rooted_stake = vec![];
-        rooted_stake.push((0, 5));
-        rooted_stake.push((1, 5));
-        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 0);
-        let mut rooted_stake = vec![];
-        rooted_stake.push((1, 5));
-        rooted_stake.push((0, 10));
-        rooted_stake.push((2, 5));
-        rooted_stake.push((1, 4));
-        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 1);
-    }
-
     #[test]
     fn test_highest_confirmed_slot() {
         let bank = Arc::new(Bank::new(&GenesisConfig::default()));
@@ -634,211 +392,4 @@ mod tests {
 
         assert_eq!(block_commitment_cache.calculate_highest_confirmed_slot(), 0);
     }
-
-    #[test]
-    fn test_aggregate_commitment_for_vote_account_1() {
-        let ancestors = vec![3, 4, 5, 7, 9, 11];
-        let mut commitment = HashMap::new();
-        let mut rooted_stake = vec![];
-        let lamports = 5;
-        let mut vote_state = VoteState::default();
-
-        let root = *ancestors.last().unwrap();
-        vote_state.root_slot = Some(root);
-        AggregateCommitmentService::aggregate_commitment_for_vote_account(
-            &mut commitment,
-            &mut rooted_stake,
-            &vote_state,
-            &ancestors,
-            lamports,
-        );
-
-        for a in ancestors {
-            let mut expected = BlockCommitment::default();
-            expected.increase_rooted_stake(lamports);
-            assert_eq!(*commitment.get(&a).unwrap(), expected);
-        }
-        assert_eq!(rooted_stake[0], (root, lamports));
-    }
-
-    #[test]
-    fn test_aggregate_commitment_for_vote_account_2() {
-        let ancestors = vec![3, 4, 5, 7, 9, 11];
-        let mut commitment = HashMap::new();
-        let mut rooted_stake = vec![];
-        let lamports = 5;
-        let mut vote_state = VoteState::default();
-
-        let root = ancestors[2];
-        vote_state.root_slot = Some(root);
-        vote_state.process_slot_vote_unchecked(*ancestors.last().unwrap());
-        AggregateCommitmentService::aggregate_commitment_for_vote_account(
-            &mut commitment,
-            &mut rooted_stake,
-            &vote_state,
-            &ancestors,
-            lamports,
-        );
-
-        for a in ancestors {
-            if a <= root {
-                let mut expected = BlockCommitment::default();
-                expected.increase_rooted_stake(lamports);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            } else {
-                let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(1, lamports);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            }
-        }
-        assert_eq!(rooted_stake[0], (root, lamports));
-    }
-
-    #[test]
-    fn test_aggregate_commitment_for_vote_account_3() {
-        let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
-        let mut commitment = HashMap::new();
-        let mut rooted_stake = vec![];
-        let lamports = 5;
-        let mut vote_state = VoteState::default();
-
-        let root = ancestors[2];
-        vote_state.root_slot = Some(root);
-        assert!(ancestors[4] + 2 >= ancestors[6]);
-        vote_state.process_slot_vote_unchecked(ancestors[4]);
-        vote_state.process_slot_vote_unchecked(ancestors[6]);
-        AggregateCommitmentService::aggregate_commitment_for_vote_account(
-            &mut commitment,
-            &mut rooted_stake,
-            &vote_state,
-            &ancestors,
-            lamports,
-        );
-
-        for (i, a) in ancestors.iter().enumerate() {
-            if *a <= root {
-                let mut expected = BlockCommitment::default();
-                expected.increase_rooted_stake(lamports);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            } else if i <= 4 {
-                let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(2, lamports);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            } else if i <= 6 {
-                let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(1, lamports);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            }
-        }
-        assert_eq!(rooted_stake[0], (root, lamports));
-    }
-
-    #[test]
-    fn test_aggregate_commitment_validity() {
-        let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
-        let GenesisConfigInfo {
-            mut genesis_config, ..
-        } = create_genesis_config(10_000);
-
-        let rooted_stake_amount = 40;
-
-        let sk1 = Pubkey::new_rand();
-        let pk1 = Pubkey::new_rand();
-        let mut vote_account1 = vote_state::create_account(&pk1, &Pubkey::new_rand(), 0, 100);
-        let stake_account1 =
-            stake_state::create_account(&sk1, &pk1, &vote_account1, &genesis_config.rent, 100);
-        let sk2 = Pubkey::new_rand();
-        let pk2 = Pubkey::new_rand();
-        let mut vote_account2 = vote_state::create_account(&pk2, &Pubkey::new_rand(), 0, 50);
-        let stake_account2 =
-            stake_state::create_account(&sk2, &pk2, &vote_account2, &genesis_config.rent, 50);
-        let sk3 = Pubkey::new_rand();
-        let pk3 = Pubkey::new_rand();
-        let mut vote_account3 = vote_state::create_account(&pk3, &Pubkey::new_rand(), 0, 1);
-        let stake_account3 = stake_state::create_account(
-            &sk3,
-            &pk3,
-            &vote_account3,
-            &genesis_config.rent,
-            rooted_stake_amount,
-        );
-        let sk4 = Pubkey::new_rand();
-        let pk4 = Pubkey::new_rand();
-        let mut vote_account4 = vote_state::create_account(&pk4, &Pubkey::new_rand(), 0, 1);
-        let stake_account4 = stake_state::create_account(
-            &sk4,
-            &pk4,
-            &vote_account4,
-            &genesis_config.rent,
-            rooted_stake_amount,
-        );
-
-        genesis_config.accounts.extend(vec![
-            (pk1, vote_account1.clone()),
-            (sk1, stake_account1),
-            (pk2, vote_account2.clone()),
-            (sk2, stake_account2),
-            (pk3, vote_account3.clone()),
-            (sk3, stake_account3),
-            (pk4, vote_account4.clone()),
-            (sk4, stake_account4),
-        ]);
-
-        // Create bank
-        let bank = Arc::new(Bank::new(&genesis_config));
-
-        let mut vote_state1 = VoteState::from(&vote_account1).unwrap();
-        vote_state1.process_slot_vote_unchecked(3);
-        vote_state1.process_slot_vote_unchecked(5);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state1));
-        VoteState::to(&versioned, &mut vote_account1).unwrap();
-        bank.store_account(&pk1, &vote_account1);
-
-        let mut vote_state2 = VoteState::from(&vote_account2).unwrap();
-        vote_state2.process_slot_vote_unchecked(9);
-        vote_state2.process_slot_vote_unchecked(10);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state2));
-        VoteState::to(&versioned, &mut vote_account2).unwrap();
-        bank.store_account(&pk2, &vote_account2);
-
-        let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
-        vote_state3.root_slot = Some(1);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
-        VoteState::to(&versioned, &mut vote_account3).unwrap();
-        bank.store_account(&pk3, &vote_account3);
-
-        let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
-        vote_state4.root_slot = Some(2);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
-        VoteState::to(&versioned, &mut vote_account4).unwrap();
-        bank.store_account(&pk4, &vote_account4);
-
-        let (commitment, rooted_stake) =
-            AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
-
-        for a in ancestors {
-            if a <= 3 {
-                let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(2, 150);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            } else if a <= 5 {
-                let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(1, 100);
-                expected.increase_confirmation_stake(2, 50);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            } else if a <= 9 {
-                let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(2, 50);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            } else if a <= 10 {
-                let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(1, 50);
-                assert_eq!(*commitment.get(&a).unwrap(), expected);
-            } else {
-                assert!(commitment.get(&a).is_none());
-            }
-        }
-        assert_eq!(rooted_stake.len(), 2);
-        assert_eq!(get_largest_confirmed_root(rooted_stake, 100), 1)
-    }
 }

core/src/commitment_service.rs

@@ -0,0 +1,454 @@
+use crate::{
+    commitment::{BlockCommitment, BlockCommitmentCache, VOTE_THRESHOLD_SIZE},
+    rpc_subscriptions::{CacheSlotInfo, RpcSubscriptions},
+};
+use solana_measure::measure::Measure;
+use solana_metrics::datapoint_info;
+use solana_runtime::bank::Bank;
+use solana_sdk::clock::Slot;
+use solana_vote_program::vote_state::VoteState;
+use std::{
+    collections::HashMap,
+    sync::atomic::{AtomicBool, Ordering},
+    sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender},
+    sync::{Arc, RwLock},
+    thread::{self, Builder, JoinHandle},
+    time::Duration,
+};
+
+pub struct CommitmentAggregationData {
+    bank: Arc<Bank>,
+    root: Slot,
+    total_staked: u64,
+}
+
+impl CommitmentAggregationData {
+    pub fn new(bank: Arc<Bank>, root: Slot, total_staked: u64) -> Self {
+        Self {
+            bank,
+            root,
+            total_staked,
+        }
+    }
+}
+
+fn get_largest_confirmed_root(mut rooted_stake: Vec<(Slot, u64)>, total_stake: u64) -> Slot {
+    rooted_stake.sort_by(|a, b| a.0.cmp(&b.0).reverse());
+    let mut stake_sum = 0;
+    for (root, stake) in rooted_stake {
+        stake_sum += stake;
+        if (stake_sum as f64 / total_stake as f64) > VOTE_THRESHOLD_SIZE {
+            return root;
+        }
+    }
+    0
+}
+
+pub struct AggregateCommitmentService {
+    t_commitment: JoinHandle<()>,
+}
+
+impl AggregateCommitmentService {
+    pub fn new(
+        exit: &Arc<AtomicBool>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        subscriptions: Arc<RpcSubscriptions>,
+    ) -> (Sender<CommitmentAggregationData>, Self) {
+        let (sender, receiver): (
+            Sender<CommitmentAggregationData>,
+            Receiver<CommitmentAggregationData>,
+        ) = channel();
+        let exit_ = exit.clone();
+        (
+            sender,
+            Self {
+                t_commitment: Builder::new()
+                    .name("solana-aggregate-stake-lockouts".to_string())
+                    .spawn(move || loop {
+                        if exit_.load(Ordering::Relaxed) {
+                            break;
+                        }
+
+                        if let Err(RecvTimeoutError::Disconnected) =
+                            Self::run(&receiver, &block_commitment_cache, &subscriptions, &exit_)
+                        {
+                            break;
+                        }
+                    })
+                    .unwrap(),
+            },
+        )
+    }
+
+    fn run(
+        receiver: &Receiver<CommitmentAggregationData>,
+        block_commitment_cache: &RwLock<BlockCommitmentCache>,
+        subscriptions: &Arc<RpcSubscriptions>,
+        exit: &Arc<AtomicBool>,
+    ) -> Result<(), RecvTimeoutError> {
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                return Ok(());
+            }
+
+            let mut aggregation_data = receiver.recv_timeout(Duration::from_secs(1))?;
+
+            while let Ok(new_data) = receiver.try_recv() {
+                aggregation_data = new_data;
+            }
+
+            let ancestors = aggregation_data.bank.status_cache_ancestors();
+            if ancestors.is_empty() {
+                continue;
+            }
+
+            let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
+            let (block_commitment, rooted_stake) =
+                Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+
+            let largest_confirmed_root =
+                get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
+
+            let mut new_block_commitment = BlockCommitmentCache::new(
+                block_commitment,
+                largest_confirmed_root,
+                aggregation_data.total_staked,
+                aggregation_data.bank,
+                block_commitment_cache.read().unwrap().blockstore.clone(),
+                aggregation_data.root,
+                aggregation_data.root,
+            );
+            new_block_commitment.highest_confirmed_slot =
+                new_block_commitment.calculate_highest_confirmed_slot();
+
+            let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+
+            std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+            aggregate_commitment_time.stop();
+            datapoint_info!(
+                "block-commitment-cache",
+                (
+                    "aggregate-commitment-ms",
+                    aggregate_commitment_time.as_ms() as i64,
+                    i64
+                )
+            );
+
+            subscriptions.notify_subscribers(CacheSlotInfo {
+                current_slot: w_block_commitment_cache.slot(),
+                node_root: w_block_commitment_cache.root(),
+                largest_confirmed_root: w_block_commitment_cache.largest_confirmed_root(),
+                highest_confirmed_slot: w_block_commitment_cache.highest_confirmed_slot(),
+            });
+        }
+    }
+
+    pub fn aggregate_commitment(
+        ancestors: &[Slot],
+        bank: &Bank,
+    ) -> (HashMap<Slot, BlockCommitment>, Vec<(Slot, u64)>) {
+        assert!(!ancestors.is_empty());
+
+        // Check ancestors is sorted
+        for a in ancestors.windows(2) {
+            assert!(a[0] < a[1]);
+        }
+
+        let mut commitment = HashMap::new();
+        let mut rooted_stake: Vec<(Slot, u64)> = Vec::new();
+        for (_, (lamports, account)) in bank.vote_accounts().into_iter() {
+            if lamports == 0 {
+                continue;
+            }
+            let vote_state = VoteState::from(&account);
+            if vote_state.is_none() {
+                continue;
+            }
+
+            let vote_state = vote_state.unwrap();
+            Self::aggregate_commitment_for_vote_account(
+                &mut commitment,
+                &mut rooted_stake,
+                &vote_state,
+                ancestors,
+                lamports,
+            );
+        }
+
+        (commitment, rooted_stake)
+    }
+
+    fn aggregate_commitment_for_vote_account(
+        commitment: &mut HashMap<Slot, BlockCommitment>,
+        rooted_stake: &mut Vec<(Slot, u64)>,
+        vote_state: &VoteState,
+        ancestors: &[Slot],
+        lamports: u64,
+    ) {
+        assert!(!ancestors.is_empty());
+        let mut ancestors_index = 0;
+        if let Some(root) = vote_state.root_slot {
+            for (i, a) in ancestors.iter().enumerate() {
+                if *a <= root {
+                    commitment
+                        .entry(*a)
+                        .or_insert_with(BlockCommitment::default)
+                        .increase_rooted_stake(lamports);
+                } else {
+                    ancestors_index = i;
+                    break;
+                }
+            }
+            rooted_stake.push((root, lamports));
+        }
+
+        for vote in &vote_state.votes {
+            while ancestors[ancestors_index] <= vote.slot {
+                commitment
+                    .entry(ancestors[ancestors_index])
+                    .or_insert_with(BlockCommitment::default)
+                    .increase_confirmation_stake(vote.confirmation_count as usize, lamports);
+                ancestors_index += 1;
+
+                if ancestors_index == ancestors.len() {
+                    return;
+                }
+            }
+        }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_commitment.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_sdk::pubkey::Pubkey;
+    use solana_stake_program::stake_state;
+    use solana_vote_program::vote_state::{self, VoteStateVersions};
+
+    #[test]
+    fn test_get_largest_confirmed_root() {
+        assert_eq!(get_largest_confirmed_root(vec![], 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((0, 5));
+        rooted_stake.push((1, 5));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((1, 5));
+        rooted_stake.push((0, 10));
+        rooted_stake.push((2, 5));
+        rooted_stake.push((1, 4));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 1);
+    }
+
+    #[test]
+    fn test_aggregate_commitment_for_vote_account_1() {
+        let ancestors = vec![3, 4, 5, 7, 9, 11];
+        let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
+        let lamports = 5;
+        let mut vote_state = VoteState::default();
+
+        let root = *ancestors.last().unwrap();
+        vote_state.root_slot = Some(root);
+        AggregateCommitmentService::aggregate_commitment_for_vote_account(
+            &mut commitment,
+            &mut rooted_stake,
+            &vote_state,
+            &ancestors,
+            lamports,
+        );
+
+        for a in ancestors {
+            let mut expected = BlockCommitment::default();
+            expected.increase_rooted_stake(lamports);
+            assert_eq!(*commitment.get(&a).unwrap(), expected);
+        }
+        assert_eq!(rooted_stake[0], (root, lamports));
+    }
+
+    #[test]
+    fn test_aggregate_commitment_for_vote_account_2() {
+        let ancestors = vec![3, 4, 5, 7, 9, 11];
+        let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
+        let lamports = 5;
+        let mut vote_state = VoteState::default();
+
+        let root = ancestors[2];
+        vote_state.root_slot = Some(root);
+        vote_state.process_slot_vote_unchecked(*ancestors.last().unwrap());
+        AggregateCommitmentService::aggregate_commitment_for_vote_account(
+            &mut commitment,
+            &mut rooted_stake,
+            &vote_state,
+            &ancestors,
+            lamports,
+        );
+
+        for a in ancestors {
+            if a <= root {
+                let mut expected = BlockCommitment::default();
+                expected.increase_rooted_stake(lamports);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            } else {
+                let mut expected = BlockCommitment::default();
+                expected.increase_confirmation_stake(1, lamports);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            }
+        }
+        assert_eq!(rooted_stake[0], (root, lamports));
+    }
+
+    #[test]
+    fn test_aggregate_commitment_for_vote_account_3() {
+        let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
+        let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
+        let lamports = 5;
+        let mut vote_state = VoteState::default();
+
+        let root = ancestors[2];
+        vote_state.root_slot = Some(root);
+        assert!(ancestors[4] + 2 >= ancestors[6]);
+        vote_state.process_slot_vote_unchecked(ancestors[4]);
+        vote_state.process_slot_vote_unchecked(ancestors[6]);
+        AggregateCommitmentService::aggregate_commitment_for_vote_account(
+            &mut commitment,
+            &mut rooted_stake,
+            &vote_state,
+            &ancestors,
+            lamports,
+        );
+
+        for (i, a) in ancestors.iter().enumerate() {
+            if *a <= root {
+                let mut expected = BlockCommitment::default();
+                expected.increase_rooted_stake(lamports);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            } else if i <= 4 {
+                let mut expected = BlockCommitment::default();
+                expected.increase_confirmation_stake(2, lamports);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            } else if i <= 6 {
+                let mut expected = BlockCommitment::default();
+                expected.increase_confirmation_stake(1, lamports);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            }
+        }
+        assert_eq!(rooted_stake[0], (root, lamports));
+    }
+
+    #[test]
+    fn test_aggregate_commitment_validity() {
+        let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
+        let GenesisConfigInfo {
+            mut genesis_config, ..
+        } = create_genesis_config(10_000);
+
+        let rooted_stake_amount = 40;
+
+        let sk1 = Pubkey::new_rand();
+        let pk1 = Pubkey::new_rand();
+        let mut vote_account1 = vote_state::create_account(&pk1, &Pubkey::new_rand(), 0, 100);
+        let stake_account1 =
+            stake_state::create_account(&sk1, &pk1, &vote_account1, &genesis_config.rent, 100);
+        let sk2 = Pubkey::new_rand();
+        let pk2 = Pubkey::new_rand();
+        let mut vote_account2 = vote_state::create_account(&pk2, &Pubkey::new_rand(), 0, 50);
+        let stake_account2 =
+            stake_state::create_account(&sk2, &pk2, &vote_account2, &genesis_config.rent, 50);
+        let sk3 = Pubkey::new_rand();
+        let pk3 = Pubkey::new_rand();
+        let mut vote_account3 = vote_state::create_account(&pk3, &Pubkey::new_rand(), 0, 1);
+        let stake_account3 = stake_state::create_account(
+            &sk3,
+            &pk3,
+            &vote_account3,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
+        let sk4 = Pubkey::new_rand();
+        let pk4 = Pubkey::new_rand();
+        let mut vote_account4 = vote_state::create_account(&pk4, &Pubkey::new_rand(), 0, 1);
+        let stake_account4 = stake_state::create_account(
+            &sk4,
+            &pk4,
+            &vote_account4,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
+
+        genesis_config.accounts.extend(vec![
+            (pk1, vote_account1.clone()),
+            (sk1, stake_account1),
+            (pk2, vote_account2.clone()),
+            (sk2, stake_account2),
+            (pk3, vote_account3.clone()),
+            (sk3, stake_account3),
+            (pk4, vote_account4.clone()),
+            (sk4, stake_account4),
+        ]);
+
+        // Create bank
+        let bank = Arc::new(Bank::new(&genesis_config));
+
+        let mut vote_state1 = VoteState::from(&vote_account1).unwrap();
+        vote_state1.process_slot_vote_unchecked(3);
+        vote_state1.process_slot_vote_unchecked(5);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state1));
+        VoteState::to(&versioned, &mut vote_account1).unwrap();
+        bank.store_account(&pk1, &vote_account1);
+
+        let mut vote_state2 = VoteState::from(&vote_account2).unwrap();
+        vote_state2.process_slot_vote_unchecked(9);
+        vote_state2.process_slot_vote_unchecked(10);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state2));
+        VoteState::to(&versioned, &mut vote_account2).unwrap();
+        bank.store_account(&pk2, &vote_account2);
+
+        let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
+        vote_state3.root_slot = Some(1);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
+        VoteState::to(&versioned, &mut vote_account3).unwrap();
+        bank.store_account(&pk3, &vote_account3);
+
+        let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
+        vote_state4.root_slot = Some(2);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
+        VoteState::to(&versioned, &mut vote_account4).unwrap();
+        bank.store_account(&pk4, &vote_account4);
+
+        let (commitment, rooted_stake) =
+            AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
+
+        for a in ancestors {
+            if a <= 3 {
+                let mut expected = BlockCommitment::default();
+                expected.increase_confirmation_stake(2, 150);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            } else if a <= 5 {
+                let mut expected = BlockCommitment::default();
+                expected.increase_confirmation_stake(1, 100);
+                expected.increase_confirmation_stake(2, 50);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            } else if a <= 9 {
+                let mut expected = BlockCommitment::default();
+                expected.increase_confirmation_stake(2, 50);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            } else if a <= 10 {
+                let mut expected = BlockCommitment::default();
+                expected.increase_confirmation_stake(1, 50);
+                assert_eq!(*commitment.get(&a).unwrap(), expected);
+            } else {
+                assert!(commitment.get(&a).is_none());
+            }
+        }
+        assert_eq!(rooted_stake.len(), 2);
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 100), 1)
+    }
+}

core/src/consensus.rs

@@ -1,4 +1,5 @@
 use crate::{
+    commitment::VOTE_THRESHOLD_SIZE,
     progress_map::{LockoutIntervals, ProgressMap},
     pubkey_references::PubkeyReferences,
 };
@@ -9,19 +10,55 @@ use solana_sdk::{
     account::Account,
     clock::{Slot, UnixTimestamp},
     hash::Hash,
+    instruction::Instruction,
     pubkey::Pubkey,
 };
-use solana_vote_program::vote_state::{
-    BlockTimestamp, Lockout, Vote, VoteState, MAX_LOCKOUT_HISTORY, TIMESTAMP_SLOT_INTERVAL,
+use solana_vote_program::{
+    vote_instruction,
+    vote_state::{
+        BlockTimestamp, Lockout, Vote, VoteState, MAX_LOCKOUT_HISTORY, TIMESTAMP_SLOT_INTERVAL,
+    },
 };
 use std::{
     collections::{BTreeMap, HashMap, HashSet},
     ops::Bound::{Included, Unbounded},
-    sync::Arc,
+    sync::{Arc, RwLock},
 };
 
+#[derive(PartialEq, Clone, Debug)]
+pub enum SwitchForkDecision {
+    SwitchProof(Hash),
+    NoSwitch,
+    FailedSwitchThreshold,
+}
+
+impl SwitchForkDecision {
+    pub fn to_vote_instruction(
+        &self,
+        vote: Vote,
+        vote_account_pubkey: &Pubkey,
+        authorized_voter_pubkey: &Pubkey,
+    ) -> Option<Instruction> {
+        match self {
+            SwitchForkDecision::FailedSwitchThreshold => None,
+            SwitchForkDecision::NoSwitch => Some(vote_instruction::vote(
+                vote_account_pubkey,
+                authorized_voter_pubkey,
+                vote,
+            )),
+            SwitchForkDecision::SwitchProof(switch_proof_hash) => {
+                Some(vote_instruction::vote_switch(
+                    vote_account_pubkey,
+                    authorized_voter_pubkey,
+                    vote,
+                    *switch_proof_hash,
+                ))
+            }
+        }
+    }
+}
+
 pub const VOTE_THRESHOLD_DEPTH: usize = 8;
-pub const VOTE_THRESHOLD_SIZE: f64 = 2f64 / 3f64;
 pub const SWITCH_FORK_THRESHOLD: f64 = 0.38;
 
 #[derive(Default, Debug, Clone)]
@@ -42,6 +79,14 @@ impl StakeLockout {
     }
 }
 
+pub(crate) struct ComputedBankState {
+    pub stake_lockouts: HashMap<Slot, StakeLockout>,
+    pub total_staked: u64,
+    pub bank_weight: u128,
+    pub lockout_intervals: LockoutIntervals,
+    pub pubkey_votes: Vec<(Pubkey, Slot)>,
+}
+
 pub struct Tower {
     node_pubkey: Pubkey,
     threshold_depth: usize,
@@ -65,10 +110,14 @@ impl Default for Tower {
 }
 
 impl Tower {
-    pub fn new(node_pubkey: &Pubkey, vote_account_pubkey: &Pubkey, bank_forks: &BankForks) -> Self {
+    pub fn new(
+        node_pubkey: &Pubkey,
+        vote_account_pubkey: &Pubkey,
+        root: Slot,
+        heaviest_bank: &Bank,
+    ) -> Self {
         let mut tower = Self::new_with_key(node_pubkey);
-
-        tower.initialize_lockouts_from_bank_forks(&bank_forks, vote_account_pubkey);
+        tower.initialize_lockouts_from_bank_forks(vote_account_pubkey, root, heaviest_bank);
 
         tower
     }
@@ -89,27 +138,28 @@ impl Tower {
         }
     }
 
-    pub fn collect_vote_lockouts<F>(
-        &self,
+    pub(crate) fn collect_vote_lockouts<F>(
+        node_pubkey: &Pubkey,
         bank_slot: u64,
         vote_accounts: F,
         ancestors: &HashMap<Slot, HashSet<u64>>,
         all_pubkeys: &mut PubkeyReferences,
-    ) -> (HashMap<Slot, StakeLockout>, u64, u128, LockoutIntervals)
+    ) -> ComputedBankState
     where
         F: Iterator<Item = (Pubkey, (u64, Account))>,
     {
         let mut stake_lockouts = HashMap::new();
-        let mut total_stake = 0;
-        let mut total_weight = 0;
+        let mut total_staked = 0;
+        let mut bank_weight = 0;
         // Tree of intervals of lockouts of the form [slot, slot + slot.lockout],
         // keyed by end of the range
         let mut lockout_intervals = BTreeMap::new();
+        let mut pubkey_votes = vec![];
         for (key, (lamports, account)) in vote_accounts {
             if lamports == 0 {
                 continue;
             }
-            trace!("{} {} with stake {}", self.node_pubkey, key, lamports);
+            trace!("{} {} with stake {}", node_pubkey, key, lamports);
             let vote_state = VoteState::from(&account);
             if vote_state.is_none() {
                 datapoint_warn!(
@@ -128,11 +178,11 @@ impl Tower {
                 let key = all_pubkeys.get_or_insert(&key);
                 lockout_intervals
                     .entry(vote.expiration_slot())
-                    .or_insert_with(|| vec![])
+                    .or_insert_with(Vec::new)
                     .push((vote.slot, key));
             }
 
-            if key == self.node_pubkey || vote_state.node_pubkey == self.node_pubkey {
+            if key == *node_pubkey || vote_state.node_pubkey == *node_pubkey {
                 debug!("vote state {:?}", vote_state);
                 debug!(
                     "observed slot {}",
@@ -151,10 +201,15 @@ impl Tower {
             }
             let start_root = vote_state.root_slot;
 
+            // Add the latest vote to update the `heaviest_subtree_fork_choice`
+            if let Some(latest_vote) = vote_state.votes.back() {
+                pubkey_votes.push((key, latest_vote.slot));
+            }
+
             vote_state.process_slot_vote_unchecked(bank_slot);
 
             for vote in &vote_state.votes {
-                total_weight += vote.lockout() as u128 * lamports as u128;
+                bank_weight += vote.lockout() as u128 * lamports as u128;
                 Self::update_ancestor_lockouts(&mut stake_lockouts, &vote, ancestors);
             }
 
@@ -165,7 +220,7 @@ impl Tower {
                         slot: root,
                     };
                     trace!("ROOT: {}", vote.slot);
-                    total_weight += vote.lockout() as u128 * lamports as u128;
+                    bank_weight += vote.lockout() as u128 * lamports as u128;
                     Self::update_ancestor_lockouts(&mut stake_lockouts, &vote, ancestors);
                 }
             }
@@ -174,7 +229,7 @@ impl Tower {
                     confirmation_count: MAX_LOCKOUT_HISTORY as u32,
                     slot: root,
                 };
-                total_weight += vote.lockout() as u128 * lamports as u128;
+                bank_weight += vote.lockout() as u128 * lamports as u128;
                 Self::update_ancestor_lockouts(&mut stake_lockouts, &vote, ancestors);
             }
 
@@ -195,9 +250,16 @@ impl Tower {
                 // Update all the parents of this last vote with the stake of this vote account
                 Self::update_ancestor_stakes(&mut stake_lockouts, vote.slot, lamports, ancestors);
             }
-            total_stake += lamports;
+            total_staked += lamports;
+        }
+
+        ComputedBankState {
+            stake_lockouts,
+            total_staked,
+            bank_weight,
+            lockout_intervals,
+            pubkey_votes,
         }
-        (stake_lockouts, total_stake, total_weight, lockout_intervals)
     }
 
     pub fn is_slot_confirmed(
@@ -345,7 +407,7 @@ impl Tower {
         progress: &ProgressMap,
         total_stake: u64,
         epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
-    ) -> bool {
+    ) -> SwitchForkDecision {
         self.last_vote()
             .slots
             .last()
@@ -355,14 +417,18 @@ impl Tower {
 
                 if switch_slot == *last_vote || switch_slot_ancestors.contains(last_vote) {
                     // If the `switch_slot is a descendant of the last vote,
-                    // no switching proof is neceessary
-                    return true;
+                    // no switching proof is necessary
+                    return SwitchForkDecision::NoSwitch;
                 }
 
                 // Should never consider switching to an ancestor
                 // of your last vote
                 assert!(!last_vote_ancestors.contains(&switch_slot));
 
+                // By this point, we know the `switch_slot` is on a different fork
+                // (is neither an ancestor nor descendant of `last_vote`), so a
+                // switching proof is necessary
+                let switch_proof = Hash::default();
                 let mut locked_out_stake = 0;
                 let mut locked_out_vote_accounts = HashSet::new();
                 for (candidate_slot, descendants) in descendants.iter() {
@@ -423,9 +489,14 @@ impl Tower {
                         }
                     }
                 }
-                (locked_out_stake as f64 / total_stake as f64) > SWITCH_FORK_THRESHOLD
+
+                if (locked_out_stake as f64 / total_stake as f64) > SWITCH_FORK_THRESHOLD {
+                    SwitchForkDecision::SwitchProof(switch_proof)
+                } else {
+                    SwitchForkDecision::FailedSwitchThreshold
+                }
             })
-            .unwrap_or(true)
+            .unwrap_or(SwitchForkDecision::NoSwitch)
     }
 
     pub fn check_vote_stake_threshold(
@@ -463,7 +534,7 @@ impl Tower {
     }
 
     /// Update lockouts for all the ancestors
-    fn update_ancestor_lockouts(
+    pub(crate) fn update_ancestor_lockouts(
         stake_lockouts: &mut HashMap<Slot, StakeLockout>,
         vote: &Lockout,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
@@ -483,6 +554,28 @@ impl Tower {
         }
     }
 
+    pub(crate) fn find_heaviest_bank(
+        bank_forks: &RwLock<BankForks>,
+        node_pubkey: &Pubkey,
+    ) -> Option<Arc<Bank>> {
+        let ancestors = bank_forks.read().unwrap().ancestors();
+        let mut bank_weights: Vec<_> = bank_forks
+            .read()
+            .unwrap()
+            .frozen_banks()
+            .values()
+            .map(|b| {
+                (
+                    Self::bank_weight(node_pubkey, b, &ancestors),
+                    b.parents().len(),
+                    b.clone(),
+                )
+            })
+            .collect();
+        bank_weights.sort_by_key(|b| (b.0, b.1));
+        bank_weights.pop().map(|b| b.2)
+    }
+
     /// Update stake for all the ancestors.
     /// Note, stake is the same for all the ancestor.
     fn update_ancestor_stakes(
@@ -491,9 +584,8 @@ impl Tower {
         lamports: u64,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
     ) {
-        // If there's no ancestors, that means this slot must be from before the current root,
-        // in which case the lockouts won't be calculated in bank_weight anyways, so ignore
-        // this slot
+        // If there's no ancestors, that means this slot must be from
+        // before the current root, so ignore this slot
         let vote_slot_ancestors = ancestors.get(&slot);
         if vote_slot_ancestors.is_none() {
             return;
@@ -506,8 +598,13 @@ impl Tower {
         }
     }
 
-    fn bank_weight(&self, bank: &Bank, ancestors: &HashMap<Slot, HashSet<Slot>>) -> u128 {
-        let (_, _, bank_weight, _) = self.collect_vote_lockouts(
+    fn bank_weight(
+        node_pubkey: &Pubkey,
+        bank: &Bank,
+        ancestors: &HashMap<Slot, HashSet<Slot>>,
+    ) -> u128 {
+        let ComputedBankState { bank_weight, .. } = Self::collect_vote_lockouts(
+            node_pubkey,
             bank.slot(),
             bank.vote_accounts().into_iter(),
             ancestors,
@@ -516,47 +613,28 @@ impl Tower {
         bank_weight
     }
 
-    fn find_heaviest_bank(&self, bank_forks: &BankForks) -> Option<Arc<Bank>> {
-        let ancestors = bank_forks.ancestors();
-        let mut bank_weights: Vec<_> = bank_forks
-            .frozen_banks()
-            .values()
-            .map(|b| {
-                (
-                    self.bank_weight(b, &ancestors),
-                    b.parents().len(),
-                    b.clone(),
-                )
-            })
-            .collect();
-        bank_weights.sort_by_key(|b| (b.0, b.1));
-        bank_weights.pop().map(|b| b.2)
-    }
-
     fn initialize_lockouts_from_bank_forks(
         &mut self,
-        bank_forks: &BankForks,
         vote_account_pubkey: &Pubkey,
+        root: Slot,
+        heaviest_bank: &Bank,
     ) {
-        if let Some(bank) = self.find_heaviest_bank(bank_forks) {
-            let root = bank_forks.root();
-            if let Some((_stake, vote_account)) = bank.vote_accounts().get(vote_account_pubkey) {
-                let mut vote_state = VoteState::deserialize(&vote_account.data)
-                    .expect("vote_account isn't a VoteState?");
-                vote_state.root_slot = Some(root);
-                vote_state.votes.retain(|v| v.slot > root);
-                trace!(
-                    "{} lockouts initialized to {:?}",
-                    self.node_pubkey,
-                    vote_state
-                );
-
-                assert_eq!(
-                    vote_state.node_pubkey, self.node_pubkey,
-                    "vote account's node_pubkey doesn't match",
-                );
-                self.lockouts = vote_state;
-            }
+        if let Some((_stake, vote_account)) = heaviest_bank.vote_accounts().get(vote_account_pubkey)
+        {
+            let mut vote_state = VoteState::deserialize(&vote_account.data)
+                .expect("vote_account isn't a VoteState?");
+            vote_state.root_slot = Some(root);
+            vote_state.votes.retain(|v| v.slot > root);
+            trace!(
+                "{} lockouts initialized to {:?}",
+                self.node_pubkey,
+                vote_state
+            );
+            assert_eq!(
+                vote_state.node_pubkey, self.node_pubkey,
+                "vote account's node_pubkey doesn't match",
+            );
+            self.lockouts = vote_state;
         }
     }
 
@@ -580,8 +658,11 @@ impl Tower {
 pub mod test {
     use super::*;
     use crate::{
+        bank_weight_fork_choice::BankWeightForkChoice,
         cluster_info_vote_listener::VoteTracker,
         cluster_slots::ClusterSlots,
+        fork_choice::SelectVoteAndResetForkResult,
+        heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
         progress_map::ForkProgress,
         replay_stage::{HeaviestForkFailures, ReplayStage},
     };
@@ -599,7 +680,7 @@ pub mod test {
         signature::{Keypair, Signer},
     };
     use solana_vote_program::{
-        vote_state::{Vote, VoteStateVersions},
+        vote_state::{Vote, VoteStateVersions, MAX_LOCKOUT_HISTORY},
         vote_transaction,
     };
     use std::{
@@ -616,18 +697,26 @@ pub mod test {
         pub vote_pubkeys: Vec<Pubkey>,
         pub bank_forks: RwLock<BankForks>,
         pub progress: ProgressMap,
+        pub heaviest_subtree_fork_choice: HeaviestSubtreeForkChoice,
     }
 
     impl VoteSimulator {
         pub(crate) fn new(num_keypairs: usize) -> Self {
-            let (validator_keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress) =
-                Self::init_state(num_keypairs);
+            let (
+                validator_keypairs,
+                node_pubkeys,
+                vote_pubkeys,
+                bank_forks,
+                progress,
+                heaviest_subtree_fork_choice,
+            ) = Self::init_state(num_keypairs);
             Self {
                 validator_keypairs,
                 node_pubkeys,
                 vote_pubkeys,
                 bank_forks: RwLock::new(bank_forks),
                 progress,
+                heaviest_subtree_fork_choice,
             }
         }
         pub(crate) fn fill_bank_forks(
@@ -670,6 +759,8 @@ pub mod test {
                     }
                 }
                 new_bank.freeze();
+                self.heaviest_subtree_fork_choice
+                    .add_new_leaf_slot(new_bank.slot(), Some(new_bank.parent_slot()));
                 self.bank_forks.write().unwrap().insert(new_bank);
                 walk.forward();
             }
@@ -704,6 +795,8 @@ pub mod test {
                 &ClusterSlots::default(),
                 &self.bank_forks,
                 &mut PubkeyReferences::default(),
+                &mut self.heaviest_subtree_fork_choice,
+                &mut BankWeightForkChoice::default(),
             );
 
             let vote_bank = self
@@ -716,8 +809,11 @@ pub mod test {
 
             // Try to vote on the given slot
             let descendants = self.bank_forks.read().unwrap().descendants();
-            let (_, _, failure_reasons) = ReplayStage::select_vote_and_reset_forks(
-                &Some(vote_bank.clone()),
+            let SelectVoteAndResetForkResult {
+                heaviest_fork_failures,
+                ..
+            } = ReplayStage::select_vote_and_reset_forks(
+                &vote_bank,
                 &None,
                 &ancestors,
                 &descendants,
@@ -727,8 +823,8 @@ pub mod test {
 
             // Make sure this slot isn't locked out or failing threshold
             info!("Checking vote: {}", vote_bank.slot());
-            if !failure_reasons.is_empty() {
-                return failure_reasons;
+            if !heaviest_fork_failures.is_empty() {
+                return heaviest_fork_failures;
             }
             let vote = tower.new_vote_from_bank(&vote_bank, &my_vote_pubkey).0;
             if let Some(new_root) = tower.record_bank_vote(vote) {
@@ -746,6 +842,7 @@ pub mod test {
                 &None,
                 &mut PubkeyReferences::default(),
                 None,
+                &mut self.heaviest_subtree_fork_choice,
             )
         }
 
@@ -831,6 +928,7 @@ pub mod test {
             Vec<Pubkey>,
             BankForks,
             ProgressMap,
+            HeaviestSubtreeForkChoice,
         ) {
             let keypairs: HashMap<_, _> = std::iter::repeat_with(|| {
                 let node_keypair = Keypair::new();
@@ -853,8 +951,16 @@ pub mod test {
                 .map(|keys| keys.vote_keypair.pubkey())
                 .collect();
 
-            let (bank_forks, progress) = initialize_state(&keypairs, 10_000);
-            (keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress)
+            let (bank_forks, progress, heaviest_subtree_fork_choice) =
+                initialize_state(&keypairs, 10_000);
+            (
+                keypairs,
+                node_pubkeys,
+                vote_pubkeys,
+                bank_forks,
+                progress,
+                heaviest_subtree_fork_choice,
+            )
         }
     }
 
@@ -862,7 +968,7 @@ pub mod test {
     pub(crate) fn initialize_state(
         validator_keypairs_map: &HashMap<Pubkey, ValidatorVoteKeypairs>,
         stake: u64,
-    ) -> (BankForks, ProgressMap) {
+    ) -> (BankForks, ProgressMap, HeaviestSubtreeForkChoice) {
         let validator_keypairs: Vec<_> = validator_keypairs_map.values().collect();
         let GenesisConfigInfo {
             genesis_config,
@@ -882,7 +988,10 @@ pub mod test {
             0,
             ForkProgress::new(bank0.last_blockhash(), None, None, 0, 0),
         );
-        (BankForks::new(0, bank0), progress)
+        let bank_forks = BankForks::new(bank0);
+        let heaviest_subtree_fork_choice =
+            HeaviestSubtreeForkChoice::new_from_bank_forks(&bank_forks);
+        (bank_forks, progress, heaviest_subtree_fork_choice)
     }
 
     fn gen_stakes(stake_votes: &[(u64, &[u64])]) -> Vec<(Pubkey, (u64, Account))> {
@@ -905,6 +1014,34 @@ pub mod test {
         stakes
     }
 
+    #[test]
+    fn test_to_vote_instruction() {
+        let vote = Vote::default();
+        let mut decision = SwitchForkDecision::FailedSwitchThreshold;
+        assert!(decision
+            .to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default())
+            .is_none());
+        decision = SwitchForkDecision::NoSwitch;
+        assert_eq!(
+            decision.to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default()),
+            Some(vote_instruction::vote(
+                &Pubkey::default(),
+                &Pubkey::default(),
+                vote.clone(),
+            ))
+        );
+        decision = SwitchForkDecision::SwitchProof(Hash::default());
+        assert_eq!(
+            decision.to_vote_instruction(vote.clone(), &Pubkey::default(), &Pubkey::default()),
+            Some(vote_instruction::vote_switch(
+                &Pubkey::default(),
+                &Pubkey::default(),
+                vote,
+                Hash::default()
+            ))
+        );
+    }
+
     #[test]
     fn test_simple_votes() {
         // Init state
@@ -975,85 +1112,106 @@ pub mod test {
         tower.record_vote(47, Hash::default());
 
         // Trying to switch to a descendant of last vote should always work
-        assert!(tower.check_switch_threshold(
-            48,
-            &ancestors,
-            &descendants,
-            &vote_simulator.progress,
-            total_stake,
-            bank0.epoch_vote_accounts(0).unwrap(),
-        ));
+        assert_eq!(
+            tower.check_switch_threshold(
+                48,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::NoSwitch
+        );
 
         // Trying to switch to another fork at 110 should fail
-        assert!(!tower.check_switch_threshold(
-            110,
-            &ancestors,
-            &descendants,
-            &vote_simulator.progress,
-            total_stake,
-            bank0.epoch_vote_accounts(0).unwrap(),
-        ));
+        assert_eq!(
+            tower.check_switch_threshold(
+                110,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::FailedSwitchThreshold
+        );
 
         // Adding another validator lockout on a descendant of last vote should
         // not count toward the switch threshold
         vote_simulator.simulate_lockout_interval(50, (49, 100), &other_vote_account);
-        assert!(!tower.check_switch_threshold(
-            110,
-            &ancestors,
-            &descendants,
-            &vote_simulator.progress,
-            total_stake,
-            bank0.epoch_vote_accounts(0).unwrap(),
-        ));
+        assert_eq!(
+            tower.check_switch_threshold(
+                110,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::FailedSwitchThreshold
+        );
 
         // Adding another validator lockout on an ancestor of last vote should
         // not count toward the switch threshold
         vote_simulator.simulate_lockout_interval(50, (45, 100), &other_vote_account);
-        assert!(!tower.check_switch_threshold(
-            110,
-            &ancestors,
-            &descendants,
-            &vote_simulator.progress,
-            total_stake,
-            bank0.epoch_vote_accounts(0).unwrap(),
-        ));
+        assert_eq!(
+            tower.check_switch_threshold(
+                110,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::FailedSwitchThreshold
+        );
 
         // Adding another validator lockout on a different fork, but the lockout
         // doesn't cover the last vote, should not satisfy the switch threshold
         vote_simulator.simulate_lockout_interval(14, (12, 46), &other_vote_account);
-        assert!(!tower.check_switch_threshold(
-            110,
-            &ancestors,
-            &descendants,
-            &vote_simulator.progress,
-            total_stake,
-            bank0.epoch_vote_accounts(0).unwrap(),
-        ));
+        assert_eq!(
+            tower.check_switch_threshold(
+                110,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::FailedSwitchThreshold
+        );
 
         // Adding another validator lockout on a different fork, and the lockout
         // covers the last vote, should satisfy the switch threshold
         vote_simulator.simulate_lockout_interval(14, (12, 47), &other_vote_account);
-        assert!(tower.check_switch_threshold(
-            110,
-            &ancestors,
-            &descendants,
-            &vote_simulator.progress,
-            total_stake,
-            bank0.epoch_vote_accounts(0).unwrap(),
-        ));
+        assert_eq!(
+            tower.check_switch_threshold(
+                110,
+                &ancestors,
+                &descendants,
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::SwitchProof(Hash::default())
+        );
 
         // If we set a root, then any lockout intervals below the root shouldn't
         // count toward the switch threshold. This means the other validator's
         // vote lockout no longer counts
         vote_simulator.set_root(43);
-        assert!(!tower.check_switch_threshold(
-            110,
-            &vote_simulator.bank_forks.read().unwrap().ancestors(),
-            &vote_simulator.bank_forks.read().unwrap().descendants(),
-            &vote_simulator.progress,
-            total_stake,
-            bank0.epoch_vote_accounts(0).unwrap(),
-        ));
+        assert_eq!(
+            tower.check_switch_threshold(
+                110,
+                &vote_simulator.bank_forks.read().unwrap().ancestors(),
+                &vote_simulator.bank_forks.read().unwrap().descendants(),
+                &vote_simulator.progress,
+                total_stake,
+                bank0.epoch_vote_accounts(0).unwrap(),
+            ),
+            SwitchForkDecision::FailedSwitchThreshold
+        );
     }
 
     #[test]
@@ -1193,20 +1351,32 @@ pub mod test {
     #[test]
     fn test_collect_vote_lockouts_sums() {
         //two accounts voting for slot 0 with 1 token staked
-        let accounts = gen_stakes(&[(1, &[0]), (1, &[0])]);
-        let tower = Tower::new_for_tests(0, 0.67);
+        let mut accounts = gen_stakes(&[(1, &[0]), (1, &[0])]);
+        accounts.sort_by_key(|(pk, _)| *pk);
+        let account_latest_votes: Vec<(Pubkey, Slot)> =
+            accounts.iter().map(|(pubkey, _)| (*pubkey, 0)).collect();
+
         let ancestors = vec![(1, vec![0].into_iter().collect()), (0, HashSet::new())]
             .into_iter()
             .collect();
-        let (staked_lockouts, total_staked, bank_weight, _) = tower.collect_vote_lockouts(
+        let ComputedBankState {
+            stake_lockouts,
+            total_staked,
+            bank_weight,
+            mut pubkey_votes,
+            ..
+        } = Tower::collect_vote_lockouts(
+            &Pubkey::default(),
             1,
             accounts.into_iter(),
             &ancestors,
             &mut PubkeyReferences::default(),
         );
-        assert_eq!(staked_lockouts[&0].stake, 2);
-        assert_eq!(staked_lockouts[&0].lockout, 2 + 2 + 4 + 4);
+        assert_eq!(stake_lockouts[&0].stake, 2);
+        assert_eq!(stake_lockouts[&0].lockout, 2 + 2 + 4 + 4);
         assert_eq!(total_staked, 2);
+        pubkey_votes.sort();
+        assert_eq!(pubkey_votes, account_latest_votes);
 
         // Each acccount has 1 vote in it. After simulating a vote in collect_vote_lockouts,
         // the account will have 2 votes, with lockout 2 + 4 = 6. So expected weight for
@@ -1218,7 +1388,12 @@ pub mod test {
     fn test_collect_vote_lockouts_root() {
         let votes: Vec<u64> = (0..MAX_LOCKOUT_HISTORY as u64).collect();
         //two accounts voting for slots 0..MAX_LOCKOUT_HISTORY with 1 token staked
-        let accounts = gen_stakes(&[(1, &votes), (1, &votes)]);
+        let mut accounts = gen_stakes(&[(1, &votes), (1, &votes)]);
+        accounts.sort_by_key(|(pk, _)| *pk);
+        let account_latest_votes: Vec<(Pubkey, Slot)> = accounts
+            .iter()
+            .map(|(pubkey, _)| (*pubkey, (MAX_LOCKOUT_HISTORY - 1) as Slot))
+            .collect();
         let mut tower = Tower::new_for_tests(0, 0.67);
         let mut ancestors = HashMap::new();
         for i in 0..(MAX_LOCKOUT_HISTORY + 1) {
@@ -1239,18 +1414,27 @@ pub mod test {
             + root_weight;
         let expected_bank_weight = 2 * vote_account_expected_weight;
         assert_eq!(tower.lockouts.root_slot, Some(0));
-        let (staked_lockouts, _total_staked, bank_weight, _) = tower.collect_vote_lockouts(
+        let ComputedBankState {
+            stake_lockouts,
+            bank_weight,
+            mut pubkey_votes,
+            ..
+        } = Tower::collect_vote_lockouts(
+            &Pubkey::default(),
             MAX_LOCKOUT_HISTORY as u64,
             accounts.into_iter(),
             &ancestors,
             &mut PubkeyReferences::default(),
         );
         for i in 0..MAX_LOCKOUT_HISTORY {
-            assert_eq!(staked_lockouts[&(i as u64)].stake, 2);
+            assert_eq!(stake_lockouts[&(i as u64)].stake, 2);
         }
+
         // should be the sum of all the weights for root
-        assert!(staked_lockouts[&0].lockout > (2 * (1 << MAX_LOCKOUT_HISTORY)));
+        assert!(stake_lockouts[&0].lockout > (2 * (1 << MAX_LOCKOUT_HISTORY)));
         assert_eq!(bank_weight, expected_bank_weight);
+        pubkey_votes.sort();
+        assert_eq!(pubkey_votes, account_latest_votes);
     }
 
     #[test]
@@ -1283,7 +1467,7 @@ pub mod test {
             );
             tower.record_vote(i, Hash::default());
         }
-        assert!(!tower.check_vote_stake_threshold(MAX_LOCKOUT_HISTORY as u64 + 1, &stakes, 2));
+        assert!(!tower.check_vote_stake_threshold(MAX_LOCKOUT_HISTORY as u64 + 1, &stakes, 2,));
     }
 
     #[test]
@@ -1511,48 +1695,7 @@ pub mod test {
         tower.record_vote(0, Hash::default());
         tower.record_vote(1, Hash::default());
         tower.record_vote(2, Hash::default());
-        assert!(tower.check_vote_stake_threshold(6, &stakes, 2));
-    }
-
-    #[test]
-    fn test_lockout_is_updated_for_entire_branch() {
-        let mut stake_lockouts = HashMap::new();
-        let vote = Lockout {
-            slot: 2,
-            confirmation_count: 1,
-        };
-        let set: HashSet<u64> = vec![0u64, 1u64].into_iter().collect();
-        let mut ancestors = HashMap::new();
-        ancestors.insert(2, set);
-        let set: HashSet<u64> = vec![0u64].into_iter().collect();
-        ancestors.insert(1, set);
-        Tower::update_ancestor_lockouts(&mut stake_lockouts, &vote, &ancestors);
-        assert_eq!(stake_lockouts[&0].lockout, 2);
-        assert_eq!(stake_lockouts[&1].lockout, 2);
-        assert_eq!(stake_lockouts[&2].lockout, 2);
-    }
-
-    #[test]
-    fn test_lockout_is_updated_for_slot_or_lower() {
-        let mut stake_lockouts = HashMap::new();
-        let set: HashSet<u64> = vec![0u64, 1u64].into_iter().collect();
-        let mut ancestors = HashMap::new();
-        ancestors.insert(2, set);
-        let set: HashSet<u64> = vec![0u64].into_iter().collect();
-        ancestors.insert(1, set);
-        let vote = Lockout {
-            slot: 2,
-            confirmation_count: 1,
-        };
-        Tower::update_ancestor_lockouts(&mut stake_lockouts, &vote, &ancestors);
-        let vote = Lockout {
-            slot: 1,
-            confirmation_count: 2,
-        };
-        Tower::update_ancestor_lockouts(&mut stake_lockouts, &vote, &ancestors);
-        assert_eq!(stake_lockouts[&0].lockout, 2 + 4);
-        assert_eq!(stake_lockouts[&1].lockout, 2 + 4);
-        assert_eq!(stake_lockouts[&2].lockout, 2);
+        assert!(tower.check_vote_stake_threshold(6, &stakes, 2,));
     }
 
     #[test]
@@ -1631,7 +1774,7 @@ pub mod test {
         let total_stake = 4;
         let threshold_size = 0.67;
         let threshold_stake = (f64::ceil(total_stake as f64 * threshold_size)) as u64;
-        let tower_votes: Vec<u64> = (0..VOTE_THRESHOLD_DEPTH as u64).collect();
+        let tower_votes: Vec<Slot> = (0..VOTE_THRESHOLD_DEPTH as u64).collect();
         let accounts = gen_stakes(&[
             (threshold_stake, &[(VOTE_THRESHOLD_DEPTH - 2) as u64]),
             (total_stake - threshold_stake, &tower_votes[..]),
@@ -1648,29 +1791,35 @@ pub mod test {
         for vote in &tower_votes {
             tower.record_vote(*vote, Hash::default());
         }
-        let (staked_lockouts, total_staked, _, _) = tower.collect_vote_lockouts(
+        let ComputedBankState {
+            stake_lockouts,
+            total_staked,
+            ..
+        } = Tower::collect_vote_lockouts(
+            &Pubkey::default(),
             vote_to_evaluate,
             accounts.clone().into_iter(),
             &ancestors,
             &mut PubkeyReferences::default(),
         );
-        assert!(tower.check_vote_stake_threshold(vote_to_evaluate, &staked_lockouts, total_staked));
+        assert!(tower.check_vote_stake_threshold(vote_to_evaluate, &stake_lockouts, total_staked,));
 
         // CASE 2: Now we want to evaluate a vote for slot VOTE_THRESHOLD_DEPTH + 1. This slot
         // will expire the vote in one of the vote accounts, so we should have insufficient
         // stake to pass the threshold
         let vote_to_evaluate = VOTE_THRESHOLD_DEPTH as u64 + 1;
-        let (staked_lockouts, total_staked, _, _) = tower.collect_vote_lockouts(
+        let ComputedBankState {
+            stake_lockouts,
+            total_staked,
+            ..
+        } = Tower::collect_vote_lockouts(
+            &Pubkey::default(),
             vote_to_evaluate,
             accounts.into_iter(),
             &ancestors,
             &mut PubkeyReferences::default(),
         );
-        assert!(!tower.check_vote_stake_threshold(
-            vote_to_evaluate,
-            &staked_lockouts,
-            total_staked
-        ));
+        assert!(!tower.check_vote_stake_threshold(vote_to_evaluate, &stake_lockouts, total_staked,));
     }
 
     fn vote_and_check_recent(num_votes: usize) {

core/src/crds.rs

@@ -36,6 +36,7 @@ use std::collections::HashMap;
 pub struct Crds {
     /// Stores the map of labels and values
     pub table: IndexMap<CrdsValueLabel, VersionedCrdsValue>,
+    pub num_inserts: usize,
 }
 
 #[derive(PartialEq, Debug)]
@@ -84,6 +85,7 @@ impl Default for Crds {
     fn default() -> Self {
         Crds {
             table: IndexMap::new(),
+            num_inserts: 0,
         }
     }
 }
@@ -93,6 +95,24 @@ impl Crds {
     pub fn new_versioned(&self, local_timestamp: u64, value: CrdsValue) -> VersionedCrdsValue {
         VersionedCrdsValue::new(local_timestamp, value)
     }
+    pub fn would_insert(
+        &self,
+        value: CrdsValue,
+        local_timestamp: u64,
+    ) -> Option<VersionedCrdsValue> {
+        let new_value = self.new_versioned(local_timestamp, value);
+        let label = new_value.value.label();
+        let would_insert = self
+            .table
+            .get(&label)
+            .map(|current| new_value > *current)
+            .unwrap_or(true);
+        if would_insert {
+            Some(new_value)
+        } else {
+            None
+        }
+    }
     /// insert the new value, returns the old value if insert succeeds
     pub fn insert_versioned(
         &mut self,
@@ -107,6 +127,7 @@ impl Crds {
             .unwrap_or(true);
         if do_insert {
             let old = self.table.insert(label, new_value);
+            self.num_inserts += 1;
             Ok(old)
         } else {
             trace!("INSERT FAILED data: {} new.wallclock: {}", label, wallclock,);

core/src/crds_gossip.rs

@@ -6,7 +6,7 @@
 use crate::{
     crds::{Crds, VersionedCrdsValue},
     crds_gossip_error::CrdsGossipError,
-    crds_gossip_pull::{CrdsFilter, CrdsGossipPull},
+    crds_gossip_pull::{CrdsFilter, CrdsGossipPull, ProcessPullStats},
     crds_gossip_push::{CrdsGossipPush, CRDS_GOSSIP_NUM_ACTIVE},
     crds_value::{CrdsValue, CrdsValueLabel},
 };
@@ -76,17 +76,10 @@ impl CrdsGossip {
         stakes: &HashMap<Pubkey, u64>,
     ) -> HashMap<Pubkey, HashSet<Pubkey>> {
         let id = &self.id;
-        let crds = &self.crds;
         let push = &mut self.push;
-        let versioned = labels
-            .into_iter()
-            .filter_map(|label| crds.lookup_versioned(&label));
-
         let mut prune_map: HashMap<Pubkey, HashSet<_>> = HashMap::new();
-        for val in versioned {
-            let origin = val.value.pubkey();
-            let hash = val.value_hash;
-            let peers = push.prune_received_cache(id, &origin, hash, stakes);
+        for origin in labels.iter().map(|k| k.pubkey()) {
+            let peers = push.prune_received_cache(id, &origin, stakes);
             for from in peers {
                 prune_map.entry(from).or_default().insert(origin);
             }
@@ -113,7 +106,7 @@ impl CrdsGossip {
             return Err(CrdsGossipError::PruneMessageTimeout);
         }
         if self.id == *destination {
-            self.push.process_prune_msg(peer, origin);
+            self.push.process_prune_msg(&self.id, peer, origin);
             Ok(())
         } else {
             Err(CrdsGossipError::BadPruneDestination)
@@ -158,24 +151,47 @@ impl CrdsGossip {
         self.pull.mark_pull_request_creation_time(from, now)
     }
     /// process a pull request and create a response
-    pub fn process_pull_requests(
-        &mut self,
-        filters: Vec<(CrdsValue, CrdsFilter)>,
-        now: u64,
-    ) -> Vec<Vec<CrdsValue>> {
+    pub fn process_pull_requests(&mut self, filters: Vec<(CrdsValue, CrdsFilter)>, now: u64) {
         self.pull
-            .process_pull_requests(&mut self.crds, filters, now)
+            .process_pull_requests(&mut self.crds, filters, now);
     }
-    /// process a pull response
-    pub fn process_pull_response(
-        &mut self,
-        from: &Pubkey,
+
+    pub fn generate_pull_responses(
+        &self,
+        filters: &[(CrdsValue, CrdsFilter)],
+    ) -> Vec<Vec<CrdsValue>> {
+        self.pull.generate_pull_responses(&self.crds, filters)
+    }
+
+    pub fn filter_pull_responses(
+        &self,
         timeouts: &HashMap<Pubkey, u64>,
         response: Vec<CrdsValue>,
         now: u64,
-    ) -> usize {
+        process_pull_stats: &mut ProcessPullStats,
+    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>) {
         self.pull
-            .process_pull_response(&mut self.crds, from, timeouts, response, now)
+            .filter_pull_responses(&self.crds, timeouts, response, now, process_pull_stats)
+    }
+
+    /// process a pull response
+    pub fn process_pull_responses(
+        &mut self,
+        from: &Pubkey,
+        responses: Vec<VersionedCrdsValue>,
+        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        now: u64,
+        process_pull_stats: &mut ProcessPullStats,
+    ) {
+        let success = self.pull.process_pull_responses(
+            &mut self.crds,
+            from,
+            responses,
+            responses_expired_timeout,
+            now,
+            process_pull_stats,
+        );
+        self.push.push_pull_responses(success, now);
     }
 
     pub fn make_timeouts_test(&self) -> HashMap<Pubkey, u64> {

core/src/crds_gossip_error.rs

@@ -2,7 +2,6 @@
 pub enum CrdsGossipError {
     NoPeers,
     PushMessageTimeout,
-    PushMessageAlreadyReceived,
     PushMessageOldVersion,
     BadPruneDestination,
     PruneMessageTimeout,

core/src/crds_gossip_pull.rs

@@ -10,7 +10,7 @@
 //! of false positives.
 
 use crate::contact_info::ContactInfo;
-use crate::crds::Crds;
+use crate::crds::{Crds, VersionedCrdsValue};
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
@@ -20,8 +20,8 @@ use solana_runtime::bloom::Bloom;
 use solana_sdk::hash::Hash;
 use solana_sdk::pubkey::Pubkey;
 use std::cmp;
-use std::collections::HashMap;
 use std::collections::VecDeque;
+use std::collections::{HashMap, HashSet};
 
 pub const CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS: u64 = 15000;
 // The maximum age of a value received over pull responses
@@ -118,6 +118,14 @@ impl CrdsFilter {
     }
 }
 
+#[derive(Default)]
+pub struct ProcessPullStats {
+    pub success: usize,
+    pub failed_insert: usize,
+    pub failed_timeout: usize,
+    pub timeout_count: usize,
+}
+
 #[derive(Clone)]
 pub struct CrdsGossipPull {
     /// timestamp of last request
@@ -126,6 +134,7 @@ pub struct CrdsGossipPull {
     purged_values: VecDeque<(Hash, u64)>,
     pub crds_timeout: u64,
     pub msg_timeout: u64,
+    pub num_pulls: usize,
 }
 
 impl Default for CrdsGossipPull {
@@ -135,6 +144,7 @@ impl Default for CrdsGossipPull {
             pull_request_time: HashMap::new(),
             crds_timeout: CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS,
             msg_timeout: CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS,
+            num_pulls: 0,
         }
     }
 }
@@ -204,14 +214,13 @@ impl CrdsGossipPull {
         self.purged_values.push_back((hash, timestamp))
     }
 
-    /// process a pull request and create a response
+    /// process a pull request
     pub fn process_pull_requests(
         &mut self,
         crds: &mut Crds,
         requests: Vec<(CrdsValue, CrdsFilter)>,
         now: u64,
-    ) -> Vec<Vec<CrdsValue>> {
-        let rv = self.filter_crds_values(crds, &requests);
+    ) {
         requests.into_iter().for_each(|(caller, _)| {
             let key = caller.label().pubkey();
             let old = crds.insert(caller, now);
@@ -221,19 +230,33 @@ impl CrdsGossipPull {
             }
             crds.update_record_timestamp(&key, now);
         });
-        rv
     }
-    /// process a pull response
-    pub fn process_pull_response(
-        &mut self,
-        crds: &mut Crds,
-        from: &Pubkey,
+
+    /// Create gossip responses to pull requests
+    pub fn generate_pull_responses(
+        &self,
+        crds: &Crds,
+        requests: &[(CrdsValue, CrdsFilter)],
+    ) -> Vec<Vec<CrdsValue>> {
+        self.filter_crds_values(crds, requests)
+    }
+
+    // Checks if responses should be inserted and
+    // returns those responses converted to VersionedCrdsValue
+    // Separated in two vecs as:
+    //  .0 => responses that update the owner timestamp
+    //  .1 => responses that do not update the owner timestamp
+    pub fn filter_pull_responses(
+        &self,
+        crds: &Crds,
         timeouts: &HashMap<Pubkey, u64>,
-        response: Vec<CrdsValue>,
+        responses: Vec<CrdsValue>,
         now: u64,
-    ) -> usize {
-        let mut failed = 0;
-        for r in response {
+        stats: &mut ProcessPullStats,
+    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>) {
+        let mut versioned = vec![];
+        let mut versioned_expired_timestamp = vec![];
+        for r in responses {
             let owner = r.label().pubkey();
             // Check if the crds value is older than the msg_timeout
             if now
@@ -252,11 +275,8 @@ impl CrdsGossipPull {
                         if now > r.wallclock().checked_add(timeout).unwrap_or_else(|| 0)
                             || now + timeout < r.wallclock()
                         {
-                            inc_new_counter_warn!(
-                                "cluster_info-gossip_pull_response_value_timeout",
-                                1
-                            );
-                            failed += 1;
+                            stats.timeout_count += 1;
+                            stats.failed_timeout += 1;
                             continue;
                         }
                     }
@@ -264,32 +284,69 @@ impl CrdsGossipPull {
                         // Before discarding this value, check if a ContactInfo for the owner
                         // exists in the table. If it doesn't, that implies that this value can be discarded
                         if crds.lookup(&CrdsValueLabel::ContactInfo(owner)).is_none() {
-                            inc_new_counter_warn!(
-                                "cluster_info-gossip_pull_response_value_timeout",
-                                1
-                            );
-                            failed += 1;
+                            stats.timeout_count += 1;
+                            stats.failed_timeout += 1;
                             continue;
                         } else {
                             // Silently insert this old value without bumping record timestamps
-                            failed += crds.insert(r, now).is_err() as usize;
+                            match crds.would_insert(r, now) {
+                                Some(resp) => versioned_expired_timestamp.push(resp),
+                                None => stats.failed_insert += 1,
+                            }
                             continue;
                         }
                     }
                 }
             }
-            let old = crds.insert(r, now);
-            failed += old.is_err() as usize;
+            match crds.would_insert(r, now) {
+                Some(resp) => versioned.push(resp),
+                None => stats.failed_insert += 1,
+            }
+        }
+        (versioned, versioned_expired_timestamp)
+    }
+
+    /// process a vec of pull responses
+    pub fn process_pull_responses(
+        &mut self,
+        crds: &mut Crds,
+        from: &Pubkey,
+        responses: Vec<VersionedCrdsValue>,
+        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        now: u64,
+        stats: &mut ProcessPullStats,
+    ) -> Vec<(CrdsValueLabel, Hash, u64)> {
+        let mut success = vec![];
+        let mut owners = HashSet::new();
+        for r in responses_expired_timeout {
+            stats.failed_insert += crds.insert_versioned(r).is_err() as usize;
+        }
+        for r in responses {
+            let owner = r.value.label().pubkey();
+            let label = r.value.label();
+            let wc = r.value.wallclock();
+            let hash = r.value_hash;
+            let old = crds.insert_versioned(r);
+            if old.is_err() {
+                stats.failed_insert += 1;
+            } else {
+                stats.success += 1;
+                self.num_pulls += 1;
+                success.push((label, hash, wc));
+            }
             old.ok().map(|opt| {
-                crds.update_record_timestamp(&owner, now);
+                owners.insert(owner);
                 opt.map(|val| {
                     self.purged_values
                         .push_back((val.value_hash, val.local_timestamp))
                 })
             });
         }
-        crds.update_record_timestamp(from, now);
-        failed
+        owners.insert(*from);
+        for owner in owners {
+            crds.update_record_timestamp(&owner, now);
+        }
+        success
     }
     // build a set of filters of the current crds table
     // num_filters - used to increase the likelyhood of a value in crds being added to some filter
@@ -379,6 +436,34 @@ impl CrdsGossipPull {
             .count();
         self.purged_values.drain(..cnt);
     }
+
+    /// For legacy tests
+    #[cfg(test)]
+    pub fn process_pull_response(
+        &mut self,
+        crds: &mut Crds,
+        from: &Pubkey,
+        timeouts: &HashMap<Pubkey, u64>,
+        response: Vec<CrdsValue>,
+        now: u64,
+    ) -> (usize, usize, usize) {
+        let mut stats = ProcessPullStats::default();
+        let (versioned, versioned_expired_timeout) =
+            self.filter_pull_responses(crds, timeouts, response, now, &mut stats);
+        self.process_pull_responses(
+            crds,
+            from,
+            versioned,
+            versioned_expired_timeout,
+            now,
+            &mut stats,
+        );
+        (
+            stats.failed_timeout + stats.failed_insert,
+            stats.timeout_count,
+            stats.success,
+        )
+    }
 }
 #[cfg(test)]
 mod test {
@@ -578,8 +663,9 @@ mod test {
         let mut dest_crds = Crds::default();
         let mut dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
-        let filters = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.process_pull_requests(&mut dest_crds, filters, 1);
+        let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
+        let rsp = dest.generate_pull_responses(&dest_crds, &filters);
+        dest.process_pull_requests(&mut dest_crds, filters, 1);
         assert!(rsp.iter().all(|rsp| rsp.is_empty()));
         assert!(dest_crds.lookup(&caller.label()).is_some());
         assert_eq!(
@@ -648,8 +734,9 @@ mod test {
                 PACKET_DATA_SIZE,
             );
             let (_, filters, caller) = req.unwrap();
-            let filters = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-            let mut rsp = dest.process_pull_requests(&mut dest_crds, filters, 0);
+            let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
+            let mut rsp = dest.generate_pull_responses(&dest_crds, &filters);
+            dest.process_pull_requests(&mut dest_crds, filters, 0);
             // if there is a false positive this is empty
             // prob should be around 0.1 per iteration
             if rsp.is_empty() {
@@ -660,13 +747,15 @@ mod test {
                 continue;
             }
             assert_eq!(rsp.len(), 1);
-            let failed = node.process_pull_response(
-                &mut node_crds,
-                &node_pubkey,
-                &node.make_timeouts_def(&node_pubkey, &HashMap::new(), 0, 1),
-                rsp.pop().unwrap(),
-                1,
-            );
+            let failed = node
+                .process_pull_response(
+                    &mut node_crds,
+                    &node_pubkey,
+                    &node.make_timeouts_def(&node_pubkey, &HashMap::new(), 0, 1),
+                    rsp.pop().unwrap(),
+                    1,
+                )
+                .0;
             assert_eq!(failed, 0);
             assert_eq!(
                 node_crds
@@ -827,7 +916,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry.clone()],
                 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -843,7 +933,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry.clone(), unstaked_peer_entry],
                 node.msg_timeout + 100,
-            ),
+            )
+            .0,
             2
         );
 
@@ -856,7 +947,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -872,7 +964,8 @@ mod test {
                 &timeouts,
                 vec![peer_vote.clone()],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -885,7 +978,8 @@ mod test {
                 &timeouts,
                 vec![peer_vote],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             1
         );
     }

core/src/crds_gossip_push.rs

@@ -5,7 +5,7 @@
 //!
 //! Main differences are:
 //! 1. There is no `max hop`.  Messages are signed with a local wallclock.  If they are outside of
-//!    the local nodes wallclock window they are drooped silently.
+//!    the local nodes wallclock window they are dropped silently.
 //! 2. The prune set is stored in a Bloom filter.
 
 use crate::{
@@ -35,6 +35,7 @@ pub const CRDS_GOSSIP_PUSH_FANOUT: usize = 6;
 pub const CRDS_GOSSIP_PUSH_MSG_TIMEOUT_MS: u64 = 30000;
 pub const CRDS_GOSSIP_PRUNE_MSG_TIMEOUT_MS: u64 = 500;
 pub const CRDS_GOSSIP_PRUNE_STAKE_THRESHOLD_PCT: f64 = 0.15;
+pub const CRDS_GOSSIP_PRUNE_MIN_INGRESS_NODES: usize = 2;
 
 #[derive(Clone)]
 pub struct CrdsGossipPush {
@@ -44,12 +45,18 @@ pub struct CrdsGossipPush {
     active_set: IndexMap<Pubkey, Bloom<Pubkey>>,
     /// push message queue
     push_messages: HashMap<CrdsValueLabel, Hash>,
-    /// cache that tracks which validators a message was received from
-    received_cache: HashMap<Hash, (u64, HashSet<Pubkey>)>,
+    /// Cache that tracks which validators a message was received from
+    /// bool indicates it has been pruned.
+    /// This cache represents a lagging view of which validators
+    /// currently have this node in their `active_set`
+    received_cache: HashMap<Pubkey, HashMap<Pubkey, (bool, u64)>>,
     pub num_active: usize,
     pub push_fanout: usize,
     pub msg_timeout: u64,
     pub prune_timeout: u64,
+    pub num_total: usize,
+    pub num_old: usize,
+    pub num_pushes: usize,
 }
 
 impl Default for CrdsGossipPush {
@@ -64,6 +71,9 @@ impl Default for CrdsGossipPush {
             push_fanout: CRDS_GOSSIP_PUSH_FANOUT,
             msg_timeout: CRDS_GOSSIP_PUSH_MSG_TIMEOUT_MS,
             prune_timeout: CRDS_GOSSIP_PRUNE_MSG_TIMEOUT_MS,
+            num_total: 0,
+            num_old: 0,
+            num_pushes: 0,
         }
     }
 }
@@ -81,18 +91,21 @@ impl CrdsGossipPush {
         &mut self,
         self_pubkey: &Pubkey,
         origin: &Pubkey,
-        hash: Hash,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<Pubkey> {
         let origin_stake = stakes.get(origin).unwrap_or(&0);
         let self_stake = stakes.get(self_pubkey).unwrap_or(&0);
-        let cache = self.received_cache.get(&hash);
+        let cache = self.received_cache.get(origin);
         if cache.is_none() {
             return Vec::new();
         }
+        let peers = cache.unwrap();
 
-        let peers = &cache.unwrap().1;
-        let peer_stake_total: u64 = peers.iter().map(|p| stakes.get(p).unwrap_or(&0)).sum();
+        let peer_stake_total: u64 = peers
+            .iter()
+            .filter(|v| !(v.1).0)
+            .map(|v| stakes.get(v.0).unwrap_or(&0))
+            .sum();
         let prune_stake_threshold = Self::prune_stake_threshold(*self_stake, *origin_stake);
         if peer_stake_total < prune_stake_threshold {
             return Vec::new();
@@ -100,7 +113,8 @@ impl CrdsGossipPush {
 
         let staked_peers: Vec<(Pubkey, u64)> = peers
             .iter()
-            .filter_map(|p| stakes.get(p).map(|s| (*p, *s)))
+            .filter(|v| !(v.1).0)
+            .filter_map(|p| stakes.get(p.0).map(|s| (*p.0, *s)))
             .filter(|(_, s)| *s > 0)
             .collect();
 
@@ -117,16 +131,27 @@ impl CrdsGossipPush {
             let (next_peer, next_stake) = staked_peers[next];
             keep.insert(next_peer);
             peer_stake_sum += next_stake;
-            if peer_stake_sum >= prune_stake_threshold {
+            if peer_stake_sum >= prune_stake_threshold
+                && keep.len() >= CRDS_GOSSIP_PRUNE_MIN_INGRESS_NODES
+            {
                 break;
             }
         }
 
-        peers
-            .iter()
+        let pruned_peers: Vec<Pubkey> = peers
+            .keys()
             .filter(|p| !keep.contains(p))
             .cloned()
-            .collect()
+            .collect();
+        pruned_peers.iter().for_each(|p| {
+            self.received_cache
+                .get_mut(origin)
+                .unwrap()
+                .get_mut(p)
+                .unwrap()
+                .0 = true;
+        });
+        pruned_peers
     }
 
     /// process a push message to the network
@@ -137,6 +162,7 @@ impl CrdsGossipPush {
         value: CrdsValue,
         now: u64,
     ) -> Result<Option<VersionedCrdsValue>, CrdsGossipError> {
+        self.num_total += 1;
         if now
             > value
                 .wallclock()
@@ -149,21 +175,32 @@ impl CrdsGossipPush {
             return Err(CrdsGossipError::PushMessageTimeout);
         }
         let label = value.label();
+        let origin = label.pubkey();
         let new_value = crds.new_versioned(now, value);
         let value_hash = new_value.value_hash;
-        if let Some((_, ref mut received_set)) = self.received_cache.get_mut(&value_hash) {
-            received_set.insert(from.clone());
-            return Err(CrdsGossipError::PushMessageAlreadyReceived);
-        }
+        let received_set = self
+            .received_cache
+            .entry(origin)
+            .or_insert_with(HashMap::new);
+        received_set.entry(*from).or_insert((false, 0)).1 = now;
+
         let old = crds.insert_versioned(new_value);
         if old.is_err() {
+            self.num_old += 1;
             return Err(CrdsGossipError::PushMessageOldVersion);
         }
-        let mut received_set = HashSet::new();
-        received_set.insert(from.clone());
         self.push_messages.insert(label, value_hash);
-        self.received_cache.insert(value_hash, (now, received_set));
-        Ok(old.ok().and_then(|opt| opt))
+        Ok(old.unwrap())
+    }
+
+    /// push pull responses
+    pub fn push_pull_responses(&mut self, values: Vec<(CrdsValueLabel, Hash, u64)>, now: u64) {
+        for (label, value_hash, wc) in values {
+            if now > wc.checked_add(self.msg_timeout).unwrap_or_else(|| 0) {
+                continue;
+            }
+            self.push_messages.insert(label, value_hash);
+        }
     }
 
     /// New push message to broadcast to peers.
@@ -172,18 +209,10 @@ impl CrdsGossipPush {
     /// The list of push messages is created such that all the randomly selected peers have not
     /// pruned the source addresses.
     pub fn new_push_messages(&mut self, crds: &Crds, now: u64) -> HashMap<Pubkey, Vec<CrdsValue>> {
-        let max = self.active_set.len();
-        let mut nodes: Vec<_> = (0..max).collect();
-        nodes.shuffle(&mut rand::thread_rng());
-        let peers: Vec<Pubkey> = nodes
-            .into_iter()
-            .filter_map(|n| self.active_set.get_index(n))
-            .take(self.push_fanout)
-            .map(|n| *n.0)
-            .collect();
         let mut total_bytes: usize = 0;
         let mut values = vec![];
         let mut push_messages: HashMap<Pubkey, Vec<CrdsValue>> = HashMap::new();
+        trace!("new_push_messages {}", self.push_messages.len());
         for (label, hash) in &self.push_messages {
             let res = crds.lookup_versioned(label);
             if res.is_none() {
@@ -203,21 +232,37 @@ impl CrdsGossipPush {
             }
             values.push(value.clone());
         }
+        trace!(
+            "new_push_messages {} {}",
+            values.len(),
+            self.active_set.len()
+        );
         for v in values {
-            for p in peers.iter() {
-                let filter = self.active_set.get_mut(p);
-                if filter.is_some() && !filter.unwrap().contains(&v.label().pubkey()) {
-                    push_messages.entry(*p).or_default().push(v.clone());
+            //use a consistent index for the same origin so
+            //the active set learns the MST for that origin
+            let start = v.label().pubkey().as_ref()[0] as usize;
+            let max = self.push_fanout.min(self.active_set.len());
+            for i in start..(start + max) {
+                let ix = i % self.active_set.len();
+                if let Some((p, filter)) = self.active_set.get_index(ix) {
+                    if !filter.contains(&v.label().pubkey()) {
+                        trace!("new_push_messages insert {} {:?}", *p, v);
+                        push_messages.entry(*p).or_default().push(v.clone());
+                        self.num_pushes += 1;
+                    }
                 }
+                self.push_messages.remove(&v.label());
             }
-            self.push_messages.remove(&v.label());
         }
         push_messages
     }
 
     /// add the `from` to the peer's filter of nodes
-    pub fn process_prune_msg(&mut self, peer: &Pubkey, origins: &[Pubkey]) {
+    pub fn process_prune_msg(&mut self, self_pubkey: &Pubkey, peer: &Pubkey, origins: &[Pubkey]) {
         for origin in origins {
+            if origin == self_pubkey {
+                continue;
+            }
             if let Some(p) = self.active_set.get_mut(peer) {
                 p.add(origin)
             }
@@ -339,15 +384,11 @@ impl CrdsGossipPush {
 
     /// purge received push message cache
     pub fn purge_old_received_cache(&mut self, min_time: u64) {
-        let old_msgs: Vec<Hash> = self
-            .received_cache
-            .iter()
-            .filter_map(|(k, (rcvd_time, _))| if *rcvd_time < min_time { Some(k) } else { None })
-            .cloned()
-            .collect();
-        for k in old_msgs {
-            self.received_cache.remove(&k);
-        }
+        self.received_cache
+            .iter_mut()
+            .for_each(|v| v.1.retain(|_, v| v.1 > min_time));
+
+        self.received_cache.retain(|_, v| !v.is_empty());
     }
 }
 
@@ -371,7 +412,6 @@ mod test {
         let value = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &origin, 0,
         )));
-        let label = value.label();
         let low_staked_peers = (0..10).map(|_| Pubkey::new_rand());
         let mut low_staked_set = HashSet::new();
         low_staked_peers.for_each(|p| {
@@ -380,11 +420,7 @@ mod test {
             stakes.insert(p, 1);
         });
 
-        let versioned = crds
-            .lookup_versioned(&label)
-            .expect("versioned value should exist");
-        let hash = versioned.value_hash;
-        let pruned = push.prune_received_cache(&self_id, &origin, hash, &stakes);
+        let pruned = push.prune_received_cache(&self_id, &origin, &stakes);
         assert!(
             pruned.is_empty(),
             "should not prune if min threshold has not been reached"
@@ -395,7 +431,7 @@ mod test {
         stakes.insert(high_staked_peer, high_stake);
         let _ = push.process_push_message(&mut crds, &high_staked_peer, value, 0);
 
-        let pruned = push.prune_received_cache(&self_id, &origin, hash, &stakes);
+        let pruned = push.prune_received_cache(&self_id, &origin, &stakes);
         assert!(
             pruned.len() < low_staked_set.len() + 1,
             "should not prune all peers"
@@ -409,7 +445,7 @@ mod test {
     }
 
     #[test]
-    fn test_process_push() {
+    fn test_process_push_one() {
         let mut crds = Crds::default();
         let mut push = CrdsGossipPush::default();
         let value = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -425,9 +461,9 @@ mod test {
         assert_eq!(crds.lookup(&label), Some(&value));
 
         // push it again
-        assert_eq!(
+        assert_matches!(
             push.process_push_message(&mut crds, &Pubkey::default(), value, 0),
-            Err(CrdsGossipError::PushMessageAlreadyReceived)
+            Err(CrdsGossipError::PushMessageOldVersion)
         );
     }
     #[test]
@@ -690,6 +726,7 @@ mod test {
     #[test]
     fn test_process_prune() {
         let mut crds = Crds::default();
+        let self_id = Pubkey::new_rand();
         let mut push = CrdsGossipPush::default();
         let peer = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -707,7 +744,11 @@ mod test {
             push.process_push_message(&mut crds, &Pubkey::default(), new_msg.clone(), 0),
             Ok(None)
         );
-        push.process_prune_msg(&peer.label().pubkey(), &[new_msg.label().pubkey()]);
+        push.process_prune_msg(
+            &self_id,
+            &peer.label().pubkey(),
+            &[new_msg.label().pubkey()],
+        );
         assert_eq!(push.new_push_messages(&crds, 0), expected);
     }
     #[test]
@@ -749,9 +790,9 @@ mod test {
         assert_eq!(crds.lookup(&label), Some(&value));
 
         // push it again
-        assert_eq!(
+        assert_matches!(
             push.process_push_message(&mut crds, &Pubkey::default(), value.clone(), 0),
-            Err(CrdsGossipError::PushMessageAlreadyReceived)
+            Err(CrdsGossipError::PushMessageOldVersion)
         );
 
         // purge the old pushed

core/src/fork_choice.rs

@@ -0,0 +1,40 @@
+use crate::{
+    consensus::{ComputedBankState, SwitchForkDecision, Tower},
+    progress_map::ProgressMap,
+    replay_stage::HeaviestForkFailures,
+};
+use solana_ledger::bank_forks::BankForks;
+use solana_runtime::bank::Bank;
+use std::{
+    collections::{HashMap, HashSet},
+    sync::{Arc, RwLock},
+};
+
+pub(crate) struct SelectVoteAndResetForkResult {
+    pub vote_bank: Option<(Arc<Bank>, SwitchForkDecision)>,
+    pub reset_bank: Option<Arc<Bank>>,
+    pub heaviest_fork_failures: Vec<HeaviestForkFailures>,
+}
+
+pub(crate) trait ForkChoice {
+    fn compute_bank_stats(
+        &mut self,
+        bank: &Bank,
+        tower: &Tower,
+        progress: &mut ProgressMap,
+        computed_bank_stats: &ComputedBankState,
+    );
+
+    // Returns:
+    // 1) The heaviest overall bbank
+    // 2) The heavest bank on the same fork as the last vote (doesn't require a
+    // switching proof to vote for)
+    fn select_forks(
+        &self,
+        frozen_banks: &[Arc<Bank>],
+        tower: &Tower,
+        progress: &ProgressMap,
+        ancestors: &HashMap<u64, HashSet<u64>>,
+        bank_forks: &RwLock<BankForks>,
+    ) -> (Arc<Bank>, Option<Arc<Bank>>);
+}

core/src/ledger_cleanup_service.rs

@@ -1,9 +1,9 @@
 //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
 
-use solana_ledger::blockstore::Blockstore;
+use solana_ledger::blockstore::{Blockstore, PurgeType};
 use solana_ledger::blockstore_db::Result as BlockstoreResult;
 use solana_measure::measure::Measure;
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Slot, DEFAULT_TICKS_PER_SLOT, TICKS_PER_DAY};
 use std::string::ToString;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
@@ -29,8 +29,9 @@ pub const DEFAULT_MIN_MAX_LEDGER_SHREDS: u64 = 50_000_000;
 // and starve other blockstore users.
 pub const DEFAULT_PURGE_SLOT_INTERVAL: u64 = 512;
 
-// Delay between purges to cooperate with other blockstore users
-pub const DEFAULT_DELAY_BETWEEN_PURGES: Duration = Duration::from_millis(500);
+// Compacting at a slower interval than purging helps keep IOPS down.
+// Once a day should be ample
+const DEFAULT_COMPACTION_SLOT_INTERVAL: u64 = TICKS_PER_DAY / DEFAULT_TICKS_PER_SLOT;
 
 pub struct LedgerCleanupService {
     t_cleanup: JoinHandle<()>,
@@ -49,6 +50,8 @@ impl LedgerCleanupService {
         );
         let exit = exit.clone();
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
+
         let t_cleanup = Builder::new()
             .name("solana-ledger-cleanup".to_string())
             .spawn(move || loop {
@@ -61,7 +64,8 @@ impl LedgerCleanupService {
                     max_ledger_shreds,
                     &mut last_purge_slot,
                     DEFAULT_PURGE_SLOT_INTERVAL,
-                    Some(DEFAULT_DELAY_BETWEEN_PURGES),
+                    &mut last_compaction_slot,
+                    DEFAULT_COMPACTION_SLOT_INTERVAL,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -116,7 +120,7 @@ impl LedgerCleanupService {
             }
         }
 
-        (true, lowest_cleanup_slot, first_slot, total_shreds)
+        (true, first_slot, lowest_cleanup_slot, total_shreds)
     }
 
     fn receive_new_roots(new_root_receiver: &Receiver<Slot>) -> Result<Slot, RecvTimeoutError> {
@@ -134,7 +138,8 @@ impl LedgerCleanupService {
         max_ledger_shreds: u64,
         last_purge_slot: &mut u64,
         purge_interval: u64,
-        delay_between_purges: Option<Duration>,
+        last_compaction_slot: &mut u64,
+        compaction_interval: u64,
     ) -> Result<(), RecvTimeoutError> {
         let root = Self::receive_new_roots(new_root_receiver)?;
         if root - *last_purge_slot <= purge_interval {
@@ -143,19 +148,21 @@ impl LedgerCleanupService {
 
         let disk_utilization_pre = blockstore.storage_size();
         info!(
-            "purge: last_root={}, last_purge_slot={}, purge_interval={}, disk_utilization={:?}",
-            root, last_purge_slot, purge_interval, disk_utilization_pre
+            "purge: last_root={}, last_purge_slot={}, purge_interval={}, last_compaction_slot={}, disk_utilization={:?}",
+            root, last_purge_slot, purge_interval, last_compaction_slot, disk_utilization_pre
         );
+
         *last_purge_slot = root;
 
-        let (slots_to_clean, lowest_cleanup_slot, first_slot, total_shreds) =
+        let (slots_to_clean, purge_first_slot, lowest_cleanup_slot, total_shreds) =
             Self::find_slots_to_clean(&blockstore, root, max_ledger_shreds);
 
         if slots_to_clean {
-            info!(
-                "purging data from slots {} to {}",
-                first_slot, lowest_cleanup_slot
-            );
+            let mut compact_first_slot = std::u64::MAX;
+            if lowest_cleanup_slot.saturating_sub(*last_compaction_slot) > compaction_interval {
+                compact_first_slot = *last_compaction_slot;
+                *last_compaction_slot = lowest_cleanup_slot;
+            }
 
             let purge_complete = Arc::new(AtomicBool::new(false));
             let blockstore = blockstore.clone();
@@ -167,14 +174,36 @@ impl LedgerCleanupService {
                     *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_cleanup_slot;
                     slot_update_time.stop();
 
-                    let mut purge_time = Measure::start("purge_slots_with_delay");
-                    blockstore.purge_slots_with_delay(
-                        first_slot,
+                    info!(
+                        "purging data from slots {} to {}",
+                        purge_first_slot, lowest_cleanup_slot
+                    );
+
+                    let mut purge_time = Measure::start("purge_slots");
+                    blockstore.purge_slots(
+                        purge_first_slot,
                         lowest_cleanup_slot,
-                        delay_between_purges,
+                        PurgeType::PrimaryIndex,
                     );
                     purge_time.stop();
                     info!("{}", purge_time);
+
+                    if compact_first_slot < lowest_cleanup_slot {
+                        info!(
+                            "compacting data from slots {} to {}",
+                            compact_first_slot, lowest_cleanup_slot
+                        );
+                        if let Err(err) =
+                            blockstore.compact_storage(compact_first_slot, lowest_cleanup_slot)
+                        {
+                            // This error is not fatal and indicates an internal error?
+                            error!(
+                                "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
+                                err, compact_first_slot, lowest_cleanup_slot
+                            );
+                        }
+                    }
+
                     purge_complete1.store(true, Ordering::Relaxed);
                 })
                 .unwrap();
@@ -233,6 +262,7 @@ mod tests {
 
         //send a signal to kill all but 5 shreds, which will be in the newest slots
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         sender.send(50).unwrap();
         LedgerCleanupService::cleanup_ledger(
             &receiver,
@@ -240,7 +270,8 @@ mod tests {
             5,
             &mut last_purge_slot,
             10,
-            None,
+            &mut last_compaction_slot,
+            10,
         )
         .unwrap();
 
@@ -272,6 +303,7 @@ mod tests {
         info!("{}", first_insert);
 
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         let mut slot = initial_slots;
         let mut num_slots = 6;
         for _ in 0..5 {
@@ -295,7 +327,8 @@ mod tests {
                 initial_slots,
                 &mut last_purge_slot,
                 10,
-                None,
+                &mut last_compaction_slot,
+                10,
             )
             .unwrap();
             time.stop();

core/src/lib.rs

@@ -11,10 +11,12 @@ pub mod banking_stage;
 pub mod broadcast_stage;
 pub mod cluster_info_vote_listener;
 pub mod commitment;
+pub mod commitment_service;
 mod deprecated;
 pub mod shred_fetch_stage;
 #[macro_use]
 pub mod contact_info;
+pub mod bank_weight_fork_choice;
 pub mod cluster_info;
 pub mod cluster_slots;
 pub mod consensus;
@@ -26,8 +28,10 @@ pub mod crds_gossip_push;
 pub mod crds_value;
 pub mod epoch_slots;
 pub mod fetch_stage;
+pub mod fork_choice;
 pub mod gen_keys;
 pub mod gossip_service;
+pub mod heaviest_subtree_fork_choice;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
 pub mod non_circulating_supply;
@@ -43,10 +47,12 @@ pub mod retransmit_stage;
 pub mod rewards_recorder_service;
 pub mod rpc;
 pub mod rpc_error;
+pub mod rpc_health;
 pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
+pub mod send_transaction_service;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;

core/src/non_circulating_supply.rs

@@ -57,8 +57,8 @@ solana_sdk::pubkeys!(
     [
         "9huDUZfxoJ7wGMTffUE7vh1xePqef7gyrLJu9NApncqA",
         "GK2zqSsXLA2rwVZk347RYhh6jJpRsCA69FjLW93ZGi3B",
+        "CWeRmXme7LmbaUWTZWFLt6FMnpzLCHaQLuR2TdgFn4Lq",
         "HCV5dGFJXRrJ3jhDYA4DCeb9TEDTwGGYXtT3wHksu2Zr",
-        "25odAafVXnd63L6Hq5Cx6xGmhKqkhE2y6UrLVuqUfWZj",
         "14FUT96s9swbmH7ZjpDvfEDywnAYy9zaNhv4xvezySGu",
         "HbZ5FfmKWNHC7uwk6TF1hVi6TCs7dtYfdjEcuPGgzFAg",
         "C7C8odR8oashR5Feyrq2tJKaXL18id1dSj2zbkDGL2C2",
@@ -76,6 +76,7 @@ solana_sdk::pubkeys!(
         "FR84wZQy3Y3j2gWz6pgETUiUoJtreMEuWfbg6573UCj9",
         "5q54XjQ7vDx4y6KphPeE97LUNiYGtP55spjvXAWPGBuf",
         "3o6xgkJ9sTmDeQWyfj3sxwon18fXJB9PV5LDc8sfgR4a",
+        "GumSE5HsMV5HCwBTv2D2D81yy9x17aDkvobkqAfTRgmo",
     ]
 );
 
@@ -85,6 +86,8 @@ solana_sdk::pubkeys!(
     [
         "8CUUMKYNGxdgYio5CLHRHyzMEhhVRMcqefgE6dLqnVRK",
         "3FFaheyqtyAXZSYxDzsr5CVKvJuvZD1WE1VEsBtDbRqB",
+        "FdGYQdiRky8NZzN9wZtczTBcWLYYRXrJ3LMDhqDPn5rM",
+        "4e6KwQpyzGQPfgVr5Jn3g5jLjbXB4pKPa2jRLohEb1QA",
     ]
 );
 

core/src/poh_recorder.rs

@@ -354,7 +354,7 @@ impl PohRecorder {
     pub fn tick(&mut self) {
         let now = Instant::now();
         let poh_entry = self.poh.lock().unwrap().tick();
-        inc_new_counter_warn!(
+        inc_new_counter_info!(
             "poh_recorder-tick_lock_contention",
             timing::duration_as_us(&now.elapsed()) as usize
         );
@@ -364,7 +364,7 @@ impl PohRecorder {
             trace!("tick_height {}", self.tick_height);
 
             if self.leader_first_tick_height.is_none() {
-                inc_new_counter_warn!(
+                inc_new_counter_info!(
                     "poh_recorder-tick_overhead",
                     timing::duration_as_us(&now.elapsed()) as usize
                 );
@@ -380,7 +380,7 @@ impl PohRecorder {
             self.tick_cache.push((entry, self.tick_height));
             let _ = self.flush_cache(true);
         }
-        inc_new_counter_warn!(
+        inc_new_counter_info!(
             "poh_recorder-tick_overhead",
             timing::duration_as_us(&now.elapsed()) as usize
         );

core/src/poh_service.rs

@@ -89,18 +89,22 @@ impl PohService {
         let poh = poh_recorder.lock().unwrap().poh.clone();
         let mut now = Instant::now();
         let mut num_ticks = 0;
+        let mut num_hashes = 0;
         loop {
+            num_hashes += NUM_HASHES_PER_BATCH;
             if poh.lock().unwrap().hash(NUM_HASHES_PER_BATCH) {
                 // Lock PohRecorder only for the final hash...
                 poh_recorder.lock().unwrap().tick();
                 num_ticks += 1;
                 if num_ticks >= DEFAULT_TICKS_PER_SLOT * 2 {
-                    datapoint_debug!(
+                    datapoint_info!(
                         "poh-service",
                         ("ticks", num_ticks as i64, i64),
+                        ("hashes", num_hashes as i64, i64),
                         ("elapsed_ms", now.elapsed().as_millis() as i64, i64),
                     );
                     num_ticks = 0;
+                    num_hashes = 0;
                     now = Instant::now();
                 }
                 if poh_exit.load(Ordering::Relaxed) {

core/src/progress_map.rs

@@ -42,7 +42,16 @@ impl ReplaySlotStats {
                 self.fetch_fail_elapsed as i64,
                 i64
             ),
-            ("entry_verification_time", self.verify_elapsed as i64, i64),
+            (
+                "entry_poh_verification_time",
+                self.poh_verify_elapsed as i64,
+                i64
+            ),
+            (
+                "entry_transaction_verification_time",
+                self.transaction_verify_elapsed as i64,
+                i64
+            ),
             ("replay_time", self.replay_elapsed as i64, i64),
             (
                 "replay_total_elapsed",

core/src/repair_service.rs

@@ -2,12 +2,16 @@
 //! regularly finds missing shreds in the ledger and sends repair requests for those shreds
 use crate::{
     cluster_info::ClusterInfo,
+    cluster_info_vote_listener::VoteTracker,
     cluster_slots::ClusterSlots,
-    consensus::VOTE_THRESHOLD_SIZE,
+    commitment::VOTE_THRESHOLD_SIZE,
     result::Result,
     serve_repair::{RepairType, ServeRepair, DEFAULT_NONCE},
 };
 use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
+use rand::distributions::{Distribution, WeightedIndex};
+use rand::{thread_rng, Rng, SeedableRng};
+use rand_chacha::ChaChaRng;
 use solana_ledger::{
     bank_forks::BankForks,
     blockstore::{Blockstore, CompletedSlotsReceiver, SlotMeta},
@@ -30,22 +34,39 @@ use std::{
 pub type DuplicateSlotsResetSender = CrossbeamSender<Slot>;
 pub type DuplicateSlotsResetReceiver = CrossbeamReceiver<Slot>;
 
-#[derive(Default)]
+#[derive(Default, Debug)]
+pub struct SlotRepairs {
+    highest_shred_index: u64,
+    // map from pubkey to total number of requests
+    pubkey_repairs: HashMap<Pubkey, u64>,
+}
+
+#[derive(Default, Debug)]
 pub struct RepairStatsGroup {
     pub count: u64,
     pub min: u64,
     pub max: u64,
+    pub slot_pubkeys: HashMap<Slot, SlotRepairs>,
 }
 
 impl RepairStatsGroup {
-    pub fn update(&mut self, slot: u64) {
+    pub fn update(&mut self, repair_peer_id: &Pubkey, slot: Slot, shred_index: u64) {
         self.count += 1;
+        let slot_repairs = self.slot_pubkeys.entry(slot).or_default();
+        // Increment total number of repairs of this type for this pubkey by 1
+        *slot_repairs
+            .pubkey_repairs
+            .entry(*repair_peer_id)
+            .or_default() += 1;
+        // Update the max requested shred index for this slot
+        slot_repairs.highest_shred_index =
+            std::cmp::max(slot_repairs.highest_shred_index, shred_index);
         self.min = std::cmp::min(self.min, slot);
         self.max = std::cmp::max(self.max, slot);
     }
 }
 
-#[derive(Default)]
+#[derive(Default, Debug)]
 pub struct RepairStats {
     pub shred: RepairStatsGroup,
     pub highest_shred: RepairStatsGroup,
@@ -82,7 +103,7 @@ impl Default for RepairSlotRange {
 #[derive(Default, Clone)]
 pub struct DuplicateSlotRepairStatus {
     start: u64,
-    repair_addr: Option<SocketAddr>,
+    repair_pubkey_and_addr: Option<(Pubkey, SocketAddr)>,
 }
 
 pub struct RepairService {
@@ -97,6 +118,7 @@ impl RepairService {
         cluster_info: Arc<ClusterInfo>,
         repair_info: RepairInfo,
         cluster_slots: Arc<ClusterSlots>,
+        vote_tracker: Arc<VoteTracker>,
     ) -> Self {
         let t_repair = Builder::new()
             .name("solana-repair-service".to_string())
@@ -108,6 +130,7 @@ impl RepairService {
                     cluster_info,
                     repair_info,
                     &cluster_slots,
+                    vote_tracker,
                 )
             })
             .unwrap();
@@ -122,13 +145,14 @@ impl RepairService {
         cluster_info: Arc<ClusterInfo>,
         repair_info: RepairInfo,
         cluster_slots: &ClusterSlots,
+        vote_tracker: Arc<VoteTracker>,
     ) {
         let serve_repair = ServeRepair::new(cluster_info.clone());
         let id = cluster_info.id();
         Self::initialize_lowest_slot(id, blockstore, &cluster_info);
         let mut repair_stats = RepairStats::default();
         let mut last_stats = Instant::now();
-        let mut duplicate_slot_repair_statuses = HashMap::new();
+        let duplicate_slot_repair_statuses = HashMap::new();
         Self::initialize_epoch_slots(
             blockstore,
             &cluster_info,
@@ -146,7 +170,7 @@ impl RepairService {
                 Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
                 Self::update_completed_slots(&repair_info.completed_slots_receiver, &cluster_info);
                 cluster_slots.update(new_root, &cluster_info, &repair_info.bank_forks);
-                let new_duplicate_slots = Self::find_new_duplicate_slots(
+                /*let new_duplicate_slots = Self::find_new_duplicate_slots(
                     &duplicate_slot_repair_statuses,
                     blockstore,
                     cluster_slots,
@@ -168,12 +192,13 @@ impl RepairService {
                     &serve_repair,
                     &mut repair_stats,
                     &repair_socket,
-                );
+                );*/
                 Self::generate_repairs(
                     blockstore,
                     root_bank.slot(),
                     MAX_REPAIR_LENGTH,
                     &duplicate_slot_repair_statuses,
+                    &vote_tracker,
                 )
             };
 
@@ -194,10 +219,11 @@ impl RepairService {
                 });
             }
 
-            if last_stats.elapsed().as_secs() > 1 {
+            if last_stats.elapsed().as_secs() > 2 {
                 let repair_total = repair_stats.shred.count
                     + repair_stats.highest_shred.count
                     + repair_stats.orphan.count;
+                info!("repair_stats: {:?}", repair_stats);
                 if repair_total > 0 {
                     datapoint_info!(
                         "serve_repair-repair",
@@ -254,6 +280,7 @@ impl RepairService {
         root: Slot,
         max_repairs: usize,
         duplicate_slot_repair_statuses: &HashMap<Slot, DuplicateSlotRepairStatus>,
+        vote_tracker: &Arc<VoteTracker>,
     ) -> Result<Vec<RepairType>> {
         // Slot height and shred indexes for shreds we want to repair
         let mut repairs: Vec<RepairType> = vec![];
@@ -263,10 +290,9 @@ impl RepairService {
             max_repairs,
             root,
             duplicate_slot_repair_statuses,
+            vote_tracker,
         );
 
-        // TODO: Incorporate gossip to determine priorities for repair?
-
         // Try to resolve orphans in blockstore
         let orphans = blockstore.orphans_iterator(root + 1).unwrap();
         Self::generate_repairs_for_orphans(orphans, &mut repairs);
@@ -274,6 +300,7 @@ impl RepairService {
         Ok(repairs)
     }
 
+    #[allow(dead_code)]
     fn generate_duplicate_repairs_for_slot(
         blockstore: &Blockstore,
         slot: Slot,
@@ -298,6 +325,7 @@ impl RepairService {
         }
     }
 
+    #[allow(dead_code)]
     fn generate_and_send_duplicate_repairs(
         duplicate_slot_repair_statuses: &mut HashMap<Slot, DuplicateSlotRepairStatus>,
         cluster_slots: &ClusterSlots,
@@ -308,7 +336,7 @@ impl RepairService {
     ) {
         duplicate_slot_repair_statuses.retain(|slot, status| {
             Self::update_duplicate_slot_repair_addr(*slot, status, cluster_slots, serve_repair);
-            if let Some(repair_addr) = status.repair_addr {
+            if let Some((repair_pubkey, repair_addr)) = status.repair_pubkey_and_addr {
                 let repairs = Self::generate_duplicate_repairs_for_slot(&blockstore, *slot);
 
                 if let Some(repairs) = repairs {
@@ -316,12 +344,16 @@ impl RepairService {
                         if let Err(e) = Self::serialize_and_send_request(
                             &repair_type,
                             repair_socket,
+                            &repair_pubkey,
                             &repair_addr,
                             serve_repair,
                             repair_stats,
                             DEFAULT_NONCE,
                         ) {
-                            info!("repair req send_to({}) error {:?}", repair_addr, e);
+                            info!(
+                                "repair req send_to {} ({}) error {:?}",
+                                repair_pubkey, repair_addr, e
+                            );
                         }
                     }
                     true
@@ -334,19 +366,23 @@ impl RepairService {
         })
     }
 
+    #[allow(dead_code)]
     fn serialize_and_send_request(
         repair_type: &RepairType,
         repair_socket: &UdpSocket,
+        repair_pubkey: &Pubkey,
         to: &SocketAddr,
         serve_repair: &ServeRepair,
         repair_stats: &mut RepairStats,
         nonce: Nonce,
     ) -> Result<()> {
-        let req = serve_repair.map_repair_request(&repair_type, repair_stats, nonce)?;
+        let req =
+            serve_repair.map_repair_request(&repair_type, repair_pubkey, repair_stats, nonce)?;
         repair_socket.send_to(&req, to)?;
         Ok(())
     }
 
+    #[allow(dead_code)]
     fn update_duplicate_slot_repair_addr(
         slot: Slot,
         status: &mut DuplicateSlotRepairStatus,
@@ -354,16 +390,17 @@ impl RepairService {
         serve_repair: &ServeRepair,
     ) {
         let now = timestamp();
-        if status.repair_addr.is_none()
+        if status.repair_pubkey_and_addr.is_none()
             || now.saturating_sub(status.start) >= MAX_DUPLICATE_WAIT_MS as u64
         {
-            let repair_addr =
+            let repair_pubkey_and_addr =
                 serve_repair.repair_request_duplicate_compute_best_peer(slot, cluster_slots);
-            status.repair_addr = repair_addr.ok();
+            status.repair_pubkey_and_addr = repair_pubkey_and_addr.ok();
             status.start = timestamp();
         }
     }
 
+    #[allow(dead_code)]
     fn process_new_duplicate_slots(
         new_duplicate_slots: &[Slot],
         duplicate_slot_repair_statuses: &mut HashMap<Slot, DuplicateSlotRepairStatus>,
@@ -396,17 +433,18 @@ impl RepairService {
 
             // Mark this slot as special repair, try to download from single
             // validator to avoid corruption
-            let repair_addr = serve_repair
+            let repair_pubkey_and_addr = serve_repair
                 .repair_request_duplicate_compute_best_peer(*slot, cluster_slots)
                 .ok();
             let new_duplicate_slot_repair_status = DuplicateSlotRepairStatus {
                 start: timestamp(),
-                repair_addr,
+                repair_pubkey_and_addr,
             };
             duplicate_slot_repair_statuses.insert(*slot, new_duplicate_slot_repair_status);
         }
     }
 
+    #[allow(dead_code)]
     fn find_new_duplicate_slots(
         duplicate_slot_repair_statuses: &HashMap<Slot, DuplicateSlotRepairStatus>,
         blockstore: &Blockstore,
@@ -424,7 +462,7 @@ impl RepairService {
                     warn!(
                         "Repaired version of slot {} most recently (but maybe not entirely)
                         from {:?} has failed again",
-                        dead_slot, status.repair_addr
+                        dead_slot, status.repair_pubkey_and_addr
                     );
                 }
                 cluster_slots
@@ -503,27 +541,64 @@ impl RepairService {
         max_repairs: usize,
         slot: Slot,
         duplicate_slot_repair_statuses: &HashMap<Slot, DuplicateSlotRepairStatus>,
+        vote_tracker: &Arc<VoteTracker>,
     ) {
+        let mut seed = [0u8; 32];
+        thread_rng().fill(&mut seed);
+        let rng = &mut ChaChaRng::from_seed(seed);
         let mut pending_slots = vec![slot];
         while repairs.len() < max_repairs && !pending_slots.is_empty() {
-            let slot = pending_slots.pop().unwrap();
-            if duplicate_slot_repair_statuses.contains_key(&slot) {
-                // These are repaired through a different path
-                continue;
+            pending_slots.retain(|slot| !duplicate_slot_repair_statuses.contains_key(slot));
+            let mut next_pending_slots = vec![];
+            let mut level_repairs = HashMap::new();
+            for slot in &pending_slots {
+                if let Some(slot_meta) = blockstore.meta(*slot).unwrap() {
+                    let new_repairs = Self::generate_repairs_for_slot(
+                        blockstore,
+                        *slot,
+                        &slot_meta,
+                        std::usize::MAX,
+                    );
+                    if !new_repairs.is_empty() {
+                        level_repairs.insert(*slot, new_repairs);
+                    }
+                    next_pending_slots.extend(slot_meta.next_slots);
+                }
             }
-            if let Some(slot_meta) = blockstore.meta(slot).unwrap() {
-                let new_repairs = Self::generate_repairs_for_slot(
-                    blockstore,
-                    slot,
-                    &slot_meta,
-                    max_repairs - repairs.len(),
-                );
-                repairs.extend(new_repairs);
-                let next_slots = slot_meta.next_slots;
-                pending_slots.extend(next_slots);
-            } else {
-                break;
+
+            if !level_repairs.is_empty() {
+                let mut slots_to_repair: Vec<_> = level_repairs.keys().cloned().collect();
+                let mut weights: Vec<_> = {
+                    let r_vote_tracker = vote_tracker.slot_vote_trackers.read().unwrap();
+                    slots_to_repair
+                        .iter()
+                        .map(|slot| {
+                            if let Some(slot_vote_tracker) = r_vote_tracker.get(slot) {
+                                std::cmp::max(slot_vote_tracker.read().unwrap().total_stake, 1)
+                            } else {
+                                // should it be something else?
+                                1
+                            }
+                        })
+                        .collect()
+                };
+
+                let mut weighted_index = WeightedIndex::new(weights.clone()).unwrap();
+                while repairs.len() < max_repairs && !level_repairs.is_empty() {
+                    let index = weighted_index.sample(rng);
+                    let slot_repairs = level_repairs.get_mut(&slots_to_repair[index]).unwrap();
+                    repairs.push(slot_repairs.remove(0));
+                    if slot_repairs.is_empty() {
+                        level_repairs.remove(&slots_to_repair[index]);
+                        slots_to_repair.remove(index);
+                        weights.remove(index);
+                        if !weights.is_empty() {
+                            weighted_index = WeightedIndex::new(weights.clone()).unwrap();
+                        }
+                    }
+                }
             }
+            pending_slots = next_pending_slots;
         }
     }
 
@@ -610,8 +685,10 @@ mod test {
             let (shreds2, _) = make_slot_entries(5, 2, 1);
             shreds.extend(shreds2);
             blockstore.insert_shreds(shreds, None, false).unwrap();
+            let vote_tracker = Arc::new(VoteTracker::default());
             assert_eq!(
-                RepairService::generate_repairs(&blockstore, 0, 2, &HashMap::new()).unwrap(),
+                RepairService::generate_repairs(&blockstore, 0, 2, &HashMap::new(), &vote_tracker)
+                    .unwrap(),
                 vec![RepairType::HighestShred(0, 0), RepairType::Orphan(2)]
             );
         }
@@ -631,9 +708,11 @@ mod test {
             // any shreds for
             blockstore.insert_shreds(shreds, None, false).unwrap();
 
+            let vote_tracker = Arc::new(VoteTracker::default());
             // Check that repair tries to patch the empty slot
             assert_eq!(
-                RepairService::generate_repairs(&blockstore, 0, 2, &HashMap::new()).unwrap(),
+                RepairService::generate_repairs(&blockstore, 0, 2, &HashMap::new(), &vote_tracker)
+                    .unwrap(),
                 vec![RepairType::HighestShred(0, 0)]
             );
         }
@@ -678,9 +757,16 @@ mod test {
                 })
                 .collect();
 
+            let vote_tracker = Arc::new(VoteTracker::default());
             assert_eq!(
-                RepairService::generate_repairs(&blockstore, 0, std::usize::MAX, &HashMap::new())
-                    .unwrap(),
+                RepairService::generate_repairs(
+                    &blockstore,
+                    0,
+                    std::usize::MAX,
+                    &HashMap::new(),
+                    &vote_tracker
+                )
+                .unwrap(),
                 expected
             );
 
@@ -689,7 +775,8 @@ mod test {
                     &blockstore,
                     0,
                     expected.len() - 2,
-                    &HashMap::new()
+                    &HashMap::new(),
+                    &vote_tracker,
                 )
                 .unwrap()[..],
                 expected[0..expected.len() - 2]
@@ -698,6 +785,55 @@ mod test {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
+    #[test]
+    pub fn test_repairs_distributed_across_slots() {
+        solana_logger::setup();
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+
+            let num_entries_per_slot = 100;
+
+            // Create some shreds
+            for i in 1..10 {
+                let (shreds, _) = make_slot_entries(i, 0, num_entries_per_slot as u64);
+
+                // Only insert the first shred
+                blockstore
+                    .insert_shreds(shreds[..1].to_vec(), None, false)
+                    .unwrap();
+            }
+
+            let vote_tracker = Arc::new(VoteTracker::default());
+            let repairs = RepairService::generate_repairs(
+                &blockstore,
+                0,
+                num_entries_per_slot,
+                &HashMap::new(),
+                &vote_tracker,
+            )
+            .unwrap();
+            let mut repairs_slots = HashMap::new();
+            for repair in repairs {
+                match repair {
+                    RepairType::Shred(slot, _shred_index) => {
+                        *repairs_slots.entry(slot).or_insert(0) += 1;
+                    }
+                    RepairType::HighestShred(slot, _shred_index) => {
+                        *repairs_slots.entry(slot).or_insert(0) += 1;
+                    }
+                    RepairType::Orphan(slot) => {
+                        *repairs_slots.entry(slot).or_insert(0) += 1;
+                    }
+                }
+            }
+            for i in 1..10 {
+                assert!(repairs_slots.contains_key(&i));
+            }
+        }
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
     #[test]
     pub fn test_generate_highest_repair() {
         let blockstore_path = get_tmp_ledger_path!();
@@ -719,9 +855,16 @@ mod test {
             let expected: Vec<RepairType> =
                 vec![RepairType::HighestShred(0, num_shreds_per_slot - 1)];
 
+            let vote_tracker = Arc::new(VoteTracker::default());
             assert_eq!(
-                RepairService::generate_repairs(&blockstore, 0, std::usize::MAX, &HashMap::new())
-                    .unwrap(),
+                RepairService::generate_repairs(
+                    &blockstore,
+                    0,
+                    std::usize::MAX,
+                    &HashMap::new(),
+                    &vote_tracker
+                )
+                .unwrap(),
                 expected
             );
         }
@@ -874,7 +1017,7 @@ mod test {
         let receive_socket = &UdpSocket::bind("0.0.0.0:0").unwrap();
         let duplicate_status = DuplicateSlotRepairStatus {
             start: std::u64::MAX,
-            repair_addr: None,
+            repair_pubkey_and_addr: None,
         };
 
         // Insert some shreds to create a SlotMeta,
@@ -899,7 +1042,7 @@ mod test {
         assert!(duplicate_slot_repair_statuses
             .get(&dead_slot)
             .unwrap()
-            .repair_addr
+            .repair_pubkey_and_addr
             .is_none());
         assert!(duplicate_slot_repair_statuses.get(&dead_slot).is_some());
 
@@ -907,7 +1050,8 @@ mod test {
         duplicate_slot_repair_statuses
             .get_mut(&dead_slot)
             .unwrap()
-            .repair_addr = Some(receive_socket.local_addr().unwrap());
+            .repair_pubkey_and_addr =
+            Some((Pubkey::default(), receive_socket.local_addr().unwrap()));
 
         // Slot is not yet full, should not get filtered from `duplicate_slot_repair_statuses`
         RepairService::generate_and_send_duplicate_repairs(
@@ -939,7 +1083,10 @@ mod test {
 
     #[test]
     pub fn test_update_duplicate_slot_repair_addr() {
-        let dummy_addr = Some(UdpSocket::bind("0.0.0.0:0").unwrap().local_addr().unwrap());
+        let dummy_addr = Some((
+            Pubkey::default(),
+            UdpSocket::bind("0.0.0.0:0").unwrap().local_addr().unwrap(),
+        ));
         let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(
             Node::new_localhost().info,
         ));
@@ -957,7 +1104,7 @@ mod test {
         // address
         let mut duplicate_status = DuplicateSlotRepairStatus {
             start: std::u64::MAX,
-            repair_addr: dummy_addr,
+            repair_pubkey_and_addr: dummy_addr,
         };
         RepairService::update_duplicate_slot_repair_addr(
             dead_slot,
@@ -965,12 +1112,12 @@ mod test {
             &cluster_slots,
             &serve_repair,
         );
-        assert_eq!(duplicate_status.repair_addr, dummy_addr);
+        assert_eq!(duplicate_status.repair_pubkey_and_addr, dummy_addr);
 
         // If the repair address is None, should try to update
         let mut duplicate_status = DuplicateSlotRepairStatus {
             start: std::u64::MAX,
-            repair_addr: None,
+            repair_pubkey_and_addr: None,
         };
         RepairService::update_duplicate_slot_repair_addr(
             dead_slot,
@@ -978,12 +1125,12 @@ mod test {
             &cluster_slots,
             &serve_repair,
         );
-        assert!(duplicate_status.repair_addr.is_some());
+        assert!(duplicate_status.repair_pubkey_and_addr.is_some());
 
-        // If sufficient time has passssed, should try to update
+        // If sufficient time has passed, should try to update
         let mut duplicate_status = DuplicateSlotRepairStatus {
             start: timestamp() - MAX_DUPLICATE_WAIT_MS as u64,
-            repair_addr: dummy_addr,
+            repair_pubkey_and_addr: dummy_addr,
         };
         RepairService::update_duplicate_slot_repair_addr(
             dead_slot,
@@ -991,7 +1138,7 @@ mod test {
             &cluster_slots,
             &serve_repair,
         );
-        assert_ne!(duplicate_status.repair_addr, dummy_addr);
+        assert_ne!(duplicate_status.repair_pubkey_and_addr, dummy_addr);
     }
 
     #[test]

core/src/retransmit_stage.rs

@@ -1,8 +1,10 @@
 //! The `retransmit_stage` retransmits shreds between validators
 
+use crate::cluster_info_vote_listener::VoteTracker;
 use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
     cluster_slots::ClusterSlots,
+    contact_info::ContactInfo,
     repair_service::DuplicateSlotsResetSender,
     repair_service::RepairInfo,
     result::{Error, Result},
@@ -18,8 +20,9 @@ use solana_ledger::{
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_error;
 use solana_perf::packet::Packets;
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Epoch, Slot};
 use solana_sdk::epoch_schedule::EpochSchedule;
+use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
 use solana_streamer::streamer::PacketReceiver;
 use std::{
@@ -44,6 +47,8 @@ struct RetransmitStats {
     total_packets: AtomicU64,
     total_batches: AtomicU64,
     total_time: AtomicU64,
+    epoch_fetch: AtomicU64,
+    epoch_cache_update: AtomicU64,
     repair_total: AtomicU64,
     discard_total: AtomicU64,
     retransmit_total: AtomicU64,
@@ -65,6 +70,8 @@ fn update_retransmit_stats(
     peers_len: usize,
     packets_by_slot: HashMap<Slot, usize>,
     packets_by_source: HashMap<String, usize>,
+    epoch_fetch: u64,
+    epoch_cach_update: u64,
 ) {
     stats.total_time.fetch_add(total_time, Ordering::Relaxed);
     stats
@@ -83,6 +90,10 @@ fn update_retransmit_stats(
         .compute_turbine_peers_total
         .fetch_add(compute_turbine_peers_total, Ordering::Relaxed);
     stats.total_batches.fetch_add(1, Ordering::Relaxed);
+    stats.epoch_fetch.fetch_add(epoch_fetch, Ordering::Relaxed);
+    stats
+        .epoch_cache_update
+        .fetch_add(epoch_cach_update, Ordering::Relaxed);
     {
         let mut stats_packets_by_slot = stats.packets_by_slot.lock().unwrap();
         for (slot, count) in packets_by_slot {
@@ -107,6 +118,16 @@ fn update_retransmit_stats(
                 stats.total_time.swap(0, Ordering::Relaxed) as i64,
                 i64
             ),
+            (
+                "epoch_fetch",
+                stats.epoch_fetch.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "epoch_cache_update",
+                stats.epoch_cache_update.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
             (
                 "total_batches",
                 stats.total_batches.swap(0, Ordering::Relaxed) as i64,
@@ -139,15 +160,23 @@ fn update_retransmit_stats(
             ),
         );
         let mut packets_by_slot = stats.packets_by_slot.lock().unwrap();
-        info!("retransmit: packets_by_slot: {:#?}", packets_by_slot);
+        info!("retransmit: packets_by_slot: {:?}", packets_by_slot);
         packets_by_slot.clear();
         drop(packets_by_slot);
         let mut packets_by_source = stats.packets_by_source.lock().unwrap();
-        info!("retransmit: packets_by_source: {:#?}", packets_by_source);
+        info!("retransmit: packets_by_source: {:?}", packets_by_source);
         packets_by_source.clear();
     }
 }
 
+#[derive(Default)]
+struct EpochStakesCache {
+    epoch: Epoch,
+    stakes: Option<Arc<HashMap<Pubkey, u64>>>,
+    peers: Vec<ContactInfo>,
+    stakes_and_index: Vec<(u64, usize)>,
+}
+
 fn retransmit(
     bank_forks: &Arc<RwLock<BankForks>>,
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
@@ -156,6 +185,8 @@ fn retransmit(
     sock: &UdpSocket,
     id: u32,
     stats: &Arc<RetransmitStats>,
+    epoch_stakes_cache: &Arc<RwLock<EpochStakesCache>>,
+    last_peer_update: &Arc<AtomicU64>,
 ) -> Result<()> {
     let timer = Duration::new(1, 0);
     let r_lock = r.lock().unwrap();
@@ -172,12 +203,42 @@ fn retransmit(
     }
     drop(r_lock);
 
+    let mut epoch_fetch = Measure::start("retransmit_epoch_fetch");
     let r_bank = bank_forks.read().unwrap().working_bank();
     let bank_epoch = r_bank.get_leader_schedule_epoch(r_bank.slot());
+    epoch_fetch.stop();
+
+    let mut epoch_cache_update = Measure::start("retransmit_epoch_cach_update");
+    let mut r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    if r_epoch_stakes_cache.epoch != bank_epoch {
+        drop(r_epoch_stakes_cache);
+        let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
+        if w_epoch_stakes_cache.epoch != bank_epoch {
+            let stakes = staking_utils::staked_nodes_at_epoch(&r_bank, bank_epoch);
+            let stakes = stakes.map(Arc::new);
+            w_epoch_stakes_cache.stakes = stakes;
+            w_epoch_stakes_cache.epoch = bank_epoch;
+        }
+        drop(w_epoch_stakes_cache);
+        r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    }
+
+    let now = timestamp();
+    let last = last_peer_update.load(Ordering::Relaxed);
+    if now - last > 1000 && last_peer_update.compare_and_swap(last, now, Ordering::Relaxed) == last
+    {
+        drop(r_epoch_stakes_cache);
+        let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
+        let (peers, stakes_and_index) =
+            cluster_info.sorted_retransmit_peers_and_stakes(w_epoch_stakes_cache.stakes.clone());
+        w_epoch_stakes_cache.peers = peers;
+        w_epoch_stakes_cache.stakes_and_index = stakes_and_index;
+        drop(w_epoch_stakes_cache);
+        r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    }
     let mut peers_len = 0;
-    let stakes = staking_utils::staked_nodes_at_epoch(&r_bank, bank_epoch);
-    let stakes = stakes.map(Arc::new);
-    let (peers, stakes_and_index) = cluster_info.sorted_retransmit_peers_and_stakes(stakes);
+    epoch_cache_update.stop();
+
     let my_id = cluster_info.id();
     let mut discard_total = 0;
     let mut repair_total = 0;
@@ -202,8 +263,8 @@ fn retransmit(
             let mut compute_turbine_peers = Measure::start("turbine_start");
             let (my_index, mut shuffled_stakes_and_index) = ClusterInfo::shuffle_peers_and_index(
                 &my_id,
-                &peers,
-                &stakes_and_index,
+                &r_epoch_stakes_cache.peers,
+                &r_epoch_stakes_cache.stakes_and_index,
                 packet.meta.seed,
             );
             peers_len = cmp::max(peers_len, shuffled_stakes_and_index.len());
@@ -216,8 +277,14 @@ fn retransmit(
 
             let (neighbors, children) =
                 compute_retransmit_peers(DATA_PLANE_FANOUT, my_index, indexes);
-            let neighbors: Vec<_> = neighbors.into_iter().map(|index| &peers[index]).collect();
-            let children: Vec<_> = children.into_iter().map(|index| &peers[index]).collect();
+            let neighbors: Vec<_> = neighbors
+                .into_iter()
+                .map(|index| &r_epoch_stakes_cache.peers[index])
+                .collect();
+            let children: Vec<_> = children
+                .into_iter()
+                .map(|index| &r_epoch_stakes_cache.peers[index])
+                .collect();
             compute_turbine_peers.stop();
             compute_turbine_peers_total += compute_turbine_peers.as_us();
 
@@ -258,6 +325,8 @@ fn retransmit(
         peers_len,
         packets_by_slot,
         packets_by_source,
+        epoch_fetch.as_us(),
+        epoch_cache_update.as_us(),
     );
 
     Ok(())
@@ -287,6 +356,8 @@ pub fn retransmitter(
             let r = r.clone();
             let cluster_info = cluster_info.clone();
             let stats = stats.clone();
+            let epoch_stakes_cache = Arc::new(RwLock::new(EpochStakesCache::default()));
+            let last_peer_update = Arc::new(AtomicU64::new(0));
 
             Builder::new()
                 .name("solana-retransmitter".to_string())
@@ -301,6 +372,8 @@ pub fn retransmitter(
                             &sockets[s],
                             s as u32,
                             &stats,
+                            &epoch_stakes_cache,
+                            &last_peer_update,
                         ) {
                             match e {
                                 Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
@@ -341,6 +414,7 @@ impl RetransmitStage {
         shred_version: u16,
         cluster_slots: Arc<ClusterSlots>,
         duplicate_slots_reset_sender: DuplicateSlotsResetSender,
+        vote_tracker: Arc<VoteTracker>,
     ) -> Self {
         let (retransmit_sender, retransmit_receiver) = channel();
 
@@ -385,6 +459,7 @@ impl RetransmitStage {
                 rv && is_connected
             },
             cluster_slots,
+            vote_tracker,
         );
 
         let thread_hdls = t_retransmit;

core/src/rpc_error.rs

@@ -3,6 +3,7 @@ use solana_sdk::clock::Slot;
 
 const JSON_RPC_SERVER_ERROR_0: i64 = -32000;
 const JSON_RPC_SERVER_ERROR_1: i64 = -32001;
+const JSON_RPC_SERVER_ERROR_2: i64 = -32002;
 
 pub enum RpcCustomError {
     NonexistentClusterRoot {
@@ -13,6 +14,9 @@ pub enum RpcCustomError {
         slot: Slot,
         first_available_block: Slot,
     },
+    SendTransactionPreflightFailure {
+        message: String,
+    },
 }
 
 impl From<RpcCustomError> for Error {
@@ -40,6 +44,11 @@ impl From<RpcCustomError> for Error {
                 ),
                 data: None,
             },
+            RpcCustomError::SendTransactionPreflightFailure { message } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_2),
+                message,
+                data: None,
+            },
         }
     }
 }

core/src/rpc_health.rs

@@ -0,0 +1,118 @@
+use crate::cluster_info::ClusterInfo;
+use solana_sdk::pubkey::Pubkey;
+use std::{
+    collections::HashSet,
+    sync::atomic::{AtomicBool, Ordering},
+    sync::Arc,
+};
+
+#[derive(PartialEq, Clone, Copy)]
+pub enum RpcHealthStatus {
+    Ok,
+    Behind, // Validator is behind its trusted validators
+}
+
+pub struct RpcHealth {
+    cluster_info: Arc<ClusterInfo>,
+    trusted_validators: Option<HashSet<Pubkey>>,
+    health_check_slot_distance: u64,
+    override_health_check: Arc<AtomicBool>,
+    #[cfg(test)]
+    stub_health_status: std::sync::RwLock<Option<RpcHealthStatus>>,
+}
+
+impl RpcHealth {
+    pub fn new(
+        cluster_info: Arc<ClusterInfo>,
+        trusted_validators: Option<HashSet<Pubkey>>,
+        health_check_slot_distance: u64,
+        override_health_check: Arc<AtomicBool>,
+    ) -> Self {
+        Self {
+            cluster_info,
+            trusted_validators,
+            health_check_slot_distance,
+            override_health_check,
+            #[cfg(test)]
+            stub_health_status: std::sync::RwLock::new(None),
+        }
+    }
+
+    pub fn check(&self) -> RpcHealthStatus {
+        #[cfg(test)]
+        {
+            if let Some(stub_health_status) = *self.stub_health_status.read().unwrap() {
+                return stub_health_status;
+            }
+        }
+
+        if self.override_health_check.load(Ordering::Relaxed) {
+            RpcHealthStatus::Ok
+        } else if let Some(trusted_validators) = &self.trusted_validators {
+            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
+                (
+                    self.cluster_info
+                        .get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| {
+                            hashes
+                                .iter()
+                                .max_by(|a, b| a.0.cmp(&b.0))
+                                .map(|slot_hash| slot_hash.0)
+                        })
+                        .flatten()
+                        .unwrap_or(0),
+                    trusted_validators
+                        .iter()
+                        .map(|trusted_validator| {
+                            self.cluster_info
+                                .get_accounts_hash_for_node(&trusted_validator, |hashes| {
+                                    hashes
+                                        .iter()
+                                        .max_by(|a, b| a.0.cmp(&b.0))
+                                        .map(|slot_hash| slot_hash.0)
+                                })
+                                .flatten()
+                                .unwrap_or(0)
+                        })
+                        .max()
+                        .unwrap_or(0),
+                )
+            };
+
+            // This validator is considered healthy if its latest account hash slot is within
+            // `health_check_slot_distance` of the latest trusted validator's account hash slot
+            if latest_account_hash_slot > 0
+                && latest_trusted_validator_account_hash_slot > 0
+                && latest_account_hash_slot
+                    > latest_trusted_validator_account_hash_slot
+                        .saturating_sub(self.health_check_slot_distance)
+            {
+                RpcHealthStatus::Ok
+            } else {
+                warn!(
+                    "health check: me={}, latest trusted_validator={}",
+                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
+                );
+                RpcHealthStatus::Behind
+            }
+        } else {
+            // No trusted validator point of reference available, so this validator is healthy
+            // because it's running
+            RpcHealthStatus::Ok
+        }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn stub() -> Arc<Self> {
+        Arc::new(Self::new(
+            Arc::new(ClusterInfo::default()),
+            None,
+            42,
+            Arc::new(AtomicBool::new(false)),
+        ))
+    }
+
+    #[cfg(test)]
+    pub(crate) fn stub_set_health_status(&self, stub_health_status: Option<RpcHealthStatus>) {
+        *self.stub_health_status.write().unwrap() = stub_health_status;
+    }
+}

core/src/rpc_pubsub.rs

@@ -355,8 +355,8 @@ mod tests {
     use super::*;
     use crate::{
         cluster_info_vote_listener::{ClusterInfoVoteListener, VoteTracker},
-        commitment::{BlockCommitmentCache, CacheSlotInfo},
-        rpc_subscriptions::tests::robust_poll_or_panic,
+        commitment::BlockCommitmentCache,
+        rpc_subscriptions::{tests::robust_poll_or_panic, CacheSlotInfo},
     };
     use crossbeam_channel::unbounded;
     use jsonrpc_core::{futures::sync::mpsc, Response};
@@ -374,6 +374,7 @@ mod tests {
     };
     use solana_sdk::{
         hash::Hash,
+        message::Message,
         pubkey::Pubkey,
         signature::{Keypair, Signer},
         system_program, system_transaction,
@@ -420,7 +421,7 @@ mod tests {
         let bob_pubkey = bob.pubkey();
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let rpc = RpcSolPubSubImpl {
@@ -471,7 +472,7 @@ mod tests {
         let bob_pubkey = Pubkey::new_rand();
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
@@ -528,7 +529,7 @@ mod tests {
         let budget_program_id = solana_budget_program::id();
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let bank0 = bank_forks.read().unwrap().get(0).unwrap().clone();
         let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
         bank_forks.write().unwrap().insert(bank1);
@@ -569,11 +570,8 @@ mod tests {
             None,
             51,
         );
-        let tx = Transaction::new_signed_instructions(
-            &[&contract_funds, &contract_state],
-            &ixs,
-            blockhash,
-        );
+        let message = Message::new(&ixs, Some(&contract_funds.pubkey()));
+        let tx = Transaction::new(&[&contract_funds, &contract_state], message, blockhash);
         process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap();
         sleep(Duration::from_millis(200));
 
@@ -615,7 +613,8 @@ mod tests {
             &contract_state.pubkey(),
             &bob_pubkey,
         );
-        let tx = Transaction::new_signed_instructions(&[&witness], &[ix], blockhash);
+        let message = Message::new(&[ix], Some(&witness.pubkey()));
+        let tx = Transaction::new(&[&witness], message, blockhash);
         process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap();
         sleep(Duration::from_millis(200));
 
@@ -638,7 +637,7 @@ mod tests {
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, Bank::new(&genesis_config))));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(&genesis_config))));
 
         let mut io = PubSubHandler::default();
         let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
@@ -680,7 +679,7 @@ mod tests {
         } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(1, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
@@ -731,7 +730,7 @@ mod tests {
         } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let bank0 = bank_forks.read().unwrap().get(0).unwrap().clone();
         let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
         bank_forks.write().unwrap().insert(bank1);
@@ -807,7 +806,7 @@ mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
@@ -837,7 +836,7 @@ mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
@@ -884,7 +883,7 @@ mod tests {
             create_genesis_config_with_vote_accounts(10_000, &validator_voting_keypairs, 100);
         let exit = Arc::new(AtomicBool::new(false));
         let bank = Bank::new(&genesis_config);
-        let bank_forks = BankForks::new(0, bank);
+        let bank_forks = BankForks::new(bank);
         let bank = bank_forks.get(0).unwrap().clone();
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 
@@ -944,7 +943,7 @@ mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let rpc = RpcSolPubSubImpl::default_with_blockstore_bank_forks(blockstore, bank_forks);
         let session = create_session();
         let (subscriber, _id_receiver, _) = Subscriber::new_test("voteNotification");

core/src/rpc_pubsub_service.rs

@@ -93,7 +93,7 @@ mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let subscriptions = Arc::new(RpcSubscriptions::new(
             &exit,
             bank_forks,

core/src/rpc_service.rs

@@ -1,7 +1,8 @@
 //! The `rpc_service` module implements the Solana JSON RPC service.
 
 use crate::{
-    cluster_info::ClusterInfo, commitment::BlockCommitmentCache, rpc::*, validator::ValidatorExit,
+    cluster_info::ClusterInfo, commitment::BlockCommitmentCache, rpc::*, rpc_health::*,
+    send_transaction_service::SendTransactionService, validator::ValidatorExit,
 };
 use jsonrpc_core::MetaIoHandler;
 use jsonrpc_http_server::{
@@ -19,20 +20,17 @@ use std::{
     collections::HashSet,
     net::SocketAddr,
     path::{Path, PathBuf},
+    sync::atomic::{AtomicBool, Ordering},
     sync::{mpsc::channel, Arc, RwLock},
     thread::{self, Builder, JoinHandle},
 };
 use tokio::prelude::Future;
 
-// If trusted validators are specified, consider this validator healthy if its latest account hash
-// is no further behind than this distance from the latest trusted validator account hash
-const HEALTH_CHECK_SLOT_DISTANCE: u64 = 150;
-
 pub struct JsonRpcService {
     thread_hdl: JoinHandle<()>,
 
     #[cfg(test)]
-    pub request_processor: Arc<RwLock<JsonRpcRequestProcessor>>, // Used only by test_rpc_new()...
+    pub request_processor: JsonRpcRequestProcessor, // Used only by test_rpc_new()...
 
     close_handle: Option<CloseHandle>,
 }
@@ -41,18 +39,16 @@ struct RpcRequestMiddleware {
     ledger_path: PathBuf,
     snapshot_archive_path_regex: Regex,
     snapshot_config: Option<SnapshotConfig>,
-    cluster_info: Arc<ClusterInfo>,
-    trusted_validators: Option<HashSet<Pubkey>>,
     bank_forks: Arc<RwLock<BankForks>>,
+    health: Arc<RpcHealth>,
 }
 
 impl RpcRequestMiddleware {
     pub fn new(
         ledger_path: PathBuf,
         snapshot_config: Option<SnapshotConfig>,
-        cluster_info: Arc<ClusterInfo>,
-        trusted_validators: Option<HashSet<Pubkey>>,
         bank_forks: Arc<RwLock<BankForks>>,
+        health: Arc<RpcHealth>,
     ) -> Self {
         Self {
             ledger_path,
@@ -61,9 +57,8 @@ impl RpcRequestMiddleware {
             )
             .unwrap(),
             snapshot_config,
-            cluster_info,
-            trusted_validators,
             bank_forks,
+            health,
         }
     }
 
@@ -134,58 +129,10 @@ impl RpcRequestMiddleware {
     }
 
     fn health_check(&self) -> &'static str {
-        let response = if let Some(trusted_validators) = &self.trusted_validators {
-            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
-                (
-                    self.cluster_info
-                        .get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| {
-                            hashes
-                                .iter()
-                                .max_by(|a, b| a.0.cmp(&b.0))
-                                .map(|slot_hash| slot_hash.0)
-                        })
-                        .flatten()
-                        .unwrap_or(0),
-                    trusted_validators
-                        .iter()
-                        .map(|trusted_validator| {
-                            self.cluster_info
-                                .get_accounts_hash_for_node(&trusted_validator, |hashes| {
-                                    hashes
-                                        .iter()
-                                        .max_by(|a, b| a.0.cmp(&b.0))
-                                        .map(|slot_hash| slot_hash.0)
-                                })
-                                .flatten()
-                                .unwrap_or(0)
-                        })
-                        .max()
-                        .unwrap_or(0),
-                )
-            };
-
-            // This validator is considered healthy if its latest account hash slot is within
-            // `HEALTH_CHECK_SLOT_DISTANCE` of the latest trusted validator's account hash slot
-            if latest_account_hash_slot > 0
-                && latest_trusted_validator_account_hash_slot > 0
-                && latest_account_hash_slot
-                    > latest_trusted_validator_account_hash_slot
-                        .saturating_sub(HEALTH_CHECK_SLOT_DISTANCE)
-            {
-                "ok"
-            } else {
-                warn!(
-                    "health check: me={}, latest trusted_validator={}",
-                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
-                );
-                "behind"
-            }
-        } else {
-            // No trusted validator point of reference available, so this validator is healthy
-            // because it's running
-            "ok"
+        let response = match self.health.check() {
+            RpcHealthStatus::Ok => "ok",
+            RpcHealthStatus::Behind => "behind",
         };
-
         info!("health check: {}", response);
         response
     }
@@ -290,16 +237,36 @@ impl JsonRpcService {
         ledger_path: &Path,
         validator_exit: Arc<RwLock<Option<ValidatorExit>>>,
         trusted_validators: Option<HashSet<Pubkey>>,
+        override_health_check: Arc<AtomicBool>,
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
-        let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new(
+
+        let health = Arc::new(RpcHealth::new(
+            cluster_info.clone(),
+            trusted_validators,
+            config.health_check_slot_distance,
+            override_health_check,
+        ));
+
+        let exit_send_transaction_service = Arc::new(AtomicBool::new(false));
+        let send_transaction_service = Arc::new(SendTransactionService::new(
+            &cluster_info,
+            &bank_forks,
+            &exit_send_transaction_service,
+        ));
+
+        let request_processor = JsonRpcRequestProcessor::new(
             config,
             bank_forks.clone(),
             block_commitment_cache,
             blockstore,
             validator_exit.clone(),
-        )));
+            health.clone(),
+            cluster_info,
+            genesis_hash,
+            send_transaction_service,
+        );
 
         #[cfg(test)]
         let test_request_processor = request_processor.clone();
@@ -317,17 +284,12 @@ impl JsonRpcService {
                 let request_middleware = RpcRequestMiddleware::new(
                     ledger_path,
                     snapshot_config,
-                    cluster_info.clone(),
-                    trusted_validators,
                     bank_forks.clone(),
+                    health.clone(),
                 );
                 let server = ServerBuilder::with_meta_extractor(
                     io,
-                    move |_req: &hyper::Request<hyper::Body>| Meta {
-                        request_processor: request_processor.clone(),
-                        cluster_info: cluster_info.clone(),
-                        genesis_hash,
-                    },
+                    move |_req: &hyper::Request<hyper::Body>| request_processor.clone(),
                 )
                 .threads(num_cpus::get())
                 .cors(DomainsValidation::AllowOnly(vec![
@@ -350,6 +312,7 @@ impl JsonRpcService {
                 let server = server.unwrap();
                 close_handle_sender.send(server.close_handle()).unwrap();
                 server.wait();
+                exit_send_transaction_service.store(true, Ordering::Relaxed);
             })
             .unwrap();
 
@@ -383,7 +346,6 @@ impl JsonRpcService {
 mod tests {
     use super::*;
     use crate::{
-        contact_info::ContactInfo,
         crds_value::{CrdsData, CrdsValue, SnapshotHash},
         rpc::tests::create_validator_exit,
     };
@@ -391,11 +353,11 @@ mod tests {
         bank_forks::CompressionType,
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
+        snapshot_utils::SnapshotVersion,
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::signature::Signer;
-    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
-    use std::sync::atomic::AtomicBool;
+    use std::net::{IpAddr, Ipv4Addr};
 
     #[test]
     fn test_rpc_new() {
@@ -407,13 +369,13 @@ mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let validator_exit = create_validator_exit(&exit);
         let bank = Bank::new(&genesis_config);
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
+        let cluster_info = Arc::new(ClusterInfo::default());
         let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
         let rpc_addr = SocketAddr::new(
             ip_addr,
             solana_net_utils::find_available_port_in_range(ip_addr, (10000, 65535)).unwrap(),
         );
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let block_commitment_cache = Arc::new(RwLock::new(
@@ -431,6 +393,7 @@ mod tests {
             &PathBuf::from("farf"),
             validator_exit,
             None,
+            Arc::new(AtomicBool::new(false)),
         );
         let thread = rpc_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
@@ -439,9 +402,7 @@ mod tests {
             10_000,
             rpc_service
                 .request_processor
-                .read()
-                .unwrap()
-                .get_balance(Ok(mint_keypair.pubkey()), None)
+                .get_balance(&mint_keypair.pubkey(), None)
                 .unwrap()
                 .value
         );
@@ -452,7 +413,7 @@ mod tests {
     fn create_bank_forks() -> Arc<RwLock<BankForks>> {
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)))
+        Arc::new(RwLock::new(BankForks::new(bank)))
     }
 
     #[test]
@@ -472,15 +433,12 @@ mod tests {
 
     #[test]
     fn test_is_file_get_path() {
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
         let bank_forks = create_bank_forks();
-
         let rrm = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             None,
-            cluster_info.clone(),
-            None,
             bank_forks.clone(),
+            RpcHealth::stub(),
         );
         let rrm_with_snapshot_config = RpcRequestMiddleware::new(
             PathBuf::from("/"),
@@ -489,10 +447,10 @@ mod tests {
                 snapshot_package_output_path: PathBuf::from("/"),
                 snapshot_path: PathBuf::from("/"),
                 compression: CompressionType::Bzip2,
+                snapshot_version: SnapshotVersion::default(),
             }),
-            cluster_info,
-            None,
             bank_forks,
+            RpcHealth::stub(),
         );
 
         assert!(rrm.is_file_get_path("/genesis.tar.bz2"));
@@ -518,30 +476,30 @@ mod tests {
 
     #[test]
     fn test_health_check_with_no_trusted_validators() {
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
-
         let rm = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             None,
-            cluster_info,
-            None,
             create_bank_forks(),
+            RpcHealth::stub(),
         );
         assert_eq!(rm.health_check(), "ok");
     }
 
     #[test]
     fn test_health_check_with_trusted_validators() {
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
-
+        let cluster_info = Arc::new(ClusterInfo::default());
+        let health_check_slot_distance = 123;
+        let override_health_check = Arc::new(AtomicBool::new(false));
         let trusted_validators = vec![Pubkey::new_rand(), Pubkey::new_rand(), Pubkey::new_rand()];
-        let rm = RpcRequestMiddleware::new(
-            PathBuf::from("/"),
-            None,
+
+        let health = Arc::new(RpcHealth::new(
             cluster_info.clone(),
             Some(trusted_validators.clone().into_iter().collect()),
-            create_bank_forks(),
-        );
+            health_check_slot_distance,
+            override_health_check.clone(),
+        ));
+
+        let rm = RpcRequestMiddleware::new(PathBuf::from("/"), None, create_bank_forks(), health);
 
         // No account hashes for this node or any trusted validators == "behind"
         assert_eq!(rm.health_check(), "behind");
@@ -549,6 +507,9 @@ mod tests {
         // No account hashes for any trusted validators == "behind"
         cluster_info.push_accounts_hashes(vec![(1000, Hash::default()), (900, Hash::default())]);
         assert_eq!(rm.health_check(), "behind");
+        override_health_check.store(true, Ordering::Relaxed);
+        assert_eq!(rm.health_check(), "ok");
+        override_health_check.store(false, Ordering::Relaxed);
 
         // This node is ahead of the trusted validators == "ok"
         cluster_info
@@ -579,7 +540,7 @@ mod tests {
             .insert(
                 CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
                     trusted_validators[1],
-                    vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE - 1, Hash::default())],
+                    vec![(1000 + health_check_slot_distance - 1, Hash::default())],
                 ))),
                 1,
             )
@@ -595,7 +556,7 @@ mod tests {
             .insert(
                 CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
                     trusted_validators[2],
-                    vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE, Hash::default())],
+                    vec![(1000 + health_check_slot_distance, Hash::default())],
                 ))),
                 1,
             )

core/src/rpc_subscriptions.rs

@@ -1,6 +1,6 @@
 //! The `pubsub` module implements a threaded subscription service on client RPC request
 
-use crate::commitment::{BlockCommitmentCache, CacheSlotInfo};
+use crate::commitment::BlockCommitmentCache;
 use core::hash::Hash;
 use jsonrpc_core::futures::Future;
 use jsonrpc_pubsub::{
@@ -44,6 +44,14 @@ pub struct SlotInfo {
     pub root: Slot,
 }
 
+#[derive(Default)]
+pub struct CacheSlotInfo {
+    pub current_slot: Slot,
+    pub node_root: Slot,
+    pub largest_confirmed_root: Slot,
+    pub highest_confirmed_slot: Slot,
+}
+
 // A more human-friendly version of Vote, with the bank state signature base58 encoded.
 #[derive(Serialize, Deserialize, Debug)]
 pub struct RpcVote {
@@ -56,6 +64,7 @@ enum NotificationEntry {
     Slot(SlotInfo),
     Vote(Vote),
     Root(Slot),
+    Frozen(Slot),
     Bank(CacheSlotInfo),
     Gossip(Slot),
 }
@@ -64,6 +73,7 @@ impl std::fmt::Debug for NotificationEntry {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {
             NotificationEntry::Root(root) => write!(f, "Root({})", root),
+            NotificationEntry::Frozen(slot) => write!(f, "Frozen({})", slot),
             NotificationEntry::Vote(vote) => write!(f, "Vote({:?})", vote),
             NotificationEntry::Slot(slot_info) => write!(f, "Slot({:?})", slot_info),
             NotificationEntry::Bank(cache_slot_info) => write!(
@@ -219,6 +229,8 @@ fn filter_account_result(
     last_notified_slot: Slot,
 ) -> (Box<dyn Iterator<Item = RpcAccount>>, Slot) {
     if let Some((account, fork)) = result {
+        // If fork < last_notified_slot this means that we last notified for a fork
+        // and should notify that the account state has been reverted.
         if fork != last_notified_slot {
             return (Box::new(iter::once(RpcAccount::encode(account))), fork);
         }
@@ -639,6 +651,10 @@ impl RpcSubscriptions {
         self.enqueue_notification(NotificationEntry::Vote(vote.clone()));
     }
 
+    pub fn notify_frozen(&self, frozen_slot: Slot) {
+        self.enqueue_notification(NotificationEntry::Frozen(frozen_slot));
+    }
+
     pub fn add_root_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber<Slot>) {
         let sink = subscriber.assign_id(sub_id.clone()).unwrap();
         let mut subscriptions = self.subscriptions.root_subscriptions.write().unwrap();
@@ -682,6 +698,7 @@ impl RpcSubscriptions {
         bank_forks: Arc<RwLock<BankForks>>,
         last_checked_slots: Arc<RwLock<HashMap<CommitmentLevel, Slot>>>,
     ) {
+        let mut pending_gossip_notifications = HashSet::new();
         loop {
             if exit.load(Ordering::Relaxed) {
                 break;
@@ -712,6 +729,12 @@ impl RpcSubscriptions {
                         for (_, sink) in subscriptions.iter() {
                             notifier.notify(root, sink);
                         }
+
+                        // Prune old pending notifications
+                        pending_gossip_notifications = pending_gossip_notifications
+                            .into_iter()
+                            .filter(|&s| s > root)
+                            .collect();
                     }
                     NotificationEntry::Bank(cache_slot_info) => {
                         RpcSubscriptions::notify_accounts_programs_signatures(
@@ -723,23 +746,36 @@ impl RpcSubscriptions {
                             &notifier,
                         )
                     }
+                    NotificationEntry::Frozen(slot) => {
+                        if pending_gossip_notifications.remove(&slot) {
+                            Self::process_gossip_notification(
+                                slot,
+                                &notifier,
+                                &subscriptions,
+                                &bank_forks,
+                                &last_checked_slots,
+                            );
+                        }
+                    }
                     NotificationEntry::Gossip(slot) => {
-                        let _ = last_checked_slots
-                            .write()
+                        let bank_frozen = bank_forks
+                            .read()
                             .unwrap()
-                            .insert(CommitmentLevel::SingleGossip, slot);
-                        let cache_slot_info = CacheSlotInfo {
-                            highest_confirmed_slot: slot,
-                            ..CacheSlotInfo::default()
-                        };
-                        RpcSubscriptions::notify_accounts_programs_signatures(
-                            &subscriptions.gossip_account_subscriptions,
-                            &subscriptions.gossip_program_subscriptions,
-                            &subscriptions.gossip_signature_subscriptions,
-                            &bank_forks,
-                            &cache_slot_info,
-                            &notifier,
-                        )
+                            .get(slot)
+                            .filter(|b| b.is_frozen())
+                            .is_some();
+
+                        if !bank_frozen {
+                            pending_gossip_notifications.insert(slot);
+                        } else {
+                            Self::process_gossip_notification(
+                                slot,
+                                &notifier,
+                                &subscriptions,
+                                &bank_forks,
+                                &last_checked_slots,
+                            );
+                        }
                     }
                 },
                 Err(RecvTimeoutError::Timeout) => {
@@ -753,6 +789,42 @@ impl RpcSubscriptions {
         }
     }
 
+    fn process_gossip_notification(
+        slot: Slot,
+        notifier: &RpcNotifier,
+        subscriptions: &Subscriptions,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        last_checked_slots: &Arc<RwLock<HashMap<CommitmentLevel, Slot>>>,
+    ) {
+        let mut last_checked_slots_lock = last_checked_slots.write().unwrap();
+        let last_checked_slot = last_checked_slots_lock
+            .get(&CommitmentLevel::SingleGossip)
+            .cloned()
+            .unwrap_or_default();
+
+        if slot > last_checked_slot {
+            last_checked_slots_lock.insert(CommitmentLevel::SingleGossip, slot);
+        } else {
+            // Avoid sending stale or duplicate notifications
+            return;
+        }
+
+        drop(last_checked_slots_lock);
+
+        let cache_slot_info = CacheSlotInfo {
+            highest_confirmed_slot: slot,
+            ..CacheSlotInfo::default()
+        };
+        RpcSubscriptions::notify_accounts_programs_signatures(
+            &subscriptions.gossip_account_subscriptions,
+            &subscriptions.gossip_program_subscriptions,
+            &subscriptions.gossip_signature_subscriptions,
+            &bank_forks,
+            &cache_slot_info,
+            &notifier,
+        );
+    }
+
     fn notify_accounts_programs_signatures(
         account_subscriptions: &Arc<RpcAccountSubscriptions>,
         program_subscriptions: &Arc<RpcProgramSubscriptions>,
@@ -880,7 +952,7 @@ pub(crate) mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let bank0 = bank_forks.read().unwrap().get(0).unwrap().clone();
         let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
         bank_forks.write().unwrap().insert(bank1);
@@ -974,7 +1046,7 @@ pub(crate) mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let alice = Keypair::new();
         let tx = system_transaction::create_account(
             &mint_keypair,
@@ -1062,7 +1134,7 @@ pub(crate) mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let mut bank_forks = BankForks::new(0, bank);
+        let mut bank_forks = BankForks::new(bank);
         let alice = Keypair::new();
 
         let past_bank_tx =
@@ -1218,7 +1290,7 @@ pub(crate) mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let subscriptions = RpcSubscriptions::new(
             &exit,
             bank_forks,
@@ -1270,7 +1342,7 @@ pub(crate) mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let subscriptions = RpcSubscriptions::new(
             &exit,
             bank_forks,
@@ -1368,10 +1440,12 @@ pub(crate) mod tests {
         let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
         let bank0 = bank_forks.read().unwrap().get(0).unwrap().clone();
         let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
         bank_forks.write().unwrap().insert(bank1);
+        let bank2 = Bank::new_from_parent(&bank0, &Pubkey::default(), 2);
+        bank_forks.write().unwrap().insert(bank2);
         let alice = Keypair::new();
 
         let (subscriber0, _id_receiver, transport_receiver0) =
@@ -1397,17 +1471,10 @@ pub(crate) mod tests {
             sub_id0.clone(),
             subscriber0,
         );
-        let sub_id1 = SubscriptionId::Number(1 as u64);
-        subscriptions.add_account_subscription(
-            alice.pubkey(),
-            Some(CommitmentConfig::recent()),
-            sub_id1.clone(),
-            subscriber1,
-        );
 
         assert!(subscriptions
             .subscriptions
-            .account_subscriptions
+            .gossip_account_subscriptions
             .read()
             .unwrap()
             .contains_key(&alice.pubkey()));
@@ -1420,37 +1487,27 @@ pub(crate) mod tests {
             16,
             &solana_budget_program::id(),
         );
+
+        // Add the transaction to the 1st bank and then freeze the bank
+        let bank1 = bank_forks.write().unwrap().get(1).cloned().unwrap();
+        bank1.process_transaction(&tx).unwrap();
+        bank1.freeze();
+
+        // Add the same transaction to the unfrozen 2nd bank
         bank_forks
             .write()
             .unwrap()
-            .get(1)
+            .get(2)
             .unwrap()
             .process_transaction(&tx)
             .unwrap();
-        let mut cache_slot_info = CacheSlotInfo::default();
-        cache_slot_info.current_slot = 1;
-        subscriptions.notify_subscribers(cache_slot_info);
-        let (response, _) = robust_poll_or_panic(transport_receiver1);
-        let expected = json!({
-           "jsonrpc": "2.0",
-           "method": "accountNotification",
-           "params": {
-               "result": {
-                   "context": { "slot": 1 },
-                   "value": {
-                       "data": "1111111111111111",
-                       "executable": false,
-                       "lamports": 1,
-                       "owner": "Budget1111111111111111111111111111111111111",
-                       "rentEpoch": 1,
-                    },
-               },
-               "subscription": 1,
-           }
-        });
-        assert_eq!(serde_json::to_string(&expected).unwrap(), response);
 
+        // First, notify the unfrozen bank first to queue pending notification
+        subscriptions.notify_gossip_subscribers(2);
+
+        // Now, notify the frozen bank and ensure its notifications are processed
         subscriptions.notify_gossip_subscribers(1);
+
         let (response, _) = robust_poll_or_panic(transport_receiver0);
         let expected = json!({
            "jsonrpc": "2.0",
@@ -1470,18 +1527,41 @@ pub(crate) mod tests {
            }
         });
         assert_eq!(serde_json::to_string(&expected).unwrap(), response);
-
         subscriptions.remove_account_subscription(&sub_id0);
-        assert!(subscriptions
-            .subscriptions
-            .account_subscriptions
-            .read()
-            .unwrap()
-            .contains_key(&alice.pubkey()));
+
+        let sub_id1 = SubscriptionId::Number(1 as u64);
+        subscriptions.add_account_subscription(
+            alice.pubkey(),
+            Some(CommitmentConfig::single_gossip()),
+            sub_id1.clone(),
+            subscriber1,
+        );
+
+        subscriptions.notify_frozen(2);
+        let (response, _) = robust_poll_or_panic(transport_receiver1);
+        let expected = json!({
+           "jsonrpc": "2.0",
+           "method": "accountNotification",
+           "params": {
+               "result": {
+                   "context": { "slot": 2 },
+                   "value": {
+                       "data": "1111111111111111",
+                       "executable": false,
+                       "lamports": 1,
+                       "owner": "Budget1111111111111111111111111111111111111",
+                       "rentEpoch": 1,
+                    },
+               },
+               "subscription": 1,
+           }
+        });
+        assert_eq!(serde_json::to_string(&expected).unwrap(), response);
         subscriptions.remove_account_subscription(&sub_id1);
+
         assert!(!subscriptions
             .subscriptions
-            .account_subscriptions
+            .gossip_account_subscriptions
             .read()
             .unwrap()
             .contains_key(&alice.pubkey()));

core/src/send_transaction_service.rs

@@ -0,0 +1,377 @@
+use crate::cluster_info::ClusterInfo;
+use solana_ledger::bank_forks::BankForks;
+use solana_metrics::{datapoint_warn, inc_new_counter_info};
+use solana_runtime::bank::Bank;
+use solana_sdk::{clock::Slot, signature::Signature};
+use std::{
+    collections::HashMap,
+    net::{SocketAddr, UdpSocket},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::{channel, Receiver, Sender},
+        Arc, Mutex, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+/// Maximum size of the transaction queue
+const MAX_TRANSACTION_QUEUE_SIZE: usize = 10_000; // This seems like a lot but maybe it needs to be bigger one day
+
+pub struct SendTransactionService {
+    thread: JoinHandle<()>,
+    sender: Mutex<Sender<TransactionInfo>>,
+    send_socket: UdpSocket,
+    tpu_address: SocketAddr,
+}
+
+struct TransactionInfo {
+    signature: Signature,
+    wire_transaction: Vec<u8>,
+    last_valid_slot: Slot,
+}
+
+#[derive(Default, Debug, PartialEq)]
+struct ProcessTransactionsResult {
+    rooted: u64,
+    expired: u64,
+    retried: u64,
+    failed: u64,
+    retained: u64,
+}
+
+impl SendTransactionService {
+    pub fn new(
+        cluster_info: &Arc<ClusterInfo>,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        exit: &Arc<AtomicBool>,
+    ) -> Self {
+        let (sender, receiver) = channel::<TransactionInfo>();
+        let tpu_address = cluster_info.my_contact_info().tpu;
+
+        let thread = Self::retry_thread(receiver, bank_forks.clone(), tpu_address, exit.clone());
+        Self {
+            thread,
+            sender: Mutex::new(sender),
+            send_socket: UdpSocket::bind("0.0.0.0:0").unwrap(),
+            tpu_address,
+        }
+    }
+
+    fn retry_thread(
+        receiver: Receiver<TransactionInfo>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        tpu_address: SocketAddr,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        let mut last_status_check = Instant::now();
+        let mut transactions = HashMap::new();
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+
+        Builder::new()
+            .name("send-tx-svc".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+
+                if let Ok(transaction_info) = receiver.recv_timeout(Duration::from_secs(1)) {
+                    if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
+                        transactions.insert(transaction_info.signature, transaction_info);
+                    } else {
+                        datapoint_warn!("send_transaction_service-queue-overflow");
+                    }
+                }
+
+                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
+                    if !transactions.is_empty() {
+                        datapoint_info!(
+                            "send_transaction_service-queue-size",
+                            ("len", transactions.len(), i64)
+                        );
+                        let bank_forks = bank_forks.read().unwrap();
+                        let root_bank = bank_forks.root_bank();
+                        let working_bank = bank_forks.working_bank();
+
+                        let _result = Self::process_transactions(
+                            &working_bank,
+                            &root_bank,
+                            &send_socket,
+                            &tpu_address,
+                            &mut transactions,
+                        );
+                    }
+                    last_status_check = Instant::now();
+                }
+            })
+            .unwrap()
+    }
+
+    fn process_transactions(
+        working_bank: &Arc<Bank>,
+        root_bank: &Arc<Bank>,
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        transactions: &mut HashMap<Signature, TransactionInfo>,
+    ) -> ProcessTransactionsResult {
+        let mut result = ProcessTransactionsResult::default();
+
+        transactions.retain(|signature, transaction_info| {
+            if root_bank.has_signature(signature) {
+                info!("Transaction is rooted: {}", signature);
+                result.rooted += 1;
+                inc_new_counter_info!("send_transaction_service-rooted", 1);
+                false
+            } else if transaction_info.last_valid_slot < root_bank.slot() {
+                info!("Dropping expired transaction: {}", signature);
+                result.expired += 1;
+                inc_new_counter_info!("send_transaction_service-expired", 1);
+                false
+            } else {
+                match working_bank.get_signature_status_slot(signature) {
+                    None => {
+                        // Transaction is unknown to the working bank, it might have been
+                        // dropped or landed in another fork.  Re-send it
+                        info!("Retrying transaction: {}", signature);
+                        result.retried += 1;
+                        inc_new_counter_info!("send_transaction_service-retry", 1);
+                        Self::send_transaction(
+                            &send_socket,
+                            &tpu_address,
+                            &transaction_info.wire_transaction,
+                        );
+                        true
+                    }
+                    Some((_slot, status)) => {
+                        if status.is_err() {
+                            info!("Dropping failed transaction: {}", signature);
+                            result.failed += 1;
+                            inc_new_counter_info!("send_transaction_service-failed", 1);
+                            false
+                        } else {
+                            result.retained += 1;
+                            true
+                        }
+                    }
+                }
+            }
+        });
+
+        result
+    }
+
+    fn send_transaction(
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        wire_transaction: &[u8],
+    ) {
+        if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
+            warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
+        }
+    }
+
+    pub fn send(&self, signature: Signature, wire_transaction: Vec<u8>, last_valid_slot: Slot) {
+        inc_new_counter_info!("send_transaction_service-enqueue", 1, 1);
+        Self::send_transaction(&self.send_socket, &self.tpu_address, &wire_transaction);
+
+        self.sender
+            .lock()
+            .unwrap()
+            .send(TransactionInfo {
+                signature,
+                wire_transaction,
+                last_valid_slot,
+            })
+            .unwrap_or_else(|err| warn!("Failed to enqueue transaction: {}", err));
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::rpc::tests::new_bank_forks;
+    use solana_sdk::{pubkey::Pubkey, signature::Signer};
+
+    #[test]
+    fn service_exit() {
+        let cluster_info = Arc::new(ClusterInfo::default());
+        let bank_forks = new_bank_forks().0;
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let send_tranaction_service =
+            SendTransactionService::new(&cluster_info, &bank_forks, &exit);
+
+        exit.store(true, Ordering::Relaxed);
+        send_tranaction_service.join().unwrap();
+    }
+
+    #[test]
+    fn process_transactions() {
+        solana_logger::setup();
+
+        let (bank_forks, mint_keypair, _voting_keypair) = new_bank_forks();
+        let cluster_info = ClusterInfo::default();
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let tpu_address = cluster_info.my_contact_info().tpu;
+
+        let root_bank = Arc::new(Bank::new_from_parent(
+            &bank_forks.read().unwrap().working_bank(),
+            &Pubkey::default(),
+            1,
+        ));
+        let rooted_signature = root_bank
+            .transfer(1, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let working_bank = Arc::new(Bank::new_from_parent(&root_bank, &Pubkey::default(), 2));
+
+        let non_rooted_signature = working_bank
+            .transfer(2, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let failed_signature = {
+            let blockhash = working_bank.last_blockhash();
+            let transaction = solana_sdk::system_transaction::transfer(
+                &mint_keypair,
+                &Pubkey::default(),
+                1,
+                blockhash,
+            );
+            let signature = transaction.signatures[0];
+            working_bank.process_transaction(&transaction).unwrap_err();
+            signature
+        };
+
+        let mut transactions = HashMap::new();
+
+        info!("Expired transactions are dropped..");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo {
+                signature: Signature::default(),
+                wire_transaction: vec![],
+                last_valid_slot: root_bank.slot() - 1,
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                expired: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Rooted transactions are dropped...");
+        transactions.insert(
+            rooted_signature,
+            TransactionInfo {
+                signature: rooted_signature,
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                rooted: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Failed transactions are dropped...");
+        transactions.insert(
+            failed_signature,
+            TransactionInfo {
+                signature: failed_signature,
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                failed: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Non-rooted transactions are kept...");
+        transactions.insert(
+            non_rooted_signature,
+            TransactionInfo {
+                signature: non_rooted_signature,
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retained: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        transactions.clear();
+
+        info!("Unknown transactions are retried...");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo {
+                signature: Signature::default(),
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retried: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+    }
+}

core/src/serve_repair.rs

@@ -397,7 +397,13 @@ impl ServeRepair {
         let (repair_peers, weights) = cache.get(&slot).unwrap();
         let n = weighted_best(&weights, Pubkey::new_rand().to_bytes());
         let addr = repair_peers[n].serve_repair; // send the request to the peer's serve_repair port
-        let out = self.map_repair_request(&repair_request, repair_stats, DEFAULT_NONCE)?;
+        let repair_peer_id = repair_peers[n].id;
+        let out = self.map_repair_request(
+            &repair_request,
+            &repair_peer_id,
+            repair_stats,
+            DEFAULT_NONCE,
+        )?;
         Ok((addr, out))
     }
 
@@ -405,33 +411,38 @@ impl ServeRepair {
         &self,
         slot: Slot,
         cluster_slots: &ClusterSlots,
-    ) -> Result<SocketAddr> {
+    ) -> Result<(Pubkey, SocketAddr)> {
         let repair_peers: Vec<_> = self.cluster_info.repair_peers(slot);
         if repair_peers.is_empty() {
             return Err(ClusterInfoError::NoPeers.into());
         }
         let weights = cluster_slots.compute_weights_exclude_noncomplete(slot, &repair_peers);
         let n = weighted_best(&weights, Pubkey::new_rand().to_bytes());
-        Ok(repair_peers[n].serve_repair)
+        Ok((repair_peers[n].id, repair_peers[n].serve_repair))
     }
 
     pub fn map_repair_request(
         &self,
         repair_request: &RepairType,
+        repair_peer_id: &Pubkey,
         repair_stats: &mut RepairStats,
         nonce: Nonce,
     ) -> Result<Vec<u8>> {
         match repair_request {
             RepairType::Shred(slot, shred_index) => {
-                repair_stats.shred.update(*slot);
+                repair_stats
+                    .shred
+                    .update(repair_peer_id, *slot, *shred_index);
                 Ok(self.window_index_request_bytes(*slot, *shred_index, nonce)?)
             }
             RepairType::HighestShred(slot, shred_index) => {
-                repair_stats.highest_shred.update(*slot);
+                repair_stats
+                    .highest_shred
+                    .update(repair_peer_id, *slot, *shred_index);
                 Ok(self.window_highest_index_request_bytes(*slot, *shred_index, nonce)?)
             }
             RepairType::Orphan(slot) => {
-                repair_stats.orphan.update(*slot);
+                repair_stats.orphan.update(repair_peer_id, *slot, 0);
                 Ok(self.orphan_bytes(*slot, nonce)?)
             }
         }
@@ -532,6 +543,8 @@ impl ServeRepair {
                 );
                 if let Some(packet) = packet {
                     res.packets.push(packet);
+                } else {
+                    break;
                 }
                 if meta.is_parent_set() && res.packets.len() <= max_responses {
                     slot = meta.parent_slot;
@@ -864,4 +877,64 @@ mod tests {
 
         Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
     }
+
+    #[test]
+    fn run_orphan_corrupted_shred_size() {
+        solana_logger::setup();
+        let recycler = PacketsRecycler::default();
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+            // Create slots [1, 2] with 1 shred apiece
+            let (mut shreds, _) = make_many_slot_entries(1, 2, 1);
+
+            // Make shred for slot 1 too large
+            assert_eq!(shreds[0].slot(), 1);
+            assert_eq!(shreds[0].index(), 0);
+            shreds[0].payload.push(10);
+            blockstore
+                .insert_shreds(shreds, None, false)
+                .expect("Expect successful ledger write");
+            let nonce = 42;
+            // Make sure repair response is corrupted
+            assert!(repair_response::repair_response_packet(
+                &blockstore,
+                1,
+                0,
+                &socketaddr_any!(),
+                nonce,
+            )
+            .is_none());
+
+            // Orphan request for slot 2 should only return slot 1 since
+            // calling `repair_response_packet` on slot 1's shred will
+            // be corrupted
+            let rv: Vec<_> = ServeRepair::run_orphan(
+                &recycler,
+                &socketaddr_any!(),
+                Some(&blockstore),
+                2,
+                5,
+                nonce,
+            )
+            .expect("run_orphan packets")
+            .packets
+            .iter()
+            .cloned()
+            .collect();
+
+            // Verify responses
+            let expected = vec![repair_response::repair_response_packet(
+                &blockstore,
+                2,
+                0,
+                &socketaddr_any!(),
+                nonce,
+            )
+            .unwrap()];
+            assert_eq!(rv, expected);
+        }
+
+        Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
+    }
 }

core/src/sigverify_shreds.rs

@@ -127,7 +127,7 @@ pub mod tests {
         let bank =
             Bank::new(&create_genesis_config_with_leader(100, &leader_pubkey, 10).genesis_config);
         let cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank));
-        let bf = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let bf = Arc::new(RwLock::new(BankForks::new(bank)));
         let verifier = ShredSigVerifier::new(bf, cache);
 
         let mut batch = vec![Packets::default()];

core/src/snapshot_packager_service.rs

@@ -81,7 +81,7 @@ mod tests {
     use solana_ledger::bank_forks::CompressionType;
     use solana_ledger::{
         snapshot_package::AccountsPackage,
-        snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME},
+        snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
     };
     use solana_runtime::{accounts_db::AccountStorageEntry, bank::BankSlotDelta};
     use solana_sdk::hash::Hash;
@@ -173,6 +173,7 @@ mod tests {
             output_tar_path.clone(),
             Hash::default(),
             CompressionType::Bzip2,
+            SnapshotVersion::default(),
         );
 
         // Make tarball from packageable snapshot

core/src/tvu.rs

@@ -146,6 +146,7 @@ impl Tvu {
             tvu_config.shred_version,
             cluster_slots.clone(),
             duplicate_slots_reset_sender,
+            vote_tracker.clone(),
         );
 
         let (ledger_cleanup_slot_sender, ledger_cleanup_slot_receiver) = channel();
@@ -256,7 +257,7 @@ pub mod tests {
         let starting_balance = 10_000;
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(starting_balance);
 
-        let bank_forks = BankForks::new(0, Bank::new(&genesis_config));
+        let bank_forks = BankForks::new(Bank::new(&genesis_config));
 
         //start cluster_info1
         let cluster_info1 = ClusterInfo::new_with_invalid_keypair(target1.info.clone());

core/src/validator.rs

@@ -23,10 +23,11 @@ use crate::{
     tvu::{Sockets, Tvu, TvuConfig},
 };
 use crossbeam_channel::unbounded;
+use rand::{thread_rng, Rng};
 use solana_ledger::{
     bank_forks::{BankForks, SnapshotConfig},
     bank_forks_utils,
-    blockstore::{Blockstore, CompletedSlotsReceiver},
+    blockstore::{Blockstore, CompletedSlotsReceiver, PurgeType},
     blockstore_processor, create_new_tmp_ledger,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     leader_schedule::FixedSchedule,
@@ -60,6 +61,7 @@ use std::{
 pub struct ValidatorConfig {
     pub dev_halt_at_slot: Option<Slot>,
     pub expected_genesis_hash: Option<Hash>,
+    pub expected_bank_hash: Option<Hash>,
     pub expected_shred_version: Option<u16>,
     pub voting_disabled: bool,
     pub account_paths: Vec<PathBuf>,
@@ -86,6 +88,7 @@ impl Default for ValidatorConfig {
         Self {
             dev_halt_at_slot: None,
             expected_genesis_hash: None,
+            expected_bank_hash: None,
             expected_shred_version: None,
             voting_disabled: false,
             max_ledger_shreds: None,
@@ -172,10 +175,24 @@ impl Validator {
 
         info!("entrypoint: {:?}", entrypoint_info_option);
 
-        info!("Initializing sigverify, this could take a while...");
+        if solana_perf::perf_libs::api().is_some() {
+            info!("Initializing sigverify, this could take a while...");
+        } else {
+            info!("Initializing sigverify...");
+        }
         sigverify::init();
         info!("Done.");
 
+        if let Some(shred_version) = config.expected_shred_version {
+            if let Some(wait_for_supermajority_slot) = config.wait_for_supermajority {
+                backup_and_clear_blockstore(
+                    ledger_path,
+                    wait_for_supermajority_slot + 1,
+                    shred_version,
+                );
+            }
+        }
+
         info!("creating bank...");
         let (
             genesis_config,
@@ -215,8 +232,8 @@ impl Validator {
         if let Some(expected_shred_version) = config.expected_shred_version {
             if expected_shred_version != node.info.shred_version {
                 error!(
-                    "shred version mismatch: expected {}",
-                    expected_shred_version
+                    "shred version mismatch: expected {} found: {}",
+                    expected_shred_version, node.info.shred_version,
                 );
                 process::exit(1);
             }
@@ -234,6 +251,7 @@ impl Validator {
             block_commitment_cache.clone(),
         ));
 
+        let rpc_override_health_check = Arc::new(AtomicBool::new(false));
         let rpc_service = config.rpc_ports.map(|(rpc_port, rpc_pubsub_port)| {
             if ContactInfo::is_valid_address(&node.info.rpc) {
                 assert!(ContactInfo::is_valid_address(&node.info.rpc_pubsub));
@@ -255,6 +273,7 @@ impl Validator {
                     ledger_path,
                     validator_exit.clone(),
                     config.trusted_validators.clone(),
+                    rpc_override_health_check.clone(),
                 ),
                 PubSubService::new(
                     &subscriptions,
@@ -374,7 +393,9 @@ impl Validator {
                 (None, None)
             };
 
-        wait_for_supermajority(config, &bank, &cluster_info);
+        if wait_for_supermajority(config, &bank, &cluster_info, rpc_override_health_check) {
+            std::process::exit(1);
+        }
 
         let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
         assert_eq!(
@@ -620,9 +641,95 @@ fn new_banks_from_blockstore(
     )
 }
 
-fn wait_for_supermajority(config: &ValidatorConfig, bank: &Bank, cluster_info: &ClusterInfo) {
-    if config.wait_for_supermajority != Some(bank.slot()) {
-        return;
+fn backup_and_clear_blockstore(ledger_path: &Path, start_slot: Slot, shred_version: u16) {
+    use std::time::Instant;
+    let blockstore = Blockstore::open(ledger_path).unwrap();
+    let mut do_copy_and_clear = false;
+
+    // Search for shreds with incompatible version in blockstore
+    if let Ok(slot_meta_iterator) = blockstore.slot_meta_iterator(start_slot) {
+        for (slot, _meta) in slot_meta_iterator {
+            if let Ok(shreds) = blockstore.get_data_shreds_for_slot(slot, 0) {
+                for shred in &shreds {
+                    if shred.version() != shred_version {
+                        do_copy_and_clear = true;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    // If found, then copy shreds to another db and clear from start_slot
+    if do_copy_and_clear {
+        let folder_name = format!("backup_rocksdb_{}", thread_rng().gen_range(0, 99999));
+        let backup_blockstore = Blockstore::open(&ledger_path.join(folder_name));
+        let mut last_print = Instant::now();
+        let mut copied = 0;
+        let mut last_slot = None;
+        let slot_meta_iterator = blockstore.slot_meta_iterator(start_slot).unwrap();
+        for (slot, _meta) in slot_meta_iterator {
+            if let Ok(shreds) = blockstore.get_data_shreds_for_slot(slot, 0) {
+                if let Ok(ref backup_blockstore) = backup_blockstore {
+                    copied += shreds.len();
+                    let _ = backup_blockstore.insert_shreds(shreds, None, true);
+                }
+            }
+            if last_print.elapsed().as_millis() > 3000 {
+                info!(
+                    "Copying shreds from slot {} copied {} so far.",
+                    start_slot, copied
+                );
+                last_print = Instant::now();
+            }
+            last_slot = Some(slot);
+        }
+
+        let end_slot = last_slot.unwrap();
+        info!("Purging slots {} to {}", start_slot, end_slot);
+        blockstore.purge_slots(start_slot, end_slot, PurgeType::Exact);
+        blockstore.purge_from_next_slots(start_slot, end_slot);
+        info!("Purging done, compacting db..");
+        if let Err(e) = blockstore.compact_storage(start_slot, end_slot) {
+            warn!(
+                "Error from compacting storage from {} to {}: {:?}",
+                start_slot, end_slot, e
+            );
+        }
+        info!("done");
+    }
+    drop(blockstore);
+}
+
+// Return true on error, indicating the validator should exit.
+fn wait_for_supermajority(
+    config: &ValidatorConfig,
+    bank: &Bank,
+    cluster_info: &ClusterInfo,
+    rpc_override_health_check: Arc<AtomicBool>,
+) -> bool {
+    if let Some(wait_for_supermajority) = config.wait_for_supermajority {
+        match wait_for_supermajority.cmp(&bank.slot()) {
+            std::cmp::Ordering::Less => return false,
+            std::cmp::Ordering::Greater => {
+                error!("Ledger does not have enough data to wait for supermajority, please enable snapshot fetch. Has {} needs {}", bank.slot(), wait_for_supermajority);
+                return true;
+            }
+            _ => {}
+        }
+    } else {
+        return false;
+    }
+
+    if let Some(expected_bank_hash) = config.expected_bank_hash {
+        if bank.hash() != expected_bank_hash {
+            error!(
+                "Bank hash({}) does not match expected value: {}",
+                bank.hash(),
+                expected_bank_hash
+            );
+            return true;
+        }
     }
 
     info!(
@@ -635,8 +742,14 @@ fn wait_for_supermajority(config: &ValidatorConfig, bank: &Bank, cluster_info: &
         if gossip_stake_percent >= 80 {
             break;
         }
+        // The normal RPC health checks don't apply as the node is waiting, so feign health to
+        // prevent load balancers from removing the node from their list of candidates during a
+        // manual restart.
+        rpc_override_health_check.store(true, Ordering::Relaxed);
         sleep(Duration::new(1, 0));
     }
+    rpc_override_health_check.store(false, Ordering::Relaxed);
+    false
 }
 
 pub struct TestValidator {
@@ -886,6 +999,43 @@ mod tests {
         remove_dir_all(validator_ledger_path).unwrap();
     }
 
+    #[test]
+    fn test_backup_and_clear_blockstore() {
+        use std::time::Instant;
+        solana_logger::setup();
+        use solana_ledger::get_tmp_ledger_path;
+        use solana_ledger::{blockstore, entry};
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+
+            let entries = entry::create_ticks(1, 0, Hash::default());
+
+            info!("creating shreds");
+            let mut last_print = Instant::now();
+            for i in 1..10 {
+                let shreds = blockstore::entries_to_test_shreds(entries.clone(), i, i - 1, true, 1);
+                blockstore.insert_shreds(shreds, None, true).unwrap();
+                if last_print.elapsed().as_millis() > 5000 {
+                    info!("inserted {}", i);
+                    last_print = Instant::now();
+                }
+            }
+            drop(blockstore);
+
+            backup_and_clear_blockstore(&blockstore_path, 5, 2);
+
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            assert!(blockstore.meta(4).unwrap().unwrap().next_slots.is_empty());
+            for i in 5..10 {
+                assert!(blockstore
+                    .get_data_shreds_for_slot(i, 0)
+                    .unwrap()
+                    .is_empty());
+            }
+        }
+    }
+
     #[test]
     fn validator_parallel_exit() {
         let leader_keypair = Keypair::new();
@@ -934,4 +1084,56 @@ mod tests {
             remove_dir_all(path).unwrap();
         }
     }
+
+    #[test]
+    fn test_wait_for_supermajority() {
+        solana_logger::setup();
+        use solana_sdk::genesis_config::create_genesis_config;
+        use solana_sdk::hash::hash;
+        let node_keypair = Arc::new(Keypair::new());
+        let cluster_info = ClusterInfo::new(
+            ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
+            node_keypair,
+        );
+
+        let (genesis_config, _mint_keypair) = create_genesis_config(1);
+        let bank = Arc::new(Bank::new(&genesis_config));
+        let mut config = ValidatorConfig::default();
+        let rpc_override_health_check = Arc::new(AtomicBool::new(false));
+        assert!(!wait_for_supermajority(
+            &config,
+            &bank,
+            &cluster_info,
+            rpc_override_health_check.clone()
+        ));
+
+        // bank=0, wait=1, should fail
+        config.wait_for_supermajority = Some(1);
+        assert!(wait_for_supermajority(
+            &config,
+            &bank,
+            &cluster_info,
+            rpc_override_health_check.clone()
+        ));
+
+        // bank=1, wait=0, should pass, bank is past the wait slot
+        let bank = Bank::new_from_parent(&bank, &Pubkey::default(), 1);
+        config.wait_for_supermajority = Some(0);
+        assert!(!wait_for_supermajority(
+            &config,
+            &bank,
+            &cluster_info,
+            rpc_override_health_check.clone()
+        ));
+
+        // bank=1, wait=1, equal, but bad hash provided
+        config.wait_for_supermajority = Some(1);
+        config.expected_bank_hash = Some(hash(&[1]));
+        assert!(wait_for_supermajority(
+            &config,
+            &bank,
+            &cluster_info,
+            rpc_override_health_check
+        ));
+    }
 }

core/src/window_service.rs

@@ -3,6 +3,7 @@
 //!
 use crate::{
     cluster_info::ClusterInfo,
+    cluster_info_vote_listener::VoteTracker,
     cluster_slots::ClusterSlots,
     repair_response,
     repair_service::{RepairInfo, RepairService},
@@ -301,6 +302,7 @@ impl WindowService {
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         shred_filter: F,
         cluster_slots: Arc<ClusterSlots>,
+        vote_tracker: Arc<VoteTracker>,
     ) -> WindowService
     where
         F: 'static
@@ -317,6 +319,7 @@ impl WindowService {
             cluster_info.clone(),
             repair_info,
             cluster_slots,
+            vote_tracker,
         );
 
         let (insert_sender, insert_receiver) = unbounded();

core/tests/bank_forks.rs

@@ -1,5 +1,36 @@
 // Long-running bank_forks tests
 
+macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
+    ($x:ident) => {
+        #[allow(non_snake_case)]
+        mod $x {
+            use super::*;
+
+            const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x;
+
+            #[test]
+            fn test_bank_forks_status_cache_snapshot_n() {
+                run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION)
+            }
+
+            #[test]
+            fn test_bank_forks_snapshot_n() {
+                run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION)
+            }
+
+            #[test]
+            fn test_concurrent_snapshot_packaging() {
+                run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION)
+            }
+
+            #[test]
+            fn test_slots_to_snapshot() {
+                run_test_slots_to_snapshot(SNAPSHOT_VERSION)
+            }
+        }
+    };
+}
+
 #[cfg(test)]
 mod tests {
     use bincode::serialize_into;
@@ -13,6 +44,7 @@ mod tests {
         bank_forks::{BankForks, SnapshotConfig},
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         snapshot_utils,
+        snapshot_utils::SnapshotVersion,
     };
     use solana_runtime::{
         bank::{Bank, BankSlotDelta},
@@ -29,6 +61,9 @@ mod tests {
     use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
     use tempfile::TempDir;
 
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0);
+
     struct SnapshotTestConfig {
         accounts_dir: TempDir,
         snapshot_dir: TempDir,
@@ -38,41 +73,47 @@ mod tests {
         genesis_config_info: GenesisConfigInfo,
     }
 
-    fn setup_snapshot_test(snapshot_interval_slots: u64) -> SnapshotTestConfig {
-        let accounts_dir = TempDir::new().unwrap();
-        let snapshot_dir = TempDir::new().unwrap();
-        let snapshot_output_path = TempDir::new().unwrap();
-        let genesis_config_info = create_genesis_config(10_000);
-        let bank0 = Bank::new_with_paths(
-            &genesis_config_info.genesis_config,
-            vec![accounts_dir.path().to_path_buf()],
-            &[],
-        );
-        bank0.freeze();
-        let mut bank_forks = BankForks::new(0, bank0);
-        bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
+    impl SnapshotTestConfig {
+        fn new(
+            snapshot_version: SnapshotVersion,
+            snapshot_interval_slots: u64,
+        ) -> SnapshotTestConfig {
+            let accounts_dir = TempDir::new().unwrap();
+            let snapshot_dir = TempDir::new().unwrap();
+            let snapshot_output_path = TempDir::new().unwrap();
+            let genesis_config_info = create_genesis_config(10_000);
+            let bank0 = Bank::new_with_paths(
+                &genesis_config_info.genesis_config,
+                vec![accounts_dir.path().to_path_buf()],
+                &[],
+            );
+            bank0.freeze();
+            let mut bank_forks = BankForks::new(bank0);
+            bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
 
-        let snapshot_config = SnapshotConfig {
-            snapshot_interval_slots,
-            snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
-            snapshot_path: PathBuf::from(snapshot_dir.path()),
-            compression: CompressionType::Bzip2,
-        };
-        bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
-        SnapshotTestConfig {
-            accounts_dir,
-            snapshot_dir,
-            _snapshot_output_path: snapshot_output_path,
-            snapshot_config,
-            bank_forks,
-            genesis_config_info,
+            let snapshot_config = SnapshotConfig {
+                snapshot_interval_slots,
+                snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
+                snapshot_path: PathBuf::from(snapshot_dir.path()),
+                compression: CompressionType::Bzip2,
+                snapshot_version,
+            };
+            bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
+            SnapshotTestConfig {
+                accounts_dir,
+                snapshot_dir,
+                _snapshot_output_path: snapshot_output_path,
+                snapshot_config,
+                bank_forks,
+                genesis_config_info,
+            }
         }
     }
 
     fn restore_from_snapshot(
         old_bank_forks: &BankForks,
         old_last_slot: Slot,
-        account_paths: Vec<PathBuf>,
+        account_paths: &[PathBuf],
     ) {
         let (snapshot_path, snapshot_package_output_path) = old_bank_forks
             .snapshot_config
@@ -118,17 +159,20 @@ mod tests {
     // also marks each bank as root and generates snapshots
     // finally tries to restore from the last bank's snapshot and compares the restored bank to the
     // `last_slot` bank
-    fn run_bank_forks_snapshot_n<F>(last_slot: Slot, f: F, set_root_interval: u64)
-    where
+    fn run_bank_forks_snapshot_n<F>(
+        snapshot_version: SnapshotVersion,
+        last_slot: Slot,
+        f: F,
+        set_root_interval: u64,
+    ) where
         F: Fn(&mut Bank, &Keypair),
     {
         solana_logger::setup();
         // Set up snapshotting config
-        let mut snapshot_test_config = setup_snapshot_test(1);
+        let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
 
         let bank_forks = &mut snapshot_test_config.bank_forks;
         let accounts_dir = &snapshot_test_config.accounts_dir;
-        let snapshot_config = &snapshot_test_config.snapshot_config;
         let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
 
         let (s, _r) = channel();
@@ -146,36 +190,33 @@ mod tests {
         }
         // Generate a snapshot package for last bank
         let last_bank = bank_forks.get(last_slot).unwrap();
-        let storages: Vec<_> = last_bank.get_snapshot_storages();
-        let slot_snapshot_paths =
-            snapshot_utils::get_snapshot_paths(&snapshot_config.snapshot_path);
+        let snapshot_config = &snapshot_test_config.snapshot_config;
+        let snapshot_path = &snapshot_config.snapshot_path;
+        let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
+            .pop()
+            .expect("no snapshots found in path");
         let snapshot_package = snapshot_utils::package_snapshot(
             last_bank,
-            slot_snapshot_paths
-                .last()
-                .expect("no snapshots found in path"),
-            &snapshot_config.snapshot_path,
+            &last_slot_snapshot_path,
+            snapshot_path,
             &last_bank.src.roots(),
             &snapshot_config.snapshot_package_output_path,
-            storages,
+            last_bank.get_snapshot_storages(),
             CompressionType::Bzip2,
+            snapshot_version,
         )
         .unwrap();
 
         snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
 
-        restore_from_snapshot(
-            bank_forks,
-            last_slot,
-            vec![accounts_dir.path().to_path_buf()],
-        );
+        restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
     }
 
-    #[test]
-    fn test_bank_forks_snapshot_n() {
+    fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
         // create banks upto slot 4 and create 1 new account in each bank. test that bank 4 snapshots
         // and restores correctly
         run_bank_forks_snapshot_n(
+            snapshot_version,
             4,
             |bank, mint_keypair| {
                 let key1 = Keypair::new().pubkey();
@@ -206,24 +247,25 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_concurrent_snapshot_packaging() {
+    fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) {
         solana_logger::setup();
 
         // Set up snapshotting config
-        let mut snapshot_test_config = setup_snapshot_test(1);
+        let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
 
         let bank_forks = &mut snapshot_test_config.bank_forks;
         let accounts_dir = &snapshot_test_config.accounts_dir;
         let snapshots_dir = &snapshot_test_config.snapshot_dir;
         let snapshot_config = &snapshot_test_config.snapshot_config;
+        let snapshot_path = &snapshot_config.snapshot_path;
+        let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
         let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
         let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
 
         // Take snapshot of zeroth bank
         let bank0 = bank_forks.get(0).unwrap();
-        let storages: Vec<_> = bank0.get_snapshot_storages();
-        snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0, &storages).unwrap();
+        let storages = bank0.get_snapshot_storages();
+        snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
 
         // Set up snapshotting channels
         let (sender, receiver) = channel();
@@ -273,7 +315,7 @@ mod tests {
             if slot == saved_slot as u64 {
                 let options = CopyOptions::new();
                 fs_extra::dir::copy(accounts_dir, &saved_accounts_dir, &options).unwrap();
-                let snapshot_paths: Vec<_> = fs::read_dir(&snapshot_config.snapshot_path)
+                let last_snapshot_path = fs::read_dir(snapshot_path)
                     .unwrap()
                     .filter_map(|entry| {
                         let e = entry.unwrap();
@@ -285,17 +327,13 @@ mod tests {
                             .unwrap_or(None)
                     })
                     .sorted()
-                    .collect();
+                    .last()
+                    .unwrap();
                 // only save off the snapshot of this slot, we don't need the others.
-                fs_extra::dir::copy(
-                    &snapshot_paths.last().unwrap(),
-                    &saved_snapshots_dir,
-                    &options,
-                )
-                .unwrap();
+                fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
 
                 saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
-                    &snapshot_config.snapshot_package_output_path,
+                    snapshot_package_output_path,
                     &(slot, accounts_hash),
                     &CompressionType::Bzip2,
                 ));
@@ -305,12 +343,10 @@ mod tests {
         // Purge all the outdated snapshots, including the ones needed to generate the package
         // currently sitting in the channel
         bank_forks.purge_old_snapshots();
-        let mut snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshots_dir);
-        snapshot_paths.sort();
-        assert_eq!(
-            snapshot_paths.iter().map(|path| path.slot).collect_vec(),
-            (3..=MAX_CACHE_ENTRIES as u64 + 2).collect_vec()
-        );
+        assert!(snapshot_utils::get_snapshot_paths(&snapshots_dir)
+            .into_iter()
+            .map(|path| path.slot)
+            .eq(3..=MAX_CACHE_ENTRIES as u64 + 2));
 
         // Create a SnapshotPackagerService to create tarballs from all the pending
         // SnapshotPackage's on the channel. By the time this service starts, we have already
@@ -339,13 +375,12 @@ mod tests {
         // before we compare, stick an empty status_cache in this dir so that the package comparision works
         // This is needed since the status_cache is added by the packager and is not collected from
         // the source dir for snapshots
-        let dummy_slot_deltas: Vec<BankSlotDelta> = vec![];
         snapshot_utils::serialize_snapshot_data_file(
             &saved_snapshots_dir
                 .path()
                 .join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME),
             |stream| {
-                serialize_into(stream, &dummy_slot_deltas)?;
+                serialize_into(stream, &[] as &[BankSlotDelta])?;
                 Ok(())
             },
         )
@@ -361,16 +396,17 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_slots_to_snapshot() {
+    fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) {
         solana_logger::setup();
         let num_set_roots = MAX_CACHE_ENTRIES * 2;
 
         for add_root_interval in &[1, 3, 9] {
             let (snapshot_sender, _snapshot_receiver) = channel();
             // Make sure this test never clears bank.slots_since_snapshot
-            let mut snapshot_test_config =
-                setup_snapshot_test((add_root_interval * num_set_roots * 2) as u64);
+            let mut snapshot_test_config = SnapshotTestConfig::new(
+                snapshot_version,
+                (*add_root_interval * num_set_roots * 2) as u64,
+            );
             let mut current_bank = snapshot_test_config.bank_forks[0].clone();
             let snapshot_sender = Some(snapshot_sender);
             for _ in 0..num_set_roots {
@@ -389,21 +425,20 @@ mod tests {
             }
 
             let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1;
-            let expected_slots_to_snapshot = (num_old_slots as u64
-                ..=num_set_roots as u64 * *add_root_interval as u64)
-                .collect_vec();
+            let expected_slots_to_snapshot =
+                num_old_slots as u64..=num_set_roots as u64 * *add_root_interval as u64;
 
-            let rooted_bank = snapshot_test_config
+            let slots_to_snapshot = snapshot_test_config
                 .bank_forks
                 .get(snapshot_test_config.bank_forks.root())
-                .unwrap();
-            let slots_to_snapshot = rooted_bank.src.roots();
-            assert_eq!(slots_to_snapshot, expected_slots_to_snapshot);
+                .unwrap()
+                .src
+                .roots();
+            assert!(slots_to_snapshot.into_iter().eq(expected_slots_to_snapshot));
         }
     }
 
-    #[test]
-    fn test_bank_forks_status_cache_snapshot_n() {
+    fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) {
         // create banks upto slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time
         // this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves
         // ahead. Also tests the status_cache purge and status cache snapshotting.
@@ -412,6 +447,7 @@ mod tests {
         let key2 = Keypair::new().pubkey();
         for set_root_interval in &[1, 4] {
             run_bank_forks_snapshot_n(
+                snapshot_version,
                 (MAX_CACHE_ENTRIES * 2 + 1) as u64,
                 |bank, mint_keypair| {
                     let tx = system_transaction::transfer(

core/tests/client.rs

@@ -96,7 +96,7 @@ fn test_slot_subscription() {
     let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
     let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
     let bank = Bank::new(&genesis_config);
-    let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+    let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
     let subscriptions = Arc::new(RpcSubscriptions::new(
         &exit,
         bank_forks,

core/tests/crds_gossip.rs

@@ -5,7 +5,7 @@ use solana_core::cluster_info;
 use solana_core::contact_info::ContactInfo;
 use solana_core::crds_gossip::*;
 use solana_core::crds_gossip_error::CrdsGossipError;
-use solana_core::crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
+use solana_core::crds_gossip_pull::{ProcessPullStats, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS};
 use solana_core::crds_gossip_push::CRDS_GOSSIP_PUSH_MSG_TIMEOUT_MS;
 use solana_core::crds_value::CrdsValueLabel;
 use solana_core::crds_value::{CrdsData, CrdsValue};
@@ -426,36 +426,35 @@ fn network_run_pull(
                     .map(|f| f.filter.bits.len() as usize / 8)
                     .sum::<usize>();
                 bytes += serialized_size(&caller_info).unwrap() as usize;
-                let filters = filters
+                let filters: Vec<_> = filters
                     .into_iter()
                     .map(|f| (caller_info.clone(), f))
                     .collect();
-                let rsp = network
+                let rsp: Vec<_> = network
                     .get(&to)
                     .map(|node| {
-                        let mut rsp = vec![];
-                        rsp.append(
-                            &mut node
-                                .lock()
-                                .unwrap()
-                                .process_pull_requests(filters, now)
-                                .into_iter()
-                                .flatten()
-                                .collect(),
-                        );
+                        let rsp = node
+                            .lock()
+                            .unwrap()
+                            .generate_pull_responses(&filters)
+                            .into_iter()
+                            .flatten()
+                            .collect();
+                        node.lock().unwrap().process_pull_requests(filters, now);
                         rsp
                     })
                     .unwrap();
                 bytes += serialized_size(&rsp).unwrap() as usize;
                 msgs += rsp.len();
                 if let Some(node) = network.get(&from) {
-                    node.lock()
-                        .unwrap()
-                        .mark_pull_request_creation_time(&from, now);
-                    overhead += node
-                        .lock()
-                        .unwrap()
-                        .process_pull_response(&from, &timeouts, rsp, now);
+                    let mut node = node.lock().unwrap();
+                    node.mark_pull_request_creation_time(&from, now);
+                    let mut stats = ProcessPullStats::default();
+                    let (vers, vers_expired_timeout) =
+                        node.filter_pull_responses(&timeouts, rsp, now, &mut stats);
+                    node.process_pull_responses(&from, vers, vers_expired_timeout, now, &mut stats);
+                    overhead += stats.failed_insert;
+                    overhead += stats.failed_timeout;
                 }
                 (bytes, msgs, overhead)
             })

core/tests/gossip.rs

@@ -4,13 +4,14 @@ extern crate log;
 use rayon::iter::*;
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::gossip_service::GossipService;
+use solana_ledger::bank_forks::BankForks;
 
 use solana_perf::packet::Packet;
 use solana_sdk::signature::{Keypair, Signer};
 use solana_sdk::timing::timestamp;
 use std::net::UdpSocket;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use std::thread::sleep;
 use std::time::Duration;
 
@@ -27,6 +28,28 @@ fn test_node(exit: &Arc<AtomicBool>) -> (Arc<ClusterInfo>, GossipService, UdpSoc
     )
 }
 
+fn test_node_with_bank(
+    node_keypair: Keypair,
+    exit: &Arc<AtomicBool>,
+    bank_forks: Arc<RwLock<BankForks>>,
+) -> (Arc<ClusterInfo>, GossipService, UdpSocket) {
+    let keypair = Arc::new(node_keypair);
+    let mut test_node = Node::new_localhost_with_pubkey(&keypair.pubkey());
+    let cluster_info = Arc::new(ClusterInfo::new(test_node.info.clone(), keypair));
+    let gossip_service = GossipService::new(
+        &cluster_info,
+        Some(bank_forks),
+        test_node.sockets.gossip,
+        exit,
+    );
+    let _ = cluster_info.my_contact_info();
+    (
+        cluster_info,
+        gossip_service,
+        test_node.sockets.tvu.pop().unwrap(),
+    )
+}
+
 /// Test that the network converges.
 /// Run until every node in the network has a full ContactInfo set.
 /// Check that nodes stop sending updates after all the ContactInfo has been shared.
@@ -181,3 +204,120 @@ pub fn cluster_info_retransmit() {
     dr2.join().unwrap();
     dr3.join().unwrap();
 }
+
+#[test]
+#[ignore]
+pub fn cluster_info_scale() {
+    use solana_measure::measure::Measure;
+    use solana_perf::test_tx::test_tx;
+    use solana_runtime::bank::Bank;
+    use solana_runtime::genesis_utils::{
+        create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs,
+    };
+    solana_logger::setup();
+    let exit = Arc::new(AtomicBool::new(false));
+    let num_nodes: usize = std::env::var("NUM_NODES")
+        .unwrap_or_else(|_| "10".to_string())
+        .parse()
+        .expect("could not parse NUM_NODES as a number");
+
+    let vote_keypairs: Vec<_> = (0..num_nodes)
+        .map(|_| ValidatorVoteKeypairs::new_rand())
+        .collect();
+    let genesis_config_info = create_genesis_config_with_vote_accounts(10_000, &vote_keypairs, 100);
+    let bank0 = Bank::new(&genesis_config_info.genesis_config);
+    let bank_forks = Arc::new(RwLock::new(BankForks::new(bank0)));
+
+    let nodes: Vec<_> = vote_keypairs
+        .into_iter()
+        .map(|keypairs| test_node_with_bank(keypairs.node_keypair, &exit, bank_forks.clone()))
+        .collect();
+    let ci0 = nodes[0].0.my_contact_info();
+    for node in &nodes[1..] {
+        node.0.insert_info(ci0.clone());
+    }
+
+    let mut time = Measure::start("time");
+    let mut done;
+    let mut success = false;
+    for _ in 0..30 {
+        done = true;
+        for (i, node) in nodes.iter().enumerate() {
+            warn!("node {} peers: {}", i, node.0.gossip_peers().len());
+            if node.0.gossip_peers().len() != num_nodes - 1 {
+                done = false;
+                break;
+            }
+        }
+        if done {
+            success = true;
+            break;
+        }
+        sleep(Duration::from_secs(1));
+    }
+    time.stop();
+    warn!("found {} nodes in {} success: {}", num_nodes, time, success);
+
+    for num_votes in 1..1000 {
+        let mut time = Measure::start("votes");
+        let tx = test_tx();
+        warn!("tx.message.account_keys: {:?}", tx.message.account_keys);
+        nodes[0].0.push_vote(0, tx.clone());
+        let mut success = false;
+        for _ in 0..(30 * 5) {
+            let mut not_done = 0;
+            let mut num_old = 0;
+            let mut num_push_total = 0;
+            let mut num_pushes = 0;
+            let mut num_pulls = 0;
+            let mut num_inserts = 0;
+            for node in nodes.iter() {
+                //if node.0.get_votes(0).1.len() != (num_nodes * num_votes) {
+                let has_tx = node
+                    .0
+                    .get_votes(0)
+                    .1
+                    .iter()
+                    .filter(|v| v.message.account_keys == tx.message.account_keys)
+                    .count();
+                num_old += node.0.gossip.read().unwrap().push.num_old;
+                num_push_total += node.0.gossip.read().unwrap().push.num_total;
+                num_pushes += node.0.gossip.read().unwrap().push.num_pushes;
+                num_pulls += node.0.gossip.read().unwrap().pull.num_pulls;
+                num_inserts += node.0.gossip.read().unwrap().crds.num_inserts;
+                if has_tx == 0 {
+                    not_done += 1;
+                }
+            }
+            warn!("not_done: {}/{}", not_done, nodes.len());
+            warn!("num_old: {}", num_old);
+            warn!("num_push_total: {}", num_push_total);
+            warn!("num_pushes: {}", num_pushes);
+            warn!("num_pulls: {}", num_pulls);
+            warn!("num_inserts: {}", num_inserts);
+            success = not_done < (nodes.len() / 20);
+            if success {
+                break;
+            }
+            sleep(Duration::from_millis(200));
+        }
+        time.stop();
+        warn!(
+            "propagated vote {} in {} success: {}",
+            num_votes, time, success
+        );
+        sleep(Duration::from_millis(200));
+        for node in nodes.iter() {
+            node.0.gossip.write().unwrap().push.num_old = 0;
+            node.0.gossip.write().unwrap().push.num_total = 0;
+            node.0.gossip.write().unwrap().push.num_pushes = 0;
+            node.0.gossip.write().unwrap().pull.num_pulls = 0;
+            node.0.gossip.write().unwrap().crds.num_inserts = 0;
+        }
+    }
+
+    exit.store(true, Ordering::Relaxed);
+    for node in nodes {
+        node.1.join().unwrap();
+    }
+}

core/tests/ledger_cleanup.rs

@@ -374,14 +374,16 @@ mod tests {
         // send signal to cleanup slots
         let (sender, receiver) = channel();
         sender.send(n).unwrap();
-        let mut next_purge_batch = 0;
+        let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         LedgerCleanupService::cleanup_ledger(
             &receiver,
             &blockstore,
             max_ledger_shreds,
-            &mut next_purge_batch,
+            &mut last_purge_slot,
+            10,
+            &mut last_compaction_slot,
             10,
-            None,
         )
         .unwrap();
 

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.2.0"
+version = "1.2.8"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

docs/art/sdk-tools.bob

@@ -1,20 +1,20 @@
 
-                            .----------------------------------------.
-                            |            Solana Runtime              |
-                            |                                        |
-                            |   .------------.   .------------.      |
-                            |   |            |   |            |      |
-                      .-------->|  Verifier  +-->|  Accounts  |      |
-                      |     |   |            |   |            |      |
-  .----------.        |     |   `------------`   `------------`      |
-  |          +--------`     |                           ^            |
-  |  Client  |              |             LoadAccounts  |            |
-  |          +--------.     |          .----------------`            |
-  `----------`        |     |          |                             |
-                      |     |   .------+-----.   .-------------.     |
-                      |     |   |            |   |             |     |
-                      `-------->|  Loader    +-->| Interpreter |     |
-                            |   |            |   |             |     |
-                            |   `------------`   `-------------`     |
-                            |                                        |
-                            `----------------------------------------`
+                 .-------------------------------------.
+                 | Solana Runtime                      |
+                 |                                     |
+  .----------.   |   .------------.   .------------.   |
+  | Program  |   |   | BPF        |   | Executable |   |
+  | Author   +------>| Bytecode   +-->| Account    |   |
+  |          |   |   | Verifier   |   |            |   |
+  `----------`   |   `------------`   `------------`   |
+                 |                           |         |
+                 |          .----------------`         |
+                 |          |  LoadAccounts            |
+                 |          V                          |
+  .----------.   |   .------------.   .-------------.  |
+  |          |   |   |  BPF       |   | BPF         |  |
+  |  Client  +------>|  Loader    +-->| Interpreter |  |
+  |          |   |   |            |   |             |  |
+  `----------`   |   `------------`   `-------------`  |
+                 |                                     |
+                 `-------------------------------------`

docs/set-solana-release-tag.sh

@@ -5,12 +5,21 @@ cd "$(dirname "$0")"
 
 if [[ -n $CI_TAG ]]; then
   LATEST_SOLANA_RELEASE_VERSION=$CI_TAG
-else
+elif [[ -z $CI_PULL_REQUEST ]]; then
   LATEST_SOLANA_RELEASE_VERSION=$(\
     curl -sSfL https://api.github.com/repos/solana-labs/solana/releases/latest \
     | grep -m 1 tag_name \
     | sed -ne 's/^ *"tag_name": "\([^"]*\)",$/\1/p' \
   )
+else
+  # Don't bother the `api.github.com` on pull requests to avoid getting rate
+  # limited
+  LATEST_SOLANA_RELEASE_VERSION=unknown-version
+fi
+
+if [[ -z "$LATEST_SOLANA_RELEASE_VERSION" ]]; then
+  echo Error: release version not defined
+  exit 1
 fi
 
 set -x