Use max commitment when fetching epoch info for block production

(cherry picked from commit 2724f37d0e)

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,11 +18,11 @@ lazy_static = "1.4.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
+solana-config-program = { path = "../programs/config", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"
 

account-decoder/src/parse_account_data.rs

@@ -118,7 +118,7 @@ mod test {
 
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
         let parsed = parse_account_data(
             &account_pubkey,

account-decoder/src/parse_vote.rs

@@ -128,7 +128,7 @@ mod test {
     fn test_parse_vote() {
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
         let expected_vote_state = UiVoteState {
             node_pubkey: Pubkey::default().to_string(),

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-measure = { path = "../measure", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-streamer = { path = "../streamer", version = "1.5.0" }
-solana-perf = { path = "../perf", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-core = { path = "../core", version = "1.5.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-streamer = { path = "../streamer", version = "1.5.1" }
+solana-perf = { path = "../perf", version = "1.5.1" }
+solana-ledger = { path = "../ledger", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-measure = { path = "../measure", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,15 +12,15 @@ edition = "2018"
 bincode = "1.3.1"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
+solana-banks-interface = { path = "../banks-interface", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
 tarpc = { version = "0.23.0", features = ["full"] }
-tokio = { version = "0.3", features = ["full"] }
+tokio = { version = "0.3.5", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-banks-server = { path = "../banks-server", version = "1.5.0" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-banks-server = { path = "../banks-server", version = "1.5.1" }
 
 [lib]
 crate-type = ["lib"]

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,11 +11,11 @@ edition = "2018"
 [dependencies]
 mio = "0.7.6"
 serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
 tarpc = { version = "0.23.0", features = ["full"] }
 
 [dev-dependencies]
-tokio = { version = "0.3", features = ["full"] }
+tokio = { version = "0.3.5", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,10 +13,10 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
+solana-banks-interface = { path = "../banks-interface", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-metrics = { path = "../metrics", version = "1.5.1" }
 tarpc = { version = "0.23.0", features = ["full"] }
 tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-core = { path = "../core", version = "1.5.0" }
-solana-genesis = { path = "../genesis", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-core = { path = "../core", version = "1.5.1" }
+solana-genesis = { path = "../genesis", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-faucet = { path = "../faucet", version = "1.5.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-metrics = { path = "../metrics", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.5.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-streamer = { path = "../streamer", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-streamer = { path = "../streamer", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,23 +15,23 @@ log = "0.4.11"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-core = { path = "../core", version = "1.5.0" }
-solana-genesis = { path = "../genesis", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-core = { path = "../core", version = "1.5.1" }
+solana-genesis = { path = "../genesis", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-faucet = { path = "../faucet", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-metrics = { path = "../metrics", version = "1.5.1" }
+solana-measure = { path = "../measure", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.5.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

ci/test-checks.sh

@@ -78,11 +78,6 @@ cargo_audit_ignores=(
   #
   # Blocked on multiple crates updating `time` to >= 0.2.23
   --ignore RUSTSEC-2020-0071
-
-  # memmap crate is unmaintained
-  #
-  # Blocked on us releasing new solana crates
-  --ignore RUSTSEC-2020-0077
 )
 _ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit "${cargo_audit_ignores[@]}"
 

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/input_validators.rs

@@ -1,7 +1,7 @@
 use crate::keypair::{parse_keypair_path, KeypairUrl, ASK_KEYWORD};
 use chrono::DateTime;
 use solana_sdk::{
-    clock::Slot,
+    clock::{Epoch, Slot},
     hash::Hash,
     pubkey::Pubkey,
     signature::{read_keypair_file, Signature},
@@ -148,6 +148,13 @@ where
     }
 }
 
+pub fn is_epoch<T>(epoch: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<Epoch, _>(epoch)
+}
+
 pub fn is_slot<T>(slot: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,

clap-utils/src/keypair.rs

@@ -58,6 +58,15 @@ impl CliSignerInfo {
             Some(0)
         }
     }
+    pub fn index_of_or_none(&self, pubkey: Option<Pubkey>) -> Option<usize> {
+        if let Some(pubkey) = pubkey {
+            self.signers
+                .iter()
+                .position(|signer| signer.pubkey() == pubkey)
+        } else {
+            None
+        }
+    }
 }
 
 pub struct DefaultSigner {

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,13 +17,13 @@ indicatif = "0.15.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -241,6 +241,9 @@ impl fmt::Display for CliEpochInfo {
         )?;
         writeln_name_value(f, "Slot:", &self.epoch_info.absolute_slot.to_string())?;
         writeln_name_value(f, "Epoch:", &self.epoch_info.epoch.to_string())?;
+        if let Some(transaction_count) = &self.epoch_info.transaction_count {
+            writeln_name_value(f, "Transaction Count:", &transaction_count.to_string())?;
+        }
         let start_slot = self.epoch_info.absolute_slot - self.epoch_info.slot_index;
         let end_slot = start_slot + self.epoch_info.slots_in_epoch;
         writeln_name_value(
@@ -646,13 +649,13 @@ fn show_epoch_rewards(
         writeln!(f, "Epoch Rewards:")?;
         writeln!(
             f,
-            "  {:<8}  {:<11}  {:<15}  {:<15}  {:>14}  {:>14}",
+            "  {:<6}  {:<11}  {:<16}  {:<16}  {:>14}  {:>14}",
             "Epoch", "Reward Slot", "Amount", "New Balance", "Percent Change", "APR"
         )?;
         for reward in epoch_rewards {
             writeln!(
                 f,
-                "  {:<8}  {:<11}  ◎{:<14.9}  ◎{:<14.9}  {:>13.9}%  {}",
+                "  {:<6}  {:<11}  ◎{:<16.9}  ◎{:<14.9}  {:>13.2}%  {}",
                 reward.epoch,
                 reward.effective_slot,
                 lamports_to_sol(reward.amount),
@@ -660,7 +663,7 @@ fn show_epoch_rewards(
                 reward.percent_change,
                 reward
                     .apr
-                    .map(|apr| format!("{:>13.9}%", apr))
+                    .map(|apr| format!("{:>13.2}%", apr))
                     .unwrap_or_default(),
             )?;
         }

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,29 +27,29 @@ reqwest = { version = "0.10.8", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-cli-config = { path = "../cli-config", version = "1.5.0" }
-solana-cli-output = { path = "../cli-output", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-config-program = { path = "../programs/config", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.1" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-cli-config = { path = "../cli-config", version = "1.5.1" }
+solana-cli-output = { path = "../cli-output", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-config-program = { path = "../programs/config", version = "1.5.1" }
+solana-faucet = { path = "../faucet", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
 solana_rbpf = "=0.2.2"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.5.0" }
+solana-core = { path = "../core", version = "1.5.1" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cluster_query.rs

@@ -132,7 +132,17 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .help("Slot number of the block to query")
             )
         )
-        .subcommand(SubCommand::with_name("leader-schedule").about("Display leader schedule"))
+        .subcommand(SubCommand::with_name("leader-schedule")
+            .about("Display leader schedule")
+            .arg(
+                Arg::with_name("epoch")
+                    .long("epoch")
+                    .takes_value(true)
+                    .value_name("EPOCH")
+                    .validator(is_epoch)
+                    .help("Epoch to show leader schedule for. (default: current)")
+            )
+        )
         .subcommand(
             SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
@@ -713,9 +723,23 @@ pub fn process_first_available_block(rpc_client: &RpcClient) -> ProcessResult {
     Ok(format!("{}", first_available_block))
 }
 
-pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
+pub fn parse_leader_schedule(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let epoch = value_of(matches, "epoch");
+    Ok(CliCommandInfo {
+        command: CliCommand::LeaderSchedule { epoch },
+        signers: vec![],
+    })
+}
+
+pub fn process_leader_schedule(rpc_client: &RpcClient, epoch: Option<Epoch>) -> ProcessResult {
     let epoch_info = rpc_client.get_epoch_info()?;
-    let first_slot_in_epoch = epoch_info.absolute_slot - epoch_info.slot_index;
+    let epoch = epoch.unwrap_or(epoch_info.epoch);
+    if epoch > epoch_info.epoch {
+        return Err(format!("Epoch {} is in the future", epoch).into());
+    }
+
+    let epoch_schedule = rpc_client.get_epoch_schedule()?;
+    let first_slot_in_epoch = epoch_schedule.get_first_slot_in_epoch(epoch);
 
     let leader_schedule = rpc_client.get_leader_schedule(Some(first_slot_in_epoch))?;
     if leader_schedule.is_none() {
@@ -800,8 +824,9 @@ pub fn process_get_block(
                     format!(
                         "◎{:<19.9}  {:>13.9}%",
                         lamports_to_sol(reward.post_balance),
-                        reward.lamports.abs() as f64
-                            / (reward.post_balance as f64 - reward.lamports as f64)
+                        (reward.lamports.abs() as f64
+                            / (reward.post_balance as f64 - reward.lamports as f64))
+                            * 100.0
                     )
                 }
             );
@@ -886,7 +911,7 @@ pub fn process_show_block_production(
     slot_limit: Option<u64>,
 ) -> ProcessResult {
     let epoch_schedule = rpc_client.get_epoch_schedule()?;
-    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::root())?;
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(CommitmentConfig::max())?;
 
     let epoch = epoch.unwrap_or(epoch_info.epoch);
     if epoch > epoch_info.epoch {

cli/src/lib.rs

@@ -26,6 +26,7 @@ pub mod cluster_query;
 pub mod feature;
 pub mod inflation;
 pub mod nonce;
+pub mod program;
 pub mod send_tpu;
 pub mod spend_utils;
 pub mod stake;

cli/src/main.rs

@@ -182,11 +182,16 @@ pub fn parse_args<'a>(
             OutputFormat::Display
         });
 
-    let commitment = matches
-        .subcommand_name()
-        .and_then(|name| matches.subcommand_matches(name))
-        .and_then(|sub_matches| commitment_of(sub_matches, COMMITMENT_ARG.long))
-        .unwrap_or_default();
+    let commitment = {
+        let mut sub_matches = matches;
+        while let Some(subcommand_name) = sub_matches.subcommand_name() {
+            sub_matches = sub_matches
+                .subcommand_matches(subcommand_name)
+                .expect("subcommand_matches");
+        }
+        commitment_of(sub_matches, COMMITMENT_ARG.long)
+    }
+    .unwrap_or_default();
 
     let address_labels = if matches.is_present("no_address_labels") {
         HashMap::new()
@@ -206,7 +211,10 @@ pub fn parse_args<'a>(
             verbose,
             output_format,
             commitment,
-            send_transaction_config: RpcSendTransactionConfig::default(),
+            send_transaction_config: RpcSendTransactionConfig {
+                preflight_commitment: Some(commitment.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
             address_labels,
         },
         signers,

cli/src/stake.rs

@@ -1662,13 +1662,13 @@ pub(crate) fn fetch_epoch_rewards(
                 .find(|reward| reward.pubkey == address.to_string())
             {
                 if reward.post_balance > reward.lamports.try_into().unwrap_or(0) {
-                    let percent_change = reward.lamports.abs() as f64
+                    let rate_change = reward.lamports.abs() as f64
                         / (reward.post_balance as f64 - reward.lamports as f64);
 
                     let apr = wallclock_epoch_duration.map(|wallclock_epoch_duration| {
                         let wallclock_epochs_per_year =
                             (SECONDS_PER_DAY * 356) as f64 / wallclock_epoch_duration;
-                        percent_change * wallclock_epochs_per_year
+                        rate_change * wallclock_epochs_per_year
                     });
 
                     all_epoch_rewards.push(CliEpochReward {
@@ -1676,8 +1676,8 @@ pub(crate) fn fetch_epoch_rewards(
                         effective_slot,
                         amount: reward.lamports.abs() as u64,
                         post_balance: reward.post_balance,
-                        percent_change,
-                        apr,
+                        percent_change: rate_change * 100.0,
+                        apr: apr.map(|r| r * 100.0),
                     });
                 }
             }

cli/tests/program.rs

@@ -1,5 +1,8 @@
 use serde_json::Value;
-use solana_cli::cli::{process_command, CliCommand, CliConfig};
+use solana_cli::{
+    cli::{process_command, CliCommand, CliConfig},
+    program::ProgramCliCommand,
+};
 use solana_client::rpc_client::RpcClient;
 use solana_core::test_validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
@@ -13,7 +16,7 @@ use solana_sdk::{
 use std::{fs::File, io::Read, path::PathBuf, str::FromStr, sync::mpsc::channel};
 
 #[test]
-fn test_cli_deploy_program() {
+fn test_cli_program_deploy_non_upgradeable() {
     solana_logger::setup();
 
     let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -41,25 +44,21 @@ fn test_cli_deploy_program() {
     let mut config = CliConfig::recent_for_tests();
     let keypair = Keypair::new();
     config.json_rpc_url = test_validator.rpc_url();
+    config.signers = vec![&keypair];
     config.command = CliCommand::Airdrop {
         faucet_host: None,
         faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 4 * minimum_balance_for_rent_exemption, // min balance for rent exemption for three programs + leftover for tx processing
     };
-    config.signers = vec![&keypair];
     process_command(&config).unwrap();
 
-    config.command = CliCommand::ProgramDeploy {
+    config.command = CliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: None,
+        address: None,
         use_deprecated_loader: false,
-        use_upgradeable_loader: false,
         allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: None,
     };
-
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let program_id_str = json
@@ -78,24 +77,19 @@ fn test_cli_deploy_program() {
     assert_eq!(account0.lamports, minimum_balance_for_rent_exemption);
     assert_eq!(account0.owner, bpf_loader::id());
     assert_eq!(account0.executable, true);
-
     let mut file = File::open(pathbuf.to_str().unwrap().to_string()).unwrap();
     let mut elf = Vec::new();
     file.read_to_end(&mut elf).unwrap();
-
     assert_eq!(account0.data, elf);
 
     // Test custom address
     let custom_address_keypair = Keypair::new();
     config.signers = vec![&keypair, &custom_address_keypair];
-    config.command = CliCommand::ProgramDeploy {
+    config.command = CliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: Some(1),
+        address: Some(1),
         use_deprecated_loader: false,
-        use_upgradeable_loader: false,
         allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: None,
     };
     process_command(&config).unwrap();
     let account1 = rpc_client
@@ -106,43 +100,36 @@ fn test_cli_deploy_program() {
     assert_eq!(account1.lamports, minimum_balance_for_rent_exemption);
     assert_eq!(account1.owner, bpf_loader::id());
     assert_eq!(account1.executable, true);
-    assert_eq!(account0.data, account1.data);
+    assert_eq!(account1.data, account0.data);
 
     // Attempt to redeploy to the same address
     process_command(&config).unwrap_err();
 
     // Attempt to deploy to account with excess balance
     let custom_address_keypair = Keypair::new();
+    config.signers = vec![&custom_address_keypair];
     config.command = CliCommand::Airdrop {
         faucet_host: None,
         faucet_port: faucet_addr.port(),
         pubkey: None,
         lamports: 2 * minimum_balance_for_rent_exemption, // Anything over minimum_balance_for_rent_exemption should trigger err
     };
-    config.signers = vec![&custom_address_keypair];
     process_command(&config).unwrap();
-
     config.signers = vec![&keypair, &custom_address_keypair];
-    config.command = CliCommand::ProgramDeploy {
+    config.command = CliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: Some(1),
+        address: Some(1),
         use_deprecated_loader: false,
-        use_upgradeable_loader: false,
         allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: None,
     };
     process_command(&config).unwrap_err();
 
     // Use forcing parameter to deploy to account with excess balance
-    config.command = CliCommand::ProgramDeploy {
+    config.command = CliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: Some(1),
+        address: Some(1),
         use_deprecated_loader: false,
-        use_upgradeable_loader: false,
         allow_excessive_balance: true,
-        upgrade_authority: None,
-        max_len: None,
     };
     process_command(&config).unwrap();
     let account2 = rpc_client
@@ -153,11 +140,11 @@ fn test_cli_deploy_program() {
     assert_eq!(account2.lamports, 2 * minimum_balance_for_rent_exemption);
     assert_eq!(account2.owner, bpf_loader::id());
     assert_eq!(account2.executable, true);
-    assert_eq!(account0.data, account2.data);
+    assert_eq!(account2.data, account0.data);
 }
 
 #[test]
-fn test_cli_deploy_upgradeable_program() {
+fn test_cli_program_deploy_no_authority() {
     solana_logger::setup();
 
     let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -179,11 +166,6 @@ fn test_cli_deploy_upgradeable_program() {
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
-    println!(
-        "max_len {:?} {:?}",
-        max_len,
-        UpgradeableLoaderState::programdata_len(max_len)
-    );
     let minimum_balance_for_programdata = rpc_client
         .get_minimum_balance_for_rent_exemption(
             UpgradeableLoaderState::programdata_len(max_len).unwrap(),
@@ -206,16 +188,18 @@ fn test_cli_deploy_upgradeable_program() {
     config.signers = vec![&keypair];
     process_command(&config).unwrap();
 
-    // Deploy and attempt to upgrade a non-upgradeable program
-    config.command = CliCommand::ProgramDeploy {
+    // Deploy a program with no authority
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: None,
-        use_deprecated_loader: false,
-        use_upgradeable_loader: true,
+        program_signer_index: None,
+        program_pubkey: None,
+        buffer_signer_index: None,
         allow_excessive_balance: false,
-        upgrade_authority: None,
-        max_len: Some(max_len),
-    };
+        upgrade_authority_signer_index: None,
+        upgrade_authority_pubkey: None,
+        max_len: None,
+    });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let program_id_str = json
@@ -227,25 +211,132 @@ fn test_cli_deploy_upgradeable_program() {
         .unwrap();
     let program_id = Pubkey::from_str(&program_id_str).unwrap();
 
+    // Attempt to upgrade the program
     config.signers = vec![&keypair, &upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
+        program_signer_index: None,
+        program_pubkey: Some(program_id),
+        buffer_signer_index: None,
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: Some(1),
+        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
+        max_len: None,
+    });
     process_command(&config).unwrap_err();
+}
+
+#[test]
+fn test_cli_program_deploy_with_authority() {
+    solana_logger::setup();
+
+    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    pathbuf.push("tests");
+    pathbuf.push("fixtures");
+    pathbuf.push("noop");
+    pathbuf.set_extension("so");
+
+    let mint_keypair = Keypair::new();
+    let test_validator = TestValidator::with_no_fees(mint_keypair.pubkey());
+
+    let (sender, receiver) = channel();
+    run_local_faucet(mint_keypair, sender, None);
+    let faucet_addr = receiver.recv().unwrap();
+
+    let rpc_client = RpcClient::new(test_validator.rpc_url());
+
+    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut program_data = Vec::new();
+    file.read_to_end(&mut program_data).unwrap();
+    let max_len = program_data.len();
+    let minimum_balance_for_programdata = rpc_client
+        .get_minimum_balance_for_rent_exemption(
+            UpgradeableLoaderState::programdata_len(max_len).unwrap(),
+        )
+        .unwrap();
+    let minimum_balance_for_program = rpc_client
+        .get_minimum_balance_for_rent_exemption(UpgradeableLoaderState::program_len().unwrap())
+        .unwrap();
+    let upgrade_authority = Keypair::new();
+
+    let mut config = CliConfig::recent_for_tests();
+    let keypair = Keypair::new();
+    config.json_rpc_url = test_validator.rpc_url();
+    config.signers = vec![&keypair];
+    config.command = CliCommand::Airdrop {
+        faucet_host: None,
+        faucet_port: faucet_addr.port(),
+        pubkey: None,
+        lamports: 100 * minimum_balance_for_programdata + minimum_balance_for_program,
+    };
+    process_command(&config).unwrap();
+
+    // Deploy the upgradeable program with specified program_id
+    let program_keypair = Keypair::new();
+    config.signers = vec![&keypair, &upgrade_authority, &program_keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
+        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_signer_index: Some(2),
+        program_pubkey: Some(program_keypair.pubkey()),
+        buffer_signer_index: None,
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: Some(1),
+        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
+        max_len: Some(max_len),
+    });
+    let response = process_command(&config);
+    let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
+    let program_id_str = json
+        .as_object()
+        .unwrap()
+        .get("programId")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    assert_eq!(
+        program_keypair.pubkey(),
+        Pubkey::from_str(&program_id_str).unwrap()
+    );
+    let program_account = rpc_client
+        .get_account_with_commitment(&program_keypair.pubkey(), CommitmentConfig::recent())
+        .unwrap()
+        .value
+        .unwrap();
+    assert_eq!(program_account.lamports, minimum_balance_for_program);
+    assert_eq!(program_account.owner, bpf_loader_upgradeable::id());
+    assert_eq!(program_account.executable, true);
+    let (programdata_pubkey, _) = Pubkey::find_program_address(
+        &[program_keypair.pubkey().as_ref()],
+        &bpf_loader_upgradeable::id(),
+    );
+    let programdata_account = rpc_client
+        .get_account_with_commitment(&programdata_pubkey, CommitmentConfig::recent())
+        .unwrap()
+        .value
+        .unwrap();
+    assert_eq!(
+        programdata_account.lamports,
+        minimum_balance_for_programdata
+    );
+    assert_eq!(programdata_account.owner, bpf_loader_upgradeable::id());
+    assert_eq!(programdata_account.executable, false);
+    assert_eq!(
+        programdata_account.data[UpgradeableLoaderState::programdata_data_offset().unwrap()..],
+        program_data[..]
+    );
 
     // Deploy the upgradeable program
-    config.command = CliCommand::ProgramDeploy {
+    config.signers = vec![&keypair, &upgrade_authority];
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        buffer: None,
-        use_deprecated_loader: false,
-        use_upgradeable_loader: true,
+        program_signer_index: None,
+        program_pubkey: None,
+        buffer_signer_index: None,
         allow_excessive_balance: false,
-        upgrade_authority: Some(upgrade_authority.pubkey()),
+        upgrade_authority_signer_index: Some(1),
+        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
         max_len: Some(max_len),
-    };
+    });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let program_id_str = json
@@ -284,12 +375,16 @@ fn test_cli_deploy_upgradeable_program() {
 
     // Upgrade the program
     config.signers = vec![&keypair, &upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
+        program_signer_index: None,
+        program_pubkey: Some(program_id),
+        buffer_signer_index: None,
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: Some(1),
+        upgrade_authority_pubkey: Some(upgrade_authority.pubkey()),
+        max_len: Some(max_len),
+    });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let program_id_str = json
@@ -329,11 +424,11 @@ fn test_cli_deploy_upgradeable_program() {
     // Set a new authority
     let new_upgrade_authority = Keypair::new();
     config.signers = vec![&keypair, &upgrade_authority];
-    config.command = CliCommand::SetProgramUpgradeAuthority {
+    config.command = CliCommand::Program(ProgramCliCommand::SetUpgradeAuthority {
         program: program_id,
-        upgrade_authority: 1,
+        upgrade_authority_index: Some(1),
         new_upgrade_authority: Some(new_upgrade_authority.pubkey()),
-    };
+    });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let new_upgrade_authority_str = json
@@ -350,12 +445,16 @@ fn test_cli_deploy_upgradeable_program() {
 
     // Upgrade with new authority
     config.signers = vec![&keypair, &new_upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
+        program_signer_index: None,
+        program_pubkey: Some(program_id),
+        buffer_signer_index: None,
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: Some(1),
+        upgrade_authority_pubkey: Some(new_upgrade_authority.pubkey()),
+        max_len: None,
+    });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let program_id_str = json
@@ -392,13 +491,13 @@ fn test_cli_deploy_upgradeable_program() {
         program_data[..]
     );
 
-    // Set a no authority
+    // Set no authority
     config.signers = vec![&keypair, &new_upgrade_authority];
-    config.command = CliCommand::SetProgramUpgradeAuthority {
+    config.command = CliCommand::Program(ProgramCliCommand::SetUpgradeAuthority {
         program: program_id,
-        upgrade_authority: 1,
+        upgrade_authority_index: Some(1),
         new_upgrade_authority: None,
-    };
+    });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
     let new_upgrade_authority_str = json
@@ -412,11 +511,15 @@ fn test_cli_deploy_upgradeable_program() {
 
     // Upgrade with no authority
     config.signers = vec![&keypair, &new_upgrade_authority];
-    config.command = CliCommand::ProgramUpgrade {
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
-        program: program_id,
-        buffer: None,
-        upgrade_authority: 1,
-    };
+        program_signer_index: None,
+        program_pubkey: Some(program_id),
+        buffer_signer_index: None,
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: Some(1),
+        upgrade_authority_pubkey: Some(new_upgrade_authority.pubkey()),
+        max_len: None,
+    });
     process_command(&config).unwrap_err();
 }

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -23,13 +23,13 @@ semver = "0.11.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -38,7 +38,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "15.0.0"
 jsonrpc-http-server = "15.0.0"
-solana-logger = { path = "../logger", version = "1.5.0" }
+solana-logger = { path = "../logger", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/client_error.rs

@@ -5,6 +5,8 @@ use solana_sdk::{
 use std::io;
 use thiserror::Error;
 
+pub use reqwest; // export `reqwest` for clients
+
 #[derive(Error, Debug)]
 pub enum ClientErrorKind {
     #[error(transparent)]

client/src/mock_sender.rs

@@ -48,6 +48,10 @@ impl RpcSender for MockSender {
             return Ok(Value::Null);
         }
         let val = match request {
+            RpcRequest::GetAccountInfo => serde_json::to_value(Response {
+                context: RpcResponseContext { slot: 1 },
+                value: Value::Null,
+            })?,
             RpcRequest::GetBalance => serde_json::to_value(Response {
                 context: RpcResponseContext { slot: 1 },
                 value: Value::Number(Number::from(50)),
@@ -65,6 +69,7 @@ impl RpcSender for MockSender {
                 slots_in_epoch: 32,
                 absolute_slot: 34,
                 block_height: 34,
+                transaction_count: Some(123),
             })?,
             RpcRequest::GetFeeCalculatorForBlockhash => {
                 let value = if self.url == "blockhash_expired" {

client/src/rpc_custom_error.rs

@@ -10,6 +10,7 @@ pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE: i64
 pub const JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE: i64 = -32004;
 pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY: i64 = -32005;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_PRECOMPILE_VERIFICATION_FAILURE: i64 = -32006;
+pub const JSON_RPC_SERVER_ERROR_SLOT_SKIPPED: i64 = -32007;
 
 pub enum RpcCustomError {
     BlockCleanedUp {
@@ -26,6 +27,9 @@ pub enum RpcCustomError {
     },
     RpcNodeUnhealthy,
     TransactionPrecompileVerificationFailure(solana_sdk::transaction::TransactionError),
+    SlotSkipped {
+        slot: Slot,
+    },
 }
 
 impl From<RpcCustomError> for Error {
@@ -73,6 +77,14 @@ impl From<RpcCustomError> for Error {
                 message: format!("Transaction precompile verification failure {:?}", e),
                 data: None,
             },
+            RpcCustomError::SlotSkipped { slot } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_SLOT_SKIPPED),
+                message: format!(
+                    "Slot {} was skipped, or missing due to ledger jump to recent snapshot",
+                    slot
+                ),
+                data: None,
+            },
         }
     }
 }

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -46,40 +46,43 @@ raptorq = "1.4.2"
 rayon = "1.4.1"
 regex = "1.3.9"
 serde = "1.0.112"
+serde_bytes = "0.11"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.5.0" }
-solana-banks-server = { path = "../banks-server", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-perf = { path = "../perf", version = "1.5.0" }
-solana-program-test = { path = "../program-test", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "1.5.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.0" }
-solana-streamer = { path = "../streamer", version = "1.5.0" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
+solana-account-decoder = { path = "../account-decoder", version = "1.5.1" }
+solana-banks-server = { path = "../banks-server", version = "1.5.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-faucet = { path = "../faucet", version = "1.5.1" }
+solana-ledger = { path = "../ledger", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.5.1" }
+solana-metrics = { path = "../metrics", version = "1.5.1" }
+solana-measure = { path = "../measure", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
+solana-perf = { path = "../perf", version = "1.5.1" }
+solana-program-test = { path = "../program-test", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "1.5.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.1" }
+solana-streamer = { path = "../streamer", version = "1.5.1" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.5.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "0.2", features = ["full"] }
 tokio_01 = { version = "0.1", package = "tokio" }
+tokio_01_bytes = { version = "0.4.7", package = "bytes" }
 tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
 tokio_io_01 = { version = "0.1", package = "tokio-io" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.0" }
+tokio_codec_01 = { version = "0.1", package = "tokio-codec" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.1" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/src/accounts_hash_verifier.rs

@@ -254,6 +254,9 @@ mod tests {
                 0,
                 100,
             );
+            // sleep for 1ms to create a newer timestmap for gossip entry
+            // otherwise the timestamp won't be newer.
+            std::thread::sleep(Duration::from_millis(1));
         }
         cluster_info.flush_push_queue();
         let cluster_hashes = cluster_info

core/src/banking_stage.rs

@@ -4,7 +4,7 @@
 use crate::{
     cluster_info::ClusterInfo,
     poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
-    poh_service::PohService,
+    poh_service::{self, PohService},
 };
 use crossbeam_channel::{Receiver as CrossbeamReceiver, RecvTimeoutError};
 use itertools::Itertools;
@@ -58,7 +58,7 @@ type PacketsAndOffsets = (Packets, Vec<usize>);
 pub type UnprocessedPackets = Vec<PacketsAndOffsets>;
 
 /// Transaction forwarding
-pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 1;
+pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 2;
 
 // Fixed thread size seems to be fastest on GCP setup
 pub const NUM_THREADS: u32 = 4;
@@ -1088,7 +1088,13 @@ pub fn create_test_recorder(
     poh_recorder.set_bank(&bank);
 
     let poh_recorder = Arc::new(Mutex::new(poh_recorder));
-    let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
+    let poh_service = PohService::new(
+        poh_recorder.clone(),
+        &poh_config,
+        &exit,
+        bank.ticks_per_slot(),
+        poh_service::DEFAULT_PINNED_CPU_CORE,
+    );
 
     (exit, poh_recorder, poh_service, entry_receiver)
 }

core/src/broadcast_stage.rs

@@ -17,7 +17,7 @@ use crossbeam_channel::{
     Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
     Sender as CrossbeamSender,
 };
-use solana_ledger::{blockstore::Blockstore, shred::Shred, staking_utils};
+use solana_ledger::{blockstore::Blockstore, shred::Shred};
 use solana_measure::measure::Measure;
 use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
 use solana_runtime::bank::Bank;
@@ -306,7 +306,7 @@ impl BroadcastStage {
 
         for (_, bank) in retransmit_slots.iter() {
             let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-            let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+            let stakes = bank.epoch_staked_nodes(bank_epoch);
             let stakes = stakes.map(Arc::new);
             let data_shreds = Arc::new(
                 blockstore
@@ -347,7 +347,7 @@ fn update_peer_stats(
 ) {
     let now = timestamp();
     let last = last_datapoint_submit.load(Ordering::Relaxed);
-    if now - last > 1000
+    if now.saturating_sub(last) > 1000
         && last_datapoint_submit.compare_and_swap(last, now, Ordering::Relaxed) == last
     {
         datapoint_info!(

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -102,7 +102,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         blockstore_sender.send((data_shreds.clone(), None))?;
         // 4) Start broadcast step
         let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-        let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+        let stakes = bank.epoch_staked_nodes(bank_epoch);
         let stakes = stakes.map(Arc::new);
         socket_sender.send(((stakes.clone(), data_shreds), None))?;
         if let Some((good_last_data_shred, bad_last_data_shred)) = last_shreds {

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -213,7 +213,7 @@ impl StandardBroadcastRun {
 
         let mut get_leader_schedule_time = Measure::start("broadcast_get_leader_schedule");
         let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-        let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+        let stakes = bank.epoch_staked_nodes(bank_epoch);
         let stakes = stakes.map(Arc::new);
 
         // Broadcast the last shred of the interrupted slot if necessary

core/src/cluster_info.rs

@@ -36,10 +36,10 @@ use solana_sdk::sanitize::{Sanitize, SanitizeError};
 use bincode::{serialize, serialized_size};
 use core::cmp;
 use itertools::Itertools;
+use rand::thread_rng;
 use rayon::prelude::*;
 use rayon::{ThreadPool, ThreadPoolBuilder};
 use serde::ser::Serialize;
-use solana_ledger::staking_utils;
 use solana_measure::measure::Measure;
 use solana_measure::thread_mem_usage;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_error};
@@ -69,8 +69,11 @@ use std::{
     cmp::min,
     collections::{hash_map::Entry, HashMap, HashSet, VecDeque},
     fmt::{self, Debug},
+    fs::{self, File},
+    io::BufReader,
     net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket},
     ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
     sync::atomic::{AtomicBool, AtomicU64, Ordering},
     sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
     thread::{sleep, Builder, JoinHandle},
@@ -107,7 +110,10 @@ const MAX_PRUNE_DATA_NODES: usize = 32;
 const GOSSIP_PING_TOKEN_SIZE: usize = 32;
 const GOSSIP_PING_CACHE_CAPACITY: usize = 16384;
 const GOSSIP_PING_CACHE_TTL: Duration = Duration::from_secs(640);
-pub const DEFAULT_CONTACT_DEBUG_INTERVAL: u64 = 10_000;
+pub const DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS: u64 = 10_000;
+pub const DEFAULT_CONTACT_SAVE_INTERVAL_MILLIS: u64 = 60_000;
+/// Minimum serialized size of a Protocol::PullResponse packet.
+const PULL_RESPONSE_MIN_SERIALIZED_SIZE: usize = 167;
 
 #[derive(Debug, PartialEq, Eq)]
 pub enum ClusterInfoError {
@@ -235,10 +241,8 @@ struct GossipStats {
     entrypoint2: Counter,
     gossip_packets_dropped_count: Counter,
     push_vote_read: Counter,
-    vote_process_push: Counter,
     get_votes: Counter,
     get_accounts_hash: Counter,
-    get_snapshot_hash: Counter,
     all_tvu_peers: Counter,
     tvu_peers: Counter,
     retransmit_peers: Counter,
@@ -271,8 +275,6 @@ struct GossipStats {
     pull_request_ping_pong_check_failed_count: Counter,
     purge: Counter,
     epoch_slots_lookup: Counter,
-    epoch_slots_push: Counter,
-    push_message: Counter,
     new_pull_requests: Counter,
     new_pull_requests_count: Counter,
     mark_pull_request: Counter,
@@ -290,8 +292,8 @@ pub struct ClusterInfo {
     pub gossip: RwLock<CrdsGossip>,
     /// set the keypair that will be used to sign crds values generated. It is unset only in tests.
     pub(crate) keypair: Arc<Keypair>,
-    /// The network entrypoint
-    entrypoint: RwLock<Option<ContactInfo>>,
+    /// Network entrypoints
+    entrypoints: RwLock<Vec<ContactInfo>>,
     outbound_budget: DataBudget,
     my_contact_info: RwLock<ContactInfo>,
     ping_cache: RwLock<PingCache>,
@@ -299,8 +301,10 @@ pub struct ClusterInfo {
     stats: GossipStats,
     socket: UdpSocket,
     local_message_pending_push_queue: RwLock<Vec<(CrdsValue, u64)>>,
-    contact_debug_interval: u64,
+    contact_debug_interval: u64, // milliseconds, 0 = disabled
+    contact_save_interval: u64,  // milliseconds, 0 = disabled
     instance: NodeInstance,
+    contact_info_path: PathBuf,
 }
 
 impl Default for ClusterInfo {
@@ -425,7 +429,7 @@ pub fn make_accounts_hashes_message(
 type Ping = ping_pong::Ping<[u8; GOSSIP_PING_TOKEN_SIZE]>;
 
 // TODO These messages should go through the gpu pipeline for spam filtering
-#[frozen_abi(digest = "6PpTdBvyX37y5ERokb8DejgKobpsuTbFJC39f8Eqz7Vy")]
+#[frozen_abi(digest = "HAFjUDgiGthYTiAg6CYJxA8PqfwuhrC82NtHYYmee4vb")]
 #[derive(Serialize, Deserialize, Debug, AbiEnumVisitor, AbiExample)]
 #[allow(clippy::large_enum_variant)]
 enum Protocol {
@@ -545,7 +549,7 @@ impl ClusterInfo {
         let me = Self {
             gossip: RwLock::new(CrdsGossip::default()),
             keypair,
-            entrypoint: RwLock::new(None),
+            entrypoints: RwLock::new(vec![]),
             outbound_budget: DataBudget::default(),
             my_contact_info: RwLock::new(contact_info),
             ping_cache: RwLock::new(PingCache::new(
@@ -556,8 +560,10 @@ impl ClusterInfo {
             stats: GossipStats::default(),
             socket: UdpSocket::bind("0.0.0.0:0").unwrap(),
             local_message_pending_push_queue: RwLock::new(vec![]),
-            contact_debug_interval: DEFAULT_CONTACT_DEBUG_INTERVAL,
-            instance: NodeInstance::new(&mut rand::thread_rng(), id, timestamp()),
+            contact_debug_interval: DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
+            instance: NodeInstance::new(&mut thread_rng(), id, timestamp()),
+            contact_info_path: PathBuf::default(),
+            contact_save_interval: 0, // disabled
         };
         {
             let mut gossip = me.gossip.write().unwrap();
@@ -578,7 +584,7 @@ impl ClusterInfo {
         ClusterInfo {
             gossip: RwLock::new(gossip),
             keypair: self.keypair.clone(),
-            entrypoint: RwLock::new(self.entrypoint.read().unwrap().clone()),
+            entrypoints: RwLock::new(self.entrypoints.read().unwrap().clone()),
             outbound_budget: self.outbound_budget.clone_non_atomic(),
             my_contact_info: RwLock::new(my_contact_info),
             ping_cache: RwLock::new(self.ping_cache.read().unwrap().mock_clone()),
@@ -592,7 +598,9 @@ impl ClusterInfo {
                     .clone(),
             ),
             contact_debug_interval: self.contact_debug_interval,
-            instance: NodeInstance::new(&mut rand::thread_rng(), *new_id, timestamp()),
+            instance: NodeInstance::new(&mut thread_rng(), *new_id, timestamp()),
+            contact_info_path: PathBuf::default(),
+            contact_save_interval: 0, // disabled
         }
     }
 
@@ -644,7 +652,122 @@ impl ClusterInfo {
     }
 
     pub fn set_entrypoint(&self, entrypoint: ContactInfo) {
-        *self.entrypoint.write().unwrap() = Some(entrypoint)
+        self.set_entrypoints(vec![entrypoint]);
+    }
+
+    pub fn set_entrypoints(&self, entrypoints: Vec<ContactInfo>) {
+        *self.entrypoints.write().unwrap() = entrypoints;
+    }
+
+    pub fn save_contact_info(&self) {
+        let nodes = {
+            let gossip = self.gossip.read().unwrap();
+            let entrypoint_gossip_addrs = self
+                .entrypoints
+                .read()
+                .unwrap()
+                .iter()
+                .map(|contact_info| contact_info.gossip)
+                .collect::<HashSet<_>>();
+
+            gossip
+                .crds
+                .get_nodes()
+                .filter_map(|v| {
+                    // Don't save:
+                    // 1. Our ContactInfo. No point
+                    // 2. Entrypoint ContactInfo. This will avoid adopting the incorrect shred
+                    //    version on restart if the entrypoint shred version changes.  Also
+                    //    there's not much point in saving entrypoint ContactInfo since by
+                    //    definition that information is already available
+                    let contact_info = v.value.contact_info().unwrap();
+                    if contact_info.id != self.id()
+                        && !entrypoint_gossip_addrs.contains(&contact_info.gossip)
+                    {
+                        return Some(v.value.clone());
+                    }
+                    None
+                })
+                .collect::<Vec<_>>()
+        };
+
+        if nodes.is_empty() {
+            return;
+        }
+
+        let filename = self.contact_info_path.join("contact-info.bin");
+        let tmp_filename = &filename.with_extension("tmp");
+
+        match File::create(&tmp_filename) {
+            Ok(mut file) => {
+                if let Err(err) = bincode::serialize_into(&mut file, &nodes) {
+                    warn!(
+                        "Failed to serialize contact info info {}: {}",
+                        tmp_filename.display(),
+                        err
+                    );
+                    return;
+                }
+            }
+            Err(err) => {
+                warn!("Failed to create {}: {}", tmp_filename.display(), err);
+                return;
+            }
+        }
+
+        match fs::rename(&tmp_filename, &filename) {
+            Ok(()) => {
+                info!(
+                    "Saved contact info for {} nodes into {}",
+                    nodes.len(),
+                    filename.display()
+                );
+            }
+            Err(err) => {
+                warn!(
+                    "Failed to rename {} to {}: {}",
+                    tmp_filename.display(),
+                    filename.display(),
+                    err
+                );
+            }
+        }
+    }
+
+    pub fn restore_contact_info(&mut self, contact_info_path: &Path, contact_save_interval: u64) {
+        self.contact_info_path = contact_info_path.into();
+        self.contact_save_interval = contact_save_interval;
+
+        let filename = contact_info_path.join("contact-info.bin");
+        if !filename.exists() {
+            return;
+        }
+
+        let nodes: Vec<CrdsValue> = match File::open(&filename) {
+            Ok(file) => {
+                bincode::deserialize_from(&mut BufReader::new(file)).unwrap_or_else(|err| {
+                    warn!("Failed to deserialize {}: {}", filename.display(), err);
+                    vec![]
+                })
+            }
+            Err(err) => {
+                warn!("Failed to open {}: {}", filename.display(), err);
+                vec![]
+            }
+        };
+
+        info!(
+            "Loaded contact info for {} nodes from {}",
+            nodes.len(),
+            filename.display()
+        );
+        let now = timestamp();
+        let mut gossip = self.gossip.write().unwrap();
+        for node in nodes {
+            if let Err(err) = gossip.crds.insert(node, now) {
+                warn!("crds insert failed {:?}", err);
+            }
+        }
     }
 
     pub fn id(&self) -> Pubkey {
@@ -1014,35 +1137,21 @@ impl ClusterInfo {
         let (labels, txs): (Vec<CrdsValueLabel>, Vec<Transaction>) = self
             .time_gossip_read_lock("get_votes", &self.stats.get_votes)
             .crds
-            .iter()
-            .filter(|(_, x)| x.insert_timestamp > since)
-            .filter_map(|(label, x)| {
-                max_ts = std::cmp::max(x.insert_timestamp, max_ts);
-                x.value
-                    .vote()
-                    .map(|v| (label.clone(), v.transaction.clone()))
+            .get_votes()
+            .filter(|vote| vote.insert_timestamp > since)
+            .map(|vote| {
+                max_ts = std::cmp::max(vote.insert_timestamp, max_ts);
+                let transaction = match &vote.value.data {
+                    CrdsData::Vote(_, vote) => vote.transaction.clone(),
+                    _ => panic!("this should not happen!"),
+                };
+                (vote.value.label(), transaction)
             })
             .unzip();
         inc_new_counter_info!("cluster_info-get_votes-count", txs.len());
         (labels, txs, max_ts)
     }
 
-    pub fn get_snapshot_hash(&self, slot: Slot) -> Vec<(Pubkey, Hash)> {
-        self.time_gossip_read_lock("get_snapshot_hash", &self.stats.get_snapshot_hash)
-            .crds
-            .values()
-            .filter_map(|x| x.value.snapshot_hash())
-            .filter_map(|x| {
-                for (table_slot, hash) in &x.hashes {
-                    if *table_slot == slot {
-                        return Some((x.from, *hash));
-                    }
-                }
-                None
-            })
-            .collect()
-    }
-
     pub fn get_accounts_hash_for_node<F, Y>(&self, pubkey: &Pubkey, map: F) -> Option<Y>
     where
         F: FnOnce(&Vec<(Slot, Hash)>) -> Y,
@@ -1500,52 +1609,49 @@ impl ClusterInfo {
         thread_pool: &ThreadPool,
         pulls: &mut Vec<(Pubkey, CrdsFilter, SocketAddr, CrdsValue)>,
     ) {
-        let pull_from_entrypoint = {
-            let mut w_entrypoint = self.entrypoint.write().unwrap();
-            if let Some(ref mut entrypoint) = &mut *w_entrypoint {
+        let entrypoint_id_and_gossip = {
+            let mut entrypoints = self.entrypoints.write().unwrap();
+            if entrypoints.is_empty() {
+                None
+            } else {
+                let i = thread_rng().gen_range(0, entrypoints.len());
+                let entrypoint = &mut entrypoints[i];
+
                 if pulls.is_empty() {
-                    // Nobody else to pull from, try the entrypoint
-                    true
+                    // Nobody else to pull from, try an entrypoint
+                    Some((entrypoint.id, entrypoint.gossip))
                 } else {
                     let now = timestamp();
-                    // Only consider pulling from the entrypoint periodically to avoid spamming it
-                    if timestamp() - entrypoint.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS / 2 {
-                        false
+                    if now - entrypoint.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS / 2 {
+                        None
                     } else {
                         entrypoint.wallclock = now;
-                        let found_entrypoint = self
+                        if self
                             .time_gossip_read_lock("entrypoint", &self.stats.entrypoint)
                             .crds
                             .get_nodes_contact_info()
-                            .any(|node| node.gossip == entrypoint.gossip);
-                        !found_entrypoint
+                            .any(|node| node.gossip == entrypoint.gossip)
+                        {
+                            None // Found the entrypoint, no need to pull from it
+                        } else {
+                            Some((entrypoint.id, entrypoint.gossip))
+                        }
                     }
                 }
-            } else {
-                false
             }
         };
 
-        if pull_from_entrypoint {
-            let id_and_gossip = {
-                self.entrypoint
-                    .read()
-                    .unwrap()
-                    .as_ref()
-                    .map(|e| (e.id, e.gossip))
-            };
-            if let Some((id, gossip)) = id_and_gossip {
-                let r_gossip = self.time_gossip_read_lock("entrypoint", &self.stats.entrypoint2);
-                let self_info = r_gossip
-                    .crds
-                    .lookup(&CrdsValueLabel::ContactInfo(self.id()))
-                    .unwrap_or_else(|| panic!("self_id invalid {}", self.id()));
-                r_gossip
-                    .pull
-                    .build_crds_filters(thread_pool, &r_gossip.crds, MAX_BLOOM_SIZE)
-                    .into_iter()
-                    .for_each(|filter| pulls.push((id, filter, gossip, self_info.clone())));
-            }
+        if let Some((id, gossip)) = entrypoint_id_and_gossip {
+            let r_gossip = self.time_gossip_read_lock("entrypoint", &self.stats.entrypoint2);
+            let self_info = r_gossip
+                .crds
+                .lookup(&CrdsValueLabel::ContactInfo(self.id()))
+                .unwrap_or_else(|| panic!("self_id invalid {}", self.id()));
+            r_gossip
+                .pull
+                .build_crds_filters(thread_pool, &r_gossip.crds, MAX_BLOOM_SIZE)
+                .into_iter()
+                .for_each(|filter| pulls.push((id, filter, gossip, self_info.clone())));
         }
     }
 
@@ -1730,41 +1836,55 @@ impl ClusterInfo {
         Ok(())
     }
 
-    fn handle_adopt_shred_version(self: &Arc<Self>, adopt_shred_version: &mut bool) {
-        // Adopt the entrypoint's `shred_version` if ours is unset
-        if *adopt_shred_version {
-            // If gossip was given an entrypoint, look up the ContactInfo by the given
-            // entrypoint gossip adddress
-            let gossip_addr = self.entrypoint.read().unwrap().as_ref().map(|e| e.gossip);
+    fn process_entrypoints(&self, entrypoints_processed: &mut bool) {
+        if *entrypoints_processed {
+            return;
+        }
 
-            if let Some(gossip_addr) = gossip_addr {
-                // If a pull from the entrypoint was successful, it should exist in the crds table
-                let entrypoint = self.lookup_contact_info_by_gossip_addr(&gossip_addr);
-                if let Some(entrypoint) = entrypoint {
-                    if entrypoint.shred_version == 0 {
-                        info!("Unable to adopt entrypoint's shred version");
-                    } else {
-                        info!(
-                            "Setting shred version to {:?} from entrypoint {:?}",
-                            entrypoint.shred_version, entrypoint.id
-                        );
-                        self.my_contact_info.write().unwrap().shred_version =
-                            entrypoint.shred_version;
-                        self.gossip
-                            .write()
-                            .unwrap()
-                            .set_shred_version(entrypoint.shred_version);
-                        self.insert_self();
-                        *self.entrypoint.write().unwrap() = Some(entrypoint);
-                        *adopt_shred_version = false;
-                    }
+        let mut entrypoints = self.entrypoints.write().unwrap();
+        if entrypoints.is_empty() {
+            // No entrypoint specified.  Nothing more to process
+            *entrypoints_processed = true;
+            return;
+        }
+
+        for entrypoint in entrypoints.iter_mut() {
+            if entrypoint.id == Pubkey::default() {
+                // If a pull from the entrypoint was successful it should exist in the CRDS table
+                if let Some(entrypoint_from_gossip) =
+                    self.lookup_contact_info_by_gossip_addr(&entrypoint.gossip)
+                {
+                    // Update the entrypoint's id so future entrypoint pulls correctly reference it
+                    *entrypoint = entrypoint_from_gossip;
                 }
             }
         }
+        // Adopt an entrypoint's `shred_version` if ours is unset
+        if self.my_shred_version() == 0 {
+            if let Some(entrypoint) = entrypoints
+                .iter()
+                .find(|entrypoint| entrypoint.shred_version != 0)
+            {
+                info!(
+                    "Setting shred version to {:?} from entrypoint {:?}",
+                    entrypoint.shred_version, entrypoint.id
+                );
+                self.my_contact_info.write().unwrap().shred_version = entrypoint.shred_version;
+                self.gossip
+                    .write()
+                    .unwrap()
+                    .set_shred_version(entrypoint.shred_version);
+            }
+        }
+
+        *entrypoints_processed = self.my_shred_version() != 0
+            && entrypoints
+                .iter()
+                .all(|entrypoint| entrypoint.id != Pubkey::default());
     }
 
     fn handle_purge(
-        self: &Arc<Self>,
+        &self,
         thread_pool: &ThreadPool,
         bank_forks: &Option<Arc<RwLock<BankForks>>>,
         stakes: &HashMap<Pubkey, u64>,
@@ -1806,7 +1926,8 @@ impl ClusterInfo {
             .spawn(move || {
                 let mut last_push = timestamp();
                 let mut last_contact_info_trace = timestamp();
-                let mut adopt_shred_version = self.my_shred_version() == 0;
+                let mut last_contact_info_save = timestamp();
+                let mut entrypoints_processed = false;
                 let recycler = PacketsRecycler::default();
                 let crds_data = vec![
                     CrdsData::Version(Version::new(self.id())),
@@ -1823,7 +1944,7 @@ impl ClusterInfo {
                     if self.contact_debug_interval != 0
                         && start - last_contact_info_trace > self.contact_debug_interval
                     {
-                        // Log contact info every 10 seconds
+                        // Log contact info
                         info!(
                             "\n{}\n\n{}",
                             self.contact_info_trace(),
@@ -1832,9 +1953,16 @@ impl ClusterInfo {
                         last_contact_info_trace = start;
                     }
 
+                    if self.contact_save_interval != 0
+                        && start - last_contact_info_save > self.contact_save_interval
+                    {
+                        self.save_contact_info();
+                        last_contact_info_save = start;
+                    }
+
                     let stakes: HashMap<_, _> = match bank_forks {
                         Some(ref bank_forks) => {
-                            staking_utils::staked_nodes(&bank_forks.read().unwrap().working_bank())
+                            bank_forks.read().unwrap().root_bank().staked_nodes()
                         }
                         None => HashMap::new(),
                     };
@@ -1853,7 +1981,7 @@ impl ClusterInfo {
 
                     self.handle_purge(&thread_pool, &bank_forks, &stakes);
 
-                    self.handle_adopt_shred_version(&mut adopt_shred_version);
+                    self.process_entrypoints(&mut entrypoints_processed);
 
                     //TODO: possibly tune this parameter
                     //we saw a deadlock passing an self.read().unwrap().timeout into sleep
@@ -1974,7 +2102,7 @@ impl ClusterInfo {
         }
     }
 
-    fn update_data_budget(&self, num_staked: usize) {
+    fn update_data_budget(&self, num_staked: usize) -> usize {
         const INTERVAL_MS: u64 = 100;
         // allow 50kBps per staked validator, epoch slots + votes ~= 1.5kB/slot ~= 4kB/s
         const BYTES_PER_INTERVAL: usize = 5000;
@@ -1985,7 +2113,7 @@ impl ClusterInfo {
                 bytes + num_staked * BYTES_PER_INTERVAL,
                 MAX_BUDGET_MULTIPLE * num_staked * BYTES_PER_INTERVAL,
             )
-        });
+        })
     }
 
     // Returns a predicate checking if the pull request is from a valid
@@ -2046,7 +2174,8 @@ impl ClusterInfo {
         let callers = crds_value::filter_current(requests.iter().map(|r| &r.caller));
         self.time_gossip_write_lock("process_pull_reqs", &self.stats.process_pull_requests)
             .process_pull_requests(callers.cloned(), timestamp());
-        self.update_data_budget(stakes.len());
+        let output_size_limit =
+            self.update_data_budget(stakes.len()) / PULL_RESPONSE_MIN_SERIALIZED_SIZE;
         let mut packets = Packets::new_with_recycler(recycler.clone(), 64, "handle_pull_requests");
         let (caller_and_filters, addrs): (Vec<_>, Vec<_>) = {
             let mut rng = rand::thread_rng();
@@ -2066,7 +2195,7 @@ impl ClusterInfo {
                 "generate_pull_responses",
                 &self.stats.generate_pull_responses,
             )
-            .generate_pull_responses(&caller_and_filters, now);
+            .generate_pull_responses(&caller_and_filters, output_size_limit, now);
 
         let pull_responses: Vec<_> = pull_responses
             .into_iter()
@@ -2484,24 +2613,24 @@ impl ClusterInfo {
 
     fn get_stakes_and_epoch_time(
         bank_forks: Option<&Arc<RwLock<BankForks>>>,
-    ) -> (HashMap<Pubkey, u64>, u64) {
-        let epoch_time_ms;
-        let stakes: HashMap<_, _> = match bank_forks {
+    ) -> (
+        HashMap<Pubkey, u64>, // staked nodes
+        u64,                  // epoch time ms
+    ) {
+        match bank_forks {
             Some(ref bank_forks) => {
-                let bank = bank_forks.read().unwrap().working_bank();
+                let bank = bank_forks.read().unwrap().root_bank();
                 let epoch = bank.epoch();
-                let epoch_schedule = bank.epoch_schedule();
-                epoch_time_ms = epoch_schedule.get_slots_in_epoch(epoch) * DEFAULT_MS_PER_SLOT;
-                staking_utils::staked_nodes(&bank)
+                (
+                    bank.staked_nodes(),
+                    bank.get_slots_in_epoch(epoch) * DEFAULT_MS_PER_SLOT,
+                )
             }
             None => {
                 inc_new_counter_info!("cluster_info-purge-no_working_bank", 1);
-                epoch_time_ms = CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
-                HashMap::new()
+                (HashMap::new(), CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS)
             }
-        };
-
-        (stakes, epoch_time_ms)
+        }
     }
 
     fn process_packets(
@@ -2650,11 +2779,6 @@ impl ClusterInfo {
                 ("entrypoint", self.stats.entrypoint.clear(), i64),
                 ("entrypoint2", self.stats.entrypoint2.clear(), i64),
                 ("push_vote_read", self.stats.push_vote_read.clear(), i64),
-                (
-                    "vote_process_push",
-                    self.stats.vote_process_push.clear(),
-                    i64
-                ),
                 ("get_votes", self.stats.get_votes.clear(), i64),
                 (
                     "get_accounts_hash",
@@ -2806,8 +2930,6 @@ impl ClusterInfo {
                     self.stats.epoch_slots_lookup.clear(),
                     i64
                 ),
-                ("epoch_slots_push", self.stats.epoch_slots_push.clear(), i64),
-                ("push_message", self.stats.push_message.clear(), i64),
                 (
                     "new_pull_requests",
                     self.stats.new_pull_requests.clear(),
@@ -3467,6 +3589,17 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_pull_response_min_serialized_size() {
+        let mut rng = rand::thread_rng();
+        for _ in 0..100 {
+            let crds_values = vec![CrdsValue::new_rand(&mut rng, None)];
+            let pull_response = Protocol::PullResponse(Pubkey::new_unique(), crds_values);
+            let size = serialized_size(&pull_response).unwrap();
+            assert!(PULL_RESPONSE_MIN_SERIALIZED_SIZE as u64 <= size);
+        }
+    }
+
     #[test]
     fn test_cluster_spy_gossip() {
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
@@ -3898,7 +4031,7 @@ mod tests {
         );
         let pulls = cluster_info.new_pull_requests(&thread_pool, None, &HashMap::new());
         assert_eq!(1, pulls.len() as u64);
-        assert_eq!(*cluster_info.entrypoint.read().unwrap(), Some(entrypoint));
+        assert_eq!(*cluster_info.entrypoints.read().unwrap(), vec![entrypoint]);
     }
 
     #[test]
@@ -4084,13 +4217,7 @@ mod tests {
 
         // Pull request 2: pretend it's been a while since we've pulled from `entrypoint`.  There should
         // now be two pull requests
-        cluster_info
-            .entrypoint
-            .write()
-            .unwrap()
-            .as_mut()
-            .unwrap()
-            .wallclock = 0;
+        cluster_info.entrypoints.write().unwrap()[0].wallclock = 0;
         let pulls = cluster_info.new_pull_requests(&thread_pool, None, &stakes);
         assert_eq!(2, pulls.len() as u64);
         assert_eq!(pulls.get(0).unwrap().0, other_node.gossip);
@@ -4212,12 +4339,98 @@ mod tests {
     }
 
     #[test]
-    fn test_handle_adopt_shred_version() {
+    fn test_process_entrypoint_adopt_shred_version() {
         let node_keypair = Arc::new(Keypair::new());
         let cluster_info = Arc::new(ClusterInfo::new(
             ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
             node_keypair,
         ));
+        assert_eq!(cluster_info.my_shred_version(), 0);
+
+        // Simulating starting up with two entrypoints, no known id, only a gossip
+        // address
+        let entrypoint1_gossip_addr = socketaddr!("127.0.0.2:1234");
+        let mut entrypoint1 = ContactInfo::new_localhost(&Pubkey::default(), timestamp());
+        entrypoint1.gossip = entrypoint1_gossip_addr;
+        assert_eq!(entrypoint1.shred_version, 0);
+
+        let entrypoint2_gossip_addr = socketaddr!("127.0.0.2:5678");
+        let mut entrypoint2 = ContactInfo::new_localhost(&Pubkey::default(), timestamp());
+        entrypoint2.gossip = entrypoint2_gossip_addr;
+        assert_eq!(entrypoint2.shred_version, 0);
+        cluster_info.set_entrypoints(vec![entrypoint1, entrypoint2]);
+
+        // Simulate getting entrypoint ContactInfo from gossip with an entrypoint1 shred version of
+        // 0
+        let mut gossiped_entrypoint1_info =
+            ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), timestamp());
+        gossiped_entrypoint1_info.gossip = entrypoint1_gossip_addr;
+        gossiped_entrypoint1_info.shred_version = 0;
+        cluster_info.insert_info(gossiped_entrypoint1_info.clone());
+        assert!(!cluster_info
+            .entrypoints
+            .read()
+            .unwrap()
+            .iter()
+            .any(|entrypoint| *entrypoint == gossiped_entrypoint1_info));
+
+        // Adopt the entrypoint's gossiped contact info and verify
+        let mut entrypoints_processed = false;
+        ClusterInfo::process_entrypoints(&cluster_info, &mut entrypoints_processed);
+        assert_eq!(cluster_info.entrypoints.read().unwrap().len(), 2);
+        assert!(cluster_info
+            .entrypoints
+            .read()
+            .unwrap()
+            .iter()
+            .any(|entrypoint| *entrypoint == gossiped_entrypoint1_info));
+
+        assert!(!entrypoints_processed); // <--- entrypoint processing incomplete because shred adoption still pending
+        assert_eq!(cluster_info.my_shred_version(), 0); // <-- shred version still 0
+
+        // Simulate getting entrypoint ContactInfo from gossip with an entrypoint2 shred version of
+        // !0
+        let mut gossiped_entrypoint2_info =
+            ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), timestamp());
+        gossiped_entrypoint2_info.gossip = entrypoint2_gossip_addr;
+        gossiped_entrypoint2_info.shred_version = 1;
+        cluster_info.insert_info(gossiped_entrypoint2_info.clone());
+        assert!(!cluster_info
+            .entrypoints
+            .read()
+            .unwrap()
+            .iter()
+            .any(|entrypoint| *entrypoint == gossiped_entrypoint2_info));
+
+        // Adopt the entrypoint's gossiped contact info and verify
+        error!("Adopt the entrypoint's gossiped contact info and verify");
+        let mut entrypoints_processed = false;
+        ClusterInfo::process_entrypoints(&cluster_info, &mut entrypoints_processed);
+        assert_eq!(cluster_info.entrypoints.read().unwrap().len(), 2);
+        assert!(cluster_info
+            .entrypoints
+            .read()
+            .unwrap()
+            .iter()
+            .any(|entrypoint| *entrypoint == gossiped_entrypoint2_info));
+
+        assert!(entrypoints_processed);
+        assert_eq!(cluster_info.my_shred_version(), 1); // <-- shred version now adopted from entrypoint2
+    }
+
+    #[test]
+    fn test_process_entrypoint_without_adopt_shred_version() {
+        let node_keypair = Arc::new(Keypair::new());
+        let cluster_info = Arc::new(ClusterInfo::new(
+            {
+                let mut contact_info =
+                    ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp());
+                contact_info.shred_version = 2;
+                contact_info
+            },
+            node_keypair,
+        ));
+        assert_eq!(cluster_info.my_shred_version(), 2);
 
         // Simulating starting up with default entrypoint, no known id, only a gossip
         // address
@@ -4235,11 +4448,14 @@ mod tests {
         cluster_info.insert_info(gossiped_entrypoint_info.clone());
 
         // Adopt the entrypoint's gossiped contact info and verify
-        ClusterInfo::handle_adopt_shred_version(&cluster_info, &mut true);
+        let mut entrypoints_processed = false;
+        ClusterInfo::process_entrypoints(&cluster_info, &mut entrypoints_processed);
+        assert_eq!(cluster_info.entrypoints.read().unwrap().len(), 1);
         assert_eq!(
-            cluster_info.entrypoint.read().unwrap().as_ref().unwrap(),
-            &gossiped_entrypoint_info
+            cluster_info.entrypoints.read().unwrap()[0],
+            gossiped_entrypoint_info
         );
-        assert_eq!(cluster_info.my_shred_version(), 1);
+        assert!(entrypoints_processed);
+        assert_eq!(cluster_info.my_shred_version(), 2); // <--- No change to shred version
     }
 }

core/src/cluster_slots.rs

@@ -85,7 +85,7 @@ impl ClusterSlots {
     }
 
     fn update_peers(&self, cluster_info: &ClusterInfo, bank_forks: &RwLock<BankForks>) {
-        let root_bank = bank_forks.read().unwrap().root_bank().clone();
+        let root_bank = bank_forks.read().unwrap().root_bank();
         let root_epoch = root_bank.epoch();
         let my_epoch = *self.epoch.read().unwrap();
 

core/src/commitment_service.rs

@@ -434,26 +434,26 @@ mod tests {
         let mut vote_state1 = VoteState::from(&vote_account1).unwrap();
         vote_state1.process_slot_vote_unchecked(3);
         vote_state1.process_slot_vote_unchecked(5);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state1));
+        let versioned = VoteStateVersions::new_current(vote_state1);
         VoteState::to(&versioned, &mut vote_account1).unwrap();
         bank.store_account(&pk1, &vote_account1);
 
         let mut vote_state2 = VoteState::from(&vote_account2).unwrap();
         vote_state2.process_slot_vote_unchecked(9);
         vote_state2.process_slot_vote_unchecked(10);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state2));
+        let versioned = VoteStateVersions::new_current(vote_state2);
         VoteState::to(&versioned, &mut vote_account2).unwrap();
         bank.store_account(&pk2, &vote_account2);
 
         let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
         vote_state3.root_slot = Some(1);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
+        let versioned = VoteStateVersions::new_current(vote_state3);
         VoteState::to(&versioned, &mut vote_account3).unwrap();
         bank.store_account(&pk3, &vote_account3);
 
         let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
         vote_state4.root_slot = Some(2);
-        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
+        let versioned = VoteStateVersions::new_current(vote_state4);
         VoteState::to(&versioned, &mut vote_account4).unwrap();
         bank.store_account(&pk4, &vote_account4);
 

core/src/consensus.rs

@@ -26,7 +26,10 @@ use std::{
     collections::{HashMap, HashSet},
     fs::{self, File},
     io::BufReader,
-    ops::Bound::{Included, Unbounded},
+    ops::{
+        Bound::{Included, Unbounded},
+        Deref,
+    },
     path::{Path, PathBuf},
     sync::Arc,
 };
@@ -183,7 +186,7 @@ impl Tower {
         let root_bank = bank_forks.root_bank();
         let (_progress, heaviest_subtree_fork_choice) =
             crate::replay_stage::ReplayStage::initialize_progress_and_fork_choice(
-                root_bank,
+                root_bank.deref(),
                 bank_forks.frozen_banks().values().cloned().collect(),
                 &my_pubkey,
                 &vote_account,
@@ -216,6 +219,7 @@ impl Tower {
     where
         F: IntoIterator<Item = (Pubkey, (u64, ArcVoteAccount))>,
     {
+        let mut vote_slots = HashSet::new();
         let mut voted_stakes = HashMap::new();
         let mut total_stake = 0;
         let mut bank_weight = 0;
@@ -278,7 +282,7 @@ impl Tower {
 
             for vote in &vote_state.votes {
                 bank_weight += vote.lockout() as u128 * voted_stake as u128;
-                Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
+                vote_slots.insert(vote.slot);
             }
 
             if start_root != vote_state.root_slot {
@@ -289,7 +293,7 @@ impl Tower {
                     };
                     trace!("ROOT: {}", vote.slot);
                     bank_weight += vote.lockout() as u128 * voted_stake as u128;
-                    Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
+                    vote_slots.insert(vote.slot);
                 }
             }
             if let Some(root) = vote_state.root_slot {
@@ -298,7 +302,7 @@ impl Tower {
                     slot: root,
                 };
                 bank_weight += vote.lockout() as u128 * voted_stake as u128;
-                Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
+                vote_slots.insert(vote.slot);
             }
 
             // The last vote in the vote stack is a simulated vote on bank_slot, which
@@ -326,6 +330,9 @@ impl Tower {
             total_stake += voted_stake;
         }
 
+        // TODO: populate_ancestor_voted_stakes only adds zeros. Comment why
+        // that is necessary (if so).
+        Self::populate_ancestor_voted_stakes(&mut voted_stakes, vote_slots, ancestors);
         ComputedBankState {
             voted_stakes,
             total_stake,
@@ -766,20 +773,19 @@ impl Tower {
     /// Update lockouts for all the ancestors
     pub(crate) fn populate_ancestor_voted_stakes(
         voted_stakes: &mut VotedStakes,
-        vote: &Lockout,
+        vote_slots: impl IntoIterator<Item = Slot>,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
     ) {
         // If there's no ancestors, that means this slot must be from before the current root,
         // in which case the lockouts won't be calculated in bank_weight anyways, so ignore
         // this slot
-        let vote_slot_ancestors = ancestors.get(&vote.slot);
-        if vote_slot_ancestors.is_none() {
-            return;
-        }
-        let mut slot_with_ancestors = vec![vote.slot];
-        slot_with_ancestors.extend(vote_slot_ancestors.unwrap());
-        for slot in slot_with_ancestors {
-            voted_stakes.entry(slot).or_default();
+        for vote_slot in vote_slots {
+            if let Some(slot_ancestors) = ancestors.get(&vote_slot) {
+                voted_stakes.entry(vote_slot).or_default();
+                for slot in slot_ancestors {
+                    voted_stakes.entry(*slot).or_default();
+                }
+            }
         }
     }
 
@@ -793,15 +799,11 @@ impl Tower {
     ) {
         // If there's no ancestors, that means this slot must be from
         // before the current root, so ignore this slot
-        let vote_slot_ancestors = ancestors.get(&voted_slot);
-        if vote_slot_ancestors.is_none() {
-            return;
-        }
-        let mut slot_with_ancestors = vec![voted_slot];
-        slot_with_ancestors.extend(vote_slot_ancestors.unwrap());
-        for slot in slot_with_ancestors {
-            let current = voted_stakes.entry(slot).or_default();
-            *current += voted_stake;
+        if let Some(vote_slot_ancestors) = ancestors.get(&voted_slot) {
+            *voted_stakes.entry(voted_slot).or_default() += voted_stake;
+            for slot in vote_slot_ancestors {
+                *voted_stakes.entry(*slot).or_default() += voted_stake;
+            }
         }
     }
 
@@ -1584,7 +1586,7 @@ pub mod test {
                 vote_state.process_slot_vote_unchecked(*slot);
             }
             VoteState::serialize(
-                &VoteStateVersions::Current(Box::new(vote_state)),
+                &VoteStateVersions::new_current(vote_state),
                 &mut account.data,
             )
             .expect("serialize state");

core/src/crds.rs

@@ -28,7 +28,7 @@ use crate::contact_info::ContactInfo;
 use crate::crds_shards::CrdsShards;
 use crate::crds_value::{CrdsData, CrdsValue, CrdsValueLabel, LowestSlot};
 use bincode::serialize;
-use indexmap::map::{rayon::ParValues, Entry, IndexMap, Iter, Values};
+use indexmap::map::{rayon::ParValues, Entry, IndexMap, Values};
 use indexmap::set::IndexSet;
 use rayon::{prelude::*, ThreadPool};
 use solana_sdk::hash::{hash, Hash};
@@ -47,8 +47,8 @@ pub struct Crds {
     table: IndexMap<CrdsValueLabel, VersionedCrdsValue>,
     pub num_inserts: usize, // Only used in tests.
     shards: CrdsShards,
-    // Indices of all crds values which are node ContactInfo.
-    nodes: IndexSet<usize>,
+    nodes: IndexSet<usize>, // Indices of nodes' ContactInfo.
+    votes: IndexSet<usize>, // Indices of Vote crds values.
     // Indices of all crds values associated with a node.
     records: HashMap<Pubkey, IndexSet<usize>>,
 }
@@ -109,6 +109,7 @@ impl Default for Crds {
             num_inserts: 0,
             shards: CrdsShards::new(CRDS_SHARDS_BITS),
             nodes: IndexSet::default(),
+            votes: IndexSet::default(),
             records: HashMap::default(),
         }
     }
@@ -141,9 +142,15 @@ impl Crds {
             Entry::Vacant(entry) => {
                 let entry_index = entry.index();
                 self.shards.insert(entry_index, &new_value);
-                if let CrdsData::ContactInfo(_) = new_value.value.data {
-                    self.nodes.insert(entry_index);
-                }
+                match new_value.value.data {
+                    CrdsData::ContactInfo(_) => {
+                        self.nodes.insert(entry_index);
+                    }
+                    CrdsData::Vote(_, _) => {
+                        self.votes.insert(entry_index);
+                    }
+                    _ => (),
+                };
                 self.records
                     .entry(new_value.value.pubkey())
                     .or_default()
@@ -215,6 +222,11 @@ impl Crds {
         })
     }
 
+    /// Returns all entries which are Vote.
+    pub(crate) fn get_votes(&self) -> impl Iterator<Item = &VersionedCrdsValue> {
+        self.votes.iter().map(move |i| self.table.index(*i))
+    }
+
     pub fn len(&self) -> usize {
         self.table.len()
     }
@@ -223,10 +235,6 @@ impl Crds {
         self.table.is_empty()
     }
 
-    pub fn iter(&self) -> Iter<'_, CrdsValueLabel, VersionedCrdsValue> {
-        self.table.iter()
-    }
-
     pub fn values(&self) -> Values<'_, CrdsValueLabel, VersionedCrdsValue> {
         self.table.values()
     }
@@ -267,6 +275,7 @@ impl Crds {
         now: u64,
         timeouts: &HashMap<Pubkey, u64>,
     ) -> Vec<CrdsValueLabel> {
+        // TODO: need custom logic for purging duplicate shreds.
         let default_timeout = *timeouts
             .get(&Pubkey::default())
             .expect("must have default timeout");
@@ -289,8 +298,14 @@ impl Crds {
     pub fn remove(&mut self, key: &CrdsValueLabel) -> Option<VersionedCrdsValue> {
         let (index, _ /*label*/, value) = self.table.swap_remove_full(key)?;
         self.shards.remove(index, &value);
-        if let CrdsData::ContactInfo(_) = value.value.data {
-            self.nodes.swap_remove(&index);
+        match value.value.data {
+            CrdsData::ContactInfo(_) => {
+                self.nodes.swap_remove(&index);
+            }
+            CrdsData::Vote(_, _) => {
+                self.votes.swap_remove(&index);
+            }
+            _ => (),
         }
         // Remove the index from records associated with the value's pubkey.
         let pubkey = value.value.pubkey();
@@ -312,10 +327,17 @@ impl Crds {
             let value = self.table.index(index);
             self.shards.remove(size, value);
             self.shards.insert(index, value);
-            if let CrdsData::ContactInfo(_) = value.value.data {
-                self.nodes.swap_remove(&size);
-                self.nodes.insert(index);
-            }
+            match value.value.data {
+                CrdsData::ContactInfo(_) => {
+                    self.nodes.swap_remove(&size);
+                    self.nodes.insert(index);
+                }
+                CrdsData::Vote(_, _) => {
+                    self.votes.swap_remove(&size);
+                    self.votes.insert(index);
+                }
+                _ => (),
+            };
             let pubkey = value.value.pubkey();
             let records = self.records.get_mut(&pubkey).unwrap();
             records.swap_remove(&size);
@@ -536,51 +558,67 @@ mod test {
     }
 
     #[test]
-    fn test_crds_nodes() {
-        fn check_crds_nodes(crds: &Crds) -> usize {
+    fn test_crds_value_indices() {
+        fn check_crds_value_indices(crds: &Crds) -> (usize, usize) {
             let num_nodes = crds
                 .table
                 .values()
                 .filter(|value| matches!(value.value.data, CrdsData::ContactInfo(_)))
                 .count();
+            let num_votes = crds
+                .table
+                .values()
+                .filter(|value| matches!(value.value.data, CrdsData::Vote(_, _)))
+                .count();
             assert_eq!(num_nodes, crds.get_nodes_contact_info().count());
-            num_nodes
+            assert_eq!(num_votes, crds.get_votes().count());
+            for vote in crds.get_votes() {
+                match vote.value.data {
+                    CrdsData::Vote(_, _) => (),
+                    _ => panic!("not a vote!"),
+                }
+            }
+            (num_nodes, num_votes)
         }
         let mut rng = thread_rng();
-        let keypairs: Vec<_> = std::iter::repeat_with(Keypair::new).take(256).collect();
+        let keypairs: Vec<_> = repeat_with(Keypair::new).take(128).collect();
         let mut crds = Crds::default();
         let mut num_inserts = 0;
         let mut num_overrides = 0;
-        for _ in 0..4096 {
+        for k in 0..4096 {
             let keypair = &keypairs[rng.gen_range(0, keypairs.len())];
             let value = VersionedCrdsValue::new_rand(&mut rng, Some(keypair));
             match crds.insert_versioned(value) {
                 Ok(None) => {
                     num_inserts += 1;
-                    check_crds_nodes(&crds);
                 }
                 Ok(Some(_)) => {
                     num_inserts += 1;
                     num_overrides += 1;
-                    check_crds_nodes(&crds);
                 }
                 Err(_) => (),
             }
+            if k % 64 == 0 {
+                check_crds_value_indices(&crds);
+            }
         }
         assert_eq!(num_inserts, crds.num_inserts);
         assert!(num_inserts > 700);
         assert!(num_overrides > 500);
         assert!(crds.table.len() > 200);
         assert!(num_inserts > crds.table.len());
-        let num_nodes = check_crds_nodes(&crds);
+        let (num_nodes, num_votes) = check_crds_value_indices(&crds);
         assert!(num_nodes * 3 < crds.table.len());
-        assert!(num_nodes > 150);
+        assert!(num_nodes > 100, "num nodes: {}", num_nodes);
+        assert!(num_votes > 100, "num votes: {}", num_votes);
         // Remove values one by one and assert that nodes indices stay valid.
         while !crds.table.is_empty() {
             let index = rng.gen_range(0, crds.table.len());
             let key = crds.table.get_index(index).unwrap().0.clone();
             crds.remove(&key);
-            check_crds_nodes(&crds);
+            if crds.table.len() % 64 == 0 {
+                check_crds_value_indices(&crds);
+            }
         }
     }
 

core/src/crds_gossip.rs

@@ -184,9 +184,11 @@ impl CrdsGossip {
     pub fn generate_pull_responses(
         &self,
         filters: &[(CrdsValue, CrdsFilter)],
+        output_size_limit: usize, // Limit number of crds values returned.
         now: u64,
     ) -> Vec<Vec<CrdsValue>> {
-        self.pull.generate_pull_responses(&self.crds, filters, now)
+        self.pull
+            .generate_pull_responses(&self.crds, filters, output_size_limit, now)
     }
 
     pub fn filter_pull_responses(

core/src/crds_gossip_pull.rs

@@ -14,6 +14,7 @@ use crate::crds::{Crds, VersionedCrdsValue};
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
+use itertools::Itertools;
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::Rng;
 use rayon::{prelude::*, ThreadPool};
@@ -304,9 +305,10 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         requests: &[(CrdsValue, CrdsFilter)],
+        output_size_limit: usize, // Limit number of crds values returned.
         now: u64,
     ) -> Vec<Vec<CrdsValue>> {
-        self.filter_crds_values(crds, requests, now)
+        self.filter_crds_values(crds, requests, output_size_limit, now)
     }
 
     // Checks if responses should be inserted and
@@ -474,6 +476,7 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         filters: &[(CrdsValue, CrdsFilter)],
+        mut output_size_limit: usize, // Limit number of crds values returned.
         now: u64,
     ) -> Vec<Vec<CrdsValue>> {
         let msg_timeout = CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
@@ -483,16 +486,20 @@ impl CrdsGossipPull {
         let past = now.saturating_sub(msg_timeout);
         let mut dropped_requests = 0;
         let mut total_skipped = 0;
-        let ret = filters
+        let ret: Vec<_> = filters
             .iter()
             .map(|(caller, filter)| {
+                if output_size_limit == 0 {
+                    return None;
+                }
                 let caller_wallclock = caller.wallclock();
                 if caller_wallclock >= future || caller_wallclock < past {
                     dropped_requests += 1;
-                    return vec![];
+                    return Some(vec![]);
                 }
                 let caller_wallclock = caller_wallclock.checked_add(jitter).unwrap_or(0);
-                crds.filter_bitmask(filter.mask, filter.mask_bits)
+                let out: Vec<_> = crds
+                    .filter_bitmask(filter.mask, filter.mask_bits)
                     .filter_map(|item| {
                         debug_assert!(filter.test_mask(&item.value_hash));
                         //skip values that are too new
@@ -505,12 +512,16 @@ impl CrdsGossipPull {
                             Some(item.value.clone())
                         }
                     })
-                    .collect()
+                    .take(output_size_limit)
+                    .collect();
+                output_size_limit -= out.len();
+                Some(out)
             })
+            .while_some()
             .collect();
         inc_new_counter_info!(
             "gossip_filter_crds_values-dropped_requests",
-            dropped_requests
+            dropped_requests + filters.len() - ret.len()
         );
         inc_new_counter_info!("gossip_filter_crds_values-dropped_values", total_skipped);
         ret
@@ -1029,7 +1040,12 @@ mod test {
         let dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
         let mut filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            0,
+        );
 
         assert_eq!(rsp[0].len(), 0);
 
@@ -1042,8 +1058,12 @@ mod test {
             .unwrap();
 
         //should skip new value since caller is to old
-        let rsp =
-            dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS,
+        );
         assert_eq!(rsp[0].len(), 0);
 
         assert_eq!(filters.len(), 1);
@@ -1054,8 +1074,12 @@ mod test {
             CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS + 1,
         )));
 
-        let rsp =
-            dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS,
+        );
         assert_eq!(rsp.len(), 2);
         assert_eq!(rsp[0].len(), 0);
         assert_eq!(rsp[1].len(), 1); // Orders are also preserved.
@@ -1092,7 +1116,12 @@ mod test {
         let mut dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
         let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
+        let rsp = dest.generate_pull_responses(
+            &dest_crds,
+            &filters,
+            /*output_size_limit=*/ usize::MAX,
+            0,
+        );
         dest.process_pull_requests(
             &mut dest_crds,
             filters.into_iter().map(|(caller, _)| caller),
@@ -1170,7 +1199,12 @@ mod test {
             );
             let (_, filters, caller) = req.unwrap();
             let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-            let mut rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
+            let mut rsp = dest.generate_pull_responses(
+                &dest_crds,
+                &filters,
+                /*output_size_limit=*/ usize::MAX,
+                0,
+            );
             dest.process_pull_requests(
                 &mut dest_crds,
                 filters.into_iter().map(|(caller, _)| caller),

core/src/crds_value.rs

@@ -1,7 +1,10 @@
-use crate::cluster_info::MAX_SNAPSHOT_HASHES;
-use crate::contact_info::ContactInfo;
-use crate::deprecated;
-use crate::epoch_slots::EpochSlots;
+use crate::{
+    cluster_info::MAX_SNAPSHOT_HASHES,
+    contact_info::ContactInfo,
+    deprecated,
+    duplicate_shred::{DuplicateShred, DuplicateShredIndex},
+    epoch_slots::EpochSlots,
+};
 use bincode::{serialize, serialized_size};
 use rand::{CryptoRng, Rng};
 use solana_sdk::sanitize::{Sanitize, SanitizeError};
@@ -80,6 +83,7 @@ pub enum CrdsData {
     LegacyVersion(LegacyVersion),
     Version(Version),
     NodeInstance(NodeInstance),
+    DuplicateShred(DuplicateShred),
 }
 
 impl Sanitize for CrdsData {
@@ -109,6 +113,7 @@ impl Sanitize for CrdsData {
             CrdsData::LegacyVersion(version) => version.sanitize(),
             CrdsData::Version(version) => version.sanitize(),
             CrdsData::NodeInstance(node) => node.sanitize(),
+            CrdsData::DuplicateShred(shred) => shred.sanitize(),
         }
     }
 }
@@ -122,7 +127,7 @@ pub(crate) fn new_rand_timestamp<R: Rng>(rng: &mut R) -> u64 {
 impl CrdsData {
     /// New random CrdsData for tests and benchmarks.
     fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> CrdsData {
-        let kind = rng.gen_range(0, 5);
+        let kind = rng.gen_range(0, 6);
         // TODO: Implement other kinds of CrdsData here.
         // TODO: Assign ranges to each arm proportional to their frequency in
         // the mainnet crds table.
@@ -131,7 +136,8 @@ impl CrdsData {
             1 => CrdsData::LowestSlot(rng.gen(), LowestSlot::new_rand(rng, pubkey)),
             2 => CrdsData::SnapshotHashes(SnapshotHash::new_rand(rng, pubkey)),
             3 => CrdsData::AccountsHashes(SnapshotHash::new_rand(rng, pubkey)),
-            _ => CrdsData::Version(Version::new_rand(rng, pubkey)),
+            4 => CrdsData::Version(Version::new_rand(rng, pubkey)),
+            _ => CrdsData::Vote(rng.gen_range(0, MAX_VOTES), Vote::new_rand(rng, pubkey)),
         }
     }
 }
@@ -145,9 +151,7 @@ pub struct SnapshotHash {
 
 impl Sanitize for SnapshotHash {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         for (slot, _) in &self.hashes {
             if *slot >= MAX_SLOT {
                 return Err(SanitizeError::ValueOutOfBounds);
@@ -220,9 +224,7 @@ impl LowestSlot {
 
 impl Sanitize for LowestSlot {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         if self.lowest >= MAX_SLOT {
             return Err(SanitizeError::ValueOutOfBounds);
         }
@@ -248,9 +250,7 @@ pub struct Vote {
 
 impl Sanitize for Vote {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()?;
         self.transaction.sanitize()
     }
@@ -264,6 +264,15 @@ impl Vote {
             wallclock,
         }
     }
+
+    /// New random Vote for tests and benchmarks.
+    fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
+        Self {
+            from: pubkey.unwrap_or_else(pubkey::new_rand),
+            transaction: Transaction::default(),
+            wallclock: new_rand_timestamp(rng),
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, AbiExample)]
@@ -275,9 +284,7 @@ pub struct LegacyVersion {
 
 impl Sanitize for LegacyVersion {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()?;
         self.version.sanitize()
     }
@@ -292,9 +299,7 @@ pub struct Version {
 
 impl Sanitize for Version {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()?;
         self.version.sanitize()
     }
@@ -370,9 +375,7 @@ impl NodeInstance {
 
 impl Sanitize for NodeInstance {
     fn sanitize(&self) -> Result<(), SanitizeError> {
-        if self.wallclock >= MAX_WALLCLOCK {
-            return Err(SanitizeError::ValueOutOfBounds);
-        }
+        sanitize_wallclock(self.wallclock)?;
         self.from.sanitize()
     }
 }
@@ -390,6 +393,7 @@ pub enum CrdsValueLabel {
     LegacyVersion(Pubkey),
     Version(Pubkey),
     NodeInstance(Pubkey, u64 /*token*/),
+    DuplicateShred(DuplicateShredIndex, Pubkey),
 }
 
 impl fmt::Display for CrdsValueLabel {
@@ -403,7 +407,8 @@ impl fmt::Display for CrdsValueLabel {
             CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
             CrdsValueLabel::LegacyVersion(_) => write!(f, "LegacyVersion({})", self.pubkey()),
             CrdsValueLabel::Version(_) => write!(f, "Version({})", self.pubkey()),
-            CrdsValueLabel::NodeInstance(_, _) => write!(f, "NodeInstance({})", self.pubkey()),
+            CrdsValueLabel::NodeInstance(pk, token) => write!(f, "NodeInstance({}, {})", pk, token),
+            CrdsValueLabel::DuplicateShred(ix, pk) => write!(f, "DuplicateShred({:?}, {})", ix, pk),
         }
     }
 }
@@ -420,6 +425,7 @@ impl CrdsValueLabel {
             CrdsValueLabel::LegacyVersion(p) => *p,
             CrdsValueLabel::Version(p) => *p,
             CrdsValueLabel::NodeInstance(p, _ /*token*/) => *p,
+            CrdsValueLabel::DuplicateShred(_, p) => *p,
         }
     }
 }
@@ -467,6 +473,7 @@ impl CrdsValue {
             CrdsData::LegacyVersion(version) => version.wallclock,
             CrdsData::Version(version) => version.wallclock,
             CrdsData::NodeInstance(node) => node.wallclock,
+            CrdsData::DuplicateShred(shred) => shred.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
@@ -480,6 +487,7 @@ impl CrdsValue {
             CrdsData::LegacyVersion(version) => version.from,
             CrdsData::Version(version) => version.from,
             CrdsData::NodeInstance(node) => node.from,
+            CrdsData::DuplicateShred(shred) => shred.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
@@ -492,7 +500,10 @@ impl CrdsValue {
             CrdsData::EpochSlots(ix, _) => CrdsValueLabel::EpochSlots(*ix, self.pubkey()),
             CrdsData::LegacyVersion(_) => CrdsValueLabel::LegacyVersion(self.pubkey()),
             CrdsData::Version(_) => CrdsValueLabel::Version(self.pubkey()),
-            CrdsData::NodeInstance(node) => CrdsValueLabel::NodeInstance(self.pubkey(), node.token),
+            CrdsData::NodeInstance(node) => CrdsValueLabel::NodeInstance(node.from, node.token),
+            CrdsData::DuplicateShred(shred) => {
+                CrdsValueLabel::DuplicateShred(DuplicateShredIndex::from(shred), shred.from)
+            }
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -623,6 +634,14 @@ where
     out.into_iter().map(|(_, (v, _))| v)
 }
 
+pub(crate) fn sanitize_wallclock(wallclock: u64) -> Result<(), SanitizeError> {
+    if wallclock >= MAX_WALLCLOCK {
+        Err(SanitizeError::ValueOutOfBounds)
+    } else {
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -812,7 +831,7 @@ mod test {
             let index = rng.gen_range(0, keys.len());
             CrdsValue::new_rand(&mut rng, Some(&keys[index]))
         })
-        .take(256)
+        .take(2048)
         .collect();
         let mut currents = HashMap::new();
         for value in filter_current(&values) {
@@ -834,9 +853,9 @@ mod test {
         }
         assert_eq!(count, currents.len());
         // Currently CrdsData::new_rand is only implemented for 5 different
-        // kinds and excludes Vote and EpochSlots, and so the unique labels
-        // cannot be more than 5 times number of keys.
-        assert!(currents.len() <= keys.len() * 5);
+        // kinds and excludes EpochSlots, and so the unique labels cannot be
+        // more than (5 + MAX_VOTES) times number of keys.
+        assert!(currents.len() <= keys.len() * (5 + MAX_VOTES as usize));
     }
 
     #[test]

core/src/data_budget.rs

@@ -52,27 +52,28 @@ impl DataBudget {
         }
     }
 
-    // Updates the budget if at least given milliseconds has passed since last
-    // update. Updater function maps current value of bytes to the new one.
-    pub fn update<F>(&self, duration_millis: u64, updater: F)
+    /// Updates the budget if at least given milliseconds has passed since last
+    /// update. Updater function maps current value of bytes to the new one.
+    /// Returns current data-budget after the update.
+    pub fn update<F>(&self, duration_millis: u64, updater: F) -> usize
     where
         F: Fn(usize) -> usize,
     {
-        if !self.can_update(duration_millis) {
-            return;
-        }
-        let mut bytes = self.bytes.load(Ordering::Acquire);
-        loop {
-            match self.bytes.compare_exchange_weak(
-                bytes,
-                updater(bytes),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-            ) {
-                Ok(_) => break,
-                Err(b) => bytes = b,
+        if self.can_update(duration_millis) {
+            let mut bytes = self.bytes.load(Ordering::Acquire);
+            loop {
+                match self.bytes.compare_exchange_weak(
+                    bytes,
+                    updater(bytes),
+                    Ordering::AcqRel,
+                    Ordering::Acquire,
+                ) {
+                    Ok(_) => break,
+                    Err(b) => bytes = b,
+                }
             }
         }
+        self.bytes.load(Ordering::Acquire)
     }
 
     // Non-atomic clone only for tests and simulations.
@@ -94,16 +95,16 @@ mod tests {
         let budget = DataBudget::default();
         assert!(!budget.take(1)); // budget = 0.
 
-        budget.update(1000, |bytes| bytes + 5); // budget updates to 5.
+        assert_eq!(budget.update(1000, |bytes| bytes + 5), 5); // budget updates to 5.
         assert!(budget.take(1));
         assert!(budget.take(2));
         assert!(!budget.take(3)); // budget = 2, out of budget.
 
-        budget.update(30, |_| 10); // no update, budget = 2.
+        assert_eq!(budget.update(30, |_| 10), 2); // no update, budget = 2.
         assert!(!budget.take(3)); // budget = 2, out of budget.
 
         std::thread::sleep(Duration::from_millis(50));
-        budget.update(30, |bytes| bytes * 2); // budget updates to 4.
+        assert_eq!(budget.update(30, |bytes| bytes * 2), 4); // budget updates to 4.
 
         assert!(budget.take(3));
         assert!(budget.take(1));

core/src/duplicate_shred.rs

@@ -0,0 +1,367 @@
+use crate::crds_value::sanitize_wallclock;
+use itertools::Itertools;
+use solana_ledger::{
+    blockstore_meta::DuplicateSlotProof,
+    shred::{Shred, ShredError, ShredType},
+};
+use solana_sdk::{
+    clock::Slot,
+    pubkey::Pubkey,
+    sanitize::{Sanitize, SanitizeError},
+};
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    convert::TryFrom,
+    num::TryFromIntError,
+};
+use thiserror::Error;
+
+const DUPLICATE_SHRED_HEADER_SIZE: usize = 63;
+
+/// Function returning leader at a given slot.
+pub trait LeaderScheduleFn: FnOnce(Slot) -> Option<Pubkey> {}
+impl<F> LeaderScheduleFn for F where F: FnOnce(Slot) -> Option<Pubkey> {}
+
+#[derive(Clone, Debug, PartialEq, AbiExample, Deserialize, Serialize)]
+pub struct DuplicateShred {
+    pub(crate) from: Pubkey,
+    pub(crate) wallclock: u64,
+    slot: Slot,
+    shred_index: u32,
+    shred_type: ShredType,
+    // Serialized DuplicateSlotProof split into chunks.
+    num_chunks: u8,
+    chunk_index: u8,
+    #[serde(with = "serde_bytes")]
+    chunk: Vec<u8>,
+}
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct DuplicateShredIndex {
+    slot: Slot,
+    shred_index: u32,
+    shred_type: ShredType,
+    num_chunks: u8,
+    chunk_index: u8,
+}
+
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("data chunk mismatch")]
+    DataChunkMismatch,
+    #[error("decoding error")]
+    DecodingError(std::io::Error),
+    #[error("encoding error")]
+    EncodingError(std::io::Error),
+    #[error("invalid chunk index")]
+    InvalidChunkIndex,
+    #[error("invalid duplicate shreds")]
+    InvalidDuplicateShreds,
+    #[error("invalid duplicate slot proof")]
+    InvalidDuplicateSlotProof,
+    #[error("invalid signature")]
+    InvalidSignature,
+    #[error("invalid size limit")]
+    InvalidSizeLimit,
+    #[error("invalid shred")]
+    InvalidShred(#[from] ShredError),
+    #[error("number of chunks mismatch")]
+    NumChunksMismatch,
+    #[error("missing data chunk")]
+    MissingDataChunk,
+    #[error("(de)serialization error")]
+    SerializationError(#[from] bincode::Error),
+    #[error("shred index mismatch")]
+    ShredIndexMismatch,
+    #[error("shred type mismatch")]
+    ShredTypeMismatch,
+    #[error("slot mismatch")]
+    SlotMismatch,
+    #[error("type conversion error")]
+    TryFromIntError(#[from] TryFromIntError),
+    #[error("unknown slot leader")]
+    UnknownSlotLeader,
+}
+
+// Asserts that the two shreds can indicate duplicate proof for
+// the same triplet of (slot, shred-index, and shred-type_), and
+// that they have valid signatures from the slot leader.
+fn check_shreds(
+    leader: impl LeaderScheduleFn,
+    shred1: &Shred,
+    shred2: &Shred,
+) -> Result<(), Error> {
+    if shred1.slot() != shred2.slot() {
+        Err(Error::SlotMismatch)
+    } else if shred1.index() != shred2.index() {
+        Err(Error::ShredIndexMismatch)
+    } else if shred1.common_header.shred_type != shred2.common_header.shred_type {
+        Err(Error::ShredTypeMismatch)
+    } else if shred1.payload == shred2.payload {
+        Err(Error::InvalidDuplicateShreds)
+    } else {
+        let slot_leader = leader(shred1.slot()).ok_or(Error::UnknownSlotLeader)?;
+        if !shred1.verify(&slot_leader) || !shred2.verify(&slot_leader) {
+            Err(Error::InvalidSignature)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Splits a DuplicateSlotProof into DuplicateShred
+/// chunks with a size limit on each chunk.
+pub fn from_duplicate_slot_proof(
+    proof: &DuplicateSlotProof,
+    self_pubkey: Pubkey, // Pubkey of my node broadcasting crds value.
+    leader: impl LeaderScheduleFn,
+    wallclock: u64,
+    max_size: usize, // Maximum serialized size of each DuplicateShred.
+    encoder: impl FnOnce(Vec<u8>) -> Result<Vec<u8>, std::io::Error>,
+) -> Result<impl Iterator<Item = DuplicateShred>, Error> {
+    if proof.shred1 == proof.shred2 {
+        return Err(Error::InvalidDuplicateSlotProof);
+    }
+    let shred1 = Shred::new_from_serialized_shred(proof.shred1.clone())?;
+    let shred2 = Shred::new_from_serialized_shred(proof.shred2.clone())?;
+    check_shreds(leader, &shred1, &shred2)?;
+    let (slot, shred_index, shred_type) = (
+        shred1.slot(),
+        shred1.index(),
+        shred1.common_header.shred_type,
+    );
+    let data = bincode::serialize(proof)?;
+    let data = encoder(data).map_err(Error::EncodingError)?;
+    let chunk_size = if DUPLICATE_SHRED_HEADER_SIZE < max_size {
+        max_size - DUPLICATE_SHRED_HEADER_SIZE
+    } else {
+        return Err(Error::InvalidSizeLimit);
+    };
+    let chunks: Vec<_> = data.chunks(chunk_size).map(Vec::from).collect();
+    let num_chunks = u8::try_from(chunks.len())?;
+    let chunks = chunks
+        .into_iter()
+        .enumerate()
+        .map(move |(i, chunk)| DuplicateShred {
+            from: self_pubkey,
+            wallclock,
+            slot,
+            shred_index,
+            shred_type,
+            num_chunks,
+            chunk_index: i as u8,
+            chunk,
+        });
+    Ok(chunks)
+}
+
+// Returns a predicate checking if a duplicate-shred chunk matches
+// (slot, shred_index, shred_type) and has valid chunk_index.
+fn check_chunk(
+    slot: Slot,
+    shred_index: u32,
+    shred_type: ShredType,
+    num_chunks: u8,
+) -> impl Fn(&DuplicateShred) -> Result<(), Error> {
+    move |dup| {
+        if dup.slot != slot {
+            Err(Error::SlotMismatch)
+        } else if dup.shred_index != shred_index {
+            Err(Error::ShredIndexMismatch)
+        } else if dup.shred_type != shred_type {
+            Err(Error::ShredTypeMismatch)
+        } else if dup.num_chunks != num_chunks {
+            Err(Error::NumChunksMismatch)
+        } else if dup.chunk_index >= num_chunks {
+            Err(Error::InvalidChunkIndex)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Reconstructs the duplicate shreds from chunks of DuplicateShred.
+pub fn into_shreds(
+    chunks: impl IntoIterator<Item = DuplicateShred>,
+    leader: impl LeaderScheduleFn,
+    decoder: impl FnOnce(Vec<u8>) -> Result<Vec<u8>, std::io::Error>,
+) -> Result<(Shred, Shred), Error> {
+    let mut chunks = chunks.into_iter();
+    let DuplicateShred {
+        slot,
+        shred_index,
+        shred_type,
+        num_chunks,
+        chunk_index,
+        chunk,
+        ..
+    } = match chunks.next() {
+        None => return Err(Error::InvalidDuplicateShreds),
+        Some(chunk) => chunk,
+    };
+    let slot_leader = leader(slot).ok_or(Error::UnknownSlotLeader)?;
+    let check_chunk = check_chunk(slot, shred_index, shred_type, num_chunks);
+    let mut data = HashMap::new();
+    data.insert(chunk_index, chunk);
+    for chunk in chunks {
+        check_chunk(&chunk)?;
+        match data.entry(chunk.chunk_index) {
+            Entry::Vacant(entry) => {
+                entry.insert(chunk.chunk);
+            }
+            Entry::Occupied(entry) => {
+                if *entry.get() != chunk.chunk {
+                    return Err(Error::DataChunkMismatch);
+                }
+            }
+        }
+    }
+    if data.len() != num_chunks as usize {
+        return Err(Error::MissingDataChunk);
+    }
+    let data = (0..num_chunks).map(|k| data.remove(&k).unwrap());
+    let data = decoder(data.concat()).map_err(Error::DecodingError)?;
+    let proof: DuplicateSlotProof = bincode::deserialize(&data)?;
+    if proof.shred1 == proof.shred2 {
+        return Err(Error::InvalidDuplicateSlotProof);
+    }
+    let shred1 = Shred::new_from_serialized_shred(proof.shred1)?;
+    let shred2 = Shred::new_from_serialized_shred(proof.shred2)?;
+    if shred1.slot() != slot || shred2.slot() != slot {
+        Err(Error::SlotMismatch)
+    } else if shred1.index() != shred_index || shred2.index() != shred_index {
+        Err(Error::ShredIndexMismatch)
+    } else if shred1.common_header.shred_type != shred_type
+        || shred2.common_header.shred_type != shred_type
+    {
+        Err(Error::ShredTypeMismatch)
+    } else if shred1.payload == shred2.payload {
+        Err(Error::InvalidDuplicateShreds)
+    } else if !shred1.verify(&slot_leader) || !shred2.verify(&slot_leader) {
+        Err(Error::InvalidSignature)
+    } else {
+        Ok((shred1, shred2))
+    }
+}
+
+impl Sanitize for DuplicateShred {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        sanitize_wallclock(self.wallclock)?;
+        if self.chunk_index >= self.num_chunks {
+            return Err(SanitizeError::IndexOutOfBounds);
+        }
+        self.from.sanitize()
+    }
+}
+
+impl From<&DuplicateShred> for DuplicateShredIndex {
+    fn from(shred: &DuplicateShred) -> Self {
+        Self {
+            slot: shred.slot,
+            shred_index: shred.shred_index,
+            shred_type: shred.shred_type,
+            num_chunks: shred.num_chunks,
+            chunk_index: shred.chunk_index,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rand::Rng;
+    use solana_ledger::{entry::Entry, shred::Shredder};
+    use solana_sdk::{hash, signature::Keypair, signature::Signer, system_transaction};
+    use std::sync::Arc;
+
+    #[test]
+    fn test_duplicate_shred_header_size() {
+        let dup = DuplicateShred {
+            from: Pubkey::new_unique(),
+            wallclock: u64::MAX,
+            slot: Slot::MAX,
+            shred_index: u32::MAX,
+            shred_type: ShredType(u8::MAX),
+            num_chunks: u8::MAX,
+            chunk_index: u8::MAX,
+            chunk: Vec::default(),
+        };
+        assert_eq!(
+            bincode::serialize(&dup).unwrap().len(),
+            DUPLICATE_SHRED_HEADER_SIZE
+        );
+        assert_eq!(
+            bincode::serialized_size(&dup).unwrap(),
+            DUPLICATE_SHRED_HEADER_SIZE as u64
+        );
+    }
+
+    fn new_rand_shred<R: Rng>(rng: &mut R, next_shred_index: u32, shredder: &Shredder) -> Shred {
+        let entries: Vec<_> = std::iter::repeat_with(|| {
+            let tx = system_transaction::transfer(
+                &Keypair::new(),       // from
+                &Pubkey::new_unique(), // to
+                rng.gen(),             // lamports
+                hash::new_rand(rng),   // recent blockhash
+            );
+            Entry::new(
+                &hash::new_rand(rng), // prev_hash
+                1,                    // num_hashes,
+                vec![tx],             // transactions
+            )
+        })
+        .take(5)
+        .collect();
+        let (mut data_shreds, _coding_shreds, _last_shred_index) = shredder.entries_to_shreds(
+            &entries,
+            true, // is_last_in_slot
+            next_shred_index,
+        );
+        data_shreds.swap_remove(0)
+    }
+
+    #[test]
+    fn test_duplicate_shred_round_trip() {
+        let mut rng = rand::thread_rng();
+        let leader = Arc::new(Keypair::new());
+        let (slot, parent_slot, fec_rate, reference_tick, version) =
+            (53084024, 53084023, 0.0, 0, 0);
+        let shredder = Shredder::new(
+            slot,
+            parent_slot,
+            fec_rate,
+            leader.clone(),
+            reference_tick,
+            version,
+        )
+        .unwrap();
+        let next_shred_index = rng.gen();
+        let shred1 = new_rand_shred(&mut rng, next_shred_index, &shredder);
+        let shred2 = new_rand_shred(&mut rng, next_shred_index, &shredder);
+        let leader = |s| {
+            if s == slot {
+                Some(leader.pubkey())
+            } else {
+                None
+            }
+        };
+        let proof = DuplicateSlotProof {
+            shred1: shred1.payload.clone(),
+            shred2: shred2.payload.clone(),
+        };
+        let chunks: Vec<_> = from_duplicate_slot_proof(
+            &proof,
+            Pubkey::new_unique(), // self_pubkey
+            leader,
+            rng.gen(), // wallclock
+            512,       // max_size
+            Ok,        // encoder
+        )
+        .unwrap()
+        .collect();
+        assert!(chunks.len() > 4);
+        let (shred3, shred4) = into_shreds(chunks, leader, Ok).unwrap();
+        assert_eq!(shred1, shred3);
+        assert_eq!(shred2, shred4);
+    }
+}

core/src/lib.rs

@@ -31,6 +31,7 @@ pub mod crds_gossip_push;
 pub mod crds_shards;
 pub mod crds_value;
 pub mod data_budget;
+pub mod duplicate_shred;
 pub mod epoch_slots;
 pub mod fetch_stage;
 pub mod fork_choice;
@@ -41,6 +42,7 @@ pub mod ledger_cleanup_service;
 pub mod non_circulating_supply;
 pub mod optimistic_confirmation_verifier;
 pub mod optimistically_confirmed_bank_tracker;
+pub mod packet_hasher;
 pub mod ping_pong;
 pub mod poh_recorder;
 pub mod poh_service;

core/src/optimistically_confirmed_bank_tracker.rs

@@ -23,7 +23,7 @@ pub struct OptimisticallyConfirmedBank {
 impl OptimisticallyConfirmedBank {
     pub fn locked_from_bank_forks_root(bank_forks: &Arc<RwLock<BankForks>>) -> Arc<RwLock<Self>> {
         Arc::new(RwLock::new(Self {
-            bank: bank_forks.read().unwrap().root_bank().clone(),
+            bank: bank_forks.read().unwrap().root_bank(),
         }))
     }
 }

core/src/packet_hasher.rs

@@ -0,0 +1,34 @@
+// Get a unique hash value for a packet
+// Used in retransmit and shred fetch to prevent dos with same packet data.
+
+use ahash::AHasher;
+use rand::{thread_rng, Rng};
+use solana_perf::packet::Packet;
+use std::hash::Hasher;
+
+#[derive(Clone)]
+pub struct PacketHasher {
+    seed1: u128,
+    seed2: u128,
+}
+
+impl Default for PacketHasher {
+    fn default() -> Self {
+        Self {
+            seed1: thread_rng().gen::<u128>(),
+            seed2: thread_rng().gen::<u128>(),
+        }
+    }
+}
+
+impl PacketHasher {
+    pub fn hash_packet(&self, packet: &Packet) -> u64 {
+        let mut hasher = AHasher::new_with_keys(self.seed1, self.seed2);
+        hasher.write(&packet.data[0..packet.meta.size]);
+        hasher.finish()
+    }
+
+    pub fn reset(&mut self) {
+        *self = Self::default();
+    }
+}

core/src/poh_service.rs

@@ -1,7 +1,6 @@
 //! The `poh_service` module implements a service that records the passing of
 //! "ticks", a measure of time in the PoH stream
 use crate::poh_recorder::PohRecorder;
-use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
 use solana_sdk::poh_config::PohConfig;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
@@ -20,11 +19,15 @@ pub struct PohService {
 // See benches/poh.rs for some benchmarks that attempt to justify this magic number.
 pub const NUM_HASHES_PER_BATCH: u64 = 1;
 
+pub const DEFAULT_PINNED_CPU_CORE: usize = 0;
+
 impl PohService {
     pub fn new(
         poh_recorder: Arc<Mutex<PohRecorder>>,
         poh_config: &Arc<PohConfig>,
         poh_exit: &Arc<AtomicBool>,
+        ticks_per_slot: u64,
+        pinned_cpu_core: usize,
     ) -> Self {
         let poh_exit_ = poh_exit.clone();
         let poh_config = poh_config.clone();
@@ -47,9 +50,14 @@ impl PohService {
                     // Let's dedicate one of the CPU cores to this thread so that it can gain
                     // from cache performance.
                     if let Some(cores) = core_affinity::get_core_ids() {
-                        core_affinity::set_for_current(cores[0]);
+                        core_affinity::set_for_current(cores[pinned_cpu_core]);
                     }
-                    Self::tick_producer(poh_recorder, &poh_exit_);
+                    Self::tick_producer(
+                        poh_recorder,
+                        &poh_exit_,
+                        poh_config.target_tick_duration.as_nanos() as u64,
+                        ticks_per_slot,
+                    );
                 }
                 poh_exit_.store(true, Ordering::Relaxed);
             })
@@ -85,27 +93,48 @@ impl PohService {
         }
     }
 
-    fn tick_producer(poh_recorder: Arc<Mutex<PohRecorder>>, poh_exit: &AtomicBool) {
+    fn tick_producer(
+        poh_recorder: Arc<Mutex<PohRecorder>>,
+        poh_exit: &AtomicBool,
+        target_tick_ns: u64,
+        ticks_per_slot: u64,
+    ) {
+        info!("starting with target ns: {}", target_tick_ns);
         let poh = poh_recorder.lock().unwrap().poh.clone();
         let mut now = Instant::now();
+        let mut last_metric = Instant::now();
         let mut num_ticks = 0;
         let mut num_hashes = 0;
+        let mut total_sleep_us = 0;
         loop {
             num_hashes += NUM_HASHES_PER_BATCH;
             if poh.lock().unwrap().hash(NUM_HASHES_PER_BATCH) {
                 // Lock PohRecorder only for the final hash...
                 poh_recorder.lock().unwrap().tick();
                 num_ticks += 1;
-                if num_ticks >= DEFAULT_TICKS_PER_SLOT * 2 {
+                let elapsed_ns = now.elapsed().as_nanos() as u64;
+                // sleep is not accurate enough to get a predictable time.
+                // Kernel can not schedule the thread for a while.
+                while (now.elapsed().as_nanos() as u64) < target_tick_ns {
+                    std::sync::atomic::spin_loop_hint();
+                }
+                total_sleep_us += (now.elapsed().as_nanos() as u64 - elapsed_ns) / 1000;
+                now = Instant::now();
+
+                if last_metric.elapsed().as_millis() > 1000 {
+                    let elapsed_ms = last_metric.elapsed().as_millis() as u64;
+                    let ms_per_slot = (elapsed_ms * ticks_per_slot) / num_ticks;
                     datapoint_info!(
                         "poh-service",
                         ("ticks", num_ticks as i64, i64),
                         ("hashes", num_hashes as i64, i64),
-                        ("elapsed_ms", now.elapsed().as_millis() as i64, i64),
+                        ("elapsed_ms", ms_per_slot, i64),
+                        ("total_sleep_ms", total_sleep_us / 1000, i64),
                     );
+                    total_sleep_us = 0;
                     num_ticks = 0;
                     num_hashes = 0;
-                    now = Instant::now();
+                    last_metric = Instant::now();
                 }
                 if poh_exit.load(Ordering::Relaxed) {
                     break;
@@ -189,7 +218,13 @@ mod tests {
                     .unwrap()
             };
 
-            let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
+            let poh_service = PohService::new(
+                poh_recorder.clone(),
+                &poh_config,
+                &exit,
+                0,
+                DEFAULT_PINNED_CPU_CORE,
+            );
             poh_recorder.lock().unwrap().set_working_bank(working_bank);
 
             // get some events

core/src/replay_stage.rs

@@ -616,7 +616,7 @@ impl ReplayStage {
         let (root_bank, frozen_banks) = {
             let bank_forks = bank_forks.read().unwrap();
             (
-                bank_forks.root_bank().clone(),
+                bank_forks.root_bank(),
                 bank_forks.frozen_banks().values().cloned().collect(),
             )
         };
@@ -630,7 +630,7 @@ impl ReplayStage {
     }
 
     pub(crate) fn initialize_progress_and_fork_choice(
-        root_bank: &Arc<Bank>,
+        root_bank: &Bank,
         mut frozen_banks: Vec<Arc<Bank>>,
         my_pubkey: &Pubkey,
         vote_account: &Pubkey,
@@ -2558,11 +2558,11 @@ pub(crate) mod tests {
 
     #[test]
     fn test_replay_commitment_cache() {
-        fn leader_vote(bank: &Arc<Bank>, pubkey: &Pubkey) {
+        fn leader_vote(vote_slot: Slot, bank: &Arc<Bank>, pubkey: &Pubkey) {
             let mut leader_vote_account = bank.get_account(&pubkey).unwrap();
             let mut vote_state = VoteState::from(&leader_vote_account).unwrap();
-            vote_state.process_slot_vote_unchecked(bank.slot());
-            let versioned = VoteStateVersions::Current(Box::new(vote_state));
+            vote_state.process_slot_vote_unchecked(vote_slot);
+            let versioned = VoteStateVersions::new_current(vote_state);
             VoteState::to(&versioned, &mut leader_vote_account).unwrap();
             bank.store_account(&pubkey, &leader_vote_account);
         }
@@ -2581,10 +2581,7 @@ pub(crate) mod tests {
         }
         bank0.freeze();
         let arc_bank0 = Arc::new(bank0);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new_from_banks(
-            &[arc_bank0.clone()],
-            0,
-        )));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new_from_banks(&[arc_bank0], 0)));
 
         let exit = Arc::new(AtomicBool::new(false));
         let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
@@ -2608,44 +2605,33 @@ pub(crate) mod tests {
             .get_block_commitment(1)
             .is_none());
 
-        let bank1 = Bank::new_from_parent(&arc_bank0, &Pubkey::default(), arc_bank0.slot() + 1);
-        let _res = bank1.transfer(
-            10,
-            &genesis_config_info.mint_keypair,
-            &solana_sdk::pubkey::new_rand(),
-        );
-        for _ in 0..genesis_config.ticks_per_slot {
-            bank1.register_tick(&Hash::default());
+        for i in 1..=3 {
+            let prev_bank = bank_forks.read().unwrap().get(i - 1).unwrap().clone();
+            let bank = Bank::new_from_parent(&prev_bank, &Pubkey::default(), prev_bank.slot() + 1);
+            let _res = bank.transfer(
+                10,
+                &genesis_config_info.mint_keypair,
+                &solana_sdk::pubkey::new_rand(),
+            );
+            for _ in 0..genesis_config.ticks_per_slot {
+                bank.register_tick(&Hash::default());
+            }
+            bank_forks.write().unwrap().insert(bank);
+            let arc_bank = bank_forks.read().unwrap().get(i).unwrap().clone();
+            leader_vote(i - 1, &arc_bank, &leader_voting_pubkey);
+            ReplayStage::update_commitment_cache(
+                arc_bank.clone(),
+                0,
+                leader_lamports,
+                &lockouts_sender,
+            );
+            arc_bank.freeze();
         }
-        bank1.freeze();
-        bank_forks.write().unwrap().insert(bank1);
-        let arc_bank1 = bank_forks.read().unwrap().get(1).unwrap().clone();
-        leader_vote(&arc_bank1, &leader_voting_pubkey);
-        ReplayStage::update_commitment_cache(
-            arc_bank1.clone(),
-            0,
-            leader_lamports,
-            &lockouts_sender,
-        );
 
-        let bank2 = Bank::new_from_parent(&arc_bank1, &Pubkey::default(), arc_bank1.slot() + 1);
-        let _res = bank2.transfer(
-            10,
-            &genesis_config_info.mint_keypair,
-            &solana_sdk::pubkey::new_rand(),
-        );
-        for _ in 0..genesis_config.ticks_per_slot {
-            bank2.register_tick(&Hash::default());
-        }
-        bank2.freeze();
-        bank_forks.write().unwrap().insert(bank2);
-        let arc_bank2 = bank_forks.read().unwrap().get(2).unwrap().clone();
-        leader_vote(&arc_bank2, &leader_voting_pubkey);
-        ReplayStage::update_commitment_cache(arc_bank2, 0, leader_lamports, &lockouts_sender);
         thread::sleep(Duration::from_millis(200));
 
         let mut expected0 = BlockCommitment::default();
-        expected0.increase_confirmation_stake(2, leader_lamports);
+        expected0.increase_confirmation_stake(3, leader_lamports);
         assert_eq!(
             block_commitment_cache
                 .read()
@@ -3134,7 +3120,7 @@ pub(crate) mod tests {
     ) {
         let stake = 10_000;
         let (bank_forks, _, _) = initialize_state(&all_keypairs, stake);
-        let root_bank = bank_forks.root_bank().clone();
+        let root_bank = bank_forks.root_bank();
         let mut propagated_stats = PropagatedStats {
             total_epoch_stake: stake * all_keypairs.len() as u64,
             ..PropagatedStats::default()
@@ -3819,7 +3805,7 @@ pub(crate) mod tests {
             ..
         } = replay_blockstore_components();
 
-        let root_bank = bank_forks.read().unwrap().root_bank().clone();
+        let root_bank = bank_forks.read().unwrap().root_bank();
         let my_pubkey = leader_schedule_cache
             .slot_leader_at(root_bank.slot(), Some(&root_bank))
             .unwrap();

core/src/retransmit_stage.rs

@@ -1,8 +1,6 @@
 //! The `retransmit_stage` retransmits shreds between validators
 #![allow(clippy::rc_buffer)]
 
-use crate::shred_fetch_stage::ShredFetchStage;
-use crate::shred_fetch_stage::ShredFetchStats;
 use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
     cluster_info_vote_listener::VerifiedVoteReceiver,
@@ -15,25 +13,22 @@ use crate::{
     result::{Error, Result},
     window_service::{should_retransmit_and_persist, WindowService},
 };
-use ahash::AHasher;
 use crossbeam_channel::Receiver;
 use lru::LruCache;
-use rand::{thread_rng, Rng};
+use solana_ledger::shred::{get_shred_slot_index_type, ShredFetchStats};
 use solana_ledger::{
     blockstore::{Blockstore, CompletedSlotsReceiver},
     leader_schedule_cache::LeaderScheduleCache,
-    staking_utils,
 };
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_error;
-use solana_perf::packet::Packets;
+use solana_perf::packet::{Packet, Packets};
 use solana_runtime::bank_forks::BankForks;
 use solana_sdk::clock::{Epoch, Slot};
 use solana_sdk::epoch_schedule::EpochSchedule;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
 use solana_streamer::streamer::PacketReceiver;
-use std::hash::Hasher;
 use std::{
     cmp,
     collections::hash_set::HashSet,
@@ -122,7 +117,9 @@ fn update_retransmit_stats(
 
     let now = timestamp();
     let last = stats.last_ts.load(Ordering::Relaxed);
-    if now - last > 2000 && stats.last_ts.compare_and_swap(last, now, Ordering::Relaxed) == last {
+    if now.saturating_sub(last) > 2000
+        && stats.last_ts.compare_and_swap(last, now, Ordering::Relaxed) == last
+    {
         datapoint_info!("retransmit-num_nodes", ("count", peers_len, i64));
         datapoint_info!(
             "retransmit-stage",
@@ -206,7 +203,42 @@ struct EpochStakesCache {
     stakes_and_index: Vec<(u64, usize)>,
 }
 
-pub type ShredFilterAndSeeds = (LruCache<(Slot, u32), Vec<u64>>, u128, u128);
+use crate::packet_hasher::PacketHasher;
+// Map of shred (slot, index, is_data) => list of hash values seen for that key.
+pub type ShredFilter = LruCache<(Slot, u32, bool), Vec<u64>>;
+
+pub type ShredFilterAndHasher = (ShredFilter, PacketHasher);
+
+// Return true if shred is already received and should skip retransmit
+fn check_if_already_received(
+    packet: &Packet,
+    shreds_received: &Arc<Mutex<ShredFilterAndHasher>>,
+) -> bool {
+    match get_shred_slot_index_type(packet, &mut ShredFetchStats::default()) {
+        Some(slot_index) => {
+            let mut received = shreds_received.lock().unwrap();
+            let hasher = received.1.clone();
+            if let Some(sent) = received.0.get_mut(&slot_index) {
+                if sent.len() < MAX_DUPLICATE_COUNT {
+                    let hash = hasher.hash_packet(packet);
+                    if sent.contains(&hash) {
+                        return true;
+                    }
+
+                    sent.push(hash);
+                } else {
+                    return true;
+                }
+            } else {
+                let hash = hasher.hash_packet(&packet);
+                received.0.put(slot_index, vec![hash]);
+            }
+
+            false
+        }
+        None => true,
+    }
+}
 
 #[allow(clippy::too_many_arguments)]
 fn retransmit(
@@ -219,7 +251,7 @@ fn retransmit(
     stats: &Arc<RetransmitStats>,
     epoch_stakes_cache: &Arc<RwLock<EpochStakesCache>>,
     last_peer_update: &Arc<AtomicU64>,
-    shreds_received: &Arc<Mutex<ShredFilterAndSeeds>>,
+    shreds_received: &Arc<Mutex<ShredFilterAndHasher>>,
 ) -> Result<()> {
     let timer = Duration::new(1, 0);
     let r_lock = r.lock().unwrap();
@@ -247,7 +279,7 @@ fn retransmit(
         drop(r_epoch_stakes_cache);
         let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
         if w_epoch_stakes_cache.epoch != bank_epoch {
-            let stakes = staking_utils::staked_nodes_at_epoch(&r_bank, bank_epoch);
+            let stakes = r_bank.epoch_staked_nodes(bank_epoch);
             let stakes = stakes.map(Arc::new);
             w_epoch_stakes_cache.stakes = stakes;
             w_epoch_stakes_cache.epoch = bank_epoch;
@@ -258,7 +290,8 @@ fn retransmit(
 
     let now = timestamp();
     let last = last_peer_update.load(Ordering::Relaxed);
-    if now - last > 1000 && last_peer_update.compare_and_swap(last, now, Ordering::Relaxed) == last
+    if now.saturating_sub(last) > 1000
+        && last_peer_update.compare_and_swap(last, now, Ordering::Relaxed) == last
     {
         drop(r_epoch_stakes_cache);
         let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
@@ -271,8 +304,7 @@ fn retransmit(
         {
             let mut sr = shreds_received.lock().unwrap();
             sr.0.clear();
-            sr.1 = thread_rng().gen::<u128>();
-            sr.2 = thread_rng().gen::<u128>();
+            sr.1.reset();
         }
     }
     let mut peers_len = 0;
@@ -299,33 +331,10 @@ fn retransmit(
                 continue;
             }
 
-            match ShredFetchStage::get_slot_index(packet, &mut ShredFetchStats::default()) {
-                Some(slot_index) => {
-                    let mut received = shreds_received.lock().unwrap();
-                    let seed1 = received.1;
-                    let seed2 = received.2;
-                    if let Some(sent) = received.0.get_mut(&slot_index) {
-                        if sent.len() < MAX_DUPLICATE_COUNT {
-                            let mut hasher = AHasher::new_with_keys(seed1, seed2);
-                            hasher.write(&packet.data[0..packet.meta.size]);
-                            let hash = hasher.finish();
-                            if sent.contains(&hash) {
-                                continue;
-                            }
-
-                            sent.push(hash);
-                        } else {
-                            continue;
-                        }
-                    } else {
-                        let mut hasher = AHasher::new_with_keys(seed1, seed2);
-                        hasher.write(&packet.data[0..packet.meta.size]);
-                        let hash = hasher.finish();
-                        received.0.put(slot_index, vec![hash]);
-                    }
-                }
-                None => continue,
+            if check_if_already_received(packet, shreds_received) {
+                continue;
             }
+
             let mut compute_turbine_peers = Measure::start("turbine_start");
             let (my_index, mut shuffled_stakes_and_index) = ClusterInfo::shuffle_peers_and_index(
                 &my_id,
@@ -414,7 +423,10 @@ pub fn retransmitter(
     r: Arc<Mutex<PacketReceiver>>,
 ) -> Vec<JoinHandle<()>> {
     let stats = Arc::new(RetransmitStats::default());
-    let shreds_received = Arc::new(Mutex::new((LruCache::new(DEFAULT_LRU_SIZE), 0, 0)));
+    let shreds_received = Arc::new(Mutex::new((
+        LruCache::new(DEFAULT_LRU_SIZE),
+        PacketHasher::default(),
+    )));
     (0..sockets.len())
         .map(|s| {
             let sockets = sockets.clone();
@@ -568,6 +580,7 @@ mod tests {
     use solana_ledger::blockstore_processor::{process_blockstore, ProcessOptions};
     use solana_ledger::create_new_tmp_ledger;
     use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::shred::Shred;
     use solana_net_utils::find_available_port_in_range;
     use solana_perf::packet::{Packet, Packets};
     use std::net::{IpAddr, Ipv4Addr};
@@ -616,8 +629,7 @@ mod tests {
         );
         let _thread_hdls = vec![t_retransmit];
 
-        let mut shred =
-            solana_ledger::shred::Shred::new_from_data(0, 0, 0, None, true, true, 0, 0x20, 0);
+        let mut shred = Shred::new_from_data(0, 0, 0, None, true, true, 0, 0x20, 0);
         let mut packet = Packet::default();
         shred.copy_to_packet(&mut packet);
 
@@ -642,4 +654,52 @@ mod tests {
         assert_eq!(packets.packets.len(), 1);
         assert_eq!(packets.packets[0].meta.repair, false);
     }
+
+    #[test]
+    fn test_already_received() {
+        let mut packet = Packet::default();
+        let slot = 1;
+        let index = 5;
+        let version = 0x40;
+        let shred = Shred::new_from_data(slot, index, 0, None, true, true, 0, version, 0);
+        shred.copy_to_packet(&mut packet);
+        let shreds_received = Arc::new(Mutex::new((LruCache::new(100), PacketHasher::default())));
+        // unique shred for (1, 5) should pass
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // duplicate shred for (1, 5) blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_from_data(slot, index, 2, None, true, true, 0, version, 0);
+        shred.copy_to_packet(&mut packet);
+        // first duplicate shred for (1, 5) passed
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // then blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_from_data(slot, index, 8, None, true, true, 0, version, 0);
+        shred.copy_to_packet(&mut packet);
+        // 2nd duplicate shred for (1, 5) blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_empty_coding(slot, index, 0, 1, 1, 0, version);
+        shred.copy_to_packet(&mut packet);
+        // Coding at (1, 5) passes
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // then blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_empty_coding(slot, index, 2, 1, 1, 0, version);
+        shred.copy_to_packet(&mut packet);
+        // 2nd unique coding at (1, 5) passes
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // same again is blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_empty_coding(slot, index, 3, 1, 1, 0, version);
+        shred.copy_to_packet(&mut packet);
+        // Another unique coding at (1, 5) always blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+        assert!(check_if_already_received(&packet, &shreds_received));
+    }
 }

core/src/rpc.rs

@@ -39,14 +39,16 @@ use solana_metrics::inc_new_counter_info;
 use solana_perf::packet::PACKET_DATA_SIZE;
 use solana_runtime::{
     accounts::AccountAddressFilter,
+    accounts_index::{AccountIndex, IndexKey},
     bank::Bank,
     bank_forks::BankForks,
     commitment::{BlockCommitmentArray, BlockCommitmentCache, CommitmentSlots},
+    inline_spl_token_v2_0::{SPL_TOKEN_ACCOUNT_MINT_OFFSET, SPL_TOKEN_ACCOUNT_OWNER_OFFSET},
 };
 use solana_sdk::{
     account::Account,
     account_utils::StateMut,
-    clock::{Slot, UnixTimestamp},
+    clock::{Slot, UnixTimestamp, MAX_RECENT_BLOCKHASHES},
     commitment_config::{CommitmentConfig, CommitmentLevel},
     epoch_info::EpochInfo,
     epoch_schedule::EpochSchedule,
@@ -109,6 +111,7 @@ pub struct JsonRpcConfig {
     pub enable_bigtable_ledger_storage: bool,
     pub enable_bigtable_ledger_upload: bool,
     pub max_multiple_accounts: Option<usize>,
+    pub account_indexes: HashSet<AccountIndex>,
 }
 
 #[derive(Clone)]
@@ -187,7 +190,7 @@ impl JsonRpcRequestProcessor {
                 "Bank with {:?} not found at slot: {:?}",
                 commitment_level, slot
             );
-            r_bank_forks.root_bank().clone()
+            r_bank_forks.root_bank()
         })
     }
 
@@ -237,7 +240,7 @@ impl JsonRpcRequestProcessor {
         let cluster_info = Arc::new(ClusterInfo::default());
         let tpu_address = cluster_info.my_contact_info().tpu;
         let (sender, receiver) = channel();
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         Self {
             config: JsonRpcConfig::default(),
@@ -315,12 +318,19 @@ impl JsonRpcRequestProcessor {
         let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
         let data_slice_config = config.data_slice;
         check_slice_and_encoding(&encoding, data_slice_config.is_some())?;
-        let keyed_accounts = get_filtered_program_accounts(&bank, program_id, filters);
+        let keyed_accounts = {
+            if let Some(owner) = get_spl_token_owner_filter(program_id, &filters) {
+                self.get_filtered_spl_token_accounts_by_owner(&bank, &owner, filters)
+            } else {
+                self.get_filtered_program_accounts(&bank, program_id, filters)
+            }
+        };
         let result =
             if program_id == &spl_token_id_v2_0() && encoding == UiAccountEncoding::JsonParsed {
-                get_parsed_token_accounts(bank, keyed_accounts).collect()
+                get_parsed_token_accounts(bank, keyed_accounts.into_iter()).collect()
             } else {
                 keyed_accounts
+                    .into_iter()
                     .map(|(pubkey, account)| RpcKeyedAccount {
                         pubkey: pubkey.to_string(),
                         account: UiAccount::encode(
@@ -601,7 +611,7 @@ impl JsonRpcRequestProcessor {
         }
     }
 
-    fn check_blockstore_max_root<T>(
+    fn check_blockstore_root<T>(
         &self,
         result: &std::result::Result<T, BlockstoreError>,
         slot: Slot,
@@ -612,7 +622,7 @@ impl JsonRpcRequestProcessor {
         if result.is_err() {
             let err = result.as_ref().unwrap_err();
             debug!(
-                "check_blockstore_max_root, slot: {:?}, max root: {:?}, err: {:?}",
+                "check_blockstore_root, slot: {:?}, max root: {:?}, err: {:?}",
                 slot,
                 self.blockstore.max_root(),
                 err
@@ -620,6 +630,9 @@ impl JsonRpcRequestProcessor {
             if slot >= self.blockstore.max_root() {
                 return Err(RpcCustomError::BlockNotAvailable { slot }.into());
             }
+            if self.blockstore.is_skipped(slot) {
+                return Err(RpcCustomError::SlotSkipped { slot }.into());
+            }
         }
         Ok(())
     }
@@ -662,7 +675,7 @@ impl JsonRpcRequestProcessor {
                     .highest_confirmed_root()
         {
             let result = self.blockstore.get_confirmed_block(slot);
-            self.check_blockstore_max_root(&result, slot)?;
+            self.check_blockstore_root(&result, slot)?;
             if result.is_err() {
                 if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
                     return Ok(self
@@ -768,8 +781,8 @@ impl JsonRpcRequestProcessor {
                 .highest_confirmed_root()
         {
             let result = self.blockstore.get_block_time(slot);
-            self.check_blockstore_max_root(&result, slot)?;
-            if result.is_err() {
+            self.check_blockstore_root(&result, slot)?;
+            if result.is_err() || matches!(result, Ok(None)) {
                 if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
                     return Ok(self
                         .runtime_handle
@@ -1152,29 +1165,20 @@ impl JsonRpcRequestProcessor {
                 "Invalid param: not a v2.0 Token mint".to_string(),
             ));
         }
-        let filters = vec![
-            // Filter on Mint address
-            RpcFilterType::Memcmp(Memcmp {
-                offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(mint.to_string()),
-                encoding: None,
-            }),
-            // Filter on Token Account state
-            RpcFilterType::DataSize(TokenAccount::get_packed_len() as u64),
-        ];
-        let mut token_balances: Vec<RpcTokenAccountBalance> =
-            get_filtered_program_accounts(&bank, &mint_owner, filters)
-                .map(|(address, account)| {
-                    let amount = TokenAccount::unpack(&account.data)
-                        .map(|account| account.amount)
-                        .unwrap_or(0);
-                    let amount = token_amount_to_ui_amount(amount, decimals);
-                    RpcTokenAccountBalance {
-                        address: address.to_string(),
-                        amount,
-                    }
-                })
-                .collect();
+        let mut token_balances: Vec<RpcTokenAccountBalance> = self
+            .get_filtered_spl_token_accounts_by_mint(&bank, &mint, vec![])
+            .into_iter()
+            .map(|(address, account)| {
+                let amount = TokenAccount::unpack(&account.data)
+                    .map(|account| account.amount)
+                    .unwrap_or(0);
+                let amount = token_amount_to_ui_amount(amount, decimals);
+                RpcTokenAccountBalance {
+                    address: address.to_string(),
+                    amount,
+                }
+            })
+            .collect();
         token_balances.sort_by(|a, b| {
             a.amount
                 .amount
@@ -1198,18 +1202,9 @@ impl JsonRpcRequestProcessor {
         let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
         let data_slice_config = config.data_slice;
         check_slice_and_encoding(&encoding, data_slice_config.is_some())?;
-        let (token_program_id, mint) = get_token_program_id_and_mint(&bank, token_account_filter)?;
+        let (_, mint) = get_token_program_id_and_mint(&bank, token_account_filter)?;
 
-        let mut filters = vec![
-            // Filter on Owner address
-            RpcFilterType::Memcmp(Memcmp {
-                offset: 32,
-                bytes: MemcmpEncodedBytes::Binary(owner.to_string()),
-                encoding: None,
-            }),
-            // Filter on Token Account state
-            RpcFilterType::DataSize(TokenAccount::get_packed_len() as u64),
-        ];
+        let mut filters = vec![];
         if let Some(mint) = mint {
             // Optional filter on Mint address
             filters.push(RpcFilterType::Memcmp(Memcmp {
@@ -1218,11 +1213,13 @@ impl JsonRpcRequestProcessor {
                 encoding: None,
             }));
         }
-        let keyed_accounts = get_filtered_program_accounts(&bank, &token_program_id, filters);
+
+        let keyed_accounts = self.get_filtered_spl_token_accounts_by_owner(&bank, owner, filters);
         let accounts = if encoding == UiAccountEncoding::JsonParsed {
-            get_parsed_token_accounts(bank.clone(), keyed_accounts).collect()
+            get_parsed_token_accounts(bank.clone(), keyed_accounts.into_iter()).collect()
         } else {
             keyed_accounts
+                .into_iter()
                 .map(|(pubkey, account)| RpcKeyedAccount {
                     pubkey: pubkey.to_string(),
                     account: UiAccount::encode(
@@ -1266,22 +1263,22 @@ impl JsonRpcRequestProcessor {
                 bytes: MemcmpEncodedBytes::Binary(delegate.to_string()),
                 encoding: None,
             }),
-            // Filter on Token Account state
-            RpcFilterType::DataSize(TokenAccount::get_packed_len() as u64),
         ];
-        if let Some(mint) = mint {
-            // Optional filter on Mint address
-            filters.push(RpcFilterType::Memcmp(Memcmp {
-                offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(mint.to_string()),
-                encoding: None,
-            }));
-        }
-        let keyed_accounts = get_filtered_program_accounts(&bank, &token_program_id, filters);
+        // Optional filter on Mint address, uses mint account index for scan
+        let keyed_accounts = if let Some(mint) = mint {
+            self.get_filtered_spl_token_accounts_by_mint(&bank, &mint, filters)
+        } else {
+            // Filter on Token Account state
+            filters.push(RpcFilterType::DataSize(
+                TokenAccount::get_packed_len() as u64
+            ));
+            self.get_filtered_program_accounts(&bank, &token_program_id, filters)
+        };
         let accounts = if encoding == UiAccountEncoding::JsonParsed {
-            get_parsed_token_accounts(bank.clone(), keyed_accounts).collect()
+            get_parsed_token_accounts(bank.clone(), keyed_accounts.into_iter()).collect()
         } else {
             keyed_accounts
+                .into_iter()
                 .map(|(pubkey, account)| RpcKeyedAccount {
                     pubkey: pubkey.to_string(),
                     account: UiAccount::encode(
@@ -1296,6 +1293,111 @@ impl JsonRpcRequestProcessor {
         };
         Ok(new_response(&bank, accounts))
     }
+
+    /// Use a set of filters to get an iterator of keyed program accounts from a bank
+    fn get_filtered_program_accounts(
+        &self,
+        bank: &Arc<Bank>,
+        program_id: &Pubkey,
+        filters: Vec<RpcFilterType>,
+    ) -> Vec<(Pubkey, Account)> {
+        let filter_closure = |account: &Account| {
+            filters.iter().all(|filter_type| match filter_type {
+                RpcFilterType::DataSize(size) => account.data.len() as u64 == *size,
+                RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data),
+            })
+        };
+        if self
+            .config
+            .account_indexes
+            .contains(&AccountIndex::ProgramId)
+        {
+            bank.get_filtered_indexed_accounts(&IndexKey::ProgramId(*program_id), |account| {
+                account.owner == *program_id && filter_closure(account)
+            })
+        } else {
+            bank.get_filtered_program_accounts(program_id, filter_closure)
+        }
+    }
+
+    /// Get an iterator of spl-token accounts by owner address
+    fn get_filtered_spl_token_accounts_by_owner(
+        &self,
+        bank: &Arc<Bank>,
+        owner_key: &Pubkey,
+        mut filters: Vec<RpcFilterType>,
+    ) -> Vec<(Pubkey, Account)> {
+        // The by-owner accounts index checks for Token Account state and Owner address on inclusion.
+        // However, due to the current AccountsDB implementation, accounts may remain in storage as
+        // be zero-lamport Account::Default() after being wiped and reinitialized in a later updates.
+        // We include the redundant filters here to avoid returning these accounts.
+        //
+        // Filter on Token Account state
+        filters.push(RpcFilterType::DataSize(
+            TokenAccount::get_packed_len() as u64
+        ));
+        // Filter on Owner address
+        filters.push(RpcFilterType::Memcmp(Memcmp {
+            offset: SPL_TOKEN_ACCOUNT_OWNER_OFFSET,
+            bytes: MemcmpEncodedBytes::Binary(owner_key.to_string()),
+            encoding: None,
+        }));
+
+        if self
+            .config
+            .account_indexes
+            .contains(&AccountIndex::SplTokenOwner)
+        {
+            bank.get_filtered_indexed_accounts(&IndexKey::SplTokenOwner(*owner_key), |account| {
+                account.owner == spl_token_id_v2_0()
+                    && filters.iter().all(|filter_type| match filter_type {
+                        RpcFilterType::DataSize(size) => account.data.len() as u64 == *size,
+                        RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data),
+                    })
+            })
+        } else {
+            self.get_filtered_program_accounts(bank, &spl_token_id_v2_0(), filters)
+        }
+    }
+
+    /// Get an iterator of spl-token accounts by mint address
+    fn get_filtered_spl_token_accounts_by_mint(
+        &self,
+        bank: &Arc<Bank>,
+        mint_key: &Pubkey,
+        mut filters: Vec<RpcFilterType>,
+    ) -> Vec<(Pubkey, Account)> {
+        // The by-mint accounts index checks for Token Account state and Mint address on inclusion.
+        // However, due to the current AccountsDB implementation, accounts may remain in storage as
+        // be zero-lamport Account::Default() after being wiped and reinitialized in a later updates.
+        // We include the redundant filters here to avoid returning these accounts.
+        //
+        // Filter on Token Account state
+        filters.push(RpcFilterType::DataSize(
+            TokenAccount::get_packed_len() as u64
+        ));
+        // Filter on Mint address
+        filters.push(RpcFilterType::Memcmp(Memcmp {
+            offset: SPL_TOKEN_ACCOUNT_MINT_OFFSET,
+            bytes: MemcmpEncodedBytes::Binary(mint_key.to_string()),
+            encoding: None,
+        }));
+        if self
+            .config
+            .account_indexes
+            .contains(&AccountIndex::SplTokenMint)
+        {
+            bank.get_filtered_indexed_accounts(&IndexKey::SplTokenMint(*mint_key), |account| {
+                account.owner == spl_token_id_v2_0()
+                    && filters.iter().all(|filter_type| match filter_type {
+                        RpcFilterType::DataSize(size) => account.data.len() as u64 == *size,
+                        RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data),
+                    })
+            })
+        } else {
+            self.get_filtered_program_accounts(bank, &spl_token_id_v2_0(), filters)
+        }
+    }
 }
 
 fn verify_transaction(transaction: &Transaction) -> Result<()> {
@@ -1394,20 +1496,32 @@ fn get_encoded_account(
     Ok(response)
 }
 
-/// Use a set of filters to get an iterator of keyed program accounts from a bank
-fn get_filtered_program_accounts(
-    bank: &Arc<Bank>,
-    program_id: &Pubkey,
-    filters: Vec<RpcFilterType>,
-) -> impl Iterator<Item = (Pubkey, Account)> {
-    bank.get_program_accounts(&program_id)
-        .into_iter()
-        .filter(move |(_, account)| {
-            filters.iter().all(|filter_type| match filter_type {
-                RpcFilterType::DataSize(size) => account.data.len() as u64 == *size,
-                RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data),
-            })
-        })
+fn get_spl_token_owner_filter(program_id: &Pubkey, filters: &[RpcFilterType]) -> Option<Pubkey> {
+    if program_id != &spl_token_id_v2_0() {
+        return None;
+    }
+    let mut data_size_filter: Option<u64> = None;
+    let mut owner_key: Option<Pubkey> = None;
+    for filter in filters {
+        match filter {
+            RpcFilterType::DataSize(size) => data_size_filter = Some(*size),
+            RpcFilterType::Memcmp(Memcmp {
+                offset: SPL_TOKEN_ACCOUNT_OWNER_OFFSET,
+                bytes: MemcmpEncodedBytes::Binary(bytes),
+                ..
+            }) => {
+                if let Ok(key) = Pubkey::from_str(bytes) {
+                    owner_key = Some(key)
+                }
+            }
+            _ => {}
+        }
+    }
+    if data_size_filter == Some(TokenAccount::get_packed_len() as u64) {
+        owner_key
+    } else {
+        None
+    }
 }
 
 pub(crate) fn get_parsed_token_account(
@@ -1872,12 +1986,18 @@ fn _send_transaction(
     transaction: Transaction,
     wire_transaction: Vec<u8>,
     last_valid_slot: Slot,
+    durable_nonce_info: Option<(Pubkey, Hash)>,
 ) -> Result<String> {
     if transaction.signatures.is_empty() {
         return Err(RpcCustomError::TransactionSignatureVerificationFailure.into());
     }
     let signature = transaction.signatures[0];
-    let transaction_info = TransactionInfo::new(signature, wire_transaction, last_valid_slot);
+    let transaction_info = TransactionInfo::new(
+        signature,
+        wire_transaction,
+        last_valid_slot,
+        durable_nonce_info,
+    );
     meta.transaction_sender
         .lock()
         .unwrap()
@@ -2306,7 +2426,7 @@ impl RpcSol for RpcSolImpl {
             Error::internal_error()
         })?;
 
-        _send_transaction(meta, transaction, wire_transaction, last_valid_slot)
+        _send_transaction(meta, transaction, wire_transaction, last_valid_slot, None)
     }
 
     fn send_transaction(
@@ -2319,11 +2439,29 @@ impl RpcSol for RpcSolImpl {
         let config = config.unwrap_or_default();
         let encoding = config.encoding.unwrap_or(UiTransactionEncoding::Base58);
         let (wire_transaction, transaction) = deserialize_transaction(data, encoding)?;
-        let bank = &*meta.bank(None);
-        let last_valid_slot = bank
+
+        let preflight_commitment = config
+            .preflight_commitment
+            .map(|commitment| CommitmentConfig { commitment });
+        let preflight_bank = &*meta.bank(preflight_commitment);
+
+        let mut last_valid_slot = preflight_bank
             .get_blockhash_last_valid_slot(&transaction.message.recent_blockhash)
             .unwrap_or(0);
 
+        let durable_nonce_info = solana_sdk::transaction::uses_durable_nonce(&transaction)
+            .and_then(|nonce_ix| {
+                solana_sdk::transaction::get_nonce_pubkey_from_instruction(&nonce_ix, &transaction)
+            })
+            .map(|&pubkey| (pubkey, transaction.message.recent_blockhash));
+        if durable_nonce_info.is_some() {
+            // While it uses a defined constant, this last_valid_slot value is chosen arbitrarily.
+            // It provides a fallback timeout for durable-nonce transaction retries in case of
+            // malicious packing of the retry queue. Durable-nonce transactions are otherwise
+            // retried until the nonce is advanced.
+            last_valid_slot = preflight_bank.slot() + MAX_RECENT_BLOCKHASHES as u64;
+        }
+
         if !config.skip_preflight {
             if let Err(e) = verify_transaction(&transaction) {
                 return Err(e);
@@ -2332,11 +2470,6 @@ impl RpcSol for RpcSolImpl {
             if meta.health.check() != RpcHealthStatus::Ok {
                 return Err(RpcCustomError::RpcNodeUnhealthy.into());
             }
-
-            let preflight_commitment = config
-                .preflight_commitment
-                .map(|commitment| CommitmentConfig { commitment });
-            let preflight_bank = &*meta.bank(preflight_commitment);
             if let (Err(err), logs) = preflight_bank.simulate_transaction(transaction.clone()) {
                 return Err(RpcCustomError::SendTransactionPreflightFailure {
                     message: format!("Transaction simulation failed: {}", err),
@@ -2349,7 +2482,13 @@ impl RpcSol for RpcSolImpl {
             }
         }
 
-        _send_transaction(meta, transaction, wire_transaction, last_valid_slot)
+        _send_transaction(
+            meta,
+            transaction,
+            wire_transaction,
+            last_valid_slot,
+            durable_nonce_info,
+        )
     }
 
     fn simulate_transaction(
@@ -2903,7 +3042,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         cluster_info.insert_info(ContactInfo::new_with_pubkey_socketaddr(
             &leader_pubkey,
@@ -4303,7 +4442,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         let mut bad_transaction = system_transaction::transfer(
             &mint_keypair,
@@ -4499,7 +4638,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
         assert_eq!(request_processor.validator_exit(), false);
         assert_eq!(exit.load(Ordering::Relaxed), false);
     }
@@ -4531,7 +4670,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
         assert_eq!(request_processor.validator_exit(), true);
         assert_eq!(exit.load(Ordering::Relaxed), true);
     }
@@ -4622,7 +4761,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
         assert_eq!(
             request_processor.get_block_commitment(0),
             RpcBlockCommitment {
@@ -5755,6 +5894,54 @@ pub mod tests {
         );
     }
 
+    #[test]
+    fn test_get_spl_token_owner_filter() {
+        let owner = Pubkey::new_unique();
+        assert_eq!(
+            get_spl_token_owner_filter(
+                &Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(),
+                &[
+                    RpcFilterType::Memcmp(Memcmp {
+                        offset: 32,
+                        bytes: MemcmpEncodedBytes::Binary(owner.to_string()),
+                        encoding: None
+                    }),
+                    RpcFilterType::DataSize(165)
+                ],
+            )
+            .unwrap(),
+            owner
+        );
+
+        // Filtering on mint instead of owner
+        assert!(get_spl_token_owner_filter(
+            &Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(),
+            &[
+                RpcFilterType::Memcmp(Memcmp {
+                    offset: 0,
+                    bytes: MemcmpEncodedBytes::Binary(owner.to_string()),
+                    encoding: None
+                }),
+                RpcFilterType::DataSize(165)
+            ],
+        )
+        .is_none());
+
+        // Wrong program id
+        assert!(get_spl_token_owner_filter(
+            &Pubkey::new_unique(),
+            &[
+                RpcFilterType::Memcmp(Memcmp {
+                    offset: 32,
+                    bytes: MemcmpEncodedBytes::Binary(owner.to_string()),
+                    encoding: None
+                }),
+                RpcFilterType::DataSize(165)
+            ],
+        )
+        .is_none());
+    }
+
     #[test]
     fn test_rpc_single_gossip() {
         let exit = Arc::new(AtomicBool::new(false));

core/src/rpc_service.rs

@@ -17,6 +17,7 @@ use jsonrpc_http_server::{
 };
 use regex::Regex;
 use solana_ledger::blockstore::Blockstore;
+use solana_metrics::inc_new_counter_info;
 use solana_runtime::{
     bank_forks::{BankForks, SnapshotConfig},
     commitment::BlockCommitmentCache,
@@ -61,7 +62,7 @@ impl RpcRequestMiddleware {
         Self {
             ledger_path,
             snapshot_archive_path_regex: Regex::new(
-                r"/snapshot-\d+-[[:alnum:]]+\.tar\.(bz2|zst|gz)$",
+                r"/snapshot-\d+-[[:alnum:]]+\.(tar|tar\.bz2|tar\.zst|tar\.gz)$",
             )
             .unwrap(),
             snapshot_config,
@@ -85,6 +86,7 @@ impl RpcRequestMiddleware {
             .unwrap()
     }
 
+    #[allow(dead_code)]
     fn internal_server_error() -> hyper::Response<hyper::Body> {
         hyper::Response::builder()
             .status(hyper::StatusCode::INTERNAL_SERVER_ERROR)
@@ -112,27 +114,42 @@ impl RpcRequestMiddleware {
         let stem = path.split_at(1).1; // Drop leading '/' from path
         let filename = {
             match path {
-                "/genesis.tar.bz2" => self.ledger_path.join(stem),
-                _ => self
-                    .snapshot_config
-                    .as_ref()
-                    .unwrap()
-                    .snapshot_package_output_path
-                    .join(stem),
+                "/genesis.tar.bz2" => {
+                    inc_new_counter_info!("rpc-get_genesis", 1);
+                    self.ledger_path.join(stem)
+                }
+                _ => {
+                    inc_new_counter_info!("rpc-get_snapshot", 1);
+                    self.snapshot_config
+                        .as_ref()
+                        .unwrap()
+                        .snapshot_package_output_path
+                        .join(stem)
+                }
             }
         };
 
-        info!("get {} -> {:?}", path, filename);
+        let file_length = std::fs::metadata(&filename)
+            .map(|m| m.len())
+            .unwrap_or(0)
+            .to_string();
+        info!("get {} -> {:?} ({} bytes)", path, filename, file_length);
 
         RequestMiddlewareAction::Respond {
             should_validate_hosts: true,
             response: Box::new(
                 tokio_fs_01::file::File::open(filename)
                     .and_then(|file| {
-                        let buf: Vec<u8> = Vec::new();
-                        tokio_io_01::io::read_to_end(file, buf)
-                            .and_then(|item| Ok(hyper::Response::new(item.1.into())))
-                            .or_else(|_| Ok(RpcRequestMiddleware::internal_server_error()))
+                        use tokio_codec_01::{BytesCodec, FramedRead};
+
+                        let stream = FramedRead::new(file, BytesCodec::new())
+                            .map(tokio_01_bytes::BytesMut::freeze);
+                        let body = hyper::Body::wrap_stream(stream);
+
+                        Ok(hyper::Response::builder()
+                            .header(hyper::header::CONTENT_LENGTH, file_length)
+                            .body(body)
+                            .unwrap())
                     })
                     .or_else(|_| Ok(RpcRequestMiddleware::not_found())),
             ),
@@ -251,6 +268,8 @@ impl JsonRpcService {
         trusted_validators: Option<HashSet<Pubkey>>,
         override_health_check: Arc<AtomicBool>,
         optimistically_confirmed_bank: Arc<RwLock<OptimisticallyConfirmedBank>>,
+        send_transaction_retry_ms: u64,
+        send_transaction_leader_forward_count: u64,
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
@@ -281,17 +300,22 @@ impl JsonRpcService {
                     .map(|bigtable_ledger_storage| {
                         info!("BigTable ledger storage initialized");
 
-                        let bigtable_ledger_upload_service = Arc::new(BigTableUploadService::new(
-                            runtime.handle().clone(),
-                            bigtable_ledger_storage.clone(),
-                            blockstore.clone(),
-                            block_commitment_cache.clone(),
-                            exit_bigtable_ledger_upload_service.clone(),
-                        ));
+                        let bigtable_ledger_upload_service = if config.enable_bigtable_ledger_upload
+                        {
+                            Some(Arc::new(BigTableUploadService::new(
+                                runtime.handle().clone(),
+                                bigtable_ledger_storage.clone(),
+                                blockstore.clone(),
+                                block_commitment_cache.clone(),
+                                exit_bigtable_ledger_upload_service.clone(),
+                            )))
+                        } else {
+                            None
+                        };
 
                         (
                             Some(bigtable_ledger_storage),
-                            Some(bigtable_ledger_upload_service),
+                            bigtable_ledger_upload_service,
                         )
                     })
                     .unwrap_or_else(|err| {
@@ -323,6 +347,8 @@ impl JsonRpcService {
             &bank_forks,
             leader_info,
             receiver,
+            send_transaction_retry_ms,
+            send_transaction_leader_forward_count,
         ));
 
         #[cfg(test)]
@@ -456,6 +482,8 @@ mod tests {
             None,
             Arc::new(AtomicBool::new(false)),
             optimistically_confirmed_bank,
+            1000,
+            1,
         );
         let thread = rpc_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
@@ -524,6 +552,13 @@ mod tests {
         assert!(rrm_with_snapshot_config.is_file_get_path(
             "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
         ));
+        assert!(rrm_with_snapshot_config.is_file_get_path(
+            "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst"
+        ));
+        assert!(rrm_with_snapshot_config
+            .is_file_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.gz"));
+        assert!(rrm_with_snapshot_config
+            .is_file_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar"));
 
         assert!(!rrm.is_file_get_path(
             "/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"

core/src/sample_performance_service.rs

@@ -49,7 +49,7 @@ impl SamplePerformanceService {
         exit: Arc<AtomicBool>,
     ) {
         let forks = bank_forks.read().unwrap();
-        let bank = forks.root_bank().clone();
+        let bank = forks.root_bank();
         let highest_slot = forks.highest_slot();
         drop(forks);
 

core/src/send_transaction_service.rs

@@ -4,7 +4,13 @@ use crate::poh_recorder::PohRecorder;
 use log::*;
 use solana_metrics::{datapoint_warn, inc_new_counter_info};
 use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature};
+use solana_sdk::{
+    clock::{Slot, NUM_CONSECUTIVE_LEADER_SLOTS},
+    hash::Hash,
+    nonce_account,
+    pubkey::Pubkey,
+    signature::Signature,
+};
 use std::sync::Mutex;
 use std::{
     collections::HashMap,
@@ -28,14 +34,21 @@ pub struct TransactionInfo {
     pub signature: Signature,
     pub wire_transaction: Vec<u8>,
     pub last_valid_slot: Slot,
+    pub durable_nonce_info: Option<(Pubkey, Hash)>,
 }
 
 impl TransactionInfo {
-    pub fn new(signature: Signature, wire_transaction: Vec<u8>, last_valid_slot: Slot) -> Self {
+    pub fn new(
+        signature: Signature,
+        wire_transaction: Vec<u8>,
+        last_valid_slot: Slot,
+        durable_nonce_info: Option<(Pubkey, Hash)>,
+    ) -> Self {
         Self {
             signature,
             wire_transaction,
             last_valid_slot,
+            durable_nonce_info,
         }
     }
 }
@@ -64,12 +77,21 @@ impl LeaderInfo {
             .collect();
     }
 
-    pub fn get_leader_tpu(&self) -> Option<&SocketAddr> {
-        self.poh_recorder
-            .lock()
-            .unwrap()
-            .leader_after_n_slots(0)
-            .and_then(|leader| self.recent_peers.get(&leader))
+    pub fn get_leader_tpus(&self, max_count: u64) -> Vec<&SocketAddr> {
+        let recorder = self.poh_recorder.lock().unwrap();
+        let leaders: Vec<_> = (0..max_count)
+            .filter_map(|i| recorder.leader_after_n_slots(i * NUM_CONSECUTIVE_LEADER_SLOTS))
+            .collect();
+        drop(recorder);
+        let mut unique_leaders = vec![];
+        for leader in leaders.iter() {
+            if let Some(addr) = self.recent_peers.get(leader) {
+                if !unique_leaders.contains(&addr) {
+                    unique_leaders.push(addr);
+                }
+            }
+        }
+        unique_leaders
     }
 }
 
@@ -88,8 +110,17 @@ impl SendTransactionService {
         bank_forks: &Arc<RwLock<BankForks>>,
         leader_info: Option<LeaderInfo>,
         receiver: Receiver<TransactionInfo>,
+        retry_rate_ms: u64,
+        leader_forward_count: u64,
     ) -> Self {
-        let thread = Self::retry_thread(tpu_address, receiver, bank_forks.clone(), leader_info);
+        let thread = Self::retry_thread(
+            tpu_address,
+            receiver,
+            bank_forks.clone(),
+            leader_info,
+            retry_rate_ms,
+            leader_forward_count,
+        );
         Self { thread }
     }
 
@@ -98,8 +129,11 @@ impl SendTransactionService {
         receiver: Receiver<TransactionInfo>,
         bank_forks: Arc<RwLock<BankForks>>,
         mut leader_info: Option<LeaderInfo>,
+        retry_rate_ms: u64,
+        leader_forward_count: u64,
     ) -> JoinHandle<()> {
         let mut last_status_check = Instant::now();
+        let mut last_leader_refresh = Instant::now();
         let mut transactions = HashMap::new();
         let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
 
@@ -110,19 +144,29 @@ impl SendTransactionService {
         Builder::new()
             .name("send-tx-sv2".to_string())
             .spawn(move || loop {
-                match receiver.recv_timeout(Duration::from_secs(1)) {
+                match receiver.recv_timeout(Duration::from_millis(1000.min(retry_rate_ms))) {
                     Err(RecvTimeoutError::Disconnected) => break,
                     Err(RecvTimeoutError::Timeout) => {}
                     Ok(transaction_info) => {
-                        let address = leader_info
+                        let addresses = leader_info
                             .as_ref()
-                            .and_then(|leader_info| leader_info.get_leader_tpu())
-                            .unwrap_or(&tpu_address);
-                        Self::send_transaction(
-                            &send_socket,
-                            address,
-                            &transaction_info.wire_transaction,
-                        );
+                            .map(|leader_info| leader_info.get_leader_tpus(leader_forward_count));
+                        let addresses = addresses
+                            .map(|address_list| {
+                                if address_list.is_empty() {
+                                    vec![&tpu_address]
+                                } else {
+                                    address_list
+                                }
+                            })
+                            .unwrap_or_else(|| vec![&tpu_address]);
+                        for address in addresses {
+                            Self::send_transaction(
+                                &send_socket,
+                                address,
+                                &transaction_info.wire_transaction,
+                            );
+                        }
                         if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
                             transactions.insert(transaction_info.signature, transaction_info);
                         } else {
@@ -131,15 +175,19 @@ impl SendTransactionService {
                     }
                 }
 
-                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
+                if last_status_check.elapsed().as_millis() as u64 >= retry_rate_ms {
                     if !transactions.is_empty() {
                         datapoint_info!(
                             "send_transaction_service-queue-size",
                             ("len", transactions.len(), i64)
                         );
-                        let bank_forks = bank_forks.read().unwrap();
-                        let root_bank = bank_forks.root_bank();
-                        let working_bank = bank_forks.working_bank();
+                        let (root_bank, working_bank) = {
+                            let bank_forks = bank_forks.read().unwrap();
+                            (
+                                bank_forks.root_bank().clone(),
+                                bank_forks.working_bank().clone(),
+                            )
+                        };
 
                         let _result = Self::process_transactions(
                             &working_bank,
@@ -151,8 +199,11 @@ impl SendTransactionService {
                         );
                     }
                     last_status_check = Instant::now();
-                    if let Some(leader_info) = leader_info.as_mut() {
-                        leader_info.refresh_recent_peers();
+                    if last_leader_refresh.elapsed().as_millis() > 1000 {
+                        if let Some(leader_info) = leader_info.as_mut() {
+                            leader_info.refresh_recent_peers();
+                        }
+                        last_leader_refresh = Instant::now();
                     }
                 }
             })
@@ -170,44 +221,68 @@ impl SendTransactionService {
         let mut result = ProcessTransactionsResult::default();
 
         transactions.retain(|signature, transaction_info| {
+            if transaction_info.durable_nonce_info.is_some() {
+                inc_new_counter_info!("send_transaction_service-nonced", 1);
+            }
             if root_bank.has_signature(signature) {
                 info!("Transaction is rooted: {}", signature);
                 result.rooted += 1;
                 inc_new_counter_info!("send_transaction_service-rooted", 1);
-                false
-            } else if transaction_info.last_valid_slot < root_bank.slot() {
+                return false;
+            }
+            if let Some((nonce_pubkey, durable_nonce)) = transaction_info.durable_nonce_info {
+                let nonce_account = working_bank.get_account(&nonce_pubkey).unwrap_or_default();
+                if !nonce_account::verify_nonce_account(&nonce_account, &durable_nonce)
+                    && working_bank.get_signature_status_slot(signature).is_none()
+                {
+                    info!("Dropping expired durable-nonce transaction: {}", signature);
+                    result.expired += 1;
+                    inc_new_counter_info!("send_transaction_service-expired", 1);
+                    return false;
+                }
+            }
+            if transaction_info.last_valid_slot < root_bank.slot() {
                 info!("Dropping expired transaction: {}", signature);
                 result.expired += 1;
                 inc_new_counter_info!("send_transaction_service-expired", 1);
-                false
-            } else {
-                match working_bank.get_signature_status_slot(signature) {
-                    None => {
-                        // Transaction is unknown to the working bank, it might have been
-                        // dropped or landed in another fork.  Re-send it
-                        info!("Retrying transaction: {}", signature);
-                        result.retried += 1;
-                        inc_new_counter_info!("send_transaction_service-retry", 1);
-                        Self::send_transaction(
-                            &send_socket,
-                            leader_info
-                                .as_ref()
-                                .and_then(|leader_info| leader_info.get_leader_tpu())
-                                .unwrap_or(&tpu_address),
-                            &transaction_info.wire_transaction,
-                        );
-                        true
-                    }
-                    Some((_slot, status)) => {
-                        if status.is_err() {
-                            info!("Dropping failed transaction: {}", signature);
-                            result.failed += 1;
-                            inc_new_counter_info!("send_transaction_service-failed", 1);
-                            false
+                return false;
+            }
+
+            match working_bank.get_signature_status_slot(signature) {
+                None => {
+                    // Transaction is unknown to the working bank, it might have been
+                    // dropped or landed in another fork.  Re-send it
+                    info!("Retrying transaction: {}", signature);
+                    result.retried += 1;
+                    inc_new_counter_info!("send_transaction_service-retry", 1);
+                    let leaders = leader_info
+                        .as_ref()
+                        .map(|leader_info| leader_info.get_leader_tpus(1));
+                    let leader = if let Some(leaders) = leaders {
+                        if leaders.is_empty() {
+                            &tpu_address
                         } else {
-                            result.retained += 1;
-                            true
+                            leaders[0]
                         }
+                    } else {
+                        &tpu_address
+                    };
+                    Self::send_transaction(
+                        &send_socket,
+                        leader,
+                        &transaction_info.wire_transaction,
+                    );
+                    true
+                }
+                Some((_slot, status)) => {
+                    if status.is_err() {
+                        info!("Dropping failed transaction: {}", signature);
+                        result.failed += 1;
+                        inc_new_counter_info!("send_transaction_service-failed", 1);
+                        false
+                    } else {
+                        result.retained += 1;
+                        true
                     }
                 }
             }
@@ -234,9 +309,23 @@ impl SendTransactionService {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::contact_info::ContactInfo;
+    use solana_ledger::{
+        blockstore::Blockstore, get_tmp_ledger_path, leader_schedule_cache::LeaderScheduleCache,
+    };
+    use solana_runtime::genesis_utils::{
+        create_genesis_config_with_vote_accounts, GenesisConfigInfo, ValidatorVoteKeypairs,
+    };
     use solana_sdk::{
-        genesis_config::create_genesis_config, pubkey::Pubkey, signature::Signer,
-        system_transaction,
+        account::Account,
+        fee_calculator::FeeCalculator,
+        genesis_config::create_genesis_config,
+        nonce,
+        poh_config::PohConfig,
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
+        system_program, system_transaction,
+        timing::timestamp,
     };
     use std::sync::mpsc::channel;
 
@@ -248,7 +337,7 @@ mod test {
         let (sender, receiver) = channel();
 
         let send_tranaction_service =
-            SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+            SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         drop(sender);
         send_tranaction_service.join().unwrap();
@@ -290,10 +379,10 @@ mod test {
 
         let mut transactions = HashMap::new();
 
-        info!("Expired transactions are dropped..");
+        info!("Expired transactions are dropped...");
         transactions.insert(
             Signature::default(),
-            TransactionInfo::new(Signature::default(), vec![], root_bank.slot() - 1),
+            TransactionInfo::new(Signature::default(), vec![], root_bank.slot() - 1, None),
         );
         let result = SendTransactionService::process_transactions(
             &working_bank,
@@ -315,7 +404,7 @@ mod test {
         info!("Rooted transactions are dropped...");
         transactions.insert(
             rooted_signature,
-            TransactionInfo::new(rooted_signature, vec![], working_bank.slot()),
+            TransactionInfo::new(rooted_signature, vec![], working_bank.slot(), None),
         );
         let result = SendTransactionService::process_transactions(
             &working_bank,
@@ -337,7 +426,7 @@ mod test {
         info!("Failed transactions are dropped...");
         transactions.insert(
             failed_signature,
-            TransactionInfo::new(failed_signature, vec![], working_bank.slot()),
+            TransactionInfo::new(failed_signature, vec![], working_bank.slot(), None),
         );
         let result = SendTransactionService::process_transactions(
             &working_bank,
@@ -359,7 +448,7 @@ mod test {
         info!("Non-rooted transactions are kept...");
         transactions.insert(
             non_rooted_signature,
-            TransactionInfo::new(non_rooted_signature, vec![], working_bank.slot()),
+            TransactionInfo::new(non_rooted_signature, vec![], working_bank.slot(), None),
         );
         let result = SendTransactionService::process_transactions(
             &working_bank,
@@ -382,7 +471,7 @@ mod test {
         info!("Unknown transactions are retried...");
         transactions.insert(
             Signature::default(),
-            TransactionInfo::new(Signature::default(), vec![], working_bank.slot()),
+            TransactionInfo::new(Signature::default(), vec![], working_bank.slot(), None),
         );
         let result = SendTransactionService::process_transactions(
             &working_bank,
@@ -401,4 +490,377 @@ mod test {
             }
         );
     }
+
+    #[test]
+    fn test_retry_durable_nonce_transactions() {
+        solana_logger::setup();
+
+        let (genesis_config, mint_keypair) = create_genesis_config(4);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let tpu_address = "127.0.0.1:0".parse().unwrap();
+
+        let root_bank = Arc::new(Bank::new_from_parent(
+            &bank_forks.read().unwrap().working_bank(),
+            &Pubkey::default(),
+            1,
+        ));
+        let rooted_signature = root_bank
+            .transfer(1, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let nonce_address = Pubkey::new_unique();
+        let durable_nonce = Hash::new_unique();
+        let nonce_state =
+            nonce::state::Versions::new_current(nonce::State::Initialized(nonce::state::Data {
+                authority: Pubkey::default(),
+                blockhash: durable_nonce,
+                fee_calculator: FeeCalculator::new(42),
+            }));
+        let nonce_account = Account::new_data(43, &nonce_state, &system_program::id()).unwrap();
+        root_bank.store_account(&nonce_address, &nonce_account);
+
+        let working_bank = Arc::new(Bank::new_from_parent(&root_bank, &Pubkey::default(), 2));
+        let non_rooted_signature = working_bank
+            .transfer(2, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let last_valid_slot = working_bank.slot() + 300;
+
+        let failed_signature = {
+            let blockhash = working_bank.last_blockhash();
+            let transaction =
+                system_transaction::transfer(&mint_keypair, &Pubkey::default(), 1, blockhash);
+            let signature = transaction.signatures[0];
+            working_bank.process_transaction(&transaction).unwrap_err();
+            signature
+        };
+
+        let mut transactions = HashMap::new();
+
+        info!("Rooted durable-nonce transactions are dropped...");
+        transactions.insert(
+            rooted_signature,
+            TransactionInfo::new(
+                rooted_signature,
+                vec![],
+                last_valid_slot,
+                Some((nonce_address, durable_nonce)),
+            ),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                rooted: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        // Nonce expired case
+        transactions.insert(
+            rooted_signature,
+            TransactionInfo::new(
+                rooted_signature,
+                vec![],
+                last_valid_slot,
+                Some((nonce_address, Hash::new_unique())),
+            ),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                rooted: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        // Expired durable-nonce transactions are dropped; nonce has advanced...
+        info!("Expired durable-nonce transactions are dropped...");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo::new(
+                Signature::default(),
+                vec![],
+                last_valid_slot,
+                Some((nonce_address, Hash::new_unique())),
+            ),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                expired: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        // ... or last_valid_slot timeout has passed
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo::new(
+                Signature::default(),
+                vec![],
+                root_bank.slot() - 1,
+                Some((nonce_address, durable_nonce)),
+            ),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                expired: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Failed durable-nonce transactions are dropped...");
+        transactions.insert(
+            failed_signature,
+            TransactionInfo::new(
+                failed_signature,
+                vec![],
+                last_valid_slot,
+                Some((nonce_address, Hash::new_unique())), // runtime should advance nonce on failed transactions
+            ),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                failed: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Non-rooted durable-nonce transactions are kept...");
+        transactions.insert(
+            non_rooted_signature,
+            TransactionInfo::new(
+                non_rooted_signature,
+                vec![],
+                last_valid_slot,
+                Some((nonce_address, Hash::new_unique())), // runtime advances nonce when transaction lands
+            ),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retained: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        transactions.clear();
+
+        info!("Unknown durable-nonce transactions are retried until nonce advances...");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo::new(
+                Signature::default(),
+                vec![],
+                last_valid_slot,
+                Some((nonce_address, durable_nonce)),
+            ),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retried: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        // Advance nonce
+        let new_durable_nonce = Hash::new_unique();
+        let new_nonce_state =
+            nonce::state::Versions::new_current(nonce::State::Initialized(nonce::state::Data {
+                authority: Pubkey::default(),
+                blockhash: new_durable_nonce,
+                fee_calculator: FeeCalculator::new(42),
+            }));
+        let nonce_account = Account::new_data(43, &new_nonce_state, &system_program::id()).unwrap();
+        working_bank.store_account(&nonce_address, &nonce_account);
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+            &None,
+        );
+        assert_eq!(transactions.len(), 0);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                expired: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+    }
+
+    #[test]
+    fn test_get_leader_tpus() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&ledger_path).unwrap();
+
+            let validator_vote_keypairs0 = ValidatorVoteKeypairs::new_rand();
+            let validator_vote_keypairs1 = ValidatorVoteKeypairs::new_rand();
+            let validator_vote_keypairs2 = ValidatorVoteKeypairs::new_rand();
+            let validator_keypairs = vec![
+                &validator_vote_keypairs0,
+                &validator_vote_keypairs1,
+                &validator_vote_keypairs2,
+            ];
+            let GenesisConfigInfo {
+                genesis_config,
+                mint_keypair: _,
+                voting_keypair: _,
+            } = create_genesis_config_with_vote_accounts(
+                1_000_000_000,
+                &validator_keypairs,
+                vec![10_000; 3],
+            );
+            let bank = Arc::new(Bank::new(&genesis_config));
+
+            let (poh_recorder, _entry_receiver) = PohRecorder::new(
+                0,
+                bank.last_blockhash(),
+                0,
+                Some((2, 2)),
+                bank.ticks_per_slot(),
+                &Pubkey::default(),
+                &Arc::new(blockstore),
+                &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
+                &Arc::new(PohConfig::default()),
+            );
+
+            let node_keypair = Arc::new(Keypair::new());
+            let cluster_info = Arc::new(ClusterInfo::new(
+                ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
+                node_keypair,
+            ));
+
+            let validator0_socket = SocketAddr::from(([127, 0, 0, 1], 1111));
+            let validator1_socket = SocketAddr::from(([127, 0, 0, 1], 2222));
+            let validator2_socket = SocketAddr::from(([127, 0, 0, 1], 3333));
+            let recent_peers: HashMap<_, _> = vec![
+                (
+                    validator_vote_keypairs0.node_keypair.pubkey(),
+                    validator0_socket,
+                ),
+                (
+                    validator_vote_keypairs1.node_keypair.pubkey(),
+                    validator1_socket,
+                ),
+                (
+                    validator_vote_keypairs2.node_keypair.pubkey(),
+                    validator2_socket,
+                ),
+            ]
+            .iter()
+            .cloned()
+            .collect();
+            let leader_info = LeaderInfo {
+                cluster_info,
+                poh_recorder: Arc::new(Mutex::new(poh_recorder)),
+                recent_peers: recent_peers.clone(),
+            };
+
+            let slot = bank.slot();
+            let first_leader =
+                solana_ledger::leader_schedule_utils::slot_leader_at(slot, &bank).unwrap();
+            assert_eq!(
+                leader_info.get_leader_tpus(1),
+                vec![recent_peers.get(&first_leader).unwrap()]
+            );
+
+            let second_leader = solana_ledger::leader_schedule_utils::slot_leader_at(
+                slot + NUM_CONSECUTIVE_LEADER_SLOTS,
+                &bank,
+            )
+            .unwrap();
+            let mut expected_leader_sockets = vec![
+                recent_peers.get(&first_leader).unwrap(),
+                recent_peers.get(&second_leader).unwrap(),
+            ];
+            expected_leader_sockets.dedup();
+            assert_eq!(leader_info.get_leader_tpus(2), expected_leader_sockets);
+
+            let third_leader = solana_ledger::leader_schedule_utils::slot_leader_at(
+                slot + (2 * NUM_CONSECUTIVE_LEADER_SLOTS),
+                &bank,
+            )
+            .unwrap();
+            let mut expected_leader_sockets = vec![
+                recent_peers.get(&first_leader).unwrap(),
+                recent_peers.get(&second_leader).unwrap(),
+                recent_peers.get(&third_leader).unwrap(),
+            ];
+            expected_leader_sockets.dedup();
+            assert_eq!(leader_info.get_leader_tpus(3), expected_leader_sockets);
+
+            for x in 4..8 {
+                assert!(leader_info.get_leader_tpus(x).len() <= recent_peers.len());
+            }
+        }
+        Blockstore::destroy(&ledger_path).unwrap();
+    }
 }

core/src/shred_fetch_stage.rs

@@ -1,17 +1,10 @@
 //! The `shred_fetch_stage` pulls shreds from UDP sockets and sends it to a channel.
 
-use ahash::AHasher;
+use crate::packet_hasher::PacketHasher;
 use lru::LruCache;
-use rand::{thread_rng, Rng};
-use std::hash::Hasher;
-
-use solana_ledger::blockstore::MAX_DATA_SHREDS_PER_SLOT;
-use solana_ledger::shred::{
-    CODING_SHRED, DATA_SHRED, OFFSET_OF_SHRED_INDEX, OFFSET_OF_SHRED_SLOT, OFFSET_OF_SHRED_TYPE,
-    SIZE_OF_SHRED_INDEX, SIZE_OF_SHRED_SLOT,
-};
+use solana_ledger::shred::{get_shred_slot_index_type, ShredFetchStats};
 use solana_perf::cuda_runtime::PinnedVec;
-use solana_perf::packet::{limited_deserialize, Packet, PacketsRecycler};
+use solana_perf::packet::{Packet, PacketsRecycler};
 use solana_perf::recycler::Recycler;
 use solana_runtime::bank_forks::BankForks;
 use solana_sdk::clock::{Slot, DEFAULT_MS_PER_SLOT};
@@ -27,48 +20,11 @@ use std::time::Instant;
 const DEFAULT_LRU_SIZE: usize = 10_000;
 pub type ShredsReceived = LruCache<u64, ()>;
 
-#[derive(Default)]
-pub struct ShredFetchStats {
-    index_overrun: usize,
-    shred_count: usize,
-    index_bad_deserialize: usize,
-    index_out_of_bounds: usize,
-    slot_bad_deserialize: usize,
-    duplicate_shred: usize,
-    slot_out_of_range: usize,
-}
-
 pub struct ShredFetchStage {
     thread_hdls: Vec<JoinHandle<()>>,
 }
 
 impl ShredFetchStage {
-    pub fn get_slot_index(p: &Packet, stats: &mut ShredFetchStats) -> Option<(u64, u32)> {
-        let index_start = OFFSET_OF_SHRED_INDEX;
-        let index_end = index_start + SIZE_OF_SHRED_INDEX;
-        let slot_start = OFFSET_OF_SHRED_SLOT;
-        let slot_end = slot_start + SIZE_OF_SHRED_SLOT;
-
-        if index_end <= p.meta.size {
-            if let Ok(index) = limited_deserialize::<u32>(&p.data[index_start..index_end]) {
-                if index < MAX_DATA_SHREDS_PER_SLOT as u32 && slot_end <= p.meta.size {
-                    if let Ok(slot) = limited_deserialize::<Slot>(&p.data[slot_start..slot_end]) {
-                        return Some((slot, index));
-                    } else {
-                        stats.slot_bad_deserialize += 1;
-                    }
-                } else {
-                    stats.index_out_of_bounds += 1;
-                }
-            } else {
-                stats.index_bad_deserialize += 1;
-            }
-        } else {
-            stats.index_overrun += 1;
-        }
-        None
-    }
-
     fn process_packet<F>(
         p: &mut Packet,
         shreds_received: &mut ShredsReceived,
@@ -77,32 +33,24 @@ impl ShredFetchStage {
         last_slot: Slot,
         slots_per_epoch: u64,
         modify: &F,
-        seeds: (u128, u128),
+        packet_hasher: &PacketHasher,
     ) where
         F: Fn(&mut Packet),
     {
         p.meta.discard = true;
-        if let Some((slot, _index)) = Self::get_slot_index(p, stats) {
+        if let Some((slot, _index, _shred_type)) = get_shred_slot_index_type(p, stats) {
             // Seems reasonable to limit shreds to 2 epochs away
-            if slot > last_root
-                && slot < (last_slot + 2 * slots_per_epoch)
-                && p.meta.size > OFFSET_OF_SHRED_TYPE
-            {
-                let shred_type = p.data[OFFSET_OF_SHRED_TYPE];
-                if shred_type == DATA_SHRED || shred_type == CODING_SHRED {
-                    // Shred filter
+            if slot > last_root && slot < (last_slot + 2 * slots_per_epoch) {
+                // Shred filter
 
-                    let mut hasher = AHasher::new_with_keys(seeds.0, seeds.1);
-                    hasher.write(&p.data[0..p.meta.size]);
-                    let hash = hasher.finish();
+                let hash = packet_hasher.hash_packet(p);
 
-                    if shreds_received.get(&hash).is_none() {
-                        shreds_received.put(hash, ());
-                        p.meta.discard = false;
-                        modify(p);
-                    } else {
-                        stats.duplicate_shred += 1;
-                    }
+                if shreds_received.get(&hash).is_none() {
+                    shreds_received.put(hash, ());
+                    p.meta.discard = false;
+                    modify(p);
+                } else {
+                    stats.duplicate_shred += 1;
                 }
             } else {
                 stats.slot_out_of_range += 1;
@@ -130,12 +78,12 @@ impl ShredFetchStage {
 
         let mut last_stats = Instant::now();
         let mut stats = ShredFetchStats::default();
-        let mut seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+        let mut packet_hasher = PacketHasher::default();
 
         while let Some(mut p) = recvr.iter().next() {
             if last_updated.elapsed().as_millis() as u64 > DEFAULT_MS_PER_SLOT {
                 last_updated = Instant::now();
-                seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+                packet_hasher.reset();
                 shreds_received.clear();
                 if let Some(bank_forks) = bank_forks.as_ref() {
                     let bank_forks_r = bank_forks.read().unwrap();
@@ -156,7 +104,7 @@ impl ShredFetchStage {
                     last_slot,
                     slots_per_epoch,
                     &modify,
-                    seeds,
+                    &packet_hasher,
                 );
             });
             if last_stats.elapsed().as_millis() > 1000 {
@@ -274,6 +222,7 @@ impl ShredFetchStage {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use solana_ledger::blockstore::MAX_DATA_SHREDS_PER_SLOT;
     use solana_ledger::shred::Shred;
 
     #[test]
@@ -287,7 +236,7 @@ mod tests {
         let shred = Shred::new_from_data(slot, 3, 0, None, true, true, 0, 0, 0);
         shred.copy_to_packet(&mut packet);
 
-        let seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+        let hasher = PacketHasher::default();
 
         let last_root = 0;
         let last_slot = 100;
@@ -300,7 +249,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(!packet.meta.discard);
 
@@ -315,7 +264,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(!packet.meta.discard);
     }
@@ -329,7 +278,9 @@ mod tests {
         let last_root = 0;
         let last_slot = 100;
         let slots_per_epoch = 10;
-        let seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+
+        let hasher = PacketHasher::default();
+
         // packet size is 0, so cannot get index
         ShredFetchStage::process_packet(
             &mut packet,
@@ -339,7 +290,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert_eq!(stats.index_overrun, 1);
         assert!(packet.meta.discard);
@@ -355,7 +306,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
 
@@ -368,7 +319,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(!packet.meta.discard);
 
@@ -381,7 +332,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
 
@@ -397,7 +348,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
 
@@ -412,20 +363,8 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
     }
-
-    #[test]
-    fn test_shred_offsets() {
-        let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
-        let mut packet = Packet::default();
-        shred.copy_to_packet(&mut packet);
-        let mut stats = ShredFetchStats::default();
-        assert_eq!(
-            Some((1, 3)),
-            ShredFetchStage::get_slot_index(&packet, &mut stats)
-        );
-    }
 }

core/src/test_validator.rs

@@ -24,8 +24,13 @@ use {
         signature::{read_keypair_file, write_keypair_file, Keypair, Signer},
     },
     std::{
-        collections::HashMap, fs::remove_dir_all, net::SocketAddr, path::PathBuf, sync::Arc,
-        thread::sleep, time::Duration,
+        collections::HashMap,
+        fs::remove_dir_all,
+        net::{IpAddr, Ipv4Addr, SocketAddr},
+        path::PathBuf,
+        sync::Arc,
+        thread::sleep,
+        time::Duration,
     },
 };
 
@@ -328,13 +333,19 @@ impl TestValidator {
         }
 
         let vote_account_address = validator_vote_account.pubkey();
-        let rpc_url = format!("http://{}:{}", node.info.rpc.ip(), node.info.rpc.port());
+        let rpc_url = format!("http://{}", node.info.rpc);
         let rpc_pubsub_url = format!("ws://{}/", node.info.rpc_pubsub);
         let tpu = node.info.tpu;
         let gossip = node.info.gossip;
 
         let validator_config = ValidatorConfig {
-            rpc_addrs: Some((node.info.rpc, node.info.rpc_pubsub)),
+            rpc_addrs: Some((
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()),
+                SocketAddr::new(
+                    IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
+                    node.info.rpc_pubsub.port(),
+                ),
+            )),
             rpc_config: config.rpc_config.clone(),
             accounts_hash_interval_slots: 100,
             account_paths: vec![ledger_path.join("accounts")],
@@ -346,6 +357,7 @@ impl TestValidator {
                 compression: CompressionType::NoCompression,
                 snapshot_version: SnapshotVersion::default(),
             }),
+            enforce_ulimit_nofile: false,
             ..ValidatorConfig::default()
         };
 
@@ -355,7 +367,7 @@ impl TestValidator {
             &ledger_path,
             &validator_vote_account.pubkey(),
             vec![Arc::new(validator_vote_account)],
-            None,
+            vec![],
             &validator_config,
         ));
 

core/src/tvu.rs

@@ -341,7 +341,7 @@ pub mod tests {
             ledger_signal_receiver,
             completed_slots_receiver,
             ..
-        } = Blockstore::open_with_signal(&blockstore_path, None)
+        } = Blockstore::open_with_signal(&blockstore_path, None, true)
             .expect("Expected to successfully open ledger");
         let blockstore = Arc::new(blockstore);
         let bank = bank_forks.working_bank();

core/src/validator.rs

@@ -3,7 +3,10 @@
 use crate::{
     broadcast_stage::BroadcastStageType,
     cache_block_time_service::{CacheBlockTimeSender, CacheBlockTimeService},
-    cluster_info::{ClusterInfo, Node, DEFAULT_CONTACT_DEBUG_INTERVAL},
+    cluster_info::{
+        ClusterInfo, Node, DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
+        DEFAULT_CONTACT_SAVE_INTERVAL_MILLIS,
+    },
     cluster_info_vote_listener::VoteTracker,
     completed_data_sets_service::CompletedDataSetsService,
     consensus::{reconcile_blockstore_roots_with_tower, Tower},
@@ -13,7 +16,7 @@ use crate::{
         OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker,
     },
     poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
-    poh_service::PohService,
+    poh_service::{self, PohService},
     rewards_recorder_service::{RewardsRecorderSender, RewardsRecorderService},
     rpc::JsonRpcConfig,
     rpc_pubsub_service::{PubSubConfig, PubSubService},
@@ -37,10 +40,12 @@ use solana_ledger::{
     blockstore_processor::{self, TransactionStatusSender},
     leader_schedule::FixedSchedule,
     leader_schedule_cache::LeaderScheduleCache,
+    poh::compute_hash_time_ns,
 };
 use solana_measure::measure::Measure;
 use solana_metrics::datapoint_info;
 use solana_runtime::{
+    accounts_index::AccountIndex,
     bank::Bank,
     bank_forks::{BankForks, SnapshotConfig},
     commitment::BlockCommitmentCache,
@@ -61,6 +66,7 @@ use std::time::Instant;
 use std::{
     collections::HashSet,
     net::SocketAddr,
+    ops::Deref,
     path::{Path, PathBuf},
     sync::atomic::{AtomicBool, Ordering},
     sync::mpsc::Receiver,
@@ -79,6 +85,7 @@ pub struct ValidatorConfig {
     pub expected_shred_version: Option<u16>,
     pub voting_disabled: bool,
     pub account_paths: Vec<PathBuf>,
+    pub account_shrink_paths: Option<Vec<PathBuf>>,
     pub rpc_config: JsonRpcConfig,
     pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub)
     pub pubsub_config: PubSubConfig,
@@ -86,6 +93,7 @@ pub struct ValidatorConfig {
     pub max_ledger_shreds: Option<u64>,
     pub broadcast_stage_type: BroadcastStageType,
     pub enable_partition: Option<Arc<AtomicBool>>,
+    pub enforce_ulimit_nofile: bool,
     pub fixed_leader_schedule: Option<FixedSchedule>,
     pub wait_for_supermajority: Option<Slot>,
     pub new_hard_forks: Option<Vec<Slot>>,
@@ -104,7 +112,13 @@ pub struct ValidatorConfig {
     pub require_tower: bool,
     pub debug_keys: Option<Arc<HashSet<Pubkey>>>,
     pub contact_debug_interval: u64,
+    pub contact_save_interval: u64,
     pub bpf_jit: bool,
+    pub send_transaction_retry_ms: u64,
+    pub send_transaction_leader_forward_count: u64,
+    pub no_poh_speed_test: bool,
+    pub poh_pinned_cpu_core: usize,
+    pub account_indexes: HashSet<AccountIndex>,
 }
 
 impl Default for ValidatorConfig {
@@ -117,12 +131,14 @@ impl Default for ValidatorConfig {
             voting_disabled: false,
             max_ledger_shreds: None,
             account_paths: Vec::new(),
+            account_shrink_paths: None,
             rpc_config: JsonRpcConfig::default(),
             rpc_addrs: None,
             pubsub_config: PubSubConfig::default(),
             snapshot_config: None,
             broadcast_stage_type: BroadcastStageType::Standard,
             enable_partition: None,
+            enforce_ulimit_nofile: true,
             fixed_leader_schedule: None,
             wait_for_supermajority: None,
             new_hard_forks: None,
@@ -140,8 +156,14 @@ impl Default for ValidatorConfig {
             cuda: false,
             require_tower: false,
             debug_keys: None,
-            contact_debug_interval: DEFAULT_CONTACT_DEBUG_INTERVAL,
+            contact_debug_interval: DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
+            contact_save_interval: DEFAULT_CONTACT_SAVE_INTERVAL_MILLIS,
             bpf_jit: false,
+            send_transaction_retry_ms: 2000,
+            send_transaction_leader_forward_count: 2,
+            no_poh_speed_test: true,
+            poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE,
+            account_indexes: HashSet::new(),
         }
     }
 }
@@ -214,7 +236,7 @@ impl Validator {
         ledger_path: &Path,
         vote_account: &Pubkey,
         mut authorized_voter_keypairs: Vec<Arc<Keypair>>,
-        cluster_entrypoint: Option<&ContactInfo>,
+        cluster_entrypoints: Vec<ContactInfo>,
         config: &ValidatorConfig,
     ) -> Self {
         let id = identity_keypair.pubkey();
@@ -233,7 +255,9 @@ impl Validator {
         }
         report_target_features();
 
-        info!("entrypoint: {:?}", cluster_entrypoint);
+        for cluster_entrypoint in &cluster_entrypoints {
+            info!("entrypoint: {:?}", cluster_entrypoint);
+        }
 
         if solana_perf::perf_libs::api().is_some() {
             info!("Initializing sigverify, this could take a while...");
@@ -266,6 +290,11 @@ impl Validator {
         for accounts_path in &config.account_paths {
             cleanup_accounts_path(accounts_path);
         }
+        if let Some(ref shrink_paths) = config.account_shrink_paths {
+            for accounts_path in shrink_paths {
+                cleanup_accounts_path(accounts_path);
+            }
+        }
         start.stop();
         info!("done. {}", start);
 
@@ -300,10 +329,14 @@ impl Validator {
             ledger_path,
             config.poh_verify,
             &exit,
+            config.enforce_ulimit_nofile,
         );
 
         let leader_schedule_cache = Arc::new(leader_schedule_cache);
         let bank = bank_forks.working_bank();
+        if let Some(ref shrink_paths) = config.account_shrink_paths {
+            bank.set_shrink_paths(shrink_paths.clone());
+        }
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 
         let sample_performance_service =
@@ -345,6 +378,8 @@ impl Validator {
 
         let mut cluster_info = ClusterInfo::new(node.info.clone(), identity_keypair.clone());
         cluster_info.set_contact_debug_interval(config.contact_debug_interval);
+        cluster_info.set_entrypoints(cluster_entrypoints);
+        cluster_info.restore_contact_info(ledger_path, config.contact_save_interval);
         let cluster_info = Arc::new(cluster_info);
         let mut block_commitment_cache = BlockCommitmentCache::default();
         block_commitment_cache.initialize_slots(bank.slot());
@@ -430,6 +465,8 @@ impl Validator {
                         config.trusted_validators.clone(),
                         rpc_override_health_check.clone(),
                         optimistically_confirmed_bank.clone(),
+                        config.send_transaction_retry_ms,
+                        config.send_transaction_leader_forward_count,
                     ),
                     pubsub_service: PubSubService::new(
                         config.pubsub_config.clone(),
@@ -473,7 +510,6 @@ impl Validator {
             config.gossip_validators.clone(),
             &exit,
         );
-
         let serve_repair = Arc::new(RwLock::new(ServeRepair::new(cluster_info.clone())));
         let serve_repair_service = ServeRepairService::new(
             &serve_repair,
@@ -482,12 +518,6 @@ impl Validator {
             &exit,
         );
 
-        // Insert the entrypoint info, should only be None if this node
-        // is the bootstrap validator
-        if let Some(cluster_entrypoint) = cluster_entrypoint {
-            cluster_info.set_entrypoint(cluster_entrypoint.clone());
-        }
-
         let (snapshot_packager_service, snapshot_config_and_package_sender) =
             if let Some(snapshot_config) = config.snapshot_config.clone() {
                 if is_snapshot_config_invalid(
@@ -509,18 +539,30 @@ impl Validator {
                 (None, None)
             };
 
+        if !config.no_poh_speed_test {
+            check_poh_speed(&genesis_config, None);
+        }
+
         if wait_for_supermajority(config, &bank, &cluster_info, rpc_override_health_check) {
             abort();
         }
 
-        let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
+        let poh_service = PohService::new(
+            poh_recorder.clone(),
+            &poh_config,
+            &exit,
+            bank.ticks_per_slot(),
+            config.poh_pinned_cpu_core,
+        );
         assert_eq!(
             blockstore.new_shreds_signals.len(),
             1,
             "New shred signal for the TVU should be the same as the clear bank signal."
         );
 
-        let vote_tracker = Arc::new(VoteTracker::new(bank_forks.read().unwrap().root_bank()));
+        let vote_tracker = Arc::new(VoteTracker::new(
+            bank_forks.read().unwrap().root_bank().deref(),
+        ));
 
         let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
         let (verified_vote_sender, verified_vote_receiver) = unbounded();
@@ -717,7 +759,7 @@ impl Validator {
         self.completed_data_sets_service
             .join()
             .expect("completed_data_sets_service");
-        self.ip_echo_server.shutdown_now();
+        self.ip_echo_server.shutdown_background();
     }
 }
 
@@ -730,6 +772,35 @@ fn active_vote_account_exists_in_bank(bank: &Arc<Bank>, vote_account: &Pubkey) -
     false
 }
 
+fn check_poh_speed(genesis_config: &GenesisConfig, maybe_hash_samples: Option<u64>) {
+    if let Some(hashes_per_tick) = genesis_config.hashes_per_tick() {
+        let ticks_per_slot = genesis_config.ticks_per_slot();
+        let hashes_per_slot = hashes_per_tick * ticks_per_slot;
+
+        let hash_samples = maybe_hash_samples.unwrap_or(hashes_per_slot);
+        let hash_time_ns = compute_hash_time_ns(hash_samples);
+
+        let my_ns_per_slot = (hash_time_ns * hashes_per_slot) / hash_samples;
+        debug!("computed: ns_per_slot: {}", my_ns_per_slot);
+        let target_ns_per_slot = genesis_config.ns_per_slot() as u64;
+        debug!(
+            "cluster ns_per_hash: {}ns ns_per_slot: {}",
+            target_ns_per_slot / hashes_per_slot,
+            target_ns_per_slot
+        );
+        if my_ns_per_slot < target_ns_per_slot {
+            let extra_ns = target_ns_per_slot - my_ns_per_slot;
+            info!("PoH speed check: Will sleep {}ns per slot.", extra_ns);
+        } else {
+            error!(
+                "PoH is slower than cluster target tick rate! mine: {} cluster: {}. If you wish to continue, try --no-poh-speed-test",
+                my_ns_per_slot, target_ns_per_slot,
+            );
+            abort();
+        }
+    }
+}
+
 fn post_process_restored_tower(
     restored_tower: crate::consensus::Result<Tower>,
     validator_identity: &Pubkey,
@@ -822,6 +893,7 @@ fn new_banks_from_ledger(
     ledger_path: &Path,
     poh_verify: bool,
     exit: &Arc<AtomicBool>,
+    enforce_ulimit_nofile: bool,
 ) -> (
     GenesisConfig,
     BankForks,
@@ -859,8 +931,12 @@ fn new_banks_from_ledger(
         ledger_signal_receiver,
         completed_slots_receiver,
         ..
-    } = Blockstore::open_with_signal(ledger_path, config.wal_recovery_mode.clone())
-        .expect("Failed to open ledger database");
+    } = Blockstore::open_with_signal(
+        ledger_path,
+        config.wal_recovery_mode.clone(),
+        enforce_ulimit_nofile,
+    )
+    .expect("Failed to open ledger database");
     blockstore.set_no_compaction(config.no_rocksdb_compaction);
 
     let restored_tower = Tower::restore(ledger_path, &validator_identity);
@@ -878,6 +954,7 @@ fn new_banks_from_ledger(
         new_hard_forks: config.new_hard_forks.clone(),
         frozen_accounts: config.frozen_accounts.clone(),
         debug_keys: config.debug_keys.clone(),
+        account_indexes: config.account_indexes.clone(),
         ..blockstore_processor::ProcessOptions::default()
     };
 
@@ -893,6 +970,7 @@ fn new_banks_from_ledger(
         &genesis_config,
         &blockstore,
         config.account_paths.clone(),
+        config.account_shrink_paths.clone(),
         config.snapshot_config.as_ref(),
         process_options,
         transaction_history_services
@@ -1140,10 +1218,10 @@ fn get_stake_percent_in_gossip(bank: &Bank, cluster_info: &ClusterInfo, log: boo
     let my_shred_version = cluster_info.my_shred_version();
     let my_id = cluster_info.id();
 
-    for (activated_stake, vote_account) in bank.vote_accounts().values() {
+    for (_, (activated_stake, vote_account)) in bank.vote_accounts() {
         total_activated_stake += activated_stake;
 
-        if *activated_stake == 0 {
+        if activated_stake == 0 {
             continue;
         }
         let vote_state_node_pubkey = vote_account
@@ -1165,13 +1243,13 @@ fn get_stake_percent_in_gossip(bank: &Bank, cluster_info: &ClusterInfo, log: boo
                 online_stake += activated_stake;
             } else {
                 wrong_shred_stake += activated_stake;
-                wrong_shred_nodes.push((*activated_stake, vote_state_node_pubkey));
+                wrong_shred_nodes.push((activated_stake, vote_state_node_pubkey));
             }
         } else if vote_state_node_pubkey == my_id {
             online_stake += activated_stake; // This node is online
         } else {
             offline_stake += activated_stake;
-            offline_nodes.push((*activated_stake, vote_state_node_pubkey));
+            offline_nodes.push((activated_stake, vote_state_node_pubkey));
         }
     }
 
@@ -1236,6 +1314,8 @@ pub fn is_snapshot_config_invalid(
 mod tests {
     use super::*;
     use solana_ledger::{create_new_tmp_ledger, genesis_utils::create_genesis_config_with_leader};
+    use solana_sdk::genesis_config::create_genesis_config;
+    use solana_sdk::poh_config::PohConfig;
     use std::fs::remove_dir_all;
 
     #[test]
@@ -1262,7 +1342,7 @@ mod tests {
             &validator_ledger_path,
             &voting_keypair.pubkey(),
             vec![voting_keypair.clone()],
-            Some(&leader_node.info),
+            vec![leader_node.info],
             &config,
         );
         validator.close();
@@ -1332,7 +1412,7 @@ mod tests {
                     &validator_ledger_path,
                     &vote_account_keypair.pubkey(),
                     vec![Arc::new(vote_account_keypair)],
-                    Some(&leader_node.info),
+                    vec![leader_node.info.clone()],
                     &config,
                 )
             })
@@ -1354,7 +1434,6 @@ mod tests {
     #[test]
     fn test_wait_for_supermajority() {
         solana_logger::setup();
-        use solana_sdk::genesis_config::create_genesis_config;
         use solana_sdk::hash::hash;
         let node_keypair = Arc::new(Keypair::new());
         let cluster_info = ClusterInfo::new(
@@ -1411,4 +1490,38 @@ mod tests {
         assert!(!is_snapshot_config_invalid(500, 100));
         assert!(!is_snapshot_config_invalid(5, 5));
     }
+
+    #[test]
+    #[should_panic]
+    fn test_poh_speed() {
+        solana_logger::setup();
+        let poh_config = PohConfig {
+            target_tick_duration: Duration::from_millis(solana_sdk::clock::MS_PER_TICK),
+            // make PoH rate really fast to cause the panic condition
+            hashes_per_tick: Some(
+                100 * solana_sdk::clock::DEFAULT_HASHES_PER_SECOND
+                    / solana_sdk::clock::DEFAULT_TICKS_PER_SECOND,
+            ),
+            ..PohConfig::default()
+        };
+        let genesis_config = GenesisConfig {
+            poh_config,
+            ..GenesisConfig::default()
+        };
+        check_poh_speed(&genesis_config, Some(10_000));
+    }
+
+    #[test]
+    fn test_poh_speed_no_hashes_per_tick() {
+        let poh_config = PohConfig {
+            target_tick_duration: Duration::from_millis(solana_sdk::clock::MS_PER_TICK),
+            hashes_per_tick: None,
+            ..PohConfig::default()
+        };
+        let genesis_config = GenesisConfig {
+            poh_config,
+            ..GenesisConfig::default()
+        };
+        check_poh_speed(&genesis_config, Some(10_000));
+    }
 }

core/tests/crds_gossip.rs

@@ -458,7 +458,11 @@ fn network_run_pull(
                         let rsp = node
                             .lock()
                             .unwrap()
-                            .generate_pull_responses(&filters, now)
+                            .generate_pull_responses(
+                                &filters,
+                                /*output_size_limit=*/ usize::MAX,
+                                now,
+                            )
                             .into_iter()
                             .flatten()
                             .collect();

core/tests/snapshots.rs

@@ -59,7 +59,10 @@ mod tests {
         signature::{Keypair, Signer},
         system_transaction,
     };
-    use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
+    use std::{
+        collections::HashSet, fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel,
+        sync::Arc,
+    };
     use tempfile::TempDir;
 
     DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Development, V1_2_0_Development);
@@ -93,6 +96,7 @@ mod tests {
                 &[],
                 None,
                 None,
+                HashSet::new(),
             );
             bank0.freeze();
             let mut bank_forks = BankForks::new(bank0);
@@ -148,6 +152,7 @@ mod tests {
             old_genesis_config,
             None,
             None,
+            HashSet::new(),
         )
         .unwrap();
 

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

docs/src/cluster/vote-signing.md

@@ -24,19 +24,19 @@ The vote signing service consists of a JSON RPC server and a request processor.
 
 1. Register a new validator node
 
-- The request must contain validator's identity \(public key\)
-- The request must be signed with the validator's private key
-- The service drops the request if signature of the request cannot be verified
-- The service creates a new voting asymmetric key for the validator, and returns the public key as a response
-- If a validator tries to register again, the service returns the public key from the pre-existing keypair
+    - The request must contain validator's identity \(public key\)
+    - The request must be signed with the validator's private key
+    - The service drops the request if signature of the request cannot be verified
+    - The service creates a new voting asymmetric key for the validator, and returns the public key as a response
+    - If a validator tries to register again, the service returns the public key from the pre-existing keypair
 
-  2. Sign a vote
+2. Sign a vote
 
-- The request must contain a voting transaction and all verification data
-- The request must be signed with the validator's private key
-- The service drops the request if signature of the request cannot be verified
-- The service verifies the voting data
-- The service returns a signature for the transaction
+    - The request must contain a voting transaction and all verification data
+    - The request must be signed with the validator's private key
+    - The service drops the request if signature of the request cannot be verified
+    - The service verifies the voting data
+    - The service returns a signature for the transaction
 
 ## Validator voting
 

docs/src/clusters.md

@@ -119,7 +119,7 @@ Currently, rewards and inflation are disabled.
   - Note: If you are using a non-command-line wallet such as
     [Solflare](wallet-guide/solflare.md),
     the wallet will always be connecting to Mainnet Beta.
-- Gossip entrypoint for Mainnet Beta: `mainnet-beta.solana.com:8001`
+- Gossip entrypoint for Mainnet Beta: `entrypoint.mainnet-beta.solana.com:8001`
 - Metrics environment variable for Mainnet Beta:
 ```bash
 export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=mainnet-beta,u=mainnet-beta_write,p=password"
@@ -147,7 +147,11 @@ $ solana-validator \
     --rpc-port 8899 \
     --private-rpc \
     --dynamic-port-range 8000-8010 \
-    --entrypoint mainnet-beta.solana.com:8001 \
+    --entrypoint entrypoint.mainnet-beta.solana.com:8001 \
+    --entrypoint entrypoint2.mainnet-beta.solana.com:8001 \
+    --entrypoint entrypoint3.mainnet-beta.solana.com:8001 \
+    --entrypoint entrypoint4.mainnet-beta.solana.com:8001 \
+    --entrypoint entrypoint5.mainnet-beta.solana.com:8001 \
     --expected-genesis-hash 5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d \
     --wal-recovery-mode skip_any_corrupted_record \
     --limit-ledger-size

docs/src/developing/clients/jsonrpc-api.md

@@ -2571,7 +2571,7 @@ curl http://localhost:8899 -X POST -H "Content-Type: application/json" -d '
 
 Result:
 ```json
-{"jsonrpc":"2.0","result":{"solana-core": "1.5.0"},"id":1}
+{"jsonrpc":"2.0","result":{"solana-core": "1.5.1"},"id":1}
 ```
 
 ### getVoteAccounts

docs/src/running-validator/validator-start.md

@@ -371,17 +371,37 @@ Once your validator is operating normally, you can reduce the time it takes to
 restart your validator by adding the `--no-port-check` flag to your
 `solana-validator` command-line.
 
-### Using a ramdisk for the accounts database to reduce SSD wear
-If your machine has plenty of RAM, a ramdisk
+### Disable snapshot compression to reduce CPU usage
+If you are not serving snapshots to other validators, snapshot compression can
+be disabled to reduce CPU load at the expense of slightly more disk usage for
+local snapshot storage.
+
+Add the `--snapshot-compression none` argument to your `solana-validator`
+command-line arguments and restart the validator.
+
+### Using a ramdisk with spill-over into swap for the accounts database to reduce SSD wear
+If your machine has plenty of RAM, a tmpfs ramdisk
 ([tmpfs](https://man7.org/linux/man-pages/man5/tmpfs.5.html)) may be used to hold
-the accounts database.
+the accounts database
 
-Assuming your tmpfs ramdisk is mounted at `/mnt/solana-accounts` and writable
-by the validator user, add the `--accounts /mnt/solana-accounts` argument to
-your validator command-line to use it.  Once your validator restarts, you should
-now see activity in `/mnt/solana-accounts` instead of the `accounts/`
-subdirectory of your ledger (`--ledger` argument) directory.
+When using tmpfs it's essential to also configure swap on your machine as well to
+avoid running out of tmpfs space periodically.
 
-For Mainnet Beta it's recommended that a ramdisk of 64GB be used.  Note that if
-your machine has less than 100GB of RAM in total, using a ramdisk is not
-recommended.
+A 300GB tmpfs partition is recommended, with an accompanying 250GB swap
+partition.
+
+Example configuration:
+1. `sudo mkdir /mnt/solana-accounts`
+2. Add a 300GB tmpfs parition by adding a new line containing `tmpfs
+   /mnt/solana-accounts tmpfs rw,size=300G,user=sol 0 0` to `/etc/fstab`
+   (assuming your validator is running under the user "sol").  **CAREFUL: If you
+   incorrectly edit /etc/fstab your machine may no longer boot**
+3. Create at least 250GB of swap space
+  - Choose a device to use in place of `SWAPDEV` for the remainder of these instructions.  Ideally select a free disk partition of 250GB or greater on a fast disk. If one is not available, create a swap file with `sudo fallocate -l 250G /swapfile`, set its permissions with `sudo chmod 0600 /swapfile` and use `/swapfile` as `SWAPDEV` for the remainder of these instructions
+  - Format the device for usage as swap with `sudo mkswap SWAPDEV`
+4. Add the swap file to `/etc/fstab` with a new line containing `SWAPDEV swap swap defaults 0 0`
+5. Enable swap with `sudo swapon -a` and mount the tmpfs with `sudo mount /mnt/solana-accounts/`
+6. Confirm swap is active with `free -g` and the tmpfs is mounted with `mount`
+
+Now add the `--accounts /mnt/solana-accounts` argument to your `solana-validator`
+command-line arguments and restart the validator.

dos/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-dos"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,15 +14,15 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-core = { path = "../core", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-core = { path = "../core", version = "1.5.1" }
+solana-ledger = { path = "../ledger", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

download-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-download-utils"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana Download Utils"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,8 +14,8 @@ console = "0.11.3"
 indicatif = "0.15.0"
 log = "0.4.11"
 reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
 tar = "0.4.28"
 
 [lib]

download-utils/src/lib.rs

@@ -35,7 +35,7 @@ pub fn download_file(
 
     fs::create_dir_all(destination_file.parent().unwrap()).map_err(|err| err.to_string())?;
 
-    let temp_destination_file = destination_file.with_extension(".tmp");
+    let temp_destination_file = destination_file.with_extension("tmp");
 
     let progress_bar = new_spinner_progress_bar();
     if use_progress_bar {
@@ -82,6 +82,7 @@ pub fn download_file(
         response: R,
         last_print: Instant,
         current_bytes: usize,
+        last_print_bytes: usize,
         download_size: f32,
         use_progress_bar: bool,
     }
@@ -94,14 +95,16 @@ pub fn download_file(
                 } else {
                     self.current_bytes += n;
                     if self.last_print.elapsed().as_secs() > 5 {
-                        let bytes_f32 = self.current_bytes as f32;
+                        let total_bytes_f32 = self.current_bytes as f32;
+                        let diff_bytes_f32 = (self.current_bytes - self.last_print_bytes) as f32;
                         info!(
                             "downloaded {} bytes {:.1}% {:.1} bytes/s",
                             self.current_bytes,
-                            100f32 * (bytes_f32 / self.download_size),
-                            bytes_f32 / self.last_print.elapsed().as_secs_f32(),
+                            100f32 * (total_bytes_f32 / self.download_size),
+                            diff_bytes_f32 / self.last_print.elapsed().as_secs_f32(),
                         );
                         self.last_print = Instant::now();
+                        self.last_print_bytes = self.current_bytes;
                     }
                 }
                 n
@@ -114,6 +117,7 @@ pub fn download_file(
         response,
         last_print: Instant::now(),
         current_bytes: 0,
+        last_print_bytes: 0,
         download_size: (download_size as f32).max(1f32),
         use_progress_bar,
     };
@@ -168,52 +172,40 @@ pub fn download_snapshot(
     desired_snapshot_hash: (Slot, Hash),
     use_progress_bar: bool,
 ) -> Result<(), String> {
-    // Remove all snapshot not matching the desired hash
-    let snapshot_packages = snapshot_utils::get_snapshot_archives(ledger_path);
-    let mut found_package = false;
-    for (snapshot_package, (snapshot_slot, snapshot_hash, _compression)) in snapshot_packages.iter()
-    {
-        if (*snapshot_slot, *snapshot_hash) != desired_snapshot_hash {
-            info!("Removing old snapshot: {:?}", snapshot_package);
-            fs::remove_file(snapshot_package)
-                .unwrap_or_else(|err| info!("Failed to remove old snapshot: {:}", err));
-        } else {
-            found_package = true;
+    snapshot_utils::purge_old_snapshot_archives(ledger_path);
+
+    for compression in &[
+        CompressionType::Zstd,
+        CompressionType::Gzip,
+        CompressionType::Bzip2,
+    ] {
+        let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
+            ledger_path,
+            &desired_snapshot_hash,
+            compression,
+        );
+
+        if desired_snapshot_package.is_file() {
+            return Ok(());
+        }
+
+        if download_file(
+            &format!(
+                "http://{}/{}",
+                rpc_addr,
+                desired_snapshot_package
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+            ),
+            &desired_snapshot_package,
+            use_progress_bar,
+        )
+        .is_ok()
+        {
+            return Ok(());
         }
     }
-
-    if found_package {
-        Ok(())
-    } else {
-        for compression in &[
-            CompressionType::Zstd,
-            CompressionType::Gzip,
-            CompressionType::Bzip2,
-        ] {
-            let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
-                ledger_path,
-                &desired_snapshot_hash,
-                compression,
-            );
-
-            if download_file(
-                &format!(
-                    "http://{}/{}",
-                    rpc_addr,
-                    desired_snapshot_package
-                        .file_name()
-                        .unwrap()
-                        .to_str()
-                        .unwrap()
-                ),
-                &desired_snapshot_package,
-                use_progress_bar,
-            )
-            .is_ok()
-            {
-                return Ok(());
-            }
-        }
-        Err("Snapshot couldn't be downloaded".to_string())
-    }
+    Err("Snapshot couldn't be downloaded".to_string())
 }

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,19 +11,17 @@ edition = "2018"
 [dependencies]
 bincode = "1.3.1"
 byteorder = "1.3.4"
-bytes = "0.4"
 clap = "2.33"
 log = "0.4.11"
 serde = "1.0.112"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-cli-config = { path = "../cli-config", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-tokio = "0.1"
-tokio-codec = "0.1"
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-cli-config = { path = "../cli-config", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-metrics = { path = "../metrics", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
+tokio = { version = "0.3.5", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

faucet/src/bin/faucet.rs

@@ -11,7 +11,8 @@ use std::{
     thread,
 };
 
-fn main() {
+#[tokio::main]
+async fn main() {
     let default_keypair = solana_cli_config::Config::default().keypair_path;
 
     solana_logger::setup_with_default("solana=info");
@@ -76,5 +77,5 @@ fn main() {
         faucet1.lock().unwrap().clear_request_count();
     });
 
-    run_faucet(faucet, faucet_addr, None);
+    run_faucet(faucet, faucet_addr, None).await;
 }

faucet/src/faucet.rs

@@ -4,9 +4,8 @@
 //! checking requests against a request cap for a given time time_slice
 //! and (to come) an IP rate limit.
 
-use bincode::{deserialize, serialize};
+use bincode::{deserialize, serialize, serialized_size};
 use byteorder::{ByteOrder, LittleEndian};
-use bytes::{Bytes, BytesMut};
 use log::*;
 use serde_derive::{Deserialize, Serialize};
 use solana_metrics::datapoint_info;
@@ -20,18 +19,17 @@ use solana_sdk::{
     transaction::Transaction,
 };
 use std::{
-    io::{self, Error, ErrorKind},
+    io::{self, Error, ErrorKind, Read, Write},
     net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream},
     sync::{mpsc::Sender, Arc, Mutex},
     thread,
     time::Duration,
 };
 use tokio::{
-    self,
-    net::TcpListener,
-    prelude::{Future, Read, Sink, Stream, Write},
+    io::{AsyncReadExt, AsyncWriteExt},
+    net::{TcpListener, TcpStream as TokioTcpStream},
+    runtime::Runtime,
 };
-use tokio_codec::{BytesCodec, Decoder};
 
 #[macro_export]
 macro_rules! socketaddr {
@@ -58,6 +56,16 @@ pub enum FaucetRequest {
     },
 }
 
+impl Default for FaucetRequest {
+    fn default() -> Self {
+        Self::GetAirdrop {
+            lamports: u64::default(),
+            to: Pubkey::default(),
+            blockhash: Hash::default(),
+        }
+    }
+}
+
 pub struct Faucet {
     faucet_keypair: Keypair,
     ip_cache: Vec<IpAddr>,
@@ -154,7 +162,7 @@ impl Faucet {
             }
         }
     }
-    pub fn process_faucet_request(&mut self, bytes: &BytesMut) -> Result<Bytes, io::Error> {
+    pub fn process_faucet_request(&mut self, bytes: &[u8]) -> Result<Vec<u8>, io::Error> {
         let req: FaucetRequest = deserialize(bytes).map_err(|err| {
             io::Error::new(
                 io::ErrorKind::Other,
@@ -177,9 +185,8 @@ impl Faucet {
                 LittleEndian::write_u16(&mut response_vec_with_length, response_vec.len() as u16);
                 response_vec_with_length.extend_from_slice(&response_vec);
 
-                let response_bytes = Bytes::from(response_vec_with_length);
                 info!("Airdrop transaction granted");
-                Ok(response_bytes)
+                Ok(response_vec_with_length)
             }
             Err(err) => {
                 warn!("Airdrop transaction failed: {:?}", err);
@@ -270,7 +277,8 @@ pub fn run_local_faucet_with_port(
             per_time_cap,
             None,
         )));
-        run_faucet(faucet, faucet_addr, Some(sender));
+        let runtime = Runtime::new().unwrap();
+        runtime.block_on(run_faucet(faucet, faucet_addr, Some(sender)));
     });
 }
 
@@ -283,14 +291,14 @@ pub fn run_local_faucet(
     run_local_faucet_with_port(faucet_keypair, sender, per_time_cap, 0)
 }
 
-pub fn run_faucet(
+pub async fn run_faucet(
     faucet: Arc<Mutex<Faucet>>,
     faucet_addr: SocketAddr,
     send_addr: Option<Sender<SocketAddr>>,
 ) {
-    let socket = TcpListener::bind(&faucet_addr).unwrap();
+    let listener = TcpListener::bind(&faucet_addr).await.unwrap();
     if let Some(send_addr) = send_addr {
-        send_addr.send(socket.local_addr().unwrap()).unwrap();
+        send_addr.send(listener.local_addr().unwrap()).unwrap();
     }
     info!("Faucet started. Listening on: {}", faucet_addr);
     info!(
@@ -298,43 +306,48 @@ pub fn run_faucet(
         faucet.lock().unwrap().faucet_keypair.pubkey()
     );
 
-    let done = socket
-        .incoming()
-        .map_err(|e| debug!("failed to accept socket; error = {:?}", e))
-        .for_each(move |socket| {
-            let faucet2 = faucet.clone();
-            let framed = BytesCodec::new().framed(socket);
-            let (writer, reader) = framed.split();
+    loop {
+        let _faucet = faucet.clone();
+        match listener.accept().await {
+            Ok((stream, _)) => {
+                tokio::spawn(async move {
+                    if let Err(e) = process(stream, _faucet).await {
+                        info!("failed to process request; error = {:?}", e);
+                    }
+                });
+            }
+            Err(e) => debug!("failed to accept socket; error = {:?}", e),
+        }
+    }
+}
 
-            let processor = reader.and_then(move |bytes| {
-                match faucet2.lock().unwrap().process_faucet_request(&bytes) {
-                    Ok(response_bytes) => {
-                        trace!("Airdrop response_bytes: {:?}", response_bytes.to_vec());
-                        Ok(response_bytes)
-                    }
-                    Err(e) => {
-                        info!("Error in request: {:?}", e);
-                        Ok(Bytes::from(0u16.to_le_bytes().to_vec()))
-                    }
-                }
-            });
-            let server = writer
-                .send_all(processor.or_else(|err| {
-                    Err(io::Error::new(
-                        io::ErrorKind::Other,
-                        format!("Faucet response: {:?}", err),
-                    ))
-                }))
-                .then(|_| Ok(()));
-            tokio::spawn(server)
-        });
-    tokio::run(done);
+async fn process(
+    mut stream: TokioTcpStream,
+    faucet: Arc<Mutex<Faucet>>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let mut request = vec![0u8; serialized_size(&FaucetRequest::default()).unwrap() as usize];
+    while stream.read_exact(&mut request).await.is_ok() {
+        trace!("{:?}", request);
+
+        let response = match faucet.lock().unwrap().process_faucet_request(&request) {
+            Ok(response_bytes) => {
+                trace!("Airdrop response_bytes: {:?}", response_bytes);
+                response_bytes
+            }
+            Err(e) => {
+                info!("Error in request: {:?}", e);
+                0u16.to_le_bytes().to_vec()
+            }
+        };
+        stream.write_all(&response).await?;
+    }
+
+    Ok(())
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use bytes::BufMut;
     use solana_sdk::system_instruction::SystemInstruction;
     use std::time::Duration;
 
@@ -446,8 +459,6 @@ mod tests {
             to,
         };
         let req = serialize(&req).unwrap();
-        let mut bytes = BytesMut::with_capacity(req.len());
-        bytes.put(&req[..]);
 
         let keypair = Keypair::new();
         let expected_instruction = system_instruction::transfer(&keypair.pubkey(), &to, lamports);
@@ -459,12 +470,11 @@ mod tests {
         expected_vec_with_length.extend_from_slice(&expected_bytes);
 
         let mut faucet = Faucet::new(keypair, None, None, None);
-        let response = faucet.process_faucet_request(&bytes);
+        let response = faucet.process_faucet_request(&req);
         let response_vec = response.unwrap().to_vec();
         assert_eq!(expected_vec_with_length, response_vec);
 
-        let mut bad_bytes = BytesMut::with_capacity(9);
-        bad_bytes.put("bad bytes");
+        let bad_bytes = "bad bytes".as_bytes();
         assert!(faucet.process_faucet_request(&bad_bytes).is_err());
     }
 }

frozen-abi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana Frozen ABI"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,11 +15,11 @@ log = "0.4.11"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 sha2 = "0.8.2"
-solana-frozen-abi-macro = { path = "macro", version = "1.5.0" }
+solana-frozen-abi-macro = { path = "macro", version = "1.5.1" }
 thiserror = "1.0"
 
 [target.'cfg(not(target_arch = "bpf"))'.dependencies]
-solana-logger = { path = "../logger", version = "1.5.0" }
+solana-logger = { path = "../logger", version = "1.5.1" }
 generic-array = { version = "0.14.3", default-features = false, features = ["serde", "more_lengths"]}
 memmap2 = "0.1.0"
 

frozen-abi/macro/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi-macro"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana Frozen ABI Macro"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,18 +15,18 @@ chrono = "0.4"
 serde = "1.0.112"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-budget-program = { path = "../programs/budget", version = "1.5.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-cli-config = { path = "../cli-config", version = "1.5.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vest-program = { path = "../programs/vest", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-budget-program = { path = "../programs/budget", version = "1.5.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-cli-config = { path = "../cli-config", version = "1.5.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.5.1" }
+solana-ledger = { path = "../ledger", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
+solana-vest-program = { path = "../programs/vest", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
 tempfile = "3.1.0"
 
 [[bin]]

gossip/Cargo.toml

@@ -3,20 +3,20 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-core = { path = "../core", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-net-utils = { path = "../net-utils", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-core = { path = "../core", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-net-utils = { path = "../net-utils", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,12 +24,12 @@ reqwest = { version = "0.10.8", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-config-program = { path = "../programs/config", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-config-program = { path = "../programs/config", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 semver = "0.9.0"
 tar = "0.4.28"
 tempfile = "3.1.0"
@@ -39,13 +39,5 @@ url = "2.1.1"
 winapi = "0.3.8"
 winreg = "0.7"
 
-[[bin]]
-name = "solana-install"
-path = "src/main-install.rs"
-
-[[bin]]
-name = "solana-install-init"
-path = "src/main-install-init.rs"
-
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

install/src/command.rs

@@ -369,7 +369,9 @@ fn add_to_path(new_path: &str) -> bool {
     use winreg::enums::{RegType, HKEY_CURRENT_USER, KEY_READ, KEY_WRITE};
     use winreg::{RegKey, RegValue};
 
-    let old_path = if let Some(s) = get_windows_path_var()? {
+    let old_path = if let Some(s) =
+        get_windows_path_var().unwrap_or_else(|err| panic!("Unable to get PATH: {}", err))
+    {
         s
     } else {
         return false;
@@ -385,7 +387,7 @@ fn add_to_path(new_path: &str) -> bool {
         let root = RegKey::predef(HKEY_CURRENT_USER);
         let environment = root
             .open_subkey_with_flags("Environment", KEY_READ | KEY_WRITE)
-            .map_err(|err| format!("Unable to open HKEY_CURRENT_USER\\Environment: {}", err))?;
+            .unwrap_or_else(|err| panic!("Unable to open HKEY_CURRENT_USER\\Environment: {}", err));
 
         let reg_value = RegValue {
             bytes: string_to_winreg_bytes(&new_path),
@@ -394,7 +396,9 @@ fn add_to_path(new_path: &str) -> bool {
 
         environment
             .set_raw_value("PATH", &reg_value)
-            .map_err(|err| format!("Unable set HKEY_CURRENT_USER\\Environment\\PATH: {}", err))?;
+            .unwrap_or_else(|err| {
+                panic!("Unable set HKEY_CURRENT_USER\\Environment\\PATH: {}", err)
+            });
 
         // Tell other processes to update their environment
         unsafe {

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,11 +13,11 @@ bs58 = "0.3.1"
 clap = "2.33"
 dirs-next = "2.0.0"
 num_cpus = "1.13.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-cli-config = { path = "../cli-config", version = "1.5.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-cli-config = { path = "../cli-config", version = "1.5.1" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 tiny-bip39 = "0.7.0"
 
 [[bin]]

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -22,18 +22,18 @@ regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-cli-output = { path = "../cli-output", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-cli-output = { path = "../cli-output", version = "1.5.1" }
+solana-ledger = { path = "../ledger", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-measure = { path = "../measure", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
 tempfile = "3.1.0"
 tokio = { version = "0.2.22", features = ["full"] }
 

ledger-tool/src/main.rs

@@ -615,7 +615,7 @@ fn open_blockstore(
     access_type: AccessType,
     wal_recovery_mode: Option<BlockstoreRecoveryMode>,
 ) -> Blockstore {
-    match Blockstore::open_with_access_type(ledger_path, access_type, wal_recovery_mode) {
+    match Blockstore::open_with_access_type(ledger_path, access_type, wal_recovery_mode, true) {
         Ok(blockstore) => blockstore,
         Err(err) => {
             eprintln!("Failed to open ledger at {:?}: {:?}", ledger_path, err);
@@ -694,6 +694,7 @@ fn load_bank_forks(
         &genesis_config,
         &blockstore,
         account_paths,
+        None,
         snapshot_config.as_ref(),
         process_options,
         None,

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.5.0"
+version = "1.5.1"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -32,20 +32,22 @@ reed-solomon-erasure = { version = "4.0.2", features = ["simd-accel"] }
 serde = "1.0.112"
 serde_bytes = "0.11.4"
 sha2 = "0.8.2"
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-measure = { path = "../measure", version = "1.5.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.5.0" }
-solana-metrics = { path = "../metrics", version = "1.5.0" }
-solana-perf = { path = "../perf", version = "1.5.0" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.0" }
-solana-storage-proto = { path = "../storage-proto", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.5.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "1.5.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.5.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-measure = { path = "../measure", version = "1.5.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.5.1" }
+solana-metrics = { path = "../metrics", version = "1.5.1" }
+solana-perf = { path = "../perf", version = "1.5.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.5.1" }
+solana-storage-proto = { path = "../storage-proto", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "0.2.22", features = ["full"] }
@@ -61,7 +63,10 @@ features = ["lz4"]
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-budget-program = { path = "../programs/budget", version = "1.5.0" }
+solana-budget-program = { path = "../programs/budget", version = "1.5.1" }
+
+[build-dependencies]
+rustc_version = "0.2"
 
 [lib]
 crate-type = ["lib"]

ledger/build.rs

@@ -0,0 +1,27 @@
+extern crate rustc_version;
+use rustc_version::{version_meta, Channel};
+
+fn main() {
+    // Copied and adapted from
+    // https://github.com/Kimundi/rustc-version-rs/blob/1d692a965f4e48a8cb72e82cda953107c0d22f47/README.md#example
+    // Licensed under Apache-2.0 + MIT
+    match version_meta().unwrap().channel {
+        Channel::Stable => {
+            println!("cargo:rustc-cfg=RUSTC_WITHOUT_SPECIALIZATION");
+        }
+        Channel::Beta => {
+            println!("cargo:rustc-cfg=RUSTC_WITHOUT_SPECIALIZATION");
+        }
+        Channel::Nightly => {
+            println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION");
+        }
+        Channel::Dev => {
+            println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION");
+            // See https://github.com/solana-labs/solana/issues/11055
+            // We may be running the custom `rust-bpf-builder` toolchain,
+            // which currently needs `#![feature(proc_macro_hygiene)]` to
+            // be applied.
+            println!("cargo:rustc-cfg=RUSTC_NEEDS_PROC_MACRO_HYGIENE");
+        }
+    }
+}

ledger/src/bank_forks_utils.rs

@@ -33,6 +33,7 @@ pub fn load(
     genesis_config: &GenesisConfig,
     blockstore: &Blockstore,
     account_paths: Vec<PathBuf>,
+    shrink_paths: Option<Vec<PathBuf>>,
     snapshot_config: Option<&SnapshotConfig>,
     process_options: ProcessOptions,
     transaction_status_sender: Option<TransactionStatusSender>,
@@ -66,12 +67,13 @@ pub fn load(
                     compression,
                     genesis_config,
                     process_options.debug_keys.clone(),
-                    Some(&crate::builtins::get(
-                        genesis_config.cluster_type,
-                        process_options.bpf_jit,
-                    )),
+                    Some(&crate::builtins::get(process_options.bpf_jit)),
+                    process_options.account_indexes.clone(),
                 )
                 .expect("Load from snapshot failed");
+                if let Some(shrink_paths) = shrink_paths {
+                    deserialized_bank.set_shrink_paths(shrink_paths);
+                }
 
                 let deserialized_snapshot_hash = (
                     deserialized_bank.slot(),

ledger/src/blockstore.rs

@@ -253,26 +253,33 @@ impl Blockstore {
 
     /// Opens a Ledger in directory, provides "infinite" window of shreds
     pub fn open(ledger_path: &Path) -> Result<Blockstore> {
-        Self::do_open(ledger_path, AccessType::PrimaryOnly, None)
+        Self::do_open(ledger_path, AccessType::PrimaryOnly, None, true)
     }
 
     pub fn open_with_access_type(
         ledger_path: &Path,
         access_type: AccessType,
         recovery_mode: Option<BlockstoreRecoveryMode>,
+        enforce_ulimit_nofile: bool,
     ) -> Result<Blockstore> {
-        Self::do_open(ledger_path, access_type, recovery_mode)
+        Self::do_open(
+            ledger_path,
+            access_type,
+            recovery_mode,
+            enforce_ulimit_nofile,
+        )
     }
 
     fn do_open(
         ledger_path: &Path,
         access_type: AccessType,
         recovery_mode: Option<BlockstoreRecoveryMode>,
+        enforce_ulimit_nofile: bool,
     ) -> Result<Blockstore> {
         fs::create_dir_all(&ledger_path)?;
         let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY);
 
-        adjust_ulimit_nofile()?;
+        adjust_ulimit_nofile(enforce_ulimit_nofile)?;
 
         // Open the database
         let mut measure = Measure::start("open");
@@ -363,9 +370,14 @@ impl Blockstore {
     pub fn open_with_signal(
         ledger_path: &Path,
         recovery_mode: Option<BlockstoreRecoveryMode>,
+        enforce_ulimit_nofile: bool,
     ) -> Result<BlockstoreSignals> {
-        let mut blockstore =
-            Self::open_with_access_type(ledger_path, AccessType::PrimaryOnly, recovery_mode)?;
+        let mut blockstore = Self::open_with_access_type(
+            ledger_path,
+            AccessType::PrimaryOnly,
+            recovery_mode,
+            enforce_ulimit_nofile,
+        )?;
         let (ledger_signal_sender, ledger_signal_receiver) = sync_channel(1);
         let (completed_slots_sender, completed_slots_receiver) =
             sync_channel(MAX_COMPLETED_SLOTS_IN_CHANNEL);
@@ -2649,6 +2661,21 @@ impl Blockstore {
         matches!(self.db.get::<cf::Root>(slot), Ok(Some(true)))
     }
 
+    /// Returns true if a slot is between the rooted slot bounds of the ledger, but has not itself
+    /// been rooted. This is either because the slot was skipped, or due to a gap in ledger data,
+    /// as when booting from a newer snapshot.
+    pub fn is_skipped(&self, slot: Slot) -> bool {
+        let lowest_root = self
+            .rooted_slot_iterator(0)
+            .ok()
+            .and_then(|mut iter| iter.next())
+            .unwrap_or_default();
+        match self.db.get::<cf::Root>(slot).ok().flatten() {
+            Some(_) => false,
+            None => slot < self.max_root() && slot > lowest_root,
+        }
+    }
+
     pub fn set_roots(&self, rooted_slots: &[u64]) -> Result<()> {
         let mut write_batch = self.db.batch()?;
         for slot in rooted_slots {
@@ -3267,7 +3294,7 @@ pub fn create_new_ledger(
     genesis_config.write(&ledger_path)?;
 
     // Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger.
-    let blockstore = Blockstore::open_with_access_type(ledger_path, access_type, None)?;
+    let blockstore = Blockstore::open_with_access_type(ledger_path, access_type, None, false)?;
     let ticks_per_slot = genesis_config.ticks_per_slot;
     let hashes_per_tick = genesis_config.poh_config.hashes_per_tick.unwrap_or(0);
     let entries = create_ticks(ticks_per_slot, hashes_per_tick, genesis_config.hash());
@@ -3511,12 +3538,12 @@ pub fn make_chaining_slot_entries(
 }
 
 #[cfg(not(unix))]
-fn adjust_ulimit_nofile() -> Result<()> {
+fn adjust_ulimit_nofile(_enforce_ulimit_nofile: bool) -> Result<()> {
     Ok(())
 }
 
 #[cfg(unix)]
-fn adjust_ulimit_nofile() -> Result<()> {
+fn adjust_ulimit_nofile(enforce_ulimit_nofile: bool) -> Result<()> {
     // Rocks DB likes to have many open files.  The default open file descriptor limit is
     // usually not enough
     let desired_nofile = 500000;
@@ -3547,7 +3574,9 @@ fn adjust_ulimit_nofile() -> Result<()> {
                     desired_nofile, desired_nofile,
                 );
             }
-            return Err(BlockstoreError::UnableToSetOpenFileDescriptorLimit);
+            if enforce_ulimit_nofile {
+                return Err(BlockstoreError::UnableToSetOpenFileDescriptorLimit);
+            }
         }
 
         nofile = get_nofile();
@@ -4208,7 +4237,7 @@ pub mod tests {
     fn test_data_set_completed_on_insert() {
         let ledger_path = get_tmp_ledger_path!();
         let BlockstoreSignals { blockstore, .. } =
-            Blockstore::open_with_signal(&ledger_path, None).unwrap();
+            Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
 
         // Create enough entries to fill 2 shreds, only the later one is data complete
         let slot = 0;
@@ -4249,7 +4278,7 @@ pub mod tests {
             blockstore: ledger,
             ledger_signal_receiver: recvr,
             ..
-        } = Blockstore::open_with_signal(&ledger_path, None).unwrap();
+        } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 50;
@@ -4333,7 +4362,7 @@ pub mod tests {
             blockstore: ledger,
             completed_slots_receiver: recvr,
             ..
-        } = Blockstore::open_with_signal(&ledger_path, None).unwrap();
+        } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 10;
@@ -4359,7 +4388,7 @@ pub mod tests {
             blockstore: ledger,
             completed_slots_receiver: recvr,
             ..
-        } = Blockstore::open_with_signal(&ledger_path, None).unwrap();
+        } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 10;
@@ -4403,7 +4432,7 @@ pub mod tests {
             blockstore: ledger,
             completed_slots_receiver: recvr,
             ..
-        } = Blockstore::open_with_signal(&ledger_path, None).unwrap();
+        } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 10;
@@ -5523,6 +5552,25 @@ pub mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
+    #[test]
+    fn test_is_skipped() {
+        let blockstore_path = get_tmp_ledger_path!();
+        let blockstore = Blockstore::open(&blockstore_path).unwrap();
+        let roots = vec![2, 4, 7, 12, 15];
+        blockstore.set_roots(&roots).unwrap();
+
+        for i in 0..20 {
+            if i < 2 || roots.contains(&i) || i > 15 {
+                assert!(!blockstore.is_skipped(i));
+            } else {
+                assert!(blockstore.is_skipped(i));
+            }
+        }
+
+        drop(blockstore);
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
     #[test]
     fn test_iter_bounds() {
         let blockstore_path = get_tmp_ledger_path!();

ledger/src/blockstore_processor.rs

@@ -15,6 +15,7 @@ use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::{datapoint_error, inc_new_counter_debug};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_runtime::{
+    accounts_index::AccountIndex,
     bank::{
         Bank, InnerInstructionsList, TransactionBalancesSet, TransactionExecutionResult,
         TransactionLogMessages, TransactionResults,
@@ -344,6 +345,7 @@ pub struct ProcessOptions {
     pub new_hard_forks: Option<Vec<Slot>>,
     pub frozen_accounts: Vec<Pubkey>,
     pub debug_keys: Option<Arc<HashSet<Pubkey>>>,
+    pub account_indexes: HashSet<AccountIndex>,
 }
 
 pub fn process_blockstore(
@@ -367,10 +369,8 @@ pub fn process_blockstore(
         account_paths,
         &opts.frozen_accounts,
         opts.debug_keys.clone(),
-        Some(&crate::builtins::get(
-            genesis_config.cluster_type,
-            opts.bpf_jit,
-        )),
+        Some(&crate::builtins::get(opts.bpf_jit)),
+        opts.account_indexes.clone(),
     );
     let bank0 = Arc::new(bank0);
     info!("processing ledger for slot 0...");
@@ -928,7 +928,7 @@ fn load_frozen_forks(
                 leader_schedule_cache.set_root(&new_root_bank);
                 new_root_bank.squash();
 
-                if last_free.elapsed() > Duration::from_secs(30) {
+                if last_free.elapsed() > Duration::from_secs(10) {
                     // This could take few secs; so update last_free later
                     new_root_bank.exhaustively_free_unused_resource();
                     last_free = Instant::now();
@@ -2894,7 +2894,14 @@ pub mod tests {
         genesis_config: &GenesisConfig,
         account_paths: Vec<PathBuf>,
     ) -> EpochSchedule {
-        let bank = Bank::new_with_paths(&genesis_config, account_paths, &[], None, None);
+        let bank = Bank::new_with_paths(
+            &genesis_config,
+            account_paths,
+            &[],
+            None,
+            None,
+            HashSet::new(),
+        );
         *bank.epoch_schedule()
     }
 
@@ -3220,7 +3227,7 @@ pub mod tests {
                         vote_state.root_slot = Some(root);
                         let mut vote_account =
                             Account::new(1, VoteState::size_of(), &solana_vote_program::id());
-                        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+                        let versioned = VoteStateVersions::new_current(vote_state);
                         VoteState::serialize(&versioned, &mut vote_account.data).unwrap();
                         (
                             solana_sdk::pubkey::new_rand(),

ledger/src/builtins.rs

@@ -2,7 +2,7 @@ use solana_runtime::{
     bank::{Builtin, Builtins},
     builtins::ActivationType,
 };
-use solana_sdk::{feature_set, genesis_config::ClusterType, pubkey::Pubkey};
+use solana_sdk::{feature_set, pubkey::Pubkey};
 
 macro_rules! to_builtin {
     ($b:expr) => {
@@ -11,47 +11,33 @@ macro_rules! to_builtin {
 }
 
 /// Builtin programs that are always available
-fn genesis_builtins(cluster_type: ClusterType, bpf_jit: bool) -> Vec<Builtin> {
-    if cluster_type != ClusterType::MainnetBeta {
-        vec![
-            to_builtin!(solana_bpf_loader_deprecated_program!()),
-            if bpf_jit {
-                to_builtin!(solana_bpf_loader_program_with_jit!())
-            } else {
-                to_builtin!(solana_bpf_loader_program!())
-            },
-            if bpf_jit {
-                to_builtin!(solana_bpf_loader_upgradeable_program_with_jit!())
-            } else {
-                to_builtin!(solana_bpf_loader_upgradeable_program!())
-            },
-        ]
-    } else {
-        // Remove this `else` block and the `cluster_type` argument to this function once
-        // `feature_set::bpf_loader2_program::id()` is active on Mainnet Beta
-        vec![to_builtin!(solana_bpf_loader_deprecated_program!())]
-    }
-}
-
-/// Builtin programs activated dynamically by feature
-fn feature_builtins() -> Vec<(Builtin, Pubkey, ActivationType)> {
+fn genesis_builtins(bpf_jit: bool) -> Vec<Builtin> {
     vec![
-        (
-            to_builtin!(solana_bpf_loader_program!()),
-            feature_set::bpf_loader2_program::id(),
-            ActivationType::NewProgram,
-        ),
-        (
-            to_builtin!(solana_bpf_loader_upgradeable_program!()),
-            feature_set::bpf_loader_upgradeable_program::id(),
-            ActivationType::NewProgram,
-        ),
+        to_builtin!(solana_bpf_loader_deprecated_program!()),
+        if bpf_jit {
+            to_builtin!(solana_bpf_loader_program_with_jit!())
+        } else {
+            to_builtin!(solana_bpf_loader_program!())
+        },
     ]
 }
 
-pub(crate) fn get(cluster_type: ClusterType, bpf_jit: bool) -> Builtins {
+/// Builtin programs activated dynamically by feature
+fn feature_builtins(bpf_jit: bool) -> Vec<(Builtin, Pubkey, ActivationType)> {
+    vec![(
+        if bpf_jit {
+            to_builtin!(solana_bpf_loader_upgradeable_program_with_jit!())
+        } else {
+            to_builtin!(solana_bpf_loader_upgradeable_program!())
+        },
+        feature_set::bpf_loader_upgradeable_program::id(),
+        ActivationType::NewProgram,
+    )]
+}
+
+pub(crate) fn get(bpf_jit: bool) -> Builtins {
     Builtins {
-        genesis_builtins: genesis_builtins(cluster_type, bpf_jit),
-        feature_builtins: feature_builtins(),
+        genesis_builtins: genesis_builtins(bpf_jit),
+        feature_builtins: feature_builtins(bpf_jit),
     }
 }

ledger/src/leader_schedule_utils.rs

@@ -1,5 +1,4 @@
 use crate::leader_schedule::LeaderSchedule;
-use crate::staking_utils;
 use solana_runtime::bank::Bank;
 use solana_sdk::{
     clock::{Epoch, Slot, NUM_CONSECUTIVE_LEADER_SLOTS},
@@ -8,7 +7,7 @@ use solana_sdk::{
 
 /// Return the leader schedule for the given epoch.
 pub fn leader_schedule(epoch: Epoch, bank: &Bank) -> Option<LeaderSchedule> {
-    staking_utils::staked_nodes_at_epoch(bank, epoch).map(|stakes| {
+    bank.epoch_staked_nodes(epoch).map(|stakes| {
         let mut seed = [0u8; 32];
         seed[0..8].copy_from_slice(&epoch.to_le_bytes());
         let mut stakes: Vec<_> = stakes.into_iter().collect();
@@ -66,7 +65,7 @@ mod tests {
                 .genesis_config;
         let bank = Bank::new(&genesis_config);
 
-        let pubkeys_and_stakes: Vec<_> = staking_utils::staked_nodes(&bank).into_iter().collect();
+        let pubkeys_and_stakes: Vec<_> = bank.staked_nodes().into_iter().collect();
         let seed = [0u8; 32];
         let leader_schedule = LeaderSchedule::new(
             &pubkeys_and_stakes,

ledger/src/lib.rs

@@ -1,3 +1,4 @@
+#![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))]
 #[macro_use]
 extern crate solana_bpf_loader_program;
 
@@ -32,3 +33,6 @@ extern crate log;
 
 #[macro_use]
 extern crate lazy_static;
+
+#[macro_use]
+extern crate solana_frozen_abi_macro;

ledger/src/poh.rs

@@ -82,18 +82,18 @@ impl Poh {
     }
 }
 
-pub fn compute_hashes_per_tick(duration: Duration, hashes_sample_size: u64) -> u64 {
-    // calculate hash rate with the system under maximum load
-    info!(
-        "Running {} hashes in parallel on all threads...",
-        hashes_sample_size
-    );
+pub fn compute_hash_time_ns(hashes_sample_size: u64) -> u64 {
+    info!("Running {} hashes...", hashes_sample_size);
     let mut v = Hash::default();
     let start = Instant::now();
     for _ in 0..hashes_sample_size {
         v = hash(&v.as_ref());
     }
-    let elapsed = start.elapsed().as_millis() as u64;
+    start.elapsed().as_nanos() as u64
+}
+
+pub fn compute_hashes_per_tick(duration: Duration, hashes_sample_size: u64) -> u64 {
+    let elapsed = compute_hash_time_ns(hashes_sample_size) / (1000 * 1000);
     duration.as_millis() as u64 * hashes_sample_size / elapsed
 }
 

ledger/src/shred.rs

@@ -1,5 +1,6 @@
 //! The `shred` module defines data structures and methods to pull MTU sized data frames from the network.
 use crate::{
+    blockstore::MAX_DATA_SHREDS_PER_SLOT,
     entry::{create_ticks, Entry},
     erasure::Session,
 };
@@ -12,7 +13,7 @@ use rayon::{
 };
 use serde::{Deserialize, Serialize};
 use solana_measure::measure::Measure;
-use solana_perf::packet::Packet;
+use solana_perf::packet::{limited_deserialize, Packet};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_sdk::{
     clock::Slot,
@@ -118,7 +119,7 @@ pub enum ShredError {
 
 pub type Result<T> = std::result::Result<T, ShredError>;
 
-#[derive(Serialize, Clone, Deserialize, PartialEq, Debug)]
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, AbiExample, Deserialize, Serialize)]
 pub struct ShredType(pub u8);
 impl Default for ShredType {
     fn default() -> Self {
@@ -309,6 +310,27 @@ impl Shred {
         Ok(shred)
     }
 
+    pub fn new_empty_coding(
+        slot: Slot,
+        index: u32,
+        fec_set_index: u32,
+        num_data: usize,
+        num_code: usize,
+        position: usize,
+        version: u16,
+    ) -> Self {
+        let (header, coding_header) = Shredder::new_coding_shred_header(
+            slot,
+            index,
+            fec_set_index,
+            num_data,
+            num_code,
+            position,
+            version,
+        );
+        Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header)
+    }
+
     pub fn new_empty_from_header(
         common_header: ShredCommonHeader,
         data_header: DataShredHeader,
@@ -699,7 +721,7 @@ impl Shredder {
             // Create empty coding shreds, with correctly populated headers
             let mut coding_shreds = Vec::with_capacity(num_coding);
             (0..num_coding).for_each(|i| {
-                let (header, coding_header) = Self::new_coding_shred_header(
+                let shred = Shred::new_empty_coding(
                     slot,
                     start_index + i as u32,
                     start_index,
@@ -708,8 +730,6 @@ impl Shredder {
                     i,
                     version,
                 );
-                let shred =
-                    Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header);
                 coding_shreds.push(shred.payload);
             });
 
@@ -730,7 +750,7 @@ impl Shredder {
                 .into_iter()
                 .enumerate()
                 .map(|(i, payload)| {
-                    let (common_header, coding_header) = Self::new_coding_shred_header(
+                    let mut shred = Shred::new_empty_coding(
                         slot,
                         start_index + i as u32,
                         start_index,
@@ -739,12 +759,8 @@ impl Shredder {
                         i,
                         version,
                     );
-                    Shred {
-                        common_header,
-                        data_header: DataShredHeader::default(),
-                        coding_header,
-                        payload,
-                    }
+                    shred.payload = payload;
+                    shred
                 })
                 .collect()
         } else {
@@ -963,6 +979,71 @@ impl Shredder {
     }
 }
 
+#[derive(Default, Debug, Eq, PartialEq)]
+pub struct ShredFetchStats {
+    pub index_overrun: usize,
+    pub shred_count: usize,
+    pub index_bad_deserialize: usize,
+    pub index_out_of_bounds: usize,
+    pub slot_bad_deserialize: usize,
+    pub duplicate_shred: usize,
+    pub slot_out_of_range: usize,
+    pub bad_shred_type: usize,
+}
+
+// Get slot, index, and type from a packet with partial deserialize
+pub fn get_shred_slot_index_type(
+    p: &Packet,
+    stats: &mut ShredFetchStats,
+) -> Option<(Slot, u32, bool)> {
+    let index_start = OFFSET_OF_SHRED_INDEX;
+    let index_end = index_start + SIZE_OF_SHRED_INDEX;
+    let slot_start = OFFSET_OF_SHRED_SLOT;
+    let slot_end = slot_start + SIZE_OF_SHRED_SLOT;
+
+    debug_assert!(index_end > slot_end);
+    debug_assert!(index_end > OFFSET_OF_SHRED_TYPE);
+
+    if index_end > p.meta.size {
+        stats.index_overrun += 1;
+        return None;
+    }
+
+    let index;
+    match limited_deserialize::<u32>(&p.data[index_start..index_end]) {
+        Ok(x) => index = x,
+        Err(_e) => {
+            stats.index_bad_deserialize += 1;
+            return None;
+        }
+    }
+
+    if index >= MAX_DATA_SHREDS_PER_SLOT as u32 {
+        stats.index_out_of_bounds += 1;
+        return None;
+    }
+
+    let slot;
+    match limited_deserialize::<Slot>(&p.data[slot_start..slot_end]) {
+        Ok(x) => {
+            slot = x;
+        }
+        Err(_e) => {
+            stats.slot_bad_deserialize += 1;
+            return None;
+        }
+    }
+
+    let shred_type = p.data[OFFSET_OF_SHRED_TYPE];
+    if shred_type == DATA_SHRED || shred_type == CODING_SHRED {
+        return Some((slot, index, shred_type == DATA_SHRED));
+    } else {
+        stats.bad_shred_type += 1;
+    }
+
+    None
+}
+
 pub fn max_ticks_per_n_shreds(num_shreds: u64, shred_data_size: Option<usize>) -> u64 {
     let ticks = create_ticks(1, 0, Hash::default());
     max_entries_per_n_shred(&ticks[0], num_shreds, shred_data_size)
@@ -1707,4 +1788,60 @@ pub mod tests {
             })
         );
     }
+
+    #[test]
+    fn test_shred_offsets() {
+        solana_logger::setup();
+        let mut packet = Packet::default();
+        let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
+        shred.copy_to_packet(&mut packet);
+        let mut stats = ShredFetchStats::default();
+        let ret = get_shred_slot_index_type(&packet, &mut stats);
+        assert_eq!(Some((1, 3, true)), ret);
+        assert_eq!(stats, ShredFetchStats::default());
+
+        packet.meta.size = OFFSET_OF_SHRED_TYPE;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 1);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 2);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX + 1;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 3);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX - 1;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 4);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX;
+        assert_eq!(
+            Some((1, 3, true)),
+            get_shred_slot_index_type(&packet, &mut stats)
+        );
+        assert_eq!(stats.index_overrun, 4);
+
+        let shred = Shred::new_empty_coding(8, 2, 10, 30, 4, 7, 200);
+        shred.copy_to_packet(&mut packet);
+        assert_eq!(
+            Some((8, 2, false)),
+            get_shred_slot_index_type(&packet, &mut stats)
+        );
+
+        let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, None, true, true, 0, 0, 0);
+        shred.copy_to_packet(&mut packet);
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(1, stats.index_out_of_bounds);
+
+        let (mut header, coding_header) =
+            Shredder::new_coding_shred_header(8, 2, 10, 30, 4, 7, 200);
+        header.shred_type = ShredType(u8::MAX);
+        let shred = Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header);
+        shred.copy_to_packet(&mut packet);
+
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(1, stats.bad_shred_type);
+    }
 }

ledger/src/staking_utils.rs

@@ -1,9 +1,9 @@
-use solana_runtime::{bank::Bank, vote_account::ArcVoteAccount};
+use solana_runtime::bank::Bank;
 use solana_sdk::{
     clock::{Epoch, Slot},
     pubkey::Pubkey,
 };
-use std::{borrow::Borrow, collections::HashMap};
+use std::collections::HashMap;
 
 /// Looks through vote accounts, and finds the latest slot that has achieved
 /// supermajority lockout
@@ -24,36 +24,6 @@ pub fn vote_account_stakes(bank: &Bank) -> HashMap<Pubkey, u64> {
         .collect()
 }
 
-/// Collect the staked nodes, as named by staked vote accounts from the given bank
-pub fn staked_nodes(bank: &Bank) -> HashMap<Pubkey, u64> {
-    to_staked_nodes(bank.vote_accounts())
-}
-
-/// At the specified epoch, collect the delegate account balance and vote states for delegates
-/// that have non-zero balance in any of their managed staking accounts
-pub fn staked_nodes_at_epoch(bank: &Bank, epoch: Epoch) -> Option<HashMap<Pubkey, u64>> {
-    bank.epoch_vote_accounts(epoch).map(to_staked_nodes)
-}
-
-fn to_staked_nodes<I, K, V>(
-    vote_accounts: I,
-) -> HashMap<Pubkey /*VoteState.node_pubkey*/, u64 /*stake*/>
-where
-    I: IntoIterator<Item = (K /*vote pubkey*/, V)>,
-    V: Borrow<(u64 /*stake*/, ArcVoteAccount)>,
-{
-    let mut out: HashMap<Pubkey, u64> = HashMap::new();
-    for (_ /*vote pubkey*/, stake_vote_account) in vote_accounts {
-        let (stake, vote_account) = stake_vote_account.borrow();
-        if let Ok(vote_state) = vote_account.vote_state().as_ref() {
-            out.entry(vote_state.node_pubkey)
-                .and_modify(|s| *s += *stake)
-                .or_insert(*stake);
-        }
-    }
-    out
-}
-
 fn epoch_stakes_and_lockouts(bank: &Bank, epoch: Epoch) -> Vec<(u64, Option<u64>)> {
     bank.epoch_vote_accounts(epoch)
         .expect("Bank state for epoch is missing")
@@ -96,6 +66,7 @@ pub(crate) mod tests {
         bootstrap_validator_stake_lamports, create_genesis_config, GenesisConfigInfo,
     };
     use rand::Rng;
+    use solana_runtime::vote_account::{ArcVoteAccount, VoteAccounts};
     use solana_sdk::{
         account::{from_account, Account},
         clock::Clock,
@@ -340,14 +311,14 @@ pub(crate) mod tests {
         let vote_accounts = stakes.into_iter().map(|(stake, vote_state)| {
             let account = Account::new_data(
                 rng.gen(), // lamports
-                &VoteStateVersions::Current(Box::new(vote_state)),
+                &VoteStateVersions::new_current(vote_state),
                 &Pubkey::new_unique(), // owner
             )
             .unwrap();
             let vote_pubkey = Pubkey::new_unique();
             (vote_pubkey, (stake, ArcVoteAccount::from(account)))
         });
-        let result = to_staked_nodes(vote_accounts);
+        let result = vote_accounts.collect::<VoteAccounts>().staked_nodes();
         assert_eq!(result.len(), 2);
         assert_eq!(result[&node1], 3);
         assert_eq!(result[&node2], 5);

local-cluster/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-local-cluster"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,21 +15,21 @@ gag = "0.1.10"
 fs_extra = "1.1.0"
 log = "0.4.11"
 rand = "0.7.0"
-solana-config-program = { path = "../programs/config", version = "1.5.0" }
-solana-core = { path = "../core", version = "1.5.0" }
-solana-client = { path = "../client", version = "1.5.0" }
-solana-download-utils = { path = "../download-utils", version = "1.5.0" }
-solana-faucet = { path = "../faucet", version = "1.5.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.5.0" }
-solana-ledger = { path = "../ledger", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-runtime = { path = "../runtime", version = "1.5.0" }
-solana-sdk = { path = "../sdk", version = "1.5.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.5.0" }
-solana-vest-program = { path = "../programs/vest", version = "1.5.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.5.0" }
+solana-config-program = { path = "../programs/config", version = "1.5.1" }
+solana-core = { path = "../core", version = "1.5.1" }
+solana-client = { path = "../client", version = "1.5.1" }
+solana-download-utils = { path = "../download-utils", version = "1.5.1" }
+solana-faucet = { path = "../faucet", version = "1.5.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.5.1" }
+solana-ledger = { path = "../ledger", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-runtime = { path = "../runtime", version = "1.5.1" }
+solana-sdk = { path = "../sdk", version = "1.5.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.5.1" }
+solana-vest-program = { path = "../programs/vest", version = "1.5.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.5.1" }
 tempfile = "3.1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.5.1" }
 
 [dev-dependencies]
 assert_matches = "1.3.0"

local-cluster/src/local_cluster.rs

@@ -205,7 +205,7 @@ impl LocalCluster {
             &leader_ledger_path,
             &leader_vote_keypair.pubkey(),
             vec![leader_vote_keypair.clone()],
-            None,
+            vec![],
             &leader_config,
         );
 
@@ -348,7 +348,7 @@ impl LocalCluster {
             &ledger_path,
             &voting_keypair.pubkey(),
             vec![voting_keypair.clone()],
-            Some(&self.entry_point_info),
+            vec![self.entry_point_info.clone()],
             &config,
         );
 
@@ -660,7 +660,9 @@ impl Cluster for LocalCluster {
             &validator_info.ledger_path,
             &validator_info.voting_keypair.pubkey(),
             vec![validator_info.voting_keypair.clone()],
-            entry_point_info.as_ref(),
+            entry_point_info
+                .map(|entry_point_info| vec![entry_point_info])
+                .unwrap_or_default(),
             &cluster_validator_info.config,
         );
         cluster_validator_info.validator = Some(restarted_node);

local-cluster/tests/local_cluster.rs

@@ -783,6 +783,7 @@ fn test_mainnet_beta_cluster_type() {
         &solana_stake_program::id(),
         &solana_vote_program::id(),
         &solana_sdk::bpf_loader_deprecated::id(),
+        &solana_sdk::bpf_loader::id(),
     ]
     .iter()
     {
@@ -798,7 +799,12 @@ fn test_mainnet_beta_cluster_type() {
     }
 
     // Programs that are not available at epoch 0
-    for program_id in [&solana_sdk::bpf_loader::id(), &solana_vest_program::id()].iter() {
+    for program_id in [
+        &solana_sdk::bpf_loader_upgradeable::id(),
+        &solana_vest_program::id(),
+    ]
+    .iter()
+    {
         assert_eq!(
             (
                 program_id,
@@ -1657,11 +1663,10 @@ fn test_validator_saves_tower() {
 }
 
 fn open_blockstore(ledger_path: &Path) -> Blockstore {
-    Blockstore::open_with_access_type(ledger_path, AccessType::PrimaryOnly, None).unwrap_or_else(
-        |e| {
+    Blockstore::open_with_access_type(ledger_path, AccessType::PrimaryOnly, None, true)
+        .unwrap_or_else(|e| {
             panic!("Failed to open ledger at {:?}, err: {}", ledger_path, e);
-        },
-    )
+        })
 }
 
 fn purge_slots(blockstore: &Blockstore, start_slot: Slot, slot_count: Slot) {
@@ -1881,6 +1886,7 @@ fn do_test_optimistic_confirmation_violation_with_or_without_tower(with_tower: b
                 &val_a_ledger_path,
                 AccessType::TryPrimaryThenSecondary,
                 None,
+                true,
             )
             .unwrap();
             let mut ancestors = AncestorIterator::new(last_vote, &blockstore);

log-analyzer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-log-analyzer"
 description = "The solana cluster network analysis tool"
-version = "1.5.0"
+version = "1.5.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,9 +14,9 @@ byte-unit = "4.0.8"
 clap = "2.33.1"
 serde = "1.0.112"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "1.5.0" }
-solana-logger = { path = "../logger", version = "1.5.0" }
-solana-version = { path = "../version", version = "1.5.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.5.1" }
+solana-logger = { path = "../logger", version = "1.5.1" }
+solana-version = { path = "../version", version = "1.5.1" }
 
 [[bin]]
 name = "solana-log-analyzer"