Different error if block status is not yet available (#20407) (#21029)

* Different error if block is not available

* Add slot to error message

* Make and use helper function

* Check finalized path as well

Co-authored-by: Tyera Eulberg <tyera@solana.com>
(cherry picked from commit 700e42d556)

Co-authored-by: sakridge <sakridge@gmail.com>

.travis.yml

@@ -61,6 +61,12 @@ jobs:
     - <<: *release-artifacts
       name: "Windows release artifacts"
       os: windows
+      install:
+        - choco install openssl
+        - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64"
+        - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
     #  Linux release artifacts are still built by ci/buildkite-secondary.yml
     #- <<: *release-artifacts
     #  name: "Linux release artifacts"
@@ -117,7 +123,7 @@ jobs:
       if: type IN (push, pull_request) OR tag IS present
       language: node_js
       node_js:
-        - "node"
+        - "lts/*"
 
       services:
         - docker

Cargo.toml

@@ -1,5 +1,8 @@
 [workspace]
 members = [
+    "accountsdb-plugin-interface",
+    "accountsdb-plugin-manager",
+    "accountsdb-plugin-postgres",
     "accounts-cluster-bench",
     "bench-exchange",
     "bench-streamer",
@@ -43,11 +46,10 @@ members = [
     "poh-bench",
     "program-test",
     "programs/bpf_loader",
+    "programs/compute-budget",
     "programs/config",
     "programs/exchange",
-    "programs/failure",
-    "programs/noop",
-    "programs/ownable",
+    "programs/ed25519",
     "programs/secp256k1",
     "programs/stake",
     "programs/vote",

README.md

@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://solana.com">
-    <img alt="Solana" src="https://i.imgur.com/uBVzyX3.png" width="250" />
+    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
   </a>
 </p>
 
@@ -51,11 +51,6 @@ $ cd solana
 $ cargo build
 ```
 
-## **4. Run a minimal local cluster.**
-```bash
-$ ./run.sh
-```
-
 # Testing
 
 **Run the test suite:**

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.7.12"
+version = "1.8.2"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,9 +19,9 @@ lazy_static = "1.4.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.12" }
+solana-config-program = { path = "../programs/config", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.2" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.2.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"

account-decoder/src/lib.rs

@@ -49,7 +49,7 @@ pub enum UiAccountData {
     Binary(String, UiAccountEncoding),
 }
 
-#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq)]
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)]
 #[serde(rename_all = "camelCase")]
 pub enum UiAccountEncoding {
     Binary, // Legacy. Retained for RPC backwards compatibility
@@ -179,7 +179,7 @@ impl Default for UiFeeCalculator {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UiDataSliceConfig {
     pub offset: usize,

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-measure = { path = "../measure", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-measure = { path = "../measure", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -66,6 +66,7 @@ fn main() {
         AccountSecondaryIndexes::default(),
         false,
         AccountShrinkThreshold::default(),
+        None,
     );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-cluster-bench"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,24 +13,24 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.12" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-client = { path = "../client", version = "=1.7.12" }
-solana-core = { path = "../core", version = "=1.7.12" }
-solana-faucet = { path = "../faucet", version = "=1.7.12" }
-solana-gossip = { path = "../gossip", version = "=1.7.12" }
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-measure = { path = "../measure", version = "=1.7.12" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.12" }
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-streamer = { path = "../streamer", version = "=1.7.12" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.2" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-client = { path = "../client", version = "=1.8.2" }
+solana-core = { path = "../core", version = "=1.8.2" }
+solana-faucet = { path = "../faucet", version = "=1.8.2" }
+solana-gossip = { path = "../gossip", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-measure = { path = "../measure", version = "=1.8.2" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-streamer = { path = "../streamer", version = "=1.8.2" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.2.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.12" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.8.2" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accountsdb-plugin-interface"
+description = "The Solana AccountsDb plugin interface."
+version = "1.8.2"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[dependencies]
+log = "0.4.11"
+thiserror = "1.0.29"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/README.md

@@ -0,0 +1,20 @@
+<p align="center">
+  <a href="https://solana.com">
+    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
+  </a>
+</p>
+
+# Solana AccountsDb Plugin Interface
+
+This crate enables an AccountsDb plugin to be plugged into the Solana Validator runtime to take actions
+at the time of each account update; for example, saving the account state to an external database. The plugin must implement the `AccountsDbPlugin` trait. Please see the detail of the `accountsdb_plugin_interface.rs` for the interface definition.
+
+The plugin should produce a `cdylib` dynamic library, which must expose a `C` function `_create_plugin()` that
+instantiates the implementation of the interface.
+
+The `solana-accountsdb-plugin-postgres` crate provides an example of how to create a plugin which saves the accounts data into an
+external PostgreSQL databases.
+
+More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
+
+Still have questions?  Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

accountsdb-plugin-interface/src/accountsdb_plugin_interface.rs

@@ -0,0 +1,99 @@
+/// The interface for AccountsDb plugins. A plugin must implement
+/// the AccountsDbPlugin trait to work with the runtime.
+/// In addition, the dynamic library must export a "C" function _create_plugin which
+/// creates the implementation of the plugin.
+use {
+    std::{any::Any, error, io},
+    thiserror::Error,
+};
+
+impl Eq for ReplicaAccountInfo<'_> {}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct ReplicaAccountInfo<'a> {
+    pub pubkey: &'a [u8],
+    pub lamports: u64,
+    pub owner: &'a [u8],
+    pub executable: bool,
+    pub rent_epoch: u64,
+    pub data: &'a [u8],
+    pub write_version: u64,
+}
+
+pub enum ReplicaAccountInfoVersions<'a> {
+    V0_0_1(&'a ReplicaAccountInfo<'a>),
+}
+
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginError {
+    #[error("Error opening config file. Error detail: ({0}).")]
+    ConfigFileOpenError(#[from] io::Error),
+
+    #[error("Error reading config file. Error message: ({msg})")]
+    ConfigFileReadError { msg: String },
+
+    #[error("Error updating account. Error message: ({msg})")]
+    AccountsUpdateError { msg: String },
+
+    #[error("Error updating slot status. Error message: ({msg})")]
+    SlotStatusUpdateError { msg: String },
+
+    #[error("Plugin-defined custom error. Error message: ({0})")]
+    Custom(Box<dyn error::Error + Send + Sync>),
+}
+
+#[derive(Debug, Clone)]
+pub enum SlotStatus {
+    Processed,
+    Rooted,
+    Confirmed,
+}
+
+impl SlotStatus {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            SlotStatus::Confirmed => "confirmed",
+            SlotStatus::Processed => "processed",
+            SlotStatus::Rooted => "rooted",
+        }
+    }
+}
+
+pub type Result<T> = std::result::Result<T, AccountsDbPluginError>;
+
+pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
+    fn name(&self) -> &'static str;
+
+    /// The callback called when a plugin is loaded by the system,
+    /// used for doing whatever initialization is required by the plugin.
+    /// The _config_file contains the name of the
+    /// of the config file. The config must be in JSON format and
+    /// include a field "libpath" indicating the full path
+    /// name of the shared library implementing this interface.
+    fn on_load(&mut self, _config_file: &str) -> Result<()> {
+        Ok(())
+    }
+
+    /// The callback called right before a plugin is unloaded by the system
+    /// Used for doing cleanup before unload.
+    fn on_unload(&mut self) {}
+
+    /// Called when an account is updated at a slot.
+    fn update_account(
+        &mut self,
+        account: ReplicaAccountInfoVersions,
+        slot: u64,
+        is_startup: bool,
+    ) -> Result<()>;
+
+    /// Called when all accounts are notified of during startup.
+    fn notify_end_of_startup(&mut self) -> Result<()>;
+
+    /// Called when a slot status is updated
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()>;
+}

accountsdb-plugin-interface/src/lib.rs

@@ -0,0 +1 @@
+pub mod accountsdb_plugin_interface;

accountsdb-plugin-manager/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accountsdb-plugin-manager"
+description = "The Solana AccountsDb plugin manager."
+version = "1.8.2"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[dependencies]
+bs58 = "0.4.0"
+crossbeam-channel = "0.4"
+libloading = "0.7.0"
+log = "0.4.11"
+serde = "1.0.130"
+serde_derive = "1.0.103"
+serde_json = "1.0.67"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-measure = { path = "../measure", version = "=1.8.2" }
+solana-metrics = { path = "../metrics", version = "=1.8.2" }
+solana-rpc = { path = "../rpc", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+thiserror = "1.0.21"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-manager/src/accounts_update_notifier.rs

@@ -0,0 +1,227 @@
+/// Module responsible for notifying plugins of account updates
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaAccountInfo, ReplicaAccountInfoVersions, SlotStatus,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_runtime::{
+        accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
+        append_vec::{StoredAccountMeta, StoredMeta},
+    },
+    solana_sdk::{
+        account::{AccountSharedData, ReadableAccount},
+        clock::Slot,
+    },
+    std::sync::{Arc, RwLock},
+};
+#[derive(Debug)]
+pub(crate) struct AccountsUpdateNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
+    fn notify_account_update(&self, slot: Slot, meta: &StoredMeta, account: &AccountSharedData) {
+        if let Some(account_info) = self.accountinfo_from_shared_account_data(meta, account) {
+            self.notify_plugins_of_account_update(account_info, slot, false);
+        }
+    }
+
+    fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta) {
+        let mut measure_all = Measure::start("accountsdb-plugin-notify-account-restore-all");
+        let mut measure_copy = Measure::start("accountsdb-plugin-copy-stored-account-info");
+
+        let account = self.accountinfo_from_stored_account_meta(account);
+        measure_copy.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-copy-stored-account-info-us",
+            measure_copy.as_us() as usize,
+            100000,
+            100000
+        );
+
+        if let Some(account_info) = account {
+            self.notify_plugins_of_account_update(account_info, slot, true);
+        }
+        measure_all.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify-account-restore-all-us",
+            measure_all.as_us() as usize,
+            100000,
+            100000
+        );
+    }
+
+    fn notify_end_of_restore_from_snapshot(&self) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-end-of-restore-from-snapshot");
+            match plugin.notify_end_of_startup() {
+                Err(err) => {
+                    error!(
+                        "Failed to notify the end of restore from snapshot, error: {} to plugin {}",
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully notified the end of restore from snapshot to plugin {}",
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-end-of-restore-from-snapshot",
+                measure.as_us() as usize
+            );
+        }
+    }
+
+    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Confirmed);
+    }
+
+    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Processed);
+    }
+
+    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Rooted);
+    }
+}
+
+impl AccountsUpdateNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        AccountsUpdateNotifierImpl { plugin_manager }
+    }
+
+    fn accountinfo_from_shared_account_data<'a>(
+        &self,
+        meta: &'a StoredMeta,
+        account: &'a AccountSharedData,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: meta.pubkey.as_ref(),
+            lamports: account.lamports(),
+            owner: account.owner().as_ref(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch(),
+            data: account.data(),
+            write_version: meta.write_version,
+        })
+    }
+
+    fn accountinfo_from_stored_account_meta<'a>(
+        &self,
+        stored_account_meta: &'a StoredAccountMeta,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: stored_account_meta.meta.pubkey.as_ref(),
+            lamports: stored_account_meta.account_meta.lamports,
+            owner: stored_account_meta.account_meta.owner.as_ref(),
+            executable: stored_account_meta.account_meta.executable,
+            rent_epoch: stored_account_meta.account_meta.rent_epoch,
+            data: stored_account_meta.data,
+            write_version: stored_account_meta.meta.write_version,
+        })
+    }
+
+    fn notify_plugins_of_account_update(
+        &self,
+        account: ReplicaAccountInfo,
+        slot: Slot,
+        is_startup: bool,
+    ) {
+        let mut measure2 = Measure::start("accountsdb-plugin-notify_plugins_of_account_update");
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-account");
+            match plugin.update_account(
+                ReplicaAccountInfoVersions::V0_0_1(&account),
+                slot,
+                is_startup,
+            ) {
+                Err(err) => {
+                    error!(
+                        "Failed to update account {} at slot {}, error: {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated account {} at slot {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-account-us",
+                measure.as_us() as usize,
+                100000,
+                100000
+            );
+        }
+        measure2.stop();
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify_plugins_of_account_update-us",
+            measure2.as_us() as usize,
+            100000,
+            100000
+        );
+    }
+
+    pub fn notify_slot_status(&self, slot: Slot, parent: Option<Slot>, slot_status: SlotStatus) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-slot");
+            match plugin.update_slot_status(slot, parent, slot_status.clone()) {
+                Err(err) => {
+                    error!(
+                        "Failed to update slot status at slot {}, error: {} to plugin {}",
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated slot status at slot {} to plugin {}",
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-slot-us",
+                measure.as_us() as usize,
+                1000,
+                1000
+            );
+        }
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_manager.rs

@@ -0,0 +1,55 @@
+/// Managing the AccountsDb plugins
+use {
+    libloading::{Library, Symbol},
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::AccountsDbPlugin,
+    std::error::Error,
+};
+
+#[derive(Default, Debug)]
+pub struct AccountsDbPluginManager {
+    pub plugins: Vec<Box<dyn AccountsDbPlugin>>,
+    libs: Vec<Library>,
+}
+
+impl AccountsDbPluginManager {
+    pub fn new() -> Self {
+        AccountsDbPluginManager {
+            plugins: Vec::default(),
+            libs: Vec::default(),
+        }
+    }
+
+    /// # Safety
+    ///
+    /// This function loads the dynamically linked library specified in the path. The library
+    /// must do necessary initializations.
+    pub unsafe fn load_plugin(
+        &mut self,
+        libpath: &str,
+        config_file: &str,
+    ) -> Result<(), Box<dyn Error>> {
+        type PluginConstructor = unsafe fn() -> *mut dyn AccountsDbPlugin;
+        let lib = Library::new(libpath)?;
+        let constructor: Symbol<PluginConstructor> = lib.get(b"_create_plugin")?;
+        let plugin_raw = constructor();
+        let mut plugin = Box::from_raw(plugin_raw);
+        plugin.on_load(config_file)?;
+        self.plugins.push(plugin);
+        self.libs.push(lib);
+        Ok(())
+    }
+
+    /// Unload all plugins and loaded plugin libraries, making sure to fire
+    /// their `on_plugin_unload()` methods so they can do any necessary cleanup.
+    pub fn unload(&mut self) {
+        for mut plugin in self.plugins.drain(..) {
+            info!("Unloading plugin for {:?}", plugin.name());
+            plugin.on_unload();
+        }
+
+        for lib in self.libs.drain(..) {
+            drop(lib);
+        }
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_service.rs

@@ -0,0 +1,157 @@
+use {
+    crate::{
+        accounts_update_notifier::AccountsUpdateNotifierImpl,
+        accountsdb_plugin_manager::AccountsDbPluginManager,
+        slot_status_observer::SlotStatusObserver,
+    },
+    crossbeam_channel::Receiver,
+    log::*,
+    serde_json,
+    solana_rpc::optimistically_confirmed_bank_tracker::BankNotification,
+    solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier,
+    std::{
+        fs::File,
+        io::Read,
+        path::{Path, PathBuf},
+        sync::{Arc, RwLock},
+        thread,
+    },
+    thiserror::Error,
+};
+
+#[derive(Error, Debug)]
+pub enum AccountsdbPluginServiceError {
+    #[error("Cannot open the the plugin config file")]
+    CannotOpenConfigFile(String),
+
+    #[error("Cannot read the the plugin config file")]
+    CannotReadConfigFile(String),
+
+    #[error("The config file is not in a valid Json format")]
+    InvalidConfigFileFormat(String),
+
+    #[error("Plugin library path is not specified in the config file")]
+    LibPathNotSet,
+
+    #[error("Invalid plugin path")]
+    InvalidPluginPath,
+
+    #[error("Cannot load plugin shared library")]
+    PluginLoadError(String),
+}
+
+/// The service managing the AccountsDb plugin workflow.
+pub struct AccountsDbPluginService {
+    slot_status_observer: SlotStatusObserver,
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+    accounts_update_notifier: AccountsUpdateNotifier,
+}
+
+impl AccountsDbPluginService {
+    /// Creates and returns the AccountsDbPluginService.
+    /// # Arguments
+    /// * `confirmed_bank_receiver` - The receiver for confirmed bank notification
+    /// * `accountsdb_plugin_config_file` - The config file path for the plugin. The
+    ///    config file controls the plugin responsible
+    ///    for transporting the data to external data stores. It is defined in JSON format.
+    ///    The `libpath` field should be pointed to the full path of the dynamic shared library
+    ///    (.so file) to be loaded. The shared library must implement the `AccountsDbPlugin`
+    ///    trait. And the shared library shall export a `C` function `_create_plugin` which
+    ///    shall create the implementation of `AccountsDbPlugin` and returns to the caller.
+    ///    The rest of the JSON fields' definition is up to to the concrete plugin implementation
+    ///    It is usually used to configure the connection information for the external data store.
+
+    pub fn new(
+        confirmed_bank_receiver: Receiver<BankNotification>,
+        accountsdb_plugin_config_files: &[PathBuf],
+    ) -> Result<Self, AccountsdbPluginServiceError> {
+        info!(
+            "Starting AccountsDbPluginService from config files: {:?}",
+            accountsdb_plugin_config_files
+        );
+        let mut plugin_manager = AccountsDbPluginManager::new();
+
+        for accountsdb_plugin_config_file in accountsdb_plugin_config_files {
+            Self::load_plugin(&mut plugin_manager, accountsdb_plugin_config_file)?;
+        }
+
+        let plugin_manager = Arc::new(RwLock::new(plugin_manager));
+        let accounts_update_notifier = Arc::new(RwLock::new(AccountsUpdateNotifierImpl::new(
+            plugin_manager.clone(),
+        )));
+        let slot_status_observer =
+            SlotStatusObserver::new(confirmed_bank_receiver, accounts_update_notifier.clone());
+
+        info!("Started AccountsDbPluginService");
+        Ok(AccountsDbPluginService {
+            slot_status_observer,
+            plugin_manager,
+            accounts_update_notifier,
+        })
+    }
+
+    fn load_plugin(
+        plugin_manager: &mut AccountsDbPluginManager,
+        accountsdb_plugin_config_file: &Path,
+    ) -> Result<(), AccountsdbPluginServiceError> {
+        let mut file = match File::open(accountsdb_plugin_config_file) {
+            Ok(file) => file,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::CannotOpenConfigFile(format!(
+                    "Failed to open the plugin config file {:?}, error: {:?}",
+                    accountsdb_plugin_config_file, err
+                )));
+            }
+        };
+
+        let mut contents = String::new();
+        if let Err(err) = file.read_to_string(&mut contents) {
+            return Err(AccountsdbPluginServiceError::CannotReadConfigFile(format!(
+                "Failed to read the plugin config file {:?}, error: {:?}",
+                accountsdb_plugin_config_file, err
+            )));
+        }
+
+        let result: serde_json::Value = match serde_json::from_str(&contents) {
+            Ok(value) => value,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::InvalidConfigFileFormat(
+                    format!(
+                        "The config file {:?} is not in a valid Json format, error: {:?}",
+                        accountsdb_plugin_config_file, err
+                    ),
+                ));
+            }
+        };
+
+        let libpath = result["libpath"]
+            .as_str()
+            .ok_or(AccountsdbPluginServiceError::LibPathNotSet)?;
+        let config_file = accountsdb_plugin_config_file
+            .as_os_str()
+            .to_str()
+            .ok_or(AccountsdbPluginServiceError::InvalidPluginPath)?;
+
+        unsafe {
+            let result = plugin_manager.load_plugin(libpath, config_file);
+            if let Err(err) = result {
+                let msg = format!(
+                    "Failed to load the plugin library: {:?}, error: {:?}",
+                    libpath, err
+                );
+                return Err(AccountsdbPluginServiceError::PluginLoadError(msg));
+            }
+        }
+        Ok(())
+    }
+
+    pub fn get_accounts_update_notifier(&self) -> AccountsUpdateNotifier {
+        self.accounts_update_notifier.clone()
+    }
+
+    pub fn join(mut self) -> thread::Result<()> {
+        self.slot_status_observer.join()?;
+        self.plugin_manager.write().unwrap().unload();
+        Ok(())
+    }
+}

accountsdb-plugin-manager/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod accounts_update_notifier;
+pub mod accountsdb_plugin_manager;
+pub mod accountsdb_plugin_service;
+pub mod slot_status_observer;

accountsdb-plugin-manager/src/slot_status_observer.rs

@@ -0,0 +1,80 @@
+use {
+    crossbeam_channel::Receiver,
+    solana_rpc::optimistically_confirmed_bank_tracker::BankNotification,
+    solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier,
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::{self, Builder, JoinHandle},
+    },
+};
+
+#[derive(Debug)]
+pub(crate) struct SlotStatusObserver {
+    bank_notification_receiver_service: Option<JoinHandle<()>>,
+    exit_updated_slot_server: Arc<AtomicBool>,
+}
+
+impl SlotStatusObserver {
+    pub fn new(
+        bank_notification_receiver: Receiver<BankNotification>,
+        accounts_update_notifier: AccountsUpdateNotifier,
+    ) -> Self {
+        let exit_updated_slot_server = Arc::new(AtomicBool::new(false));
+
+        Self {
+            bank_notification_receiver_service: Some(Self::run_bank_notification_receiver(
+                bank_notification_receiver,
+                exit_updated_slot_server.clone(),
+                accounts_update_notifier,
+            )),
+            exit_updated_slot_server,
+        }
+    }
+
+    pub fn join(&mut self) -> thread::Result<()> {
+        self.exit_updated_slot_server.store(true, Ordering::Relaxed);
+        self.bank_notification_receiver_service
+            .take()
+            .map(JoinHandle::join)
+            .unwrap()
+    }
+
+    fn run_bank_notification_receiver(
+        bank_notification_receiver: Receiver<BankNotification>,
+        exit: Arc<AtomicBool>,
+        accounts_update_notifier: AccountsUpdateNotifier,
+    ) -> JoinHandle<()> {
+        Builder::new()
+            .name("bank_notification_receiver".to_string())
+            .spawn(move || {
+                while !exit.load(Ordering::Relaxed) {
+                    if let Ok(slot) = bank_notification_receiver.recv() {
+                        match slot {
+                            BankNotification::OptimisticallyConfirmed(slot) => {
+                                accounts_update_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_confirmed(slot, None);
+                            }
+                            BankNotification::Frozen(bank) => {
+                                accounts_update_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_processed(bank.slot(), Some(bank.parent_slot()));
+                            }
+                            BankNotification::Root(bank) => {
+                                accounts_update_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_rooted(bank.slot(), Some(bank.parent_slot()));
+                            }
+                        }
+                    }
+                }
+            })
+            .unwrap()
+    }
+}

accountsdb-plugin-postgres/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accountsdb-plugin-postgres"
+description = "The Solana AccountsDb plugin for PostgreSQL database."
+version = "1.8.2"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+bs58 = "0.4.0"
+chrono = { version = "0.4.11", features = ["serde"] }
+crossbeam-channel = "0.5"
+log = "0.4.14"
+postgres = { version = "0.19.1", features = ["with-chrono-0_4"] }
+serde = "1.0.130"
+serde_derive = "1.0.103"
+serde_json = "1.0.67"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-measure = { path = "../measure", version = "=1.8.2" }
+solana-metrics = { path = "../metrics", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+thiserror = "1.0.21"
+tokio-postgres = "0.7.3"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-postgres/README.md

@@ -0,0 +1,5 @@
+This is an example implementing the AccountsDb plugin for PostgreSQL database.
+Please see the `src/accountsdb_plugin_postgres.rs` for the format of the plugin's configuration file.
+
+To create the schema objects for the database, please use `scripts/create_schema.sql`.
+`scripts/drop_schema.sql` can be used to tear down the schema objects.

accountsdb-plugin-postgres/scripts/create_schema.sql

@@ -0,0 +1,54 @@
+/**
+ * This plugin implementation for PostgreSQL requires the following tables
+ */
+-- The table storing accounts
+
+
+CREATE TABLE account (
+    pubkey BYTEA PRIMARY KEY,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    write_version BIGINT NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+-- The table storing slot information
+CREATE TABLE slot (
+    slot BIGINT PRIMARY KEY,
+    parent BIGINT,
+    status varchar(16) NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+/**
+ * The following is for keeping historical data for accounts and is not required for plugin to work.
+ */
+-- The table storing historical data for accounts
+CREATE TABLE account_audit (
+    pubkey BYTEA,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    write_version BIGINT NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+CREATE FUNCTION audit_account_update() RETURNS trigger AS $audit_account_update$
+    BEGIN
+		INSERT INTO account_audit (pubkey, owner, lamports, slot, executable, rent_epoch, data, write_version, updated_on)
+            VALUES (OLD.pubkey, OLD.owner, OLD.lamports, OLD.slot,
+                    OLD.executable, OLD.rent_epoch, OLD.data, OLD.write_version, OLD.updated_on);
+        RETURN NEW;
+    END;
+
+$audit_account_update$ LANGUAGE plpgsql;
+
+CREATE TRIGGER account_update_trigger AFTER UPDATE OR DELETE ON account
+    FOR EACH ROW EXECUTE PROCEDURE audit_account_update();

accountsdb-plugin-postgres/scripts/drop_schema.sql

@@ -0,0 +1,9 @@
+/**
+ * Script for cleaning up the schema for PostgreSQL used for the AccountsDb plugin.
+ */
+
+DROP TRIGGER account_update_trigger;
+DROP FUNCTION audit_account_update;
+DROP TABLE account_audit;
+DROP TABLE account;
+DROP TABLE slot;

accountsdb-plugin-postgres/src/accounts_selector.rs

@@ -0,0 +1,69 @@
+use {log::*, std::collections::HashSet};
+
+#[derive(Debug)]
+pub(crate) struct AccountsSelector {
+    pub accounts: HashSet<Vec<u8>>,
+    pub owners: HashSet<Vec<u8>>,
+    pub select_all_accounts: bool,
+}
+
+impl AccountsSelector {
+    pub fn default() -> Self {
+        AccountsSelector {
+            accounts: HashSet::default(),
+            owners: HashSet::default(),
+            select_all_accounts: true,
+        }
+    }
+
+    pub fn new(accounts: &[String], owners: &[String]) -> Self {
+        info!(
+            "Creating AccountsSelector from accounts: {:?}, owners: {:?}",
+            accounts, owners
+        );
+
+        let select_all_accounts = accounts.iter().any(|key| key == "*");
+        if select_all_accounts {
+            return AccountsSelector {
+                accounts: HashSet::default(),
+                owners: HashSet::default(),
+                select_all_accounts,
+            };
+        }
+        let accounts = accounts
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        let owners = owners
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        AccountsSelector {
+            accounts,
+            owners,
+            select_all_accounts,
+        }
+    }
+
+    pub fn is_account_selected(&self, account: &[u8], owner: &[u8]) -> bool {
+        self.select_all_accounts || self.accounts.contains(account) || self.owners.contains(owner)
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_accounts_selector() {
+        AccountsSelector::new(
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+            &[],
+        );
+
+        AccountsSelector::new(
+            &[],
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+        );
+    }
+}

accountsdb-plugin-postgres/src/accountsdb_plugin_postgres.rs

@@ -0,0 +1,333 @@
+use solana_measure::measure::Measure;
+
+/// Main entry for the PostgreSQL plugin
+use {
+    crate::{
+        accounts_selector::AccountsSelector,
+        postgres_client::{ParallelPostgresClient, PostgresClientBuilder},
+    },
+    bs58,
+    log::*,
+    serde_derive::{Deserialize, Serialize},
+    serde_json,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions, Result, SlotStatus,
+    },
+    solana_metrics::*,
+    std::{fs::File, io::Read},
+    thiserror::Error,
+};
+
+#[derive(Default)]
+pub struct AccountsDbPluginPostgres {
+    client: Option<ParallelPostgresClient>,
+    accounts_selector: Option<AccountsSelector>,
+}
+
+impl std::fmt::Debug for AccountsDbPluginPostgres {
+    fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct AccountsDbPluginPostgresConfig {
+    pub host: String,
+    pub user: String,
+    pub threads: Option<usize>,
+    pub port: Option<u16>,
+    pub batch_size: Option<usize>,
+}
+
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginPostgresError {
+    #[error("Error connecting to the backend data store. Error message: ({msg})")]
+    DataStoreConnectionError { msg: String },
+
+    #[error("Error preparing data store schema. Error message: ({msg})")]
+    DataSchemaError { msg: String },
+}
+
+impl AccountsDbPlugin for AccountsDbPluginPostgres {
+    fn name(&self) -> &'static str {
+        "AccountsDbPluginPostgres"
+    }
+
+    /// Do initialization for the PostgreSQL plugin.
+    /// # Arguments
+    ///
+    /// Format of the config file:
+    /// The `accounts_selector` section allows the user to controls accounts selections.
+    /// "accounts_selector" : {
+    ///     "accounts" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
+    /// }
+    /// or:
+    /// "accounts_selector" = {
+    ///     "owners" : \["pubkey-1", "pubkey-2", ..., "pubkey-m"\]
+    /// }
+    /// Accounts either satisyfing the accounts condition or owners condition will be selected.
+    /// When only owners is specified,
+    /// all accounts belonging to the owners will be streamed.
+    /// The accounts field support wildcard to select all accounts:
+    /// "accounts_selector" : {
+    ///     "accounts" : \["*"\],
+    /// }
+    /// "host" specifies the PostgreSQL server.
+    /// "user" specifies the PostgreSQL user.
+    /// "threads" optional, specifies the number of worker threads for the plugin. A thread
+    /// maintains a PostgreSQL connection to the server. The default is 10.
+    /// "batch_size" optional, specifies the batch size of bulk insert when the AccountsDb is created
+    /// from restoring a snapshot. The default is "10".
+    /// # Examples
+    /// {
+    ///    "libpath": "/home/solana/target/release/libsolana_accountsdb_plugin_postgres.so",
+    ///    "host": "host_foo",
+    ///    "user": "solana",
+    ///    "threads": 10,
+    ///    "accounts_selector" : {
+    ///       "owners" : ["9oT9R5ZyRovSVnt37QvVoBttGpNqR3J7unkb567NP8k3"]
+    /// }
+
+    fn on_load(&mut self, config_file: &str) -> Result<()> {
+        solana_logger::setup_with_default("info");
+        info!(
+            "Loading plugin {:?} from config_file {:?}",
+            self.name(),
+            config_file
+        );
+        let mut file = File::open(config_file)?;
+        let mut contents = String::new();
+        file.read_to_string(&mut contents)?;
+
+        let result: serde_json::Value = serde_json::from_str(&contents).unwrap();
+        self.accounts_selector = Some(Self::create_accounts_selector_from_config(&result));
+
+        let result: serde_json::Result<AccountsDbPluginPostgresConfig> =
+            serde_json::from_str(&contents);
+        match result {
+            Err(err) => {
+                return Err(AccountsDbPluginError::ConfigFileReadError {
+                    msg: format!(
+                        "The config file is not in the JSON format expected: {:?}",
+                        err
+                    ),
+                })
+            }
+            Ok(config) => {
+                let client = PostgresClientBuilder::build_pararallel_postgres_client(&config)?;
+                self.client = Some(client);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn on_unload(&mut self) {
+        info!("Unloading plugin: {:?}", self.name());
+
+        match &mut self.client {
+            None => {}
+            Some(client) => {
+                client.join().unwrap();
+            }
+        }
+    }
+
+    fn update_account(
+        &mut self,
+        account: ReplicaAccountInfoVersions,
+        slot: u64,
+        is_startup: bool,
+    ) -> Result<()> {
+        let mut measure_all = Measure::start("accountsdb-plugin-postgres-update-account-main");
+        match account {
+            ReplicaAccountInfoVersions::V0_0_1(account) => {
+                let mut measure_select =
+                    Measure::start("accountsdb-plugin-postgres-update-account-select");
+                if let Some(accounts_selector) = &self.accounts_selector {
+                    if !accounts_selector.is_account_selected(account.pubkey, account.owner) {
+                        return Ok(());
+                    }
+                } else {
+                    return Ok(());
+                }
+                measure_select.stop();
+                inc_new_counter_debug!(
+                    "accountsdb-plugin-postgres-update-account-select-us",
+                    measure_select.as_us() as usize,
+                    100000,
+                    100000
+                );
+
+                debug!(
+                    "Updating account {:?} with owner {:?} at slot {:?} using account selector {:?}",
+                    bs58::encode(account.pubkey).into_string(),
+                    bs58::encode(account.owner).into_string(),
+                    slot,
+                    self.accounts_selector.as_ref().unwrap()
+                );
+
+                match &mut self.client {
+                    None => {
+                        return Err(AccountsDbPluginError::Custom(Box::new(
+                            AccountsDbPluginPostgresError::DataStoreConnectionError {
+                                msg: "There is no connection to the PostgreSQL database."
+                                    .to_string(),
+                            },
+                        )));
+                    }
+                    Some(client) => {
+                        let mut measure_update =
+                            Measure::start("accountsdb-plugin-postgres-update-account-client");
+                        let result = { client.update_account(account, slot, is_startup) };
+                        measure_update.stop();
+
+                        inc_new_counter_debug!(
+                            "accountsdb-plugin-postgres-update-account-client-us",
+                            measure_update.as_us() as usize,
+                            100000,
+                            100000
+                        );
+
+                        if let Err(err) = result {
+                            return Err(AccountsDbPluginError::AccountsUpdateError {
+                                msg: format!("Failed to persist the update of account to the PostgreSQL database. Error: {:?}", err)
+                            });
+                        }
+                    }
+                }
+            }
+        }
+
+        measure_all.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-postgres-update-account-main-us",
+            measure_all.as_us() as usize,
+            100000,
+            100000
+        );
+
+        Ok(())
+    }
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()> {
+        info!("Updating slot {:?} at with status {:?}", slot, status);
+
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => {
+                let result = client.update_slot_status(slot, parent, status);
+
+                if let Err(err) = result {
+                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                        msg: format!("Failed to persist the update of slot to the PostgreSQL database. Error: {:?}", err)
+                    });
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn notify_end_of_startup(&mut self) -> Result<()> {
+        info!("Notifying the end of startup for accounts notifications");
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => {
+                let result = client.notify_end_of_startup();
+
+                if let Err(err) = result {
+                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                        msg: format!("Failed to notify the end of startup for accounts notifications. Error: {:?}", err)
+                    });
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+impl AccountsDbPluginPostgres {
+    fn create_accounts_selector_from_config(config: &serde_json::Value) -> AccountsSelector {
+        let accounts_selector = &config["accounts_selector"];
+
+        if accounts_selector.is_null() {
+            AccountsSelector::default()
+        } else {
+            let accounts = &accounts_selector["accounts"];
+            let accounts: Vec<String> = if accounts.is_array() {
+                accounts
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            let owners = &accounts_selector["owners"];
+            let owners: Vec<String> = if owners.is_array() {
+                owners
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            AccountsSelector::new(&accounts, &owners)
+        }
+    }
+
+    pub fn new() -> Self {
+        AccountsDbPluginPostgres {
+            client: None,
+            accounts_selector: None,
+        }
+    }
+}
+
+#[no_mangle]
+#[allow(improper_ctypes_definitions)]
+/// # Safety
+///
+/// This function returns the AccountsDbPluginPostgres pointer as trait AccountsDbPlugin.
+pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin {
+    let plugin = AccountsDbPluginPostgres::new();
+    let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin);
+    Box::into_raw(plugin)
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use {super::*, serde_json};
+
+    #[test]
+    fn test_accounts_selector_from_config() {
+        let config = "{\"accounts_selector\" : { \
+           \"owners\" : [\"9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin\"] \
+        }}";
+
+        let config: serde_json::Value = serde_json::from_str(config).unwrap();
+        AccountsDbPluginPostgres::create_accounts_selector_from_config(&config);
+    }
+}

accountsdb-plugin-postgres/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod accounts_selector;
+pub mod accountsdb_plugin_postgres;
+pub mod postgres_client;

accountsdb-plugin-postgres/src/postgres_client.rs

@@ -0,0 +1,776 @@
+#![allow(clippy::integer_arithmetic)]
+
+/// A concurrent implementation for writing accounts into the PostgreSQL in parallel.
+use {
+    crate::accountsdb_plugin_postgres::{
+        AccountsDbPluginPostgresConfig, AccountsDbPluginPostgresError,
+    },
+    chrono::Utc,
+    crossbeam_channel::{bounded, Receiver, RecvTimeoutError, Sender},
+    log::*,
+    postgres::{Client, NoTls, Statement},
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPluginError, ReplicaAccountInfo, SlotStatus,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_sdk::timing::AtomicInterval,
+    std::{
+        sync::{
+            atomic::{AtomicBool, AtomicUsize, Ordering},
+            Arc, Mutex,
+        },
+        thread::{self, sleep, Builder, JoinHandle},
+        time::Duration,
+    },
+    tokio_postgres::types::ToSql,
+};
+
+/// The maximum asynchronous requests allowed in the channel to avoid excessive
+/// memory usage. The downside -- calls after this threshold is reached can get blocked.
+const MAX_ASYNC_REQUESTS: usize = 40960;
+const DEFAULT_POSTGRES_PORT: u16 = 5432;
+const DEFAULT_THREADS_COUNT: usize = 100;
+const DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE: usize = 10;
+const ACCOUNT_COLUMN_COUNT: usize = 9;
+
+struct PostgresSqlClientWrapper {
+    client: Client,
+    update_account_stmt: Statement,
+    bulk_account_insert_stmt: Statement,
+}
+
+pub struct SimplePostgresClient {
+    batch_size: usize,
+    pending_account_updates: Vec<DbAccountInfo>,
+    client: Mutex<PostgresSqlClientWrapper>,
+}
+
+struct PostgresClientWorker {
+    client: SimplePostgresClient,
+    /// Indicating if accounts notification during startup is done.
+    is_startup_done: bool,
+}
+
+impl Eq for DbAccountInfo {}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct DbAccountInfo {
+    pub pubkey: Vec<u8>,
+    pub lamports: i64,
+    pub owner: Vec<u8>,
+    pub executable: bool,
+    pub rent_epoch: i64,
+    pub data: Vec<u8>,
+    pub slot: i64,
+    pub write_version: i64,
+}
+
+impl DbAccountInfo {
+    fn new<T: ReadableAccountInfo>(account: &T, slot: u64) -> DbAccountInfo {
+        let data = account.data().to_vec();
+        Self {
+            pubkey: account.pubkey().to_vec(),
+            lamports: account.lamports() as i64,
+            owner: account.owner().to_vec(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch() as i64,
+            data,
+            slot: slot as i64,
+            write_version: account.write_version(),
+        }
+    }
+}
+
+pub trait ReadableAccountInfo: Sized {
+    fn pubkey(&self) -> &[u8];
+    fn owner(&self) -> &[u8];
+    fn lamports(&self) -> i64;
+    fn executable(&self) -> bool;
+    fn rent_epoch(&self) -> i64;
+    fn data(&self) -> &[u8];
+    fn write_version(&self) -> i64;
+}
+
+impl ReadableAccountInfo for DbAccountInfo {
+    fn pubkey(&self) -> &[u8] {
+        &self.pubkey
+    }
+
+    fn owner(&self) -> &[u8] {
+        &self.owner
+    }
+
+    fn lamports(&self) -> i64 {
+        self.lamports
+    }
+
+    fn executable(&self) -> bool {
+        self.executable
+    }
+
+    fn rent_epoch(&self) -> i64 {
+        self.rent_epoch
+    }
+
+    fn data(&self) -> &[u8] {
+        &self.data
+    }
+
+    fn write_version(&self) -> i64 {
+        self.write_version
+    }
+}
+
+impl<'a> ReadableAccountInfo for ReplicaAccountInfo<'a> {
+    fn pubkey(&self) -> &[u8] {
+        self.pubkey
+    }
+
+    fn owner(&self) -> &[u8] {
+        self.owner
+    }
+
+    fn lamports(&self) -> i64 {
+        self.lamports as i64
+    }
+
+    fn executable(&self) -> bool {
+        self.executable
+    }
+
+    fn rent_epoch(&self) -> i64 {
+        self.rent_epoch as i64
+    }
+
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+
+    fn write_version(&self) -> i64 {
+        self.write_version as i64
+    }
+}
+
+pub trait PostgresClient {
+    fn join(&mut self) -> thread::Result<()> {
+        Ok(())
+    }
+
+    fn update_account(
+        &mut self,
+        account: DbAccountInfo,
+        is_startup: bool,
+    ) -> Result<(), AccountsDbPluginError>;
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<(), AccountsDbPluginError>;
+
+    fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError>;
+}
+
+impl SimplePostgresClient {
+    fn connect_to_db(
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<Client, AccountsDbPluginError> {
+        let port = config.port.unwrap_or(DEFAULT_POSTGRES_PORT);
+
+        let connection_str = format!("host={} user={} port={}", config.host, config.user, port);
+
+        match Client::connect(&connection_str, NoTls) {
+            Err(err) => {
+                let msg = format!(
+                        "Error in connecting to the PostgreSQL database: {:?} host: {:?} user: {:?} config: {:?}",
+                        err, config.host, config.user, connection_str);
+                error!("{}", msg);
+                Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError { msg },
+                )))
+            }
+            Ok(client) => Ok(client),
+        }
+    }
+
+    fn build_bulk_account_insert_statement(
+        client: &mut Client,
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<Statement, AccountsDbPluginError> {
+        let batch_size = config
+            .batch_size
+            .unwrap_or(DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE);
+        let mut stmt = String::from("INSERT INTO account AS acct (pubkey, slot, owner, lamports, executable, rent_epoch, data, write_version, updated_on) VALUES");
+        for j in 0..batch_size {
+            let row = j * ACCOUNT_COLUMN_COUNT;
+            let val_str = format!(
+                "(${}, ${}, ${}, ${}, ${}, ${}, ${}, ${}, ${})",
+                row + 1,
+                row + 2,
+                row + 3,
+                row + 4,
+                row + 5,
+                row + 6,
+                row + 7,
+                row + 8,
+                row + 9,
+            );
+
+            if j == 0 {
+                stmt = format!("{} {}", &stmt, val_str);
+            } else {
+                stmt = format!("{}, {}", &stmt, val_str);
+            }
+        }
+
+        let handle_conflict = "ON CONFLICT (pubkey) DO UPDATE SET slot=excluded.slot, owner=excluded.owner, lamports=excluded.lamports, executable=excluded.executable, rent_epoch=excluded.rent_epoch, \
+            data=excluded.data, write_version=excluded.write_version, updated_on=excluded.updated_on WHERE acct.slot < excluded.slot OR (\
+            acct.slot = excluded.slot AND acct.write_version < excluded.write_version)";
+
+        stmt = format!("{} {}", stmt, handle_conflict);
+
+        info!("{}", stmt);
+        let bulk_stmt = client.prepare(&stmt);
+
+        match bulk_stmt {
+            Err(err) => {
+                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
+                    msg: format!(
+                        "Error in preparing for the accounts update PostgreSQL database: {} host: {} user: {} config: {:?}",
+                        err, config.host, config.user, config
+                    ),
+                })));
+            }
+            Ok(update_account_stmt) => Ok(update_account_stmt),
+        }
+    }
+
+    fn build_single_account_upsert_statement(
+        client: &mut Client,
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<Statement, AccountsDbPluginError> {
+        let stmt = "INSERT INTO account AS acct (pubkey, slot, owner, lamports, executable, rent_epoch, data, write_version, updated_on) \
+        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) \
+        ON CONFLICT (pubkey) DO UPDATE SET slot=excluded.slot, owner=excluded.owner, lamports=excluded.lamports, executable=excluded.executable, rent_epoch=excluded.rent_epoch, \
+        data=excluded.data, write_version=excluded.write_version, updated_on=excluded.updated_on  WHERE acct.slot < excluded.slot OR (\
+        acct.slot = excluded.slot AND acct.write_version < excluded.write_version)";
+
+        let stmt = client.prepare(stmt);
+
+        match stmt {
+            Err(err) => {
+                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
+                    msg: format!(
+                        "Error in preparing for the accounts update PostgreSQL database: {} host: {} user: {} config: {:?}",
+                        err, config.host, config.user, config
+                    ),
+                })));
+            }
+            Ok(update_account_stmt) => Ok(update_account_stmt),
+        }
+    }
+
+    /// Internal function for updating or inserting a single account
+    fn upsert_account_internal(
+        account: &DbAccountInfo,
+        statement: &Statement,
+        client: &mut Client,
+    ) -> Result<(), AccountsDbPluginError> {
+        let lamports = account.lamports() as i64;
+        let rent_epoch = account.rent_epoch() as i64;
+        let updated_on = Utc::now().naive_utc();
+        let result = client.query(
+            statement,
+            &[
+                &account.pubkey(),
+                &account.slot,
+                &account.owner(),
+                &lamports,
+                &account.executable(),
+                &rent_epoch,
+                &account.data(),
+                &account.write_version(),
+                &updated_on,
+            ],
+        );
+
+        if let Err(err) = result {
+            let msg = format!(
+                "Failed to persist the update of account to the PostgreSQL database. Error: {:?}",
+                err
+            );
+            error!("{}", msg);
+            return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+        }
+
+        Ok(())
+    }
+
+    /// Update or insert a single account
+    fn upsert_account(&mut self, account: &DbAccountInfo) -> Result<(), AccountsDbPluginError> {
+        let client = self.client.get_mut().unwrap();
+        let statement = &client.update_account_stmt;
+        let client = &mut client.client;
+        Self::upsert_account_internal(account, statement, client)
+    }
+
+    /// Insert accounts in batch to reduce network overhead
+    fn insert_accounts_in_batch(
+        &mut self,
+        account: DbAccountInfo,
+    ) -> Result<(), AccountsDbPluginError> {
+        self.pending_account_updates.push(account);
+
+        if self.pending_account_updates.len() == self.batch_size {
+            let mut measure = Measure::start("accountsdb-plugin-postgres-prepare-values");
+
+            let mut values: Vec<&(dyn ToSql + Sync)> =
+                Vec::with_capacity(self.batch_size * ACCOUNT_COLUMN_COUNT);
+            let updated_on = Utc::now().naive_utc();
+            for j in 0..self.batch_size {
+                let account = &self.pending_account_updates[j];
+
+                values.push(&account.pubkey);
+                values.push(&account.slot);
+                values.push(&account.owner);
+                values.push(&account.lamports);
+                values.push(&account.executable);
+                values.push(&account.rent_epoch);
+                values.push(&account.data);
+                values.push(&account.write_version);
+                values.push(&updated_on);
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-postgres-prepare-values-us",
+                measure.as_us() as usize,
+                10000,
+                10000
+            );
+
+            let mut measure = Measure::start("accountsdb-plugin-postgres-update-account");
+            let client = self.client.get_mut().unwrap();
+            let result = client
+                .client
+                .query(&client.bulk_account_insert_stmt, &values);
+
+            self.pending_account_updates.clear();
+            if let Err(err) = result {
+                let msg = format!(
+                    "Failed to persist the update of account to the PostgreSQL database. Error: {:?}",
+                    err
+                );
+                error!("{}", msg);
+                return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-postgres-update-account-us",
+                measure.as_us() as usize,
+                10000,
+                10000
+            );
+            inc_new_counter_debug!(
+                "accountsdb-plugin-postgres-update-account-count",
+                self.batch_size,
+                10000,
+                10000
+            );
+        }
+        Ok(())
+    }
+
+    /// Flush any left over accounts in batch which are not processed in the last batch
+    fn flush_buffered_writes(&mut self) -> Result<(), AccountsDbPluginError> {
+        if self.pending_account_updates.is_empty() {
+            return Ok(());
+        }
+
+        let client = self.client.get_mut().unwrap();
+        let statement = &client.update_account_stmt;
+        let client = &mut client.client;
+
+        for account in self.pending_account_updates.drain(..) {
+            Self::upsert_account_internal(&account, statement, client)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn new(config: &AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
+        info!("Creating SimplePostgresClient...");
+        let mut client = Self::connect_to_db(config)?;
+        let bulk_account_insert_stmt =
+            Self::build_bulk_account_insert_statement(&mut client, config)?;
+        let update_account_stmt = Self::build_single_account_upsert_statement(&mut client, config)?;
+
+        let batch_size = config
+            .batch_size
+            .unwrap_or(DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE);
+        info!("Created SimplePostgresClient.");
+        Ok(Self {
+            batch_size,
+            pending_account_updates: Vec::with_capacity(batch_size),
+            client: Mutex::new(PostgresSqlClientWrapper {
+                client,
+                update_account_stmt,
+                bulk_account_insert_stmt,
+            }),
+        })
+    }
+}
+
+impl PostgresClient for SimplePostgresClient {
+    fn update_account(
+        &mut self,
+        account: DbAccountInfo,
+        is_startup: bool,
+    ) -> Result<(), AccountsDbPluginError> {
+        trace!(
+            "Updating account {} with owner {} at slot {}",
+            bs58::encode(account.pubkey()).into_string(),
+            bs58::encode(account.owner()).into_string(),
+            account.slot,
+        );
+        if !is_startup {
+            return self.upsert_account(&account);
+        }
+        self.insert_accounts_in_batch(account)
+    }
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<(), AccountsDbPluginError> {
+        info!("Updating slot {:?} at with status {:?}", slot, status);
+
+        let slot = slot as i64; // postgres only supports i64
+        let parent = parent.map(|parent| parent as i64);
+        let updated_on = Utc::now().naive_utc();
+        let status_str = status.as_str();
+        let client = self.client.get_mut().unwrap();
+
+        let result = match parent {
+                        Some(parent) => {
+                            client.client.execute(
+                                "INSERT INTO slot (slot, parent, status, updated_on) \
+                                VALUES ($1, $2, $3, $4) \
+                                ON CONFLICT (slot) DO UPDATE SET parent=$2, status=$3, updated_on=$4",
+                                &[
+                                    &slot,
+                                    &parent,
+                                    &status_str,
+                                    &updated_on,
+                                ],
+                            )
+                        }
+                        None => {
+                            client.client.execute(
+                                "INSERT INTO slot (slot, status, updated_on) \
+                                VALUES ($1, $2, $3) \
+                                ON CONFLICT (slot) DO UPDATE SET status=$2, updated_on=$3",
+                                &[
+                                    &slot,
+                                    &status_str,
+                                    &updated_on,
+                                ],
+                            )
+                        }
+                };
+
+        match result {
+            Err(err) => {
+                let msg = format!(
+                    "Failed to persist the update of slot to the PostgreSQL database. Error: {:?}",
+                    err
+                );
+                error!("{:?}", msg);
+                return Err(AccountsDbPluginError::SlotStatusUpdateError { msg });
+            }
+            Ok(rows) => {
+                assert_eq!(1, rows, "Expected one rows to be updated a time");
+            }
+        }
+
+        Ok(())
+    }
+
+    fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError> {
+        self.flush_buffered_writes()
+    }
+}
+
+struct UpdateAccountRequest {
+    account: DbAccountInfo,
+    is_startup: bool,
+}
+
+struct UpdateSlotRequest {
+    slot: u64,
+    parent: Option<u64>,
+    slot_status: SlotStatus,
+}
+
+enum DbWorkItem {
+    UpdateAccount(UpdateAccountRequest),
+    UpdateSlot(UpdateSlotRequest),
+}
+
+impl PostgresClientWorker {
+    fn new(config: AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
+        let result = SimplePostgresClient::new(&config);
+        match result {
+            Ok(client) => Ok(PostgresClientWorker {
+                client,
+                is_startup_done: false,
+            }),
+            Err(err) => {
+                error!("Error in creating SimplePostgresClient: {}", err);
+                Err(err)
+            }
+        }
+    }
+
+    fn do_work(
+        &mut self,
+        receiver: Receiver<DbWorkItem>,
+        exit_worker: Arc<AtomicBool>,
+        is_startup_done: Arc<AtomicBool>,
+        startup_done_count: Arc<AtomicUsize>,
+    ) -> Result<(), AccountsDbPluginError> {
+        while !exit_worker.load(Ordering::Relaxed) {
+            let mut measure = Measure::start("accountsdb-plugin-postgres-worker-recv");
+            let work = receiver.recv_timeout(Duration::from_millis(500));
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-postgres-worker-recv-us",
+                measure.as_us() as usize,
+                100000,
+                100000
+            );
+            match work {
+                Ok(work) => match work {
+                    DbWorkItem::UpdateAccount(request) => {
+                        self.client
+                            .update_account(request.account, request.is_startup)?;
+                    }
+                    DbWorkItem::UpdateSlot(request) => {
+                        self.client.update_slot_status(
+                            request.slot,
+                            request.parent,
+                            request.slot_status,
+                        )?;
+                    }
+                },
+                Err(err) => match err {
+                    RecvTimeoutError::Timeout => {
+                        if !self.is_startup_done && is_startup_done.load(Ordering::Relaxed) {
+                            self.client.notify_end_of_startup()?;
+                            self.is_startup_done = true;
+                            startup_done_count.fetch_add(1, Ordering::Relaxed);
+                        }
+
+                        continue;
+                    }
+                    _ => {
+                        error!("Error in receiving the item {:?}", err);
+                        break;
+                    }
+                },
+            }
+        }
+        Ok(())
+    }
+}
+pub struct ParallelPostgresClient {
+    workers: Vec<JoinHandle<Result<(), AccountsDbPluginError>>>,
+    exit_worker: Arc<AtomicBool>,
+    is_startup_done: Arc<AtomicBool>,
+    startup_done_count: Arc<AtomicUsize>,
+    initialized_worker_count: Arc<AtomicUsize>,
+    sender: Sender<DbWorkItem>,
+    last_report: AtomicInterval,
+}
+
+impl ParallelPostgresClient {
+    pub fn new(config: &AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
+        info!("Creating ParallelPostgresClient...");
+        let (sender, receiver) = bounded(MAX_ASYNC_REQUESTS);
+        let exit_worker = Arc::new(AtomicBool::new(false));
+        let mut workers = Vec::default();
+        let is_startup_done = Arc::new(AtomicBool::new(false));
+        let startup_done_count = Arc::new(AtomicUsize::new(0));
+        let worker_count = config.threads.unwrap_or(DEFAULT_THREADS_COUNT);
+        let initialized_worker_count = Arc::new(AtomicUsize::new(0));
+        for i in 0..worker_count {
+            let cloned_receiver = receiver.clone();
+            let exit_clone = exit_worker.clone();
+            let is_startup_done_clone = is_startup_done.clone();
+            let startup_done_count_clone = startup_done_count.clone();
+            let initialized_worker_count_clone = initialized_worker_count.clone();
+            let config = config.clone();
+            let worker = Builder::new()
+                .name(format!("worker-{}", i))
+                .spawn(move || -> Result<(), AccountsDbPluginError> {
+                    let result = PostgresClientWorker::new(config);
+
+                    match result {
+                        Ok(mut worker) => {
+                            initialized_worker_count_clone.fetch_add(1, Ordering::Relaxed);
+                            worker.do_work(
+                                cloned_receiver,
+                                exit_clone,
+                                is_startup_done_clone,
+                                startup_done_count_clone,
+                            )?;
+                            Ok(())
+                        }
+                        Err(err) => Err(err),
+                    }
+                })
+                .unwrap();
+
+            workers.push(worker);
+        }
+
+        info!("Created ParallelPostgresClient.");
+        Ok(Self {
+            last_report: AtomicInterval::default(),
+            workers,
+            exit_worker,
+            is_startup_done,
+            startup_done_count,
+            initialized_worker_count,
+            sender,
+        })
+    }
+
+    pub fn join(&mut self) -> thread::Result<()> {
+        self.exit_worker.store(true, Ordering::Relaxed);
+        while !self.workers.is_empty() {
+            let worker = self.workers.pop();
+            if worker.is_none() {
+                break;
+            }
+            let worker = worker.unwrap();
+            let result = worker.join().unwrap();
+            if result.is_err() {
+                error!("The worker thread has failed: {:?}", result);
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn update_account(
+        &mut self,
+        account: &ReplicaAccountInfo,
+        slot: u64,
+        is_startup: bool,
+    ) -> Result<(), AccountsDbPluginError> {
+        if self.last_report.should_update(30000) {
+            datapoint_debug!(
+                "postgres-plugin-stats",
+                ("message-queue-length", self.sender.len() as i64, i64),
+            );
+        }
+        let mut measure = Measure::start("accountsdb-plugin-posgres-create-work-item");
+        let wrk_item = DbWorkItem::UpdateAccount(UpdateAccountRequest {
+            account: DbAccountInfo::new(account, slot),
+            is_startup,
+        });
+
+        measure.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-posgres-create-work-item-us",
+            measure.as_us() as usize,
+            100000,
+            100000
+        );
+
+        let mut measure = Measure::start("accountsdb-plugin-posgres-send-msg");
+
+        if let Err(err) = self.sender.send(wrk_item) {
+            return Err(AccountsDbPluginError::AccountsUpdateError {
+                msg: format!(
+                    "Failed to update the account {:?}, error: {:?}",
+                    bs58::encode(account.pubkey()).into_string(),
+                    err
+                ),
+            });
+        }
+
+        measure.stop();
+        inc_new_counter_debug!(
+            "accountsdb-plugin-posgres-send-msg-us",
+            measure.as_us() as usize,
+            100000,
+            100000
+        );
+
+        Ok(())
+    }
+
+    pub fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<(), AccountsDbPluginError> {
+        if let Err(err) = self.sender.send(DbWorkItem::UpdateSlot(UpdateSlotRequest {
+            slot,
+            parent,
+            slot_status: status,
+        })) {
+            return Err(AccountsDbPluginError::SlotStatusUpdateError {
+                msg: format!("Failed to update the slot {:?}, error: {:?}", slot, err),
+            });
+        }
+        Ok(())
+    }
+
+    pub fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError> {
+        info!("Notifying the end of startup");
+        // Ensure all items in the queue has been received by the workers
+        while !self.sender.is_empty() {
+            sleep(Duration::from_millis(100));
+        }
+        self.is_startup_done.store(true, Ordering::Relaxed);
+
+        // Wait for all worker threads to be done with flushing
+        while self.startup_done_count.load(Ordering::Relaxed)
+            != self.initialized_worker_count.load(Ordering::Relaxed)
+        {
+            info!(
+                "Startup done count: {}, good worker thread count: {}",
+                self.startup_done_count.load(Ordering::Relaxed),
+                self.initialized_worker_count.load(Ordering::Relaxed)
+            );
+            sleep(Duration::from_millis(100));
+        }
+
+        info!("Done with notifying the end of startup");
+        Ok(())
+    }
+}
+
+pub struct PostgresClientBuilder {}
+
+impl PostgresClientBuilder {
+    pub fn build_pararallel_postgres_client(
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<ParallelPostgresClient, AccountsDbPluginError> {
+        ParallelPostgresClient::new(config)
+    }
+
+    pub fn build_simple_postgres_client(
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<SimplePostgresClient, AccountsDbPluginError> {
+        SimplePostgresClient::new(config)
+    }
+}

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,18 +14,18 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-core = { path = "../core", version = "=1.7.12" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-gossip = { path = "../gossip", version = "=1.7.12" }
-solana-ledger = { path = "../ledger", version = "=1.7.12" }
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-measure = { path = "../measure", version = "=1.7.12" }
-solana-perf = { path = "../perf", version = "=1.7.12" }
-solana-poh = { path = "../poh", version = "=1.7.12" }
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-streamer = { path = "../streamer", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
+solana-core = { path = "../core", version = "=1.8.2" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-gossip = { path = "../gossip", version = "=1.8.2" }
+solana-ledger = { path = "../ledger", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-measure = { path = "../measure", version = "=1.8.2" }
+solana-perf = { path = "../perf", version = "=1.8.2" }
+solana-poh = { path = "../poh", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-streamer = { path = "../streamer", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -16,6 +16,7 @@ use solana_perf::packet::to_packets_chunked;
 use solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry};
 use solana_runtime::{
     accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
+    cost_model::CostModel,
 };
 use solana_sdk::{
     hash::Hash,
@@ -27,7 +28,7 @@ use solana_sdk::{
 };
 use solana_streamer::socket::SocketAddrSpace;
 use std::{
-    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex},
+    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
     thread::sleep,
     time::{Duration, Instant},
 };
@@ -166,6 +167,7 @@ fn main() {
 
     let (verified_sender, verified_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
+    let (tpu_vote_sender, tpu_vote_receiver) = unbounded();
     let (replay_vote_sender, _replay_vote_receiver) = unbounded();
     let bank0 = Bank::new(&genesis_config);
     let mut bank_forks = BankForks::new(bank0);
@@ -226,9 +228,11 @@ fn main() {
             &cluster_info,
             &poh_recorder,
             verified_receiver,
+            tpu_vote_receiver,
             vote_receiver,
             None,
             replay_vote_sender,
+            Arc::new(RwLock::new(CostModel::default())),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
@@ -381,6 +385,7 @@ fn main() {
         );
 
         drop(verified_sender);
+        drop(tpu_vote_sender);
         drop(vote_sender);
         exit.store(true, Ordering::Relaxed);
         banking_stage.join().unwrap();

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.7.12"
+version = "1.8.2"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,16 +15,16 @@ borsh = "0.9.0"
 borsh-derive = "0.9.0"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.12" }
-solana-program = { path = "../sdk/program", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.8.2" }
+solana-program = { path = "../sdk/program", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.12" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-banks-server = { path = "../banks-server", version = "=1.8.2" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -385,7 +385,9 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
             let mut banks_client = start_client(client_transport).await?;
 
             let recent_blockhash = banks_client.get_recent_blockhash().await?;
@@ -416,7 +418,9 @@ mod tests {
         let message = Message::new(&[instruction], Some(mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
             let mut banks_client = start_client(client_transport).await?;
             let (_, recent_blockhash, last_valid_block_height) = banks_client.get_fees().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.7.12"
+version = "1.8.2"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ edition = "2018"
 [dependencies]
 mio = "0.7.6"
 serde = { version = "1.0.122", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
 tarpc = { version = "0.24.1", features = ["full"] }
 
 [dev-dependencies]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.7.12"
+version = "1.8.2"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,10 +14,10 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.12" }
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-metrics = { path = "../metrics", version = "=1.7.12" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-metrics = { path = "../metrics", version = "=1.8.2" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -12,6 +12,7 @@ use solana_sdk::{
     account::Account,
     clock::Slot,
     commitment_config::CommitmentLevel,
+    feature_set::FeatureSet,
     fee_calculator::FeeCalculator,
     hash::Hash,
     pubkey::Pubkey,
@@ -43,6 +44,7 @@ struct BanksServer {
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     transaction_sender: Sender<TransactionInfo>,
+    poll_signature_status_sleep_duration: Duration,
 }
 
 impl BanksServer {
@@ -54,11 +56,13 @@ impl BanksServer {
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         transaction_sender: Sender<TransactionInfo>,
+        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         Self {
             bank_forks,
             block_commitment_cache,
             transaction_sender,
+            poll_signature_status_sleep_duration,
         }
     }
 
@@ -81,6 +85,7 @@ impl BanksServer {
     fn new_loopback(
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
@@ -95,7 +100,12 @@ impl BanksServer {
             .name("solana-bank-forks-client".to_string())
             .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
-        Self::new(bank_forks, block_commitment_cache, transaction_sender)
+        Self::new(
+            bank_forks,
+            block_commitment_cache,
+            transaction_sender,
+            poll_signature_status_sleep_duration,
+        )
     }
 
     fn slot(&self, commitment: CommitmentLevel) -> Slot {
@@ -120,7 +130,7 @@ impl BanksServer {
             .bank(commitment)
             .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            sleep(Duration::from_millis(200)).await;
+            sleep(self.poll_signature_status_sleep_duration).await;
             let bank = self.bank(commitment);
             if bank.block_height() > last_valid_block_height {
                 break;
@@ -133,11 +143,11 @@ impl BanksServer {
 
 fn verify_transaction(
     transaction: &Transaction,
-    libsecp256k1_0_5_upgrade_enabled: bool,
+    feature_set: &Arc<FeatureSet>,
 ) -> transaction::Result<()> {
     if let Err(err) = transaction.verify() {
         Err(err)
-    } else if let Err(err) = transaction.verify_precompiles(libsecp256k1_0_5_upgrade_enabled) {
+    } else if let Err(err) = transaction.verify_precompiles(feature_set) {
         Err(err)
     } else {
         Ok(())
@@ -227,19 +237,13 @@ impl Banks for BanksServer {
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        if let Err(err) = verify_transaction(
-            &transaction,
-            self.bank(commitment).libsecp256k1_0_5_upgrade_enabled(),
-        ) {
+        if let Err(err) = verify_transaction(&transaction, &self.bank(commitment).feature_set) {
             return Some(Err(err));
         }
 
         let blockhash = &transaction.message.recent_blockhash;
         let last_valid_block_height = self
-            .bank_forks
-            .read()
-            .unwrap()
-            .root_bank()
+            .bank(commitment)
             .get_blockhash_last_valid_block_height(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
@@ -267,8 +271,13 @@ impl Banks for BanksServer {
 pub async fn start_local_server(
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    poll_signature_status_sleep_duration: Duration,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(bank_forks, block_commitment_cache);
+    let banks_server = BanksServer::new_loopback(
+        bank_forks,
+        block_commitment_cache,
+        poll_signature_status_sleep_duration,
+    );
     let (client_transport, server_transport) = transport::channel::unbounded();
     let server = server::new(server::Config::default())
         .incoming(stream::once(future::ready(server_transport)))
@@ -303,8 +312,12 @@ pub async fn start_tcp_server(
 
             SendTransactionService::new(tpu_addr, &bank_forks, receiver);
 
-            let server =
-                BanksServer::new(bank_forks.clone(), block_commitment_cache.clone(), sender);
+            let server = BanksServer::new(
+                bank_forks.clone(),
+                block_commitment_cache.clone(),
+                sender,
+                Duration::from_millis(200),
+            );
             chan.respond_with(server.serve()).execute()
         })
         // Max 10 channels.

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,23 +18,23 @@ rand = "0.7.0"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-core = { path = "../core", version = "=1.7.12" }
-solana-genesis = { path = "../genesis", version = "=1.7.12" }
-solana-client = { path = "../client", version = "=1.7.12" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.7.12" }
-solana-faucet = { path = "../faucet", version = "=1.7.12" }
-solana-gossip = { path = "../gossip", version = "=1.7.12" }
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-metrics = { path = "../metrics", version = "=1.7.12" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.12" }
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-streamer = { path = "../streamer", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-core = { path = "../core", version = "=1.8.2" }
+solana-genesis = { path = "../genesis", version = "=1.8.2" }
+solana-client = { path = "../client", version = "=1.8.2" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.8.2" }
+solana-faucet = { path = "../faucet", version = "=1.8.2" }
+solana-gossip = { path = "../gossip", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-metrics = { path = "../metrics", version = "=1.8.2" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-streamer = { path = "../streamer", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.12" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.8.2" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-streamer = { path = "../streamer", version = "=1.7.12" }
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-streamer = { path = "../streamer", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,24 +15,24 @@ log = "0.4.11"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-core = { path = "../core", version = "=1.7.12" }
-solana-genesis = { path = "../genesis", version = "=1.7.12" }
-solana-client = { path = "../client", version = "=1.7.12" }
-solana-faucet = { path = "../faucet", version = "=1.7.12" }
-solana-gossip = { path = "../gossip", version = "=1.7.12" }
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-metrics = { path = "../metrics", version = "=1.7.12" }
-solana-measure = { path = "../measure", version = "=1.7.12" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.12" }
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-streamer = { path = "../streamer", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-core = { path = "../core", version = "=1.8.2" }
+solana-genesis = { path = "../genesis", version = "=1.8.2" }
+solana-client = { path = "../client", version = "=1.8.2" }
+solana-faucet = { path = "../faucet", version = "=1.8.2" }
+solana-gossip = { path = "../gossip", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-metrics = { path = "../metrics", version = "=1.8.2" }
+solana-measure = { path = "../measure", version = "=1.8.2" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-streamer = { path = "../streamer", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.12" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.8.2" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

ci/buildkite-pipeline.sh

@@ -137,7 +137,7 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 30
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
     wait_step
   else
     annotate --style info --context test-coverage \

ci/do-audit.sh

@@ -46,5 +46,11 @@ cargo_audit_ignores=(
   # https://github.com/paritytech/jsonrpc/issues/605
   --ignore RUSTSEC-2021-0079
 
+  # chrono: Potential segfault in `localtime_r` invocations
+  #
+  # Blocked due to no safe upgrade
+  # https://github.com/chronotope/chrono/issues/499
+  --ignore RUSTSEC-2020-0159
+
 )
 scripts/cargo-for-all-lock-files.sh stable audit "${cargo_audit_ignores[@]}"

ci/run-sanity.sh

@@ -7,7 +7,7 @@ source multinode-demo/common.sh
 
 rm -rf config/run/init-completed config/ledger config/snapshot-ledger
 
-SOLANA_RUN_SH_VALIDATOR_ARGS="--snapshot-interval-slots 200" timeout 120 ./run.sh &
+SOLANA_RUN_SH_VALIDATOR_ARGS="--snapshot-interval-slots 200" timeout 120 ./scripts/run.sh &
 pid=$!
 
 attempts=20

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.7.12"
+version = "1.8.2"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,10 +12,10 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
 thiserror = "1.0.21"
-tiny-bip39 = "0.8.0"
+tiny-bip39 = "0.8.1"
 uriparse = "0.6.3"
 url = "2.1.0"
 chrono = "0.4"

clap-utils/src/keypair.rs

@@ -1,3 +1,14 @@
+//! Loading signers and keypairs from the command line.
+//!
+//! This module contains utilities for loading [Signer]s and [Keypair]s from
+//! standard signing sources, from the command line, as in the Solana CLI.
+//!
+//! The key function here is [`signer_from_path`], which loads a `Signer` from
+//! one of several possible sources by interpreting a "path" command line
+//! argument. Its documentation includes a description of all possible signing
+//! sources supported by the Solana CLI. Many other functions here are
+//! variations on, or delegate to, `signer_from_path`.
+
 use {
     crate::{
         input_parsers::{pubkeys_sigs_of, STDOUT_OUTFILE_TOKEN},
@@ -92,14 +103,56 @@ impl CliSignerInfo {
     }
 }
 
+/// A command line argument that loads a default signer in absence of other signers.
+///
+/// This type manages a default signing source which may be overridden by other
+/// signing sources via its [`generate_unique_signers`] method.
+///
+/// [`generate_unique_signers`]: DefaultSigner::generate_unique_signers
+///
+/// `path` is a signing source as documented by [`signer_from_path`], and
+/// `arg_name` is the name of its [clap] command line argument, which is passed
+/// to `signer_from_path` as its `keypair_name` argument.
 #[derive(Debug, Default)]
 pub struct DefaultSigner {
+    /// The name of the signers command line argument.
     pub arg_name: String,
+    /// The signing source.
     pub path: String,
     is_path_checked: RefCell<bool>,
 }
 
 impl DefaultSigner {
+    /// Create a new `DefaultSigner`.
+    ///
+    /// `path` is a signing source as documented by [`signer_from_path`], and
+    /// `arg_name` is the name of its [clap] command line argument, which is
+    /// passed to `signer_from_path` as its `keypair_name` argument.
+    ///
+    /// [clap]: https://docs.rs/clap
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::DefaultSigner;
+    /// use solana_clap_utils::offline::OfflineArgs;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .offline_args();
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    ///
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// # assert!(default_signer.arg_name.len() > 0);
+    /// assert_eq!(default_signer.path, keypair_str);
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn new<AN: AsRef<str>, P: AsRef<str>>(arg_name: AN, path: P) -> Self {
         let arg_name = arg_name.as_ref().to_string();
         let path = path.as_ref().to_string();
@@ -134,6 +187,57 @@ impl DefaultSigner {
         Ok(&self.path)
     }
 
+    /// Generate a unique set of signers, possibly excluding this default signer.
+    ///
+    /// This function allows a command line application to have a default
+    /// signer, perhaps representing a default wallet, but to override that
+    /// signer and instead sign with one or more other signers.
+    ///
+    /// `bulk_signers` is a vector of signers, all of which are optional. If any
+    /// of those signers is `None`, then the default signer will be loaded; if
+    /// all of those signers are `Some`, then the default signer will not be
+    /// loaded.
+    ///
+    /// The returned value includes all of the `bulk_signers` that were not
+    /// `None`, and maybe the default signer, if it was loaded.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::{DefaultSigner, signer_from_path};
+    /// use solana_clap_utils::offline::OfflineArgs;
+    /// use solana_sdk::signer::Signer;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .arg(Arg::with_name("payer")
+    ///         .long("payer")
+    ///         .help("The account paying for the transaction"))
+    ///     .offline_args();
+    ///
+    /// let mut wallet_manager = None;
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    /// let maybe_payer = clap_matches.value_of("payer");
+    ///
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// let maybe_payer_signer = maybe_payer.map(|payer| {
+    ///     signer_from_path(&clap_matches, payer, "payer", &mut wallet_manager)
+    /// }).transpose()?;
+    /// let bulk_signers = vec![maybe_payer_signer];
+    ///
+    /// let unique_signers = default_signer.generate_unique_signers(
+    ///     bulk_signers,
+    ///     &clap_matches,
+    ///     &mut wallet_manager,
+    /// )?;
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn generate_unique_signers(
         &self,
         bulk_signers: Vec<Option<Box<dyn Signer>>>,
@@ -158,6 +262,45 @@ impl DefaultSigner {
         })
     }
 
+    /// Loads the default [Signer] from one of several possible sources.
+    ///
+    /// The `path` is not strictly a file system path, but is interpreted as
+    /// various types of _signing source_, depending on its format, one of which
+    /// is a path to a keypair file. Some sources may require user interaction
+    /// in the course of calling this function.
+    ///
+    /// This simply delegates to the [`signer_from_path`] free function, passing
+    /// it the `DefaultSigner`s `path` and `arg_name` fields as the `path` and
+    /// `keypair_name` arguments.
+    ///
+    /// See the [`signer_from_path`] free function for full documentation of how
+    /// this function interprets its arguments.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::DefaultSigner;
+    /// use solana_clap_utils::offline::OfflineArgs;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .offline_args();
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// let mut wallet_manager = None;
+    ///
+    /// let signer = default_signer.signer_from_path(
+    ///     &clap_matches,
+    ///     &mut wallet_manager,
+    /// )?;
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn signer_from_path(
         &self,
         matches: &ArgMatches,
@@ -166,6 +309,51 @@ impl DefaultSigner {
         signer_from_path(matches, self.path()?, &self.arg_name, wallet_manager)
     }
 
+    /// Loads the default [Signer] from one of several possible sources.
+    ///
+    /// The `path` is not strictly a file system path, but is interpreted as
+    /// various types of _signing source_, depending on its format, one of which
+    /// is a path to a keypair file. Some sources may require user interaction
+    /// in the course of calling this function.
+    ///
+    /// This simply delegates to the [`signer_from_path_with_config`] free
+    /// function, passing it the `DefaultSigner`s `path` and `arg_name` fields
+    /// as the `path` and `keypair_name` arguments.
+    ///
+    /// See the [`signer_from_path`] free function for full documentation of how
+    /// this function interprets its arguments.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::{SignerFromPathConfig, DefaultSigner};
+    /// use solana_clap_utils::offline::OfflineArgs;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .offline_args();
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// let mut wallet_manager = None;
+    ///
+    /// // Allow pubkey signers without accompanying signatures
+    /// let config = SignerFromPathConfig {
+    ///     allow_null_signer: true,
+    /// };
+    ///
+    /// let signer = default_signer.signer_from_path_with_config(
+    ///     &clap_matches,
+    ///     &mut wallet_manager,
+    ///     &config,
+    /// )?;
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn signer_from_path_with_config(
         &self,
         matches: &ArgMatches,
@@ -258,6 +446,15 @@ pub(crate) fn parse_signer_source<S: AsRef<str>>(
     let source = {
         #[cfg(target_family = "windows")]
         {
+            // trim matched single-quotes since cmd.exe won't
+            let mut source = source;
+            while let Some(trimmed) = source.strip_prefix('\'') {
+                source = if let Some(trimmed) = trimmed.strip_suffix('\'') {
+                    trimmed
+                } else {
+                    break;
+                }
+            }
             source.replace("\\", "/")
         }
         #[cfg(not(target_family = "windows"))]
@@ -324,19 +521,167 @@ pub fn presigner_from_pubkey_sigs(
     })
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct SignerFromPathConfig {
     pub allow_null_signer: bool,
 }
 
-impl Default for SignerFromPathConfig {
-    fn default() -> Self {
-        Self {
-            allow_null_signer: false,
-        }
-    }
-}
-
+/// Loads a [Signer] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// The result of this function is a boxed object of the [Signer] trait. To load
+/// a concrete [Keypair], use the [keypair_from_path] function, though note that
+/// it does not support all signer sources.
+///
+/// The `matches` argument is the same set of parsed [clap] matches from which
+/// `path` was parsed. It is used to parse various additional command line
+/// arguments, depending on which signing source is requested, as described
+/// below in "Signing sources".
+///
+/// [clap]: https//docs.rs/clap
+///
+/// The `keypair_name` argument is the "name" of the signer, and is typically
+/// the name of the clap argument from which the `path` argument was parsed,
+/// like "keypair", "from", or "fee-payer". It is used solely for interactively
+/// prompting the user, either when entering seed phrases or selecting from
+/// multiple hardware wallets.
+///
+/// The `wallet_manager` is used for establishing connections to a hardware
+/// device such as Ledger. If `wallet_manager` is a reference to `None`, and a
+/// hardware signer is requested, then this function will attempt to create a
+/// wallet manager, assigning it to the mutable `wallet_manager` reference. This
+/// argument is typically a reference to `None`.
+///
+/// # Signing sources
+///
+/// The `path` argument can simply be a path to a keypair file, but it may also
+/// be interpreted in several other ways, in the following order.
+///
+/// Firstly, the `path` argument may be interpreted as a [URI], with the URI
+/// scheme indicating where to load the signer from. If it parses as a URI, then
+/// the following schemes are supported:
+///
+/// - `file:` &mdash; Read the keypair from a JSON keypair file. The path portion
+///    of the URI is the file path.
+///
+/// - `stdin:` &mdash; Read the keypair from stdin, in the JSON format used by
+///   the keypair file.
+///
+///   Non-scheme parts of the URI are ignored.
+///
+/// - `prompt:` &mdash; The user will be prompted at the command line
+///   for their seed phrase and passphrase.
+///
+///   In this URI the [query string][qs] may contain zero or one of the
+///   following key/value pairs that determine the [BIP44 derivation path][dp]
+///   of the private key from the seed:
+///
+///   - `key` &mdash; In this case the value is either one or two numerical
+///     indexes separated by a slash, which represent the "account", and
+///     "change" components of the BIP44 derivation path. Example: `key=0/0`.
+///
+///   - `full-path` &mdash; In this case the value is a full derivation path,
+///     and the user is responsible for ensuring it is correct. Example:
+///     `full-path=m/44/501/0/0/0`.
+///
+///   If neither is provided, then the default derivation path is used.
+///
+///   Note that when specifying derivation paths, this routine will convert all
+///   indexes into ["hardened"] indexes, even if written as "normal" indexes.
+///
+///   Other components of the URI besides the scheme and query string are ignored.
+///
+///   If the "skip_seed_phrase_validation" argument, as defined in
+///   [SKIP_SEED_PHRASE_VALIDATION_ARG] is found in `matches`, then the keypair
+///   seed will be generated directly from the seed phrase, without parsing or
+///   validating it as a BIP39 seed phrase. This allows the use of non-BIP39 seed
+///   phrases.
+///
+/// - `usb:` &mdash; Use a USB hardware device as the signer. In this case, the
+///   URI host indicates the device type, and is required. The only currently valid host
+///   value is "ledger".
+///
+///   Optionally, the first segment of the URI path indicates the base-58
+///   encoded pubkey of the wallet, and the "account" and "change" indices of
+///   the derivation path can be specified with the `key=` query parameter, as
+///   with the `prompt:` URI.
+///
+///   Examples:
+///
+///   - `usb://ledger`
+///   - `usb://ledger?key=0/0`
+///   - `usb://ledger/9rPVSygg3brqghvdZ6wsL2i5YNQTGhXGdJzF65YxaCQd`
+///   - `usb://ledger/9rPVSygg3brqghvdZ6wsL2i5YNQTGhXGdJzF65YxaCQd?key=0/0`
+///
+/// Next the `path` argument may be one of the following strings:
+///
+/// - `-` &mdash; Read the keypair from stdin. This is the same as the `stdin:`
+///   URI scheme.
+///
+/// - `ASK` &mdash; The user will be prompted at the command line for their seed
+///   phrase and passphrase. _This uses a legacy key derivation method and should
+///   usually be avoided in favor of `prompt:`._
+///
+/// Next, if the `path` argument parses as a base-58 public key, then the signer
+/// is created without a private key, but with presigned signatures, each parsed
+/// from the additional command line arguments, provided by the `matches`
+/// argument.
+///
+/// In this case, the remaining command line arguments are searched for clap
+/// arguments named "signer", as defined by [SIGNER_ARG], and each is parsed as
+/// a key-value pair of the form "pubkey=signature", where `pubkey` is the same
+/// base-58 public key, and `signature` is a serialized signature produced by
+/// the corresponding keypair. One of the "signer" signatures must be for the
+/// pubkey specified in `path` or this function will return an error; unless the
+/// "sign_only" clap argument, as defined by [SIGN_ONLY_ARG], is present in
+/// `matches`, in which case the signer will be created with no associated
+/// signatures.
+///
+/// Finally, if `path`, interpreted as a file path, represents a file on disk,
+/// then the signer is created by reading that file as a JSON-serialized
+/// keypair. This is the same as the `file:` URI scheme.
+///
+/// [qs]: https://en.wikipedia.org/wiki/Query_string
+/// [dp]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki
+/// [URI]: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
+/// ["hardened"]: https://wiki.trezor.io/Hardened_and_non-hardened_derivation
+///
+/// # Examples
+///
+/// This shows a reasonable way to set up clap to parse all possible signer
+/// sources. Note the use of the [`OfflineArgs::offline_args`] method to add
+/// correct clap definitions of the `--signer` and `--sign-only` arguments, as
+/// required by the base-58 pubkey offline signing method.
+///
+/// [`OfflineArgs::offline_args`]: crate::offline::OfflineArgs::offline_args
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::signer_from_path;
+/// use solana_clap_utils::offline::OfflineArgs;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"))
+///     .offline_args();
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+/// let mut wallet_manager = None;
+/// let signer = signer_from_path(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     &mut wallet_manager,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn signer_from_path(
     matches: &ArgMatches,
     path: &str,
@@ -347,6 +692,63 @@ pub fn signer_from_path(
     signer_from_path_with_config(matches, path, keypair_name, wallet_manager, &config)
 }
 
+/// Loads a [Signer] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// This is the same as [`signer_from_path`] except that it additionaolly
+/// accepts a [`SignerFromPathConfig`] argument.
+///
+/// If the `allow_null_signer` field of `config` is `true`, then pubkey signers
+/// are allowed to have zero associated signatures via additional "signer"
+/// command line arguments. It the same effect as if the "sign_only" clap
+/// argument is present.
+///
+/// See [`signer_from_path`] for full documentation of how this function
+/// interprets its arguments.
+///
+/// # Examples
+///
+/// This shows a reasonable way to set up clap to parse all possible signer
+/// sources. Note the use of the [`OfflineArgs::offline_args`] method to add
+/// correct clap definitions of the `--signer` and `--sign-only` arguments, as
+/// required by the base-58 pubkey offline signing method.
+///
+/// [`OfflineArgs::offline_args`]: crate::offline::OfflineArgs::offline_args
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::{signer_from_path_with_config, SignerFromPathConfig};
+/// use solana_clap_utils::offline::OfflineArgs;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"))
+///     .offline_args();
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+/// let mut wallet_manager = None;
+///
+/// // Allow pubkey signers without accompanying signatures
+/// let config = SignerFromPathConfig {
+///     allow_null_signer: true,
+/// };
+///
+/// let signer = signer_from_path_with_config(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     &mut wallet_manager,
+///     &config,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn signer_from_path_with_config(
     matches: &ArgMatches,
     path: &str,
@@ -417,6 +819,43 @@ pub fn signer_from_path_with_config(
     }
 }
 
+/// Loads the pubkey of a [Signer] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// The only difference between this function and [`signer_from_path`] is in the
+/// case of a "pubkey" path: this function does not require that accompanying
+/// command line arguments contain an offline signature.
+///
+/// See [`signer_from_path`] for full documentation of how this function
+/// interprets its arguments.
+///
+/// # Examples
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::pubkey_from_path;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"));
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+/// let mut wallet_manager = None;
+/// let pubkey = pubkey_from_path(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     &mut wallet_manager,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn pubkey_from_path(
     matches: &ArgMatches,
     path: &str,
@@ -516,7 +955,46 @@ pub fn prompt_passphrase(prompt: &str) -> Result<String, Box<dyn error::Error>>
     Ok(passphrase)
 }
 
-/// Parses a path into a SignerSource and returns a Keypair for supporting SignerSourceKinds
+/// Loads a [Keypair] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// This is the same as [`signer_from_path`] except that it only supports
+/// signing sources that can result in a [Keypair]: prompt for seed phrase,
+/// keypair file, and stdin.
+///
+/// If `confirm_pubkey` is `true` then after deriving the pubkey, the user will
+/// be prompted to confirm that the pubkey is as expected.
+///
+/// See [`signer_from_path`] for full documentation of how this function
+/// interprets its arguments.
+///
+/// # Examples
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::keypair_from_path;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"));
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+///
+/// let signer = keypair_from_path(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     false,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn keypair_from_path(
     matches: &ArgMatches,
     path: &str,
@@ -566,9 +1044,10 @@ pub fn keypair_from_path(
     }
 }
 
-/// Reads user input from stdin to retrieve a seed phrase and passphrase for keypair derivation
-/// Optionally skips validation of seed phrase
-/// Optionally confirms recovered public key
+/// Reads user input from stdin to retrieve a seed phrase and passphrase for keypair derivation.
+///
+/// Optionally skips validation of seed phrase. Optionally confirms recovered
+/// public key.
 pub fn keypair_from_seed_phrase(
     keypair_name: &str,
     skip_validation: bool,
@@ -645,9 +1124,13 @@ fn sanitize_seed_phrase(seed_phrase: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::offline::OfflineArgs;
+    use clap::{value_t_or_exit, App, Arg};
     use solana_remote_wallet::locator::Manufacturer;
+    use solana_remote_wallet::remote_wallet::initialize_wallet_manager;
+    use solana_sdk::signer::keypair::write_keypair_file;
     use solana_sdk::system_instruction;
-    use tempfile::NamedTempFile;
+    use tempfile::{NamedTempFile, TempDir};
 
     #[test]
     fn test_sanitize_seed_phrase() {
@@ -806,4 +1289,41 @@ mod tests {
             } if p == relative_path_str)
         );
     }
+
+    #[test]
+    fn signer_from_path_with_file() -> Result<(), Box<dyn std::error::Error>> {
+        let dir = TempDir::new()?;
+        let dir = dir.path();
+        let keypair_path = dir.join("id.json");
+        let keypair_path_str = keypair_path.to_str().expect("utf-8");
+
+        let keypair = Keypair::new();
+        write_keypair_file(&keypair, &keypair_path)?;
+
+        let args = vec!["program", keypair_path_str];
+
+        let clap_app = App::new("my-program")
+            .arg(
+                Arg::with_name("keypair")
+                    .required(true)
+                    .help("The signing keypair"),
+            )
+            .offline_args();
+
+        let clap_matches = clap_app.get_matches_from(args);
+        let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+
+        let wallet_manager = initialize_wallet_manager()?;
+
+        let signer = signer_from_path(
+            &clap_matches,
+            &keypair_str,
+            "signer",
+            &mut Some(wallet_manager),
+        )?;
+
+        assert_eq!(keypair.pubkey(), signer.pubkey());
+
+        Ok(())
+    }
 }

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -20,12 +20,12 @@ indicatif = "0.15.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.12" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-client = { path = "../client", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.12" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.12" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.2" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-client = { path = "../client", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.2" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.2" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [package.metadata.docs.rs]

cli-output/src/cli_output.rs

@@ -576,7 +576,7 @@ impl fmt::Display for CliValidators {
         for (version, info) in self.stake_by_version.iter() {
             writeln!(
                 f,
-                "{:<8} - {:3} current validators ({:>5.2}%){}",
+                "{:<8} - {:4} current validators ({:>5.2}%){}",
                 version,
                 info.current_validators,
                 100. * info.current_active_stake as f64 / self.total_active_stake as f64,

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.12"
+version = "1.8.2"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -26,33 +26,34 @@ humantime = "2.0.1"
 num-traits = "0.2"
 pretty-hex = "0.2.1"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+semver = "1.0.4"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.12" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.12" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.12" }
-solana-cli-output = { path = "../cli-output", version = "=1.7.12" }
-solana-client = { path = "../client", version = "=1.7.12" }
-solana-config-program = { path = "../programs/config", version = "=1.7.12" }
-solana-faucet = { path = "../faucet", version = "=1.7.12" }
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.12" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.2" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.8.2" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-cli-config = { path = "../cli-config", version = "=1.8.2" }
+solana-cli-output = { path = "../cli-output", version = "=1.8.2" }
+solana-client = { path = "../client", version = "=1.8.2" }
+solana-config-program = { path = "../programs/config", version = "=1.8.2" }
+solana-faucet = { path = "../faucet", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.2" }
 solana_rbpf = "=0.2.11"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.12" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.2" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.21"
-tiny-bip39 = "0.7.0"
+tiny-bip39 = "0.8.1"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "=1.7.12" }
-solana-streamer = { path = "../streamer", version = "=1.7.12" }
+solana-core = { path = "../core", version = "=1.8.2" }
+solana-streamer = { path = "../streamer", version = "=1.8.2" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cluster_query.rs

@@ -1721,7 +1721,7 @@ pub fn process_show_stakes(
                 // Filter by `StakeState::Stake(_, _)`
                 rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
                     offset: 0,
-                    bytes: rpc_filter::MemcmpEncodedBytes::Binary(
+                    bytes: rpc_filter::MemcmpEncodedBytes::Base58(
                         bs58::encode([2, 0, 0, 0]).into_string(),
                     ),
                     encoding: Some(rpc_filter::MemcmpEncoding::Binary),
@@ -1729,7 +1729,7 @@ pub fn process_show_stakes(
                 // Filter by `Delegation::voter_pubkey`, which begins at byte offset 124
                 rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
                     offset: 124,
-                    bytes: rpc_filter::MemcmpEncodedBytes::Binary(
+                    bytes: rpc_filter::MemcmpEncodedBytes::Base58(
                         vote_account_pubkeys[0].to_string(),
                     ),
                     encoding: Some(rpc_filter::MemcmpEncoding::Binary),

cli/src/feature.rs

@@ -18,7 +18,12 @@ use solana_sdk::{
     pubkey::Pubkey,
     transaction::Transaction,
 };
-use std::{collections::HashMap, fmt, sync::Arc};
+use std::{
+    cmp::Ordering,
+    collections::{HashMap, HashSet},
+    fmt,
+    sync::Arc,
+};
 
 #[derive(Copy, Clone, Debug, PartialEq)]
 pub enum ForceActivation {
@@ -222,44 +227,103 @@ pub fn process_feature_subcommand(
     }
 }
 
-fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, f64>, ClientError> {
+#[derive(Debug, Default)]
+struct WorkingFeatureSetStatsEntry {
+    stake: u64,
+    rpc_nodes_count: u32,
+    software_versions: HashSet<Option<semver::Version>>,
+}
+type WorkingFeatureSetStats = HashMap<u32, WorkingFeatureSetStatsEntry>;
+
+#[derive(Debug, Default)]
+struct FeatureSetStatsEntry {
+    stake_percent: f64,
+    rpc_nodes_percent: f32,
+    software_versions: Vec<Option<semver::Version>>,
+}
+type FeatureSetStats = HashMap<u32, FeatureSetStatsEntry>;
+
+fn feature_set_stats(rpc_client: &RpcClient) -> Result<FeatureSetStats, ClientError> {
     // Validator identity -> feature set
-    let feature_set_map = rpc_client
+    let feature_sets = rpc_client
         .get_cluster_nodes()?
         .into_iter()
-        .map(|contact_info| (contact_info.pubkey, contact_info.feature_set))
-        .collect::<HashMap<_, _>>();
+        .map(|contact_info| {
+            (
+                contact_info.pubkey,
+                contact_info.feature_set,
+                contact_info.rpc.is_some(),
+                contact_info
+                    .version
+                    .and_then(|v| semver::Version::parse(&v).ok()),
+            )
+        })
+        .collect::<Vec<_>>();
 
     let vote_accounts = rpc_client.get_vote_accounts()?;
 
-    let total_active_stake: u64 = vote_accounts
-        .current
+    let mut total_active_stake: u64 = vote_accounts
+        .delinquent
         .iter()
-        .chain(vote_accounts.delinquent.iter())
         .map(|vote_account| vote_account.activated_stake)
         .sum();
 
-    // Sum all active stake by feature set
-    let mut active_stake_by_feature_set: HashMap<u32, u64> = HashMap::new();
-    for vote_account in vote_accounts.current {
-        if let Some(Some(feature_set)) = feature_set_map.get(&vote_account.node_pubkey) {
-            *active_stake_by_feature_set.entry(*feature_set).or_default() +=
-                vote_account.activated_stake;
-        } else {
-            *active_stake_by_feature_set
-                .entry(0 /* "unknown" */)
-                .or_default() += vote_account.activated_stake;
+    let vote_stakes = vote_accounts
+        .current
+        .into_iter()
+        .map(|vote_account| {
+            total_active_stake += vote_account.activated_stake;
+            (vote_account.node_pubkey, vote_account.activated_stake)
+        })
+        .collect::<HashMap<_, _>>();
+
+    let mut feature_set_stats: WorkingFeatureSetStats = HashMap::new();
+    let mut total_rpc_nodes = 0;
+    for (node_id, feature_set, is_rpc, version) in feature_sets {
+        let feature_set = feature_set.unwrap_or(0);
+        let feature_set_entry = feature_set_stats.entry(feature_set).or_default();
+
+        feature_set_entry.software_versions.insert(version);
+
+        if let Some(vote_stake) = vote_stakes.get(&node_id) {
+            feature_set_entry.stake += *vote_stake;
+        }
+
+        if is_rpc {
+            feature_set_entry.rpc_nodes_count += 1;
+            total_rpc_nodes += 1;
         }
     }
 
-    Ok(active_stake_by_feature_set
+    Ok(feature_set_stats
         .into_iter()
-        .map(|(feature_set, active_stake)| {
-            (
+        .filter_map(
+            |(
                 feature_set,
-                active_stake as f64 * 100. / total_active_stake as f64,
-            )
-        })
+                WorkingFeatureSetStatsEntry {
+                    stake,
+                    rpc_nodes_count,
+                    software_versions,
+                },
+            )| {
+                let stake_percent = (stake as f64 / total_active_stake as f64) * 100.;
+                let rpc_nodes_percent = (rpc_nodes_count as f32 / total_rpc_nodes as f32) * 100.;
+                let mut software_versions = software_versions.into_iter().collect::<Vec<_>>();
+                software_versions.sort();
+                if stake_percent >= 0.001 || rpc_nodes_percent >= 0.001 {
+                    Some((
+                        feature_set,
+                        FeatureSetStatsEntry {
+                            stake_percent,
+                            rpc_nodes_percent,
+                            software_versions,
+                        },
+                    ))
+                } else {
+                    None
+                }
+            },
+        )
         .collect())
 }
 
@@ -267,50 +331,164 @@ fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, f6
 fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<bool, ClientError> {
     let my_feature_set = solana_version::Version::default().feature_set;
 
-    let active_stake_by_feature_set = active_stake_by_feature_set(rpc_client)?;
+    let feature_set_stats = feature_set_stats(rpc_client)?;
 
-    let feature_activation_allowed = active_stake_by_feature_set
+    let (stake_allowed, rpc_allowed) = feature_set_stats
         .get(&my_feature_set)
-        .map(|percentage| *percentage >= 95.)
-        .unwrap_or(false);
+        .map(
+            |FeatureSetStatsEntry {
+                 stake_percent,
+                 rpc_nodes_percent,
+                 ..
+             }| (*stake_percent >= 95., *rpc_nodes_percent >= 95.),
+        )
+        .unwrap_or((false, false));
 
-    if !feature_activation_allowed && !quiet {
-        if active_stake_by_feature_set.get(&my_feature_set).is_none() {
+    if !stake_allowed && !rpc_allowed && !quiet {
+        if feature_set_stats.get(&my_feature_set).is_none() {
             println!(
                 "{}",
                 style("To activate features the tool and cluster feature sets must match, select a tool version that matches the cluster")
                     .bold());
         } else {
-            println!(
-                "{}",
-                style("To activate features the stake must be >= 95%").bold()
-            );
+            if !stake_allowed {
+                print!(
+                    "\n{}",
+                    style("To activate features the stake must be >= 95%")
+                        .bold()
+                        .red()
+                );
+            }
+            if !rpc_allowed {
+                print!(
+                    "\n{}",
+                    style("To activate features the RPC nodes must be >= 95%")
+                        .bold()
+                        .red()
+                );
+            }
+        }
+        println!(
+            "\n\n{}",
+            style(format!("Tool Feature Set: {}", my_feature_set)).bold()
+        );
+
+        let mut feature_set_stats = feature_set_stats.into_iter().collect::<Vec<_>>();
+        feature_set_stats.sort_by(|l, r| {
+            match l.1.software_versions[0]
+                .cmp(&r.1.software_versions[0])
+                .reverse()
+            {
+                Ordering::Equal => {
+                    match l
+                        .1
+                        .stake_percent
+                        .partial_cmp(&r.1.stake_percent)
+                        .unwrap()
+                        .reverse()
+                    {
+                        Ordering::Equal => {
+                            l.1.rpc_nodes_percent
+                                .partial_cmp(&r.1.rpc_nodes_percent)
+                                .unwrap()
+                                .reverse()
+                        }
+                        o => o,
+                    }
+                }
+                o => o,
+            }
+        });
+
+        let software_versions_title = "Software Version";
+        let feature_set_title = "Feature Set";
+        let stake_percent_title = "Stake";
+        let rpc_percent_title = "RPC";
+        let mut stats_output = Vec::new();
+        let mut max_software_versions_len = software_versions_title.len();
+        let mut max_feature_set_len = feature_set_title.len();
+        let mut max_stake_percent_len = stake_percent_title.len();
+        let mut max_rpc_percent_len = rpc_percent_title.len();
+        for (
+            feature_set,
+            FeatureSetStatsEntry {
+                stake_percent,
+                rpc_nodes_percent,
+                software_versions,
+            },
+        ) in feature_set_stats.into_iter()
+        {
+            let me = feature_set == my_feature_set;
+            let feature_set = if feature_set == 0 {
+                "unknown".to_string()
+            } else {
+                feature_set.to_string()
+            };
+            let stake_percent = format!("{:.2}%", stake_percent);
+            let rpc_percent = format!("{:.2}%", rpc_nodes_percent);
+
+            let mut has_unknown = false;
+            let mut software_versions = software_versions
+                .iter()
+                .filter_map(|v| {
+                    if v.is_none() {
+                        has_unknown = true;
+                    }
+                    v.as_ref()
+                })
+                .map(ToString::to_string)
+                .collect::<Vec<_>>();
+            if has_unknown {
+                software_versions.push("unknown".to_string());
+            }
+            let software_versions = software_versions.join(", ");
+            max_software_versions_len = max_software_versions_len.max(software_versions.len());
+
+            max_feature_set_len = max_feature_set_len.max(feature_set.len());
+            max_stake_percent_len = max_stake_percent_len.max(stake_percent.len());
+            max_rpc_percent_len = max_rpc_percent_len.max(rpc_percent.len());
+
+            stats_output.push((
+                software_versions,
+                feature_set,
+                stake_percent,
+                rpc_percent,
+                me,
+            ));
         }
         println!(
             "{}",
-            style(format!("Tool Feature Set: {}", my_feature_set)).bold()
+            style(format!(
+                "{1:<0$}  {3:<2$}  {5:<4$}  {7:<6$}",
+                max_software_versions_len,
+                software_versions_title,
+                max_feature_set_len,
+                feature_set_title,
+                max_stake_percent_len,
+                stake_percent_title,
+                max_rpc_percent_len,
+                rpc_percent_title,
+            ))
+            .bold(),
         );
-        println!("{}", style("Cluster Feature Sets and Stakes:").bold());
-        for (feature_set, percentage) in active_stake_by_feature_set.iter() {
-            if *feature_set == 0 {
-                println!("  unknown    - {:.2}%", percentage);
-            } else {
-                println!(
-                    "  {:<10} - {:.2}% {}",
-                    feature_set,
-                    percentage,
-                    if *feature_set == my_feature_set {
-                        " <-- me"
-                    } else {
-                        ""
-                    }
-                );
-            }
+        for (software_versions, feature_set, stake_percent, rpc_percent, me) in stats_output {
+            println!(
+                "{1:<0$}  {3:>2$}  {5:>4$} {7:>6$}  {8}",
+                max_software_versions_len,
+                software_versions,
+                max_feature_set_len,
+                feature_set,
+                max_stake_percent_len,
+                stake_percent,
+                max_rpc_percent_len,
+                rpc_percent,
+                if me { "<-- me" } else { "" },
+            );
         }
         println!();
     }
 
-    Ok(feature_activation_allowed)
+    Ok(stake_allowed && rpc_allowed)
 }
 
 fn status_from_account(account: Account) -> Option<CliFeatureStatus> {

cli/src/program.rs

@@ -12,9 +12,9 @@ use solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig};
 use solana_bpf_loader_program::{bpf_verifier, BpfError, ThisInstructionMeter};
 use solana_clap_utils::{self, input_parsers::*, input_validators::*, keypair::*};
 use solana_cli_output::{
-    display::new_spinner_progress_bar, CliProgram, CliProgramAccountType, CliProgramAuthority,
-    CliProgramBuffer, CliProgramId, CliUpgradeableBuffer, CliUpgradeableBuffers,
-    CliUpgradeableProgram, CliUpgradeableProgramClosed, CliUpgradeablePrograms,
+    CliProgram, CliProgramAccountType, CliProgramAuthority, CliProgramBuffer, CliProgramId,
+    CliUpgradeableBuffer, CliUpgradeableBuffers, CliUpgradeableProgram,
+    CliUpgradeableProgramClosed, CliUpgradeablePrograms,
 };
 use solana_client::{
     client_error::ClientErrorKind,
@@ -22,8 +22,6 @@ use solana_client::{
     rpc_config::RpcSendTransactionConfig,
     rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig},
     rpc_filter::{Memcmp, MemcmpEncodedBytes, RpcFilterType},
-    rpc_request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS,
-    rpc_response::Fees,
     tpu_client::{TpuClient, TpuClientConfig},
 };
 use solana_rbpf::vm::{Config, Executable};
@@ -33,7 +31,6 @@ use solana_sdk::{
     account_utils::StateMut,
     bpf_loader, bpf_loader_deprecated,
     bpf_loader_upgradeable::{self, UpgradeableLoaderState},
-    commitment_config::CommitmentConfig,
     instruction::Instruction,
     instruction::InstructionError,
     loader_instruction,
@@ -42,24 +39,18 @@ use solana_sdk::{
     packet::PACKET_DATA_SIZE,
     pubkey::Pubkey,
     signature::{keypair_from_seed, read_keypair_file, Keypair, Signature, Signer},
-    signers::Signers,
     system_instruction::{self, SystemError},
     system_program,
     transaction::Transaction,
     transaction::TransactionError,
 };
-use solana_transaction_status::TransactionConfirmationStatus;
 use std::{
-    collections::HashMap,
-    error,
     fs::File,
     io::{Read, Write},
     mem::size_of,
     path::PathBuf,
     str::FromStr,
     sync::Arc,
-    thread::sleep,
-    time::Duration,
 };
 
 #[derive(Debug, PartialEq)]
@@ -385,6 +376,7 @@ impl ProgramSubCommands for App<'_, '_> {
         .subcommand(
             SubCommand::with_name("deploy")
                 .about("Deploy a program")
+                .setting(AppSettings::Hidden)
                 .arg(
                     Arg::with_name("program_location")
                         .index(1)
@@ -1146,18 +1138,18 @@ fn get_buffers(
 ) -> Result<CliUpgradeableBuffers, Box<dyn std::error::Error>> {
     let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
         offset: 0,
-        bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 0, 0, 0]).into_string()),
+        bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 0, 0, 0]).into_string()),
         encoding: None,
     })];
     if let Some(authority_pubkey) = authority_pubkey {
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
             encoding: None,
         }));
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE + OPTION_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(
+            bytes: MemcmpEncodedBytes::Base58(
                 bs58::encode(authority_pubkey.as_ref()).into_string(),
             ),
             encoding: None,
@@ -1199,18 +1191,18 @@ fn get_programs(
 ) -> Result<CliUpgradeablePrograms, Box<dyn std::error::Error>> {
     let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
         offset: 0,
-        bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 0, 0, 0]).into_string()),
+        bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 0, 0, 0]).into_string()),
         encoding: None,
     })];
     if let Some(authority_pubkey) = authority_pubkey {
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
             encoding: None,
         }));
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE + OPTION_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(
+            bytes: MemcmpEncodedBytes::Base58(
                 bs58::encode(authority_pubkey.as_ref()).into_string(),
             ),
             encoding: None,
@@ -1234,7 +1226,7 @@ fn get_programs(
             bytes.extend_from_slice(programdata_address.as_ref());
             let filters = vec![RpcFilterType::Memcmp(Memcmp {
                 offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(bs58::encode(bytes).into_string()),
+                bytes: MemcmpEncodedBytes::Base58(bs58::encode(bytes).into_string()),
                 encoding: None,
             })];
 
@@ -2018,7 +2010,13 @@ fn complete_partial_program_init(
         return Err("Buffer account is already executable".into());
     }
     if account.owner != *loader_id && !system_program::check_id(&account.owner) {
-        return Err("Buffer account is already owned by another account".into());
+        return Err("Buffer account passed is already in use by another program".into());
+    }
+    if !account.data.is_empty() && account.data.len() < account_data_len {
+        return Err(
+            "Buffer account passed is not large enough, may have been for a different deploy?"
+                .into(),
+        );
     }
 
     if account.data.is_empty() && system_program::check_id(&account.owner) {
@@ -2029,24 +2027,24 @@ fn complete_partial_program_init(
         if account.owner != *loader_id {
             instructions.push(system_instruction::assign(elf_pubkey, loader_id));
         }
-    }
-    if account.lamports < minimum_balance {
-        let balance = minimum_balance - account.lamports;
-        instructions.push(system_instruction::transfer(
-            payer_pubkey,
-            elf_pubkey,
-            balance,
-        ));
-        balance_needed = balance;
-    } else if account.lamports > minimum_balance
-        && system_program::check_id(&account.owner)
-        && !allow_excessive_balance
-    {
-        return Err(format!(
-            "Buffer account has a balance: {:?}; it may already be in use",
-            Sol(account.lamports)
-        )
-        .into());
+        if account.lamports < minimum_balance {
+            let balance = minimum_balance - account.lamports;
+            instructions.push(system_instruction::transfer(
+                payer_pubkey,
+                elf_pubkey,
+                balance,
+            ));
+            balance_needed = balance;
+        } else if account.lamports > minimum_balance
+            && system_program::check_id(&account.owner)
+            && !allow_excessive_balance
+        {
+            return Err(format!(
+                "Buffer account has a balance: {:?}; it may already be in use",
+                Sol(account.lamports)
+            )
+            .into());
+        }
     }
     Ok((instructions, balance_needed))
 }
@@ -2109,29 +2107,29 @@ fn send_deploy_messages(
     if let Some(write_messages) = write_messages {
         if let Some(write_signer) = write_signer {
             trace!("Writing program data");
-            let Fees {
-                blockhash,
-                last_valid_block_height,
-                ..
-            } = rpc_client
-                .get_fees_with_commitment(config.commitment)?
-                .value;
-            let mut write_transactions = vec![];
-            for message in write_messages.iter() {
-                let mut tx = Transaction::new_unsigned(message.clone());
-                tx.try_sign(&[payer_signer, write_signer], blockhash)?;
-                write_transactions.push(tx);
-            }
-
-            send_and_confirm_transactions_with_spinner(
+            let tpu_client = TpuClient::new(
                 rpc_client.clone(),
                 &config.websocket_url,
-                write_transactions,
-                &[payer_signer, write_signer],
-                config.commitment,
-                last_valid_block_height,
-            )
-            .map_err(|err| format!("Data writes to account failed: {}", err))?;
+                TpuClientConfig::default(),
+            )?;
+            let transaction_errors = tpu_client
+                .send_and_confirm_messages_with_spinner(
+                    write_messages,
+                    &[payer_signer, write_signer],
+                )
+                .map_err(|err| format!("Data writes to account failed: {}", err))?
+                .into_iter()
+                .flatten()
+                .collect::<Vec<_>>();
+
+            if !transaction_errors.is_empty() {
+                for transaction_error in &transaction_errors {
+                    error!("{:?}", transaction_error);
+                }
+                return Err(
+                    format!("{} write transactions failed", transaction_errors.len()).into(),
+                );
+            }
         }
     }
 
@@ -2183,9 +2181,8 @@ fn report_ephemeral_mnemonic(words: usize, mnemonic: bip39::Mnemonic) {
         words
     );
     eprintln!("{}\n{}\n{}", divider, phrase, divider);
-    eprintln!("To resume a deploy, pass the recovered keypair as");
-    eprintln!("the [PROGRAM_ADDRESS_SIGNER] argument to `solana deploy` or");
-    eprintln!("as the [BUFFER_SIGNER] to `solana program deploy` or `solana write-buffer'.");
+    eprintln!("To resume a deploy, pass the recovered keypair as the");
+    eprintln!("[BUFFER_SIGNER] to `solana program deploy` or `solana write-buffer'.");
     eprintln!("Or to recover the account's lamports, pass it as the");
     eprintln!(
         "[BUFFER_ACCOUNT_ADDRESS] argument to `solana program close`.\n{}",
@@ -2193,134 +2190,6 @@ fn report_ephemeral_mnemonic(words: usize, mnemonic: bip39::Mnemonic) {
     );
 }
 
-fn send_and_confirm_transactions_with_spinner<T: Signers>(
-    rpc_client: Arc<RpcClient>,
-    websocket_url: &str,
-    mut transactions: Vec<Transaction>,
-    signer_keys: &T,
-    commitment: CommitmentConfig,
-    mut last_valid_block_height: u64,
-) -> Result<(), Box<dyn error::Error>> {
-    let progress_bar = new_spinner_progress_bar();
-    let mut send_retries = 5;
-
-    progress_bar.set_message("Finding leader nodes...");
-    let tpu_client = TpuClient::new(
-        rpc_client.clone(),
-        websocket_url,
-        TpuClientConfig::default(),
-    )?;
-    loop {
-        // Send all transactions
-        let mut pending_transactions = HashMap::new();
-        let num_transactions = transactions.len();
-        for transaction in transactions {
-            if !tpu_client.send_transaction(&transaction) {
-                let _result = rpc_client
-                    .send_transaction_with_config(
-                        &transaction,
-                        RpcSendTransactionConfig {
-                            preflight_commitment: Some(commitment.commitment),
-                            ..RpcSendTransactionConfig::default()
-                        },
-                    )
-                    .ok();
-            }
-            pending_transactions.insert(transaction.signatures[0], transaction);
-            progress_bar.set_message(&format!(
-                "[{}/{}] Transactions sent",
-                pending_transactions.len(),
-                num_transactions
-            ));
-
-            // Throttle transactions to about 100 TPS
-            sleep(Duration::from_millis(10));
-        }
-
-        // Collect statuses for all the transactions, drop those that are confirmed
-        loop {
-            let mut block_height = 0;
-            let pending_signatures = pending_transactions.keys().cloned().collect::<Vec<_>>();
-            for pending_signatures_chunk in
-                pending_signatures.chunks(MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS)
-            {
-                if let Ok(result) = rpc_client.get_signature_statuses(pending_signatures_chunk) {
-                    let statuses = result.value;
-                    for (signature, status) in
-                        pending_signatures_chunk.iter().zip(statuses.into_iter())
-                    {
-                        if let Some(status) = status {
-                            if let Some(confirmation_status) = &status.confirmation_status {
-                                if *confirmation_status != TransactionConfirmationStatus::Processed
-                                {
-                                    let _ = pending_transactions.remove(signature);
-                                }
-                            } else if status.confirmations.is_none()
-                                || status.confirmations.unwrap() > 1
-                            {
-                                let _ = pending_transactions.remove(signature);
-                            }
-                        }
-                    }
-                }
-
-                block_height = rpc_client.get_block_height()?;
-                progress_bar.set_message(&format!(
-                    "[{}/{}] Transactions confirmed. Retrying in {} blocks",
-                    num_transactions - pending_transactions.len(),
-                    num_transactions,
-                    last_valid_block_height.saturating_sub(block_height)
-                ));
-            }
-
-            if pending_transactions.is_empty() {
-                return Ok(());
-            }
-
-            if block_height > last_valid_block_height {
-                break;
-            }
-
-            for transaction in pending_transactions.values() {
-                if !tpu_client.send_transaction(transaction) {
-                    let _result = rpc_client
-                        .send_transaction_with_config(
-                            transaction,
-                            RpcSendTransactionConfig {
-                                preflight_commitment: Some(commitment.commitment),
-                                ..RpcSendTransactionConfig::default()
-                            },
-                        )
-                        .ok();
-                }
-            }
-
-            if cfg!(not(test)) {
-                // Retry twice a second
-                sleep(Duration::from_millis(500));
-            }
-        }
-
-        if send_retries == 0 {
-            return Err("Transactions failed".into());
-        }
-        send_retries -= 1;
-
-        // Re-sign any failed transactions with a new blockhash and retry
-        let Fees {
-            blockhash,
-            last_valid_block_height: new_last_valid_block_height,
-            ..
-        } = rpc_client.get_fees_with_commitment(commitment)?.value;
-        last_valid_block_height = new_last_valid_block_height;
-        transactions = vec![];
-        for (_, mut transaction) in pending_transactions.into_iter() {
-            transaction.try_sign(signer_keys, blockhash)?;
-            transactions.push(transaction);
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

cli/src/stake.rs

@@ -39,13 +39,11 @@ use solana_sdk::{
     stake::{
         self,
         instruction::{self as stake_instruction, LockupArgs, StakeError},
-        state::{Authorized, Lockup, Meta, StakeAuthorize, StakeState},
+        state::{Authorized, Lockup, Meta, StakeActivationStatus, StakeAuthorize, StakeState},
     },
+    stake_history::StakeHistory,
     system_instruction::SystemError,
-    sysvar::{
-        clock,
-        stake_history::{self, StakeHistory},
-    },
+    sysvar::{clock, stake_history},
     transaction::Transaction,
 };
 use solana_vote_program::vote_state::VoteState;
@@ -2020,7 +2018,11 @@ pub fn build_stake_state(
             stake,
         ) => {
             let current_epoch = clock.epoch;
-            let (active_stake, activating_stake, deactivating_stake) = stake
+            let StakeActivationStatus {
+                effective,
+                activating,
+                deactivating,
+            } = stake
                 .delegation
                 .stake_activating_and_deactivating(current_epoch, Some(stake_history));
             let lockup = if lockup.is_in_force(clock, None) {
@@ -2055,9 +2057,9 @@ pub fn build_stake_state(
                 use_lamports_unit,
                 current_epoch,
                 rent_exempt_reserve: Some(*rent_exempt_reserve),
-                active_stake: u64_some_if_not_zero(active_stake),
-                activating_stake: u64_some_if_not_zero(activating_stake),
-                deactivating_stake: u64_some_if_not_zero(deactivating_stake),
+                active_stake: u64_some_if_not_zero(effective),
+                activating_stake: u64_some_if_not_zero(activating),
+                deactivating_stake: u64_some_if_not_zero(deactivating),
                 ..CliStakeState::default()
             }
         }

cli/tests/fixtures/build.sh

@@ -5,3 +5,4 @@ cd "$(dirname "$0")"
 
 make -C ../../../programs/bpf/c/
 cp ../../../programs/bpf/c/out/noop.so .
+cat noop.so noop.so noop.so > noop_large.so

cli/tests/program.rs

@@ -22,11 +22,11 @@ use std::{env, fs::File, io::Read, path::PathBuf, str::FromStr};
 fn test_cli_program_deploy_non_upgradeable() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -37,7 +37,7 @@ fn test_cli_program_deploy_non_upgradeable() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let minimum_balance_for_rent_exemption = rpc_client
@@ -55,7 +55,7 @@ fn test_cli_program_deploy_non_upgradeable() {
     process_command(&config).unwrap();
 
     config.command = CliCommand::Deploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         address: None,
         use_deprecated_loader: false,
         allow_excessive_balance: false,
@@ -75,7 +75,7 @@ fn test_cli_program_deploy_non_upgradeable() {
     assert_eq!(account0.lamports, minimum_balance_for_rent_exemption);
     assert_eq!(account0.owner, bpf_loader::id());
     assert!(account0.executable);
-    let mut file = File::open(pathbuf.to_str().unwrap().to_string()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap().to_string()).unwrap();
     let mut elf = Vec::new();
     file.read_to_end(&mut elf).unwrap();
     assert_eq!(account0.data, elf);
@@ -84,7 +84,7 @@ fn test_cli_program_deploy_non_upgradeable() {
     let custom_address_keypair = Keypair::new();
     config.signers = vec![&keypair, &custom_address_keypair];
     config.command = CliCommand::Deploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: false,
@@ -111,7 +111,7 @@ fn test_cli_program_deploy_non_upgradeable() {
     process_command(&config).unwrap();
     config.signers = vec![&keypair, &custom_address_keypair];
     config.command = CliCommand::Deploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: false,
@@ -120,7 +120,7 @@ fn test_cli_program_deploy_non_upgradeable() {
 
     // Use forcing parameter to deploy to account with excess balance
     config.command = CliCommand::Deploy {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: true,
@@ -139,11 +139,11 @@ fn test_cli_program_deploy_non_upgradeable() {
 fn test_cli_program_deploy_no_authority() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -154,7 +154,7 @@ fn test_cli_program_deploy_no_authority() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -181,7 +181,7 @@ fn test_cli_program_deploy_no_authority() {
     // Deploy a program
     config.signers = vec![&keypair, &upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: None,
         buffer_signer_index: None,
@@ -206,7 +206,7 @@ fn test_cli_program_deploy_no_authority() {
     // Attempt to upgrade the program
     config.signers = vec![&keypair, &upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: Some(program_id),
         buffer_signer_index: None,
@@ -223,11 +223,11 @@ fn test_cli_program_deploy_no_authority() {
 fn test_cli_program_deploy_with_authority() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -238,7 +238,7 @@ fn test_cli_program_deploy_with_authority() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -266,7 +266,7 @@ fn test_cli_program_deploy_with_authority() {
     let program_keypair = Keypair::new();
     config.signers = vec![&keypair, &upgrade_authority, &program_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: Some(2),
         program_pubkey: Some(program_keypair.pubkey()),
         buffer_signer_index: None,
@@ -313,7 +313,7 @@ fn test_cli_program_deploy_with_authority() {
     // Deploy the upgradeable program
     config.signers = vec![&keypair, &upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: None,
         buffer_signer_index: None,
@@ -354,7 +354,7 @@ fn test_cli_program_deploy_with_authority() {
     // Upgrade the program
     config.signers = vec![&keypair, &upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: Some(program_pubkey),
         buffer_signer_index: None,
@@ -408,7 +408,7 @@ fn test_cli_program_deploy_with_authority() {
     // Upgrade with new authority
     config.signers = vec![&keypair, &new_upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: Some(program_pubkey),
         buffer_signer_index: None,
@@ -482,7 +482,7 @@ fn test_cli_program_deploy_with_authority() {
     // Upgrade with no authority
     config.signers = vec![&keypair, &new_upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: Some(program_pubkey),
         buffer_signer_index: None,
@@ -497,7 +497,7 @@ fn test_cli_program_deploy_with_authority() {
     // deploy with finality
     config.signers = vec![&keypair, &new_upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: None,
         buffer_signer_index: None,
@@ -556,11 +556,11 @@ fn test_cli_program_deploy_with_authority() {
 fn test_cli_program_close_program() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -571,7 +571,7 @@ fn test_cli_program_close_program() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -599,7 +599,7 @@ fn test_cli_program_close_program() {
     let program_keypair = Keypair::new();
     config.signers = vec![&keypair, &upgrade_authority, &program_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: Some(2),
         program_pubkey: Some(program_keypair.pubkey()),
         buffer_signer_index: None,
@@ -638,11 +638,17 @@ fn test_cli_program_close_program() {
 fn test_cli_program_write_buffer() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
+
+    let mut noop_large_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_large_path.push("tests");
+    noop_large_path.push("fixtures");
+    noop_large_path.push("noop_large");
+    noop_large_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -653,7 +659,7 @@ fn test_cli_program_write_buffer() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -681,7 +687,7 @@ fn test_cli_program_write_buffer() {
     // Write a buffer with default params
     config.signers = vec![&keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: None,
         buffer_pubkey: None,
         buffer_authority_signer_index: None,
@@ -715,7 +721,7 @@ fn test_cli_program_write_buffer() {
     let buffer_keypair = Keypair::new();
     config.signers = vec![&keypair, &buffer_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
@@ -776,7 +782,7 @@ fn test_cli_program_write_buffer() {
     let authority_keypair = Keypair::new();
     config.signers = vec![&keypair, &buffer_keypair, &authority_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
@@ -813,7 +819,7 @@ fn test_cli_program_write_buffer() {
     let authority_keypair = Keypair::new();
     config.signers = vec![&keypair, &buffer_keypair, &authority_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: None,
         buffer_pubkey: None,
         buffer_authority_signer_index: Some(2),
@@ -885,7 +891,7 @@ fn test_cli_program_write_buffer() {
     // Write a buffer with default params
     config.signers = vec![&keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: None,
         buffer_pubkey: None,
         buffer_authority_signer_index: None,
@@ -919,17 +925,47 @@ fn test_cli_program_write_buffer() {
         pre_lamports + minimum_balance_for_buffer,
         recipient_account.lamports
     );
+
+    // write small buffer then attempt to deploy larger program
+    let buffer_keypair = Keypair::new();
+    config.signers = vec![&keypair, &buffer_keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
+        program_location: noop_path.to_str().unwrap().to_string(),
+        buffer_signer_index: Some(1),
+        buffer_pubkey: Some(buffer_keypair.pubkey()),
+        buffer_authority_signer_index: None,
+        max_len: None, //Some(max_len),
+    });
+    process_command(&config).unwrap();
+    config.signers = vec![&keypair, &buffer_keypair];
+    config.command = CliCommand::Program(ProgramCliCommand::Deploy {
+        program_location: Some(noop_large_path.to_str().unwrap().to_string()),
+        program_signer_index: None,
+        program_pubkey: None,
+        buffer_signer_index: Some(1),
+        buffer_pubkey: Some(buffer_keypair.pubkey()),
+        allow_excessive_balance: false,
+        upgrade_authority_signer_index: 1,
+        is_final: true,
+        max_len: None,
+    });
+    config.output_format = OutputFormat::JsonCompact;
+    let error = process_command(&config).unwrap_err();
+    assert_eq!(
+        error.to_string(),
+        "Buffer account passed is not large enough, may have been for a different deploy?"
+    );
 }
 
 #[test]
 fn test_cli_program_set_buffer_authority() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -940,7 +976,7 @@ fn test_cli_program_set_buffer_authority() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -964,7 +1000,7 @@ fn test_cli_program_set_buffer_authority() {
     let buffer_keypair = Keypair::new();
     config.signers = vec![&keypair, &buffer_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
@@ -1039,11 +1075,11 @@ fn test_cli_program_set_buffer_authority() {
 fn test_cli_program_mismatch_buffer_authority() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -1054,7 +1090,7 @@ fn test_cli_program_mismatch_buffer_authority() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -1079,7 +1115,7 @@ fn test_cli_program_mismatch_buffer_authority() {
     let buffer_keypair = Keypair::new();
     config.signers = vec![&keypair, &buffer_keypair, &buffer_authority];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
@@ -1097,7 +1133,7 @@ fn test_cli_program_mismatch_buffer_authority() {
     let upgrade_authority = Keypair::new();
     config.signers = vec![&keypair, &upgrade_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: None,
         buffer_signer_index: None,
@@ -1112,7 +1148,7 @@ fn test_cli_program_mismatch_buffer_authority() {
     // Attempt to deploy matched authority
     config.signers = vec![&keypair, &buffer_authority];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: None,
         program_pubkey: None,
         buffer_signer_index: None,
@@ -1129,11 +1165,11 @@ fn test_cli_program_mismatch_buffer_authority() {
 fn test_cli_program_show() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -1144,7 +1180,7 @@ fn test_cli_program_show() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -1172,7 +1208,7 @@ fn test_cli_program_show() {
     let authority_keypair = Keypair::new();
     config.signers = vec![&keypair, &buffer_keypair, &authority_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
@@ -1227,7 +1263,7 @@ fn test_cli_program_show() {
     let program_keypair = Keypair::new();
     config.signers = vec![&keypair, &authority_keypair, &program_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::Deploy {
-        program_location: Some(pathbuf.to_str().unwrap().to_string()),
+        program_location: Some(noop_path.to_str().unwrap().to_string()),
         program_signer_index: Some(2),
         program_pubkey: Some(program_keypair.pubkey()),
         buffer_signer_index: None,
@@ -1314,11 +1350,11 @@ fn test_cli_program_show() {
 fn test_cli_program_dump() {
     solana_logger::setup();
 
-    let mut pathbuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    pathbuf.push("tests");
-    pathbuf.push("fixtures");
-    pathbuf.push("noop");
-    pathbuf.set_extension("so");
+    let mut noop_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    noop_path.push("tests");
+    noop_path.push("fixtures");
+    noop_path.push("noop");
+    noop_path.set_extension("so");
 
     let mint_keypair = Keypair::new();
     let mint_pubkey = mint_keypair.pubkey();
@@ -1329,7 +1365,7 @@ fn test_cli_program_dump() {
     let rpc_client =
         RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
 
-    let mut file = File::open(pathbuf.to_str().unwrap()).unwrap();
+    let mut file = File::open(noop_path.to_str().unwrap()).unwrap();
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data).unwrap();
     let max_len = program_data.len();
@@ -1357,7 +1393,7 @@ fn test_cli_program_dump() {
     let authority_keypair = Keypair::new();
     config.signers = vec![&keypair, &buffer_keypair, &authority_keypair];
     config.command = CliCommand::Program(ProgramCliCommand::WriteBuffer {
-        program_location: pathbuf.to_str().unwrap().to_string(),
+        program_location: noop_path.to_str().unwrap().to_string(),
         buffer_signer_index: Some(1),
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.7.12"
+version = "1.8.2"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -24,14 +24,14 @@ semver = "0.11.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.12" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-faucet = { path = "../faucet", version = "=1.7.12" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.12" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.2" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-faucet = { path = "../faucet", version = "=1.8.2" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.2" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tungstenite = "0.10.1"
@@ -40,7 +40,7 @@ url = "2.1.1"
 [dev-dependencies]
 assert_matches = "1.3.0"
 jsonrpc-http-server = "18.0.0"
-solana-logger = { path = "../logger", version = "=1.7.12" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/http_sender.rs

@@ -125,46 +125,42 @@ impl RpcSender for HttpSender {
                         .body(request_json)
                         .send()
                 })
-            };
+            }?;
 
-            match response {
-                Ok(response) => {
-                    if !response.status().is_success() {
-                        if response.status() == StatusCode::TOO_MANY_REQUESTS
-                            && too_many_requests_retries > 0
-                        {
-                            let mut duration = Duration::from_millis(500);
-                            if let Some(retry_after) = response.headers().get(RETRY_AFTER) {
-                                if let Ok(retry_after) = retry_after.to_str() {
-                                    if let Ok(retry_after) = retry_after.parse::<u64>() {
-                                        if retry_after < 120 {
-                                            duration = Duration::from_secs(retry_after);
-                                        }
-                                    }
+            if !response.status().is_success() {
+                if response.status() == StatusCode::TOO_MANY_REQUESTS
+                    && too_many_requests_retries > 0
+                {
+                    let mut duration = Duration::from_millis(500);
+                    if let Some(retry_after) = response.headers().get(RETRY_AFTER) {
+                        if let Ok(retry_after) = retry_after.to_str() {
+                            if let Ok(retry_after) = retry_after.parse::<u64>() {
+                                if retry_after < 120 {
+                                    duration = Duration::from_secs(retry_after);
                                 }
                             }
+                        }
+                    }
 
-                            too_many_requests_retries -= 1;
-                            debug!(
+                    too_many_requests_retries -= 1;
+                    debug!(
                                 "Too many requests: server responded with {:?}, {} retries left, pausing for {:?}",
                                 response, too_many_requests_retries, duration
                             );
 
-                            sleep(duration);
-                            stats_updater.add_rate_limited_time(duration);
-                            continue;
-                        }
-                        return Err(response.error_for_status().unwrap_err().into());
-                    }
+                    sleep(duration);
+                    stats_updater.add_rate_limited_time(duration);
+                    continue;
+                }
+                return Err(response.error_for_status().unwrap_err().into());
+            }
 
-                    let response_text = tokio::task::block_in_place(move || response.text())?;
-
-                    let json: serde_json::Value = serde_json::from_str(&response_text)?;
-                    if json["error"].is_object() {
-                        return match serde_json::from_value::<RpcErrorObject>(json["error"].clone())
-                        {
-                            Ok(rpc_error_object) => {
-                                let data = match rpc_error_object.code {
+            let mut json =
+                tokio::task::block_in_place(move || response.json::<serde_json::Value>())?;
+            if json["error"].is_object() {
+                return match serde_json::from_value::<RpcErrorObject>(json["error"].clone()) {
+                    Ok(rpc_error_object) => {
+                        let data = match rpc_error_object.code {
                                     rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE => {
                                         match serde_json::from_value::<RpcSimulateTransactionResult>(json["error"]["data"].clone()) {
                                             Ok(data) => RpcResponseErrorData::SendTransactionPreflightFailure(data),
@@ -185,27 +181,22 @@ impl RpcSender for HttpSender {
                                     _ => RpcResponseErrorData::Empty
                                 };
 
-                                Err(RpcError::RpcResponseError {
-                                    code: rpc_error_object.code,
-                                    message: rpc_error_object.message,
-                                    data,
-                                }
-                                .into())
-                            }
-                            Err(err) => Err(RpcError::RpcRequestError(format!(
-                                "Failed to deserialize RPC error response: {} [{}]",
-                                serde_json::to_string(&json["error"]).unwrap(),
-                                err
-                            ))
-                            .into()),
-                        };
+                        Err(RpcError::RpcResponseError {
+                            code: rpc_error_object.code,
+                            message: rpc_error_object.message,
+                            data,
+                        }
+                        .into())
                     }
-                    return Ok(json["result"].clone());
-                }
-                Err(err) => {
-                    return Err(err.into());
-                }
+                    Err(err) => Err(RpcError::RpcRequestError(format!(
+                        "Failed to deserialize RPC error response: {} [{}]",
+                        serde_json::to_string(&json["error"]).unwrap(),
+                        err
+                    ))
+                    .into()),
+                };
             }
+            return Ok(json["result"].take());
         }
     }
 }

client/src/lib.rs

@@ -18,5 +18,6 @@ pub mod rpc_filter;
 pub mod rpc_request;
 pub mod rpc_response;
 pub mod rpc_sender;
+pub mod spinner;
 pub mod thin_client;
 pub mod tpu_client;

client/src/mock_sender.rs

@@ -7,19 +7,24 @@ use {
         rpc_request::RpcRequest,
         rpc_response::{
             Response, RpcAccountBalance, RpcBlockProduction, RpcBlockProductionRange,
-            RpcConfirmedTransactionStatusWithSignature, RpcContactInfo, RpcFees, RpcPerfSample,
-            RpcResponseContext, RpcSimulateTransactionResult, RpcStakeActivation, RpcSupply,
-            RpcVersionInfo, RpcVoteAccountInfo, RpcVoteAccountStatus, StakeActivationState,
+            RpcConfirmedTransactionStatusWithSignature, RpcContactInfo, RpcFees, RpcIdentity,
+            RpcInflationGovernor, RpcInflationRate, RpcInflationReward, RpcKeyedAccount,
+            RpcPerfSample, RpcResponseContext, RpcSimulateTransactionResult, RpcStakeActivation,
+            RpcSupply, RpcVersionInfo, RpcVoteAccountInfo, RpcVoteAccountStatus,
+            StakeActivationState,
         },
         rpc_sender::*,
     },
     serde_json::{json, Number, Value},
+    solana_account_decoder::{UiAccount, UiAccountEncoding},
     solana_sdk::{
+        account::Account,
         clock::{Slot, UnixTimestamp},
         epoch_info::EpochInfo,
         fee_calculator::{FeeCalculator, FeeRateGovernor},
         instruction::InstructionError,
         message::MessageHeader,
+        pubkey::Pubkey,
         signature::Signature,
         sysvar::epoch_schedule::EpochSchedule,
         transaction::{self, Transaction, TransactionError},
@@ -31,7 +36,7 @@ use {
         UiTransactionEncoding, UiTransactionStatusMeta,
     },
     solana_version::Version,
-    std::{collections::HashMap, net::SocketAddr, sync::RwLock},
+    std::{collections::HashMap, net::SocketAddr, str::FromStr, sync::RwLock},
 };
 
 pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
@@ -141,8 +146,8 @@ impl RpcSender for MockSender {
                 value: serde_json::to_value(RpcFees {
                     blockhash: PUBKEY.to_string(),
                     fee_calculator: FeeCalculator::default(),
-                    last_valid_slot: 42,
-                    last_valid_block_height: 42,
+                    last_valid_slot: 1234,
+                    last_valid_block_height: 1234,
                 })
                 .unwrap(),
             })?,
@@ -381,6 +386,60 @@ impl RpcSender for MockSender {
                 num_slots: 123,
                 sample_period_secs: 60,
             }])?,
+            "getIdentity" => serde_json::to_value(RpcIdentity {
+                identity: PUBKEY.to_string(),
+            })?,
+            "getInflationGovernor" => serde_json::to_value(
+                RpcInflationGovernor {
+                    initial: 0.08,
+                    terminal: 0.015,
+                    taper: 0.15,
+                    foundation: 0.05,
+                    foundation_term: 7.0,
+                })?,
+            "getInflationRate" => serde_json::to_value(
+                RpcInflationRate {
+                    total: 0.08,
+                    validator: 0.076,
+                    foundation: 0.004,
+                    epoch: 0,
+                })?,
+            "getInflationReward" => serde_json::to_value(vec![
+                Some(RpcInflationReward {
+                    epoch: 2,
+                    effective_slot: 224,
+                    amount: 2500,
+                    post_balance: 499999442500,
+                    commission: None,
+                })])?,
+            "minimumLedgerSlot" => json![123],
+            "getMaxRetransmitSlot" => json![123],
+            "getMultipleAccounts" => serde_json::to_value(Response {
+                context: RpcResponseContext { slot: 1 },
+                value: vec![Value::Null, Value::Null]
+            })?,
+            "getProgramAccounts" => {
+                let pubkey = Pubkey::from_str(&PUBKEY.to_string()).unwrap();
+                let account = Account {
+                    lamports: 1_000_000,
+                    data: vec![],
+                    owner: pubkey,
+                    executable: false,
+                    rent_epoch: 0,
+                };
+                serde_json::to_value(vec![
+                    RpcKeyedAccount {
+                        pubkey: PUBKEY.to_string(),
+                        account: UiAccount::encode(
+                            &pubkey,
+                            &account,
+                            UiAccountEncoding::Base64,
+                            None,
+                            None,
+                        )
+                    }
+                ])?
+            },
             _ => Value::Null,
         };
         Ok(val)

client/src/pubsub_client.rs

@@ -22,7 +22,8 @@ use {
             mpsc::{channel, Receiver},
             Arc, RwLock,
         },
-        thread::JoinHandle,
+        thread::{sleep, JoinHandle},
+        time::Duration,
     },
     thiserror::Error,
     tungstenite::{client::AutoStream, connect, Message, WebSocket},
@@ -164,6 +165,28 @@ pub type SignatureSubscription = (
 
 pub struct PubsubClient {}
 
+fn connect_with_retry(url: Url) -> Result<WebSocket<AutoStream>, tungstenite::Error> {
+    let mut connection_retries = 5;
+    loop {
+        let result = connect(url.clone()).map(|(socket, _)| socket);
+        if let Err(tungstenite::Error::Http(status_code)) = &result {
+            if *status_code == reqwest::StatusCode::TOO_MANY_REQUESTS && connection_retries > 0 {
+                let duration = Duration::from_millis(500) * 2u32.pow(5 - connection_retries);
+
+                connection_retries -= 1;
+                debug!(
+                    "Too many requests: server responded with {:?}, {} retries left, pausing for {:?}",
+                    status_code, connection_retries, duration
+                );
+
+                sleep(duration);
+                continue;
+            }
+        }
+        return result;
+    }
+}
+
 impl PubsubClient {
     pub fn logs_subscribe(
         url: &str,
@@ -171,7 +194,7 @@ impl PubsubClient {
         config: RpcTransactionLogsConfig,
     ) -> Result<LogsSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
-        let (socket, _response) = connect(url)?;
+        let socket = connect_with_retry(url)?;
         let (sender, receiver) = channel();
 
         let socket = Arc::new(RwLock::new(socket));
@@ -226,7 +249,7 @@ impl PubsubClient {
 
     pub fn slot_subscribe(url: &str) -> Result<SlotsSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
-        let (socket, _response) = connect(url)?;
+        let socket = connect_with_retry(url)?;
         let (sender, receiver) = channel::<SlotInfo>();
 
         let socket = Arc::new(RwLock::new(socket));
@@ -282,7 +305,7 @@ impl PubsubClient {
         config: Option<RpcSignatureSubscribeConfig>,
     ) -> Result<SignatureSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
-        let (socket, _response) = connect(url)?;
+        let socket = connect_with_retry(url)?;
         let (sender, receiver) = channel();
 
         let socket = Arc::new(RwLock::new(socket));
@@ -348,7 +371,7 @@ impl PubsubClient {
         handler: impl Fn(SlotUpdate) + Send + 'static,
     ) -> Result<PubsubClientSubscription<SlotUpdate>, PubsubClientError> {
         let url = Url::parse(url)?;
-        let (socket, _response) = connect(url)?;
+        let socket = connect_with_retry(url)?;
 
         let socket = Arc::new(RwLock::new(socket));
         let exit = Arc::new(AtomicBool::new(false));

client/src/rpc_custom_error.rs

@@ -19,6 +19,7 @@ pub const JSON_RPC_SERVER_ERROR_KEY_EXCLUDED_FROM_SECONDARY_INDEX: i64 = -32010;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_HISTORY_NOT_AVAILABLE: i64 = -32011;
 pub const JSON_RPC_SCAN_ERROR: i64 = -32012;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_LEN_MISMATCH: i64 = -32013;
+pub const JSON_RPC_SERVER_ERROR_BLOCK_STATUS_NOT_AVAILABLE_YET: i64 = -32014;
 
 #[derive(Error, Debug)]
 pub enum RpcCustomError {
@@ -54,6 +55,8 @@ pub enum RpcCustomError {
     ScanError { message: String },
     #[error("TransactionSignatureLenMismatch")]
     TransactionSignatureLenMismatch,
+    #[error("BlockStatusNotAvailableYet")]
+    BlockStatusNotAvailableYet { slot: Slot },
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -161,6 +164,11 @@ impl From<RpcCustomError> for Error {
                 message: "Transaction signature length mismatch".to_string(),
                 data: None,
             },
+            RpcCustomError::BlockStatusNotAvailableYet { slot } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_BLOCK_STATUS_NOT_AVAILABLE_YET),
+                message: format!("Block status not yet available for slot {}", slot),
+                data: None,
+            },
         }
     }
 }

client/src/rpc_filter.rs

@@ -1,6 +1,11 @@
-use thiserror::Error;
+#![allow(deprecated)]
+use {std::borrow::Cow, thiserror::Error};
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+const MAX_DATA_SIZE: usize = 128;
+const MAX_DATA_BASE58_SIZE: usize = 175;
+const MAX_DATA_BASE64_SIZE: usize = 172;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum RpcFilterType {
     DataSize(u64),
@@ -15,15 +20,50 @@ impl RpcFilterType {
                 let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary);
                 match encoding {
                     MemcmpEncoding::Binary => {
-                        let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;
-
-                        if bytes.len() > 128 {
-                            Err(RpcFilterError::Base58DataTooLarge)
-                        } else {
-                            bs58::decode(&bytes)
-                                .into_vec()
-                                .map(|_| ())
-                                .map_err(|e| e.into())
+                        use MemcmpEncodedBytes::*;
+                        match &compare.bytes {
+                            // DEPRECATED
+                            Binary(bytes) => {
+                                if bytes.len() > MAX_DATA_BASE58_SIZE {
+                                    return Err(RpcFilterError::Base58DataTooLarge);
+                                }
+                                let bytes = bs58::decode(&bytes)
+                                    .into_vec()
+                                    .map_err(RpcFilterError::DecodeError)?;
+                                if bytes.len() > MAX_DATA_SIZE {
+                                    Err(RpcFilterError::Base58DataTooLarge)
+                                } else {
+                                    Ok(())
+                                }
+                            }
+                            Base58(bytes) => {
+                                if bytes.len() > MAX_DATA_BASE58_SIZE {
+                                    return Err(RpcFilterError::DataTooLarge);
+                                }
+                                let bytes = bs58::decode(&bytes).into_vec()?;
+                                if bytes.len() > MAX_DATA_SIZE {
+                                    Err(RpcFilterError::DataTooLarge)
+                                } else {
+                                    Ok(())
+                                }
+                            }
+                            Base64(bytes) => {
+                                if bytes.len() > MAX_DATA_BASE64_SIZE {
+                                    return Err(RpcFilterError::DataTooLarge);
+                                }
+                                let bytes = base64::decode(&bytes)?;
+                                if bytes.len() > MAX_DATA_SIZE {
+                                    Err(RpcFilterError::DataTooLarge)
+                                } else {
+                                    Ok(())
+                                }
+                            }
+                            Bytes(bytes) => {
+                                if bytes.len() > MAX_DATA_SIZE {
+                                    return Err(RpcFilterError::DataTooLarge);
+                                }
+                                Ok(())
+                            }
                         }
                     }
                 }
@@ -34,25 +74,46 @@ impl RpcFilterType {
 
 #[derive(Error, PartialEq, Debug)]
 pub enum RpcFilterError {
-    #[error("bs58 decode error")]
-    DecodeError(#[from] bs58::decode::Error),
+    #[error("encoded binary data should be less than 129 bytes")]
+    DataTooLarge,
+    #[deprecated(
+        since = "1.9.0",
+        note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
+    )]
     #[error("encoded binary (base 58) data should be less than 129 bytes")]
     Base58DataTooLarge,
+    #[deprecated(
+        since = "1.9.0",
+        note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
+    )]
+    #[error("bs58 decode error")]
+    DecodeError(bs58::decode::Error),
+    #[error("base58 decode error")]
+    Base58DecodeError(#[from] bs58::decode::Error),
+    #[error("base64 decode error")]
+    Base64DecodeError(#[from] base64::DecodeError),
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum MemcmpEncoding {
     Binary,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase", untagged)]
 pub enum MemcmpEncodedBytes {
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use MemcmpEncodedBytes::Base58 instead"
+    )]
     Binary(String),
+    Base58(String),
+    Base64(String),
+    Bytes(Vec<u8>),
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct Memcmp {
     /// Data offset to begin match
     pub offset: usize,
@@ -63,14 +124,18 @@ pub struct Memcmp {
 }
 
 impl Memcmp {
-    pub fn bytes_match(&self, data: &[u8]) -> bool {
+    pub fn bytes(&self) -> Option<Cow<Vec<u8>>> {
+        use MemcmpEncodedBytes::*;
         match &self.bytes {
-            MemcmpEncodedBytes::Binary(bytes) => {
-                let bytes = bs58::decode(bytes).into_vec();
-                if bytes.is_err() {
-                    return false;
-                }
-                let bytes = bytes.unwrap();
+            Binary(bytes) | Base58(bytes) => bs58::decode(bytes).into_vec().ok().map(Cow::Owned),
+            Base64(bytes) => base64::decode(bytes).ok().map(Cow::Owned),
+            Bytes(bytes) => Some(Cow::Borrowed(bytes)),
+        }
+    }
+
+    pub fn bytes_match(&self, data: &[u8]) -> bool {
+        match self.bytes() {
+            Some(bytes) => {
                 if self.offset > data.len() {
                     return false;
                 }
@@ -79,6 +144,7 @@ impl Memcmp {
                 }
                 data[self.offset..self.offset + bytes.len()] == bytes[..]
             }
+            None => false,
         }
     }
 }
@@ -87,6 +153,15 @@ impl Memcmp {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_worst_case_encoded_tx_goldens() {
+        let ff_data = vec![0xffu8; MAX_DATA_SIZE];
+        let data58 = bs58::encode(&ff_data).into_string();
+        assert_eq!(data58.len(), MAX_DATA_BASE58_SIZE);
+        let data64 = base64::encode(&ff_data);
+        assert_eq!(data64.len(), MAX_DATA_BASE64_SIZE);
+    }
+
     #[test]
     fn test_bytes_match() {
         let data = vec![1, 2, 3, 4, 5];
@@ -94,7 +169,7 @@ mod tests {
         // Exact match of data succeeds
         assert!(Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -102,7 +177,7 @@ mod tests {
         // Partial match of data succeeds
         assert!(Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -110,7 +185,7 @@ mod tests {
         // Offset partial match of data succeeds
         assert!(Memcmp {
             offset: 2,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -118,7 +193,7 @@ mod tests {
         // Incorrect partial match of data fails
         assert!(!Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![2]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -126,7 +201,7 @@ mod tests {
         // Bytes overrun data fails
         assert!(!Memcmp {
             offset: 2,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4, 5, 6]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -134,7 +209,7 @@ mod tests {
         // Offset outside data fails
         assert!(!Memcmp {
             offset: 6,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![5]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -142,7 +217,7 @@ mod tests {
         // Invalid base-58 fails
         assert!(!Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary("III".to_string()),
+            bytes: MemcmpEncodedBytes::Base58("III".to_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -157,7 +232,7 @@ mod tests {
         assert_eq!(
             RpcFilterType::Memcmp(Memcmp {
                 offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
+                bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
                 encoding: None,
             })
             .verify(),
@@ -172,11 +247,11 @@ mod tests {
         assert_eq!(
             RpcFilterType::Memcmp(Memcmp {
                 offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
+                bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
                 encoding: None,
             })
             .verify(),
-            Err(RpcFilterError::Base58DataTooLarge)
+            Err(RpcFilterError::DataTooLarge)
         );
     }
 }

client/src/spinner.rs

@@ -0,0 +1,11 @@
+//! Spinner creator
+
+use indicatif::{ProgressBar, ProgressStyle};
+
+pub(crate) fn new_progress_bar() -> ProgressBar {
+    let progress_bar = ProgressBar::new(42);
+    progress_bar
+        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
+    progress_bar.enable_steady_tick(100);
+    progress_bar
+}

client/src/tpu_client.rs

@@ -1,12 +1,21 @@
 use crate::{
+    client_error::ClientError,
     pubsub_client::{PubsubClient, PubsubClientError, PubsubClientSubscription},
     rpc_client::RpcClient,
-    rpc_response::SlotUpdate,
+    rpc_request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS,
+    rpc_response::{Fees, SlotUpdate},
+    spinner,
 };
 use bincode::serialize;
 use log::*;
 use solana_sdk::{
-    clock::Slot, commitment_config::CommitmentConfig, pubkey::Pubkey, transaction::Transaction,
+    clock::Slot,
+    commitment_config::CommitmentConfig,
+    message::Message,
+    pubkey::Pubkey,
+    signature::SignerError,
+    signers::Signers,
+    transaction::{Transaction, TransactionError},
 };
 use std::{
     collections::{HashMap, HashSet, VecDeque},
@@ -16,7 +25,7 @@ use std::{
         atomic::{AtomicBool, Ordering},
         Arc, RwLock,
     },
-    thread::JoinHandle,
+    thread::{sleep, JoinHandle},
     time::{Duration, Instant},
 };
 use thiserror::Error;
@@ -26,9 +35,13 @@ pub enum TpuSenderError {
     #[error("Pubsub error: {0:?}")]
     PubsubError(#[from] PubsubClientError),
     #[error("RPC error: {0:?}")]
-    RpcError(#[from] crate::client_error::ClientError),
+    RpcError(#[from] ClientError),
     #[error("IO error: {0:?}")]
     IoError(#[from] std::io::Error),
+    #[error("Signer error: {0:?}")]
+    SignerError(#[from] SignerError),
+    #[error("Custom error: {0}")]
+    Custom(String),
 }
 
 type Result<T> = std::result::Result<T, TpuSenderError>;
@@ -62,6 +75,7 @@ pub struct TpuClient {
     fanout_slots: u64,
     leader_tpu_service: LeaderTpuService,
     exit: Arc<AtomicBool>,
+    rpc_client: Arc<RpcClient>,
 }
 
 impl TpuClient {
@@ -96,15 +110,161 @@ impl TpuClient {
         config: TpuClientConfig,
     ) -> Result<Self> {
         let exit = Arc::new(AtomicBool::new(false));
-        let leader_tpu_service = LeaderTpuService::new(rpc_client, websocket_url, exit.clone())?;
+        let leader_tpu_service =
+            LeaderTpuService::new(rpc_client.clone(), websocket_url, exit.clone())?;
 
         Ok(Self {
             send_socket: UdpSocket::bind("0.0.0.0:0").unwrap(),
             fanout_slots: config.fanout_slots.min(MAX_FANOUT_SLOTS).max(1),
             leader_tpu_service,
             exit,
+            rpc_client,
         })
     }
+
+    pub fn send_and_confirm_messages_with_spinner<T: Signers>(
+        &self,
+        messages: &[Message],
+        signers: &T,
+    ) -> Result<Vec<Option<TransactionError>>> {
+        let mut expired_blockhash_retries = 5;
+        /* Send at ~100 TPS */
+        const SEND_TRANSACTION_INTERVAL: Duration = Duration::from_millis(10);
+        /* Retry batch send after 4 seconds */
+        const TRANSACTION_RESEND_INTERVAL: Duration = Duration::from_secs(4);
+
+        let progress_bar = spinner::new_progress_bar();
+        progress_bar.set_message("Setting up...");
+
+        let mut transactions = messages
+            .iter()
+            .enumerate()
+            .map(|(i, message)| (i, Transaction::new_unsigned(message.clone())))
+            .collect::<Vec<_>>();
+        let num_transactions = transactions.len() as f64;
+        let mut transaction_errors = vec![None; transactions.len()];
+        let set_message = |confirmed_transactions,
+                           block_height: Option<u64>,
+                           last_valid_block_height: u64,
+                           status: &str| {
+            progress_bar.set_message(&format!(
+                "{:>5.1}% | {:<40}{}",
+                confirmed_transactions as f64 * 100. / num_transactions,
+                status,
+                match block_height {
+                    Some(block_height) => format!(
+                        " [block height {}; re-sign in {} blocks]",
+                        block_height,
+                        last_valid_block_height.saturating_sub(block_height),
+                    ),
+                    None => String::new(),
+                },
+            ));
+        };
+
+        let mut confirmed_transactions = 0;
+        let mut block_height = self.rpc_client.get_block_height()?;
+        while expired_blockhash_retries > 0 {
+            let Fees {
+                blockhash,
+                fee_calculator: _,
+                last_valid_block_height,
+            } = self.rpc_client.get_fees()?;
+
+            let mut pending_transactions = HashMap::new();
+            for (i, mut transaction) in transactions {
+                transaction.try_sign(signers, blockhash)?;
+                pending_transactions.insert(transaction.signatures[0], (i, transaction));
+            }
+
+            let mut last_resend = Instant::now() - TRANSACTION_RESEND_INTERVAL;
+            while block_height <= last_valid_block_height {
+                let num_transactions = pending_transactions.len();
+
+                // Periodically re-send all pending transactions
+                if Instant::now().duration_since(last_resend) > TRANSACTION_RESEND_INTERVAL {
+                    for (index, (_i, transaction)) in pending_transactions.values().enumerate() {
+                        if !self.send_transaction(transaction) {
+                            let _result = self.rpc_client.send_transaction(transaction).ok();
+                        }
+                        set_message(
+                            confirmed_transactions,
+                            None, //block_height,
+                            last_valid_block_height,
+                            &format!("Sending {}/{} transactions", index + 1, num_transactions,),
+                        );
+                        sleep(SEND_TRANSACTION_INTERVAL);
+                    }
+                    last_resend = Instant::now();
+                }
+
+                // Wait for the next block before checking for transaction statuses
+                let mut block_height_refreshes = 10;
+                set_message(
+                    confirmed_transactions,
+                    Some(block_height),
+                    last_valid_block_height,
+                    &format!("Waiting for next block, {} pending...", num_transactions),
+                );
+                let mut new_block_height = block_height;
+                while block_height == new_block_height && block_height_refreshes > 0 {
+                    sleep(Duration::from_millis(500));
+                    new_block_height = self.rpc_client.get_block_height()?;
+                    block_height_refreshes -= 1;
+                }
+                block_height = new_block_height;
+
+                // Collect statuses for the transactions, drop those that are confirmed
+                let pending_signatures = pending_transactions.keys().cloned().collect::<Vec<_>>();
+                for pending_signatures_chunk in
+                    pending_signatures.chunks(MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS)
+                {
+                    if let Ok(result) = self
+                        .rpc_client
+                        .get_signature_statuses(pending_signatures_chunk)
+                    {
+                        let statuses = result.value;
+                        for (signature, status) in
+                            pending_signatures_chunk.iter().zip(statuses.into_iter())
+                        {
+                            if let Some(status) = status {
+                                if status.satisfies_commitment(self.rpc_client.commitment()) {
+                                    if let Some((i, _)) = pending_transactions.remove(signature) {
+                                        confirmed_transactions += 1;
+                                        if status.err.is_some() {
+                                            progress_bar.println(format!(
+                                                "Failed transaction: {:?}",
+                                                status
+                                            ));
+                                        }
+                                        transaction_errors[i] = status.err;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    set_message(
+                        confirmed_transactions,
+                        Some(block_height),
+                        last_valid_block_height,
+                        "Checking transaction status...",
+                    );
+                }
+
+                if pending_transactions.is_empty() {
+                    return Ok(transaction_errors);
+                }
+            }
+
+            transactions = pending_transactions.into_iter().map(|(_k, v)| v).collect();
+            progress_bar.println(format!(
+                "Blockhash expired. {} retries remaining",
+                expired_blockhash_retries
+            ));
+            expired_blockhash_retries -= 1;
+        }
+        Err(TpuSenderError::Custom("Max retries exceeded".into()))
+    }
 }
 
 impl Drop for TpuClient {

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.12"
+version = "1.8.2"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -27,13 +27,14 @@ ed25519-dalek = "=1.0.1"
 fs_extra = "1.2.0"
 flate2 = "1.0"
 indexmap = { version = "1.5", features = ["rayon"] }
-itertools = "0.9.0"
 libc = "0.2.81"
-log = "0.4.11"
-lru = "0.6.1"
 miow = "0.2.2"
 net2 = "0.2.37"
 num-traits = "0.2"
+histogram = "0.6.9"
+itertools = "0.10.1"
+log = "0.4.14"
+lru = "0.6.6"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 rand_core = "0.6.2"
@@ -43,45 +44,48 @@ retain_mut = "0.1.2"
 serde = "1.0.122"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.12" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.12" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.12" }
-solana-client = { path = "../client", version = "=1.7.12" }
-solana-gossip = { path = "../gossip", version = "=1.7.12" }
-solana-ledger = { path = "../ledger", version = "=1.7.12" }
-solana-logger = { path = "../logger", version = "=1.7.12" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.7.12" }
-solana-metrics = { path = "../metrics", version = "=1.7.12" }
-solana-measure = { path = "../measure", version = "=1.7.12" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.12" }
-solana-perf = { path = "../perf", version = "=1.7.12" }
-solana-poh = { path = "../poh", version = "=1.7.12" }
-solana-program-test = { path = "../program-test", version = "=1.7.12" }
-solana-rpc = { path = "../rpc", version = "=1.7.12" }
-solana-runtime = { path = "../runtime", version = "=1.7.12" }
-solana-sdk = { path = "../sdk", version = "=1.7.12" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.12" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.12" }
-solana-streamer = { path = "../streamer", version = "=1.7.12" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.12" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.2" }
+solana-accountsdb-plugin-manager = { path = "../accountsdb-plugin-manager", version = "=1.8.2" }
+solana-banks-server = { path = "../banks-server", version = "=1.8.2" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.2" }
+solana-client = { path = "../client", version = "=1.8.2" }
+solana-gossip = { path = "../gossip", version = "=1.8.2" }
+solana-ledger = { path = "../ledger", version = "=1.8.2" }
+solana-logger = { path = "../logger", version = "=1.8.2" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.8.2" }
+solana-metrics = { path = "../metrics", version = "=1.8.2" }
+solana-measure = { path = "../measure", version = "=1.8.2" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.2" }
+solana-perf = { path = "../perf", version = "=1.8.2" }
+solana-poh = { path = "../poh", version = "=1.8.2" }
+solana-program-test = { path = "../program-test", version = "=1.8.2" }
+solana-rpc = { path = "../rpc", version = "=1.8.2" }
+solana-runtime = { path = "../runtime", version = "=1.8.2" }
+solana-sdk = { path = "../sdk", version = "=1.8.2" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.8.2" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.8.2" }
+solana-streamer = { path = "../streamer", version = "=1.8.2" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.2" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.2.0", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.12" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.8.2" }
 trees = "0.2.1"
 
 [dev-dependencies]
 jsonrpc-core = "18.0.0"
 jsonrpc-core-client = { version = "18.0.0", features = ["ipc", "ws"] }
+jsonrpc-derive = "18.0.0"
+jsonrpc-pubsub = "18.0.0"
 matches = "0.1.6"
 num_cpus = "1.13.0"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde_json = "1.0.56"
 serial_test = "0.4.0"
-solana-stake-program = { path = "../programs/stake", version = "=1.7.12" }
-solana-version = { path = "../version", version = "=1.7.12" }
+solana-stake-program = { path = "../programs/stake", version = "=1.8.2" }
+solana-version = { path = "../version", version = "=1.8.2" }
 symlink = "0.1.0"
 systemstat = "0.1.5"
 tokio = { version = "1", features = ["full"] }

core/benches/banking_stage.rs

@@ -18,6 +18,7 @@ use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_poh::poh_recorder::{create_test_recorder, WorkingBankEntry};
 use solana_runtime::bank::Bank;
+use solana_runtime::cost_model::CostModel;
 use solana_sdk::genesis_config::GenesisConfig;
 use solana_sdk::hash::Hash;
 use solana_sdk::message::Message;
@@ -33,7 +34,7 @@ use solana_streamer::socket::SocketAddrSpace;
 use std::collections::VecDeque;
 use std::sync::atomic::Ordering;
 use std::sync::mpsc::Receiver;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use std::time::{Duration, Instant};
 use test::Bencher;
 
@@ -92,6 +93,7 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
                 None::<Box<dyn Fn()>>,
                 &BankingStageStats::default(),
                 &recorder,
+                &Arc::new(RwLock::new(CostModel::default())),
             );
         });
 
@@ -108,7 +110,7 @@ fn make_accounts_txs(txes: usize, mint_keypair: &Keypair, hash: Hash) -> Vec<Tra
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
-            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            let sig: Vec<_> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
             new.message.account_keys[0] = pubkey::new_rand();
             new.message.account_keys[1] = pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
@@ -158,12 +160,18 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
     genesis_config.ticks_per_slot = 10_000;
 
     let (verified_sender, verified_receiver) = unbounded();
+    let (tpu_vote_sender, tpu_vote_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
     let mut bank = Bank::new(&genesis_config);
     // Allow arbitrary transaction processing time for the purposes of this bench
     bank.ns_per_slot = std::u128::MAX;
     let bank = Arc::new(Bank::new(&genesis_config));
 
+    // set cost tracker limits to MAX so it will not filter out TXs
+    bank.write_cost_tracker()
+        .unwrap()
+        .set_limits(std::u64::MAX, std::u64::MAX);
+
     debug!("threads: {} txs: {}", num_threads, txes);
 
     let transactions = match tx_type {
@@ -213,9 +221,11 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
             &cluster_info,
             &poh_recorder,
             verified_receiver,
+            tpu_vote_receiver,
             vote_receiver,
             None,
             s,
+            Arc::new(RwLock::new(CostModel::default())),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
@@ -259,6 +269,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
             start += chunk_len;
             start %= verified.len();
         });
+        drop(tpu_vote_sender);
         drop(vote_sender);
         exit.store(true, Ordering::Relaxed);
         poh_service.join().unwrap();

core/benches/cluster_info.rs

@@ -2,24 +2,37 @@
 
 extern crate test;
 
-use rand::{thread_rng, Rng};
-use solana_core::{
-    broadcast_stage::{broadcast_metrics::TransmitShredsStats, broadcast_shreds, BroadcastStage},
-    cluster_nodes::ClusterNodes,
+use {
+    rand::{thread_rng, Rng},
+    solana_core::{
+        broadcast_stage::{
+            broadcast_metrics::TransmitShredsStats, broadcast_shreds, BroadcastStage,
+        },
+        cluster_nodes::ClusterNodesCache,
+    },
+    solana_gossip::{
+        cluster_info::{ClusterInfo, Node},
+        contact_info::ContactInfo,
+    },
+    solana_ledger::{
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        shred::Shred,
+    },
+    solana_runtime::{bank::Bank, bank_forks::BankForks},
+    solana_sdk::{
+        pubkey,
+        signature::Keypair,
+        timing::{timestamp, AtomicInterval},
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    std::{
+        collections::HashMap,
+        net::UdpSocket,
+        sync::{Arc, RwLock},
+        time::Duration,
+    },
+    test::Bencher,
 };
-use solana_gossip::{
-    cluster_info::{ClusterInfo, Node},
-    contact_info::ContactInfo,
-};
-use solana_ledger::shred::Shred;
-use solana_sdk::{
-    pubkey,
-    signature::Keypair,
-    timing::{timestamp, AtomicInterval},
-};
-use solana_streamer::socket::SocketAddrSpace;
-use std::{collections::HashMap, net::UdpSocket, sync::Arc};
-use test::Bencher;
 
 #[bench]
 fn broadcast_shreds_bench(bencher: &mut Bencher) {
@@ -33,6 +46,10 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
     );
     let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
 
+    let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+    let bank = Bank::new(&genesis_config);
+    let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+
     const NUM_SHREDS: usize = 32;
     let shreds = vec![Shred::new_empty_data_shred(); NUM_SHREDS];
     let mut stakes = HashMap::new();
@@ -44,7 +61,10 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
         stakes.insert(id, thread_rng().gen_range(1, NUM_PEERS) as u64);
     }
     let cluster_info = Arc::new(cluster_info);
-    let cluster_nodes = ClusterNodes::<BroadcastStage>::new(&cluster_info, &stakes);
+    let cluster_nodes_cache = ClusterNodesCache::<BroadcastStage>::new(
+        8,                      // cap
+        Duration::from_secs(5), // ttl
+    );
     let shreds = Arc::new(shreds);
     let last_datapoint = Arc::new(AtomicInterval::default());
     bencher.iter(move || {
@@ -52,9 +72,11 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
         broadcast_shreds(
             &socket,
             &shreds,
-            &cluster_nodes,
+            &cluster_nodes_cache,
             &last_datapoint,
             &mut TransmitShredsStats::default(),
+            &cluster_info,
+            &bank_forks,
             &SocketAddrSpace::Unspecified,
         )
         .unwrap();

core/benches/retransmit_stage.rs

@@ -3,35 +3,49 @@
 extern crate solana_core;
 extern crate test;
 
-use log::*;
-use solana_core::retransmit_stage::retransmitter;
-use solana_gossip::cluster_info::{ClusterInfo, Node};
-use solana_gossip::contact_info::ContactInfo;
-use solana_ledger::entry::Entry;
-use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
-use solana_ledger::shred::Shredder;
-use solana_measure::measure::Measure;
-use solana_perf::packet::{Packet, Packets};
-use solana_rpc::max_slots::MaxSlots;
-use solana_runtime::bank::Bank;
-use solana_runtime::bank_forks::BankForks;
-use solana_sdk::hash::Hash;
-use solana_sdk::pubkey;
-use solana_sdk::signature::{Keypair, Signer};
-use solana_sdk::system_transaction;
-use solana_sdk::timing::timestamp;
-use solana_streamer::socket::SocketAddrSpace;
-use std::net::UdpSocket;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::mpsc::channel;
-use std::sync::Mutex;
-use std::sync::{Arc, RwLock};
-use std::thread::sleep;
-use std::thread::Builder;
-use std::time::Duration;
-use test::Bencher;
+use {
+    log::*,
+    solana_core::retransmit_stage::retransmitter,
+    solana_gossip::{
+        cluster_info::{ClusterInfo, Node},
+        contact_info::ContactInfo,
+    },
+    solana_ledger::{
+        entry::Entry,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        leader_schedule_cache::LeaderScheduleCache,
+        shred::Shredder,
+    },
+    solana_measure::measure::Measure,
+    solana_runtime::{bank::Bank, bank_forks::BankForks},
+    solana_sdk::{
+        hash::Hash,
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
+        system_transaction,
+        timing::timestamp,
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    std::{
+        net::UdpSocket,
+        sync::{
+            atomic::{AtomicUsize, Ordering},
+            mpsc::channel,
+            Arc, RwLock,
+        },
+        thread::{sleep, Builder},
+        time::Duration,
+    },
+    test::Bencher,
+};
 
+// TODO: The benchmark is ignored as it currently may indefinitely block.
+// The code incorrectly expects that the node receiving the shred on tvu socket
+// retransmits that to other nodes in its neighborhood. But that is no longer
+// the case since https://github.com/solana-labs/solana/pull/17716.
+// So depending on shred seed, peers may not receive packets and the receive
+// threads loop indefinitely.
+#[ignore]
 #[bench]
 #[allow(clippy::same_item_push)]
 fn bench_retransmitter(bencher: &mut Bencher) {
@@ -44,12 +58,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     const NUM_PEERS: usize = 4;
     let mut peer_sockets = Vec::new();
     for _ in 0..NUM_PEERS {
-        // This ensures that cluster_info.id() is the root of turbine
-        // retransmit tree and so the shreds are retransmited to all other
-        // nodes in the cluster.
-        let id = std::iter::repeat_with(pubkey::new_rand)
-            .find(|pk| cluster_info.id() < *pk)
-            .unwrap();
+        let id = Pubkey::new_unique();
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut contact_info = ContactInfo::new_localhost(&id, timestamp());
         contact_info.tvu = socket.local_addr().unwrap();
@@ -68,8 +77,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     let bank_forks = BankForks::new(bank0);
     let bank = bank_forks.working_bank();
     let bank_forks = Arc::new(RwLock::new(bank_forks));
-    let (packet_sender, packet_receiver) = channel();
-    let packet_receiver = Arc::new(Mutex::new(packet_receiver));
+    let (shreds_sender, shreds_receiver) = channel();
     const NUM_THREADS: usize = 2;
     let sockets = (0..NUM_THREADS)
         .map(|_| UdpSocket::bind("0.0.0.0:0").unwrap())
@@ -99,10 +107,10 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     let retransmitter_handles = retransmitter(
         Arc::new(sockets),
         bank_forks,
-        &leader_schedule_cache,
+        leader_schedule_cache,
         cluster_info,
-        packet_receiver,
-        &Arc::new(MaxSlots::default()),
+        shreds_receiver,
+        Arc::default(), // solana_rpc::max_slots::MaxSlots
         None,
     );
 
@@ -140,9 +148,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
             shred.set_index(index);
             index += 1;
             index %= 200;
-            let mut p = Packet::default();
-            shred.copy_to_packet(&mut p);
-            let _ = packet_sender.send(Packets::new(vec![p]));
+            let _ = shreds_sender.send(vec![shred.clone()]);
         }
         slot += 1;
 
@@ -158,7 +164,5 @@ fn bench_retransmitter(bencher: &mut Bencher) {
         total.store(0, Ordering::Relaxed);
     });
 
-    for t in retransmitter_handles {
-        t.join().unwrap();
-    }
+    retransmitter_handles.join().unwrap();
 }

core/benches/sigverify_stage.rs

@@ -18,6 +18,44 @@ use std::sync::mpsc::channel;
 use std::time::{Duration, Instant};
 use test::Bencher;
 
+#[bench]
+fn bench_packet_discard(bencher: &mut Bencher) {
+    solana_logger::setup();
+    let len = 30 * 1000;
+    let chunk_size = 1024;
+    let tx = test_tx();
+    let mut batches = to_packets_chunked(&vec![tx; len], chunk_size);
+
+    let mut total = 0;
+
+    let ips: Vec<_> = (0..10_000)
+        .into_iter()
+        .map(|_| {
+            let mut addr = [0u16; 8];
+            thread_rng().fill(&mut addr);
+            addr
+        })
+        .collect();
+
+    for batch in batches.iter_mut() {
+        total += batch.packets.len();
+        for p in batch.packets.iter_mut() {
+            let ip_index = thread_rng().gen_range(0, ips.len());
+            p.meta.addr = ips[ip_index];
+        }
+    }
+    info!("total packets: {}", total);
+
+    bencher.iter(move || {
+        SigVerifyStage::discard_excess_packets(&mut batches, 10_000);
+        for batch in batches.iter_mut() {
+            for p in batch.packets.iter_mut() {
+                p.meta.discard = false;
+            }
+        }
+    });
+}
+
 #[bench]
 fn bench_sigverify_stage(bencher: &mut Bencher) {
     solana_logger::setup();

core/src/broadcast_stage.rs

@@ -1,36 +1,46 @@
 //! A stage to broadcast data from a leader node to validators
 #![allow(clippy::rc_buffer)]
-use self::{
-    broadcast_duplicates_run::BroadcastDuplicatesRun,
-    broadcast_fake_shreds_run::BroadcastFakeShredsRun, broadcast_metrics::*,
-    fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
-    standard_broadcast_run::StandardBroadcastRun,
-};
-use crate::{
-    cluster_nodes::ClusterNodes,
-    result::{Error, Result},
-};
-use crossbeam_channel::{
-    Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
-    Sender as CrossbeamSender,
-};
-use solana_gossip::cluster_info::{ClusterInfo, ClusterInfoError};
-use solana_ledger::{blockstore::Blockstore, shred::Shred};
-use solana_measure::measure::Measure;
-use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
-use solana_poh::poh_recorder::WorkingBankEntry;
-use solana_runtime::bank::Bank;
-use solana_sdk::timing::{timestamp, AtomicInterval};
-use solana_sdk::{clock::Slot, pubkey::Pubkey};
-use solana_streamer::{sendmmsg::send_mmsg, socket::SocketAddrSpace};
-use std::{
-    collections::HashMap,
-    net::UdpSocket,
-    sync::atomic::{AtomicBool, Ordering},
-    sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
-    sync::{Arc, Mutex},
-    thread::{self, Builder, JoinHandle},
-    time::{Duration, Instant},
+use {
+    self::{
+        broadcast_duplicates_run::BroadcastDuplicatesRun,
+        broadcast_fake_shreds_run::BroadcastFakeShredsRun, broadcast_metrics::*,
+        fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
+        standard_broadcast_run::StandardBroadcastRun,
+    },
+    crate::{
+        cluster_nodes::{ClusterNodes, ClusterNodesCache},
+        result::{Error, Result},
+    },
+    crossbeam_channel::{
+        Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
+        Sender as CrossbeamSender,
+    },
+    itertools::Itertools,
+    solana_gossip::cluster_info::{ClusterInfo, ClusterInfoError},
+    solana_ledger::{blockstore::Blockstore, shred::Shred},
+    solana_measure::measure::Measure,
+    solana_metrics::{inc_new_counter_error, inc_new_counter_info},
+    solana_poh::poh_recorder::WorkingBankEntry,
+    solana_runtime::{bank::Bank, bank_forks::BankForks},
+    solana_sdk::{
+        timing::{timestamp, AtomicInterval},
+        {clock::Slot, pubkey::Pubkey},
+    },
+    solana_streamer::{
+        sendmmsg::{batch_send, SendPktsError},
+        socket::SocketAddrSpace,
+    },
+    std::{
+        collections::HashMap,
+        net::UdpSocket,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
+            Arc, Mutex, RwLock,
+        },
+        thread::{self, Builder, JoinHandle},
+        time::{Duration, Instant},
+    },
 };
 
 mod broadcast_duplicates_run;
@@ -40,11 +50,14 @@ pub(crate) mod broadcast_utils;
 mod fail_entry_verification_broadcast_run;
 mod standard_broadcast_run;
 
+const CLUSTER_NODES_CACHE_NUM_EPOCH_CAP: usize = 8;
+const CLUSTER_NODES_CACHE_TTL: Duration = Duration::from_secs(5);
+
 pub(crate) const NUM_INSERT_THREADS: usize = 2;
 pub(crate) type RetransmitSlotsSender = CrossbeamSender<HashMap<Slot, Arc<Bank>>>;
 pub(crate) type RetransmitSlotsReceiver = CrossbeamReceiver<HashMap<Slot, Arc<Bank>>>;
 pub(crate) type RecordReceiver = Receiver<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>;
-pub(crate) type TransmitReceiver = Receiver<(TransmitShreds, Option<BroadcastShredBatchInfo>)>;
+pub(crate) type TransmitReceiver = Receiver<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>;
 
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum BroadcastStageReturnType {
@@ -68,6 +81,7 @@ pub enum BroadcastStageType {
 }
 
 impl BroadcastStageType {
+    #[allow(clippy::too_many_arguments)]
     pub fn new_broadcast_stage(
         &self,
         sock: Vec<UdpSocket>,
@@ -76,6 +90,7 @@ impl BroadcastStageType {
         retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
+        bank_forks: &Arc<RwLock<BankForks>>,
         shred_version: u16,
     ) -> BroadcastStage {
         let keypair = cluster_info.keypair.clone();
@@ -87,6 +102,7 @@ impl BroadcastStageType {
                 retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
+                bank_forks,
                 StandardBroadcastRun::new(keypair, shred_version),
             ),
 
@@ -97,6 +113,7 @@ impl BroadcastStageType {
                 retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
+                bank_forks,
                 FailEntryVerificationBroadcastRun::new(keypair, shred_version),
             ),
 
@@ -107,6 +124,7 @@ impl BroadcastStageType {
                 retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
+                bank_forks,
                 BroadcastFakeShredsRun::new(keypair, 0, shred_version),
             ),
 
@@ -117,19 +135,19 @@ impl BroadcastStageType {
                 retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
+                bank_forks,
                 BroadcastDuplicatesRun::new(keypair, shred_version, config.clone()),
             ),
         }
     }
 }
 
-pub type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
 trait BroadcastRun {
     fn run(
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()>;
     fn transmit(
@@ -137,6 +155,7 @@ trait BroadcastRun {
         receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &ClusterInfo,
         sock: &UdpSocket,
+        bank_forks: &Arc<RwLock<BankForks>>,
     ) -> Result<()>;
     fn record(
         &mut self,
@@ -172,7 +191,7 @@ impl BroadcastStage {
     fn run(
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         mut broadcast_stage_run: impl BroadcastRun,
     ) -> BroadcastStageReturnType {
@@ -230,6 +249,7 @@ impl BroadcastStage {
         retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
+        bank_forks: &Arc<RwLock<BankForks>>,
         broadcast_stage_run: impl BroadcastRun + Send + 'static + Clone,
     ) -> Self {
         let btree = blockstore.clone();
@@ -258,10 +278,12 @@ impl BroadcastStage {
             let socket_receiver = socket_receiver.clone();
             let mut bs_transmit = broadcast_stage_run.clone();
             let cluster_info = cluster_info.clone();
+            let bank_forks = bank_forks.clone();
             let t = Builder::new()
                 .name("solana-broadcaster-transmit".to_string())
                 .spawn(move || loop {
-                    let res = bs_transmit.transmit(&socket_receiver, &cluster_info, &sock);
+                    let res =
+                        bs_transmit.transmit(&socket_receiver, &cluster_info, &sock, &bank_forks);
                     let res = Self::handle_error(res, "solana-broadcaster-transmit");
                     if let Some(res) = res {
                         return res;
@@ -312,7 +334,7 @@ impl BroadcastStage {
     fn check_retransmit_signals(
         blockstore: &Blockstore,
         retransmit_slots_receiver: &RetransmitSlotsReceiver,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         let timer = Duration::from_millis(100);
 
@@ -323,27 +345,26 @@ impl BroadcastStage {
         }
 
         for (_, bank) in retransmit_slots.iter() {
-            let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-            let stakes = bank.epoch_staked_nodes(bank_epoch);
-            let stakes = stakes.map(Arc::new);
+            let slot = bank.slot();
             let data_shreds = Arc::new(
                 blockstore
-                    .get_data_shreds_for_slot(bank.slot(), 0)
+                    .get_data_shreds_for_slot(slot, 0)
                     .expect("My own shreds must be reconstructable"),
             );
-
+            debug_assert!(data_shreds.iter().all(|shred| shred.slot() == slot));
             if !data_shreds.is_empty() {
-                socket_sender.send(((stakes.clone(), data_shreds), None))?;
+                socket_sender.send((data_shreds, None))?;
             }
 
             let coding_shreds = Arc::new(
                 blockstore
-                    .get_coding_shreds_for_slot(bank.slot(), 0)
+                    .get_coding_shreds_for_slot(slot, 0)
                     .expect("My own shreds must be reconstructable"),
             );
 
+            debug_assert!(coding_shreds.iter().all(|shred| shred.slot() == slot));
             if !coding_shreds.is_empty() {
-                socket_sender.send(((stakes.clone(), coding_shreds), None))?;
+                socket_sender.send((coding_shreds, None))?;
             }
         }
 
@@ -359,11 +380,13 @@ impl BroadcastStage {
 }
 
 fn update_peer_stats(
-    num_live_peers: i64,
-    broadcast_len: i64,
+    cluster_nodes: &ClusterNodes<BroadcastStage>,
     last_datapoint_submit: &Arc<AtomicInterval>,
 ) {
     if last_datapoint_submit.should_update(1000) {
+        let now = timestamp();
+        let num_live_peers = cluster_nodes.num_peers_live(now);
+        let broadcast_len = cluster_nodes.num_peers() + 1;
         datapoint_info!(
             "cluster_info-num_nodes",
             ("live_count", num_live_peers, i64),
@@ -377,51 +400,51 @@ fn update_peer_stats(
 pub fn broadcast_shreds(
     s: &UdpSocket,
     shreds: &[Shred],
-    cluster_nodes: &ClusterNodes<BroadcastStage>,
+    cluster_nodes_cache: &ClusterNodesCache<BroadcastStage>,
     last_datapoint_submit: &Arc<AtomicInterval>,
     transmit_stats: &mut TransmitShredsStats,
+    cluster_info: &ClusterInfo,
+    bank_forks: &Arc<RwLock<BankForks>>,
     socket_addr_space: &SocketAddrSpace,
 ) -> Result<()> {
-    let broadcast_len = cluster_nodes.num_peers();
-    if broadcast_len == 0 {
-        update_peer_stats(1, 1, last_datapoint_submit);
-        return Ok(());
-    }
+    let mut result = Ok(());
     let mut shred_select = Measure::start("shred_select");
+    // Only the leader broadcasts shreds.
+    let leader = cluster_info.id();
+    let (root_bank, working_bank) = {
+        let bank_forks = bank_forks.read().unwrap();
+        (bank_forks.root_bank(), bank_forks.working_bank())
+    };
     let packets: Vec<_> = shreds
         .iter()
-        .filter_map(|shred| {
-            let node = cluster_nodes.get_broadcast_peer(shred.seed())?;
-            if socket_addr_space.check(&node.tvu) {
-                Some((&shred.payload, &node.tvu))
-            } else {
-                None
-            }
+        .group_by(|shred| shred.slot())
+        .into_iter()
+        .flat_map(|(slot, shreds)| {
+            let cluster_nodes =
+                cluster_nodes_cache.get(slot, &root_bank, &working_bank, cluster_info);
+            update_peer_stats(&cluster_nodes, last_datapoint_submit);
+            let root_bank = root_bank.clone();
+            shreds.filter_map(move |shred| {
+                let seed = shred.seed(leader, &root_bank);
+                let node = cluster_nodes.get_broadcast_peer(seed)?;
+                socket_addr_space
+                    .check(&node.tvu)
+                    .then(|| (&shred.payload, node.tvu))
+            })
         })
         .collect();
     shred_select.stop();
     transmit_stats.shred_select += shred_select.as_us();
 
-    let mut sent = 0;
     let mut send_mmsg_time = Measure::start("send_mmsg");
-    while sent < packets.len() {
-        match send_mmsg(s, &packets[sent..]) {
-            Ok(n) => sent += n,
-            Err(e) => {
-                return Err(Error::Io(e));
-            }
-        }
+    if let Err(SendPktsError::IoError(ioerr, num_failed)) = batch_send(s, &packets[..]) {
+        transmit_stats.dropped_packets += num_failed;
+        result = Err(Error::Io(ioerr));
     }
     send_mmsg_time.stop();
     transmit_stats.send_mmsg_elapsed += send_mmsg_time.as_us();
-
-    let num_live_peers = cluster_nodes.num_peers_live(timestamp()) as i64;
-    update_peer_stats(
-        num_live_peers,
-        broadcast_len as i64 + 1,
-        last_datapoint_submit,
-    );
-    Ok(())
+    transmit_stats.total_packets += packets.len();
+    result
 }
 
 #[cfg(test)]
@@ -447,15 +470,15 @@ pub mod test {
     };
 
     #[allow(clippy::implicit_hasher)]
-    pub fn make_transmit_shreds(
+    #[allow(clippy::type_complexity)]
+    fn make_transmit_shreds(
         slot: Slot,
         num: u64,
-        stakes: Option<Arc<HashMap<Pubkey, u64>>>,
     ) -> (
         Vec<Shred>,
         Vec<Shred>,
-        Vec<TransmitShreds>,
-        Vec<TransmitShreds>,
+        Vec<Arc<Vec<Shred>>>,
+        Vec<Arc<Vec<Shred>>>,
     ) {
         let num_entries = max_ticks_per_n_shreds(num, None);
         let (data_shreds, _) = make_slot_entries(slot, 0, num_entries);
@@ -472,11 +495,11 @@ pub mod test {
             coding_shreds.clone(),
             data_shreds
                 .into_iter()
-                .map(|s| (stakes.clone(), Arc::new(vec![s])))
+                .map(|shred| Arc::new(vec![shred]))
                 .collect(),
             coding_shreds
                 .into_iter()
-                .map(|s| (stakes.clone(), Arc::new(vec![s])))
+                .map(|shred| Arc::new(vec![shred]))
                 .collect(),
         )
     }
@@ -488,15 +511,15 @@ pub mod test {
         num_expected_data_shreds: u64,
         num_expected_coding_shreds: u64,
     ) {
-        while let Ok((new_retransmit_slots, _)) = transmit_receiver.try_recv() {
-            if new_retransmit_slots.1[0].is_data() {
-                for data_shred in new_retransmit_slots.1.iter() {
+        while let Ok((shreds, _)) = transmit_receiver.try_recv() {
+            if shreds[0].is_data() {
+                for data_shred in shreds.iter() {
                     assert_eq!(data_shred.index() as u64, data_index);
                     data_index += 1;
                 }
             } else {
-                assert_eq!(new_retransmit_slots.1[0].index() as u64, coding_index);
-                for coding_shred in new_retransmit_slots.1.iter() {
+                assert_eq!(shreds[0].index() as u64, coding_index);
+                for coding_shred in shreds.iter() {
                     assert_eq!(coding_shred.index() as u64, coding_index);
                     coding_index += 1;
                 }
@@ -520,7 +543,7 @@ pub mod test {
         // Make some shreds
         let updated_slot = 0;
         let (all_data_shreds, all_coding_shreds, _, _all_coding_transmit_shreds) =
-            make_transmit_shreds(updated_slot, 10, None);
+            make_transmit_shreds(updated_slot, 10);
         let num_data_shreds = all_data_shreds.len();
         let num_coding_shreds = all_coding_shreds.len();
         assert!(num_data_shreds >= 10);
@@ -591,7 +614,9 @@ pub mod test {
         let exit_sender = Arc::new(AtomicBool::new(false));
 
         let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
-        let bank = Arc::new(Bank::new(&genesis_config));
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let bank = bank_forks.read().unwrap().root_bank();
 
         let leader_keypair = cluster_info.keypair.clone();
         // Start up the broadcast stage
@@ -602,6 +627,7 @@ pub mod test {
             retransmit_slots_receiver,
             &exit_sender,
             &blockstore,
+            &bank_forks,
             StandardBroadcastRun::new(leader_keypair, 0),
         );
 

core/src/broadcast_stage/broadcast_duplicates_run.rs

@@ -1,16 +1,19 @@
-use super::broadcast_utils::ReceiveResults;
-use super::*;
-use log::*;
-use solana_ledger::entry::{create_ticks, Entry, EntrySlice};
-use solana_ledger::shred::Shredder;
-use solana_runtime::blockhash_queue::BlockhashQueue;
-use solana_sdk::clock::Slot;
-use solana_sdk::fee_calculator::FeeCalculator;
-use solana_sdk::hash::Hash;
-use solana_sdk::signature::{Keypair, Signer};
-use solana_sdk::transaction::Transaction;
-use std::collections::VecDeque;
-use std::sync::Mutex;
+use {
+    super::{broadcast_utils::ReceiveResults, *},
+    crate::cluster_nodes::ClusterNodesCache,
+    solana_ledger::{
+        entry::{create_ticks, Entry, EntrySlice},
+        shred::Shredder,
+    },
+    solana_runtime::blockhash_queue::BlockhashQueue,
+    solana_sdk::{
+        fee_calculator::FeeCalculator,
+        hash::Hash,
+        signature::{Keypair, Signer},
+        transaction::Transaction,
+    },
+    std::collections::VecDeque,
+};
 
 // Queue which facilitates delivering shreds with a delay
 type DelayedQueue = VecDeque<(Option<Pubkey>, Option<Vec<Shred>>)>;
@@ -29,6 +32,7 @@ pub(super) struct BroadcastDuplicatesRun {
     next_shred_index: u32,
     shred_version: u16,
     keypair: Arc<Keypair>,
+    cluster_nodes_cache: Arc<ClusterNodesCache<BroadcastStage>>,
 }
 
 impl BroadcastDuplicatesRun {
@@ -37,6 +41,10 @@ impl BroadcastDuplicatesRun {
         shred_version: u16,
         config: BroadcastDuplicatesConfig,
     ) -> Self {
+        let cluster_nodes_cache = Arc::new(ClusterNodesCache::<BroadcastStage>::new(
+            CLUSTER_NODES_CACHE_NUM_EPOCH_CAP,
+            CLUSTER_NODES_CACHE_TTL,
+        ));
         let mut delayed_queue = DelayedQueue::new();
         delayed_queue.resize(config.duplicate_send_delay, (None, None));
         Self {
@@ -49,6 +57,7 @@ impl BroadcastDuplicatesRun {
             last_duplicate_entry_hash: Hash::default(),
             shred_version,
             keypair,
+            cluster_nodes_cache,
         }
     }
 
@@ -141,7 +150,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         // 1) Pull entries from banking stage
@@ -257,29 +266,17 @@ impl BroadcastRun for BroadcastDuplicatesRun {
             }
         }
 
-        let duplicate_recipients = Arc::new(duplicate_recipients);
-        let real_recipients = Arc::new(real_recipients);
+        let _duplicate_recipients = Arc::new(duplicate_recipients);
+        let _real_recipients = Arc::new(real_recipients);
 
         let data_shreds = Arc::new(data_shreds);
         blockstore_sender.send((data_shreds.clone(), None))?;
 
         // 3) Start broadcast step
-        socket_sender.send((
-            (
-                Some(duplicate_recipients.clone()),
-                Arc::new(duplicate_data_shreds),
-            ),
-            None,
-        ))?;
-        socket_sender.send((
-            (
-                Some(duplicate_recipients),
-                Arc::new(duplicate_coding_shreds),
-            ),
-            None,
-        ))?;
-        socket_sender.send(((Some(real_recipients.clone()), data_shreds), None))?;
-        socket_sender.send(((Some(real_recipients), Arc::new(coding_shreds)), None))?;
+        socket_sender.send((Arc::new(duplicate_data_shreds), None))?;
+        socket_sender.send((Arc::new(duplicate_coding_shreds), None))?;
+        socket_sender.send((data_shreds, None))?;
+        socket_sender.send((Arc::new(coding_shreds), None))?;
 
         Ok(())
     }
@@ -288,6 +285,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
         receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &ClusterInfo,
         sock: &UdpSocket,
+        bank_forks: &Arc<RwLock<BankForks>>,
     ) -> Result<()> {
         // Check the delay queue for shreds that are ready to be sent
         let (delayed_recipient, delayed_shreds) = {
@@ -299,8 +297,15 @@ impl BroadcastRun for BroadcastDuplicatesRun {
             }
         };
 
-        let ((stakes, shreds), _) = receiver.lock().unwrap().recv()?;
-        let stakes = stakes.unwrap();
+        let (shreds, _) = receiver.lock().unwrap().recv()?;
+        if shreds.is_empty() {
+            return Ok(());
+        }
+        let slot = shreds.first().unwrap().slot();
+        assert!(shreds.iter().all(|shred| shred.slot() == slot));
+        let root_bank = bank_forks.read().unwrap().root_bank();
+        let epoch = root_bank.get_leader_schedule_epoch(slot);
+        let stakes = root_bank.epoch_staked_nodes(epoch).unwrap_or_default();
         let socket_addr_space = cluster_info.socket_addr_space();
         for peer in cluster_info.tvu_peers() {
             // Forward shreds to circumvent gossip

core/src/broadcast_stage/broadcast_fake_shreds_run.rs

@@ -28,7 +28,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         // 1) Pull entries from banking stage
@@ -84,19 +84,23 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         let data_shreds = Arc::new(data_shreds);
         blockstore_sender.send((data_shreds.clone(), None))?;
 
+        let slot = bank.slot();
+        let batch_info = BroadcastShredBatchInfo {
+            slot,
+            num_expected_batches: None,
+            slot_start_ts: Instant::now(),
+            was_interrupted: false,
+        };
         // 3) Start broadcast step
         //some indicates fake shreds
-        socket_sender.send((
-            (Some(Arc::new(HashMap::new())), Arc::new(fake_data_shreds)),
-            None,
-        ))?;
-        socket_sender.send((
-            (Some(Arc::new(HashMap::new())), Arc::new(fake_coding_shreds)),
-            None,
-        ))?;
+        let batch_info = Some(batch_info);
+        assert!(fake_data_shreds.iter().all(|shred| shred.slot() == slot));
+        assert!(fake_coding_shreds.iter().all(|shred| shred.slot() == slot));
+        socket_sender.send((Arc::new(fake_data_shreds), batch_info.clone()))?;
+        socket_sender.send((Arc::new(fake_coding_shreds), batch_info))?;
         //none indicates real shreds
-        socket_sender.send(((None, data_shreds), None))?;
-        socket_sender.send(((None, Arc::new(coding_shreds)), None))?;
+        socket_sender.send((data_shreds, None))?;
+        socket_sender.send((Arc::new(coding_shreds), None))?;
 
         Ok(())
     }
@@ -105,19 +109,17 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &ClusterInfo,
         sock: &UdpSocket,
+        _bank_forks: &Arc<RwLock<BankForks>>,
     ) -> Result<()> {
-        for ((stakes, data_shreds), _) in receiver.lock().unwrap().iter() {
+        for (data_shreds, batch_info) in receiver.lock().unwrap().iter() {
+            let fake = batch_info.is_some();
             let peers = cluster_info.tvu_peers();
             peers.iter().enumerate().for_each(|(i, peer)| {
-                if i <= self.partition && stakes.is_some() {
+                if fake == (i <= self.partition) {
                     // Send fake shreds to the first N peers
                     data_shreds.iter().for_each(|b| {
                         sock.send_to(&b.payload, &peer.tvu_forwards).unwrap();
                     });
-                } else if i > self.partition && stakes.is_none() {
-                    data_shreds.iter().for_each(|b| {
-                        sock.send_to(&b.payload, &peer.tvu_forwards).unwrap();
-                    });
                 }
             });
         }

core/src/broadcast_stage/broadcast_metrics.rs

@@ -2,7 +2,7 @@ use super::*;
 
 pub(crate) trait BroadcastStats {
     fn update(&mut self, new_stats: &Self);
-    fn report_stats(&mut self, slot: Slot, slot_start: Instant);
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant, was_interrupted: bool);
 }
 
 #[derive(Clone)]
@@ -10,6 +10,7 @@ pub(crate) struct BroadcastShredBatchInfo {
     pub(crate) slot: Slot,
     pub(crate) num_expected_batches: Option<usize>,
     pub(crate) slot_start_ts: Instant,
+    pub(crate) was_interrupted: bool,
 }
 
 #[derive(Default, Clone)]
@@ -19,6 +20,8 @@ pub struct TransmitShredsStats {
     pub get_peers_elapsed: u64,
     pub shred_select: u64,
     pub num_shreds: usize,
+    pub total_packets: usize,
+    pub dropped_packets: usize,
 }
 
 impl BroadcastStats for TransmitShredsStats {
@@ -28,24 +31,42 @@ impl BroadcastStats for TransmitShredsStats {
         self.get_peers_elapsed += new_stats.get_peers_elapsed;
         self.num_shreds += new_stats.num_shreds;
         self.shred_select += new_stats.shred_select;
+        self.total_packets += new_stats.total_packets;
+        self.dropped_packets += new_stats.dropped_packets;
     }
-    fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
-        datapoint_info!(
-            "broadcast-transmit-shreds-stats",
-            ("slot", slot as i64, i64),
-            (
-                "end_to_end_elapsed",
-                // `slot_start` signals when the first batch of shreds was
-                // received, used to measure duration of broadcast
-                slot_start.elapsed().as_micros() as i64,
-                i64
-            ),
-            ("transmit_elapsed", self.transmit_elapsed as i64, i64),
-            ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
-            ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
-            ("num_shreds", self.num_shreds as i64, i64),
-            ("shred_select", self.shred_select as i64, i64),
-        );
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant, was_interrupted: bool) {
+        if was_interrupted {
+            datapoint_info!(
+                "broadcast-transmit-shreds-interrupted-stats",
+                ("slot", slot as i64, i64),
+                ("transmit_elapsed", self.transmit_elapsed as i64, i64),
+                ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
+                ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
+                ("num_shreds", self.num_shreds as i64, i64),
+                ("shred_select", self.shred_select as i64, i64),
+                ("total_packets", self.total_packets as i64, i64),
+                ("dropped_packets", self.dropped_packets as i64, i64),
+            );
+        } else {
+            datapoint_info!(
+                "broadcast-transmit-shreds-stats",
+                ("slot", slot as i64, i64),
+                (
+                    "end_to_end_elapsed",
+                    // `slot_start` signals when the first batch of shreds was
+                    // received, used to measure duration of broadcast
+                    slot_start.elapsed().as_micros() as i64,
+                    i64
+                ),
+                ("transmit_elapsed", self.transmit_elapsed as i64, i64),
+                ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
+                ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
+                ("num_shreds", self.num_shreds as i64, i64),
+                ("shred_select", self.shred_select as i64, i64),
+                ("total_packets", self.total_packets as i64, i64),
+                ("dropped_packets", self.dropped_packets as i64, i64),
+            );
+        }
     }
 }
 
@@ -59,24 +80,37 @@ impl BroadcastStats for InsertShredsStats {
         self.insert_shreds_elapsed += new_stats.insert_shreds_elapsed;
         self.num_shreds += new_stats.num_shreds;
     }
-    fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
-        datapoint_info!(
-            "broadcast-insert-shreds-stats",
-            ("slot", slot as i64, i64),
-            (
-                "end_to_end_elapsed",
-                // `slot_start` signals when the first batch of shreds was
-                // received, used to measure duration of broadcast
-                slot_start.elapsed().as_micros() as i64,
-                i64
-            ),
-            (
-                "insert_shreds_elapsed",
-                self.insert_shreds_elapsed as i64,
-                i64
-            ),
-            ("num_shreds", self.num_shreds as i64, i64),
-        );
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant, was_interrupted: bool) {
+        if was_interrupted {
+            datapoint_info!(
+                "broadcast-insert-shreds-interrupted-stats",
+                ("slot", slot as i64, i64),
+                (
+                    "insert_shreds_elapsed",
+                    self.insert_shreds_elapsed as i64,
+                    i64
+                ),
+                ("num_shreds", self.num_shreds as i64, i64),
+            );
+        } else {
+            datapoint_info!(
+                "broadcast-insert-shreds-stats",
+                ("slot", slot as i64, i64),
+                (
+                    "end_to_end_elapsed",
+                    // `slot_start` signals when the first batch of shreds was
+                    // received, used to measure duration of broadcast
+                    slot_start.elapsed().as_micros() as i64,
+                    i64
+                ),
+                (
+                    "insert_shreds_elapsed",
+                    self.insert_shreds_elapsed as i64,
+                    i64
+                ),
+                ("num_shreds", self.num_shreds as i64, i64),
+            );
+        }
     }
 }
 
@@ -122,9 +156,11 @@ impl<T: BroadcastStats + Default> SlotBroadcastStats<T> {
                 }
                 if let Some(num_expected_batches) = slot_batch_counter.num_expected_batches {
                     if slot_batch_counter.num_batches == num_expected_batches {
-                        slot_batch_counter
-                            .broadcast_shred_stats
-                            .report_stats(batch_info.slot, batch_info.slot_start_ts);
+                        slot_batch_counter.broadcast_shred_stats.report_stats(
+                            batch_info.slot,
+                            batch_info.slot_start_ts,
+                            batch_info.was_interrupted,
+                        );
                         should_delete = true;
                     }
                 }
@@ -153,7 +189,7 @@ mod test {
             self.count += new_stats.count;
             self.sender = new_stats.sender.clone();
         }
-        fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
+        fn report_stats(&mut self, slot: Slot, slot_start: Instant, _was_interrupted: bool) {
             self.sender
                 .as_ref()
                 .unwrap()
@@ -173,11 +209,14 @@ mod test {
                 send_mmsg_elapsed: 3,
                 shred_select: 4,
                 num_shreds: 5,
+                total_packets: 6,
+                dropped_packets: 7,
             },
             &Some(BroadcastShredBatchInfo {
                 slot: 0,
                 num_expected_batches: Some(2),
                 slot_start_ts: start,
+                was_interrupted: false,
             }),
         );
 
@@ -190,14 +229,18 @@ mod test {
         assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
         assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
         assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.total_packets, 6);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.dropped_packets, 7);
 
         slot_broadcast_stats.update(
             &TransmitShredsStats {
-                transmit_elapsed: 7,
-                get_peers_elapsed: 8,
-                send_mmsg_elapsed: 9,
-                shred_select: 10,
-                num_shreds: 11,
+                transmit_elapsed: 11,
+                get_peers_elapsed: 12,
+                send_mmsg_elapsed: 13,
+                shred_select: 14,
+                num_shreds: 15,
+                total_packets: 16,
+                dropped_packets: 17,
             },
             &None,
         );
@@ -211,6 +254,8 @@ mod test {
         assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
         assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
         assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.total_packets, 6);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.dropped_packets, 7);
 
         // If another batch is given, then total number of batches == num_expected_batches == 2,
         // so the batch should be purged from the HashMap
@@ -221,11 +266,14 @@ mod test {
                 send_mmsg_elapsed: 1,
                 shred_select: 1,
                 num_shreds: 1,
+                total_packets: 1,
+                dropped_packets: 1,
             },
             &Some(BroadcastShredBatchInfo {
                 slot: 0,
                 num_expected_batches: None,
                 slot_start_ts: start,
+                was_interrupted: false,
             }),
         );
 
@@ -249,6 +297,7 @@ mod test {
                         slot,
                         num_expected_batches: None,
                         slot_start_ts: start,
+                        was_interrupted: false,
                     };
                     if i == round % num_threads {
                         broadcast_batch_info.num_expected_batches = Some(num_threads);

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -1,9 +1,10 @@
-use super::*;
-use crate::cluster_nodes::ClusterNodes;
-use solana_ledger::shred::Shredder;
-use solana_sdk::hash::Hash;
-use solana_sdk::signature::Keypair;
-use std::{thread::sleep, time::Duration};
+use {
+    super::*,
+    crate::cluster_nodes::ClusterNodesCache,
+    solana_ledger::shred::Shredder,
+    solana_sdk::{hash::Hash, signature::Keypair},
+    std::{thread::sleep, time::Duration},
+};
 
 pub const NUM_BAD_SLOTS: u64 = 10;
 pub const SLOT_TO_RESOLVE: u64 = 32;
@@ -15,16 +16,22 @@ pub(super) struct FailEntryVerificationBroadcastRun {
     good_shreds: Vec<Shred>,
     current_slot: Slot,
     next_shred_index: u32,
+    cluster_nodes_cache: Arc<ClusterNodesCache<BroadcastStage>>,
 }
 
 impl FailEntryVerificationBroadcastRun {
     pub(super) fn new(keypair: Arc<Keypair>, shred_version: u16) -> Self {
+        let cluster_nodes_cache = Arc::new(ClusterNodesCache::<BroadcastStage>::new(
+            CLUSTER_NODES_CACHE_NUM_EPOCH_CAP,
+            CLUSTER_NODES_CACHE_TTL,
+        ));
         Self {
             shred_version,
             keypair,
             good_shreds: vec![],
             current_slot: 0,
             next_shred_index: 0,
+            cluster_nodes_cache,
         }
     }
 }
@@ -34,7 +41,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         // 1) Pull entries from banking stage
@@ -101,10 +108,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         let data_shreds = Arc::new(data_shreds);
         blockstore_sender.send((data_shreds.clone(), None))?;
         // 4) Start broadcast step
-        let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-        let stakes = bank.epoch_staked_nodes(bank_epoch);
-        let stakes = stakes.map(Arc::new);
-        socket_sender.send(((stakes.clone(), data_shreds), None))?;
+        socket_sender.send((data_shreds, None))?;
         if let Some((good_last_data_shred, bad_last_data_shred)) = last_shreds {
             // Stash away the good shred so we can rewrite them later
             self.good_shreds.extend(good_last_data_shred.clone());
@@ -123,7 +127,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
             // Store the bad shred so we serve bad repairs to validators catching up
             blockstore_sender.send((bad_last_data_shred.clone(), None))?;
             // Send bad shreds to rest of network
-            socket_sender.send(((stakes, bad_last_data_shred), None))?;
+            socket_sender.send((bad_last_data_shred, None))?;
         }
         Ok(())
     }
@@ -132,23 +136,19 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &ClusterInfo,
         sock: &UdpSocket,
+        bank_forks: &Arc<RwLock<BankForks>>,
     ) -> Result<()> {
-        let ((stakes, shreds), _) = receiver.lock().unwrap().recv()?;
-        // Broadcast data
-        let cluster_nodes = ClusterNodes::<BroadcastStage>::new(
-            cluster_info,
-            stakes.as_deref().unwrap_or(&HashMap::default()),
-        );
+        let (shreds, _) = receiver.lock().unwrap().recv()?;
         broadcast_shreds(
             sock,
             &shreds,
-            &cluster_nodes,
+            &self.cluster_nodes_cache,
             &Arc::new(AtomicInterval::default()),
             &mut TransmitShredsStats::default(),
+            cluster_info,
+            bank_forks,
             cluster_info.socket_addr_space(),
-        )?;
-
-        Ok(())
+        )
     }
     fn record(
         &mut self,

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -1,23 +1,26 @@
 #![allow(clippy::rc_buffer)]
 
-use super::{
-    broadcast_utils::{self, ReceiveResults},
-    *,
-};
-use crate::{broadcast_stage::broadcast_utils::UnfinishedSlotInfo, cluster_nodes::ClusterNodes};
-use solana_ledger::{
-    entry::Entry,
-    shred::{
-        ProcessShredsStats, Shred, Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK,
-        SHRED_TICK_REFERENCE_MASK,
+use {
+    super::{
+        broadcast_utils::{self, ReceiveResults},
+        *,
     },
+    crate::{
+        broadcast_stage::broadcast_utils::UnfinishedSlotInfo, cluster_nodes::ClusterNodesCache,
+    },
+    solana_ledger::{
+        entry::Entry,
+        shred::{
+            ProcessShredsStats, Shred, Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK,
+            SHRED_TICK_REFERENCE_MASK,
+        },
+    },
+    solana_sdk::{
+        signature::Keypair,
+        timing::{duration_as_us, AtomicInterval},
+    },
+    std::{ops::Deref, sync::RwLock, time::Duration},
 };
-use solana_sdk::{
-    pubkey::Pubkey,
-    signature::Keypair,
-    timing::{duration_as_us, AtomicInterval},
-};
-use std::{collections::HashMap, ops::Deref, sync::RwLock, time::Duration};
 
 #[derive(Clone)]
 pub struct StandardBroadcastRun {
@@ -31,12 +34,16 @@ pub struct StandardBroadcastRun {
     shred_version: u16,
     last_datapoint_submit: Arc<AtomicInterval>,
     num_batches: usize,
-    cluster_nodes: Arc<RwLock<ClusterNodes<BroadcastStage>>>,
+    cluster_nodes_cache: Arc<ClusterNodesCache<BroadcastStage>>,
     last_peer_update: Arc<AtomicInterval>,
 }
 
 impl StandardBroadcastRun {
     pub(super) fn new(keypair: Arc<Keypair>, shred_version: u16) -> Self {
+        let cluster_nodes_cache = Arc::new(ClusterNodesCache::<BroadcastStage>::new(
+            CLUSTER_NODES_CACHE_NUM_EPOCH_CAP,
+            CLUSTER_NODES_CACHE_TTL,
+        ));
         Self {
             process_shreds_stats: ProcessShredsStats::default(),
             transmit_shreds_stats: Arc::default(),
@@ -48,7 +55,7 @@ impl StandardBroadcastRun {
             shred_version,
             last_datapoint_submit: Arc::default(),
             num_batches: 0,
-            cluster_nodes: Arc::default(),
+            cluster_nodes_cache,
             last_peer_update: Arc::new(AtomicInterval::default()),
         }
     }
@@ -90,7 +97,7 @@ impl StandardBroadcastRun {
                     stats,
                 );
                 shreds.insert(0, shred);
-                self.report_and_reset_stats();
+                self.report_and_reset_stats(true);
                 self.unfinished_slot = None;
                 shreds
             }
@@ -156,17 +163,19 @@ impl StandardBroadcastRun {
         sock: &UdpSocket,
         blockstore: &Arc<Blockstore>,
         receive_results: ReceiveResults,
+        bank_forks: &Arc<RwLock<BankForks>>,
     ) -> Result<()> {
         let (bsend, brecv) = channel();
         let (ssend, srecv) = channel();
         self.process_receive_results(blockstore, &ssend, &bsend, receive_results)?;
         let srecv = Arc::new(Mutex::new(srecv));
         let brecv = Arc::new(Mutex::new(brecv));
+
         //data
-        let _ = self.transmit(&srecv, cluster_info, sock);
+        let _ = self.transmit(&srecv, cluster_info, sock, bank_forks);
         let _ = self.record(&brecv, blockstore);
         //coding
-        let _ = self.transmit(&srecv, cluster_info, sock);
+        let _ = self.transmit(&srecv, cluster_info, sock, bank_forks);
         let _ = self.record(&brecv, blockstore);
         Ok(())
     }
@@ -174,7 +183,7 @@ impl StandardBroadcastRun {
     fn process_receive_results(
         &mut self,
         blockstore: &Arc<Blockstore>,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         receive_results: ReceiveResults,
     ) -> Result<()> {
@@ -226,21 +235,21 @@ impl StandardBroadcastRun {
         to_shreds_time.stop();
 
         let mut get_leader_schedule_time = Measure::start("broadcast_get_leader_schedule");
-        let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-        let stakes = bank.epoch_staked_nodes(bank_epoch).map(Arc::new);
-
         // Broadcast the last shred of the interrupted slot if necessary
         if !prev_slot_shreds.is_empty() {
+            let slot = prev_slot_shreds[0].slot();
             let batch_info = Some(BroadcastShredBatchInfo {
-                slot: prev_slot_shreds[0].slot(),
+                slot,
                 num_expected_batches: Some(old_num_batches + 1),
                 slot_start_ts: old_broadcast_start.expect(
                     "Old broadcast start time for previous slot must exist if the previous slot
                  was interrupted",
                 ),
+                was_interrupted: true,
             });
             let shreds = Arc::new(prev_slot_shreds);
-            socket_sender.send(((stakes.clone(), shreds.clone()), batch_info.clone()))?;
+            debug_assert!(shreds.iter().all(|shred| shred.slot() == slot));
+            socket_sender.send((shreds.clone(), batch_info.clone()))?;
             blockstore_sender.send((shreds, batch_info))?;
         }
 
@@ -259,6 +268,7 @@ impl StandardBroadcastRun {
             slot_start_ts: self
                 .slot_broadcast_start
                 .expect("Start timestamp must exist for a slot if we're broadcasting the slot"),
+            was_interrupted: false,
         });
         get_leader_schedule_time.stop();
 
@@ -266,7 +276,8 @@ impl StandardBroadcastRun {
 
         // Send data shreds
         let data_shreds = Arc::new(data_shreds);
-        socket_sender.send(((stakes.clone(), data_shreds.clone()), batch_info.clone()))?;
+        debug_assert!(data_shreds.iter().all(|shred| shred.slot() == bank.slot()));
+        socket_sender.send((data_shreds.clone(), batch_info.clone()))?;
         blockstore_sender.send((data_shreds, batch_info.clone()))?;
 
         // Create and send coding shreds
@@ -277,7 +288,10 @@ impl StandardBroadcastRun {
             &mut process_stats,
         );
         let coding_shreds = Arc::new(coding_shreds);
-        socket_sender.send(((stakes, coding_shreds.clone()), batch_info.clone()))?;
+        debug_assert!(coding_shreds
+            .iter()
+            .all(|shred| shred.slot() == bank.slot()));
+        socket_sender.send((coding_shreds.clone(), batch_info.clone()))?;
         blockstore_sender.send((coding_shreds, batch_info))?;
 
         coding_send_time.stop();
@@ -290,7 +304,7 @@ impl StandardBroadcastRun {
         self.process_shreds_stats.update(&process_stats);
 
         if last_tick_height == bank.max_tick_height() {
-            self.report_and_reset_stats();
+            self.report_and_reset_stats(false);
             self.unfinished_slot = None;
         }
 
@@ -335,42 +349,28 @@ impl StandardBroadcastRun {
         &mut self,
         sock: &UdpSocket,
         cluster_info: &ClusterInfo,
-        stakes: Option<&HashMap<Pubkey, u64>>,
         shreds: Arc<Vec<Shred>>,
         broadcast_shred_batch_info: Option<BroadcastShredBatchInfo>,
+        bank_forks: &Arc<RwLock<BankForks>>,
     ) -> Result<()> {
-        const BROADCAST_PEER_UPDATE_INTERVAL_MS: u64 = 1000;
         trace!("Broadcasting {:?} shreds", shreds.len());
-        // Get the list of peers to broadcast to
-        let mut get_peers_time = Measure::start("broadcast::get_peers");
-        if self
-            .last_peer_update
-            .should_update_ext(BROADCAST_PEER_UPDATE_INTERVAL_MS, false)
-        {
-            *self.cluster_nodes.write().unwrap() = ClusterNodes::<BroadcastStage>::new(
-                cluster_info,
-                stakes.unwrap_or(&HashMap::default()),
-            );
-        }
-        get_peers_time.stop();
-        let cluster_nodes = self.cluster_nodes.read().unwrap();
-
         let mut transmit_stats = TransmitShredsStats::default();
         // Broadcast the shreds
         let mut transmit_time = Measure::start("broadcast_shreds");
+
         broadcast_shreds(
             sock,
             &shreds,
-            &cluster_nodes,
+            &self.cluster_nodes_cache,
             &self.last_datapoint_submit,
             &mut transmit_stats,
+            cluster_info,
+            bank_forks,
             cluster_info.socket_addr_space(),
         )?;
-        drop(cluster_nodes);
         transmit_time.stop();
 
         transmit_stats.transmit_elapsed = transmit_time.as_us();
-        transmit_stats.get_peers_elapsed = get_peers_time.as_us();
         transmit_stats.num_shreds = shreds.len();
 
         // Process metrics
@@ -387,35 +387,59 @@ impl StandardBroadcastRun {
         transmit_shreds_stats.update(new_transmit_shreds_stats, broadcast_shred_batch_info);
     }
 
-    fn report_and_reset_stats(&mut self) {
+    fn report_and_reset_stats(&mut self, was_interrupted: bool) {
         let stats = &self.process_shreds_stats;
         let unfinished_slot = self.unfinished_slot.as_ref().unwrap();
-        datapoint_info!(
-            "broadcast-process-shreds-stats",
-            ("slot", unfinished_slot.slot as i64, i64),
-            ("shredding_time", stats.shredding_elapsed, i64),
-            ("receive_time", stats.receive_elapsed, i64),
-            (
-                "num_data_shreds",
-                unfinished_slot.next_shred_index as i64,
-                i64
-            ),
-            (
-                "slot_broadcast_time",
-                self.slot_broadcast_start.unwrap().elapsed().as_micros() as i64,
-                i64
-            ),
-            (
-                "get_leader_schedule_time",
-                stats.get_leader_schedule_elapsed,
-                i64
-            ),
-            ("serialize_shreds_time", stats.serialize_elapsed, i64),
-            ("gen_data_time", stats.gen_data_elapsed, i64),
-            ("gen_coding_time", stats.gen_coding_elapsed, i64),
-            ("sign_coding_time", stats.sign_coding_elapsed, i64),
-            ("coding_send_time", stats.coding_send_elapsed, i64),
-        );
+        if was_interrupted {
+            datapoint_info!(
+                "broadcast-process-shreds-interrupted-stats",
+                ("slot", unfinished_slot.slot as i64, i64),
+                ("shredding_time", stats.shredding_elapsed, i64),
+                ("receive_time", stats.receive_elapsed, i64),
+                (
+                    "num_data_shreds",
+                    unfinished_slot.next_shred_index as i64,
+                    i64
+                ),
+                (
+                    "get_leader_schedule_time",
+                    stats.get_leader_schedule_elapsed,
+                    i64
+                ),
+                ("serialize_shreds_time", stats.serialize_elapsed, i64),
+                ("gen_data_time", stats.gen_data_elapsed, i64),
+                ("gen_coding_time", stats.gen_coding_elapsed, i64),
+                ("sign_coding_time", stats.sign_coding_elapsed, i64),
+                ("coding_send_time", stats.coding_send_elapsed, i64),
+            );
+        } else {
+            datapoint_info!(
+                "broadcast-process-shreds-stats",
+                ("slot", unfinished_slot.slot as i64, i64),
+                ("shredding_time", stats.shredding_elapsed, i64),
+                ("receive_time", stats.receive_elapsed, i64),
+                (
+                    "num_data_shreds",
+                    unfinished_slot.next_shred_index as i64,
+                    i64
+                ),
+                (
+                    "slot_broadcast_time",
+                    self.slot_broadcast_start.unwrap().elapsed().as_micros() as i64,
+                    i64
+                ),
+                (
+                    "get_leader_schedule_time",
+                    stats.get_leader_schedule_elapsed,
+                    i64
+                ),
+                ("serialize_shreds_time", stats.serialize_elapsed, i64),
+                ("gen_data_time", stats.gen_data_elapsed, i64),
+                ("gen_coding_time", stats.gen_coding_elapsed, i64),
+                ("sign_coding_time", stats.sign_coding_elapsed, i64),
+                ("coding_send_time", stats.coding_send_elapsed, i64),
+            );
+        }
         self.process_shreds_stats.reset();
     }
 }
@@ -451,7 +475,7 @@ impl BroadcastRun for StandardBroadcastRun {
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        socket_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         let receive_results = broadcast_utils::recv_slot_entries(receiver)?;
@@ -469,9 +493,10 @@ impl BroadcastRun for StandardBroadcastRun {
         receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &ClusterInfo,
         sock: &UdpSocket,
+        bank_forks: &Arc<RwLock<BankForks>>,
     ) -> Result<()> {
-        let ((stakes, shreds), slot_start_ts) = receiver.lock().unwrap().recv()?;
-        self.broadcast(sock, cluster_info, stakes.as_deref(), shreds, slot_start_ts)
+        let (shreds, batch_info) = receiver.lock().unwrap().recv()?;
+        self.broadcast(sock, cluster_info, shreds, batch_info, bank_forks)
     }
     fn record(
         &mut self,
@@ -502,6 +527,7 @@ mod test {
     use std::sync::Arc;
     use std::time::Duration;
 
+    #[allow(clippy::type_complexity)]
     fn setup(
         num_shreds_per_slot: Slot,
     ) -> (
@@ -511,6 +537,7 @@ mod test {
         Arc<Bank>,
         Arc<Keypair>,
         UdpSocket,
+        Arc<RwLock<BankForks>>,
     ) {
         // Setup
         let ledger_path = get_tmp_ledger_path!();
@@ -528,7 +555,10 @@ mod test {
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut genesis_config = create_genesis_config(10_000).genesis_config;
         genesis_config.ticks_per_slot = max_ticks_per_n_shreds(num_shreds_per_slot, None) + 1;
-        let bank0 = Arc::new(Bank::new(&genesis_config));
+
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let bank0 = bank_forks.read().unwrap().root_bank();
         (
             blockstore,
             genesis_config,
@@ -536,6 +566,7 @@ mod test {
             bank0,
             leader_keypair,
             socket,
+            bank_forks,
         )
     }
 
@@ -578,7 +609,7 @@ mod test {
     fn test_slot_interrupt() {
         // Setup
         let num_shreds_per_slot = 2;
-        let (blockstore, genesis_config, cluster_info, bank0, leader_keypair, socket) =
+        let (blockstore, genesis_config, cluster_info, bank0, leader_keypair, socket, bank_forks) =
             setup(num_shreds_per_slot);
 
         // Insert 1 less than the number of ticks needed to finish the slot
@@ -593,7 +624,13 @@ mod test {
         // Step 1: Make an incomplete transmission for slot 0
         let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair.clone(), 0);
         standard_broadcast_run
-            .test_process_receive_results(&cluster_info, &socket, &blockstore, receive_results)
+            .test_process_receive_results(
+                &cluster_info,
+                &socket,
+                &blockstore,
+                receive_results,
+                &bank_forks,
+            )
             .unwrap();
         let unfinished_slot = standard_broadcast_run.unfinished_slot.as_ref().unwrap();
         assert_eq!(unfinished_slot.next_shred_index as u64, num_shreds_per_slot);
@@ -651,7 +688,13 @@ mod test {
             last_tick_height: (ticks1.len() - 1) as u64,
         };
         standard_broadcast_run
-            .test_process_receive_results(&cluster_info, &socket, &blockstore, receive_results)
+            .test_process_receive_results(
+                &cluster_info,
+                &socket,
+                &blockstore,
+                receive_results,
+                &bank_forks,
+            )
             .unwrap();
         let unfinished_slot = standard_broadcast_run.unfinished_slot.as_ref().unwrap();
 
@@ -692,7 +735,7 @@ mod test {
     #[test]
     fn test_buffer_data_shreds() {
         let num_shreds_per_slot = 2;
-        let (blockstore, genesis_config, _cluster_info, bank, leader_keypair, _socket) =
+        let (blockstore, genesis_config, _cluster_info, bank, leader_keypair, _socket, _bank_forks) =
             setup(num_shreds_per_slot);
         let (bsend, brecv) = channel();
         let (ssend, _srecv) = channel();
@@ -737,7 +780,7 @@ mod test {
     fn test_slot_finish() {
         // Setup
         let num_shreds_per_slot = 2;
-        let (blockstore, genesis_config, cluster_info, bank0, leader_keypair, socket) =
+        let (blockstore, genesis_config, cluster_info, bank0, leader_keypair, socket, bank_forks) =
             setup(num_shreds_per_slot);
 
         // Insert complete slot of ticks needed to finish the slot
@@ -751,7 +794,13 @@ mod test {
 
         let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair, 0);
         standard_broadcast_run
-            .test_process_receive_results(&cluster_info, &socket, &blockstore, receive_results)
+            .test_process_receive_results(
+                &cluster_info,
+                &socket,
+                &blockstore,
+                receive_results,
+                &bank_forks,
+            )
             .unwrap();
         assert!(standard_broadcast_run.unfinished_slot.is_none())
     }

core/src/cluster_info_vote_listener.rs

@@ -351,7 +351,10 @@ impl ClusterInfoVoteListener {
         labels: Vec<CrdsValueLabel>,
     ) -> (Vec<Transaction>, Vec<(CrdsValueLabel, Slot, Packets)>) {
         let mut msgs = packet::to_packets_chunked(&votes, 1);
-        sigverify::ed25519_verify_cpu(&mut msgs);
+
+        // Votes should already be filtered by this point.
+        let reject_non_vote = false;
+        sigverify::ed25519_verify_cpu(&mut msgs, reject_non_vote);
 
         let (vote_txs, packets) = izip!(labels.into_iter(), votes.into_iter(), msgs,)
             .filter_map(|(label, vote, packet)| {
@@ -1531,7 +1534,7 @@ mod tests {
         let vote_tracker = VoteTracker::new(&bank);
         let optimistically_confirmed_bank =
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
-        let subscriptions = Arc::new(RpcSubscriptions::new(
+        let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
             &exit,
             bank_forks,
             Arc::new(RwLock::new(BlockCommitmentCache::default())),
@@ -1650,7 +1653,7 @@ mod tests {
         let bank = bank_forks.read().unwrap().get(0).unwrap().clone();
         let optimistically_confirmed_bank =
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
-        let subscriptions = Arc::new(RpcSubscriptions::new(
+        let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
             &exit,
             bank_forks,
             Arc::new(RwLock::new(BlockCommitmentCache::default())),

core/src/cluster_nodes.rs

@@ -1,14 +1,27 @@
 use {
     crate::{broadcast_stage::BroadcastStage, retransmit_stage::RetransmitStage},
     itertools::Itertools,
+    lru::LruCache,
     solana_gossip::{
         cluster_info::{compute_retransmit_peers, ClusterInfo},
         contact_info::ContactInfo,
         crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS,
         weighted_shuffle::{weighted_best, weighted_shuffle},
     },
-    solana_sdk::pubkey::Pubkey,
-    std::{any::TypeId, cmp::Reverse, collections::HashMap, marker::PhantomData},
+    solana_runtime::bank::Bank,
+    solana_sdk::{
+        clock::{Epoch, Slot},
+        pubkey::Pubkey,
+    },
+    std::{
+        any::TypeId,
+        cmp::Reverse,
+        collections::HashMap,
+        marker::PhantomData,
+        ops::Deref,
+        sync::{Arc, Mutex},
+        time::{Duration, Instant},
+    },
 };
 
 enum NodeId {
@@ -35,6 +48,15 @@ pub struct ClusterNodes<T> {
     _phantom: PhantomData<T>,
 }
 
+type CacheEntry<T> = Option<(/*as of:*/ Instant, Arc<ClusterNodes<T>>)>;
+
+pub struct ClusterNodesCache<T> {
+    // Cache entries are wrapped in Arc<Mutex<...>>, so that, when needed, only
+    // one thread does the computations to update the entry for the epoch.
+    cache: Mutex<LruCache<Epoch, Arc<Mutex<CacheEntry<T>>>>>,
+    ttl: Duration, // Time to live.
+}
+
 impl Node {
     #[inline]
     fn pubkey(&self) -> Pubkey {
@@ -54,12 +76,12 @@ impl Node {
 }
 
 impl<T> ClusterNodes<T> {
-    pub fn num_peers(&self) -> usize {
+    pub(crate) fn num_peers(&self) -> usize {
         self.index.len()
     }
 
     // A peer is considered live if they generated their contact info recently.
-    pub fn num_peers_live(&self, now: u64) -> usize {
+    pub(crate) fn num_peers_live(&self, now: u64) -> usize {
         self.index
             .iter()
             .filter_map(|(_, index)| self.nodes[*index].contact_info())
@@ -82,7 +104,7 @@ impl ClusterNodes<BroadcastStage> {
 
     /// Returns the root of turbine broadcast tree, which the leader sends the
     /// shred to.
-    pub fn get_broadcast_peer(&self, shred_seed: [u8; 32]) -> Option<&ContactInfo> {
+    pub(crate) fn get_broadcast_peer(&self, shred_seed: [u8; 32]) -> Option<&ContactInfo> {
         if self.index.is_empty() {
             None
         } else {
@@ -96,33 +118,25 @@ impl ClusterNodes<BroadcastStage> {
 }
 
 impl ClusterNodes<RetransmitStage> {
-    pub fn new(cluster_info: &ClusterInfo, stakes: &HashMap<Pubkey, u64>) -> Self {
-        new_cluster_nodes(cluster_info, stakes)
-    }
-
-    pub fn get_retransmit_peers(
+    pub(crate) fn get_retransmit_peers(
         &self,
         shred_seed: [u8; 32],
         fanout: usize,
-        slot_leader: Option<Pubkey>,
+        slot_leader: Pubkey,
     ) -> (
         Vec<&ContactInfo>, // neighbors
         Vec<&ContactInfo>, // children
     ) {
         // Exclude leader from list of nodes.
-        let index = self.index.iter().copied();
-        let (weights, index): (Vec<u64>, Vec<usize>) = match slot_leader {
-            None => {
-                error!("unknown leader for shred slot");
-                index.unzip()
-            }
-            Some(slot_leader) if slot_leader == self.pubkey => {
-                error!("retransmit from slot leader: {}", slot_leader);
-                index.unzip()
-            }
-            Some(slot_leader) => index
+        let (weights, index): (Vec<u64>, Vec<usize>) = if slot_leader == self.pubkey {
+            error!("retransmit from slot leader: {}", slot_leader);
+            self.index.iter().copied().unzip()
+        } else {
+            self.index
+                .iter()
                 .filter(|(_, i)| self.nodes[*i].pubkey() != slot_leader)
-                .unzip(),
+                .copied()
+                .unzip()
         };
         let index: Vec<_> = {
             let shuffle = weighted_shuffle(&weights, shred_seed);
@@ -211,6 +225,70 @@ fn get_nodes(cluster_info: &ClusterInfo, stakes: &HashMap<Pubkey, u64>) -> Vec<N
     .collect()
 }
 
+impl<T> ClusterNodesCache<T> {
+    pub fn new(
+        // Capacity of underlying LRU-cache in terms of number of epochs.
+        cap: usize,
+        // A time-to-live eviction policy is enforced to refresh entries in
+        // case gossip contact-infos are updated.
+        ttl: Duration,
+    ) -> Self {
+        Self {
+            cache: Mutex::new(LruCache::new(cap)),
+            ttl,
+        }
+    }
+}
+
+impl<T: 'static> ClusterNodesCache<T> {
+    fn get_cache_entry(&self, epoch: Epoch) -> Arc<Mutex<CacheEntry<T>>> {
+        let mut cache = self.cache.lock().unwrap();
+        match cache.get(&epoch) {
+            Some(entry) => Arc::clone(entry),
+            None => {
+                let entry = Arc::default();
+                cache.put(epoch, Arc::clone(&entry));
+                entry
+            }
+        }
+    }
+
+    pub(crate) fn get(
+        &self,
+        shred_slot: Slot,
+        root_bank: &Bank,
+        working_bank: &Bank,
+        cluster_info: &ClusterInfo,
+    ) -> Arc<ClusterNodes<T>> {
+        let epoch = root_bank.get_leader_schedule_epoch(shred_slot);
+        let entry = self.get_cache_entry(epoch);
+        // Hold the lock on the entry here so that, if needed, only
+        // one thread recomputes cluster-nodes for this epoch.
+        let mut entry = entry.lock().unwrap();
+        if let Some((asof, nodes)) = entry.deref() {
+            if asof.elapsed() < self.ttl {
+                return Arc::clone(nodes);
+            }
+        }
+        let epoch_staked_nodes = [root_bank, working_bank]
+            .iter()
+            .find_map(|bank| bank.epoch_staked_nodes(epoch));
+        if epoch_staked_nodes.is_none() {
+            inc_new_counter_info!("cluster_nodes-unknown_epoch_staked_nodes", 1);
+            if epoch != root_bank.get_leader_schedule_epoch(root_bank.slot()) {
+                return self.get(root_bank.slot(), root_bank, working_bank, cluster_info);
+            }
+            inc_new_counter_info!("cluster_nodes-unknown_epoch_staked_nodes_root", 1);
+        }
+        let nodes = Arc::new(new_cluster_nodes::<T>(
+            cluster_info,
+            &epoch_staked_nodes.unwrap_or_default(),
+        ));
+        *entry = Some((Instant::now(), Arc::clone(&nodes)));
+        nodes
+    }
+}
+
 impl From<ContactInfo> for NodeId {
     fn from(node: ContactInfo) -> Self {
         NodeId::ContactInfo(node)
@@ -240,6 +318,7 @@ mod tests {
         super::*,
         rand::{seq::SliceRandom, Rng},
         solana_gossip::{
+            crds::GossipRoute,
             crds_value::{CrdsData, CrdsValue},
             deprecated::{
                 shuffle_peers_and_index, sorted_retransmit_peers_and_stakes,
@@ -306,7 +385,10 @@ mod tests {
             for node in nodes.iter().skip(1) {
                 let node = CrdsData::ContactInfo(node.clone());
                 let node = CrdsValue::new_unsigned(node);
-                assert_eq!(gossip.crds.insert(node, now), Ok(()));
+                assert_eq!(
+                    gossip.crds.insert(node, now, GossipRoute::LocalMessage),
+                    Ok(())
+                );
             }
         }
         (nodes, stakes, cluster_info)
@@ -319,7 +401,7 @@ mod tests {
         let this_node = cluster_info.my_contact_info();
         // ClusterInfo::tvu_peers excludes the node itself.
         assert_eq!(cluster_info.tvu_peers().len(), nodes.len() - 1);
-        let cluster_nodes = ClusterNodes::<RetransmitStage>::new(&cluster_info, &stakes);
+        let cluster_nodes = new_cluster_nodes::<RetransmitStage>(&cluster_info, &stakes);
         // All nodes with contact-info should be in the index.
         assert_eq!(cluster_nodes.index.len(), nodes.len());
         // Staked nodes with no contact-info should be included.
@@ -380,7 +462,7 @@ mod tests {
             let (neighbors_indices, children_indices) =
                 compute_retransmit_peers(fanout, self_index, &shuffled_index);
             let (neighbors, children) =
-                cluster_nodes.get_retransmit_peers(shred_seed, fanout, Some(slot_leader));
+                cluster_nodes.get_retransmit_peers(shred_seed, fanout, slot_leader);
             assert_eq!(children.len(), children_indices.len());
             for (node, index) in children.into_iter().zip(children_indices) {
                 assert_eq!(*node, peers[index]);

core/src/cost_update_service.rs

@@ -0,0 +1,303 @@
+//! this service receives instruction ExecuteTimings from replay_stage,
+//! update cost_model which is shared with banking_stage to optimize
+//! packing transactions into block; it also triggers persisting cost
+//! table to blockstore.
+
+use solana_ledger::blockstore::Blockstore;
+use solana_measure::measure::Measure;
+use solana_runtime::{bank::Bank, bank::ExecuteTimings, cost_model::CostModel};
+use solana_sdk::timing::timestamp;
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::Receiver,
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::Duration,
+};
+
+#[derive(Default)]
+pub struct CostUpdateServiceTiming {
+    last_print: u64,
+    update_cost_model_count: u64,
+    update_cost_model_elapsed: u64,
+    persist_cost_table_elapsed: u64,
+}
+
+impl CostUpdateServiceTiming {
+    fn update(
+        &mut self,
+        update_cost_model_count: u64,
+        update_cost_model_elapsed: u64,
+        persist_cost_table_elapsed: u64,
+    ) {
+        self.update_cost_model_count += update_cost_model_count;
+        self.update_cost_model_elapsed += update_cost_model_elapsed;
+        self.persist_cost_table_elapsed += persist_cost_table_elapsed;
+
+        let now = timestamp();
+        let elapsed_ms = now - self.last_print;
+        if elapsed_ms > 1000 {
+            datapoint_info!(
+                "cost-update-service-stats",
+                ("total_elapsed_us", elapsed_ms * 1000, i64),
+                (
+                    "update_cost_model_count",
+                    self.update_cost_model_count as i64,
+                    i64
+                ),
+                (
+                    "update_cost_model_elapsed",
+                    self.update_cost_model_elapsed as i64,
+                    i64
+                ),
+                (
+                    "persist_cost_table_elapsed",
+                    self.persist_cost_table_elapsed as i64,
+                    i64
+                ),
+            );
+
+            *self = CostUpdateServiceTiming::default();
+            self.last_print = now;
+        }
+    }
+}
+
+pub enum CostUpdate {
+    FrozenBank { bank: Arc<Bank> },
+    ExecuteTiming { execute_timings: ExecuteTimings },
+}
+
+pub type CostUpdateReceiver = Receiver<CostUpdate>;
+
+pub struct CostUpdateService {
+    thread_hdl: JoinHandle<()>,
+}
+
+impl CostUpdateService {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        exit: Arc<AtomicBool>,
+        blockstore: Arc<Blockstore>,
+        cost_model: Arc<RwLock<CostModel>>,
+        cost_update_receiver: CostUpdateReceiver,
+    ) -> Self {
+        let thread_hdl = Builder::new()
+            .name("solana-cost-update-service".to_string())
+            .spawn(move || {
+                Self::service_loop(exit, blockstore, cost_model, cost_update_receiver);
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+
+    fn service_loop(
+        exit: Arc<AtomicBool>,
+        blockstore: Arc<Blockstore>,
+        cost_model: Arc<RwLock<CostModel>>,
+        cost_update_receiver: CostUpdateReceiver,
+    ) {
+        let mut cost_update_service_timing = CostUpdateServiceTiming::default();
+        let mut dirty: bool;
+        let mut update_count: u64;
+        let wait_timer = Duration::from_millis(100);
+
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            dirty = false;
+            update_count = 0_u64;
+            let mut update_cost_model_time = Measure::start("update_cost_model_time");
+            for cost_update in cost_update_receiver.try_iter() {
+                match cost_update {
+                    CostUpdate::FrozenBank { bank } => {
+                        bank.read_cost_tracker().unwrap().report_stats(bank.slot());
+                    }
+                    CostUpdate::ExecuteTiming { execute_timings } => {
+                        dirty |= Self::update_cost_model(&cost_model, &execute_timings);
+                        update_count += 1;
+                    }
+                }
+            }
+            update_cost_model_time.stop();
+
+            let mut persist_cost_table_time = Measure::start("persist_cost_table_time");
+            if dirty {
+                Self::persist_cost_table(&blockstore, &cost_model);
+            }
+            persist_cost_table_time.stop();
+
+            cost_update_service_timing.update(
+                update_count,
+                update_cost_model_time.as_us(),
+                persist_cost_table_time.as_us(),
+            );
+
+            thread::sleep(wait_timer);
+        }
+    }
+
+    fn update_cost_model(cost_model: &RwLock<CostModel>, execute_timings: &ExecuteTimings) -> bool {
+        let mut dirty = false;
+        {
+            let mut cost_model_mutable = cost_model.write().unwrap();
+            for (program_id, timing) in &execute_timings.details.per_program_timings {
+                if timing.count < 1 {
+                    continue;
+                }
+                let units = timing.accumulated_units / timing.count as u64;
+                match cost_model_mutable.upsert_instruction_cost(program_id, units) {
+                    Ok(c) => {
+                        debug!(
+                            "after replayed into bank, instruction {:?} has averaged cost {}",
+                            program_id, c
+                        );
+                        dirty = true;
+                    }
+                    Err(err) => {
+                        debug!(
+                        "after replayed into bank, instruction {:?} failed to update cost, err: {}",
+                        program_id, err
+                    );
+                    }
+                }
+            }
+        }
+        debug!(
+           "after replayed into bank, updated cost model instruction cost table, current values: {:?}",
+           cost_model.read().unwrap().get_instruction_cost_table()
+        );
+        dirty
+    }
+
+    fn persist_cost_table(blockstore: &Blockstore, cost_model: &RwLock<CostModel>) {
+        let cost_model_read = cost_model.read().unwrap();
+        let cost_table = cost_model_read.get_instruction_cost_table();
+        let db_records = blockstore.read_program_costs().expect("read programs");
+
+        // delete records from blockstore if they are no longer in cost_table
+        db_records.iter().for_each(|(pubkey, _)| {
+            if cost_table.get(pubkey).is_none() {
+                blockstore
+                    .delete_program_cost(pubkey)
+                    .expect("delete old program");
+            }
+        });
+
+        for (key, cost) in cost_table.iter() {
+            blockstore
+                .write_program_cost(key, cost)
+                .expect("persist program costs to blockstore");
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::message_processor::ProgramTiming;
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_update_cost_model_with_empty_execute_timings() {
+        let cost_model = Arc::new(RwLock::new(CostModel::default()));
+        let empty_execute_timings = ExecuteTimings::default();
+        CostUpdateService::update_cost_model(&cost_model, &empty_execute_timings);
+
+        assert_eq!(
+            0,
+            cost_model
+                .read()
+                .unwrap()
+                .get_instruction_cost_table()
+                .len()
+        );
+    }
+
+    #[test]
+    fn test_update_cost_model_with_execute_timings() {
+        let cost_model = Arc::new(RwLock::new(CostModel::default()));
+        let mut execute_timings = ExecuteTimings::default();
+
+        let program_key_1 = Pubkey::new_unique();
+        let mut expected_cost: u64;
+
+        // add new program
+        {
+            let accumulated_us: u64 = 1000;
+            let accumulated_units: u64 = 100;
+            let count: u32 = 10;
+            expected_cost = accumulated_units / count as u64;
+
+            execute_timings.details.per_program_timings.insert(
+                program_key_1,
+                ProgramTiming {
+                    accumulated_us,
+                    accumulated_units,
+                    count,
+                },
+            );
+            CostUpdateService::update_cost_model(&cost_model, &execute_timings);
+            assert_eq!(
+                1,
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .len()
+            );
+            assert_eq!(
+                Some(&expected_cost),
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .get(&program_key_1)
+            );
+        }
+
+        // update program
+        {
+            let accumulated_us: u64 = 2000;
+            let accumulated_units: u64 = 200;
+            let count: u32 = 10;
+            // to expect new cost is Average(new_value, existing_value)
+            expected_cost = ((accumulated_units / count as u64) + expected_cost) / 2;
+
+            execute_timings.details.per_program_timings.insert(
+                program_key_1,
+                ProgramTiming {
+                    accumulated_us,
+                    accumulated_units,
+                    count,
+                },
+            );
+            CostUpdateService::update_cost_model(&cost_model, &execute_timings);
+            assert_eq!(
+                1,
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .len()
+            );
+            assert_eq!(
+                Some(&expected_cost),
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .get(&program_key_1)
+            );
+        }
+    }
+}

core/src/fetch_stage.rs

@@ -23,38 +23,49 @@ impl FetchStage {
     pub fn new(
         sockets: Vec<UdpSocket>,
         tpu_forwards_sockets: Vec<UdpSocket>,
+        tpu_vote_sockets: Vec<UdpSocket>,
         exit: &Arc<AtomicBool>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         coalesce_ms: u64,
-    ) -> (Self, PacketReceiver) {
+    ) -> (Self, PacketReceiver, PacketReceiver) {
         let (sender, receiver) = channel();
+        let (vote_sender, vote_receiver) = channel();
         (
             Self::new_with_sender(
                 sockets,
                 tpu_forwards_sockets,
+                tpu_vote_sockets,
                 exit,
                 &sender,
-                poh_recorder,
+                &vote_sender,
+                &poh_recorder,
                 coalesce_ms,
             ),
             receiver,
+            vote_receiver,
         )
     }
+
     pub fn new_with_sender(
         sockets: Vec<UdpSocket>,
         tpu_forwards_sockets: Vec<UdpSocket>,
+        tpu_vote_sockets: Vec<UdpSocket>,
         exit: &Arc<AtomicBool>,
         sender: &PacketSender,
+        vote_sender: &PacketSender,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         coalesce_ms: u64,
     ) -> Self {
         let tx_sockets = sockets.into_iter().map(Arc::new).collect();
         let tpu_forwards_sockets = tpu_forwards_sockets.into_iter().map(Arc::new).collect();
+        let tpu_vote_sockets = tpu_vote_sockets.into_iter().map(Arc::new).collect();
         Self::new_multi_socket(
             tx_sockets,
             tpu_forwards_sockets,
+            tpu_vote_sockets,
             exit,
             sender,
+            vote_sender,
             poh_recorder,
             coalesce_ms,
         )
@@ -98,8 +109,10 @@ impl FetchStage {
     fn new_multi_socket(
         sockets: Vec<Arc<UdpSocket>>,
         tpu_forwards_sockets: Vec<Arc<UdpSocket>>,
+        tpu_vote_sockets: Vec<Arc<UdpSocket>>,
         exit: &Arc<AtomicBool>,
         sender: &PacketSender,
+        vote_sender: &PacketSender,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         coalesce_ms: u64,
     ) -> Self {
@@ -130,6 +143,18 @@ impl FetchStage {
             )
         });
 
+        let tpu_vote_threads = tpu_vote_sockets.into_iter().map(|socket| {
+            streamer::receiver(
+                socket,
+                &exit,
+                vote_sender.clone(),
+                recycler.clone(),
+                "fetch_vote_stage",
+                coalesce_ms,
+                true,
+            )
+        });
+
         let sender = sender.clone();
         let poh_recorder = poh_recorder.clone();
 
@@ -150,7 +175,10 @@ impl FetchStage {
             })
             .unwrap();
 
-        let mut thread_hdls: Vec<_> = tpu_threads.chain(tpu_forwards_threads).collect();
+        let mut thread_hdls: Vec<_> = tpu_threads
+            .chain(tpu_forwards_threads)
+            .chain(tpu_vote_threads)
+            .collect();
         thread_hdls.push(fwd_thread_hdl);
         Self { thread_hdls }
     }

core/src/lib.rs

@@ -19,6 +19,7 @@ pub mod cluster_slots_service;
 pub mod commitment_service;
 pub mod completed_data_sets_service;
 pub mod consensus;
+pub mod cost_update_service;
 pub mod fetch_stage;
 pub mod fork_choice;
 pub mod gen_keys;
@@ -46,6 +47,7 @@ pub mod sigverify;
 pub mod sigverify_shreds;
 pub mod sigverify_stage;
 pub mod snapshot_packager_service;
+pub mod system_monitor_service;
 pub mod test_validator;
 pub mod tpu;
 pub mod tree_diff;

core/src/packet_hasher.rs

@@ -1,10 +1,13 @@
 // Get a unique hash value for a packet
 // Used in retransmit and shred fetch to prevent dos with same packet data.
 
-use ahash::AHasher;
-use rand::{thread_rng, Rng};
-use solana_perf::packet::Packet;
-use std::hash::Hasher;
+use {
+    ahash::AHasher,
+    rand::{thread_rng, Rng},
+    solana_ledger::shred::Shred,
+    solana_perf::packet::Packet,
+    std::hash::Hasher,
+};
 
 #[derive(Clone)]
 pub struct PacketHasher {
@@ -22,9 +25,18 @@ impl Default for PacketHasher {
 }
 
 impl PacketHasher {
-    pub fn hash_packet(&self, packet: &Packet) -> u64 {
+    pub(crate) fn hash_packet(&self, packet: &Packet) -> u64 {
+        let size = packet.data.len().min(packet.meta.size);
+        self.hash_data(&packet.data[..size])
+    }
+
+    pub(crate) fn hash_shred(&self, shred: &Shred) -> u64 {
+        self.hash_data(&shred.payload)
+    }
+
+    fn hash_data(&self, data: &[u8]) -> u64 {
         let mut hasher = AHasher::new_with_keys(self.seed1, self.seed2);
-        hasher.write(&packet.data[0..packet.meta.size]);
+        hasher.write(data);
         hasher.finish()
     }
 

core/src/progress_map.rs

@@ -63,6 +63,11 @@ impl ReplaySlotStats {
             ("load_us", self.execute_timings.load_us, i64),
             ("execute_us", self.execute_timings.execute_us, i64),
             ("store_us", self.execute_timings.store_us, i64),
+            (
+                "update_stakes_cache_us",
+                self.execute_timings.update_stakes_cache_us,
+                i64
+            ),
             (
                 "total_batches_len",
                 self.execute_timings.total_batches_len,
@@ -114,6 +119,43 @@ impl ReplaySlotStats {
                 i64
             ),
         );
+
+        let mut per_pubkey_timings: Vec<_> = self
+            .execute_timings
+            .details
+            .per_program_timings
+            .iter()
+            .collect();
+        per_pubkey_timings.sort_by(|a, b| b.1.accumulated_us.cmp(&a.1.accumulated_us));
+        let (total_us, total_units, total_count) =
+            per_pubkey_timings
+                .iter()
+                .fold((0, 0, 0), |(sum_us, sum_units, sum_count), a| {
+                    (
+                        sum_us + a.1.accumulated_us,
+                        sum_units + a.1.accumulated_units,
+                        sum_count + a.1.count,
+                    )
+                });
+
+        for (pubkey, time) in per_pubkey_timings.iter().take(5) {
+            datapoint_info!(
+                "per_program_timings",
+                ("slot", slot as i64, i64),
+                ("pubkey", pubkey.to_string(), String),
+                ("execute_us", time.accumulated_us, i64),
+                ("accumulated_units", time.accumulated_units, i64),
+                ("count", time.count, i64)
+            );
+        }
+        datapoint_info!(
+            "per_program_timings",
+            ("slot", slot as i64, i64),
+            ("pubkey", "all", String),
+            ("execute_us", total_us, i64),
+            ("accumulated_units", total_units, i64),
+            ("count", total_count, i64)
+        );
     }
 }
 

core/src/repair_service.rs

@@ -24,6 +24,7 @@ use solana_sdk::{
     pubkey::Pubkey,
     timing::timestamp,
 };
+use solana_streamer::sendmmsg::{batch_send, SendPktsError};
 use std::{
     collections::{HashMap, HashSet},
     iter::Iterator,
@@ -92,6 +93,8 @@ pub struct RepairTiming {
     pub get_best_orphans_elapsed: u64,
     pub get_best_shreds_elapsed: u64,
     pub send_repairs_elapsed: u64,
+    pub build_repairs_batch_elapsed: u64,
+    pub batch_send_repairs_elapsed: u64,
 }
 
 impl RepairTiming {
@@ -100,12 +103,15 @@ impl RepairTiming {
         set_root_elapsed: u64,
         get_votes_elapsed: u64,
         add_votes_elapsed: u64,
-        send_repairs_elapsed: u64,
+        build_repairs_batch_elapsed: u64,
+        batch_send_repairs_elapsed: u64,
     ) {
         self.set_root_elapsed += set_root_elapsed;
         self.get_votes_elapsed += get_votes_elapsed;
         self.add_votes_elapsed += add_votes_elapsed;
-        self.send_repairs_elapsed += send_repairs_elapsed;
+        self.build_repairs_batch_elapsed += build_repairs_batch_elapsed;
+        self.batch_send_repairs_elapsed += batch_send_repairs_elapsed;
+        self.send_repairs_elapsed += build_repairs_batch_elapsed + batch_send_repairs_elapsed;
     }
 }
 
@@ -274,29 +280,50 @@ impl RepairService {
                 )
             };
 
-            let mut send_repairs_elapsed = Measure::start("send_repairs_elapsed");
-            let mut outstanding_requests = outstanding_requests.write().unwrap();
-            repairs.into_iter().for_each(|repair_request| {
-                if let Ok((to, req)) = serve_repair.repair_request(
-                    cluster_slots,
-                    repair_request,
-                    &mut peers_cache,
-                    &mut repair_stats,
-                    &repair_info.repair_validators,
-                    &mut outstanding_requests,
-                ) {
-                    repair_socket.send_to(&req, to).unwrap_or_else(|e| {
-                        info!("{} repair req send_to({}) error {:?}", id, to, e);
-                        0
-                    });
+            let mut build_repairs_batch_elapsed = Measure::start("build_repairs_batch_elapsed");
+            let batch: Vec<(Vec<u8>, SocketAddr)> = {
+                let mut outstanding_requests = outstanding_requests.write().unwrap();
+                repairs
+                    .iter()
+                    .filter_map(|repair_request| {
+                        let (to, req) = serve_repair
+                            .repair_request(
+                                cluster_slots,
+                                *repair_request,
+                                &mut peers_cache,
+                                &mut repair_stats,
+                                &repair_info.repair_validators,
+                                &mut outstanding_requests,
+                            )
+                            .ok()?;
+                        Some((req, to))
+                    })
+                    .collect()
+            };
+            build_repairs_batch_elapsed.stop();
+
+            let mut batch_send_repairs_elapsed = Measure::start("batch_send_repairs_elapsed");
+            if !batch.is_empty() {
+                if let Err(SendPktsError::IoError(err, num_failed)) =
+                    batch_send(repair_socket, &batch)
+                {
+                    error!(
+                        "{} batch_send failed to send {}/{} packets first error {:?}",
+                        id,
+                        num_failed,
+                        batch.len(),
+                        err
+                    );
                 }
-            });
-            send_repairs_elapsed.stop();
+            }
+            batch_send_repairs_elapsed.stop();
+
             repair_timing.update(
                 set_root_elapsed.as_us(),
                 get_votes_elapsed.as_us(),
                 add_votes_elapsed.as_us(),
-                send_repairs_elapsed.as_us(),
+                build_repairs_batch_elapsed.as_us(),
+                batch_send_repairs_elapsed.as_us(),
             );
 
             if last_stats.elapsed().as_secs() > 2 {
@@ -352,6 +379,16 @@ impl RepairService {
                         repair_timing.send_repairs_elapsed,
                         i64
                     ),
+                    (
+                        "build-repairs-batch-elapsed",
+                        repair_timing.build_repairs_batch_elapsed,
+                        i64
+                    ),
+                    (
+                        "batch-send-repairs-elapsed",
+                        repair_timing.batch_send_repairs_elapsed,
+                        i64
+                    ),
                 );
                 repair_stats = RepairStats::default();
                 repair_timing = RepairTiming::default();

core/src/repair_weight.rs

@@ -160,7 +160,7 @@ impl RepairWeight {
         );
         get_best_orphans_elapsed.stop();
 
-        let mut get_best_shreds_elapsed = Measure::start("get_best_orphans");
+        let mut get_best_shreds_elapsed = Measure::start("get_best_shreds");
         // Find the best incomplete slots in rooted subtree
         self.get_best_shreds(blockstore, &mut repairs, max_new_shreds, ignore_slots);
         get_best_shreds_elapsed.stop();

core/src/replay_stage.rs

@@ -13,12 +13,12 @@ use crate::{
     consensus::{
         ComputedBankState, Stake, SwitchForkDecision, Tower, VotedStakes, SWITCH_FORK_THRESHOLD,
     },
+    cost_update_service::CostUpdate,
     fork_choice::{ForkChoice, SelectVoteAndResetForkResult},
     heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
     latest_validator_votes_for_frozen_banks::LatestValidatorVotesForFrozenBanks,
     progress_map::{ForkProgress, ProgressMap, PropagatedStats},
     repair_service::DuplicateSlotsResetReceiver,
-    result::Result,
     rewards_recorder_service::RewardsRecorderSender,
     unfrozen_gossip_verified_vote_hashes::UnfrozenGossipVerifiedVoteHashes,
     voting_service::VoteOp,
@@ -41,8 +41,11 @@ use solana_rpc::{
     rpc_subscriptions::RpcSubscriptions,
 };
 use solana_runtime::{
-    accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
-    commitment::BlockCommitmentCache, vote_sender_types::ReplayVoteSender,
+    accounts_background_service::AbsRequestSender,
+    bank::{Bank, ExecuteTimings, NewBankOptions},
+    bank_forks::BankForks,
+    commitment::BlockCommitmentCache,
+    vote_sender_types::ReplayVoteSender,
 };
 use solana_sdk::{
     clock::{Slot, MAX_PROCESSING_AGE, NUM_CONSECUTIVE_LEADER_SLOTS},
@@ -118,7 +121,7 @@ pub struct ReplayStageConfig {
     pub vote_account: Pubkey,
     pub authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
     pub exit: Arc<AtomicBool>,
-    pub subscriptions: Arc<RpcSubscriptions>,
+    pub rpc_subscriptions: Arc<RpcSubscriptions>,
     pub leader_schedule_cache: Arc<LeaderScheduleCache>,
     pub latest_root_senders: Vec<Sender<Slot>>,
     pub accounts_background_request_sender: AbsRequestSender,
@@ -128,6 +131,7 @@ pub struct ReplayStageConfig {
     pub cache_block_meta_sender: Option<CacheBlockMetaSender>,
     pub bank_notification_sender: Option<BankNotificationSender>,
     pub wait_for_vote_to_start_leader: bool,
+    pub disable_epoch_boundary_optimization: bool,
 }
 
 #[derive(Default)]
@@ -277,7 +281,7 @@ impl ReplayTiming {
                     "process_duplicate_slots_elapsed",
                     self.process_duplicate_slots_elapsed as i64,
                     i64
-                )
+                ),
             );
 
             *self = ReplayTiming::default();
@@ -287,7 +291,7 @@ impl ReplayTiming {
 }
 
 pub struct ReplayStage {
-    t_replay: JoinHandle<Result<()>>,
+    t_replay: JoinHandle<()>,
     commitment_service: AggregateCommitmentService,
 }
 
@@ -311,13 +315,14 @@ impl ReplayStage {
         gossip_verified_vote_hash_receiver: GossipVerifiedVoteHashReceiver,
         cluster_slots_update_sender: ClusterSlotsUpdateSender,
         voting_sender: Sender<VoteOp>,
+        cost_update_sender: Sender<CostUpdate>,
     ) -> Self {
         let ReplayStageConfig {
             my_pubkey,
             vote_account,
             authorized_voter_keypairs,
             exit,
-            subscriptions,
+            rpc_subscriptions,
             leader_schedule_cache,
             latest_root_senders,
             accounts_background_request_sender,
@@ -327,6 +332,7 @@ impl ReplayStage {
             cache_block_meta_sender,
             bank_notification_sender,
             wait_for_vote_to_start_leader,
+            disable_epoch_boundary_optimization,
         } = config;
 
         trace!("replay stage");
@@ -334,7 +340,7 @@ impl ReplayStage {
         let (lockouts_sender, commitment_service) = AggregateCommitmentService::new(
             &exit,
             block_commitment_cache.clone(),
-            subscriptions.clone(),
+            rpc_subscriptions.clone(),
         );
 
         #[allow(clippy::cognitive_complexity)]
@@ -378,7 +384,7 @@ impl ReplayStage {
                         &blockstore,
                         &bank_forks,
                         &leader_schedule_cache,
-                        &subscriptions,
+                        &rpc_subscriptions,
                         &mut progress,
                     );
                     generate_new_bank_forks_time.stop();
@@ -401,12 +407,13 @@ impl ReplayStage {
                         &replay_vote_sender,
                         &bank_notification_sender,
                         &rewards_recorder_sender,
-                        &subscriptions,
+                        &rpc_subscriptions,
                         &mut duplicate_slots_tracker,
                         &gossip_duplicate_confirmed_slots,
                         &mut unfrozen_gossip_verified_vote_hashes,
                         &mut latest_validator_votes_for_frozen_banks,
                         &cluster_slots_update_sender,
+                        &cost_update_sender,
                     );
                     replay_active_banks_time.stop();
 
@@ -592,7 +599,7 @@ impl ReplayStage {
                             &lockouts_sender,
                             &accounts_background_request_sender,
                             &latest_root_senders,
-                            &subscriptions,
+                            &rpc_subscriptions,
                             &block_commitment_cache,
                             &mut heaviest_subtree_fork_choice,
                             &bank_notification_sender,
@@ -684,11 +691,12 @@ impl ReplayStage {
                             &bank_forks,
                             &poh_recorder,
                             &leader_schedule_cache,
-                            &subscriptions,
+                            &rpc_subscriptions,
                             &progress,
                             &retransmit_slots_sender,
                             &mut skipped_slots_info,
                             has_new_vote_been_rooted,
+                            disable_epoch_boundary_optimization,
                         );
 
                         let poh_bank = poh_recorder.lock().unwrap().bank();
@@ -736,7 +744,6 @@ impl ReplayStage {
                         process_duplicate_slots_time.as_us(),
                     );
                 }
-                Ok(())
             })
             .unwrap();
 
@@ -1069,16 +1076,18 @@ impl ReplayStage {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn maybe_start_leader(
         my_pubkey: &Pubkey,
         bank_forks: &Arc<RwLock<BankForks>>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
-        subscriptions: &Arc<RpcSubscriptions>,
+        rpc_subscriptions: &Arc<RpcSubscriptions>,
         progress_map: &ProgressMap,
         retransmit_slots_sender: &RetransmitSlotsSender,
         skipped_slots_info: &mut SkippedSlotsInfo,
         has_new_vote_been_rooted: bool,
+        disable_epoch_boundary_optimization: bool,
     ) {
         // all the individual calls to poh_recorder.lock() are designed to
         // increase granularity, decrease contention
@@ -1195,8 +1204,11 @@ impl ReplayStage {
                 poh_slot,
                 root_slot,
                 my_pubkey,
-                subscriptions,
-                vote_only_bank,
+                rpc_subscriptions,
+                NewBankOptions {
+                    vote_only_bank,
+                    disable_epoch_boundary_optimization,
+                },
             );
 
             let tpu_bank = bank_forks.write().unwrap().insert(tpu_bank);
@@ -1245,7 +1257,7 @@ impl ReplayStage {
         bank: &Bank,
         root: Slot,
         err: &BlockstoreProcessorError,
-        subscriptions: &Arc<RpcSubscriptions>,
+        rpc_subscriptions: &Arc<RpcSubscriptions>,
         duplicate_slots_tracker: &mut DuplicateSlotsTracker,
         gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots,
         progress: &mut ProgressMap,
@@ -1279,7 +1291,7 @@ impl ReplayStage {
         blockstore
             .set_dead_slot(slot)
             .expect("Failed to mark slot as dead in blockstore");
-        subscriptions.notify_slot_update(SlotUpdate::Dead {
+        rpc_subscriptions.notify_slot_update(SlotUpdate::Dead {
             slot,
             err: format!("error: {:?}", err),
             timestamp: timestamp(),
@@ -1311,7 +1323,7 @@ impl ReplayStage {
         lockouts_sender: &Sender<CommitmentAggregationData>,
         accounts_background_request_sender: &AbsRequestSender,
         latest_root_senders: &[Sender<Slot>],
-        subscriptions: &Arc<RpcSubscriptions>,
+        rpc_subscriptions: &Arc<RpcSubscriptions>,
         block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
         heaviest_subtree_fork_choice: &mut HeaviestSubtreeForkChoice,
         bank_notification_sender: &Option<BankNotificationSender>,
@@ -1372,7 +1384,7 @@ impl ReplayStage {
                 has_new_vote_been_rooted,
                 vote_signatures,
             );
-            subscriptions.notify_roots(rooted_slots);
+            rpc_subscriptions.notify_roots(rooted_slots);
             if let Some(sender) = bank_notification_sender {
                 sender
                     .send(BankNotification::Root(root_bank))
@@ -1673,15 +1685,17 @@ impl ReplayStage {
         replay_vote_sender: &ReplayVoteSender,
         bank_notification_sender: &Option<BankNotificationSender>,
         rewards_recorder_sender: &Option<RewardsRecorderSender>,
-        subscriptions: &Arc<RpcSubscriptions>,
+        rpc_subscriptions: &Arc<RpcSubscriptions>,
         duplicate_slots_tracker: &mut DuplicateSlotsTracker,
         gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots,
         unfrozen_gossip_verified_vote_hashes: &mut UnfrozenGossipVerifiedVoteHashes,
         latest_validator_votes_for_frozen_banks: &mut LatestValidatorVotesForFrozenBanks,
         cluster_slots_update_sender: &ClusterSlotsUpdateSender,
+        cost_update_sender: &Sender<CostUpdate>,
     ) -> bool {
         let mut did_complete_bank = false;
         let mut tx_count = 0;
+        let mut execute_timings = ExecuteTimings::default();
         let active_banks = bank_forks.read().unwrap().active_banks();
         trace!("active banks {:?}", active_banks);
 
@@ -1738,7 +1752,7 @@ impl ReplayStage {
                             &bank,
                             root_slot,
                             &err,
-                            subscriptions,
+                            rpc_subscriptions,
                             duplicate_slots_tracker,
                             gossip_duplicate_confirmed_slots,
                             progress,
@@ -1752,6 +1766,12 @@ impl ReplayStage {
             }
             assert_eq!(*bank_slot, bank.slot());
             if bank.is_complete() {
+                execute_timings.accumulate(&bank_progress.replay_stats.execute_timings);
+                debug!("bank {} is completed replay from blockstore, contribute to update cost with {:?}",
+                       bank.slot(),
+                       bank_progress.replay_stats.execute_timings
+                       );
+
                 bank_progress.replay_stats.report_stats(
                     bank.slot(),
                     bank_progress.replay_progress.num_entries,
@@ -1764,6 +1784,13 @@ impl ReplayStage {
                     transaction_status_sender.send_transaction_status_freeze_message(&bank);
                 }
                 bank.freeze();
+                // report cost tracker stats
+                cost_update_sender
+                    .send(CostUpdate::FrozenBank { bank: bank.clone() })
+                    .unwrap_or_else(|err| {
+                        warn!("cost_update_sender failed sending bank stats: {:?}", err)
+                    });
+
                 let bank_hash = bank.hash();
                 assert_ne!(bank_hash, Hash::default());
                 // Needs to be updated before `check_slot_agrees_with_cluster()` so that
@@ -1813,6 +1840,14 @@ impl ReplayStage {
                 );
             }
         }
+
+        // send accumulated excute-timings to cost_update_service
+        if !execute_timings.details.per_program_timings.is_empty() {
+            cost_update_sender
+                .send(CostUpdate::ExecuteTiming { execute_timings })
+                .unwrap_or_else(|err| warn!("cost_update_sender failed: {:?}", err));
+        }
+
         inc_new_counter_info!("replay_stage-replay_transactions", tx_count);
         did_complete_bank
     }
@@ -2406,7 +2441,7 @@ impl ReplayStage {
         blockstore: &Blockstore,
         bank_forks: &RwLock<BankForks>,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
-        subscriptions: &Arc<RpcSubscriptions>,
+        rpc_subscriptions: &Arc<RpcSubscriptions>,
         progress: &mut ProgressMap,
     ) {
         // Find the next slot that chains to the old slot
@@ -2451,8 +2486,8 @@ impl ReplayStage {
                     child_slot,
                     forks.root(),
                     &leader,
-                    subscriptions,
-                    false,
+                    rpc_subscriptions,
+                    NewBankOptions::default(),
                 );
                 let empty: Vec<Pubkey> = vec![];
                 Self::update_fork_propagated_threshold_from_votes(
@@ -2478,11 +2513,11 @@ impl ReplayStage {
         slot: u64,
         root_slot: u64,
         leader: &Pubkey,
-        subscriptions: &Arc<RpcSubscriptions>,
-        vote_only_bank: bool,
+        rpc_subscriptions: &Arc<RpcSubscriptions>,
+        new_bank_options: NewBankOptions,
     ) -> Bank {
-        subscriptions.notify_slot(slot, parent.slot(), root_slot);
-        Bank::new_from_parent_with_vote_only(parent, leader, slot, vote_only_bank)
+        rpc_subscriptions.notify_slot(slot, parent.slot(), root_slot);
+        Bank::new_from_parent_with_options(parent, leader, slot, new_bank_options)
     }
 
     fn record_rewards(bank: &Bank, rewards_recorder_sender: &Option<RewardsRecorderSender>) {
@@ -2669,7 +2704,7 @@ mod tests {
         let optimistically_confirmed_bank =
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
         let exit = Arc::new(AtomicBool::new(false));
-        let rpc_subscriptions = Arc::new(RpcSubscriptions::new(
+        let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
             &exit,
             bank_forks.clone(),
             Arc::new(RwLock::new(BlockCommitmentCache::default())),
@@ -3172,8 +3207,7 @@ mod tests {
                 &replay_vote_sender,
                 &VerifyRecyclers::default(),
             );
-
-            let subscriptions = Arc::new(RpcSubscriptions::new(
+            let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
                 &exit,
                 bank_forks.clone(),
                 block_commitment_cache,
@@ -3185,7 +3219,7 @@ mod tests {
                     &bank0,
                     0,
                     err,
-                    &subscriptions,
+                    &rpc_subscriptions,
                     &mut DuplicateSlotsTracker::default(),
                     &GossipDuplicateConfirmedSlots::default(),
                     &mut progress,
@@ -3236,14 +3270,17 @@ mod tests {
 
         let exit = Arc::new(AtomicBool::new(false));
         let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
-        let subscriptions = Arc::new(RpcSubscriptions::new(
+        let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
             &exit,
             bank_forks.clone(),
             block_commitment_cache.clone(),
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         ));
-        let (lockouts_sender, _) =
-            AggregateCommitmentService::new(&exit, block_commitment_cache.clone(), subscriptions);
+        let (lockouts_sender, _) = AggregateCommitmentService::new(
+            &exit,
+            block_commitment_cache.clone(),
+            rpc_subscriptions,
+        );
 
         assert!(block_commitment_cache
             .read()
@@ -4711,7 +4748,12 @@ mod tests {
         let vote_info = voting_receiver
             .recv_timeout(Duration::from_secs(1))
             .unwrap();
-        crate::voting_service::VotingService::handle_vote(&cluster_info, &poh_recorder, vote_info);
+        crate::voting_service::VotingService::handle_vote(
+            &cluster_info,
+            &poh_recorder,
+            vote_info,
+            false,
+        );
 
         let mut cursor = Cursor::default();
         let (_, votes) = cluster_info.get_votes(&mut cursor);
@@ -4768,7 +4810,12 @@ mod tests {
         let vote_info = voting_receiver
             .recv_timeout(Duration::from_secs(1))
             .unwrap();
-        crate::voting_service::VotingService::handle_vote(&cluster_info, &poh_recorder, vote_info);
+        crate::voting_service::VotingService::handle_vote(
+            &cluster_info,
+            &poh_recorder,
+            vote_info,
+            false,
+        );
         let (_, votes) = cluster_info.get_votes(&mut cursor);
         assert_eq!(votes.len(), 1);
         let vote_tx = &votes[0];
@@ -4830,7 +4877,12 @@ mod tests {
         let vote_info = voting_receiver
             .recv_timeout(Duration::from_secs(1))
             .unwrap();
-        crate::voting_service::VotingService::handle_vote(&cluster_info, &poh_recorder, vote_info);
+        crate::voting_service::VotingService::handle_vote(
+            &cluster_info,
+            &poh_recorder,
+            vote_info,
+            false,
+        );
 
         assert!(last_vote_refresh_time.last_refresh_time > clone_refresh_time);
         let (_, votes) = cluster_info.get_votes(&mut cursor);
@@ -4901,7 +4953,6 @@ mod tests {
         );
         assert_eq!(tower.last_voted_slot().unwrap(), 1);
     }
-
     fn run_compute_and_select_forks(
         bank_forks: &RwLock<BankForks>,
         progress: &mut ProgressMap,

core/src/serve_repair.rs

@@ -738,7 +738,7 @@ mod tests {
                 repair: socketaddr!("127.0.0.1:1237"),
                 tpu: socketaddr!("127.0.0.1:1238"),
                 tpu_forwards: socketaddr!("127.0.0.1:1239"),
-                unused: socketaddr!("127.0.0.1:1240"),
+                tpu_vote: socketaddr!("127.0.0.1:1240"),
                 rpc: socketaddr!("127.0.0.1:1241"),
                 rpc_pubsub: socketaddr!("127.0.0.1:1242"),
                 serve_repair: socketaddr!("127.0.0.1:1243"),
@@ -825,7 +825,7 @@ mod tests {
             repair: socketaddr!([127, 0, 0, 1], 1237),
             tpu: socketaddr!([127, 0, 0, 1], 1238),
             tpu_forwards: socketaddr!([127, 0, 0, 1], 1239),
-            unused: socketaddr!([127, 0, 0, 1], 1240),
+            tpu_vote: socketaddr!([127, 0, 0, 1], 1240),
             rpc: socketaddr!([127, 0, 0, 1], 1241),
             rpc_pubsub: socketaddr!([127, 0, 0, 1], 1242),
             serve_repair: serve_repair_addr,
@@ -855,7 +855,7 @@ mod tests {
             repair: socketaddr!([127, 0, 0, 1], 1237),
             tpu: socketaddr!([127, 0, 0, 1], 1238),
             tpu_forwards: socketaddr!([127, 0, 0, 1], 1239),
-            unused: socketaddr!([127, 0, 0, 1], 1240),
+            tpu_vote: socketaddr!([127, 0, 0, 1], 1240),
             rpc: socketaddr!([127, 0, 0, 1], 1241),
             rpc_pubsub: socketaddr!([127, 0, 0, 1], 1242),
             serve_repair: serve_repair_addr2,

core/src/sigverify.rs

@@ -17,6 +17,16 @@ pub use solana_perf::sigverify::{
 pub struct TransactionSigVerifier {
     recycler: Recycler<TxOffset>,
     recycler_out: Recycler<PinnedVec<u8>>,
+    reject_non_vote: bool,
+}
+
+impl TransactionSigVerifier {
+    pub fn new_reject_non_vote() -> Self {
+        TransactionSigVerifier {
+            reject_non_vote: true,
+            ..TransactionSigVerifier::default()
+        }
+    }
 }
 
 impl Default for TransactionSigVerifier {
@@ -25,13 +35,19 @@ impl Default for TransactionSigVerifier {
         Self {
             recycler: Recycler::warmed(50, 4096),
             recycler_out: Recycler::warmed(50, 4096),
+            reject_non_vote: false,
         }
     }
 }
 
 impl SigVerifier for TransactionSigVerifier {
     fn verify_batch(&self, mut batch: Vec<Packets>) -> Vec<Packets> {
-        sigverify::ed25519_verify(&mut batch, &self.recycler, &self.recycler_out);
+        sigverify::ed25519_verify(
+            &mut batch,
+            &self.recycler,
+            &self.recycler_out,
+            self.reject_non_vote,
+        );
         batch
     }
 }

core/src/sigverify_stage.rs

@@ -8,18 +8,16 @@
 use crate::sigverify;
 use crossbeam_channel::{SendError, Sender as CrossbeamSender};
 use solana_measure::measure::Measure;
-use solana_metrics::datapoint_debug;
 use solana_perf::packet::Packets;
-use solana_perf::perf_libs;
 use solana_sdk::timing;
 use solana_streamer::streamer::{self, PacketReceiver, StreamerError};
+use std::collections::HashMap;
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
-use std::sync::{Arc, Mutex};
 use std::thread::{self, Builder, JoinHandle};
+use std::time::Instant;
 use thiserror::Error;
 
-const RECV_BATCH_MAX_CPU: usize = 1_000;
-const RECV_BATCH_MAX_GPU: usize = 5_000;
+const MAX_SIGVERIFY_BATCH: usize = 10_000;
 
 #[derive(Error, Debug)]
 pub enum SigVerifyServiceError {
@@ -33,7 +31,7 @@ pub enum SigVerifyServiceError {
 type Result<T> = std::result::Result<T, SigVerifyServiceError>;
 
 pub struct SigVerifyStage {
-    thread_hdls: Vec<JoinHandle<()>>,
+    thread_hdl: JoinHandle<()>,
 }
 
 pub trait SigVerifier {
@@ -43,6 +41,82 @@ pub trait SigVerifier {
 #[derive(Default, Clone)]
 pub struct DisabledSigVerifier {}
 
+#[derive(Default)]
+struct SigVerifierStats {
+    recv_batches_us_hist: histogram::Histogram, // time to call recv_batch
+    verify_batches_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch
+    batches_hist: histogram::Histogram,         // number of Packets structures per verify call
+    packets_hist: histogram::Histogram,         // number of packets per verify call
+    total_batches: usize,
+    total_packets: usize,
+}
+
+impl SigVerifierStats {
+    fn report(&self) {
+        datapoint_info!(
+            "sigverify_stage-total_verify_time",
+            (
+                "recv_batches_us_90pct",
+                self.recv_batches_us_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            (
+                "recv_batches_us_min",
+                self.recv_batches_us_hist.minimum().unwrap_or(0),
+                i64
+            ),
+            (
+                "recv_batches_us_max",
+                self.recv_batches_us_hist.maximum().unwrap_or(0),
+                i64
+            ),
+            (
+                "recv_batches_us_mean",
+                self.recv_batches_us_hist.mean().unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_90pct",
+                self.verify_batches_pp_us_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_min",
+                self.verify_batches_pp_us_hist.minimum().unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_max",
+                self.verify_batches_pp_us_hist.maximum().unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_mean",
+                self.verify_batches_pp_us_hist.mean().unwrap_or(0),
+                i64
+            ),
+            (
+                "batches_90pct",
+                self.batches_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            ("batches_min", self.batches_hist.minimum().unwrap_or(0), i64),
+            ("batches_max", self.batches_hist.maximum().unwrap_or(0), i64),
+            ("batches_mean", self.batches_hist.mean().unwrap_or(0), i64),
+            (
+                "packets_90pct",
+                self.packets_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            ("packets_min", self.packets_hist.minimum().unwrap_or(0), i64),
+            ("packets_max", self.packets_hist.maximum().unwrap_or(0), i64),
+            ("packets_mean", self.packets_hist.mean().unwrap_or(0), i64),
+            ("total_batches", self.total_batches, i64),
+            ("total_packets", self.total_packets, i64),
+        );
+    }
+}
+
 impl SigVerifier for DisabledSigVerifier {
     fn verify_batch(&self, mut batch: Vec<Packets>) -> Vec<Packets> {
         sigverify::ed25519_verify_disabled(&mut batch);
@@ -57,74 +131,103 @@ impl SigVerifyStage {
         verified_sender: CrossbeamSender<Vec<Packets>>,
         verifier: T,
     ) -> Self {
-        let thread_hdls = Self::verifier_services(packet_receiver, verified_sender, verifier);
-        Self { thread_hdls }
+        let thread_hdl = Self::verifier_services(packet_receiver, verified_sender, verifier);
+        Self { thread_hdl }
+    }
+
+    pub fn discard_excess_packets(batches: &mut Vec<Packets>, max_packets: usize) {
+        let mut received_ips = HashMap::new();
+        for (batch_index, batch) in batches.iter().enumerate() {
+            for (packet_index, packets) in batch.packets.iter().enumerate() {
+                let e = received_ips
+                    .entry(packets.meta.addr().ip())
+                    .or_insert_with(Vec::new);
+                e.push((batch_index, packet_index));
+            }
+        }
+        let mut batch_len = 0;
+        while batch_len < max_packets {
+            for (_ip, indexes) in received_ips.iter_mut() {
+                if !indexes.is_empty() {
+                    indexes.remove(0);
+                    batch_len += 1;
+                    if batch_len >= MAX_SIGVERIFY_BATCH {
+                        break;
+                    }
+                }
+            }
+        }
+        for (_addr, indexes) in received_ips {
+            for (batch_index, packet_index) in indexes {
+                batches[batch_index].packets[packet_index].meta.discard = true;
+            }
+        }
     }
 
     fn verifier<T: SigVerifier>(
-        recvr: &Arc<Mutex<PacketReceiver>>,
+        recvr: &PacketReceiver,
         sendr: &CrossbeamSender<Vec<Packets>>,
-        id: usize,
         verifier: &T,
+        stats: &mut SigVerifierStats,
     ) -> Result<()> {
-        let (batch, len, recv_time) = streamer::recv_batch(
-            &recvr.lock().expect("'recvr' lock in fn verifier"),
-            if perf_libs::api().is_some() {
-                RECV_BATCH_MAX_GPU
-            } else {
-                RECV_BATCH_MAX_CPU
-            },
-        )?;
+        let (mut batches, len, recv_time) = streamer::recv_batch(recvr)?;
 
         let mut verify_batch_time = Measure::start("sigverify_batch_time");
-        let batch_len = batch.len();
-        debug!(
-            "@{:?} verifier: verifying: {} id: {}",
-            timing::timestamp(),
-            len,
-            id
-        );
-
-        let verified_batch = verifier.verify_batch(batch);
-
-        for v in verified_batch {
-            sendr.send(vec![v])?;
+        let batches_len = batches.len();
+        debug!("@{:?} verifier: verifying: {}", timing::timestamp(), len,);
+        if len > MAX_SIGVERIFY_BATCH {
+            Self::discard_excess_packets(&mut batches, MAX_SIGVERIFY_BATCH);
         }
-
+        sendr.send(verifier.verify_batch(batches))?;
         verify_batch_time.stop();
 
         debug!(
-            "@{:?} verifier: done. batches: {} total verify time: {:?} id: {} verified: {} v/s {}",
+            "@{:?} verifier: done. batches: {} total verify time: {:?} verified: {} v/s {}",
             timing::timestamp(),
-            batch_len,
+            batches_len,
             verify_batch_time.as_ms(),
-            id,
             len,
             (len as f32 / verify_batch_time.as_s())
         );
 
         datapoint_debug!(
             "sigverify_stage-total_verify_time",
-            ("num_batches", batch_len, i64),
+            ("num_batches", batches_len, i64),
             ("num_packets", len, i64),
             ("verify_time_ms", verify_batch_time.as_ms(), i64),
             ("recv_time", recv_time, i64),
         );
 
+        stats
+            .recv_batches_us_hist
+            .increment(recv_time as u64)
+            .unwrap();
+        stats
+            .verify_batches_pp_us_hist
+            .increment(verify_batch_time.as_us() / (len as u64))
+            .unwrap();
+        stats.batches_hist.increment(batches_len as u64).unwrap();
+        stats.packets_hist.increment(len as u64).unwrap();
+        stats.total_batches += batches_len;
+        stats.total_packets += len;
+
         Ok(())
     }
 
     fn verifier_service<T: SigVerifier + 'static + Send + Clone>(
-        packet_receiver: Arc<Mutex<PacketReceiver>>,
+        packet_receiver: PacketReceiver,
         verified_sender: CrossbeamSender<Vec<Packets>>,
-        id: usize,
         verifier: &T,
     ) -> JoinHandle<()> {
         let verifier = verifier.clone();
+        let mut stats = SigVerifierStats::default();
+        let mut last_print = Instant::now();
         Builder::new()
-            .name(format!("solana-verifier-{}", id))
+            .name("solana-verifier".to_string())
             .spawn(move || loop {
-                if let Err(e) = Self::verifier(&packet_receiver, &verified_sender, id, &verifier) {
+                if let Err(e) =
+                    Self::verifier(&packet_receiver, &verified_sender, &verifier, &mut stats)
+                {
                     match e {
                         SigVerifyServiceError::Streamer(StreamerError::RecvTimeout(
                             RecvTimeoutError::Disconnected,
@@ -138,6 +241,11 @@ impl SigVerifyStage {
                         _ => error!("{:?}", e),
                     }
                 }
+                if last_print.elapsed().as_secs() > 2 {
+                    stats.report();
+                    stats = SigVerifierStats::default();
+                    last_print = Instant::now();
+                }
             })
             .unwrap()
     }
@@ -146,19 +254,43 @@ impl SigVerifyStage {
         packet_receiver: PacketReceiver,
         verified_sender: CrossbeamSender<Vec<Packets>>,
         verifier: T,
-    ) -> Vec<JoinHandle<()>> {
-        let receiver = Arc::new(Mutex::new(packet_receiver));
-        (0..4)
-            .map(|id| {
-                Self::verifier_service(receiver.clone(), verified_sender.clone(), id, &verifier)
-            })
-            .collect()
+    ) -> JoinHandle<()> {
+        Self::verifier_service(packet_receiver, verified_sender, &verifier)
     }
 
     pub fn join(self) -> thread::Result<()> {
-        for thread_hdl in self.thread_hdls {
-            thread_hdl.join()?;
-        }
-        Ok(())
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_perf::packet::Packet;
+
+    fn count_non_discard(packets: &[Packets]) -> usize {
+        packets
+            .iter()
+            .map(|pp| {
+                pp.packets
+                    .iter()
+                    .map(|p| if p.meta.discard { 0 } else { 1 })
+                    .sum::<usize>()
+            })
+            .sum::<usize>()
+    }
+
+    #[test]
+    fn test_packet_discard() {
+        solana_logger::setup();
+        let mut p = Packets::default();
+        p.packets.resize(10, Packet::default());
+        p.packets[3].meta.addr = [1u16; 8];
+        let mut packets = vec![p];
+        let max = 3;
+        SigVerifyStage::discard_excess_packets(&mut packets, max);
+        assert_eq!(count_non_discard(&packets), max);
+        assert!(!packets[0].packets[0].meta.discard);
+        assert!(!packets[0].packets[3].meta.discard);
     }
 }

core/src/system_monitor_service.rs

@@ -0,0 +1,227 @@
+use std::{
+    collections::HashMap,
+    io::BufRead,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::{self, sleep, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+#[cfg(target_os = "linux")]
+use std::{fs::File, io::BufReader, path::Path};
+
+const SAMPLE_INTERVAL: Duration = Duration::from_secs(60);
+const SLEEP_INTERVAL: Duration = Duration::from_millis(500);
+
+#[cfg(target_os = "linux")]
+const PROC_NET_SNMP_PATH: &str = "/proc/net/snmp";
+
+pub struct SystemMonitorService {
+    thread_hdl: JoinHandle<()>,
+}
+
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+struct UdpStats {
+    in_datagrams: usize,
+    no_ports: usize,
+    in_errors: usize,
+    out_datagrams: usize,
+    rcvbuf_errors: usize,
+    sndbuf_errors: usize,
+    in_csum_errors: usize,
+    ignored_multi: usize,
+}
+
+#[cfg(target_os = "linux")]
+fn read_udp_stats(file_path: impl AsRef<Path>) -> Result<UdpStats, String> {
+    let file = File::open(file_path).map_err(|e| e.to_string())?;
+    let mut reader = BufReader::new(file);
+    parse_udp_stats(&mut reader)
+}
+
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+fn parse_udp_stats(reader: &mut impl BufRead) -> Result<UdpStats, String> {
+    let mut udp_lines = Vec::default();
+    for line in reader.lines() {
+        let line = line.map_err(|e| e.to_string())?;
+        if line.starts_with("Udp:") {
+            udp_lines.push(line);
+            if udp_lines.len() == 2 {
+                break;
+            }
+        }
+    }
+    if udp_lines.len() != 2 {
+        return Err(format!(
+            "parse error, expected 2 lines, num lines: {}",
+            udp_lines.len()
+        ));
+    }
+
+    let pairs: Vec<_> = udp_lines[0]
+        .split_ascii_whitespace()
+        .zip(udp_lines[1].split_ascii_whitespace())
+        .collect();
+    let udp_stats: HashMap<String, usize> = pairs[1..]
+        .iter()
+        .map(|(label, val)| (label.to_string(), val.parse::<usize>().unwrap()))
+        .collect();
+
+    let stats = UdpStats {
+        in_datagrams: *udp_stats.get("InDatagrams").unwrap_or(&0),
+        no_ports: *udp_stats.get("NoPorts").unwrap_or(&0),
+        in_errors: *udp_stats.get("InErrors").unwrap_or(&0),
+        out_datagrams: *udp_stats.get("OutDatagrams").unwrap_or(&0),
+        rcvbuf_errors: *udp_stats.get("RcvbufErrors").unwrap_or(&0),
+        sndbuf_errors: *udp_stats.get("SndbufErrors").unwrap_or(&0),
+        in_csum_errors: *udp_stats.get("InCsumErrors").unwrap_or(&0),
+        ignored_multi: *udp_stats.get("IgnoredMulti").unwrap_or(&0),
+    };
+
+    Ok(stats)
+}
+
+#[cfg(target_os = "linux")]
+pub fn verify_udp_stats_access() -> Result<(), String> {
+    read_udp_stats(PROC_NET_SNMP_PATH)?;
+    Ok(())
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn verify_udp_stats_access() -> Result<(), String> {
+    Ok(())
+}
+
+impl SystemMonitorService {
+    pub fn new(exit: Arc<AtomicBool>) -> Self {
+        info!("Starting SystemMonitorService");
+        let thread_hdl = Builder::new()
+            .name("system-monitor".to_string())
+            .spawn(move || {
+                Self::run(exit);
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+
+    #[cfg(target_os = "linux")]
+    fn process_udp_stats(udp_stats: &mut Option<UdpStats>) {
+        match read_udp_stats(PROC_NET_SNMP_PATH) {
+            Ok(new_stats) => {
+                if let Some(old_stats) = udp_stats {
+                    SystemMonitorService::report_udp_stats(old_stats, &new_stats);
+                }
+                *udp_stats = Some(new_stats);
+            }
+            Err(e) => warn!("read_udp_stats: {}", e),
+        }
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn process_udp_stats(_udp_stats: &mut Option<UdpStats>) {}
+
+    #[cfg(target_os = "linux")]
+    fn report_udp_stats(old_stats: &UdpStats, new_stats: &UdpStats) {
+        datapoint_info!(
+            "net-stats",
+            (
+                "in_datagrams_delta",
+                new_stats.in_datagrams - old_stats.in_datagrams,
+                i64
+            ),
+            (
+                "no_ports_delta",
+                new_stats.no_ports - old_stats.no_ports,
+                i64
+            ),
+            (
+                "in_errors_delta",
+                new_stats.in_errors - old_stats.in_errors,
+                i64
+            ),
+            (
+                "out_datagrams_delta",
+                new_stats.out_datagrams - old_stats.out_datagrams,
+                i64
+            ),
+            (
+                "rcvbuf_errors_delta",
+                new_stats.rcvbuf_errors - old_stats.rcvbuf_errors,
+                i64
+            ),
+            (
+                "sndbuf_errors_delta",
+                new_stats.sndbuf_errors - old_stats.sndbuf_errors,
+                i64
+            ),
+            (
+                "in_csum_errors_delta",
+                new_stats.in_csum_errors - old_stats.in_csum_errors,
+                i64
+            ),
+            (
+                "ignored_multi_delta",
+                new_stats.ignored_multi - old_stats.ignored_multi,
+                i64
+            ),
+            ("in_errors", new_stats.in_errors, i64),
+            ("rcvbuf_errors", new_stats.rcvbuf_errors, i64),
+            ("sndbuf_errors", new_stats.sndbuf_errors, i64),
+        );
+    }
+
+    pub fn run(exit: Arc<AtomicBool>) {
+        let mut udp_stats = None;
+
+        let mut now = Instant::now();
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            if now.elapsed() >= SAMPLE_INTERVAL {
+                now = Instant::now();
+
+                SystemMonitorService::process_udp_stats(&mut udp_stats);
+            }
+
+            sleep(SLEEP_INTERVAL);
+        }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_udp_stats() {
+        let mut mock_snmp =
+b"Ip: Forwarding DefaultTTL InReceives InHdrErrors InAddrErrors ForwDatagrams InUnknownProtos InDiscards InDelivers OutRequests OutDiscards OutNoRoutes ReasmTimeout ReasmReqds ReasmOKs ReasmFails FragOKs FragFails FragCreates
+Ip: 1 64 357 0 2 0 0 0 355 315 0 6 0 0 0 0 0 0 0
+Icmp: InMsgs InErrors InCsumErrors InDestUnreachs InTimeExcds InParmProbs InSrcQuenchs InRedirects InEchos InEchoReps InTimestamps InTimestampReps InAddrMasks InAddrMaskReps OutMsgs OutErrors OutDestUnreachs OutTimeExcds OutParmProbs OutSrcQuenchs OutRedirects OutEchos OutEchoReps OutTimestamps OutTimestampReps OutAddrMasks OutAddrMaskReps
+Icmp: 3 0 0 3 0 0 0 0 0 0 0 0 0 0 7 0 7 0 0 0 0 0 0 0 0 0 0
+IcmpMsg: InType3 OutType3
+IcmpMsg: 3 7
+Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors
+Tcp: 1 200 120000 -1 29 1 0 0 5 318 279 0 0 4 0
+Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+Udp: 27 7 0 30 0 0 0 0
+UdpLite: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+UdpLite: 0 0 0 0 0 0 0 0" as &[u8];
+        let stats = parse_udp_stats(&mut mock_snmp).unwrap();
+        assert_eq!(stats.out_datagrams, 30);
+        assert_eq!(stats.no_ports, 7);
+
+        let mut mock_snmp = b"unexpected data" as &[u8];
+        let stats = parse_udp_stats(&mut mock_snmp);
+        assert!(stats.is_err());
+    }
+}

core/src/tpu.rs

@@ -22,6 +22,7 @@ use solana_rpc::{
 };
 use solana_runtime::{
     bank_forks::BankForks,
+    cost_model::CostModel,
     vote_sender_types::{ReplayVoteReceiver, ReplayVoteSender},
 };
 use std::{
@@ -39,6 +40,7 @@ pub const DEFAULT_TPU_COALESCE_MS: u64 = 5;
 pub struct Tpu {
     fetch_stage: FetchStage,
     sigverify_stage: SigVerifyStage,
+    vote_sigverify_stage: SigVerifyStage,
     banking_stage: BankingStage,
     cluster_info_vote_listener: ClusterInfoVoteListener,
     broadcast_stage: BroadcastStage,
@@ -53,6 +55,7 @@ impl Tpu {
         retransmit_slots_receiver: RetransmitSlotsReceiver,
         transactions_sockets: Vec<UdpSocket>,
         tpu_forwards_sockets: Vec<UdpSocket>,
+        tpu_vote_sockets: Vec<UdpSocket>,
         broadcast_sockets: Vec<UdpSocket>,
         subscriptions: &Arc<RpcSubscriptions>,
         transaction_status_sender: Option<TransactionStatusSender>,
@@ -69,13 +72,17 @@ impl Tpu {
         bank_notification_sender: Option<BankNotificationSender>,
         tpu_coalesce_ms: u64,
         cluster_confirmed_slot_sender: GossipDuplicateConfirmedSlotsSender,
+        cost_model: &Arc<RwLock<CostModel>>,
     ) -> Self {
         let (packet_sender, packet_receiver) = channel();
+        let (vote_packet_sender, vote_packet_receiver) = channel();
         let fetch_stage = FetchStage::new_with_sender(
             transactions_sockets,
             tpu_forwards_sockets,
+            tpu_vote_sockets,
             exit,
             &packet_sender,
+            &vote_packet_sender,
             poh_recorder,
             tpu_coalesce_ms,
         );
@@ -86,14 +93,26 @@ impl Tpu {
             SigVerifyStage::new(packet_receiver, verified_sender, verifier)
         };
 
-        let (verified_vote_packets_sender, verified_vote_packets_receiver) = unbounded();
+        let (verified_tpu_vote_packets_sender, verified_tpu_vote_packets_receiver) = unbounded();
+
+        let vote_sigverify_stage = {
+            let verifier = TransactionSigVerifier::new_reject_non_vote();
+            SigVerifyStage::new(
+                vote_packet_receiver,
+                verified_tpu_vote_packets_sender,
+                verifier,
+            )
+        };
+
+        let (verified_gossip_vote_packets_sender, verified_gossip_vote_packets_receiver) =
+            unbounded();
         let cluster_info_vote_listener = ClusterInfoVoteListener::new(
             exit,
             cluster_info.clone(),
-            verified_vote_packets_sender,
+            verified_gossip_vote_packets_sender,
             poh_recorder,
             vote_tracker,
-            bank_forks,
+            bank_forks.clone(),
             subscriptions.clone(),
             verified_vote_sender,
             gossip_verified_vote_hash_sender,
@@ -107,9 +126,11 @@ impl Tpu {
             cluster_info,
             poh_recorder,
             verified_receiver,
-            verified_vote_packets_receiver,
+            verified_tpu_vote_packets_receiver,
+            verified_gossip_vote_packets_receiver,
             transaction_status_sender,
             replay_vote_sender,
+            cost_model.clone(),
         );
 
         let broadcast_stage = broadcast_type.new_broadcast_stage(
@@ -119,12 +140,14 @@ impl Tpu {
             retransmit_slots_receiver,
             exit,
             blockstore,
+            &bank_forks,
             shred_version,
         );
 
         Self {
             fetch_stage,
             sigverify_stage,
+            vote_sigverify_stage,
             banking_stage,
             cluster_info_vote_listener,
             broadcast_stage,
@@ -135,6 +158,7 @@ impl Tpu {
         let results = vec![
             self.fetch_stage.join(),
             self.sigverify_stage.join(),
+            self.vote_sigverify_stage.join(),
             self.cluster_info_vote_listener.join(),
             self.banking_stage.join(),
         ];

core/src/tvu.rs

@@ -12,6 +12,7 @@ use crate::{
     cluster_slots::ClusterSlots,
     completed_data_sets_service::CompletedDataSetsSender,
     consensus::Tower,
+    cost_update_service::CostUpdateService,
     ledger_cleanup_service::LedgerCleanupService,
     replay_stage::{ReplayStage, ReplayStageConfig},
     retransmit_stage::RetransmitStage,
@@ -25,8 +26,7 @@ use crate::{
 use crossbeam_channel::unbounded;
 use solana_gossip::cluster_info::ClusterInfo;
 use solana_ledger::{
-    blockstore::{Blockstore, CompletedSlotsReceiver},
-    blockstore_processor::TransactionStatusSender,
+    blockstore::Blockstore, blockstore_processor::TransactionStatusSender,
     leader_schedule_cache::LeaderScheduleCache,
 };
 use solana_poh::poh_recorder::PohRecorder;
@@ -41,6 +41,7 @@ use solana_runtime::{
     accounts_db::AccountShrinkThreshold,
     bank_forks::{BankForks, SnapshotConfig},
     commitment::BlockCommitmentCache,
+    cost_model::CostModel,
     vote_sender_types::ReplayVoteSender,
 };
 use solana_sdk::{
@@ -68,6 +69,7 @@ pub struct Tvu {
     accounts_background_service: AccountsBackgroundService,
     accounts_hash_verifier: AccountsHashVerifier,
     voting_service: VotingService,
+    cost_update_service: CostUpdateService,
 }
 
 pub struct Sockets {
@@ -92,6 +94,7 @@ pub struct TvuConfig {
     pub rocksdb_max_compaction_jitter: Option<u64>,
     pub wait_for_vote_to_start_leader: bool,
     pub accounts_shrink_ratio: AccountShrinkThreshold,
+    pub disable_epoch_boundary_optimization: bool,
 }
 
 impl Tvu {
@@ -110,12 +113,11 @@ impl Tvu {
         sockets: Sockets,
         blockstore: Arc<Blockstore>,
         ledger_signal_receiver: Receiver<bool>,
-        subscriptions: &Arc<RpcSubscriptions>,
+        rpc_subscriptions: &Arc<RpcSubscriptions>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         tower: Tower,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         exit: &Arc<AtomicBool>,
-        completed_slots_receiver: CompletedSlotsReceiver,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         cfg: Option<Arc<AtomicBool>>,
         transaction_status_sender: Option<TransactionStatusSender>,
@@ -132,6 +134,7 @@ impl Tvu {
         gossip_confirmed_slots_receiver: GossipDuplicateConfirmedSlotsReceiver,
         tvu_config: TvuConfig,
         max_slots: &Arc<MaxSlots>,
+        cost_model: &Arc<RwLock<CostModel>>,
     ) -> Self {
         let keypair: Arc<Keypair> = cluster_info.keypair.clone();
 
@@ -172,14 +175,13 @@ impl Tvu {
         let (cluster_slots_update_sender, cluster_slots_update_receiver) = unbounded();
         let retransmit_stage = RetransmitStage::new(
             bank_forks.clone(),
-            leader_schedule_cache,
+            leader_schedule_cache.clone(),
             blockstore.clone(),
-            cluster_info,
+            cluster_info.clone(),
             Arc::new(retransmit_sockets),
             repair_socket,
             verified_receiver,
-            &exit,
-            completed_slots_receiver,
+            exit.clone(),
             cluster_slots_update_receiver,
             *bank_forks.read().unwrap().working_bank().epoch_schedule(),
             cfg,
@@ -189,8 +191,8 @@ impl Tvu {
             verified_vote_receiver,
             tvu_config.repair_validators,
             completed_data_sets_sender,
-            max_slots,
-            Some(subscriptions.clone()),
+            max_slots.clone(),
+            Some(rpc_subscriptions.clone()),
             duplicate_slots_sender,
         );
 
@@ -266,7 +268,7 @@ impl Tvu {
             vote_account: *vote_account,
             authorized_voter_keypairs,
             exit: exit.clone(),
-            subscriptions: subscriptions.clone(),
+            rpc_subscriptions: rpc_subscriptions.clone(),
             leader_schedule_cache: leader_schedule_cache.clone(),
             latest_root_senders: vec![ledger_cleanup_slot_sender],
             accounts_background_request_sender,
@@ -276,11 +278,24 @@ impl Tvu {
             cache_block_meta_sender,
             bank_notification_sender,
             wait_for_vote_to_start_leader: tvu_config.wait_for_vote_to_start_leader,
+            disable_epoch_boundary_optimization: tvu_config.disable_epoch_boundary_optimization,
         };
 
         let (voting_sender, voting_receiver) = channel();
-        let voting_service =
-            VotingService::new(voting_receiver, cluster_info.clone(), poh_recorder.clone());
+        let voting_service = VotingService::new(
+            voting_receiver,
+            cluster_info.clone(),
+            poh_recorder.clone(),
+            bank_forks.clone(),
+        );
+
+        let (cost_update_sender, cost_update_receiver) = channel();
+        let cost_update_service = CostUpdateService::new(
+            exit.clone(),
+            blockstore.clone(),
+            cost_model.clone(),
+            cost_update_receiver,
+        );
 
         let replay_stage = ReplayStage::new(
             replay_stage_config,
@@ -300,6 +315,7 @@ impl Tvu {
             gossip_verified_vote_hash_receiver,
             cluster_slots_update_sender,
             voting_sender,
+            cost_update_sender,
         );
 
         let ledger_cleanup_service = tvu_config.max_ledger_shreds.map(|max_ledger_shreds| {
@@ -331,6 +347,7 @@ impl Tvu {
             accounts_background_service,
             accounts_hash_verifier,
             voting_service,
+            cost_update_service,
         }
     }
 
@@ -345,6 +362,7 @@ impl Tvu {
         self.replay_stage.join()?;
         self.accounts_hash_verifier.join()?;
         self.voting_service.join()?;
+        self.cost_update_service.join()?;
         Ok(())
     }
 }
@@ -393,7 +411,6 @@ pub mod tests {
         let BlockstoreSignals {
             blockstore,
             ledger_signal_receiver,
-            completed_slots_receiver,
             ..
         } = Blockstore::open_with_signal(&blockstore_path, None, true)
             .expect("Expected to successfully open ledger");
@@ -427,7 +444,7 @@ pub mod tests {
             },
             blockstore,
             ledger_signal_receiver,
-            &Arc::new(RpcSubscriptions::new(
+            &Arc::new(RpcSubscriptions::new_for_tests(
                 &exit,
                 bank_forks.clone(),
                 block_commitment_cache.clone(),
@@ -437,7 +454,6 @@ pub mod tests {
             tower,
             &leader_schedule_cache,
             &exit,
-            completed_slots_receiver,
             block_commitment_cache,
             None,
             None,
@@ -454,6 +470,7 @@ pub mod tests {
             gossip_confirmed_slots_receiver,
             TvuConfig::default(),
             &Arc::new(MaxSlots::default()),
+            &Arc::new(RwLock::new(CostModel::default())),
         );
         exit.store(true, Ordering::Relaxed);
         tvu.join().unwrap();

core/src/validator.rs

@@ -1,5 +1,6 @@
 //! The `validator` module hosts all the validator microservices.
 
+pub use solana_perf::report_target_features;
 use {
     crate::{
         broadcast_stage::BroadcastStageType,
@@ -13,11 +14,13 @@ use {
         serve_repair_service::ServeRepairService,
         sigverify,
         snapshot_packager_service::{PendingSnapshotPackage, SnapshotPackagerService},
+        system_monitor_service::{verify_udp_stats_access, SystemMonitorService},
         tpu::{Tpu, DEFAULT_TPU_COALESCE_MS},
         tvu::{Sockets, Tvu, TvuConfig},
     },
     crossbeam_channel::{bounded, unbounded},
     rand::{thread_rng, Rng},
+    solana_accountsdb_plugin_manager::accountsdb_plugin_service::AccountsDbPluginService,
     solana_gossip::{
         cluster_info::{
             ClusterInfo, Node, DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
@@ -42,12 +45,14 @@ use {
         poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
         poh_service::{self, PohService},
     },
+    solana_rpc::send_transaction_service,
     solana_rpc::{
         max_slots::MaxSlots,
         optimistically_confirmed_bank_tracker::{
             OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker,
         },
         rpc::JsonRpcConfig,
+        rpc_completed_slots_service::RpcCompletedSlotsService,
         rpc_pubsub_service::{PubSubConfig, PubSubService},
         rpc_service::JsonRpcService,
         rpc_subscriptions::RpcSubscriptions,
@@ -56,9 +61,11 @@ use {
     solana_runtime::{
         accounts_db::AccountShrinkThreshold,
         accounts_index::AccountSecondaryIndexes,
+        accounts_update_notifier_interface::AccountsUpdateNotifier,
         bank::Bank,
         bank_forks::{BankForks, SnapshotConfig},
         commitment::BlockCommitmentCache,
+        cost_model::CostModel,
         hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     },
     solana_sdk::{
@@ -84,7 +91,7 @@ use {
             mpsc::Receiver,
             Arc, Mutex, RwLock,
         },
-        thread::{sleep, Builder},
+        thread::{sleep, Builder, JoinHandle},
         time::{Duration, Instant},
     },
 };
@@ -102,6 +109,7 @@ pub struct ValidatorConfig {
     pub account_paths: Vec<PathBuf>,
     pub account_shrink_paths: Option<Vec<PathBuf>>,
     pub rpc_config: JsonRpcConfig,
+    pub accountsdb_plugin_config_files: Option<Vec<PathBuf>>,
     pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub)
     pub pubsub_config: PubSubConfig,
     pub snapshot_config: Option<SnapshotConfig>,
@@ -131,8 +139,7 @@ pub struct ValidatorConfig {
     pub contact_debug_interval: u64,
     pub contact_save_interval: u64,
     pub bpf_jit: bool,
-    pub send_transaction_retry_ms: u64,
-    pub send_transaction_leader_forward_count: u64,
+    pub send_transaction_service_config: send_transaction_service::Config,
     pub no_poh_speed_test: bool,
     pub poh_pinned_cpu_core: usize,
     pub poh_hashes_per_batch: u64,
@@ -146,6 +153,7 @@ pub struct ValidatorConfig {
     pub validator_exit: Arc<RwLock<Exit>>,
     pub no_wait_for_vote_to_start_leader: bool,
     pub accounts_shrink_ratio: AccountShrinkThreshold,
+    pub disable_epoch_boundary_optimization: bool,
 }
 
 impl Default for ValidatorConfig {
@@ -160,6 +168,7 @@ impl Default for ValidatorConfig {
             account_paths: Vec::new(),
             account_shrink_paths: None,
             rpc_config: JsonRpcConfig::default(),
+            accountsdb_plugin_config_files: None,
             rpc_addrs: None,
             pubsub_config: PubSubConfig::default(),
             snapshot_config: None,
@@ -188,8 +197,7 @@ impl Default for ValidatorConfig {
             contact_debug_interval: DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
             contact_save_interval: DEFAULT_CONTACT_SAVE_INTERVAL_MILLIS,
             bpf_jit: false,
-            send_transaction_retry_ms: 2000,
-            send_transaction_leader_forward_count: 2,
+            send_transaction_service_config: send_transaction_service::Config::default(),
             no_poh_speed_test: true,
             poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE,
             poh_hashes_per_batch: poh_service::DEFAULT_HASHES_PER_BATCH,
@@ -203,6 +211,7 @@ impl Default for ValidatorConfig {
             validator_exit: Arc::new(RwLock::new(Exit::default())),
             no_wait_for_vote_to_start_leader: true,
             accounts_shrink_ratio: AccountShrinkThreshold::default(),
+            disable_epoch_boundary_optimization: false,
         }
     }
 }
@@ -248,10 +257,12 @@ pub struct Validator {
     validator_exit: Arc<RwLock<Exit>>,
     json_rpc_service: Option<JsonRpcService>,
     pubsub_service: Option<PubSubService>,
+    rpc_completed_slots_service: JoinHandle<()>,
     optimistically_confirmed_bank_tracker: Option<OptimisticallyConfirmedBankTracker>,
     transaction_status_service: Option<TransactionStatusService>,
     rewards_recorder_service: Option<RewardsRecorderService>,
     cache_block_meta_service: Option<CacheBlockMetaService>,
+    system_monitor_service: Option<SystemMonitorService>,
     sample_performance_service: Option<SamplePerformanceService>,
     gossip_service: GossipService,
     serve_repair_service: ServeRepairService,
@@ -262,6 +273,7 @@ pub struct Validator {
     tpu: Tpu,
     tvu: Tvu,
     ip_echo_server: Option<solana_net_utils::IpEchoServer>,
+    accountsdb_plugin_service: Option<AccountsDbPluginService>,
 }
 
 // in the distant future, get rid of ::new()/exit() and use Result properly...
@@ -298,6 +310,27 @@ impl Validator {
         warn!("identity: {}", id);
         warn!("vote account: {}", vote_account);
 
+        let mut bank_notification_senders = Vec::new();
+
+        let accountsdb_plugin_service =
+            if let Some(accountsdb_plugin_config_files) = &config.accountsdb_plugin_config_files {
+                let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded();
+                bank_notification_senders.push(confirmed_bank_sender);
+                let result = AccountsDbPluginService::new(
+                    confirmed_bank_receiver,
+                    accountsdb_plugin_config_files,
+                );
+                match result {
+                    Ok(accountsdb_plugin_service) => Some(accountsdb_plugin_service),
+                    Err(err) => {
+                        error!("Failed to load the AccountsDb plugin: {:?}", err);
+                        abort();
+                    }
+                }
+            } else {
+                None
+            };
+
         if config.voting_disabled {
             warn!("voting disabled");
             authorized_voter_keypairs.write().unwrap().clear();
@@ -391,10 +424,19 @@ impl Validator {
             config.enforce_ulimit_nofile,
             &start_progress,
             config.no_poh_speed_test,
+            accountsdb_plugin_service
+                .as_ref()
+                .map(|plugin_service| plugin_service.get_accounts_update_notifier()),
         );
 
         *start_progress.write().unwrap() = ValidatorStartProgress::StartingServices;
 
+        verify_udp_stats_access().unwrap_or_else(|err| {
+            error!("Failed to access UDP stats: {}", err);
+            abort();
+        });
+        let system_monitor_service = Some(SystemMonitorService::new(Arc::clone(&exit)));
+
         let leader_schedule_cache = Arc::new(leader_schedule_cache);
         let bank = bank_forks.working_bank();
         if let Some(ref shrink_paths) = config.account_shrink_paths {
@@ -455,12 +497,12 @@ impl Validator {
         let optimistically_confirmed_bank =
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
 
-        let subscriptions = Arc::new(RpcSubscriptions::new_with_vote_subscription(
+        let rpc_subscriptions = Arc::new(RpcSubscriptions::new_with_config(
             &exit,
             bank_forks.clone(),
             block_commitment_cache.clone(),
             optimistically_confirmed_bank.clone(),
-            config.pubsub_config.enable_vote_subscription,
+            &config.pubsub_config,
         ));
 
         let max_slots = Arc::new(MaxSlots::default());
@@ -469,7 +511,7 @@ impl Validator {
         let completed_data_sets_service = CompletedDataSetsService::new(
             completed_data_sets_receiver,
             blockstore.clone(),
-            subscriptions.clone(),
+            rpc_subscriptions.clone(),
             &exit,
             max_slots.clone(),
         );
@@ -528,6 +570,11 @@ impl Validator {
                 ));
             }
             let (bank_notification_sender, bank_notification_receiver) = unbounded();
+            let confirmed_bank_subscribers = if !bank_notification_senders.is_empty() {
+                Some(Arc::new(RwLock::new(bank_notification_senders)))
+            } else {
+                None
+            };
             (
                 Some(JsonRpcService::new(
                     rpc_addr,
@@ -544,8 +591,7 @@ impl Validator {
                     config.trusted_validators.clone(),
                     rpc_override_health_check.clone(),
                     optimistically_confirmed_bank.clone(),
-                    config.send_transaction_retry_ms,
-                    config.send_transaction_leader_forward_count,
+                    config.send_transaction_service_config.clone(),
                     max_slots.clone(),
                     leader_schedule_cache.clone(),
                     max_complete_transaction_status_slot,
@@ -553,19 +599,26 @@ impl Validator {
                 if config.rpc_config.minimal_api {
                     None
                 } else {
-                    Some(PubSubService::new(
+                    let (trigger, pubsub_service) = PubSubService::new(
                         config.pubsub_config.clone(),
-                        &subscriptions,
+                        &rpc_subscriptions,
                         rpc_pubsub_addr,
-                        &exit,
-                    ))
+                    );
+                    config
+                        .validator_exit
+                        .write()
+                        .unwrap()
+                        .register_exit(Box::new(move || trigger.cancel()));
+
+                    Some(pubsub_service)
                 },
                 Some(OptimisticallyConfirmedBankTracker::new(
                     bank_notification_receiver,
                     &exit,
                     bank_forks.clone(),
                     optimistically_confirmed_bank,
-                    subscriptions.clone(),
+                    rpc_subscriptions.clone(),
+                    confirmed_bank_subscribers,
                 )),
                 Some(bank_notification_sender),
             )
@@ -671,10 +724,18 @@ impl Validator {
             bank_forks.read().unwrap().root_bank().deref(),
         ));
 
+        let mut cost_model = CostModel::default();
+        cost_model.initialize_cost_table(&blockstore.read_program_costs().unwrap());
+        let cost_model = Arc::new(RwLock::new(cost_model));
+
         let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
         let (verified_vote_sender, verified_vote_receiver) = unbounded();
         let (gossip_verified_vote_hash_sender, gossip_verified_vote_hash_receiver) = unbounded();
         let (cluster_confirmed_slot_sender, cluster_confirmed_slot_receiver) = unbounded();
+
+        let rpc_completed_slots_service =
+            RpcCompletedSlotsService::spawn(completed_slots_receiver, rpc_subscriptions.clone());
+
         let tvu = Tvu::new(
             vote_account,
             authorized_voter_keypairs,
@@ -707,12 +768,11 @@ impl Validator {
             },
             blockstore.clone(),
             ledger_signal_receiver,
-            &subscriptions,
+            &rpc_subscriptions,
             &poh_recorder,
             tower,
             &leader_schedule_cache,
             &exit,
-            completed_slots_receiver,
             block_commitment_cache,
             config.enable_partition.clone(),
             transaction_status_sender.clone(),
@@ -742,8 +802,10 @@ impl Validator {
                 rocksdb_max_compaction_jitter: config.rocksdb_compaction_interval,
                 wait_for_vote_to_start_leader,
                 accounts_shrink_ratio: config.accounts_shrink_ratio,
+                disable_epoch_boundary_optimization: config.disable_epoch_boundary_optimization,
             },
             &max_slots,
+            &cost_model,
         );
 
         let tpu = Tpu::new(
@@ -753,8 +815,9 @@ impl Validator {
             retransmit_slots_receiver,
             node.sockets.tpu,
             node.sockets.tpu_forwards,
+            node.sockets.tpu_vote,
             node.sockets.broadcast,
-            &subscriptions,
+            &rpc_subscriptions,
             transaction_status_sender,
             &blockstore,
             &config.broadcast_stage_type,
@@ -769,6 +832,7 @@ impl Validator {
             bank_notification_sender,
             config.tpu_coalesce_ms,
             cluster_confirmed_slot_sender,
+            &cost_model,
         );
 
         datapoint_info!("validator-new", ("id", id.to_string(), String));
@@ -778,10 +842,12 @@ impl Validator {
             serve_repair_service,
             json_rpc_service,
             pubsub_service,
+            rpc_completed_slots_service,
             optimistically_confirmed_bank_tracker,
             transaction_status_service,
             rewards_recorder_service,
             cache_block_meta_service,
+            system_monitor_service,
             sample_performance_service,
             snapshot_packager_service,
             completed_data_sets_service,
@@ -791,6 +857,7 @@ impl Validator {
             poh_recorder,
             ip_echo_server,
             validator_exit: config.validator_exit.clone(),
+            accountsdb_plugin_service,
         }
     }
 
@@ -841,6 +908,10 @@ impl Validator {
             pubsub_service.join().expect("pubsub_service");
         }
 
+        self.rpc_completed_slots_service
+            .join()
+            .expect("rpc_completed_slots_service");
+
         if let Some(optimistically_confirmed_bank_tracker) =
             self.optimistically_confirmed_bank_tracker
         {
@@ -867,6 +938,12 @@ impl Validator {
                 .expect("cache_block_meta_service");
         }
 
+        if let Some(system_monitor_service) = self.system_monitor_service {
+            system_monitor_service
+                .join()
+                .expect("system_monitor_service");
+        }
+
         if let Some(sample_performance_service) = self.sample_performance_service {
             sample_performance_service
                 .join()
@@ -889,6 +966,12 @@ impl Validator {
         if let Some(ip_echo_server) = self.ip_echo_server {
             ip_echo_server.shutdown_background();
         }
+
+        if let Some(accountsdb_plugin_service) = self.accountsdb_plugin_service {
+            accountsdb_plugin_service
+                .join()
+                .expect("accountsdb_plugin_service");
+        }
     }
 }
 
@@ -1022,6 +1105,7 @@ fn post_process_restored_tower(
 }
 
 #[allow(clippy::type_complexity)]
+#[allow(clippy::too_many_arguments)]
 fn new_banks_from_ledger(
     validator_identity: &Pubkey,
     vote_account: &Pubkey,
@@ -1032,6 +1116,7 @@ fn new_banks_from_ledger(
     enforce_ulimit_nofile: bool,
     start_progress: &Arc<RwLock<ValidatorStartProgress>>,
     no_poh_speed_test: bool,
+    accounts_update_notifier: Option<AccountsUpdateNotifier>,
 ) -> (
     GenesisConfig,
     BankForks,
@@ -1148,6 +1233,7 @@ fn new_banks_from_ledger(
         transaction_history_services
             .cache_block_meta_sender
             .as_ref(),
+        accounts_update_notifier,
     )
     .unwrap_or_else(|err| {
         error!("Failed to load ledger: {:?}", err);
@@ -1428,76 +1514,6 @@ fn wait_for_supermajority(
     Ok(true)
 }
 
-fn is_rosetta_emulated() -> bool {
-    #[cfg(target_os = "macos")]
-    {
-        use std::str::FromStr;
-        std::process::Command::new("sysctl")
-            .args(&["-in", "sysctl.proc_translated"])
-            .output()
-            .map_err(|_| ())
-            .and_then(|output| String::from_utf8(output.stdout).map_err(|_| ()))
-            .and_then(|stdout| u8::from_str(stdout.trim()).map_err(|_| ()))
-            .map(|enabled| enabled == 1)
-            .unwrap_or(false)
-    }
-    #[cfg(not(target_os = "macos"))]
-    {
-        false
-    }
-}
-
-pub fn report_target_features() {
-    warn!(
-        "CUDA is {}abled",
-        if solana_perf::perf_libs::api().is_some() {
-            "en"
-        } else {
-            "dis"
-        }
-    );
-
-    if !is_rosetta_emulated() {
-        unsafe { check_avx() };
-        unsafe { check_avx2() };
-    }
-}
-
-// Validator binaries built on a machine with AVX support will generate invalid opcodes
-// when run on machines without AVX causing a non-obvious process abort.  Instead detect
-// the mismatch and error cleanly.
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx")]
-unsafe fn check_avx() {
-    if is_x86_feature_detected!("avx") {
-        info!("AVX detected");
-    } else {
-        error!(
-            "Incompatible CPU detected: missing AVX support. Please build from source on the target"
-        );
-        abort();
-    }
-}
-
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
-unsafe fn check_avx() {}
-
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx2")]
-unsafe fn check_avx2() {
-    if is_x86_feature_detected!("avx2") {
-        info!("AVX2 detected");
-    } else {
-        error!(
-            "Incompatible CPU detected: missing AVX2 support. Please build from source on the target"
-        );
-        abort();
-    }
-}
-
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
-unsafe fn check_avx2() {}
-
 // Get the activated stake percentage (based on the provided bank) that is visible in gossip
 fn get_stake_percent_in_gossip(bank: &Bank, cluster_info: &ClusterInfo, log: bool) -> u64 {
     let mut online_stake = 0;

core/src/voting_service.rs

@@ -1,8 +1,9 @@
 use solana_gossip::cluster_info::ClusterInfo;
 use solana_poh::poh_recorder::PohRecorder;
+use solana_runtime::bank_forks::BankForks;
 use solana_sdk::{clock::Slot, transaction::Transaction};
 use std::{
-    sync::{mpsc::Receiver, Arc, Mutex},
+    sync::{mpsc::Receiver, Arc, Mutex, RwLock},
     thread::{self, Builder, JoinHandle},
 };
 
@@ -38,12 +39,15 @@ impl VotingService {
         vote_receiver: Receiver<VoteOp>,
         cluster_info: Arc<ClusterInfo>,
         poh_recorder: Arc<Mutex<PohRecorder>>,
+        bank_forks: Arc<RwLock<BankForks>>,
     ) -> Self {
         let thread_hdl = Builder::new()
             .name("sol-vote-service".to_string())
             .spawn(move || {
                 for vote_op in vote_receiver.iter() {
-                    Self::handle_vote(&cluster_info, &poh_recorder, vote_op);
+                    let rooted_bank = bank_forks.read().unwrap().root_bank().clone();
+                    let send_to_tpu_vote_port = rooted_bank.send_to_tpu_vote_port_enabled();
+                    Self::handle_vote(&cluster_info, &poh_recorder, vote_op, send_to_tpu_vote_port);
                 }
             })
             .unwrap();
@@ -54,11 +58,14 @@ impl VotingService {
         cluster_info: &ClusterInfo,
         poh_recorder: &Mutex<PohRecorder>,
         vote_op: VoteOp,
+        send_to_tpu_vote_port: bool,
     ) {
-        let _ = cluster_info.send_vote(
-            vote_op.tx(),
-            crate::banking_stage::next_leader_tpu(cluster_info, poh_recorder),
-        );
+        let target_address = if send_to_tpu_vote_port {
+            crate::banking_stage::next_leader_tpu_vote(cluster_info, poh_recorder)
+        } else {
+            crate::banking_stage::next_leader_tpu(cluster_info, poh_recorder)
+        };
+        let _ = cluster_info.send_vote(vote_op.tx(), target_address);
 
         match vote_op {
             VoteOp::PushVote { tx, tower_slots } => {

core/src/window_service.rs

@@ -1,44 +1,165 @@
 //! `window_service` handles the data plane incoming shreds, storing them in
 //!   blockstore and retransmitting where required
 //!
-use crate::{
-    cluster_info_vote_listener::VerifiedVoteReceiver,
-    cluster_slots::ClusterSlots,
-    completed_data_sets_service::CompletedDataSetsSender,
-    outstanding_requests::OutstandingRequests,
-    repair_response,
-    repair_service::{OutstandingRepairs, RepairInfo, RepairService},
-    result::{Error, Result},
-};
-use crossbeam_channel::{
-    unbounded, Receiver as CrossbeamReceiver, RecvTimeoutError, Sender as CrossbeamSender,
-};
-use rayon::iter::IntoParallelRefMutIterator;
-use rayon::iter::ParallelIterator;
-use rayon::ThreadPool;
-use solana_gossip::cluster_info::ClusterInfo;
-use solana_ledger::{
-    blockstore::{self, Blockstore, BlockstoreInsertionMetrics, MAX_DATA_SHREDS_PER_SLOT},
-    leader_schedule_cache::LeaderScheduleCache,
-    shred::{Nonce, Shred},
-};
-use solana_metrics::{inc_new_counter_debug, inc_new_counter_error};
-use solana_perf::packet::Packets;
-use solana_rayon_threadlimit::get_thread_count;
-use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::{clock::Slot, packet::PACKET_DATA_SIZE, pubkey::Pubkey, timing::duration_as_ms};
-use solana_streamer::streamer::PacketSender;
-use std::collections::HashSet;
-use std::{
-    net::{SocketAddr, UdpSocket},
-    sync::atomic::{AtomicBool, Ordering},
-    sync::{Arc, RwLock},
-    thread::{self, Builder, JoinHandle},
-    time::{Duration, Instant},
+use {
+    crate::{
+        cluster_info_vote_listener::VerifiedVoteReceiver,
+        cluster_slots::ClusterSlots,
+        completed_data_sets_service::CompletedDataSetsSender,
+        repair_response,
+        repair_service::{OutstandingRepairs, RepairInfo, RepairService},
+        result::{Error, Result},
+    },
+    crossbeam_channel::{
+        unbounded, Receiver as CrossbeamReceiver, RecvTimeoutError, Sender as CrossbeamSender,
+    },
+    rayon::{prelude::*, ThreadPool},
+    solana_gossip::cluster_info::ClusterInfo,
+    solana_ledger::{
+        blockstore::{self, Blockstore, BlockstoreInsertionMetrics, MAX_DATA_SHREDS_PER_SLOT},
+        leader_schedule_cache::LeaderScheduleCache,
+        shred::{Nonce, Shred},
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::{inc_new_counter_debug, inc_new_counter_error},
+    solana_perf::packet::{Packet, Packets},
+    solana_rayon_threadlimit::get_thread_count,
+    solana_runtime::{bank::Bank, bank_forks::BankForks},
+    solana_sdk::{clock::Slot, packet::PACKET_DATA_SIZE, pubkey::Pubkey},
+    std::collections::HashSet,
+    std::{
+        cmp::Reverse,
+        collections::HashMap,
+        net::{SocketAddr, UdpSocket},
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            mpsc::Sender,
+            Arc, RwLock,
+        },
+        thread::{self, Builder, JoinHandle},
+        time::{Duration, Instant},
+    },
 };
 
-pub type DuplicateSlotSender = CrossbeamSender<Slot>;
-pub type DuplicateSlotReceiver = CrossbeamReceiver<Slot>;
+type DuplicateSlotSender = CrossbeamSender<Slot>;
+pub(crate) type DuplicateSlotReceiver = CrossbeamReceiver<Slot>;
+
+#[derive(Default)]
+struct WindowServiceMetrics {
+    run_insert_count: u64,
+    num_shreds_received: u64,
+    shred_receiver_elapsed_us: u64,
+    prune_shreds_elapsed_us: u64,
+    num_shreds_pruned_invalid_repair: usize,
+    num_errors: u64,
+    num_errors_blockstore: u64,
+    num_errors_cross_beam_recv_timeout: u64,
+    num_errors_other: u64,
+    num_errors_try_crossbeam_send: u64,
+}
+
+impl WindowServiceMetrics {
+    fn report_metrics(&self, metric_name: &'static str) {
+        datapoint_info!(
+            metric_name,
+            ("run_insert_count", self.run_insert_count as i64, i64),
+            ("num_shreds_received", self.num_shreds_received as i64, i64),
+            (
+                "shred_receiver_elapsed_us",
+                self.shred_receiver_elapsed_us as i64,
+                i64
+            ),
+            (
+                "prune_shreds_elapsed_us",
+                self.prune_shreds_elapsed_us as i64,
+                i64
+            ),
+            (
+                "num_shreds_pruned_invalid_repair",
+                self.num_shreds_pruned_invalid_repair,
+                i64
+            ),
+            ("num_errors", self.num_errors, i64),
+            ("num_errors_blockstore", self.num_errors_blockstore, i64),
+            ("num_errors_other", self.num_errors_other, i64),
+            (
+                "num_errors_try_crossbeam_send",
+                self.num_errors_try_crossbeam_send,
+                i64
+            ),
+            (
+                "num_errors_cross_beam_recv_timeout",
+                self.num_errors_cross_beam_recv_timeout,
+                i64
+            ),
+        );
+    }
+
+    fn record_error(&mut self, err: &Error) {
+        self.num_errors += 1;
+        match err {
+            Error::TryCrossbeamSend => self.num_errors_try_crossbeam_send += 1,
+            Error::CrossbeamRecvTimeout(_) => self.num_errors_cross_beam_recv_timeout += 1,
+            Error::Blockstore(err) => {
+                self.num_errors_blockstore += 1;
+                error!("blockstore error: {}", err);
+            }
+            _ => self.num_errors_other += 1,
+        }
+    }
+}
+
+#[derive(Default)]
+struct ReceiveWindowStats {
+    num_packets: usize,
+    num_shreds: usize, // num_discards: num_packets - num_shreds
+    num_repairs: usize,
+    elapsed: Duration, // excludes waiting time on the receiver channel.
+    slots: HashMap<Slot, /*num shreds:*/ usize>,
+    addrs: HashMap</*source:*/ SocketAddr, /*num packets:*/ usize>,
+    since: Option<Instant>,
+}
+
+impl ReceiveWindowStats {
+    fn maybe_submit(&mut self) {
+        const MAX_NUM_ADDRS: usize = 5;
+        const SUBMIT_CADENCE: Duration = Duration::from_secs(2);
+        let elapsed = self.since.as_ref().map(Instant::elapsed);
+        if elapsed.map(|e| e < SUBMIT_CADENCE).unwrap_or_default() {
+            return;
+        }
+        datapoint_info!(
+            "receive_window_stats",
+            ("num_packets", self.num_packets, i64),
+            ("num_shreds", self.num_shreds, i64),
+            ("num_repairs", self.num_repairs, i64),
+            ("elapsed_micros", self.elapsed.as_micros(), i64),
+        );
+        for (slot, num_shreds) in &self.slots {
+            datapoint_info!(
+                "receive_window_num_slot_shreds",
+                ("slot", *slot, i64),
+                ("num_shreds", *num_shreds, i64)
+            );
+        }
+        let mut addrs: Vec<_> = std::mem::take(&mut self.addrs).into_iter().collect();
+        let reverse_count = |(_addr, count): &_| Reverse(*count);
+        if addrs.len() > MAX_NUM_ADDRS {
+            addrs.select_nth_unstable_by_key(MAX_NUM_ADDRS, reverse_count);
+            addrs.truncate(MAX_NUM_ADDRS);
+        }
+        addrs.sort_unstable_by_key(reverse_count);
+        info!(
+            "num addresses: {}, top packets by source: {:?}",
+            self.addrs.len(),
+            addrs
+        );
+        *self = Self {
+            since: Some(Instant::now()),
+            ..Self::default()
+        };
+    }
+}
 
 fn verify_shred_slot(shred: &Shred, root: u64) -> bool {
     if shred.is_data() {
@@ -52,18 +173,15 @@ fn verify_shred_slot(shred: &Shred, root: u64) -> bool {
 
 /// drop shreds that are from myself or not from the correct leader for the
 /// shred's slot
-pub fn should_retransmit_and_persist(
+pub(crate) fn should_retransmit_and_persist(
     shred: &Shred,
     bank: Option<Arc<Bank>>,
-    leader_schedule_cache: &Arc<LeaderScheduleCache>,
+    leader_schedule_cache: &LeaderScheduleCache,
     my_pubkey: &Pubkey,
     root: u64,
     shred_version: u16,
 ) -> bool {
-    let slot_leader_pubkey = match bank {
-        None => leader_schedule_cache.slot_leader_at(shred.slot(), None),
-        Some(bank) => leader_schedule_cache.slot_leader_at(shred.slot(), Some(&bank)),
-    };
+    let slot_leader_pubkey = leader_schedule_cache.slot_leader_at(shred.slot(), bank.as_deref());
     if let Some(leader_id) = slot_leader_pubkey {
         if leader_id == *my_pubkey {
             inc_new_counter_debug!("streamer-recv_window-circular_transmission", 1);
@@ -77,6 +195,9 @@ pub fn should_retransmit_and_persist(
         } else if shred.index() >= MAX_DATA_SHREDS_PER_SLOT as u32 {
             inc_new_counter_warn!("streamer-recv_window-shred_index_overrun", 1);
             false
+        } else if shred.data_header.size as usize > shred.payload.len() {
+            inc_new_counter_warn!("streamer-recv_window-shred_bad_meta_size", 1);
+            false
         } else {
             true
         }
@@ -144,7 +265,7 @@ fn verify_repair(
 fn prune_shreds_invalid_repair(
     shreds: &mut Vec<Shred>,
     repair_infos: &mut Vec<Option<RepairMeta>>,
-    outstanding_requests: &Arc<RwLock<OutstandingRepairs>>,
+    outstanding_requests: &RwLock<OutstandingRepairs>,
 ) {
     assert_eq!(shreds.len(), repair_infos.len());
     let mut i = 0;
@@ -170,34 +291,47 @@ fn prune_shreds_invalid_repair(
 
 fn run_insert<F>(
     shred_receiver: &CrossbeamReceiver<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
-    blockstore: &Arc<Blockstore>,
-    leader_schedule_cache: &Arc<LeaderScheduleCache>,
+    blockstore: &Blockstore,
+    leader_schedule_cache: &LeaderScheduleCache,
     handle_duplicate: F,
     metrics: &mut BlockstoreInsertionMetrics,
+    ws_metrics: &mut WindowServiceMetrics,
     completed_data_sets_sender: &CompletedDataSetsSender,
-    outstanding_requests: &Arc<RwLock<OutstandingRepairs>>,
+    retransmit_sender: &Sender<Vec<Shred>>,
+    outstanding_requests: &RwLock<OutstandingRepairs>,
 ) -> Result<()>
 where
     F: Fn(Shred),
 {
+    ws_metrics.run_insert_count += 1;
+    let mut shred_receiver_elapsed = Measure::start("shred_receiver_elapsed");
     let timer = Duration::from_millis(200);
     let (mut shreds, mut repair_infos) = shred_receiver.recv_timeout(timer)?;
     while let Ok((more_shreds, more_repair_infos)) = shred_receiver.try_recv() {
         shreds.extend(more_shreds);
         repair_infos.extend(more_repair_infos);
     }
+    shred_receiver_elapsed.stop();
+    ws_metrics.shred_receiver_elapsed_us += shred_receiver_elapsed.as_us();
+    ws_metrics.num_shreds_received += shreds.len() as u64;
 
+    let mut prune_shreds_elapsed = Measure::start("prune_shreds_elapsed");
+    let num_shreds = shreds.len();
     prune_shreds_invalid_repair(&mut shreds, &mut repair_infos, outstanding_requests);
+    ws_metrics.num_shreds_pruned_invalid_repair = num_shreds - shreds.len();
     let repairs: Vec<_> = repair_infos
         .iter()
         .map(|repair_info| repair_info.is_some())
         .collect();
+    prune_shreds_elapsed.stop();
+    ws_metrics.prune_shreds_elapsed_us += prune_shreds_elapsed.as_us();
 
     let (completed_data_sets, inserted_indices) = blockstore.insert_shreds_handle_duplicate(
         shreds,
         repairs,
         Some(leader_schedule_cache),
-        false,
+        false, // is_trusted
+        Some(retransmit_sender),
         &handle_duplicate,
         metrics,
     )?;
@@ -212,105 +346,77 @@ where
 }
 
 fn recv_window<F>(
-    blockstore: &Arc<Blockstore>,
+    blockstore: &Blockstore,
+    bank_forks: &RwLock<BankForks>,
     insert_shred_sender: &CrossbeamSender<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
-    my_pubkey: &Pubkey,
     verified_receiver: &CrossbeamReceiver<Vec<Packets>>,
-    retransmit: &PacketSender,
+    retransmit_sender: &Sender<Vec<Shred>>,
     shred_filter: F,
     thread_pool: &ThreadPool,
+    stats: &mut ReceiveWindowStats,
 ) -> Result<()>
 where
-    F: Fn(&Shred, u64) -> bool + Sync,
+    F: Fn(&Shred, Arc<Bank>, /*last root:*/ Slot) -> bool + Sync,
 {
     let timer = Duration::from_millis(200);
     let mut packets = verified_receiver.recv_timeout(timer)?;
-    let mut total_packets: usize = packets.iter().map(|p| p.packets.len()).sum();
-
-    while let Ok(mut more_packets) = verified_receiver.try_recv() {
-        let count: usize = more_packets.iter().map(|p| p.packets.len()).sum();
-        total_packets += count;
-        packets.append(&mut more_packets)
-    }
-
+    packets.extend(verified_receiver.try_iter().flatten());
     let now = Instant::now();
-    inc_new_counter_debug!("streamer-recv_window-recv", total_packets);
-
     let last_root = blockstore.last_root();
+    let working_bank = bank_forks.read().unwrap().working_bank();
+    let handle_packet = |packet: &Packet| {
+        if packet.meta.discard {
+            inc_new_counter_debug!("streamer-recv_window-invalid_or_unnecessary_packet", 1);
+            return None;
+        }
+        // shred fetch stage should be sending packets
+        // with sufficiently large buffers. Needed to ensure
+        // call to `new_from_serialized_shred` is safe.
+        assert_eq!(packet.data.len(), PACKET_DATA_SIZE);
+        let serialized_shred = packet.data.to_vec();
+        let shred = Shred::new_from_serialized_shred(serialized_shred).ok()?;
+        if !shred_filter(&shred, working_bank.clone(), last_root) {
+            return None;
+        }
+        if packet.meta.repair {
+            let repair_info = RepairMeta {
+                _from_addr: packet.meta.addr(),
+                // If can't parse the nonce, dump the packet.
+                nonce: repair_response::nonce(&packet.data)?,
+            };
+            Some((shred, Some(repair_info)))
+        } else {
+            Some((shred, None))
+        }
+    };
     let (shreds, repair_infos): (Vec<_>, Vec<_>) = thread_pool.install(|| {
         packets
-            .par_iter_mut()
-            .flat_map(|packets| {
-                packets
-                    .packets
-                    .iter_mut()
-                    .filter_map(|packet| {
-                        if packet.meta.discard {
-                            inc_new_counter_debug!(
-                                "streamer-recv_window-invalid_or_unnecessary_packet",
-                                1
-                            );
-                            None
-                        } else {
-                            // shred fetch stage should be sending packets
-                            // with sufficiently large buffers. Needed to ensure
-                            // call to `new_from_serialized_shred` is safe.
-                            assert_eq!(packet.data.len(), PACKET_DATA_SIZE);
-                            let serialized_shred = packet.data.to_vec();
-                            if let Ok(shred) = Shred::new_from_serialized_shred(serialized_shred) {
-                                let repair_info = {
-                                    if packet.meta.repair {
-                                        if let Some(nonce) = repair_response::nonce(&packet.data) {
-                                            let repair_info = RepairMeta {
-                                                _from_addr: packet.meta.addr(),
-                                                nonce,
-                                            };
-                                            Some(repair_info)
-                                        } else {
-                                            // If can't parse the nonce, dump the packet
-                                            return None;
-                                        }
-                                    } else {
-                                        None
-                                    }
-                                };
-                                if shred_filter(&shred, last_root) {
-                                    packet.meta.slot = shred.slot();
-                                    packet.meta.seed = shred.seed();
-                                    Some((shred, repair_info))
-                                } else {
-                                    packet.meta.discard = true;
-                                    None
-                                }
-                            } else {
-                                packet.meta.discard = true;
-                                None
-                            }
-                        }
-                    })
-                    .collect::<Vec<_>>()
-            })
+            .par_iter()
+            .flat_map_iter(|pkt| pkt.packets.iter().filter_map(handle_packet))
             .unzip()
     });
-
-    trace!("{:?} shreds from packets", shreds.len());
-
-    trace!("{} num total shreds received: {}", my_pubkey, total_packets);
-
-    for packets in packets.into_iter() {
-        if !packets.is_empty() {
-            // Ignore the send error, as the retransmit is optional (e.g. archivers don't retransmit)
-            let _ = retransmit.send(packets);
-        }
+    // Exclude repair packets from retransmit.
+    let _ = retransmit_sender.send(
+        shreds
+            .iter()
+            .zip(&repair_infos)
+            .filter(|(_, repair_info)| repair_info.is_none())
+            .map(|(shred, _)| shred)
+            .cloned()
+            .collect(),
+    );
+    stats.num_repairs += repair_infos.iter().filter(|r| r.is_some()).count();
+    stats.num_shreds += shreds.len();
+    for shred in &shreds {
+        *stats.slots.entry(shred.slot()).or_default() += 1;
     }
-
     insert_shred_sender.send((shreds, repair_infos))?;
 
-    trace!(
-        "Elapsed processing time in recv_window(): {}",
-        duration_as_ms(&now.elapsed())
-    );
-
+    stats.num_packets += packets.iter().map(|pkt| pkt.packets.len()).sum::<usize>();
+    for packet in packets.iter().flat_map(|pkt| pkt.packets.iter()) {
+        *stats.addrs.entry(packet.meta.addr()).or_default() += 1;
+    }
+    stats.elapsed += now.elapsed();
     Ok(())
 }
 
@@ -337,7 +443,7 @@ impl Drop for Finalizer {
     }
 }
 
-pub struct WindowService {
+pub(crate) struct WindowService {
     t_window: JoinHandle<()>,
     t_insert: JoinHandle<()>,
     t_check_duplicate: JoinHandle<()>,
@@ -346,15 +452,15 @@ pub struct WindowService {
 
 impl WindowService {
     #[allow(clippy::too_many_arguments)]
-    pub fn new<F>(
+    pub(crate) fn new<F>(
         blockstore: Arc<Blockstore>,
         cluster_info: Arc<ClusterInfo>,
         verified_receiver: CrossbeamReceiver<Vec<Packets>>,
-        retransmit: PacketSender,
+        retransmit_sender: Sender<Vec<Shred>>,
         repair_socket: Arc<UdpSocket>,
-        exit: &Arc<AtomicBool>,
+        exit: Arc<AtomicBool>,
         repair_info: RepairInfo,
-        leader_schedule_cache: &Arc<LeaderScheduleCache>,
+        leader_schedule_cache: Arc<LeaderScheduleCache>,
         shred_filter: F,
         cluster_slots: Arc<ClusterSlots>,
         verified_vote_receiver: VerifiedVoteReceiver,
@@ -363,14 +469,13 @@ impl WindowService {
     ) -> WindowService
     where
         F: 'static
-            + Fn(&Pubkey, &Shred, Option<Arc<Bank>>, u64) -> bool
+            + Fn(&Pubkey, &Shred, Option<Arc<Bank>>, /*last root:*/ Slot) -> bool
             + std::marker::Send
             + std::marker::Sync,
     {
-        let outstanding_requests: Arc<RwLock<OutstandingRepairs>> =
-            Arc::new(RwLock::new(OutstandingRequests::default()));
+        let outstanding_requests = Arc::<RwLock<OutstandingRepairs>>::default();
 
-        let bank_forks = Some(repair_info.bank_forks.clone());
+        let bank_forks = repair_info.bank_forks.clone();
 
         let repair_service = RepairService::new(
             blockstore.clone(),
@@ -395,24 +500,25 @@ impl WindowService {
         );
 
         let t_insert = Self::start_window_insert_thread(
-            exit,
-            &blockstore,
+            exit.clone(),
+            blockstore.clone(),
             leader_schedule_cache,
             insert_receiver,
             duplicate_sender,
             completed_data_sets_sender,
+            retransmit_sender.clone(),
             outstanding_requests,
         );
 
         let t_window = Self::start_recv_window_thread(
             cluster_info.id(),
             exit,
-            &blockstore,
+            blockstore,
             insert_sender,
             verified_receiver,
             shred_filter,
             bank_forks,
-            retransmit,
+            retransmit_sender,
         );
 
         WindowService {
@@ -456,17 +562,15 @@ impl WindowService {
     }
 
     fn start_window_insert_thread(
-        exit: &Arc<AtomicBool>,
-        blockstore: &Arc<Blockstore>,
-        leader_schedule_cache: &Arc<LeaderScheduleCache>,
+        exit: Arc<AtomicBool>,
+        blockstore: Arc<Blockstore>,
+        leader_schedule_cache: Arc<LeaderScheduleCache>,
         insert_receiver: CrossbeamReceiver<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
         check_duplicate_sender: CrossbeamSender<Shred>,
         completed_data_sets_sender: CompletedDataSetsSender,
+        retransmit_sender: Sender<Vec<Shred>>,
         outstanding_requests: Arc<RwLock<OutstandingRepairs>>,
     ) -> JoinHandle<()> {
-        let exit = exit.clone();
-        let blockstore = blockstore.clone();
-        let leader_schedule_cache = leader_schedule_cache.clone();
         let mut handle_timeout = || {};
         let handle_error = || {
             inc_new_counter_error!("solana-window-insert-error", 1, 1);
@@ -479,6 +583,7 @@ impl WindowService {
                     let _ = check_duplicate_sender.send(shred);
                 };
                 let mut metrics = BlockstoreInsertionMetrics::default();
+                let mut ws_metrics = WindowServiceMetrics::default();
                 let mut last_print = Instant::now();
                 loop {
                     if exit.load(Ordering::Relaxed) {
@@ -491,9 +596,12 @@ impl WindowService {
                         &leader_schedule_cache,
                         &handle_duplicate,
                         &mut metrics,
+                        &mut ws_metrics,
                         &completed_data_sets_sender,
+                        &retransmit_sender,
                         &outstanding_requests,
                     ) {
+                        ws_metrics.record_error(&e);
                         if Self::should_exit_on_error(e, &mut handle_timeout, &handle_error) {
                             break;
                         }
@@ -502,6 +610,8 @@ impl WindowService {
                     if last_print.elapsed().as_secs() > 2 {
                         metrics.report_metrics("recv-window-insert-shreds");
                         metrics = BlockstoreInsertionMetrics::default();
+                        ws_metrics.report_metrics("recv-window-insert-shreds");
+                        ws_metrics = WindowServiceMetrics::default();
                         last_print = Instant::now();
                     }
                 }
@@ -509,15 +619,16 @@ impl WindowService {
             .unwrap()
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn start_recv_window_thread<F>(
         id: Pubkey,
-        exit: &Arc<AtomicBool>,
-        blockstore: &Arc<Blockstore>,
+        exit: Arc<AtomicBool>,
+        blockstore: Arc<Blockstore>,
         insert_sender: CrossbeamSender<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
         verified_receiver: CrossbeamReceiver<Vec<Packets>>,
         shred_filter: F,
-        bank_forks: Option<Arc<RwLock<BankForks>>>,
-        retransmit: PacketSender,
+        bank_forks: Arc<RwLock<BankForks>>,
+        retransmit_sender: Sender<Vec<Shred>>,
     ) -> JoinHandle<()>
     where
         F: 'static
@@ -525,8 +636,7 @@ impl WindowService {
             + std::marker::Send
             + std::marker::Sync,
     {
-        let exit = exit.clone();
-        let blockstore = blockstore.clone();
+        let mut stats = ReceiveWindowStats::default();
         Builder::new()
             .name("solana-window".to_string())
             .spawn(move || {
@@ -541,34 +651,25 @@ impl WindowService {
                     inc_new_counter_error!("solana-window-error", 1, 1);
                 };
 
-                loop {
-                    if exit.load(Ordering::Relaxed) {
-                        break;
-                    }
-
+                while !exit.load(Ordering::Relaxed) {
                     let mut handle_timeout = || {
                         if now.elapsed() > Duration::from_secs(30) {
-                            warn!("Window does not seem to be receiving data. Ensure port configuration is correct...");
+                            warn!(
+                                "Window does not seem to be receiving data. \
+                            Ensure port configuration is correct..."
+                            );
                             now = Instant::now();
                         }
                     };
                     if let Err(e) = recv_window(
                         &blockstore,
+                        &bank_forks,
                         &insert_sender,
-                        &id,
                         &verified_receiver,
-                        &retransmit,
-                        |shred, last_root| {
-                            shred_filter(
-                                &id,
-                                shred,
-                                bank_forks
-                                    .as_ref()
-                                    .map(|bank_forks| bank_forks.read().unwrap().working_bank()),
-                                last_root,
-                            )
-                        },
+                        &retransmit_sender,
+                        |shred, bank, last_root| shred_filter(&id, shred, Some(bank), last_root),
                         &thread_pool,
+                        &mut stats,
                     ) {
                         if Self::should_exit_on_error(e, &mut handle_timeout, &handle_error) {
                             break;
@@ -576,6 +677,7 @@ impl WindowService {
                     } else {
                         now = Instant::now();
                     }
+                    stats.maybe_submit();
                 }
             })
             .unwrap()
@@ -601,7 +703,7 @@ impl WindowService {
         }
     }
 
-    pub fn join(self) -> thread::Result<()> {
+    pub(crate) fn join(self) -> thread::Result<()> {
         self.t_window.join()?;
         self.t_insert.join()?;
         self.t_check_duplicate.join()?;
@@ -611,23 +713,24 @@ impl WindowService {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_gossip::contact_info::ContactInfo;
-    use solana_ledger::{
-        blockstore::{make_many_slot_entries, Blockstore},
-        entry::{create_ticks, Entry},
-        genesis_utils::create_genesis_config_with_leader,
-        get_tmp_ledger_path,
-        shred::{DataShredHeader, Shredder},
+    use {
+        super::*,
+        solana_gossip::contact_info::ContactInfo,
+        solana_ledger::{
+            blockstore::{make_many_slot_entries, Blockstore},
+            entry::{create_ticks, Entry},
+            genesis_utils::create_genesis_config_with_leader,
+            get_tmp_ledger_path,
+            shred::{DataShredHeader, Shredder},
+        },
+        solana_sdk::{
+            epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
+            hash::Hash,
+            signature::{Keypair, Signer},
+            timing::timestamp,
+        },
+        solana_streamer::socket::SocketAddrSpace,
     };
-    use solana_sdk::{
-        epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
-        hash::Hash,
-        signature::{Keypair, Signer},
-        timing::timestamp,
-    };
-    use solana_streamer::socket::SocketAddrSpace;
-    use std::sync::Arc;
 
     fn local_entries_to_shred(
         entries: &[Entry],
@@ -667,7 +770,7 @@ mod test {
         ));
         let cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank));
 
-        let mut shreds = local_entries_to_shred(&[Entry::default()], 0, 0, &leader_keypair);
+        let shreds = local_entries_to_shred(&[Entry::default()], 0, 0, &leader_keypair);
 
         // with a Bank for slot 0, shred continues
         assert!(should_retransmit_and_persist(
@@ -719,9 +822,22 @@ mod test {
         ));
 
         // with a Bank and no idea who leader is, shred gets thrown out
-        shreds[0].set_slot(MINIMUM_SLOTS_PER_EPOCH as u64 * 3);
+        let mut bad_slot_shred = shreds[0].clone();
+        bad_slot_shred.set_slot(MINIMUM_SLOTS_PER_EPOCH as u64 * 3);
         assert!(!should_retransmit_and_persist(
-            &shreds[0],
+            &bad_slot_shred,
+            Some(bank.clone()),
+            &cache,
+            &me_id,
+            0,
+            0
+        ));
+
+        // with a bad header size
+        let mut bad_header_shred = shreds[0].clone();
+        bad_header_shred.data_header.size = (bad_header_shred.payload.len() + 1) as u16;
+        assert!(!should_retransmit_and_persist(
+            &bad_header_shred,
             Some(bank.clone()),
             &cache,
             &me_id,

core/tests/client.rs

@@ -98,14 +98,14 @@ fn test_slot_subscription() {
     let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
     let optimistically_confirmed_bank =
         OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
-    let subscriptions = Arc::new(RpcSubscriptions::new(
+    let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
         &exit,
         bank_forks,
         Arc::new(RwLock::new(BlockCommitmentCache::default())),
         optimistically_confirmed_bank,
     ));
-    let pubsub_service =
-        PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr, &exit);
+    let (trigger, pubsub_service) =
+        PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr);
     std::thread::sleep(Duration::from_millis(400));
 
     let (mut client, receiver) =
@@ -138,6 +138,7 @@ fn test_slot_subscription() {
     }
 
     exit.store(true, Ordering::Relaxed);
+    trigger.cancel();
     client.shutdown().unwrap();
     pubsub_service.close().unwrap();
 

core/tests/rpc.rs

@@ -1,6 +1,7 @@
 use bincode::serialize;
 use jsonrpc_core::futures::StreamExt;
 use jsonrpc_core_client::transports::ws;
+
 use log::*;
 use reqwest::{self, header::CONTENT_TYPE};
 use serde_json::{json, Value};
@@ -10,11 +11,12 @@ use solana_client::{
     rpc_client::RpcClient,
     rpc_config::{RpcAccountInfoConfig, RpcSignatureSubscribeConfig},
     rpc_request::RpcError,
-    rpc_response::{Response, RpcSignatureResult, SlotUpdate},
+    rpc_response::{Response as RpcResponse, RpcSignatureResult, SlotUpdate},
     tpu_client::{TpuClient, TpuClientConfig},
 };
 use solana_core::test_validator::TestValidator;
 use solana_rpc::rpc_pubsub::gen_client::Client as PubsubClient;
+
 use solana_sdk::{
     commitment_config::CommitmentConfig,
     hash::Hash,
@@ -256,9 +258,9 @@ fn test_rpc_subscriptions() {
     // Track when subscriptions are ready
     let (ready_sender, ready_receiver) = channel::<()>();
     // Track account notifications are received
-    let (account_sender, account_receiver) = channel::<Response<UiAccount>>();
+    let (account_sender, account_receiver) = channel::<RpcResponse<UiAccount>>();
     // Track when status notifications are received
-    let (status_sender, status_receiver) = channel::<(String, Response<RpcSignatureResult>)>();
+    let (status_sender, status_receiver) = channel::<(String, RpcResponse<RpcSignatureResult>)>();
 
     // Create the pub sub runtime
     let rt = Runtime::new().unwrap();

core/tests/snapshots.rs

@@ -109,6 +109,7 @@ mod tests {
                 false,
                 accounts_db::AccountShrinkThreshold::default(),
                 false,
+                None,
             );
             bank0.freeze();
             let mut bank_forks = BankForks::new(bank0);
@@ -172,6 +173,7 @@ mod tests {
             accounts_db::AccountShrinkThreshold::default(),
             check_hash_calculation,
             false,
+            None,
         )
         .unwrap();
 

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.7.12"
+version = "1.8.2"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

docs/sidebars.js

@@ -41,7 +41,6 @@ module.exports = {
       "cli/choose-a-cluster",
       "cli/transfer-tokens",
       "cli/delegate-stake",
-      "cli/manage-stake-accounts",
       "cli/deploy-a-program",
       "offline-signing",
       "offline-signing/durable-nonce",
@@ -65,6 +64,7 @@ module.exports = {
         items: [
           "developing/clients/jsonrpc-api",
           "developing/clients/javascript-api",
+          "developing/clients/javascript-reference",
           "developing/clients/rust-api",
         ],
       },